2970 lines · c
1/* This file is part of the Emulex RoCE Device Driver for2 * RoCE (RDMA over Converged Ethernet) adapters.3 * Copyright (C) 2012-2015 Emulex. All rights reserved.4 * EMULEX and SLI are trademarks of Emulex.5 * www.emulex.com6 *7 * This software is available to you under a choice of one of two licenses.8 * You may choose to be licensed under the terms of the GNU General Public9 * License (GPL) Version 2, available from the file COPYING in the main10 * directory of this source tree, or the BSD license below:11 *12 * Redistribution and use in source and binary forms, with or without13 * modification, are permitted provided that the following conditions14 * are met:15 *16 * - Redistributions of source code must retain the above copyright notice,17 * this list of conditions and the following disclaimer.18 *19 * - Redistributions in binary form must reproduce the above copyright20 * notice, this list of conditions and the following disclaimer in21 * the documentation and/or other materials provided with the distribution.22 *23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"24 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,THE25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE26 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE27 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR30 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,31 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR32 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF33 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.34 *35 * Contact Information:36 * linux-drivers@emulex.com37 *38 * Emulex39 * 3333 Susan Street40 * Costa Mesa, CA 9262641 */42 43#include <linux/dma-mapping.h>44#include <net/addrconf.h>45#include <rdma/ib_verbs.h>46#include <rdma/ib_user_verbs.h>47#include <rdma/iw_cm.h>48#include <rdma/ib_umem.h>49#include <rdma/ib_addr.h>50#include <rdma/ib_cache.h>51#include <rdma/uverbs_ioctl.h>52 53#include "ocrdma.h"54#include "ocrdma_hw.h"55#include "ocrdma_verbs.h"56#include <rdma/ocrdma-abi.h>57 58int ocrdma_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)59{60 if (index > 0)61 return -EINVAL;62 63 *pkey = 0xffff;64 return 0;65}66 67int ocrdma_query_device(struct ib_device *ibdev, struct ib_device_attr *attr,68 struct ib_udata *uhw)69{70 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);71 72 if (uhw->inlen || uhw->outlen)73 return -EINVAL;74 75 memset(attr, 0, sizeof *attr);76 memcpy(&attr->fw_ver, &dev->attr.fw_ver[0],77 min(sizeof(dev->attr.fw_ver), sizeof(attr->fw_ver)));78 addrconf_addr_eui48((u8 *)&attr->sys_image_guid,79 dev->nic_info.mac_addr);80 attr->max_mr_size = dev->attr.max_mr_size;81 attr->page_size_cap = 0xffff000;82 attr->vendor_id = dev->nic_info.pdev->vendor;83 attr->vendor_part_id = dev->nic_info.pdev->device;84 attr->hw_ver = dev->asic_id;85 attr->max_qp = dev->attr.max_qp;86 attr->max_ah = OCRDMA_MAX_AH;87 attr->max_qp_wr = dev->attr.max_wqe;88 89 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |90 IB_DEVICE_RC_RNR_NAK_GEN |91 IB_DEVICE_SHUTDOWN_PORT |92 IB_DEVICE_SYS_IMAGE_GUID |93 IB_DEVICE_MEM_MGT_EXTENSIONS;94 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;95 attr->max_send_sge = dev->attr.max_send_sge;96 attr->max_recv_sge = dev->attr.max_recv_sge;97 attr->max_sge_rd = dev->attr.max_rdma_sge;98 attr->max_cq = dev->attr.max_cq;99 attr->max_cqe = dev->attr.max_cqe;100 attr->max_mr = dev->attr.max_mr;101 attr->max_mw = dev->attr.max_mw;102 attr->max_pd = dev->attr.max_pd;103 attr->atomic_cap = 0;104 attr->max_qp_rd_atom =105 min(dev->attr.max_ord_per_qp, dev->attr.max_ird_per_qp);106 attr->max_qp_init_rd_atom = dev->attr.max_ord_per_qp;107 attr->max_srq = dev->attr.max_srq;108 attr->max_srq_sge = dev->attr.max_srq_sge;109 attr->max_srq_wr = dev->attr.max_rqe;110 attr->local_ca_ack_delay = dev->attr.local_ca_ack_delay;111 attr->max_fast_reg_page_list_len = dev->attr.max_pages_per_frmr;112 attr->max_pkeys = 1;113 return 0;114}115 116static inline void get_link_speed_and_width(struct ocrdma_dev *dev,117 u16 *ib_speed, u8 *ib_width)118{119 int status;120 u8 speed;121 122 status = ocrdma_mbx_get_link_speed(dev, &speed, NULL);123 if (status)124 speed = OCRDMA_PHYS_LINK_SPEED_ZERO;125 126 switch (speed) {127 case OCRDMA_PHYS_LINK_SPEED_1GBPS:128 *ib_speed = IB_SPEED_SDR;129 *ib_width = IB_WIDTH_1X;130 break;131 132 case OCRDMA_PHYS_LINK_SPEED_10GBPS:133 *ib_speed = IB_SPEED_QDR;134 *ib_width = IB_WIDTH_1X;135 break;136 137 case OCRDMA_PHYS_LINK_SPEED_20GBPS:138 *ib_speed = IB_SPEED_DDR;139 *ib_width = IB_WIDTH_4X;140 break;141 142 case OCRDMA_PHYS_LINK_SPEED_40GBPS:143 *ib_speed = IB_SPEED_QDR;144 *ib_width = IB_WIDTH_4X;145 break;146 147 default:148 /* Unsupported */149 *ib_speed = IB_SPEED_SDR;150 *ib_width = IB_WIDTH_1X;151 }152}153 154int ocrdma_query_port(struct ib_device *ibdev,155 u32 port, struct ib_port_attr *props)156{157 enum ib_port_state port_state;158 struct ocrdma_dev *dev;159 struct net_device *netdev;160 161 /* props being zeroed by the caller, avoid zeroing it here */162 dev = get_ocrdma_dev(ibdev);163 netdev = dev->nic_info.netdev;164 if (netif_running(netdev) && netif_oper_up(netdev)) {165 port_state = IB_PORT_ACTIVE;166 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP;167 } else {168 port_state = IB_PORT_DOWN;169 props->phys_state = IB_PORT_PHYS_STATE_DISABLED;170 }171 props->max_mtu = IB_MTU_4096;172 props->active_mtu = iboe_get_mtu(netdev->mtu);173 props->lid = 0;174 props->lmc = 0;175 props->sm_lid = 0;176 props->sm_sl = 0;177 props->state = port_state;178 props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP |179 IB_PORT_DEVICE_MGMT_SUP |180 IB_PORT_VENDOR_CLASS_SUP;181 props->ip_gids = true;182 props->gid_tbl_len = OCRDMA_MAX_SGID;183 props->pkey_tbl_len = 1;184 props->bad_pkey_cntr = 0;185 props->qkey_viol_cntr = 0;186 get_link_speed_and_width(dev, &props->active_speed,187 &props->active_width);188 props->max_msg_sz = 0x80000000;189 props->max_vl_num = 4;190 return 0;191}192 193static int ocrdma_add_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,194 unsigned long len)195{196 struct ocrdma_mm *mm;197 198 mm = kzalloc(sizeof(*mm), GFP_KERNEL);199 if (mm == NULL)200 return -ENOMEM;201 mm->key.phy_addr = phy_addr;202 mm->key.len = len;203 INIT_LIST_HEAD(&mm->entry);204 205 mutex_lock(&uctx->mm_list_lock);206 list_add_tail(&mm->entry, &uctx->mm_head);207 mutex_unlock(&uctx->mm_list_lock);208 return 0;209}210 211static void ocrdma_del_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,212 unsigned long len)213{214 struct ocrdma_mm *mm, *tmp;215 216 mutex_lock(&uctx->mm_list_lock);217 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {218 if (len != mm->key.len && phy_addr != mm->key.phy_addr)219 continue;220 221 list_del(&mm->entry);222 kfree(mm);223 break;224 }225 mutex_unlock(&uctx->mm_list_lock);226}227 228static bool ocrdma_search_mmap(struct ocrdma_ucontext *uctx, u64 phy_addr,229 unsigned long len)230{231 bool found = false;232 struct ocrdma_mm *mm;233 234 mutex_lock(&uctx->mm_list_lock);235 list_for_each_entry(mm, &uctx->mm_head, entry) {236 if (len != mm->key.len && phy_addr != mm->key.phy_addr)237 continue;238 239 found = true;240 break;241 }242 mutex_unlock(&uctx->mm_list_lock);243 return found;244}245 246 247static u16 _ocrdma_pd_mgr_get_bitmap(struct ocrdma_dev *dev, bool dpp_pool)248{249 u16 pd_bitmap_idx = 0;250 unsigned long *pd_bitmap;251 252 if (dpp_pool) {253 pd_bitmap = dev->pd_mgr->pd_dpp_bitmap;254 pd_bitmap_idx = find_first_zero_bit(pd_bitmap,255 dev->pd_mgr->max_dpp_pd);256 __set_bit(pd_bitmap_idx, pd_bitmap);257 dev->pd_mgr->pd_dpp_count++;258 if (dev->pd_mgr->pd_dpp_count > dev->pd_mgr->pd_dpp_thrsh)259 dev->pd_mgr->pd_dpp_thrsh = dev->pd_mgr->pd_dpp_count;260 } else {261 pd_bitmap = dev->pd_mgr->pd_norm_bitmap;262 pd_bitmap_idx = find_first_zero_bit(pd_bitmap,263 dev->pd_mgr->max_normal_pd);264 __set_bit(pd_bitmap_idx, pd_bitmap);265 dev->pd_mgr->pd_norm_count++;266 if (dev->pd_mgr->pd_norm_count > dev->pd_mgr->pd_norm_thrsh)267 dev->pd_mgr->pd_norm_thrsh = dev->pd_mgr->pd_norm_count;268 }269 return pd_bitmap_idx;270}271 272static int _ocrdma_pd_mgr_put_bitmap(struct ocrdma_dev *dev, u16 pd_id,273 bool dpp_pool)274{275 u16 pd_count;276 u16 pd_bit_index;277 278 pd_count = dpp_pool ? dev->pd_mgr->pd_dpp_count :279 dev->pd_mgr->pd_norm_count;280 if (pd_count == 0)281 return -EINVAL;282 283 if (dpp_pool) {284 pd_bit_index = pd_id - dev->pd_mgr->pd_dpp_start;285 if (pd_bit_index >= dev->pd_mgr->max_dpp_pd) {286 return -EINVAL;287 } else {288 __clear_bit(pd_bit_index, dev->pd_mgr->pd_dpp_bitmap);289 dev->pd_mgr->pd_dpp_count--;290 }291 } else {292 pd_bit_index = pd_id - dev->pd_mgr->pd_norm_start;293 if (pd_bit_index >= dev->pd_mgr->max_normal_pd) {294 return -EINVAL;295 } else {296 __clear_bit(pd_bit_index, dev->pd_mgr->pd_norm_bitmap);297 dev->pd_mgr->pd_norm_count--;298 }299 }300 301 return 0;302}303 304static int ocrdma_put_pd_num(struct ocrdma_dev *dev, u16 pd_id,305 bool dpp_pool)306{307 int status;308 309 mutex_lock(&dev->dev_lock);310 status = _ocrdma_pd_mgr_put_bitmap(dev, pd_id, dpp_pool);311 mutex_unlock(&dev->dev_lock);312 return status;313}314 315static int ocrdma_get_pd_num(struct ocrdma_dev *dev, struct ocrdma_pd *pd)316{317 u16 pd_idx = 0;318 int status = 0;319 320 mutex_lock(&dev->dev_lock);321 if (pd->dpp_enabled) {322 /* try allocating DPP PD, if not available then normal PD */323 if (dev->pd_mgr->pd_dpp_count < dev->pd_mgr->max_dpp_pd) {324 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, true);325 pd->id = dev->pd_mgr->pd_dpp_start + pd_idx;326 pd->dpp_page = dev->pd_mgr->dpp_page_index + pd_idx;327 } else if (dev->pd_mgr->pd_norm_count <328 dev->pd_mgr->max_normal_pd) {329 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false);330 pd->id = dev->pd_mgr->pd_norm_start + pd_idx;331 pd->dpp_enabled = false;332 } else {333 status = -EINVAL;334 }335 } else {336 if (dev->pd_mgr->pd_norm_count < dev->pd_mgr->max_normal_pd) {337 pd_idx = _ocrdma_pd_mgr_get_bitmap(dev, false);338 pd->id = dev->pd_mgr->pd_norm_start + pd_idx;339 } else {340 status = -EINVAL;341 }342 }343 mutex_unlock(&dev->dev_lock);344 return status;345}346 347/*348 * NOTE:349 *350 * ocrdma_ucontext must be used here because this function is also351 * called from ocrdma_alloc_ucontext where ib_udata does not have352 * valid ib_ucontext pointer. ib_uverbs_get_context does not call353 * uobj_{alloc|get_xxx} helpers which are used to store the354 * ib_ucontext in uverbs_attr_bundle wrapping the ib_udata. so355 * ib_udata does NOT imply valid ib_ucontext here!356 */357static int _ocrdma_alloc_pd(struct ocrdma_dev *dev, struct ocrdma_pd *pd,358 struct ocrdma_ucontext *uctx,359 struct ib_udata *udata)360{361 int status;362 363 if (udata && uctx && dev->attr.max_dpp_pds) {364 pd->dpp_enabled =365 ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R;366 pd->num_dpp_qp =367 pd->dpp_enabled ? (dev->nic_info.db_page_size /368 dev->attr.wqe_size) : 0;369 }370 371 if (dev->pd_mgr->pd_prealloc_valid)372 return ocrdma_get_pd_num(dev, pd);373 374retry:375 status = ocrdma_mbx_alloc_pd(dev, pd);376 if (status) {377 if (pd->dpp_enabled) {378 pd->dpp_enabled = false;379 pd->num_dpp_qp = 0;380 goto retry;381 }382 return status;383 }384 385 return 0;386}387 388static inline int is_ucontext_pd(struct ocrdma_ucontext *uctx,389 struct ocrdma_pd *pd)390{391 return (uctx->cntxt_pd == pd);392}393 394static void _ocrdma_dealloc_pd(struct ocrdma_dev *dev,395 struct ocrdma_pd *pd)396{397 if (dev->pd_mgr->pd_prealloc_valid)398 ocrdma_put_pd_num(dev, pd->id, pd->dpp_enabled);399 else400 ocrdma_mbx_dealloc_pd(dev, pd);401}402 403static int ocrdma_alloc_ucontext_pd(struct ocrdma_dev *dev,404 struct ocrdma_ucontext *uctx,405 struct ib_udata *udata)406{407 struct ib_device *ibdev = &dev->ibdev;408 struct ib_pd *pd;409 int status;410 411 pd = rdma_zalloc_drv_obj(ibdev, ib_pd);412 if (!pd)413 return -ENOMEM;414 415 pd->device = ibdev;416 uctx->cntxt_pd = get_ocrdma_pd(pd);417 418 status = _ocrdma_alloc_pd(dev, uctx->cntxt_pd, uctx, udata);419 if (status) {420 kfree(uctx->cntxt_pd);421 goto err;422 }423 424 uctx->cntxt_pd->uctx = uctx;425 uctx->cntxt_pd->ibpd.device = &dev->ibdev;426err:427 return status;428}429 430static void ocrdma_dealloc_ucontext_pd(struct ocrdma_ucontext *uctx)431{432 struct ocrdma_pd *pd = uctx->cntxt_pd;433 struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device);434 435 if (uctx->pd_in_use) {436 pr_err("%s(%d) Freeing in use pdid=0x%x.\n",437 __func__, dev->id, pd->id);438 }439 uctx->cntxt_pd = NULL;440 _ocrdma_dealloc_pd(dev, pd);441 kfree(pd);442}443 444static struct ocrdma_pd *ocrdma_get_ucontext_pd(struct ocrdma_ucontext *uctx)445{446 struct ocrdma_pd *pd = NULL;447 448 mutex_lock(&uctx->mm_list_lock);449 if (!uctx->pd_in_use) {450 uctx->pd_in_use = true;451 pd = uctx->cntxt_pd;452 }453 mutex_unlock(&uctx->mm_list_lock);454 455 return pd;456}457 458static void ocrdma_release_ucontext_pd(struct ocrdma_ucontext *uctx)459{460 mutex_lock(&uctx->mm_list_lock);461 uctx->pd_in_use = false;462 mutex_unlock(&uctx->mm_list_lock);463}464 465int ocrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)466{467 struct ib_device *ibdev = uctx->device;468 int status;469 struct ocrdma_ucontext *ctx = get_ocrdma_ucontext(uctx);470 struct ocrdma_alloc_ucontext_resp resp = {};471 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);472 struct pci_dev *pdev = dev->nic_info.pdev;473 u32 map_len = roundup(sizeof(u32) * 2048, PAGE_SIZE);474 475 if (!udata)476 return -EFAULT;477 INIT_LIST_HEAD(&ctx->mm_head);478 mutex_init(&ctx->mm_list_lock);479 480 ctx->ah_tbl.va = dma_alloc_coherent(&pdev->dev, map_len,481 &ctx->ah_tbl.pa, GFP_KERNEL);482 if (!ctx->ah_tbl.va)483 return -ENOMEM;484 485 ctx->ah_tbl.len = map_len;486 487 resp.ah_tbl_len = ctx->ah_tbl.len;488 resp.ah_tbl_page = virt_to_phys(ctx->ah_tbl.va);489 490 status = ocrdma_add_mmap(ctx, resp.ah_tbl_page, resp.ah_tbl_len);491 if (status)492 goto map_err;493 494 status = ocrdma_alloc_ucontext_pd(dev, ctx, udata);495 if (status)496 goto pd_err;497 498 resp.dev_id = dev->id;499 resp.max_inline_data = dev->attr.max_inline_data;500 resp.wqe_size = dev->attr.wqe_size;501 resp.rqe_size = dev->attr.rqe_size;502 resp.dpp_wqe_size = dev->attr.wqe_size;503 504 memcpy(resp.fw_ver, dev->attr.fw_ver, sizeof(resp.fw_ver));505 status = ib_copy_to_udata(udata, &resp, sizeof(resp));506 if (status)507 goto cpy_err;508 return 0;509 510cpy_err:511 ocrdma_dealloc_ucontext_pd(ctx);512pd_err:513 ocrdma_del_mmap(ctx, ctx->ah_tbl.pa, ctx->ah_tbl.len);514map_err:515 dma_free_coherent(&pdev->dev, ctx->ah_tbl.len, ctx->ah_tbl.va,516 ctx->ah_tbl.pa);517 return status;518}519 520void ocrdma_dealloc_ucontext(struct ib_ucontext *ibctx)521{522 struct ocrdma_mm *mm, *tmp;523 struct ocrdma_ucontext *uctx = get_ocrdma_ucontext(ibctx);524 struct ocrdma_dev *dev = get_ocrdma_dev(ibctx->device);525 struct pci_dev *pdev = dev->nic_info.pdev;526 527 ocrdma_dealloc_ucontext_pd(uctx);528 529 ocrdma_del_mmap(uctx, uctx->ah_tbl.pa, uctx->ah_tbl.len);530 dma_free_coherent(&pdev->dev, uctx->ah_tbl.len, uctx->ah_tbl.va,531 uctx->ah_tbl.pa);532 533 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {534 list_del(&mm->entry);535 kfree(mm);536 }537}538 539int ocrdma_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)540{541 struct ocrdma_ucontext *ucontext = get_ocrdma_ucontext(context);542 struct ocrdma_dev *dev = get_ocrdma_dev(context->device);543 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;544 u64 unmapped_db = (u64) dev->nic_info.unmapped_db;545 unsigned long len = (vma->vm_end - vma->vm_start);546 int status;547 bool found;548 549 if (vma->vm_start & (PAGE_SIZE - 1))550 return -EINVAL;551 found = ocrdma_search_mmap(ucontext, vma->vm_pgoff << PAGE_SHIFT, len);552 if (!found)553 return -EINVAL;554 555 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +556 dev->nic_info.db_total_size)) &&557 (len <= dev->nic_info.db_page_size)) {558 if (vma->vm_flags & VM_READ)559 return -EPERM;560 561 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);562 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,563 len, vma->vm_page_prot);564 } else if (dev->nic_info.dpp_unmapped_len &&565 (vm_page >= (u64) dev->nic_info.dpp_unmapped_addr) &&566 (vm_page <= (u64) (dev->nic_info.dpp_unmapped_addr +567 dev->nic_info.dpp_unmapped_len)) &&568 (len <= dev->nic_info.dpp_unmapped_len)) {569 if (vma->vm_flags & VM_READ)570 return -EPERM;571 572 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);573 status = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,574 len, vma->vm_page_prot);575 } else {576 status = remap_pfn_range(vma, vma->vm_start,577 vma->vm_pgoff, len, vma->vm_page_prot);578 }579 return status;580}581 582static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd,583 struct ib_udata *udata)584{585 int status;586 u64 db_page_addr;587 u64 dpp_page_addr = 0;588 u32 db_page_size;589 struct ocrdma_alloc_pd_uresp rsp;590 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(591 udata, struct ocrdma_ucontext, ibucontext);592 593 memset(&rsp, 0, sizeof(rsp));594 rsp.id = pd->id;595 rsp.dpp_enabled = pd->dpp_enabled;596 db_page_addr = ocrdma_get_db_addr(dev, pd->id);597 db_page_size = dev->nic_info.db_page_size;598 599 status = ocrdma_add_mmap(uctx, db_page_addr, db_page_size);600 if (status)601 return status;602 603 if (pd->dpp_enabled) {604 dpp_page_addr = dev->nic_info.dpp_unmapped_addr +605 (pd->id * PAGE_SIZE);606 status = ocrdma_add_mmap(uctx, dpp_page_addr,607 PAGE_SIZE);608 if (status)609 goto dpp_map_err;610 rsp.dpp_page_addr_hi = upper_32_bits(dpp_page_addr);611 rsp.dpp_page_addr_lo = dpp_page_addr;612 }613 614 status = ib_copy_to_udata(udata, &rsp, sizeof(rsp));615 if (status)616 goto ucopy_err;617 618 pd->uctx = uctx;619 return 0;620 621ucopy_err:622 if (pd->dpp_enabled)623 ocrdma_del_mmap(pd->uctx, dpp_page_addr, PAGE_SIZE);624dpp_map_err:625 ocrdma_del_mmap(pd->uctx, db_page_addr, db_page_size);626 return status;627}628 629int ocrdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)630{631 struct ib_device *ibdev = ibpd->device;632 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);633 struct ocrdma_pd *pd;634 int status;635 u8 is_uctx_pd = false;636 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(637 udata, struct ocrdma_ucontext, ibucontext);638 639 if (udata) {640 pd = ocrdma_get_ucontext_pd(uctx);641 if (pd) {642 is_uctx_pd = true;643 goto pd_mapping;644 }645 }646 647 pd = get_ocrdma_pd(ibpd);648 status = _ocrdma_alloc_pd(dev, pd, uctx, udata);649 if (status)650 goto exit;651 652pd_mapping:653 if (udata) {654 status = ocrdma_copy_pd_uresp(dev, pd, udata);655 if (status)656 goto err;657 }658 return 0;659 660err:661 if (is_uctx_pd)662 ocrdma_release_ucontext_pd(uctx);663 else664 _ocrdma_dealloc_pd(dev, pd);665exit:666 return status;667}668 669int ocrdma_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)670{671 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);672 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);673 struct ocrdma_ucontext *uctx = NULL;674 u64 usr_db;675 676 uctx = pd->uctx;677 if (uctx) {678 u64 dpp_db = dev->nic_info.dpp_unmapped_addr +679 (pd->id * PAGE_SIZE);680 if (pd->dpp_enabled)681 ocrdma_del_mmap(pd->uctx, dpp_db, PAGE_SIZE);682 usr_db = ocrdma_get_db_addr(dev, pd->id);683 ocrdma_del_mmap(pd->uctx, usr_db, dev->nic_info.db_page_size);684 685 if (is_ucontext_pd(uctx, pd)) {686 ocrdma_release_ucontext_pd(uctx);687 return 0;688 }689 }690 _ocrdma_dealloc_pd(dev, pd);691 return 0;692}693 694static int ocrdma_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_mr *mr,695 u32 pdid, int acc, u32 num_pbls, u32 addr_check)696{697 int status;698 699 mr->hwmr.fr_mr = 0;700 mr->hwmr.local_rd = 1;701 mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;702 mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;703 mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;704 mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0;705 mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;706 mr->hwmr.num_pbls = num_pbls;707 708 status = ocrdma_mbx_alloc_lkey(dev, &mr->hwmr, pdid, addr_check);709 if (status)710 return status;711 712 mr->ibmr.lkey = mr->hwmr.lkey;713 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)714 mr->ibmr.rkey = mr->hwmr.lkey;715 return 0;716}717 718struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *ibpd, int acc)719{720 int status;721 struct ocrdma_mr *mr;722 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);723 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);724 725 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE)) {726 pr_err("%s err, invalid access rights\n", __func__);727 return ERR_PTR(-EINVAL);728 }729 730 mr = kzalloc(sizeof(*mr), GFP_KERNEL);731 if (!mr)732 return ERR_PTR(-ENOMEM);733 734 status = ocrdma_alloc_lkey(dev, mr, pd->id, acc, 0,735 OCRDMA_ADDR_CHECK_DISABLE);736 if (status) {737 kfree(mr);738 return ERR_PTR(status);739 }740 741 return &mr->ibmr;742}743 744static void ocrdma_free_mr_pbl_tbl(struct ocrdma_dev *dev,745 struct ocrdma_hw_mr *mr)746{747 struct pci_dev *pdev = dev->nic_info.pdev;748 int i = 0;749 750 if (mr->pbl_table) {751 for (i = 0; i < mr->num_pbls; i++) {752 if (!mr->pbl_table[i].va)753 continue;754 dma_free_coherent(&pdev->dev, mr->pbl_size,755 mr->pbl_table[i].va,756 mr->pbl_table[i].pa);757 }758 kfree(mr->pbl_table);759 mr->pbl_table = NULL;760 }761}762 763static int ocrdma_get_pbl_info(struct ocrdma_dev *dev, struct ocrdma_mr *mr,764 u32 num_pbes)765{766 u32 num_pbls = 0;767 u32 idx = 0;768 int status = 0;769 u32 pbl_size;770 771 do {772 pbl_size = OCRDMA_MIN_HPAGE_SIZE * (1 << idx);773 if (pbl_size > MAX_OCRDMA_PBL_SIZE) {774 status = -EFAULT;775 break;776 }777 num_pbls = roundup(num_pbes, (pbl_size / sizeof(u64)));778 num_pbls = num_pbls / (pbl_size / sizeof(u64));779 idx++;780 } while (num_pbls >= dev->attr.max_num_mr_pbl);781 782 mr->hwmr.num_pbes = num_pbes;783 mr->hwmr.num_pbls = num_pbls;784 mr->hwmr.pbl_size = pbl_size;785 return status;786}787 788static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr)789{790 int status = 0;791 int i;792 u32 dma_len = mr->pbl_size;793 struct pci_dev *pdev = dev->nic_info.pdev;794 void *va;795 dma_addr_t pa;796 797 mr->pbl_table = kcalloc(mr->num_pbls, sizeof(struct ocrdma_pbl),798 GFP_KERNEL);799 800 if (!mr->pbl_table)801 return -ENOMEM;802 803 for (i = 0; i < mr->num_pbls; i++) {804 va = dma_alloc_coherent(&pdev->dev, dma_len, &pa, GFP_KERNEL);805 if (!va) {806 ocrdma_free_mr_pbl_tbl(dev, mr);807 status = -ENOMEM;808 break;809 }810 mr->pbl_table[i].va = va;811 mr->pbl_table[i].pa = pa;812 }813 return status;814}815 816static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr)817{818 struct ocrdma_pbe *pbe;819 struct ib_block_iter biter;820 struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table;821 int pbe_cnt;822 u64 pg_addr;823 824 if (!mr->hwmr.num_pbes)825 return;826 827 pbe = (struct ocrdma_pbe *)pbl_tbl->va;828 pbe_cnt = 0;829 830 rdma_umem_for_each_dma_block (mr->umem, &biter, PAGE_SIZE) {831 /* store the page address in pbe */832 pg_addr = rdma_block_iter_dma_address(&biter);833 pbe->pa_lo = cpu_to_le32(pg_addr);834 pbe->pa_hi = cpu_to_le32(upper_32_bits(pg_addr));835 pbe_cnt += 1;836 pbe++;837 838 /* if the given pbl is full storing the pbes,839 * move to next pbl.840 */841 if (pbe_cnt == (mr->hwmr.pbl_size / sizeof(u64))) {842 pbl_tbl++;843 pbe = (struct ocrdma_pbe *)pbl_tbl->va;844 pbe_cnt = 0;845 }846 }847}848 849struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,850 u64 usr_addr, int acc, struct ib_udata *udata)851{852 int status = -ENOMEM;853 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);854 struct ocrdma_mr *mr;855 struct ocrdma_pd *pd;856 857 pd = get_ocrdma_pd(ibpd);858 859 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))860 return ERR_PTR(-EINVAL);861 862 mr = kzalloc(sizeof(*mr), GFP_KERNEL);863 if (!mr)864 return ERR_PTR(status);865 mr->umem = ib_umem_get(ibpd->device, start, len, acc);866 if (IS_ERR(mr->umem)) {867 status = -EFAULT;868 goto umem_err;869 }870 status = ocrdma_get_pbl_info(871 dev, mr, ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE));872 if (status)873 goto umem_err;874 875 mr->hwmr.pbe_size = PAGE_SIZE;876 mr->hwmr.va = usr_addr;877 mr->hwmr.len = len;878 mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;879 mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;880 mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;881 mr->hwmr.local_rd = 1;882 mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;883 status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);884 if (status)885 goto umem_err;886 build_user_pbes(dev, mr);887 status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc);888 if (status)889 goto mbx_err;890 mr->ibmr.lkey = mr->hwmr.lkey;891 if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)892 mr->ibmr.rkey = mr->hwmr.lkey;893 894 return &mr->ibmr;895 896mbx_err:897 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);898umem_err:899 kfree(mr);900 return ERR_PTR(status);901}902 903int ocrdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)904{905 struct ocrdma_mr *mr = get_ocrdma_mr(ib_mr);906 struct ocrdma_dev *dev = get_ocrdma_dev(ib_mr->device);907 908 (void) ocrdma_mbx_dealloc_lkey(dev, mr->hwmr.fr_mr, mr->hwmr.lkey);909 910 kfree(mr->pages);911 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);912 913 /* it could be user registered memory. */914 ib_umem_release(mr->umem);915 kfree(mr);916 917 /* Don't stop cleanup, in case FW is unresponsive */918 if (dev->mqe_ctx.fw_error_state) {919 pr_err("%s(%d) fw not responding.\n",920 __func__, dev->id);921 }922 return 0;923}924 925static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq,926 struct ib_udata *udata)927{928 int status;929 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(930 udata, struct ocrdma_ucontext, ibucontext);931 struct ocrdma_create_cq_uresp uresp;932 933 /* this must be user flow! */934 if (!udata)935 return -EINVAL;936 937 memset(&uresp, 0, sizeof(uresp));938 uresp.cq_id = cq->id;939 uresp.page_size = PAGE_ALIGN(cq->len);940 uresp.num_pages = 1;941 uresp.max_hw_cqe = cq->max_hw_cqe;942 uresp.page_addr[0] = virt_to_phys(cq->va);943 uresp.db_page_addr = ocrdma_get_db_addr(dev, uctx->cntxt_pd->id);944 uresp.db_page_size = dev->nic_info.db_page_size;945 uresp.phase_change = cq->phase_change ? 1 : 0;946 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));947 if (status) {948 pr_err("%s(%d) copy error cqid=0x%x.\n",949 __func__, dev->id, cq->id);950 goto err;951 }952 status = ocrdma_add_mmap(uctx, uresp.db_page_addr, uresp.db_page_size);953 if (status)954 goto err;955 status = ocrdma_add_mmap(uctx, uresp.page_addr[0], uresp.page_size);956 if (status) {957 ocrdma_del_mmap(uctx, uresp.db_page_addr, uresp.db_page_size);958 goto err;959 }960 cq->ucontext = uctx;961err:962 return status;963}964 965int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,966 struct uverbs_attr_bundle *attrs)967{968 struct ib_udata *udata = &attrs->driver_udata;969 struct ib_device *ibdev = ibcq->device;970 int entries = attr->cqe;971 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);972 struct ocrdma_dev *dev = get_ocrdma_dev(ibdev);973 struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(974 udata, struct ocrdma_ucontext, ibucontext);975 u16 pd_id = 0;976 int status;977 struct ocrdma_create_cq_ureq ureq;978 979 if (attr->flags)980 return -EOPNOTSUPP;981 982 if (udata) {983 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))984 return -EFAULT;985 } else986 ureq.dpp_cq = 0;987 988 spin_lock_init(&cq->cq_lock);989 spin_lock_init(&cq->comp_handler_lock);990 INIT_LIST_HEAD(&cq->sq_head);991 INIT_LIST_HEAD(&cq->rq_head);992 993 if (udata)994 pd_id = uctx->cntxt_pd->id;995 996 status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq, pd_id);997 if (status)998 return status;999 1000 if (udata) {1001 status = ocrdma_copy_cq_uresp(dev, cq, udata);1002 if (status)1003 goto ctx_err;1004 }1005 cq->phase = OCRDMA_CQE_VALID;1006 dev->cq_tbl[cq->id] = cq;1007 return 0;1008 1009ctx_err:1010 ocrdma_mbx_destroy_cq(dev, cq);1011 return status;1012}1013 1014int ocrdma_resize_cq(struct ib_cq *ibcq, int new_cnt,1015 struct ib_udata *udata)1016{1017 int status = 0;1018 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);1019 1020 if (new_cnt < 1 || new_cnt > cq->max_hw_cqe) {1021 status = -EINVAL;1022 return status;1023 }1024 ibcq->cqe = new_cnt;1025 return status;1026}1027 1028static void ocrdma_flush_cq(struct ocrdma_cq *cq)1029{1030 int cqe_cnt;1031 int valid_count = 0;1032 unsigned long flags;1033 1034 struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device);1035 struct ocrdma_cqe *cqe = NULL;1036 1037 cqe = cq->va;1038 cqe_cnt = cq->cqe_cnt;1039 1040 /* Last irq might have scheduled a polling thread1041 * sync-up with it before hard flushing.1042 */1043 spin_lock_irqsave(&cq->cq_lock, flags);1044 while (cqe_cnt) {1045 if (is_cqe_valid(cq, cqe))1046 valid_count++;1047 cqe++;1048 cqe_cnt--;1049 }1050 ocrdma_ring_cq_db(dev, cq->id, false, false, valid_count);1051 spin_unlock_irqrestore(&cq->cq_lock, flags);1052}1053 1054int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)1055{1056 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);1057 struct ocrdma_eq *eq = NULL;1058 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);1059 int pdid = 0;1060 u32 irq, indx;1061 1062 dev->cq_tbl[cq->id] = NULL;1063 indx = ocrdma_get_eq_table_index(dev, cq->eqn);1064 1065 eq = &dev->eq_tbl[indx];1066 irq = ocrdma_get_irq(dev, eq);1067 synchronize_irq(irq);1068 ocrdma_flush_cq(cq);1069 1070 ocrdma_mbx_destroy_cq(dev, cq);1071 if (cq->ucontext) {1072 pdid = cq->ucontext->cntxt_pd->id;1073 ocrdma_del_mmap(cq->ucontext, (u64) cq->pa,1074 PAGE_ALIGN(cq->len));1075 ocrdma_del_mmap(cq->ucontext,1076 ocrdma_get_db_addr(dev, pdid),1077 dev->nic_info.db_page_size);1078 }1079 return 0;1080}1081 1082static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)1083{1084 int status = -EINVAL;1085 1086 if (qp->id < OCRDMA_MAX_QP && dev->qp_tbl[qp->id] == NULL) {1087 dev->qp_tbl[qp->id] = qp;1088 status = 0;1089 }1090 return status;1091}1092 1093static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)1094{1095 dev->qp_tbl[qp->id] = NULL;1096}1097 1098static int ocrdma_check_qp_params(struct ib_pd *ibpd, struct ocrdma_dev *dev,1099 struct ib_qp_init_attr *attrs,1100 struct ib_udata *udata)1101{1102 if ((attrs->qp_type != IB_QPT_GSI) &&1103 (attrs->qp_type != IB_QPT_RC) &&1104 (attrs->qp_type != IB_QPT_UC) &&1105 (attrs->qp_type != IB_QPT_UD)) {1106 pr_err("%s(%d) unsupported qp type=0x%x requested\n",1107 __func__, dev->id, attrs->qp_type);1108 return -EOPNOTSUPP;1109 }1110 /* Skip the check for QP1 to support CM size of 128 */1111 if ((attrs->qp_type != IB_QPT_GSI) &&1112 (attrs->cap.max_send_wr > dev->attr.max_wqe)) {1113 pr_err("%s(%d) unsupported send_wr=0x%x requested\n",1114 __func__, dev->id, attrs->cap.max_send_wr);1115 pr_err("%s(%d) supported send_wr=0x%x\n",1116 __func__, dev->id, dev->attr.max_wqe);1117 return -EINVAL;1118 }1119 if (!attrs->srq && (attrs->cap.max_recv_wr > dev->attr.max_rqe)) {1120 pr_err("%s(%d) unsupported recv_wr=0x%x requested\n",1121 __func__, dev->id, attrs->cap.max_recv_wr);1122 pr_err("%s(%d) supported recv_wr=0x%x\n",1123 __func__, dev->id, dev->attr.max_rqe);1124 return -EINVAL;1125 }1126 if (attrs->cap.max_inline_data > dev->attr.max_inline_data) {1127 pr_err("%s(%d) unsupported inline data size=0x%x requested\n",1128 __func__, dev->id, attrs->cap.max_inline_data);1129 pr_err("%s(%d) supported inline data size=0x%x\n",1130 __func__, dev->id, dev->attr.max_inline_data);1131 return -EINVAL;1132 }1133 if (attrs->cap.max_send_sge > dev->attr.max_send_sge) {1134 pr_err("%s(%d) unsupported send_sge=0x%x requested\n",1135 __func__, dev->id, attrs->cap.max_send_sge);1136 pr_err("%s(%d) supported send_sge=0x%x\n",1137 __func__, dev->id, dev->attr.max_send_sge);1138 return -EINVAL;1139 }1140 if (attrs->cap.max_recv_sge > dev->attr.max_recv_sge) {1141 pr_err("%s(%d) unsupported recv_sge=0x%x requested\n",1142 __func__, dev->id, attrs->cap.max_recv_sge);1143 pr_err("%s(%d) supported recv_sge=0x%x\n",1144 __func__, dev->id, dev->attr.max_recv_sge);1145 return -EINVAL;1146 }1147 /* unprivileged user space cannot create special QP */1148 if (udata && attrs->qp_type == IB_QPT_GSI) {1149 pr_err1150 ("%s(%d) Userspace can't create special QPs of type=0x%x\n",1151 __func__, dev->id, attrs->qp_type);1152 return -EINVAL;1153 }1154 /* allow creating only one GSI type of QP */1155 if (attrs->qp_type == IB_QPT_GSI && dev->gsi_qp_created) {1156 pr_err("%s(%d) GSI special QPs already created.\n",1157 __func__, dev->id);1158 return -EINVAL;1159 }1160 /* verify consumer QPs are not trying to use GSI QP's CQ */1161 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created)) {1162 if ((dev->gsi_sqcq == get_ocrdma_cq(attrs->send_cq)) ||1163 (dev->gsi_rqcq == get_ocrdma_cq(attrs->recv_cq))) {1164 pr_err("%s(%d) Consumer QP cannot use GSI CQs.\n",1165 __func__, dev->id);1166 return -EINVAL;1167 }1168 }1169 return 0;1170}1171 1172static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp,1173 struct ib_udata *udata, int dpp_offset,1174 int dpp_credit_lmt, int srq)1175{1176 int status;1177 u64 usr_db;1178 struct ocrdma_create_qp_uresp uresp;1179 struct ocrdma_pd *pd = qp->pd;1180 struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device);1181 1182 memset(&uresp, 0, sizeof(uresp));1183 usr_db = dev->nic_info.unmapped_db +1184 (pd->id * dev->nic_info.db_page_size);1185 uresp.qp_id = qp->id;1186 uresp.sq_dbid = qp->sq.dbid;1187 uresp.num_sq_pages = 1;1188 uresp.sq_page_size = PAGE_ALIGN(qp->sq.len);1189 uresp.sq_page_addr[0] = virt_to_phys(qp->sq.va);1190 uresp.num_wqe_allocated = qp->sq.max_cnt;1191 if (!srq) {1192 uresp.rq_dbid = qp->rq.dbid;1193 uresp.num_rq_pages = 1;1194 uresp.rq_page_size = PAGE_ALIGN(qp->rq.len);1195 uresp.rq_page_addr[0] = virt_to_phys(qp->rq.va);1196 uresp.num_rqe_allocated = qp->rq.max_cnt;1197 }1198 uresp.db_page_addr = usr_db;1199 uresp.db_page_size = dev->nic_info.db_page_size;1200 uresp.db_sq_offset = OCRDMA_DB_GEN2_SQ_OFFSET;1201 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET;1202 uresp.db_shift = OCRDMA_DB_RQ_SHIFT;1203 1204 if (qp->dpp_enabled) {1205 uresp.dpp_credit = dpp_credit_lmt;1206 uresp.dpp_offset = dpp_offset;1207 }1208 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));1209 if (status) {1210 pr_err("%s(%d) user copy error.\n", __func__, dev->id);1211 goto err;1212 }1213 status = ocrdma_add_mmap(pd->uctx, uresp.sq_page_addr[0],1214 uresp.sq_page_size);1215 if (status)1216 goto err;1217 1218 if (!srq) {1219 status = ocrdma_add_mmap(pd->uctx, uresp.rq_page_addr[0],1220 uresp.rq_page_size);1221 if (status)1222 goto rq_map_err;1223 }1224 return status;1225rq_map_err:1226 ocrdma_del_mmap(pd->uctx, uresp.sq_page_addr[0], uresp.sq_page_size);1227err:1228 return status;1229}1230 1231static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp,1232 struct ocrdma_pd *pd)1233{1234 if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) {1235 qp->sq_db = dev->nic_info.db +1236 (pd->id * dev->nic_info.db_page_size) +1237 OCRDMA_DB_GEN2_SQ_OFFSET;1238 qp->rq_db = dev->nic_info.db +1239 (pd->id * dev->nic_info.db_page_size) +1240 OCRDMA_DB_GEN2_RQ_OFFSET;1241 } else {1242 qp->sq_db = dev->nic_info.db +1243 (pd->id * dev->nic_info.db_page_size) +1244 OCRDMA_DB_SQ_OFFSET;1245 qp->rq_db = dev->nic_info.db +1246 (pd->id * dev->nic_info.db_page_size) +1247 OCRDMA_DB_RQ_OFFSET;1248 }1249}1250 1251static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp)1252{1253 qp->wqe_wr_id_tbl =1254 kcalloc(qp->sq.max_cnt, sizeof(*(qp->wqe_wr_id_tbl)),1255 GFP_KERNEL);1256 if (qp->wqe_wr_id_tbl == NULL)1257 return -ENOMEM;1258 qp->rqe_wr_id_tbl =1259 kcalloc(qp->rq.max_cnt, sizeof(u64), GFP_KERNEL);1260 if (qp->rqe_wr_id_tbl == NULL)1261 return -ENOMEM;1262 1263 return 0;1264}1265 1266static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp,1267 struct ocrdma_pd *pd,1268 struct ib_qp_init_attr *attrs)1269{1270 qp->pd = pd;1271 spin_lock_init(&qp->q_lock);1272 INIT_LIST_HEAD(&qp->sq_entry);1273 INIT_LIST_HEAD(&qp->rq_entry);1274 1275 qp->qp_type = attrs->qp_type;1276 qp->cap_flags = OCRDMA_QP_INB_RD | OCRDMA_QP_INB_WR;1277 qp->max_inline_data = attrs->cap.max_inline_data;1278 qp->sq.max_sges = attrs->cap.max_send_sge;1279 qp->rq.max_sges = attrs->cap.max_recv_sge;1280 qp->state = OCRDMA_QPS_RST;1281 qp->signaled = attrs->sq_sig_type == IB_SIGNAL_ALL_WR;1282}1283 1284static void ocrdma_store_gsi_qp_cq(struct ocrdma_dev *dev,1285 struct ib_qp_init_attr *attrs)1286{1287 if (attrs->qp_type == IB_QPT_GSI) {1288 dev->gsi_qp_created = 1;1289 dev->gsi_sqcq = get_ocrdma_cq(attrs->send_cq);1290 dev->gsi_rqcq = get_ocrdma_cq(attrs->recv_cq);1291 }1292}1293 1294int ocrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,1295 struct ib_udata *udata)1296{1297 int status;1298 struct ib_pd *ibpd = ibqp->pd;1299 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);1300 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);1301 struct ocrdma_dev *dev = get_ocrdma_dev(ibqp->device);1302 struct ocrdma_create_qp_ureq ureq;1303 u16 dpp_credit_lmt, dpp_offset;1304 1305 if (attrs->create_flags)1306 return -EOPNOTSUPP;1307 1308 status = ocrdma_check_qp_params(ibpd, dev, attrs, udata);1309 if (status)1310 goto gen_err;1311 1312 memset(&ureq, 0, sizeof(ureq));1313 if (udata) {1314 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq)))1315 return -EFAULT;1316 }1317 ocrdma_set_qp_init_params(qp, pd, attrs);1318 if (udata == NULL)1319 qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 |1320 OCRDMA_QP_FAST_REG);1321 1322 mutex_lock(&dev->dev_lock);1323 status = ocrdma_mbx_create_qp(qp, attrs, ureq.enable_dpp_cq,1324 ureq.dpp_cq_id,1325 &dpp_offset, &dpp_credit_lmt);1326 if (status)1327 goto mbx_err;1328 1329 /* user space QP's wr_id table are managed in library */1330 if (udata == NULL) {1331 status = ocrdma_alloc_wr_id_tbl(qp);1332 if (status)1333 goto map_err;1334 }1335 1336 status = ocrdma_add_qpn_map(dev, qp);1337 if (status)1338 goto map_err;1339 ocrdma_set_qp_db(dev, qp, pd);1340 if (udata) {1341 status = ocrdma_copy_qp_uresp(qp, udata, dpp_offset,1342 dpp_credit_lmt,1343 (attrs->srq != NULL));1344 if (status)1345 goto cpy_err;1346 }1347 ocrdma_store_gsi_qp_cq(dev, attrs);1348 qp->ibqp.qp_num = qp->id;1349 mutex_unlock(&dev->dev_lock);1350 return 0;1351 1352cpy_err:1353 ocrdma_del_qpn_map(dev, qp);1354map_err:1355 ocrdma_mbx_destroy_qp(dev, qp);1356mbx_err:1357 mutex_unlock(&dev->dev_lock);1358 kfree(qp->wqe_wr_id_tbl);1359 kfree(qp->rqe_wr_id_tbl);1360 pr_err("%s(%d) error=%d\n", __func__, dev->id, status);1361gen_err:1362 return status;1363}1364 1365int _ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,1366 int attr_mask)1367{1368 int status = 0;1369 struct ocrdma_qp *qp;1370 struct ocrdma_dev *dev;1371 enum ib_qp_state old_qps;1372 1373 qp = get_ocrdma_qp(ibqp);1374 dev = get_ocrdma_dev(ibqp->device);1375 if (attr_mask & IB_QP_STATE)1376 status = ocrdma_qp_state_change(qp, attr->qp_state, &old_qps);1377 /* if new and previous states are same hw doesn't need to1378 * know about it.1379 */1380 if (status < 0)1381 return status;1382 return ocrdma_mbx_modify_qp(dev, qp, attr, attr_mask);1383}1384 1385int ocrdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,1386 int attr_mask, struct ib_udata *udata)1387{1388 unsigned long flags;1389 int status = -EINVAL;1390 struct ocrdma_qp *qp;1391 struct ocrdma_dev *dev;1392 enum ib_qp_state old_qps, new_qps;1393 1394 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)1395 return -EOPNOTSUPP;1396 1397 qp = get_ocrdma_qp(ibqp);1398 dev = get_ocrdma_dev(ibqp->device);1399 1400 /* syncronize with multiple context trying to change, retrive qps */1401 mutex_lock(&dev->dev_lock);1402 /* syncronize with wqe, rqe posting and cqe processing contexts */1403 spin_lock_irqsave(&qp->q_lock, flags);1404 old_qps = get_ibqp_state(qp->state);1405 if (attr_mask & IB_QP_STATE)1406 new_qps = attr->qp_state;1407 else1408 new_qps = old_qps;1409 spin_unlock_irqrestore(&qp->q_lock, flags);1410 1411 if (!ib_modify_qp_is_ok(old_qps, new_qps, ibqp->qp_type, attr_mask)) {1412 pr_err("%s(%d) invalid attribute mask=0x%x specified for\n"1413 "qpn=0x%x of type=0x%x old_qps=0x%x, new_qps=0x%x\n",1414 __func__, dev->id, attr_mask, qp->id, ibqp->qp_type,1415 old_qps, new_qps);1416 goto param_err;1417 }1418 1419 status = _ocrdma_modify_qp(ibqp, attr, attr_mask);1420 if (status > 0)1421 status = 0;1422param_err:1423 mutex_unlock(&dev->dev_lock);1424 return status;1425}1426 1427static enum ib_mtu ocrdma_mtu_int_to_enum(u16 mtu)1428{1429 switch (mtu) {1430 case 256:1431 return IB_MTU_256;1432 case 512:1433 return IB_MTU_512;1434 case 1024:1435 return IB_MTU_1024;1436 case 2048:1437 return IB_MTU_2048;1438 case 4096:1439 return IB_MTU_4096;1440 default:1441 return IB_MTU_1024;1442 }1443}1444 1445static int ocrdma_to_ib_qp_acc_flags(int qp_cap_flags)1446{1447 int ib_qp_acc_flags = 0;1448 1449 if (qp_cap_flags & OCRDMA_QP_INB_WR)1450 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;1451 if (qp_cap_flags & OCRDMA_QP_INB_RD)1452 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;1453 return ib_qp_acc_flags;1454}1455 1456int ocrdma_query_qp(struct ib_qp *ibqp,1457 struct ib_qp_attr *qp_attr,1458 int attr_mask, struct ib_qp_init_attr *qp_init_attr)1459{1460 int status;1461 u32 qp_state;1462 struct ocrdma_qp_params params;1463 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);1464 struct ocrdma_dev *dev = get_ocrdma_dev(ibqp->device);1465 1466 memset(¶ms, 0, sizeof(params));1467 mutex_lock(&dev->dev_lock);1468 status = ocrdma_mbx_query_qp(dev, qp, ¶ms);1469 mutex_unlock(&dev->dev_lock);1470 if (status)1471 goto mbx_err;1472 if (qp->qp_type == IB_QPT_UD)1473 qp_attr->qkey = params.qkey;1474 qp_attr->path_mtu =1475 ocrdma_mtu_int_to_enum(params.path_mtu_pkey_indx &1476 OCRDMA_QP_PARAMS_PATH_MTU_MASK) >>1477 OCRDMA_QP_PARAMS_PATH_MTU_SHIFT;1478 qp_attr->path_mig_state = IB_MIG_MIGRATED;1479 qp_attr->rq_psn = params.hop_lmt_rq_psn & OCRDMA_QP_PARAMS_RQ_PSN_MASK;1480 qp_attr->sq_psn = params.tclass_sq_psn & OCRDMA_QP_PARAMS_SQ_PSN_MASK;1481 qp_attr->dest_qp_num =1482 params.ack_to_rnr_rtc_dest_qpn & OCRDMA_QP_PARAMS_DEST_QPN_MASK;1483 1484 qp_attr->qp_access_flags = ocrdma_to_ib_qp_acc_flags(qp->cap_flags);1485 qp_attr->cap.max_send_wr = qp->sq.max_cnt - 1;1486 qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1;1487 qp_attr->cap.max_send_sge = qp->sq.max_sges;1488 qp_attr->cap.max_recv_sge = qp->rq.max_sges;1489 qp_attr->cap.max_inline_data = qp->max_inline_data;1490 qp_init_attr->cap = qp_attr->cap;1491 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;1492 1493 rdma_ah_set_grh(&qp_attr->ah_attr, NULL,1494 params.rnt_rc_sl_fl &1495 OCRDMA_QP_PARAMS_FLOW_LABEL_MASK,1496 qp->sgid_idx,1497 (params.hop_lmt_rq_psn &1498 OCRDMA_QP_PARAMS_HOP_LMT_MASK) >>1499 OCRDMA_QP_PARAMS_HOP_LMT_SHIFT,1500 (params.tclass_sq_psn &1501 OCRDMA_QP_PARAMS_TCLASS_MASK) >>1502 OCRDMA_QP_PARAMS_TCLASS_SHIFT);1503 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid[0]);1504 1505 rdma_ah_set_port_num(&qp_attr->ah_attr, 1);1506 rdma_ah_set_sl(&qp_attr->ah_attr, (params.rnt_rc_sl_fl &1507 OCRDMA_QP_PARAMS_SL_MASK) >>1508 OCRDMA_QP_PARAMS_SL_SHIFT);1509 qp_attr->timeout = (params.ack_to_rnr_rtc_dest_qpn &1510 OCRDMA_QP_PARAMS_ACK_TIMEOUT_MASK) >>1511 OCRDMA_QP_PARAMS_ACK_TIMEOUT_SHIFT;1512 qp_attr->rnr_retry = (params.ack_to_rnr_rtc_dest_qpn &1513 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_MASK) >>1514 OCRDMA_QP_PARAMS_RNR_RETRY_CNT_SHIFT;1515 qp_attr->retry_cnt =1516 (params.rnt_rc_sl_fl & OCRDMA_QP_PARAMS_RETRY_CNT_MASK) >>1517 OCRDMA_QP_PARAMS_RETRY_CNT_SHIFT;1518 qp_attr->min_rnr_timer = 0;1519 qp_attr->pkey_index = 0;1520 qp_attr->port_num = 1;1521 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);1522 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);1523 qp_attr->alt_pkey_index = 0;1524 qp_attr->alt_port_num = 0;1525 qp_attr->alt_timeout = 0;1526 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));1527 qp_state = (params.max_sge_recv_flags & OCRDMA_QP_PARAMS_STATE_MASK) >>1528 OCRDMA_QP_PARAMS_STATE_SHIFT;1529 qp_attr->qp_state = get_ibqp_state(qp_state);1530 qp_attr->cur_qp_state = qp_attr->qp_state;1531 qp_attr->sq_draining = (qp_state == OCRDMA_QPS_SQ_DRAINING) ? 1 : 0;1532 qp_attr->max_dest_rd_atomic =1533 params.max_ord_ird >> OCRDMA_QP_PARAMS_MAX_ORD_SHIFT;1534 qp_attr->max_rd_atomic =1535 params.max_ord_ird & OCRDMA_QP_PARAMS_MAX_IRD_MASK;1536 qp_attr->en_sqd_async_notify = (params.max_sge_recv_flags &1537 OCRDMA_QP_PARAMS_FLAGS_SQD_ASYNC) ? 1 : 0;1538 /* Sync driver QP state with FW */1539 ocrdma_qp_state_change(qp, qp_attr->qp_state, NULL);1540mbx_err:1541 return status;1542}1543 1544static void ocrdma_srq_toggle_bit(struct ocrdma_srq *srq, unsigned int idx)1545{1546 unsigned int i = idx / 32;1547 u32 mask = (1U << (idx % 32));1548 1549 srq->idx_bit_fields[i] ^= mask;1550}1551 1552static int ocrdma_hwq_free_cnt(struct ocrdma_qp_hwq_info *q)1553{1554 return ((q->max_wqe_idx - q->head) + q->tail) % q->max_cnt;1555}1556 1557static int is_hw_sq_empty(struct ocrdma_qp *qp)1558{1559 return (qp->sq.tail == qp->sq.head);1560}1561 1562static int is_hw_rq_empty(struct ocrdma_qp *qp)1563{1564 return (qp->rq.tail == qp->rq.head);1565}1566 1567static void *ocrdma_hwq_head(struct ocrdma_qp_hwq_info *q)1568{1569 return q->va + (q->head * q->entry_size);1570}1571 1572static void *ocrdma_hwq_head_from_idx(struct ocrdma_qp_hwq_info *q,1573 u32 idx)1574{1575 return q->va + (idx * q->entry_size);1576}1577 1578static void ocrdma_hwq_inc_head(struct ocrdma_qp_hwq_info *q)1579{1580 q->head = (q->head + 1) & q->max_wqe_idx;1581}1582 1583static void ocrdma_hwq_inc_tail(struct ocrdma_qp_hwq_info *q)1584{1585 q->tail = (q->tail + 1) & q->max_wqe_idx;1586}1587 1588/* discard the cqe for a given QP */1589static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq)1590{1591 unsigned long cq_flags;1592 unsigned long flags;1593 u32 cur_getp, stop_getp;1594 struct ocrdma_cqe *cqe;1595 u32 qpn = 0, wqe_idx = 0;1596 1597 spin_lock_irqsave(&cq->cq_lock, cq_flags);1598 1599 /* traverse through the CQEs in the hw CQ,1600 * find the matching CQE for a given qp,1601 * mark the matching one discarded by clearing qpn.1602 * ring the doorbell in the poll_cq() as1603 * we don't complete out of order cqe.1604 */1605 1606 cur_getp = cq->getp;1607 /* find upto when do we reap the cq. */1608 stop_getp = cur_getp;1609 do {1610 if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp)))1611 break;1612 1613 cqe = cq->va + cur_getp;1614 /* if (a) done reaping whole hw cq, or1615 * (b) qp_xq becomes empty.1616 * then exit1617 */1618 qpn = cqe->cmn.qpn & OCRDMA_CQE_QPN_MASK;1619 /* if previously discarded cqe found, skip that too. */1620 /* check for matching qp */1621 if (qpn == 0 || qpn != qp->id)1622 goto skip_cqe;1623 1624 if (is_cqe_for_sq(cqe)) {1625 ocrdma_hwq_inc_tail(&qp->sq);1626 } else {1627 if (qp->srq) {1628 wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >>1629 OCRDMA_CQE_BUFTAG_SHIFT) &1630 qp->srq->rq.max_wqe_idx;1631 BUG_ON(wqe_idx < 1);1632 spin_lock_irqsave(&qp->srq->q_lock, flags);1633 ocrdma_hwq_inc_tail(&qp->srq->rq);1634 ocrdma_srq_toggle_bit(qp->srq, wqe_idx - 1);1635 spin_unlock_irqrestore(&qp->srq->q_lock, flags);1636 1637 } else {1638 ocrdma_hwq_inc_tail(&qp->rq);1639 }1640 }1641 /* mark cqe discarded so that it is not picked up later1642 * in the poll_cq().1643 */1644 cqe->cmn.qpn = 0;1645skip_cqe:1646 cur_getp = (cur_getp + 1) % cq->max_hw_cqe;1647 } while (cur_getp != stop_getp);1648 spin_unlock_irqrestore(&cq->cq_lock, cq_flags);1649}1650 1651void ocrdma_del_flush_qp(struct ocrdma_qp *qp)1652{1653 int found = false;1654 unsigned long flags;1655 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device);1656 /* sync with any active CQ poll */1657 1658 spin_lock_irqsave(&dev->flush_q_lock, flags);1659 found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp);1660 if (found)1661 list_del(&qp->sq_entry);1662 if (!qp->srq) {1663 found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp);1664 if (found)1665 list_del(&qp->rq_entry);1666 }1667 spin_unlock_irqrestore(&dev->flush_q_lock, flags);1668}1669 1670int ocrdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)1671{1672 struct ocrdma_pd *pd;1673 struct ocrdma_qp *qp;1674 struct ocrdma_dev *dev;1675 struct ib_qp_attr attrs;1676 int attr_mask;1677 unsigned long flags;1678 1679 qp = get_ocrdma_qp(ibqp);1680 dev = get_ocrdma_dev(ibqp->device);1681 1682 pd = qp->pd;1683 1684 /* change the QP state to ERROR */1685 if (qp->state != OCRDMA_QPS_RST) {1686 attrs.qp_state = IB_QPS_ERR;1687 attr_mask = IB_QP_STATE;1688 _ocrdma_modify_qp(ibqp, &attrs, attr_mask);1689 }1690 /* ensure that CQEs for newly created QP (whose id may be same with1691 * one which just getting destroyed are same), dont get1692 * discarded until the old CQEs are discarded.1693 */1694 mutex_lock(&dev->dev_lock);1695 (void) ocrdma_mbx_destroy_qp(dev, qp);1696 1697 /*1698 * acquire CQ lock while destroy is in progress, in order to1699 * protect against proessing in-flight CQEs for this QP.1700 */1701 spin_lock_irqsave(&qp->sq_cq->cq_lock, flags);1702 if (qp->rq_cq && (qp->rq_cq != qp->sq_cq)) {1703 spin_lock(&qp->rq_cq->cq_lock);1704 ocrdma_del_qpn_map(dev, qp);1705 spin_unlock(&qp->rq_cq->cq_lock);1706 } else {1707 ocrdma_del_qpn_map(dev, qp);1708 }1709 spin_unlock_irqrestore(&qp->sq_cq->cq_lock, flags);1710 1711 if (!pd->uctx) {1712 ocrdma_discard_cqes(qp, qp->sq_cq);1713 ocrdma_discard_cqes(qp, qp->rq_cq);1714 }1715 mutex_unlock(&dev->dev_lock);1716 1717 if (pd->uctx) {1718 ocrdma_del_mmap(pd->uctx, (u64) qp->sq.pa,1719 PAGE_ALIGN(qp->sq.len));1720 if (!qp->srq)1721 ocrdma_del_mmap(pd->uctx, (u64) qp->rq.pa,1722 PAGE_ALIGN(qp->rq.len));1723 }1724 1725 ocrdma_del_flush_qp(qp);1726 1727 kfree(qp->wqe_wr_id_tbl);1728 kfree(qp->rqe_wr_id_tbl);1729 return 0;1730}1731 1732static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq,1733 struct ib_udata *udata)1734{1735 int status;1736 struct ocrdma_create_srq_uresp uresp;1737 1738 memset(&uresp, 0, sizeof(uresp));1739 uresp.rq_dbid = srq->rq.dbid;1740 uresp.num_rq_pages = 1;1741 uresp.rq_page_addr[0] = virt_to_phys(srq->rq.va);1742 uresp.rq_page_size = srq->rq.len;1743 uresp.db_page_addr = dev->nic_info.unmapped_db +1744 (srq->pd->id * dev->nic_info.db_page_size);1745 uresp.db_page_size = dev->nic_info.db_page_size;1746 uresp.num_rqe_allocated = srq->rq.max_cnt;1747 if (ocrdma_get_asic_type(dev) == OCRDMA_ASIC_GEN_SKH_R) {1748 uresp.db_rq_offset = OCRDMA_DB_GEN2_RQ_OFFSET;1749 uresp.db_shift = 24;1750 } else {1751 uresp.db_rq_offset = OCRDMA_DB_RQ_OFFSET;1752 uresp.db_shift = 16;1753 }1754 1755 status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));1756 if (status)1757 return status;1758 status = ocrdma_add_mmap(srq->pd->uctx, uresp.rq_page_addr[0],1759 uresp.rq_page_size);1760 if (status)1761 return status;1762 return status;1763}1764 1765int ocrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,1766 struct ib_udata *udata)1767{1768 int status;1769 struct ocrdma_pd *pd = get_ocrdma_pd(ibsrq->pd);1770 struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device);1771 struct ocrdma_srq *srq = get_ocrdma_srq(ibsrq);1772 1773 if (init_attr->srq_type != IB_SRQT_BASIC)1774 return -EOPNOTSUPP;1775 1776 if (init_attr->attr.max_sge > dev->attr.max_recv_sge)1777 return -EINVAL;1778 if (init_attr->attr.max_wr > dev->attr.max_rqe)1779 return -EINVAL;1780 1781 spin_lock_init(&srq->q_lock);1782 srq->pd = pd;1783 srq->db = dev->nic_info.db + (pd->id * dev->nic_info.db_page_size);1784 status = ocrdma_mbx_create_srq(dev, srq, init_attr, pd);1785 if (status)1786 return status;1787 1788 if (!udata) {1789 srq->rqe_wr_id_tbl = kcalloc(srq->rq.max_cnt, sizeof(u64),1790 GFP_KERNEL);1791 if (!srq->rqe_wr_id_tbl) {1792 status = -ENOMEM;1793 goto arm_err;1794 }1795 1796 srq->bit_fields_len = (srq->rq.max_cnt / 32) +1797 (srq->rq.max_cnt % 32 ? 1 : 0);1798 srq->idx_bit_fields =1799 kmalloc_array(srq->bit_fields_len, sizeof(u32),1800 GFP_KERNEL);1801 if (!srq->idx_bit_fields) {1802 status = -ENOMEM;1803 goto arm_err;1804 }1805 memset(srq->idx_bit_fields, 0xff,1806 srq->bit_fields_len * sizeof(u32));1807 }1808 1809 if (init_attr->attr.srq_limit) {1810 status = ocrdma_mbx_modify_srq(srq, &init_attr->attr);1811 if (status)1812 goto arm_err;1813 }1814 1815 if (udata) {1816 status = ocrdma_copy_srq_uresp(dev, srq, udata);1817 if (status)1818 goto arm_err;1819 }1820 1821 return 0;1822 1823arm_err:1824 ocrdma_mbx_destroy_srq(dev, srq);1825 kfree(srq->rqe_wr_id_tbl);1826 kfree(srq->idx_bit_fields);1827 return status;1828}1829 1830int ocrdma_modify_srq(struct ib_srq *ibsrq,1831 struct ib_srq_attr *srq_attr,1832 enum ib_srq_attr_mask srq_attr_mask,1833 struct ib_udata *udata)1834{1835 int status;1836 struct ocrdma_srq *srq;1837 1838 srq = get_ocrdma_srq(ibsrq);1839 if (srq_attr_mask & IB_SRQ_MAX_WR)1840 status = -EINVAL;1841 else1842 status = ocrdma_mbx_modify_srq(srq, srq_attr);1843 return status;1844}1845 1846int ocrdma_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)1847{1848 struct ocrdma_srq *srq;1849 1850 srq = get_ocrdma_srq(ibsrq);1851 return ocrdma_mbx_query_srq(srq, srq_attr);1852}1853 1854int ocrdma_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)1855{1856 struct ocrdma_srq *srq;1857 struct ocrdma_dev *dev = get_ocrdma_dev(ibsrq->device);1858 1859 srq = get_ocrdma_srq(ibsrq);1860 1861 ocrdma_mbx_destroy_srq(dev, srq);1862 1863 if (srq->pd->uctx)1864 ocrdma_del_mmap(srq->pd->uctx, (u64) srq->rq.pa,1865 PAGE_ALIGN(srq->rq.len));1866 1867 kfree(srq->idx_bit_fields);1868 kfree(srq->rqe_wr_id_tbl);1869 return 0;1870}1871 1872/* unprivileged verbs and their support functions. */1873static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp,1874 struct ocrdma_hdr_wqe *hdr,1875 const struct ib_send_wr *wr)1876{1877 struct ocrdma_ewqe_ud_hdr *ud_hdr =1878 (struct ocrdma_ewqe_ud_hdr *)(hdr + 1);1879 struct ocrdma_ah *ah = get_ocrdma_ah(ud_wr(wr)->ah);1880 1881 ud_hdr->rsvd_dest_qpn = ud_wr(wr)->remote_qpn;1882 if (qp->qp_type == IB_QPT_GSI)1883 ud_hdr->qkey = qp->qkey;1884 else1885 ud_hdr->qkey = ud_wr(wr)->remote_qkey;1886 ud_hdr->rsvd_ahid = ah->id;1887 ud_hdr->hdr_type = ah->hdr_type;1888 if (ah->av->valid & OCRDMA_AV_VLAN_VALID)1889 hdr->cw |= (OCRDMA_FLAG_AH_VLAN_PR << OCRDMA_WQE_FLAGS_SHIFT);1890}1891 1892static void ocrdma_build_sges(struct ocrdma_hdr_wqe *hdr,1893 struct ocrdma_sge *sge, int num_sge,1894 struct ib_sge *sg_list)1895{1896 int i;1897 1898 for (i = 0; i < num_sge; i++) {1899 sge[i].lrkey = sg_list[i].lkey;1900 sge[i].addr_lo = sg_list[i].addr;1901 sge[i].addr_hi = upper_32_bits(sg_list[i].addr);1902 sge[i].len = sg_list[i].length;1903 hdr->total_len += sg_list[i].length;1904 }1905 if (num_sge == 0)1906 memset(sge, 0, sizeof(*sge));1907}1908 1909static inline uint32_t ocrdma_sglist_len(struct ib_sge *sg_list, int num_sge)1910{1911 uint32_t total_len = 0, i;1912 1913 for (i = 0; i < num_sge; i++)1914 total_len += sg_list[i].length;1915 return total_len;1916}1917 1918 1919static int ocrdma_build_inline_sges(struct ocrdma_qp *qp,1920 struct ocrdma_hdr_wqe *hdr,1921 struct ocrdma_sge *sge,1922 const struct ib_send_wr *wr, u32 wqe_size)1923{1924 int i;1925 char *dpp_addr;1926 1927 if (wr->send_flags & IB_SEND_INLINE && qp->qp_type != IB_QPT_UD) {1928 hdr->total_len = ocrdma_sglist_len(wr->sg_list, wr->num_sge);1929 if (unlikely(hdr->total_len > qp->max_inline_data)) {1930 pr_err("%s() supported_len=0x%x,\n"1931 " unsupported len req=0x%x\n", __func__,1932 qp->max_inline_data, hdr->total_len);1933 return -EINVAL;1934 }1935 dpp_addr = (char *)sge;1936 for (i = 0; i < wr->num_sge; i++) {1937 memcpy(dpp_addr,1938 (void *)(unsigned long)wr->sg_list[i].addr,1939 wr->sg_list[i].length);1940 dpp_addr += wr->sg_list[i].length;1941 }1942 1943 wqe_size += roundup(hdr->total_len, OCRDMA_WQE_ALIGN_BYTES);1944 if (0 == hdr->total_len)1945 wqe_size += sizeof(struct ocrdma_sge);1946 hdr->cw |= (OCRDMA_TYPE_INLINE << OCRDMA_WQE_TYPE_SHIFT);1947 } else {1948 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list);1949 if (wr->num_sge)1950 wqe_size += (wr->num_sge * sizeof(struct ocrdma_sge));1951 else1952 wqe_size += sizeof(struct ocrdma_sge);1953 hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);1954 }1955 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);1956 return 0;1957}1958 1959static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,1960 const struct ib_send_wr *wr)1961{1962 struct ocrdma_sge *sge;1963 u32 wqe_size = sizeof(*hdr);1964 1965 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {1966 ocrdma_build_ud_hdr(qp, hdr, wr);1967 sge = (struct ocrdma_sge *)(hdr + 2);1968 wqe_size += sizeof(struct ocrdma_ewqe_ud_hdr);1969 } else {1970 sge = (struct ocrdma_sge *)(hdr + 1);1971 }1972 1973 return ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);1974}1975 1976static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,1977 const struct ib_send_wr *wr)1978{1979 int status;1980 struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1);1981 struct ocrdma_sge *sge = ext_rw + 1;1982 u32 wqe_size = sizeof(*hdr) + sizeof(*ext_rw);1983 1984 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);1985 if (status)1986 return status;1987 ext_rw->addr_lo = rdma_wr(wr)->remote_addr;1988 ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr);1989 ext_rw->lrkey = rdma_wr(wr)->rkey;1990 ext_rw->len = hdr->total_len;1991 return 0;1992}1993 1994static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,1995 const struct ib_send_wr *wr)1996{1997 struct ocrdma_sge *ext_rw = (struct ocrdma_sge *)(hdr + 1);1998 struct ocrdma_sge *sge = ext_rw + 1;1999 u32 wqe_size = ((wr->num_sge + 1) * sizeof(struct ocrdma_sge)) +2000 sizeof(struct ocrdma_hdr_wqe);2001 2002 ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list);2003 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);2004 hdr->cw |= (OCRDMA_READ << OCRDMA_WQE_OPCODE_SHIFT);2005 hdr->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);2006 2007 ext_rw->addr_lo = rdma_wr(wr)->remote_addr;2008 ext_rw->addr_hi = upper_32_bits(rdma_wr(wr)->remote_addr);2009 ext_rw->lrkey = rdma_wr(wr)->rkey;2010 ext_rw->len = hdr->total_len;2011}2012 2013static int get_encoded_page_size(int pg_sz)2014{2015 /* Max size is 256M 4096 << 16 */2016 int i = 0;2017 for (; i < 17; i++)2018 if (pg_sz == (4096 << i))2019 break;2020 return i;2021}2022 2023static int ocrdma_build_reg(struct ocrdma_qp *qp,2024 struct ocrdma_hdr_wqe *hdr,2025 const struct ib_reg_wr *wr)2026{2027 u64 fbo;2028 struct ocrdma_ewqe_fr *fast_reg = (struct ocrdma_ewqe_fr *)(hdr + 1);2029 struct ocrdma_mr *mr = get_ocrdma_mr(wr->mr);2030 struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table;2031 struct ocrdma_pbe *pbe;2032 u32 wqe_size = sizeof(*fast_reg) + sizeof(*hdr);2033 int num_pbes = 0, i;2034 2035 wqe_size = roundup(wqe_size, OCRDMA_WQE_ALIGN_BYTES);2036 2037 hdr->cw |= (OCRDMA_FR_MR << OCRDMA_WQE_OPCODE_SHIFT);2038 hdr->cw |= ((wqe_size / OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT);2039 2040 if (wr->access & IB_ACCESS_LOCAL_WRITE)2041 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_LOCAL_WR;2042 if (wr->access & IB_ACCESS_REMOTE_WRITE)2043 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_WR;2044 if (wr->access & IB_ACCESS_REMOTE_READ)2045 hdr->rsvd_lkey_flags |= OCRDMA_LKEY_FLAG_REMOTE_RD;2046 hdr->lkey = wr->key;2047 hdr->total_len = mr->ibmr.length;2048 2049 fbo = mr->ibmr.iova - mr->pages[0];2050 2051 fast_reg->va_hi = upper_32_bits(mr->ibmr.iova);2052 fast_reg->va_lo = (u32) (mr->ibmr.iova & 0xffffffff);2053 fast_reg->fbo_hi = upper_32_bits(fbo);2054 fast_reg->fbo_lo = (u32) fbo & 0xffffffff;2055 fast_reg->num_sges = mr->npages;2056 fast_reg->size_sge = get_encoded_page_size(mr->ibmr.page_size);2057 2058 pbe = pbl_tbl->va;2059 for (i = 0; i < mr->npages; i++) {2060 u64 buf_addr = mr->pages[i];2061 2062 pbe->pa_lo = cpu_to_le32((u32) (buf_addr & PAGE_MASK));2063 pbe->pa_hi = cpu_to_le32((u32) upper_32_bits(buf_addr));2064 num_pbes += 1;2065 pbe++;2066 2067 /* if the pbl is full storing the pbes,2068 * move to next pbl.2069 */2070 if (num_pbes == (mr->hwmr.pbl_size/sizeof(u64))) {2071 pbl_tbl++;2072 pbe = (struct ocrdma_pbe *)pbl_tbl->va;2073 }2074 }2075 2076 return 0;2077}2078 2079static void ocrdma_ring_sq_db(struct ocrdma_qp *qp)2080{2081 u32 val = qp->sq.dbid | (1 << OCRDMA_DB_SQ_SHIFT);2082 2083 iowrite32(val, qp->sq_db);2084}2085 2086int ocrdma_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,2087 const struct ib_send_wr **bad_wr)2088{2089 int status = 0;2090 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);2091 struct ocrdma_hdr_wqe *hdr;2092 unsigned long flags;2093 2094 spin_lock_irqsave(&qp->q_lock, flags);2095 if (qp->state != OCRDMA_QPS_RTS && qp->state != OCRDMA_QPS_SQD) {2096 spin_unlock_irqrestore(&qp->q_lock, flags);2097 *bad_wr = wr;2098 return -EINVAL;2099 }2100 2101 while (wr) {2102 if (qp->qp_type == IB_QPT_UD &&2103 (wr->opcode != IB_WR_SEND &&2104 wr->opcode != IB_WR_SEND_WITH_IMM)) {2105 *bad_wr = wr;2106 status = -EINVAL;2107 break;2108 }2109 if (ocrdma_hwq_free_cnt(&qp->sq) == 0 ||2110 wr->num_sge > qp->sq.max_sges) {2111 *bad_wr = wr;2112 status = -ENOMEM;2113 break;2114 }2115 hdr = ocrdma_hwq_head(&qp->sq);2116 hdr->cw = 0;2117 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled)2118 hdr->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT);2119 if (wr->send_flags & IB_SEND_FENCE)2120 hdr->cw |=2121 (OCRDMA_FLAG_FENCE_L << OCRDMA_WQE_FLAGS_SHIFT);2122 if (wr->send_flags & IB_SEND_SOLICITED)2123 hdr->cw |=2124 (OCRDMA_FLAG_SOLICIT << OCRDMA_WQE_FLAGS_SHIFT);2125 hdr->total_len = 0;2126 switch (wr->opcode) {2127 case IB_WR_SEND_WITH_IMM:2128 hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT);2129 hdr->immdt = ntohl(wr->ex.imm_data);2130 fallthrough;2131 case IB_WR_SEND:2132 hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT);2133 ocrdma_build_send(qp, hdr, wr);2134 break;2135 case IB_WR_SEND_WITH_INV:2136 hdr->cw |= (OCRDMA_FLAG_INV << OCRDMA_WQE_FLAGS_SHIFT);2137 hdr->cw |= (OCRDMA_SEND << OCRDMA_WQE_OPCODE_SHIFT);2138 hdr->lkey = wr->ex.invalidate_rkey;2139 status = ocrdma_build_send(qp, hdr, wr);2140 break;2141 case IB_WR_RDMA_WRITE_WITH_IMM:2142 hdr->cw |= (OCRDMA_FLAG_IMM << OCRDMA_WQE_FLAGS_SHIFT);2143 hdr->immdt = ntohl(wr->ex.imm_data);2144 fallthrough;2145 case IB_WR_RDMA_WRITE:2146 hdr->cw |= (OCRDMA_WRITE << OCRDMA_WQE_OPCODE_SHIFT);2147 status = ocrdma_build_write(qp, hdr, wr);2148 break;2149 case IB_WR_RDMA_READ:2150 ocrdma_build_read(qp, hdr, wr);2151 break;2152 case IB_WR_LOCAL_INV:2153 hdr->cw |=2154 (OCRDMA_LKEY_INV << OCRDMA_WQE_OPCODE_SHIFT);2155 hdr->cw |= ((sizeof(struct ocrdma_hdr_wqe) +2156 sizeof(struct ocrdma_sge)) /2157 OCRDMA_WQE_STRIDE) << OCRDMA_WQE_SIZE_SHIFT;2158 hdr->lkey = wr->ex.invalidate_rkey;2159 break;2160 case IB_WR_REG_MR:2161 status = ocrdma_build_reg(qp, hdr, reg_wr(wr));2162 break;2163 default:2164 status = -EINVAL;2165 break;2166 }2167 if (status) {2168 *bad_wr = wr;2169 break;2170 }2171 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled)2172 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1;2173 else2174 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0;2175 qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id;2176 ocrdma_cpu_to_le32(hdr, ((hdr->cw >> OCRDMA_WQE_SIZE_SHIFT) &2177 OCRDMA_WQE_SIZE_MASK) * OCRDMA_WQE_STRIDE);2178 /* make sure wqe is written before adapter can access it */2179 wmb();2180 /* inform hw to start processing it */2181 ocrdma_ring_sq_db(qp);2182 2183 /* update pointer, counter for next wr */2184 ocrdma_hwq_inc_head(&qp->sq);2185 wr = wr->next;2186 }2187 spin_unlock_irqrestore(&qp->q_lock, flags);2188 return status;2189}2190 2191static void ocrdma_ring_rq_db(struct ocrdma_qp *qp)2192{2193 u32 val = qp->rq.dbid | (1 << OCRDMA_DB_RQ_SHIFT);2194 2195 iowrite32(val, qp->rq_db);2196}2197 2198static void ocrdma_build_rqe(struct ocrdma_hdr_wqe *rqe,2199 const struct ib_recv_wr *wr, u16 tag)2200{2201 u32 wqe_size = 0;2202 struct ocrdma_sge *sge;2203 if (wr->num_sge)2204 wqe_size = (wr->num_sge * sizeof(*sge)) + sizeof(*rqe);2205 else2206 wqe_size = sizeof(*sge) + sizeof(*rqe);2207 2208 rqe->cw = ((wqe_size / OCRDMA_WQE_STRIDE) <<2209 OCRDMA_WQE_SIZE_SHIFT);2210 rqe->cw |= (OCRDMA_FLAG_SIG << OCRDMA_WQE_FLAGS_SHIFT);2211 rqe->cw |= (OCRDMA_TYPE_LKEY << OCRDMA_WQE_TYPE_SHIFT);2212 rqe->total_len = 0;2213 rqe->rsvd_tag = tag;2214 sge = (struct ocrdma_sge *)(rqe + 1);2215 ocrdma_build_sges(rqe, sge, wr->num_sge, wr->sg_list);2216 ocrdma_cpu_to_le32(rqe, wqe_size);2217}2218 2219int ocrdma_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,2220 const struct ib_recv_wr **bad_wr)2221{2222 int status = 0;2223 unsigned long flags;2224 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);2225 struct ocrdma_hdr_wqe *rqe;2226 2227 spin_lock_irqsave(&qp->q_lock, flags);2228 if (qp->state == OCRDMA_QPS_RST || qp->state == OCRDMA_QPS_ERR) {2229 spin_unlock_irqrestore(&qp->q_lock, flags);2230 *bad_wr = wr;2231 return -EINVAL;2232 }2233 while (wr) {2234 if (ocrdma_hwq_free_cnt(&qp->rq) == 0 ||2235 wr->num_sge > qp->rq.max_sges) {2236 *bad_wr = wr;2237 status = -ENOMEM;2238 break;2239 }2240 rqe = ocrdma_hwq_head(&qp->rq);2241 ocrdma_build_rqe(rqe, wr, 0);2242 2243 qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id;2244 /* make sure rqe is written before adapter can access it */2245 wmb();2246 2247 /* inform hw to start processing it */2248 ocrdma_ring_rq_db(qp);2249 2250 /* update pointer, counter for next wr */2251 ocrdma_hwq_inc_head(&qp->rq);2252 wr = wr->next;2253 }2254 spin_unlock_irqrestore(&qp->q_lock, flags);2255 return status;2256}2257 2258/* cqe for srq's rqe can potentially arrive out of order.2259 * index gives the entry in the shadow table where to store2260 * the wr_id. tag/index is returned in cqe to reference back2261 * for a given rqe.2262 */2263static int ocrdma_srq_get_idx(struct ocrdma_srq *srq)2264{2265 int row = 0;2266 int indx = 0;2267 2268 for (row = 0; row < srq->bit_fields_len; row++) {2269 if (srq->idx_bit_fields[row]) {2270 indx = ffs(srq->idx_bit_fields[row]);2271 indx = (row * 32) + (indx - 1);2272 BUG_ON(indx >= srq->rq.max_cnt);2273 ocrdma_srq_toggle_bit(srq, indx);2274 break;2275 }2276 }2277 2278 BUG_ON(row == srq->bit_fields_len);2279 return indx + 1; /* Use from index 1 */2280}2281 2282static void ocrdma_ring_srq_db(struct ocrdma_srq *srq)2283{2284 u32 val = srq->rq.dbid | (1 << 16);2285 2286 iowrite32(val, srq->db + OCRDMA_DB_GEN2_SRQ_OFFSET);2287}2288 2289int ocrdma_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,2290 const struct ib_recv_wr **bad_wr)2291{2292 int status = 0;2293 unsigned long flags;2294 struct ocrdma_srq *srq;2295 struct ocrdma_hdr_wqe *rqe;2296 u16 tag;2297 2298 srq = get_ocrdma_srq(ibsrq);2299 2300 spin_lock_irqsave(&srq->q_lock, flags);2301 while (wr) {2302 if (ocrdma_hwq_free_cnt(&srq->rq) == 0 ||2303 wr->num_sge > srq->rq.max_sges) {2304 status = -ENOMEM;2305 *bad_wr = wr;2306 break;2307 }2308 tag = ocrdma_srq_get_idx(srq);2309 rqe = ocrdma_hwq_head(&srq->rq);2310 ocrdma_build_rqe(rqe, wr, tag);2311 2312 srq->rqe_wr_id_tbl[tag] = wr->wr_id;2313 /* make sure rqe is written before adapter can perform DMA */2314 wmb();2315 /* inform hw to start processing it */2316 ocrdma_ring_srq_db(srq);2317 /* update pointer, counter for next wr */2318 ocrdma_hwq_inc_head(&srq->rq);2319 wr = wr->next;2320 }2321 spin_unlock_irqrestore(&srq->q_lock, flags);2322 return status;2323}2324 2325static enum ib_wc_status ocrdma_to_ibwc_err(u16 status)2326{2327 enum ib_wc_status ibwc_status;2328 2329 switch (status) {2330 case OCRDMA_CQE_GENERAL_ERR:2331 ibwc_status = IB_WC_GENERAL_ERR;2332 break;2333 case OCRDMA_CQE_LOC_LEN_ERR:2334 ibwc_status = IB_WC_LOC_LEN_ERR;2335 break;2336 case OCRDMA_CQE_LOC_QP_OP_ERR:2337 ibwc_status = IB_WC_LOC_QP_OP_ERR;2338 break;2339 case OCRDMA_CQE_LOC_EEC_OP_ERR:2340 ibwc_status = IB_WC_LOC_EEC_OP_ERR;2341 break;2342 case OCRDMA_CQE_LOC_PROT_ERR:2343 ibwc_status = IB_WC_LOC_PROT_ERR;2344 break;2345 case OCRDMA_CQE_WR_FLUSH_ERR:2346 ibwc_status = IB_WC_WR_FLUSH_ERR;2347 break;2348 case OCRDMA_CQE_MW_BIND_ERR:2349 ibwc_status = IB_WC_MW_BIND_ERR;2350 break;2351 case OCRDMA_CQE_BAD_RESP_ERR:2352 ibwc_status = IB_WC_BAD_RESP_ERR;2353 break;2354 case OCRDMA_CQE_LOC_ACCESS_ERR:2355 ibwc_status = IB_WC_LOC_ACCESS_ERR;2356 break;2357 case OCRDMA_CQE_REM_INV_REQ_ERR:2358 ibwc_status = IB_WC_REM_INV_REQ_ERR;2359 break;2360 case OCRDMA_CQE_REM_ACCESS_ERR:2361 ibwc_status = IB_WC_REM_ACCESS_ERR;2362 break;2363 case OCRDMA_CQE_REM_OP_ERR:2364 ibwc_status = IB_WC_REM_OP_ERR;2365 break;2366 case OCRDMA_CQE_RETRY_EXC_ERR:2367 ibwc_status = IB_WC_RETRY_EXC_ERR;2368 break;2369 case OCRDMA_CQE_RNR_RETRY_EXC_ERR:2370 ibwc_status = IB_WC_RNR_RETRY_EXC_ERR;2371 break;2372 case OCRDMA_CQE_LOC_RDD_VIOL_ERR:2373 ibwc_status = IB_WC_LOC_RDD_VIOL_ERR;2374 break;2375 case OCRDMA_CQE_REM_INV_RD_REQ_ERR:2376 ibwc_status = IB_WC_REM_INV_RD_REQ_ERR;2377 break;2378 case OCRDMA_CQE_REM_ABORT_ERR:2379 ibwc_status = IB_WC_REM_ABORT_ERR;2380 break;2381 case OCRDMA_CQE_INV_EECN_ERR:2382 ibwc_status = IB_WC_INV_EECN_ERR;2383 break;2384 case OCRDMA_CQE_INV_EEC_STATE_ERR:2385 ibwc_status = IB_WC_INV_EEC_STATE_ERR;2386 break;2387 case OCRDMA_CQE_FATAL_ERR:2388 ibwc_status = IB_WC_FATAL_ERR;2389 break;2390 case OCRDMA_CQE_RESP_TIMEOUT_ERR:2391 ibwc_status = IB_WC_RESP_TIMEOUT_ERR;2392 break;2393 default:2394 ibwc_status = IB_WC_GENERAL_ERR;2395 break;2396 }2397 return ibwc_status;2398}2399 2400static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc,2401 u32 wqe_idx)2402{2403 struct ocrdma_hdr_wqe *hdr;2404 struct ocrdma_sge *rw;2405 int opcode;2406 2407 hdr = ocrdma_hwq_head_from_idx(&qp->sq, wqe_idx);2408 2409 ibwc->wr_id = qp->wqe_wr_id_tbl[wqe_idx].wrid;2410 /* Undo the hdr->cw swap */2411 opcode = le32_to_cpu(hdr->cw) & OCRDMA_WQE_OPCODE_MASK;2412 switch (opcode) {2413 case OCRDMA_WRITE:2414 ibwc->opcode = IB_WC_RDMA_WRITE;2415 break;2416 case OCRDMA_READ:2417 rw = (struct ocrdma_sge *)(hdr + 1);2418 ibwc->opcode = IB_WC_RDMA_READ;2419 ibwc->byte_len = rw->len;2420 break;2421 case OCRDMA_SEND:2422 ibwc->opcode = IB_WC_SEND;2423 break;2424 case OCRDMA_FR_MR:2425 ibwc->opcode = IB_WC_REG_MR;2426 break;2427 case OCRDMA_LKEY_INV:2428 ibwc->opcode = IB_WC_LOCAL_INV;2429 break;2430 default:2431 ibwc->status = IB_WC_GENERAL_ERR;2432 pr_err("%s() invalid opcode received = 0x%x\n",2433 __func__, hdr->cw & OCRDMA_WQE_OPCODE_MASK);2434 break;2435 }2436}2437 2438static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp,2439 struct ocrdma_cqe *cqe)2440{2441 if (is_cqe_for_sq(cqe)) {2442 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(2443 cqe->flags_status_srcqpn) &2444 ~OCRDMA_CQE_STATUS_MASK);2445 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(2446 cqe->flags_status_srcqpn) |2447 (OCRDMA_CQE_WR_FLUSH_ERR <<2448 OCRDMA_CQE_STATUS_SHIFT));2449 } else {2450 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {2451 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(2452 cqe->flags_status_srcqpn) &2453 ~OCRDMA_CQE_UD_STATUS_MASK);2454 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(2455 cqe->flags_status_srcqpn) |2456 (OCRDMA_CQE_WR_FLUSH_ERR <<2457 OCRDMA_CQE_UD_STATUS_SHIFT));2458 } else {2459 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(2460 cqe->flags_status_srcqpn) &2461 ~OCRDMA_CQE_STATUS_MASK);2462 cqe->flags_status_srcqpn = cpu_to_le32(le32_to_cpu(2463 cqe->flags_status_srcqpn) |2464 (OCRDMA_CQE_WR_FLUSH_ERR <<2465 OCRDMA_CQE_STATUS_SHIFT));2466 }2467 }2468}2469 2470static bool ocrdma_update_err_cqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,2471 struct ocrdma_qp *qp, int status)2472{2473 bool expand = false;2474 2475 ibwc->byte_len = 0;2476 ibwc->qp = &qp->ibqp;2477 ibwc->status = ocrdma_to_ibwc_err(status);2478 2479 ocrdma_flush_qp(qp);2480 ocrdma_qp_state_change(qp, IB_QPS_ERR, NULL);2481 2482 /* if wqe/rqe pending for which cqe needs to be returned,2483 * trigger inflating it.2484 */2485 if (!is_hw_rq_empty(qp) || !is_hw_sq_empty(qp)) {2486 expand = true;2487 ocrdma_set_cqe_status_flushed(qp, cqe);2488 }2489 return expand;2490}2491 2492static int ocrdma_update_err_rcqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,2493 struct ocrdma_qp *qp, int status)2494{2495 ibwc->opcode = IB_WC_RECV;2496 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];2497 ocrdma_hwq_inc_tail(&qp->rq);2498 2499 return ocrdma_update_err_cqe(ibwc, cqe, qp, status);2500}2501 2502static int ocrdma_update_err_scqe(struct ib_wc *ibwc, struct ocrdma_cqe *cqe,2503 struct ocrdma_qp *qp, int status)2504{2505 ocrdma_update_wc(qp, ibwc, qp->sq.tail);2506 ocrdma_hwq_inc_tail(&qp->sq);2507 2508 return ocrdma_update_err_cqe(ibwc, cqe, qp, status);2509}2510 2511 2512static bool ocrdma_poll_err_scqe(struct ocrdma_qp *qp,2513 struct ocrdma_cqe *cqe, struct ib_wc *ibwc,2514 bool *polled, bool *stop)2515{2516 bool expand;2517 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device);2518 int status = (le32_to_cpu(cqe->flags_status_srcqpn) &2519 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;2520 if (status < OCRDMA_MAX_CQE_ERR)2521 atomic_inc(&dev->cqe_err_stats[status]);2522 2523 /* when hw sq is empty, but rq is not empty, so we continue2524 * to keep the cqe in order to get the cq event again.2525 */2526 if (is_hw_sq_empty(qp) && !is_hw_rq_empty(qp)) {2527 /* when cq for rq and sq is same, it is safe to return2528 * flush cqe for RQEs.2529 */2530 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {2531 *polled = true;2532 status = OCRDMA_CQE_WR_FLUSH_ERR;2533 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);2534 } else {2535 /* stop processing further cqe as this cqe is used for2536 * triggering cq event on buddy cq of RQ.2537 * When QP is destroyed, this cqe will be removed2538 * from the cq's hardware q.2539 */2540 *polled = false;2541 *stop = true;2542 expand = false;2543 }2544 } else if (is_hw_sq_empty(qp)) {2545 /* Do nothing */2546 expand = false;2547 *polled = false;2548 *stop = false;2549 } else {2550 *polled = true;2551 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);2552 }2553 return expand;2554}2555 2556static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp,2557 struct ocrdma_cqe *cqe,2558 struct ib_wc *ibwc, bool *polled)2559{2560 bool expand = false;2561 int tail = qp->sq.tail;2562 u32 wqe_idx;2563 2564 if (!qp->wqe_wr_id_tbl[tail].signaled) {2565 *polled = false; /* WC cannot be consumed yet */2566 } else {2567 ibwc->status = IB_WC_SUCCESS;2568 ibwc->wc_flags = 0;2569 ibwc->qp = &qp->ibqp;2570 ocrdma_update_wc(qp, ibwc, tail);2571 *polled = true;2572 }2573 wqe_idx = (le32_to_cpu(cqe->wq.wqeidx) &2574 OCRDMA_CQE_WQEIDX_MASK) & qp->sq.max_wqe_idx;2575 if (tail != wqe_idx)2576 expand = true; /* Coalesced CQE can't be consumed yet */2577 2578 ocrdma_hwq_inc_tail(&qp->sq);2579 return expand;2580}2581 2582static bool ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,2583 struct ib_wc *ibwc, bool *polled, bool *stop)2584{2585 int status;2586 bool expand;2587 2588 status = (le32_to_cpu(cqe->flags_status_srcqpn) &2589 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;2590 2591 if (status == OCRDMA_CQE_SUCCESS)2592 expand = ocrdma_poll_success_scqe(qp, cqe, ibwc, polled);2593 else2594 expand = ocrdma_poll_err_scqe(qp, cqe, ibwc, polled, stop);2595 return expand;2596}2597 2598static int ocrdma_update_ud_rcqe(struct ocrdma_dev *dev, struct ib_wc *ibwc,2599 struct ocrdma_cqe *cqe)2600{2601 int status;2602 u16 hdr_type = 0;2603 2604 status = (le32_to_cpu(cqe->flags_status_srcqpn) &2605 OCRDMA_CQE_UD_STATUS_MASK) >> OCRDMA_CQE_UD_STATUS_SHIFT;2606 ibwc->src_qp = le32_to_cpu(cqe->flags_status_srcqpn) &2607 OCRDMA_CQE_SRCQP_MASK;2608 ibwc->pkey_index = 0;2609 ibwc->wc_flags = IB_WC_GRH;2610 ibwc->byte_len = (le32_to_cpu(cqe->ud.rxlen_pkey) >>2611 OCRDMA_CQE_UD_XFER_LEN_SHIFT) &2612 OCRDMA_CQE_UD_XFER_LEN_MASK;2613 2614 if (ocrdma_is_udp_encap_supported(dev)) {2615 hdr_type = (le32_to_cpu(cqe->ud.rxlen_pkey) >>2616 OCRDMA_CQE_UD_L3TYPE_SHIFT) &2617 OCRDMA_CQE_UD_L3TYPE_MASK;2618 ibwc->wc_flags |= IB_WC_WITH_NETWORK_HDR_TYPE;2619 ibwc->network_hdr_type = hdr_type;2620 }2621 2622 return status;2623}2624 2625static void ocrdma_update_free_srq_cqe(struct ib_wc *ibwc,2626 struct ocrdma_cqe *cqe,2627 struct ocrdma_qp *qp)2628{2629 unsigned long flags;2630 struct ocrdma_srq *srq;2631 u32 wqe_idx;2632 2633 srq = get_ocrdma_srq(qp->ibqp.srq);2634 wqe_idx = (le32_to_cpu(cqe->rq.buftag_qpn) >>2635 OCRDMA_CQE_BUFTAG_SHIFT) & srq->rq.max_wqe_idx;2636 BUG_ON(wqe_idx < 1);2637 2638 ibwc->wr_id = srq->rqe_wr_id_tbl[wqe_idx];2639 spin_lock_irqsave(&srq->q_lock, flags);2640 ocrdma_srq_toggle_bit(srq, wqe_idx - 1);2641 spin_unlock_irqrestore(&srq->q_lock, flags);2642 ocrdma_hwq_inc_tail(&srq->rq);2643}2644 2645static bool ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,2646 struct ib_wc *ibwc, bool *polled, bool *stop,2647 int status)2648{2649 bool expand;2650 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device);2651 2652 if (status < OCRDMA_MAX_CQE_ERR)2653 atomic_inc(&dev->cqe_err_stats[status]);2654 2655 /* when hw_rq is empty, but wq is not empty, so continue2656 * to keep the cqe to get the cq event again.2657 */2658 if (is_hw_rq_empty(qp) && !is_hw_sq_empty(qp)) {2659 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {2660 *polled = true;2661 status = OCRDMA_CQE_WR_FLUSH_ERR;2662 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);2663 } else {2664 *polled = false;2665 *stop = true;2666 expand = false;2667 }2668 } else if (is_hw_rq_empty(qp)) {2669 /* Do nothing */2670 expand = false;2671 *polled = false;2672 *stop = false;2673 } else {2674 *polled = true;2675 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);2676 }2677 return expand;2678}2679 2680static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp,2681 struct ocrdma_cqe *cqe, struct ib_wc *ibwc)2682{2683 struct ocrdma_dev *dev;2684 2685 dev = get_ocrdma_dev(qp->ibqp.device);2686 ibwc->opcode = IB_WC_RECV;2687 ibwc->qp = &qp->ibqp;2688 ibwc->status = IB_WC_SUCCESS;2689 2690 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI)2691 ocrdma_update_ud_rcqe(dev, ibwc, cqe);2692 else2693 ibwc->byte_len = le32_to_cpu(cqe->rq.rxlen);2694 2695 if (is_cqe_imm(cqe)) {2696 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt));2697 ibwc->wc_flags |= IB_WC_WITH_IMM;2698 } else if (is_cqe_wr_imm(cqe)) {2699 ibwc->opcode = IB_WC_RECV_RDMA_WITH_IMM;2700 ibwc->ex.imm_data = htonl(le32_to_cpu(cqe->rq.lkey_immdt));2701 ibwc->wc_flags |= IB_WC_WITH_IMM;2702 } else if (is_cqe_invalidated(cqe)) {2703 ibwc->ex.invalidate_rkey = le32_to_cpu(cqe->rq.lkey_immdt);2704 ibwc->wc_flags |= IB_WC_WITH_INVALIDATE;2705 }2706 if (qp->ibqp.srq) {2707 ocrdma_update_free_srq_cqe(ibwc, cqe, qp);2708 } else {2709 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];2710 ocrdma_hwq_inc_tail(&qp->rq);2711 }2712}2713 2714static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,2715 struct ib_wc *ibwc, bool *polled, bool *stop)2716{2717 int status;2718 bool expand = false;2719 2720 ibwc->wc_flags = 0;2721 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {2722 status = (le32_to_cpu(cqe->flags_status_srcqpn) &2723 OCRDMA_CQE_UD_STATUS_MASK) >>2724 OCRDMA_CQE_UD_STATUS_SHIFT;2725 } else {2726 status = (le32_to_cpu(cqe->flags_status_srcqpn) &2727 OCRDMA_CQE_STATUS_MASK) >> OCRDMA_CQE_STATUS_SHIFT;2728 }2729 2730 if (status == OCRDMA_CQE_SUCCESS) {2731 *polled = true;2732 ocrdma_poll_success_rcqe(qp, cqe, ibwc);2733 } else {2734 expand = ocrdma_poll_err_rcqe(qp, cqe, ibwc, polled, stop,2735 status);2736 }2737 return expand;2738}2739 2740static void ocrdma_change_cq_phase(struct ocrdma_cq *cq, struct ocrdma_cqe *cqe,2741 u16 cur_getp)2742{2743 if (cq->phase_change) {2744 if (cur_getp == 0)2745 cq->phase = (~cq->phase & OCRDMA_CQE_VALID);2746 } else {2747 /* clear valid bit */2748 cqe->flags_status_srcqpn = 0;2749 }2750}2751 2752static int ocrdma_poll_hwcq(struct ocrdma_cq *cq, int num_entries,2753 struct ib_wc *ibwc)2754{2755 u16 qpn = 0;2756 int i = 0;2757 bool expand = false;2758 int polled_hw_cqes = 0;2759 struct ocrdma_qp *qp = NULL;2760 struct ocrdma_dev *dev = get_ocrdma_dev(cq->ibcq.device);2761 struct ocrdma_cqe *cqe;2762 u16 cur_getp; bool polled = false; bool stop = false;2763 2764 cur_getp = cq->getp;2765 while (num_entries) {2766 cqe = cq->va + cur_getp;2767 /* check whether valid cqe or not */2768 if (!is_cqe_valid(cq, cqe))2769 break;2770 qpn = (le32_to_cpu(cqe->cmn.qpn) & OCRDMA_CQE_QPN_MASK);2771 /* ignore discarded cqe */2772 if (qpn == 0)2773 goto skip_cqe;2774 qp = dev->qp_tbl[qpn];2775 BUG_ON(qp == NULL);2776 2777 if (is_cqe_for_sq(cqe)) {2778 expand = ocrdma_poll_scqe(qp, cqe, ibwc, &polled,2779 &stop);2780 } else {2781 expand = ocrdma_poll_rcqe(qp, cqe, ibwc, &polled,2782 &stop);2783 }2784 if (expand)2785 goto expand_cqe;2786 if (stop)2787 goto stop_cqe;2788 /* clear qpn to avoid duplicate processing by discard_cqe() */2789 cqe->cmn.qpn = 0;2790skip_cqe:2791 polled_hw_cqes += 1;2792 cur_getp = (cur_getp + 1) % cq->max_hw_cqe;2793 ocrdma_change_cq_phase(cq, cqe, cur_getp);2794expand_cqe:2795 if (polled) {2796 num_entries -= 1;2797 i += 1;2798 ibwc = ibwc + 1;2799 polled = false;2800 }2801 }2802stop_cqe:2803 cq->getp = cur_getp;2804 2805 if (polled_hw_cqes)2806 ocrdma_ring_cq_db(dev, cq->id, false, false, polled_hw_cqes);2807 2808 return i;2809}2810 2811/* insert error cqe if the QP's SQ or RQ's CQ matches the CQ under poll. */2812static int ocrdma_add_err_cqe(struct ocrdma_cq *cq, int num_entries,2813 struct ocrdma_qp *qp, struct ib_wc *ibwc)2814{2815 int err_cqes = 0;2816 2817 while (num_entries) {2818 if (is_hw_sq_empty(qp) && is_hw_rq_empty(qp))2819 break;2820 if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) {2821 ocrdma_update_wc(qp, ibwc, qp->sq.tail);2822 ocrdma_hwq_inc_tail(&qp->sq);2823 } else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) {2824 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];2825 ocrdma_hwq_inc_tail(&qp->rq);2826 } else {2827 return err_cqes;2828 }2829 ibwc->byte_len = 0;2830 ibwc->status = IB_WC_WR_FLUSH_ERR;2831 ibwc = ibwc + 1;2832 err_cqes += 1;2833 num_entries -= 1;2834 }2835 return err_cqes;2836}2837 2838int ocrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)2839{2840 int cqes_to_poll = num_entries;2841 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);2842 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);2843 int num_os_cqe = 0, err_cqes = 0;2844 struct ocrdma_qp *qp;2845 unsigned long flags;2846 2847 /* poll cqes from adapter CQ */2848 spin_lock_irqsave(&cq->cq_lock, flags);2849 num_os_cqe = ocrdma_poll_hwcq(cq, cqes_to_poll, wc);2850 spin_unlock_irqrestore(&cq->cq_lock, flags);2851 cqes_to_poll -= num_os_cqe;2852 2853 if (cqes_to_poll) {2854 wc = wc + num_os_cqe;2855 /* adapter returns single error cqe when qp moves to2856 * error state. So insert error cqes with wc_status as2857 * FLUSHED for pending WQEs and RQEs of QP's SQ and RQ2858 * respectively which uses this CQ.2859 */2860 spin_lock_irqsave(&dev->flush_q_lock, flags);2861 list_for_each_entry(qp, &cq->sq_head, sq_entry) {2862 if (cqes_to_poll == 0)2863 break;2864 err_cqes = ocrdma_add_err_cqe(cq, cqes_to_poll, qp, wc);2865 cqes_to_poll -= err_cqes;2866 num_os_cqe += err_cqes;2867 wc = wc + err_cqes;2868 }2869 spin_unlock_irqrestore(&dev->flush_q_lock, flags);2870 }2871 return num_os_cqe;2872}2873 2874int ocrdma_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags cq_flags)2875{2876 struct ocrdma_cq *cq = get_ocrdma_cq(ibcq);2877 struct ocrdma_dev *dev = get_ocrdma_dev(ibcq->device);2878 u16 cq_id;2879 unsigned long flags;2880 bool arm_needed = false, sol_needed = false;2881 2882 cq_id = cq->id;2883 2884 spin_lock_irqsave(&cq->cq_lock, flags);2885 if (cq_flags & IB_CQ_NEXT_COMP || cq_flags & IB_CQ_SOLICITED)2886 arm_needed = true;2887 if (cq_flags & IB_CQ_SOLICITED)2888 sol_needed = true;2889 2890 ocrdma_ring_cq_db(dev, cq_id, arm_needed, sol_needed, 0);2891 spin_unlock_irqrestore(&cq->cq_lock, flags);2892 2893 return 0;2894}2895 2896struct ib_mr *ocrdma_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,2897 u32 max_num_sg)2898{2899 int status;2900 struct ocrdma_mr *mr;2901 struct ocrdma_pd *pd = get_ocrdma_pd(ibpd);2902 struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);2903 2904 if (mr_type != IB_MR_TYPE_MEM_REG)2905 return ERR_PTR(-EINVAL);2906 2907 if (max_num_sg > dev->attr.max_pages_per_frmr)2908 return ERR_PTR(-EINVAL);2909 2910 mr = kzalloc(sizeof(*mr), GFP_KERNEL);2911 if (!mr)2912 return ERR_PTR(-ENOMEM);2913 2914 mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);2915 if (!mr->pages) {2916 status = -ENOMEM;2917 goto pl_err;2918 }2919 2920 status = ocrdma_get_pbl_info(dev, mr, max_num_sg);2921 if (status)2922 goto pbl_err;2923 mr->hwmr.fr_mr = 1;2924 mr->hwmr.remote_rd = 0;2925 mr->hwmr.remote_wr = 0;2926 mr->hwmr.local_rd = 0;2927 mr->hwmr.local_wr = 0;2928 mr->hwmr.mw_bind = 0;2929 status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);2930 if (status)2931 goto pbl_err;2932 status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, 0);2933 if (status)2934 goto mbx_err;2935 mr->ibmr.rkey = mr->hwmr.lkey;2936 mr->ibmr.lkey = mr->hwmr.lkey;2937 dev->stag_arr[(mr->hwmr.lkey >> 8) & (OCRDMA_MAX_STAG - 1)] =2938 (unsigned long) mr;2939 return &mr->ibmr;2940mbx_err:2941 ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);2942pbl_err:2943 kfree(mr->pages);2944pl_err:2945 kfree(mr);2946 return ERR_PTR(-ENOMEM);2947}2948 2949static int ocrdma_set_page(struct ib_mr *ibmr, u64 addr)2950{2951 struct ocrdma_mr *mr = get_ocrdma_mr(ibmr);2952 2953 if (unlikely(mr->npages == mr->hwmr.num_pbes))2954 return -ENOMEM;2955 2956 mr->pages[mr->npages++] = addr;2957 2958 return 0;2959}2960 2961int ocrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,2962 unsigned int *sg_offset)2963{2964 struct ocrdma_mr *mr = get_ocrdma_mr(ibmr);2965 2966 mr->npages = 0;2967 2968 return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, ocrdma_set_page);2969}2970