570 lines · c
1/*2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.3 *4 * This software is available to you under a choice of one of two5 * licenses. You may choose to be licensed under the terms of the GNU6 * General Public License (GPL) Version 2, available from the file7 * COPYING in the main directory of this source tree, or the8 * OpenIB.org BSD license below:9 *10 * Redistribution and use in source and binary forms, with or11 * without modification, are permitted provided that the following12 * conditions are met:13 *14 * - Redistributions of source code must retain the above15 * copyright notice, this list of conditions and the following16 * disclaimer.17 *18 * - Redistributions in binary form must reproduce the above19 * copyright notice, this list of conditions and the following20 * disclaimer in the documentation and/or other materials21 * provided with the distribution.22 *23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE30 * SOFTWARE.31 */32#include <linux/module.h>33#include <linux/moduleparam.h>34#include <linux/device.h>35#include <linux/netdevice.h>36#include <linux/etherdevice.h>37#include <linux/delay.h>38#include <linux/errno.h>39#include <linux/list.h>40#include <linux/spinlock.h>41#include <linux/ethtool.h>42#include <linux/rtnetlink.h>43#include <linux/inetdevice.h>44#include <net/addrconf.h>45#include <linux/io.h>46 47#include <asm/irq.h>48#include <asm/byteorder.h>49 50#include <rdma/iw_cm.h>51#include <rdma/ib_verbs.h>52#include <rdma/ib_smi.h>53#include <rdma/ib_umem.h>54#include <rdma/ib_user_verbs.h>55 56#include "iw_cxgb4.h"57 58static int fastreg_support = 1;59module_param(fastreg_support, int, 0644);60MODULE_PARM_DESC(fastreg_support, "Advertise fastreg support (default=1)");61 62static void c4iw_dealloc_ucontext(struct ib_ucontext *context)63{64 struct c4iw_ucontext *ucontext = to_c4iw_ucontext(context);65 struct c4iw_dev *rhp;66 struct c4iw_mm_entry *mm, *tmp;67 68 pr_debug("context %p\n", context);69 rhp = to_c4iw_dev(ucontext->ibucontext.device);70 71 list_for_each_entry_safe(mm, tmp, &ucontext->mmaps, entry)72 kfree(mm);73 c4iw_release_dev_ucontext(&rhp->rdev, &ucontext->uctx);74}75 76static int c4iw_alloc_ucontext(struct ib_ucontext *ucontext,77 struct ib_udata *udata)78{79 struct ib_device *ibdev = ucontext->device;80 struct c4iw_ucontext *context = to_c4iw_ucontext(ucontext);81 struct c4iw_dev *rhp = to_c4iw_dev(ibdev);82 struct c4iw_alloc_ucontext_resp uresp;83 int ret = 0;84 struct c4iw_mm_entry *mm = NULL;85 86 pr_debug("ibdev %p\n", ibdev);87 c4iw_init_dev_ucontext(&rhp->rdev, &context->uctx);88 INIT_LIST_HEAD(&context->mmaps);89 spin_lock_init(&context->mmap_lock);90 91 if (udata->outlen < sizeof(uresp) - sizeof(uresp.reserved)) {92 pr_err_once("Warning - downlevel libcxgb4 (non-fatal), device status page disabled\n");93 rhp->rdev.flags |= T4_STATUS_PAGE_DISABLED;94 } else {95 mm = kmalloc(sizeof(*mm), GFP_KERNEL);96 if (!mm) {97 ret = -ENOMEM;98 goto err;99 }100 101 uresp.status_page_size = PAGE_SIZE;102 103 spin_lock(&context->mmap_lock);104 uresp.status_page_key = context->key;105 context->key += PAGE_SIZE;106 spin_unlock(&context->mmap_lock);107 108 ret = ib_copy_to_udata(udata, &uresp,109 sizeof(uresp) - sizeof(uresp.reserved));110 if (ret)111 goto err_mm;112 113 mm->key = uresp.status_page_key;114 mm->addr = virt_to_phys(rhp->rdev.status_page);115 mm->len = PAGE_SIZE;116 mm->vaddr = NULL;117 mm->dma_addr = 0;118 insert_flag_to_mmap(&rhp->rdev, mm, mm->addr);119 insert_mmap(context, mm);120 }121 return 0;122err_mm:123 kfree(mm);124err:125 return ret;126}127 128static int c4iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)129{130 int len = vma->vm_end - vma->vm_start;131 u32 key = vma->vm_pgoff << PAGE_SHIFT;132 struct c4iw_rdev *rdev;133 int ret = 0;134 struct c4iw_mm_entry *mm;135 struct c4iw_ucontext *ucontext;136 u64 addr;137 u8 mmap_flag;138 size_t size;139 void *vaddr;140 unsigned long vm_pgoff;141 dma_addr_t dma_addr;142 143 pr_debug("pgoff 0x%lx key 0x%x len %d\n", vma->vm_pgoff,144 key, len);145 146 if (vma->vm_start & (PAGE_SIZE-1))147 return -EINVAL;148 149 rdev = &(to_c4iw_dev(context->device)->rdev);150 ucontext = to_c4iw_ucontext(context);151 152 mm = remove_mmap(ucontext, key, len);153 if (!mm)154 return -EINVAL;155 addr = mm->addr;156 vaddr = mm->vaddr;157 dma_addr = mm->dma_addr;158 size = mm->len;159 mmap_flag = mm->mmap_flag;160 kfree(mm);161 162 switch (mmap_flag) {163 case CXGB4_MMAP_BAR:164 ret = io_remap_pfn_range(vma, vma->vm_start, addr >> PAGE_SHIFT,165 len,166 pgprot_noncached(vma->vm_page_prot));167 break;168 case CXGB4_MMAP_BAR_WC:169 ret = io_remap_pfn_range(vma, vma->vm_start,170 addr >> PAGE_SHIFT,171 len, t4_pgprot_wc(vma->vm_page_prot));172 break;173 case CXGB4_MMAP_CONTIG:174 ret = io_remap_pfn_range(vma, vma->vm_start,175 addr >> PAGE_SHIFT,176 len, vma->vm_page_prot);177 break;178 case CXGB4_MMAP_NON_CONTIG:179 vm_pgoff = vma->vm_pgoff;180 vma->vm_pgoff = 0;181 ret = dma_mmap_coherent(&rdev->lldi.pdev->dev, vma,182 vaddr, dma_addr, size);183 vma->vm_pgoff = vm_pgoff;184 break;185 default:186 ret = -EINVAL;187 break;188 }189 190 return ret;191}192 193static int c4iw_deallocate_pd(struct ib_pd *pd, struct ib_udata *udata)194{195 struct c4iw_dev *rhp;196 struct c4iw_pd *php;197 198 php = to_c4iw_pd(pd);199 rhp = php->rhp;200 pr_debug("ibpd %p pdid 0x%x\n", pd, php->pdid);201 c4iw_put_resource(&rhp->rdev.resource.pdid_table, php->pdid);202 mutex_lock(&rhp->rdev.stats.lock);203 rhp->rdev.stats.pd.cur--;204 mutex_unlock(&rhp->rdev.stats.lock);205 return 0;206}207 208static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_udata *udata)209{210 struct c4iw_pd *php = to_c4iw_pd(pd);211 struct ib_device *ibdev = pd->device;212 u32 pdid;213 struct c4iw_dev *rhp;214 215 pr_debug("ibdev %p\n", ibdev);216 rhp = (struct c4iw_dev *) ibdev;217 pdid = c4iw_get_resource(&rhp->rdev.resource.pdid_table);218 if (!pdid)219 return -EINVAL;220 221 php->pdid = pdid;222 php->rhp = rhp;223 if (udata) {224 struct c4iw_alloc_pd_resp uresp = {.pdid = php->pdid};225 226 if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {227 c4iw_deallocate_pd(&php->ibpd, udata);228 return -EFAULT;229 }230 }231 mutex_lock(&rhp->rdev.stats.lock);232 rhp->rdev.stats.pd.cur++;233 if (rhp->rdev.stats.pd.cur > rhp->rdev.stats.pd.max)234 rhp->rdev.stats.pd.max = rhp->rdev.stats.pd.cur;235 mutex_unlock(&rhp->rdev.stats.lock);236 pr_debug("pdid 0x%0x ptr 0x%p\n", pdid, php);237 return 0;238}239 240static int c4iw_query_gid(struct ib_device *ibdev, u32 port, int index,241 union ib_gid *gid)242{243 struct c4iw_dev *dev;244 245 pr_debug("ibdev %p, port %u, index %d, gid %p\n",246 ibdev, port, index, gid);247 if (!port)248 return -EINVAL;249 dev = to_c4iw_dev(ibdev);250 memset(&(gid->raw[0]), 0, sizeof(gid->raw));251 memcpy(&(gid->raw[0]), dev->rdev.lldi.ports[port-1]->dev_addr, 6);252 return 0;253}254 255static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *props,256 struct ib_udata *uhw)257{258 259 struct c4iw_dev *dev;260 261 pr_debug("ibdev %p\n", ibdev);262 263 if (uhw->inlen || uhw->outlen)264 return -EINVAL;265 266 dev = to_c4iw_dev(ibdev);267 addrconf_addr_eui48((u8 *)&props->sys_image_guid,268 dev->rdev.lldi.ports[0]->dev_addr);269 props->hw_ver = CHELSIO_CHIP_RELEASE(dev->rdev.lldi.adapter_type);270 props->fw_ver = dev->rdev.lldi.fw_vers;271 props->device_cap_flags = IB_DEVICE_MEM_WINDOW;272 props->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;273 if (fastreg_support)274 props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;275 props->page_size_cap = T4_PAGESIZE_MASK;276 props->vendor_id = (u32)dev->rdev.lldi.pdev->vendor;277 props->vendor_part_id = (u32)dev->rdev.lldi.pdev->device;278 props->max_mr_size = T4_MAX_MR_SIZE;279 props->max_qp = dev->rdev.lldi.vr->qp.size / 2;280 props->max_srq = dev->rdev.lldi.vr->srq.size;281 props->max_qp_wr = dev->rdev.hw_queue.t4_max_qp_depth;282 props->max_srq_wr = dev->rdev.hw_queue.t4_max_qp_depth;283 props->max_send_sge = min(T4_MAX_SEND_SGE, T4_MAX_WRITE_SGE);284 props->max_recv_sge = T4_MAX_RECV_SGE;285 props->max_srq_sge = T4_MAX_RECV_SGE;286 props->max_sge_rd = 1;287 props->max_res_rd_atom = dev->rdev.lldi.max_ird_adapter;288 props->max_qp_rd_atom = min(dev->rdev.lldi.max_ordird_qp,289 c4iw_max_read_depth);290 props->max_qp_init_rd_atom = props->max_qp_rd_atom;291 props->max_cq = dev->rdev.lldi.vr->qp.size;292 props->max_cqe = dev->rdev.hw_queue.t4_max_cq_depth;293 props->max_mr = c4iw_num_stags(&dev->rdev);294 props->max_pd = T4_MAX_NUM_PD;295 props->local_ca_ack_delay = 0;296 props->max_fast_reg_page_list_len =297 t4_max_fr_depth(dev->rdev.lldi.ulptx_memwrite_dsgl && use_dsgl);298 299 return 0;300}301 302static int c4iw_query_port(struct ib_device *ibdev, u32 port,303 struct ib_port_attr *props)304{305 int ret = 0;306 pr_debug("ibdev %p\n", ibdev);307 ret = ib_get_eth_speed(ibdev, port, &props->active_speed,308 &props->active_width);309 310 props->port_cap_flags =311 IB_PORT_CM_SUP |312 IB_PORT_SNMP_TUNNEL_SUP |313 IB_PORT_REINIT_SUP |314 IB_PORT_DEVICE_MGMT_SUP |315 IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;316 props->gid_tbl_len = 1;317 props->max_msg_sz = -1;318 319 return ret;320}321 322static ssize_t hw_rev_show(struct device *dev,323 struct device_attribute *attr, char *buf)324{325 struct c4iw_dev *c4iw_dev =326 rdma_device_to_drv_device(dev, struct c4iw_dev, ibdev);327 328 pr_debug("dev 0x%p\n", dev);329 return sysfs_emit(330 buf, "%d\n",331 CHELSIO_CHIP_RELEASE(c4iw_dev->rdev.lldi.adapter_type));332}333static DEVICE_ATTR_RO(hw_rev);334 335static ssize_t hca_type_show(struct device *dev,336 struct device_attribute *attr, char *buf)337{338 struct c4iw_dev *c4iw_dev =339 rdma_device_to_drv_device(dev, struct c4iw_dev, ibdev);340 struct ethtool_drvinfo info;341 struct net_device *lldev = c4iw_dev->rdev.lldi.ports[0];342 343 pr_debug("dev 0x%p\n", dev);344 lldev->ethtool_ops->get_drvinfo(lldev, &info);345 return sysfs_emit(buf, "%s\n", info.driver);346}347static DEVICE_ATTR_RO(hca_type);348 349static ssize_t board_id_show(struct device *dev, struct device_attribute *attr,350 char *buf)351{352 struct c4iw_dev *c4iw_dev =353 rdma_device_to_drv_device(dev, struct c4iw_dev, ibdev);354 355 pr_debug("dev 0x%p\n", dev);356 return sysfs_emit(buf, "%x.%x\n", c4iw_dev->rdev.lldi.pdev->vendor,357 c4iw_dev->rdev.lldi.pdev->device);358}359static DEVICE_ATTR_RO(board_id);360 361enum counters {362 IP4INSEGS,363 IP4OUTSEGS,364 IP4RETRANSSEGS,365 IP4OUTRSTS,366 IP6INSEGS,367 IP6OUTSEGS,368 IP6RETRANSSEGS,369 IP6OUTRSTS,370 NR_COUNTERS371};372 373static const struct rdma_stat_desc cxgb4_descs[] = {374 [IP4INSEGS].name = "ip4InSegs",375 [IP4OUTSEGS].name = "ip4OutSegs",376 [IP4RETRANSSEGS].name = "ip4RetransSegs",377 [IP4OUTRSTS].name = "ip4OutRsts",378 [IP6INSEGS].name = "ip6InSegs",379 [IP6OUTSEGS].name = "ip6OutSegs",380 [IP6RETRANSSEGS].name = "ip6RetransSegs",381 [IP6OUTRSTS].name = "ip6OutRsts"382};383 384static struct rdma_hw_stats *c4iw_alloc_device_stats(struct ib_device *ibdev)385{386 BUILD_BUG_ON(ARRAY_SIZE(cxgb4_descs) != NR_COUNTERS);387 388 /* FIXME: these look like port stats */389 return rdma_alloc_hw_stats_struct(cxgb4_descs, NR_COUNTERS,390 RDMA_HW_STATS_DEFAULT_LIFESPAN);391}392 393static int c4iw_get_mib(struct ib_device *ibdev,394 struct rdma_hw_stats *stats,395 u32 port, int index)396{397 struct tp_tcp_stats v4, v6;398 struct c4iw_dev *c4iw_dev = to_c4iw_dev(ibdev);399 400 cxgb4_get_tcp_stats(c4iw_dev->rdev.lldi.pdev, &v4, &v6);401 stats->value[IP4INSEGS] = v4.tcp_in_segs;402 stats->value[IP4OUTSEGS] = v4.tcp_out_segs;403 stats->value[IP4RETRANSSEGS] = v4.tcp_retrans_segs;404 stats->value[IP4OUTRSTS] = v4.tcp_out_rsts;405 stats->value[IP6INSEGS] = v6.tcp_in_segs;406 stats->value[IP6OUTSEGS] = v6.tcp_out_segs;407 stats->value[IP6RETRANSSEGS] = v6.tcp_retrans_segs;408 stats->value[IP6OUTRSTS] = v6.tcp_out_rsts;409 410 return stats->num_counters;411}412 413static struct attribute *c4iw_class_attributes[] = {414 &dev_attr_hw_rev.attr,415 &dev_attr_hca_type.attr,416 &dev_attr_board_id.attr,417 NULL418};419 420static const struct attribute_group c4iw_attr_group = {421 .attrs = c4iw_class_attributes,422};423 424static int c4iw_port_immutable(struct ib_device *ibdev, u32 port_num,425 struct ib_port_immutable *immutable)426{427 struct ib_port_attr attr;428 int err;429 430 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;431 432 err = ib_query_port(ibdev, port_num, &attr);433 if (err)434 return err;435 436 immutable->gid_tbl_len = attr.gid_tbl_len;437 438 return 0;439}440 441static void get_dev_fw_str(struct ib_device *dev, char *str)442{443 struct c4iw_dev *c4iw_dev = container_of(dev, struct c4iw_dev,444 ibdev);445 pr_debug("dev 0x%p\n", dev);446 447 snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u.%u.%u",448 FW_HDR_FW_VER_MAJOR_G(c4iw_dev->rdev.lldi.fw_vers),449 FW_HDR_FW_VER_MINOR_G(c4iw_dev->rdev.lldi.fw_vers),450 FW_HDR_FW_VER_MICRO_G(c4iw_dev->rdev.lldi.fw_vers),451 FW_HDR_FW_VER_BUILD_G(c4iw_dev->rdev.lldi.fw_vers));452}453 454static const struct ib_device_ops c4iw_dev_ops = {455 .owner = THIS_MODULE,456 .driver_id = RDMA_DRIVER_CXGB4,457 .uverbs_abi_ver = C4IW_UVERBS_ABI_VERSION,458 459 .alloc_hw_device_stats = c4iw_alloc_device_stats,460 .alloc_mr = c4iw_alloc_mr,461 .alloc_pd = c4iw_allocate_pd,462 .alloc_ucontext = c4iw_alloc_ucontext,463 .create_cq = c4iw_create_cq,464 .create_qp = c4iw_create_qp,465 .create_srq = c4iw_create_srq,466 .dealloc_pd = c4iw_deallocate_pd,467 .dealloc_ucontext = c4iw_dealloc_ucontext,468 .dereg_mr = c4iw_dereg_mr,469 .destroy_cq = c4iw_destroy_cq,470 .destroy_qp = c4iw_destroy_qp,471 .destroy_srq = c4iw_destroy_srq,472 .device_group = &c4iw_attr_group,473 .fill_res_cq_entry = c4iw_fill_res_cq_entry,474 .fill_res_cm_id_entry = c4iw_fill_res_cm_id_entry,475 .fill_res_mr_entry = c4iw_fill_res_mr_entry,476 .fill_res_qp_entry = c4iw_fill_res_qp_entry,477 .get_dev_fw_str = get_dev_fw_str,478 .get_dma_mr = c4iw_get_dma_mr,479 .get_hw_stats = c4iw_get_mib,480 .get_port_immutable = c4iw_port_immutable,481 .iw_accept = c4iw_accept_cr,482 .iw_add_ref = c4iw_qp_add_ref,483 .iw_connect = c4iw_connect,484 .iw_create_listen = c4iw_create_listen,485 .iw_destroy_listen = c4iw_destroy_listen,486 .iw_get_qp = c4iw_get_qp,487 .iw_reject = c4iw_reject_cr,488 .iw_rem_ref = c4iw_qp_rem_ref,489 .map_mr_sg = c4iw_map_mr_sg,490 .mmap = c4iw_mmap,491 .modify_qp = c4iw_ib_modify_qp,492 .modify_srq = c4iw_modify_srq,493 .poll_cq = c4iw_poll_cq,494 .post_recv = c4iw_post_receive,495 .post_send = c4iw_post_send,496 .post_srq_recv = c4iw_post_srq_recv,497 .query_device = c4iw_query_device,498 .query_gid = c4iw_query_gid,499 .query_port = c4iw_query_port,500 .query_qp = c4iw_ib_query_qp,501 .reg_user_mr = c4iw_reg_user_mr,502 .req_notify_cq = c4iw_arm_cq,503 504 INIT_RDMA_OBJ_SIZE(ib_cq, c4iw_cq, ibcq),505 INIT_RDMA_OBJ_SIZE(ib_mw, c4iw_mw, ibmw),506 INIT_RDMA_OBJ_SIZE(ib_pd, c4iw_pd, ibpd),507 INIT_RDMA_OBJ_SIZE(ib_qp, c4iw_qp, ibqp),508 INIT_RDMA_OBJ_SIZE(ib_srq, c4iw_srq, ibsrq),509 INIT_RDMA_OBJ_SIZE(ib_ucontext, c4iw_ucontext, ibucontext),510};511 512static int set_netdevs(struct ib_device *ib_dev, struct c4iw_rdev *rdev)513{514 int ret;515 int i;516 517 for (i = 0; i < rdev->lldi.nports; i++) {518 ret = ib_device_set_netdev(ib_dev, rdev->lldi.ports[i],519 i + 1);520 if (ret)521 return ret;522 }523 return 0;524}525 526void c4iw_register_device(struct work_struct *work)527{528 int ret;529 struct uld_ctx *ctx = container_of(work, struct uld_ctx, reg_work);530 struct c4iw_dev *dev = ctx->dev;531 532 pr_debug("c4iw_dev %p\n", dev);533 addrconf_addr_eui48((u8 *)&dev->ibdev.node_guid,534 dev->rdev.lldi.ports[0]->dev_addr);535 dev->ibdev.local_dma_lkey = 0;536 dev->ibdev.node_type = RDMA_NODE_RNIC;537 BUILD_BUG_ON(sizeof(C4IW_NODE_DESC) > IB_DEVICE_NODE_DESC_MAX);538 memcpy(dev->ibdev.node_desc, C4IW_NODE_DESC, sizeof(C4IW_NODE_DESC));539 dev->ibdev.phys_port_cnt = dev->rdev.lldi.nports;540 dev->ibdev.num_comp_vectors = dev->rdev.lldi.nciq;541 dev->ibdev.dev.parent = &dev->rdev.lldi.pdev->dev;542 543 memcpy(dev->ibdev.iw_ifname, dev->rdev.lldi.ports[0]->name,544 sizeof(dev->ibdev.iw_ifname));545 546 ib_set_device_ops(&dev->ibdev, &c4iw_dev_ops);547 ret = set_netdevs(&dev->ibdev, &dev->rdev);548 if (ret)549 goto err_dealloc_ctx;550 dma_set_max_seg_size(&dev->rdev.lldi.pdev->dev, UINT_MAX);551 ret = ib_register_device(&dev->ibdev, "cxgb4_%d",552 &dev->rdev.lldi.pdev->dev);553 if (ret)554 goto err_dealloc_ctx;555 return;556 557err_dealloc_ctx:558 pr_err("%s - Failed registering iwarp device: %d\n",559 pci_name(ctx->lldi.pdev), ret);560 c4iw_dealloc(ctx);561 return;562}563 564void c4iw_unregister_device(struct c4iw_dev *dev)565{566 pr_debug("c4iw_dev %p\n", dev);567 ib_unregister_device(&dev->ibdev);568 return;569}570