665 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause2 3/* Authors: Cheng Xu <chengyou@linux.alibaba.com> */4/* Kai Shen <kaishen@linux.alibaba.com> */5/* Copyright (c) 2020-2022, Alibaba Group. */6 7#include <linux/module.h>8#include <net/addrconf.h>9#include <rdma/erdma-abi.h>10 11#include "erdma.h"12#include "erdma_cm.h"13#include "erdma_verbs.h"14 15MODULE_AUTHOR("Cheng Xu <chengyou@linux.alibaba.com>");16MODULE_DESCRIPTION("Alibaba elasticRDMA adapter driver");17MODULE_LICENSE("Dual BSD/GPL");18 19static int erdma_netdev_event(struct notifier_block *nb, unsigned long event,20 void *arg)21{22 struct net_device *netdev = netdev_notifier_info_to_dev(arg);23 struct erdma_dev *dev = container_of(nb, struct erdma_dev, netdev_nb);24 25 if (dev->netdev == NULL || dev->netdev != netdev)26 goto done;27 28 switch (event) {29 case NETDEV_UP:30 dev->state = IB_PORT_ACTIVE;31 erdma_port_event(dev, IB_EVENT_PORT_ACTIVE);32 break;33 case NETDEV_DOWN:34 dev->state = IB_PORT_DOWN;35 erdma_port_event(dev, IB_EVENT_PORT_ERR);36 break;37 case NETDEV_CHANGEMTU:38 if (dev->mtu != netdev->mtu) {39 erdma_set_mtu(dev, netdev->mtu);40 dev->mtu = netdev->mtu;41 }42 break;43 case NETDEV_REGISTER:44 case NETDEV_UNREGISTER:45 case NETDEV_CHANGEADDR:46 case NETDEV_GOING_DOWN:47 case NETDEV_CHANGE:48 default:49 break;50 }51 52done:53 return NOTIFY_OK;54}55 56static int erdma_enum_and_get_netdev(struct erdma_dev *dev)57{58 struct net_device *netdev;59 int ret = -EPROBE_DEFER;60 61 /* Already binded to a net_device, so we skip. */62 if (dev->netdev)63 return 0;64 65 rtnl_lock();66 for_each_netdev(&init_net, netdev) {67 /*68 * In erdma, the paired netdev and ibdev should have the same69 * MAC address. erdma can get the value from its PCIe bar70 * registers. Since erdma can not get the paired netdev71 * reference directly, we do a traverse here to get the paired72 * netdev.73 */74 if (ether_addr_equal_unaligned(netdev->perm_addr,75 dev->attrs.peer_addr)) {76 ret = ib_device_set_netdev(&dev->ibdev, netdev, 1);77 if (ret) {78 rtnl_unlock();79 ibdev_warn(&dev->ibdev,80 "failed (%d) to link netdev", ret);81 return ret;82 }83 84 dev->netdev = netdev;85 break;86 }87 }88 89 rtnl_unlock();90 91 return ret;92}93 94static int erdma_device_register(struct erdma_dev *dev)95{96 struct ib_device *ibdev = &dev->ibdev;97 int ret;98 99 ret = erdma_enum_and_get_netdev(dev);100 if (ret)101 return ret;102 103 dev->mtu = dev->netdev->mtu;104 addrconf_addr_eui48((u8 *)&ibdev->node_guid, dev->netdev->dev_addr);105 106 ret = ib_register_device(ibdev, "erdma_%d", &dev->pdev->dev);107 if (ret) {108 dev_err(&dev->pdev->dev,109 "ib_register_device failed: ret = %d\n", ret);110 return ret;111 }112 113 dev->netdev_nb.notifier_call = erdma_netdev_event;114 ret = register_netdevice_notifier(&dev->netdev_nb);115 if (ret) {116 ibdev_err(&dev->ibdev, "failed to register notifier.\n");117 ib_unregister_device(ibdev);118 }119 120 return ret;121}122 123static irqreturn_t erdma_comm_irq_handler(int irq, void *data)124{125 struct erdma_dev *dev = data;126 127 erdma_cmdq_completion_handler(&dev->cmdq);128 erdma_aeq_event_handler(dev);129 130 return IRQ_HANDLED;131}132 133static int erdma_request_vectors(struct erdma_dev *dev)134{135 int expect_irq_num = min(num_possible_cpus() + 1, ERDMA_NUM_MSIX_VEC);136 int ret;137 138 ret = pci_alloc_irq_vectors(dev->pdev, 1, expect_irq_num, PCI_IRQ_MSIX);139 if (ret < 0) {140 dev_err(&dev->pdev->dev, "request irq vectors failed(%d)\n",141 ret);142 return ret;143 }144 dev->attrs.irq_num = ret;145 146 return 0;147}148 149static int erdma_comm_irq_init(struct erdma_dev *dev)150{151 snprintf(dev->comm_irq.name, ERDMA_IRQNAME_SIZE, "erdma-common@pci:%s",152 pci_name(dev->pdev));153 dev->comm_irq.msix_vector =154 pci_irq_vector(dev->pdev, ERDMA_MSIX_VECTOR_CMDQ);155 156 cpumask_set_cpu(cpumask_first(cpumask_of_pcibus(dev->pdev->bus)),157 &dev->comm_irq.affinity_hint_mask);158 irq_set_affinity_hint(dev->comm_irq.msix_vector,159 &dev->comm_irq.affinity_hint_mask);160 161 return request_irq(dev->comm_irq.msix_vector, erdma_comm_irq_handler, 0,162 dev->comm_irq.name, dev);163}164 165static void erdma_comm_irq_uninit(struct erdma_dev *dev)166{167 irq_set_affinity_hint(dev->comm_irq.msix_vector, NULL);168 free_irq(dev->comm_irq.msix_vector, dev);169}170 171static int erdma_device_init(struct erdma_dev *dev, struct pci_dev *pdev)172{173 int ret;174 175 dev->resp_pool = dma_pool_create("erdma_resp_pool", &pdev->dev,176 ERDMA_HW_RESP_SIZE, ERDMA_HW_RESP_SIZE,177 0);178 if (!dev->resp_pool)179 return -ENOMEM;180 181 dev->db_pool = dma_pool_create("erdma_db_pool", &pdev->dev,182 ERDMA_DB_SIZE, ERDMA_DB_SIZE, 0);183 if (!dev->db_pool) {184 ret = -ENOMEM;185 goto destroy_resp_pool;186 }187 188 ret = dma_set_mask_and_coherent(&pdev->dev,189 DMA_BIT_MASK(ERDMA_PCI_WIDTH));190 if (ret)191 goto destroy_db_pool;192 193 dma_set_max_seg_size(&pdev->dev, UINT_MAX);194 195 return 0;196 197destroy_db_pool:198 dma_pool_destroy(dev->db_pool);199 200destroy_resp_pool:201 dma_pool_destroy(dev->resp_pool);202 203 return ret;204}205 206static void erdma_device_uninit(struct erdma_dev *dev)207{208 dma_pool_destroy(dev->db_pool);209 dma_pool_destroy(dev->resp_pool);210}211 212static void erdma_hw_reset(struct erdma_dev *dev)213{214 u32 ctrl = FIELD_PREP(ERDMA_REG_DEV_CTRL_RESET_MASK, 1);215 216 erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG, ctrl);217}218 219static int erdma_wait_hw_init_done(struct erdma_dev *dev)220{221 int i;222 223 erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG,224 FIELD_PREP(ERDMA_REG_DEV_CTRL_INIT_MASK, 1));225 226 for (i = 0; i < ERDMA_WAIT_DEV_DONE_CNT; i++) {227 if (erdma_reg_read32_filed(dev, ERDMA_REGS_DEV_ST_REG,228 ERDMA_REG_DEV_ST_INIT_DONE_MASK))229 break;230 231 msleep(ERDMA_REG_ACCESS_WAIT_MS);232 }233 234 if (i == ERDMA_WAIT_DEV_DONE_CNT) {235 dev_err(&dev->pdev->dev, "wait init done failed.\n");236 return -ETIMEDOUT;237 }238 239 return 0;240}241 242static const struct pci_device_id erdma_pci_tbl[] = {243 { PCI_DEVICE(PCI_VENDOR_ID_ALIBABA, 0x107f) },244 {}245};246 247static int erdma_probe_dev(struct pci_dev *pdev)248{249 struct erdma_dev *dev;250 int bars, err;251 u32 version;252 253 err = pci_enable_device(pdev);254 if (err) {255 dev_err(&pdev->dev, "pci_enable_device failed(%d)\n", err);256 return err;257 }258 259 pci_set_master(pdev);260 261 dev = ib_alloc_device(erdma_dev, ibdev);262 if (!dev) {263 dev_err(&pdev->dev, "ib_alloc_device failed\n");264 err = -ENOMEM;265 goto err_disable_device;266 }267 268 pci_set_drvdata(pdev, dev);269 dev->pdev = pdev;270 dev->attrs.numa_node = dev_to_node(&pdev->dev);271 272 bars = pci_select_bars(pdev, IORESOURCE_MEM);273 err = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);274 if (bars != ERDMA_BAR_MASK || err) {275 err = err ? err : -EINVAL;276 goto err_ib_device_release;277 }278 279 dev->func_bar_addr = pci_resource_start(pdev, ERDMA_FUNC_BAR);280 dev->func_bar_len = pci_resource_len(pdev, ERDMA_FUNC_BAR);281 282 dev->func_bar =283 devm_ioremap(&pdev->dev, dev->func_bar_addr, dev->func_bar_len);284 if (!dev->func_bar) {285 dev_err(&pdev->dev, "devm_ioremap failed.\n");286 err = -EFAULT;287 goto err_release_bars;288 }289 290 version = erdma_reg_read32(dev, ERDMA_REGS_VERSION_REG);291 if (version == 0) {292 /* we knows that it is a non-functional function. */293 err = -ENODEV;294 goto err_iounmap_func_bar;295 }296 297 err = erdma_device_init(dev, pdev);298 if (err)299 goto err_iounmap_func_bar;300 301 err = erdma_request_vectors(dev);302 if (err)303 goto err_uninit_device;304 305 err = erdma_comm_irq_init(dev);306 if (err)307 goto err_free_vectors;308 309 err = erdma_aeq_init(dev);310 if (err)311 goto err_uninit_comm_irq;312 313 err = erdma_cmdq_init(dev);314 if (err)315 goto err_uninit_aeq;316 317 err = erdma_wait_hw_init_done(dev);318 if (err)319 goto err_uninit_cmdq;320 321 err = erdma_ceqs_init(dev);322 if (err)323 goto err_reset_hw;324 325 erdma_finish_cmdq_init(dev);326 327 return 0;328 329err_reset_hw:330 erdma_hw_reset(dev);331 332err_uninit_cmdq:333 erdma_cmdq_destroy(dev);334 335err_uninit_aeq:336 erdma_eq_destroy(dev, &dev->aeq);337 338err_uninit_comm_irq:339 erdma_comm_irq_uninit(dev);340 341err_free_vectors:342 pci_free_irq_vectors(dev->pdev);343 344err_uninit_device:345 erdma_device_uninit(dev);346 347err_iounmap_func_bar:348 devm_iounmap(&pdev->dev, dev->func_bar);349 350err_release_bars:351 pci_release_selected_regions(pdev, bars);352 353err_ib_device_release:354 ib_dealloc_device(&dev->ibdev);355 356err_disable_device:357 pci_disable_device(pdev);358 359 return err;360}361 362static void erdma_remove_dev(struct pci_dev *pdev)363{364 struct erdma_dev *dev = pci_get_drvdata(pdev);365 366 erdma_ceqs_uninit(dev);367 erdma_hw_reset(dev);368 erdma_cmdq_destroy(dev);369 erdma_eq_destroy(dev, &dev->aeq);370 erdma_comm_irq_uninit(dev);371 pci_free_irq_vectors(dev->pdev);372 erdma_device_uninit(dev);373 374 devm_iounmap(&pdev->dev, dev->func_bar);375 pci_release_selected_regions(pdev, ERDMA_BAR_MASK);376 377 ib_dealloc_device(&dev->ibdev);378 379 pci_disable_device(pdev);380}381 382#define ERDMA_GET_CAP(name, cap) FIELD_GET(ERDMA_CMD_DEV_CAP_##name##_MASK, cap)383 384static int erdma_dev_attrs_init(struct erdma_dev *dev)385{386 int err;387 u64 req_hdr, cap0, cap1;388 389 erdma_cmdq_build_reqhdr(&req_hdr, CMDQ_SUBMOD_RDMA,390 CMDQ_OPCODE_QUERY_DEVICE);391 392 err = erdma_post_cmd_wait(&dev->cmdq, &req_hdr, sizeof(req_hdr), &cap0,393 &cap1);394 if (err)395 return err;396 397 dev->attrs.max_cqe = 1 << ERDMA_GET_CAP(MAX_CQE, cap0);398 dev->attrs.max_mr_size = 1ULL << ERDMA_GET_CAP(MAX_MR_SIZE, cap0);399 dev->attrs.max_mw = 1 << ERDMA_GET_CAP(MAX_MW, cap1);400 dev->attrs.max_recv_wr = 1 << ERDMA_GET_CAP(MAX_RECV_WR, cap0);401 dev->attrs.local_dma_key = ERDMA_GET_CAP(DMA_LOCAL_KEY, cap1);402 dev->attrs.cc = ERDMA_GET_CAP(DEFAULT_CC, cap1);403 dev->attrs.max_qp = ERDMA_NQP_PER_QBLOCK * ERDMA_GET_CAP(QBLOCK, cap1);404 dev->attrs.max_mr = dev->attrs.max_qp << 1;405 dev->attrs.max_cq = dev->attrs.max_qp << 1;406 dev->attrs.cap_flags = ERDMA_GET_CAP(FLAGS, cap0);407 408 dev->attrs.max_send_wr = ERDMA_MAX_SEND_WR;409 dev->attrs.max_ord = ERDMA_MAX_ORD;410 dev->attrs.max_ird = ERDMA_MAX_IRD;411 dev->attrs.max_send_sge = ERDMA_MAX_SEND_SGE;412 dev->attrs.max_recv_sge = ERDMA_MAX_RECV_SGE;413 dev->attrs.max_sge_rd = ERDMA_MAX_SGE_RD;414 dev->attrs.max_pd = ERDMA_MAX_PD;415 416 dev->res_cb[ERDMA_RES_TYPE_PD].max_cap = ERDMA_MAX_PD;417 dev->res_cb[ERDMA_RES_TYPE_STAG_IDX].max_cap = dev->attrs.max_mr;418 419 erdma_cmdq_build_reqhdr(&req_hdr, CMDQ_SUBMOD_COMMON,420 CMDQ_OPCODE_QUERY_FW_INFO);421 422 err = erdma_post_cmd_wait(&dev->cmdq, &req_hdr, sizeof(req_hdr), &cap0,423 &cap1);424 if (!err)425 dev->attrs.fw_version =426 FIELD_GET(ERDMA_CMD_INFO0_FW_VER_MASK, cap0);427 428 return err;429}430 431static int erdma_device_config(struct erdma_dev *dev)432{433 struct erdma_cmdq_config_device_req req = {};434 435 if (!(dev->attrs.cap_flags & ERDMA_DEV_CAP_FLAGS_EXTEND_DB))436 return 0;437 438 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON,439 CMDQ_OPCODE_CONF_DEVICE);440 441 req.cfg = FIELD_PREP(ERDMA_CMD_CONFIG_DEVICE_PGSHIFT_MASK, PAGE_SHIFT) |442 FIELD_PREP(ERDMA_CMD_CONFIG_DEVICE_PS_EN_MASK, 1);443 444 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL);445}446 447static int erdma_res_cb_init(struct erdma_dev *dev)448{449 int i, j;450 451 for (i = 0; i < ERDMA_RES_CNT; i++) {452 dev->res_cb[i].next_alloc_idx = 1;453 spin_lock_init(&dev->res_cb[i].lock);454 dev->res_cb[i].bitmap =455 bitmap_zalloc(dev->res_cb[i].max_cap, GFP_KERNEL);456 if (!dev->res_cb[i].bitmap)457 goto err;458 }459 460 return 0;461 462err:463 for (j = 0; j < i; j++)464 bitmap_free(dev->res_cb[j].bitmap);465 466 return -ENOMEM;467}468 469static void erdma_res_cb_free(struct erdma_dev *dev)470{471 int i;472 473 for (i = 0; i < ERDMA_RES_CNT; i++)474 bitmap_free(dev->res_cb[i].bitmap);475}476 477static const struct ib_device_ops erdma_device_ops = {478 .owner = THIS_MODULE,479 .driver_id = RDMA_DRIVER_ERDMA,480 .uverbs_abi_ver = ERDMA_ABI_VERSION,481 482 .alloc_hw_port_stats = erdma_alloc_hw_port_stats,483 .alloc_mr = erdma_ib_alloc_mr,484 .alloc_pd = erdma_alloc_pd,485 .alloc_ucontext = erdma_alloc_ucontext,486 .create_cq = erdma_create_cq,487 .create_qp = erdma_create_qp,488 .dealloc_pd = erdma_dealloc_pd,489 .dealloc_ucontext = erdma_dealloc_ucontext,490 .dereg_mr = erdma_dereg_mr,491 .destroy_cq = erdma_destroy_cq,492 .destroy_qp = erdma_destroy_qp,493 .disassociate_ucontext = erdma_disassociate_ucontext,494 .get_dma_mr = erdma_get_dma_mr,495 .get_hw_stats = erdma_get_hw_stats,496 .get_port_immutable = erdma_get_port_immutable,497 .iw_accept = erdma_accept,498 .iw_add_ref = erdma_qp_get_ref,499 .iw_connect = erdma_connect,500 .iw_create_listen = erdma_create_listen,501 .iw_destroy_listen = erdma_destroy_listen,502 .iw_get_qp = erdma_get_ibqp,503 .iw_reject = erdma_reject,504 .iw_rem_ref = erdma_qp_put_ref,505 .map_mr_sg = erdma_map_mr_sg,506 .mmap = erdma_mmap,507 .mmap_free = erdma_mmap_free,508 .modify_qp = erdma_modify_qp,509 .post_recv = erdma_post_recv,510 .post_send = erdma_post_send,511 .poll_cq = erdma_poll_cq,512 .query_device = erdma_query_device,513 .query_gid = erdma_query_gid,514 .query_port = erdma_query_port,515 .query_qp = erdma_query_qp,516 .req_notify_cq = erdma_req_notify_cq,517 .reg_user_mr = erdma_reg_user_mr,518 519 INIT_RDMA_OBJ_SIZE(ib_cq, erdma_cq, ibcq),520 INIT_RDMA_OBJ_SIZE(ib_pd, erdma_pd, ibpd),521 INIT_RDMA_OBJ_SIZE(ib_ucontext, erdma_ucontext, ibucontext),522 INIT_RDMA_OBJ_SIZE(ib_qp, erdma_qp, ibqp),523};524 525static int erdma_ib_device_add(struct pci_dev *pdev)526{527 struct erdma_dev *dev = pci_get_drvdata(pdev);528 struct ib_device *ibdev = &dev->ibdev;529 u64 mac;530 int ret;531 532 ret = erdma_dev_attrs_init(dev);533 if (ret)534 return ret;535 536 ret = erdma_device_config(dev);537 if (ret)538 return ret;539 540 ibdev->node_type = RDMA_NODE_RNIC;541 memcpy(ibdev->node_desc, ERDMA_NODE_DESC, sizeof(ERDMA_NODE_DESC));542 543 /*544 * Current model (one-to-one device association):545 * One ERDMA device per net_device or, equivalently,546 * per physical port.547 */548 ibdev->phys_port_cnt = 1;549 ibdev->num_comp_vectors = dev->attrs.irq_num - 1;550 551 ib_set_device_ops(ibdev, &erdma_device_ops);552 553 INIT_LIST_HEAD(&dev->cep_list);554 555 spin_lock_init(&dev->lock);556 xa_init_flags(&dev->qp_xa, XA_FLAGS_ALLOC1);557 xa_init_flags(&dev->cq_xa, XA_FLAGS_ALLOC1);558 dev->next_alloc_cqn = 1;559 dev->next_alloc_qpn = 1;560 561 ret = erdma_res_cb_init(dev);562 if (ret)563 return ret;564 565 atomic_set(&dev->num_ctx, 0);566 567 mac = erdma_reg_read32(dev, ERDMA_REGS_NETDEV_MAC_L_REG);568 mac |= (u64)erdma_reg_read32(dev, ERDMA_REGS_NETDEV_MAC_H_REG) << 32;569 570 u64_to_ether_addr(mac, dev->attrs.peer_addr);571 572 dev->reflush_wq = alloc_workqueue("erdma-reflush-wq", WQ_UNBOUND,573 WQ_UNBOUND_MAX_ACTIVE);574 if (!dev->reflush_wq) {575 ret = -ENOMEM;576 goto err_alloc_workqueue;577 }578 579 ret = erdma_device_register(dev);580 if (ret)581 goto err_register;582 583 return 0;584 585err_register:586 destroy_workqueue(dev->reflush_wq);587err_alloc_workqueue:588 xa_destroy(&dev->qp_xa);589 xa_destroy(&dev->cq_xa);590 591 erdma_res_cb_free(dev);592 593 return ret;594}595 596static void erdma_ib_device_remove(struct pci_dev *pdev)597{598 struct erdma_dev *dev = pci_get_drvdata(pdev);599 600 unregister_netdevice_notifier(&dev->netdev_nb);601 ib_unregister_device(&dev->ibdev);602 603 destroy_workqueue(dev->reflush_wq);604 erdma_res_cb_free(dev);605 xa_destroy(&dev->qp_xa);606 xa_destroy(&dev->cq_xa);607}608 609static int erdma_probe(struct pci_dev *pdev, const struct pci_device_id *ent)610{611 int ret;612 613 ret = erdma_probe_dev(pdev);614 if (ret)615 return ret;616 617 ret = erdma_ib_device_add(pdev);618 if (ret) {619 erdma_remove_dev(pdev);620 return ret;621 }622 623 return 0;624}625 626static void erdma_remove(struct pci_dev *pdev)627{628 erdma_ib_device_remove(pdev);629 erdma_remove_dev(pdev);630}631 632static struct pci_driver erdma_pci_driver = {633 .name = DRV_MODULE_NAME,634 .id_table = erdma_pci_tbl,635 .probe = erdma_probe,636 .remove = erdma_remove637};638 639MODULE_DEVICE_TABLE(pci, erdma_pci_tbl);640 641static __init int erdma_init_module(void)642{643 int ret;644 645 ret = erdma_cm_init();646 if (ret)647 return ret;648 649 ret = pci_register_driver(&erdma_pci_driver);650 if (ret)651 erdma_cm_exit();652 653 return ret;654}655 656static void __exit erdma_exit_module(void)657{658 pci_unregister_driver(&erdma_pci_driver);659 660 erdma_cm_exit();661}662 663module_init(erdma_init_module);664module_exit(erdma_exit_module);665