550 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause2 3/* Authors: Bernard Metzler <bmt@zurich.ibm.com> */4/* Copyright (c) 2008-2019, IBM Corporation */5 6#include <linux/init.h>7#include <linux/errno.h>8#include <linux/netdevice.h>9#include <linux/inetdevice.h>10#include <net/net_namespace.h>11#include <linux/rtnetlink.h>12#include <linux/if_arp.h>13#include <linux/list.h>14#include <linux/kernel.h>15#include <linux/sched.h>16#include <linux/module.h>17#include <linux/dma-mapping.h>18 19#include <net/addrconf.h>20#include <rdma/ib_verbs.h>21#include <rdma/ib_user_verbs.h>22#include <rdma/rdma_netlink.h>23#include <linux/kthread.h>24 25#include "siw.h"26#include "siw_verbs.h"27 28MODULE_AUTHOR("Bernard Metzler");29MODULE_DESCRIPTION("Software iWARP Driver");30MODULE_LICENSE("Dual BSD/GPL");31 32/* transmit from user buffer, if possible */33const bool zcopy_tx = true;34 35/* Restrict usage of GSO, if hardware peer iwarp is unable to process36 * large packets. try_gso = true lets siw try to use local GSO,37 * if peer agrees. Not using GSO severly limits siw maximum tx bandwidth.38 */39const bool try_gso;40 41/* Attach siw also with loopback devices */42const bool loopback_enabled = true;43 44/* We try to negotiate CRC on, if true */45const bool mpa_crc_required;46 47/* MPA CRC on/off enforced */48const bool mpa_crc_strict;49 50/* Control TCP_NODELAY socket option */51const bool siw_tcp_nagle;52 53/* Select MPA version to be used during connection setup */54u_char mpa_version = MPA_REVISION_2;55 56/* Selects MPA P2P mode (additional handshake during connection57 * setup, if true.58 */59const bool peer_to_peer;60 61struct task_struct *siw_tx_thread[NR_CPUS];62struct crypto_shash *siw_crypto_shash;63 64static int siw_device_register(struct siw_device *sdev, const char *name)65{66 struct ib_device *base_dev = &sdev->base_dev;67 static int dev_id = 1;68 int rv;69 70 sdev->vendor_part_id = dev_id++;71 72 rv = ib_register_device(base_dev, name, NULL);73 if (rv) {74 pr_warn("siw: device registration error %d\n", rv);75 return rv;76 }77 78 siw_dbg(base_dev, "HWaddr=%pM\n", sdev->raw_gid);79 return 0;80}81 82static void siw_device_cleanup(struct ib_device *base_dev)83{84 struct siw_device *sdev = to_siw_dev(base_dev);85 86 xa_destroy(&sdev->qp_xa);87 xa_destroy(&sdev->mem_xa);88}89 90static int siw_dev_qualified(struct net_device *netdev)91{92 /*93 * Additional hardware support can be added here94 * (e.g. ARPHRD_FDDI, ARPHRD_ATM, ...) - see95 * <linux/if_arp.h> for type identifiers.96 */97 if (netdev->type == ARPHRD_ETHER || netdev->type == ARPHRD_IEEE802 ||98 netdev->type == ARPHRD_NONE ||99 (netdev->type == ARPHRD_LOOPBACK && loopback_enabled))100 return 1;101 102 return 0;103}104 105static DEFINE_PER_CPU(atomic_t, siw_use_cnt);106 107static struct {108 struct cpumask **tx_valid_cpus;109 int num_nodes;110} siw_cpu_info;111 112static void siw_destroy_cpulist(int number)113{114 int i = 0;115 116 while (i < number)117 kfree(siw_cpu_info.tx_valid_cpus[i++]);118 119 kfree(siw_cpu_info.tx_valid_cpus);120 siw_cpu_info.tx_valid_cpus = NULL;121}122 123static int siw_init_cpulist(void)124{125 int i, num_nodes = nr_node_ids;126 127 memset(siw_tx_thread, 0, sizeof(siw_tx_thread));128 129 siw_cpu_info.num_nodes = num_nodes;130 131 siw_cpu_info.tx_valid_cpus =132 kcalloc(num_nodes, sizeof(struct cpumask *), GFP_KERNEL);133 if (!siw_cpu_info.tx_valid_cpus) {134 siw_cpu_info.num_nodes = 0;135 return -ENOMEM;136 }137 for (i = 0; i < siw_cpu_info.num_nodes; i++) {138 siw_cpu_info.tx_valid_cpus[i] =139 kzalloc(sizeof(struct cpumask), GFP_KERNEL);140 if (!siw_cpu_info.tx_valid_cpus[i])141 goto out_err;142 143 cpumask_clear(siw_cpu_info.tx_valid_cpus[i]);144 }145 for_each_possible_cpu(i)146 cpumask_set_cpu(i, siw_cpu_info.tx_valid_cpus[cpu_to_node(i)]);147 148 return 0;149 150out_err:151 siw_cpu_info.num_nodes = 0;152 siw_destroy_cpulist(i);153 154 return -ENOMEM;155}156 157/*158 * Choose CPU with least number of active QP's from NUMA node of159 * TX interface.160 */161int siw_get_tx_cpu(struct siw_device *sdev)162{163 const struct cpumask *tx_cpumask;164 int i, num_cpus, cpu, min_use, node = sdev->numa_node, tx_cpu = -1;165 166 if (node < 0)167 tx_cpumask = cpu_online_mask;168 else169 tx_cpumask = siw_cpu_info.tx_valid_cpus[node];170 171 num_cpus = cpumask_weight(tx_cpumask);172 if (!num_cpus) {173 /* no CPU on this NUMA node */174 tx_cpumask = cpu_online_mask;175 num_cpus = cpumask_weight(tx_cpumask);176 }177 if (!num_cpus)178 goto out;179 180 cpu = cpumask_first(tx_cpumask);181 182 for (i = 0, min_use = SIW_MAX_QP; i < num_cpus;183 i++, cpu = cpumask_next(cpu, tx_cpumask)) {184 int usage;185 186 /* Skip any cores which have no TX thread */187 if (!siw_tx_thread[cpu])188 continue;189 190 usage = atomic_read(&per_cpu(siw_use_cnt, cpu));191 if (usage <= min_use) {192 tx_cpu = cpu;193 min_use = usage;194 }195 }196 siw_dbg(&sdev->base_dev,197 "tx cpu %d, node %d, %d qp's\n", tx_cpu, node, min_use);198 199out:200 if (tx_cpu >= 0)201 atomic_inc(&per_cpu(siw_use_cnt, tx_cpu));202 else203 pr_warn("siw: no tx cpu found\n");204 205 return tx_cpu;206}207 208void siw_put_tx_cpu(int cpu)209{210 atomic_dec(&per_cpu(siw_use_cnt, cpu));211}212 213static struct ib_qp *siw_get_base_qp(struct ib_device *base_dev, int id)214{215 struct siw_qp *qp = siw_qp_id2obj(to_siw_dev(base_dev), id);216 217 if (qp) {218 /*219 * siw_qp_id2obj() increments object reference count220 */221 siw_qp_put(qp);222 return &qp->base_qp;223 }224 return NULL;225}226 227static const struct ib_device_ops siw_device_ops = {228 .owner = THIS_MODULE,229 .uverbs_abi_ver = SIW_ABI_VERSION,230 .driver_id = RDMA_DRIVER_SIW,231 232 .alloc_mr = siw_alloc_mr,233 .alloc_pd = siw_alloc_pd,234 .alloc_ucontext = siw_alloc_ucontext,235 .create_cq = siw_create_cq,236 .create_qp = siw_create_qp,237 .create_srq = siw_create_srq,238 .dealloc_driver = siw_device_cleanup,239 .dealloc_pd = siw_dealloc_pd,240 .dealloc_ucontext = siw_dealloc_ucontext,241 .dereg_mr = siw_dereg_mr,242 .destroy_cq = siw_destroy_cq,243 .destroy_qp = siw_destroy_qp,244 .destroy_srq = siw_destroy_srq,245 .get_dma_mr = siw_get_dma_mr,246 .get_port_immutable = siw_get_port_immutable,247 .iw_accept = siw_accept,248 .iw_add_ref = siw_qp_get_ref,249 .iw_connect = siw_connect,250 .iw_create_listen = siw_create_listen,251 .iw_destroy_listen = siw_destroy_listen,252 .iw_get_qp = siw_get_base_qp,253 .iw_reject = siw_reject,254 .iw_rem_ref = siw_qp_put_ref,255 .map_mr_sg = siw_map_mr_sg,256 .mmap = siw_mmap,257 .mmap_free = siw_mmap_free,258 .modify_qp = siw_verbs_modify_qp,259 .modify_srq = siw_modify_srq,260 .poll_cq = siw_poll_cq,261 .post_recv = siw_post_receive,262 .post_send = siw_post_send,263 .post_srq_recv = siw_post_srq_recv,264 .query_device = siw_query_device,265 .query_gid = siw_query_gid,266 .query_port = siw_query_port,267 .query_qp = siw_query_qp,268 .query_srq = siw_query_srq,269 .req_notify_cq = siw_req_notify_cq,270 .reg_user_mr = siw_reg_user_mr,271 272 INIT_RDMA_OBJ_SIZE(ib_cq, siw_cq, base_cq),273 INIT_RDMA_OBJ_SIZE(ib_pd, siw_pd, base_pd),274 INIT_RDMA_OBJ_SIZE(ib_qp, siw_qp, base_qp),275 INIT_RDMA_OBJ_SIZE(ib_srq, siw_srq, base_srq),276 INIT_RDMA_OBJ_SIZE(ib_ucontext, siw_ucontext, base_ucontext),277};278 279static struct siw_device *siw_device_create(struct net_device *netdev)280{281 struct siw_device *sdev = NULL;282 struct ib_device *base_dev;283 int rv;284 285 sdev = ib_alloc_device(siw_device, base_dev);286 if (!sdev)287 return NULL;288 289 base_dev = &sdev->base_dev;290 sdev->netdev = netdev;291 292 if (netdev->addr_len) {293 memcpy(sdev->raw_gid, netdev->dev_addr,294 min_t(unsigned int, netdev->addr_len, ETH_ALEN));295 } else {296 /*297 * This device does not have a HW address, but298 * connection mangagement requires a unique gid.299 */300 eth_random_addr(sdev->raw_gid);301 }302 addrconf_addr_eui48((u8 *)&base_dev->node_guid, sdev->raw_gid);303 304 base_dev->uverbs_cmd_mask |= BIT_ULL(IB_USER_VERBS_CMD_POST_SEND);305 306 base_dev->node_type = RDMA_NODE_RNIC;307 memcpy(base_dev->node_desc, SIW_NODE_DESC_COMMON,308 sizeof(SIW_NODE_DESC_COMMON));309 310 /*311 * Current model (one-to-one device association):312 * One Softiwarp device per net_device or, equivalently,313 * per physical port.314 */315 base_dev->phys_port_cnt = 1;316 base_dev->num_comp_vectors = num_possible_cpus();317 318 xa_init_flags(&sdev->qp_xa, XA_FLAGS_ALLOC1);319 xa_init_flags(&sdev->mem_xa, XA_FLAGS_ALLOC1);320 321 ib_set_device_ops(base_dev, &siw_device_ops);322 rv = ib_device_set_netdev(base_dev, netdev, 1);323 if (rv)324 goto error;325 326 memcpy(base_dev->iw_ifname, netdev->name,327 sizeof(base_dev->iw_ifname));328 329 /* Disable TCP port mapping */330 base_dev->iw_driver_flags = IW_F_NO_PORT_MAP;331 332 sdev->attrs.max_qp = SIW_MAX_QP;333 sdev->attrs.max_qp_wr = SIW_MAX_QP_WR;334 sdev->attrs.max_ord = SIW_MAX_ORD_QP;335 sdev->attrs.max_ird = SIW_MAX_IRD_QP;336 sdev->attrs.max_sge = SIW_MAX_SGE;337 sdev->attrs.max_sge_rd = SIW_MAX_SGE_RD;338 sdev->attrs.max_cq = SIW_MAX_CQ;339 sdev->attrs.max_cqe = SIW_MAX_CQE;340 sdev->attrs.max_mr = SIW_MAX_MR;341 sdev->attrs.max_pd = SIW_MAX_PD;342 sdev->attrs.max_mw = SIW_MAX_MW;343 sdev->attrs.max_srq = SIW_MAX_SRQ;344 sdev->attrs.max_srq_wr = SIW_MAX_SRQ_WR;345 sdev->attrs.max_srq_sge = SIW_MAX_SGE;346 347 INIT_LIST_HEAD(&sdev->cep_list);348 INIT_LIST_HEAD(&sdev->qp_list);349 350 atomic_set(&sdev->num_ctx, 0);351 atomic_set(&sdev->num_srq, 0);352 atomic_set(&sdev->num_qp, 0);353 atomic_set(&sdev->num_cq, 0);354 atomic_set(&sdev->num_mr, 0);355 atomic_set(&sdev->num_pd, 0);356 357 sdev->numa_node = dev_to_node(&netdev->dev);358 spin_lock_init(&sdev->lock);359 360 return sdev;361error:362 ib_dealloc_device(base_dev);363 364 return NULL;365}366 367static int siw_netdev_event(struct notifier_block *nb, unsigned long event,368 void *arg)369{370 struct net_device *netdev = netdev_notifier_info_to_dev(arg);371 struct ib_device *base_dev;372 struct siw_device *sdev;373 374 dev_dbg(&netdev->dev, "siw: event %lu\n", event);375 376 base_dev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_SIW);377 if (!base_dev)378 return NOTIFY_OK;379 380 sdev = to_siw_dev(base_dev);381 382 switch (event) {383 case NETDEV_UP:384 sdev->state = IB_PORT_ACTIVE;385 siw_port_event(sdev, 1, IB_EVENT_PORT_ACTIVE);386 break;387 388 case NETDEV_DOWN:389 sdev->state = IB_PORT_DOWN;390 siw_port_event(sdev, 1, IB_EVENT_PORT_ERR);391 break;392 393 case NETDEV_REGISTER:394 /*395 * Device registration now handled only by396 * rdma netlink commands. So it shall be impossible397 * to end up here with a valid siw device.398 */399 siw_dbg(base_dev, "unexpected NETDEV_REGISTER event\n");400 break;401 402 case NETDEV_UNREGISTER:403 ib_unregister_device_queued(&sdev->base_dev);404 break;405 406 case NETDEV_CHANGEADDR:407 siw_port_event(sdev, 1, IB_EVENT_LID_CHANGE);408 break;409 /*410 * Todo: Below netdev events are currently not handled.411 */412 case NETDEV_CHANGEMTU:413 case NETDEV_CHANGE:414 break;415 416 default:417 break;418 }419 ib_device_put(&sdev->base_dev);420 421 return NOTIFY_OK;422}423 424static struct notifier_block siw_netdev_nb = {425 .notifier_call = siw_netdev_event,426};427 428static int siw_newlink(const char *basedev_name, struct net_device *netdev)429{430 struct ib_device *base_dev;431 struct siw_device *sdev = NULL;432 int rv = -ENOMEM;433 434 if (!siw_dev_qualified(netdev))435 return -EINVAL;436 437 base_dev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_SIW);438 if (base_dev) {439 ib_device_put(base_dev);440 return -EEXIST;441 }442 sdev = siw_device_create(netdev);443 if (sdev) {444 dev_dbg(&netdev->dev, "siw: new device\n");445 446 if (netif_running(netdev) && netif_carrier_ok(netdev))447 sdev->state = IB_PORT_ACTIVE;448 else449 sdev->state = IB_PORT_DOWN;450 451 ib_mark_name_assigned_by_user(&sdev->base_dev);452 rv = siw_device_register(sdev, basedev_name);453 if (rv)454 ib_dealloc_device(&sdev->base_dev);455 }456 return rv;457}458 459static struct rdma_link_ops siw_link_ops = {460 .type = "siw",461 .newlink = siw_newlink,462};463 464/*465 * siw_init_module - Initialize Softiwarp module and register with netdev466 * subsystem.467 */468static __init int siw_init_module(void)469{470 int rv;471 472 if (SENDPAGE_THRESH < SIW_MAX_INLINE) {473 pr_info("siw: sendpage threshold too small: %u\n",474 (int)SENDPAGE_THRESH);475 rv = -EINVAL;476 goto out_error;477 }478 rv = siw_init_cpulist();479 if (rv)480 goto out_error;481 482 rv = siw_cm_init();483 if (rv)484 goto out_error;485 486 if (!siw_create_tx_threads()) {487 pr_info("siw: Could not start any TX thread\n");488 rv = -ENOMEM;489 goto out_error;490 }491 /*492 * Locate CRC32 algorithm. If unsuccessful, fail493 * loading siw only, if CRC is required.494 */495 siw_crypto_shash = crypto_alloc_shash("crc32c", 0, 0);496 if (IS_ERR(siw_crypto_shash)) {497 pr_info("siw: Loading CRC32c failed: %ld\n",498 PTR_ERR(siw_crypto_shash));499 siw_crypto_shash = NULL;500 if (mpa_crc_required) {501 rv = -EOPNOTSUPP;502 goto out_error;503 }504 }505 rv = register_netdevice_notifier(&siw_netdev_nb);506 if (rv)507 goto out_error;508 509 rdma_link_register(&siw_link_ops);510 511 pr_info("SoftiWARP attached\n");512 return 0;513 514out_error:515 siw_stop_tx_threads();516 517 if (siw_crypto_shash)518 crypto_free_shash(siw_crypto_shash);519 520 pr_info("SoftIWARP attach failed. Error: %d\n", rv);521 522 siw_cm_exit();523 siw_destroy_cpulist(siw_cpu_info.num_nodes);524 525 return rv;526}527 528static void __exit siw_exit_module(void)529{530 siw_stop_tx_threads();531 532 unregister_netdevice_notifier(&siw_netdev_nb);533 rdma_link_unregister(&siw_link_ops);534 ib_unregister_driver(RDMA_DRIVER_SIW);535 536 siw_cm_exit();537 538 siw_destroy_cpulist(siw_cpu_info.num_nodes);539 540 if (siw_crypto_shash)541 crypto_free_shash(siw_crypto_shash);542 543 pr_info("SoftiWARP detached\n");544}545 546module_init(siw_init_module);547module_exit(siw_exit_module);548 549MODULE_ALIAS_RDMA_LINK("siw");550