2726 lines · c
1/*2 * Copyright (c) 2004 Topspin Communications. All rights reserved.3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.4 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.5 *6 * This software is available to you under a choice of one of two7 * licenses. You may choose to be licensed under the terms of the GNU8 * General Public License (GPL) Version 2, available from the file9 * COPYING in the main directory of this source tree, or the10 * OpenIB.org BSD license below:11 *12 * Redistribution and use in source and binary forms, with or13 * without modification, are permitted provided that the following14 * conditions are met:15 *16 * - Redistributions of source code must retain the above17 * copyright notice, this list of conditions and the following18 * disclaimer.19 *20 * - Redistributions in binary form must reproduce the above21 * copyright notice, this list of conditions and the following22 * disclaimer in the documentation and/or other materials23 * provided with the distribution.24 *25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE32 * SOFTWARE.33 */34 35#include "ipoib.h"36 37#include <linux/module.h>38 39#include <linux/init.h>40#include <linux/slab.h>41#include <linux/kernel.h>42#include <linux/vmalloc.h>43 44#include <linux/if_arp.h> /* For ARPHRD_xxx */45 46#include <linux/ip.h>47#include <linux/in.h>48 49#include <linux/jhash.h>50#include <net/arp.h>51#include <net/addrconf.h>52#include <linux/inetdevice.h>53#include <rdma/ib_cache.h>54 55MODULE_AUTHOR("Roland Dreier");56MODULE_DESCRIPTION("IP-over-InfiniBand net driver");57MODULE_LICENSE("Dual BSD/GPL");58 59int ipoib_sendq_size __read_mostly = IPOIB_TX_RING_SIZE;60int ipoib_recvq_size __read_mostly = IPOIB_RX_RING_SIZE;61 62module_param_named(send_queue_size, ipoib_sendq_size, int, 0444);63MODULE_PARM_DESC(send_queue_size, "Number of descriptors in send queue");64module_param_named(recv_queue_size, ipoib_recvq_size, int, 0444);65MODULE_PARM_DESC(recv_queue_size, "Number of descriptors in receive queue");66 67#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG68int ipoib_debug_level;69 70module_param_named(debug_level, ipoib_debug_level, int, 0644);71MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0");72#endif73 74struct ipoib_path_iter {75 struct net_device *dev;76 struct ipoib_path path;77};78 79static const u8 ipv4_bcast_addr[] = {80 0x00, 0xff, 0xff, 0xff,81 0xff, 0x12, 0x40, 0x1b, 0x00, 0x00, 0x00, 0x00,82 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff83};84 85struct workqueue_struct *ipoib_workqueue;86 87struct ib_sa_client ipoib_sa_client;88 89static int ipoib_add_one(struct ib_device *device);90static void ipoib_remove_one(struct ib_device *device, void *client_data);91static void ipoib_neigh_reclaim(struct rcu_head *rp);92static struct net_device *ipoib_get_net_dev_by_params(93 struct ib_device *dev, u32 port, u16 pkey,94 const union ib_gid *gid, const struct sockaddr *addr,95 void *client_data);96static int ipoib_set_mac(struct net_device *dev, void *addr);97static int ipoib_ioctl(struct net_device *dev, struct ifreq *ifr,98 int cmd);99 100static struct ib_client ipoib_client = {101 .name = "ipoib",102 .add = ipoib_add_one,103 .remove = ipoib_remove_one,104 .get_net_dev_by_params = ipoib_get_net_dev_by_params,105};106 107#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG108static int ipoib_netdev_event(struct notifier_block *this,109 unsigned long event, void *ptr)110{111 struct netdev_notifier_info *ni = ptr;112 struct net_device *dev = ni->dev;113 114 if (dev->netdev_ops->ndo_open != ipoib_open)115 return NOTIFY_DONE;116 117 switch (event) {118 case NETDEV_REGISTER:119 ipoib_create_debug_files(dev);120 break;121 case NETDEV_CHANGENAME:122 ipoib_delete_debug_files(dev);123 ipoib_create_debug_files(dev);124 break;125 case NETDEV_UNREGISTER:126 ipoib_delete_debug_files(dev);127 break;128 }129 130 return NOTIFY_DONE;131}132#endif133 134int ipoib_open(struct net_device *dev)135{136 struct ipoib_dev_priv *priv = ipoib_priv(dev);137 138 ipoib_dbg(priv, "bringing up interface\n");139 140 netif_carrier_off(dev);141 142 set_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags);143 144 if (ipoib_ib_dev_open(dev)) {145 if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags))146 return 0;147 goto err_disable;148 }149 150 ipoib_ib_dev_up(dev);151 152 if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {153 struct ipoib_dev_priv *cpriv;154 155 /* Bring up any child interfaces too */156 down_read(&priv->vlan_rwsem);157 list_for_each_entry(cpriv, &priv->child_intfs, list) {158 int flags;159 160 flags = cpriv->dev->flags;161 if (flags & IFF_UP)162 continue;163 164 dev_change_flags(cpriv->dev, flags | IFF_UP, NULL);165 }166 up_read(&priv->vlan_rwsem);167 } else if (priv->parent) {168 struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);169 170 if (!test_bit(IPOIB_FLAG_ADMIN_UP, &ppriv->flags))171 ipoib_dbg(priv, "parent device %s is not up, so child device may be not functioning.\n",172 ppriv->dev->name);173 }174 netif_start_queue(dev);175 176 return 0;177 178err_disable:179 clear_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags);180 181 return -EINVAL;182}183 184static int ipoib_stop(struct net_device *dev)185{186 struct ipoib_dev_priv *priv = ipoib_priv(dev);187 188 ipoib_dbg(priv, "stopping interface\n");189 190 clear_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags);191 192 netif_stop_queue(dev);193 194 ipoib_ib_dev_down(dev);195 ipoib_ib_dev_stop(dev);196 197 if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {198 struct ipoib_dev_priv *cpriv;199 200 /* Bring down any child interfaces too */201 down_read(&priv->vlan_rwsem);202 list_for_each_entry(cpriv, &priv->child_intfs, list) {203 int flags;204 205 flags = cpriv->dev->flags;206 if (!(flags & IFF_UP))207 continue;208 209 dev_change_flags(cpriv->dev, flags & ~IFF_UP, NULL);210 }211 up_read(&priv->vlan_rwsem);212 }213 214 return 0;215}216 217static netdev_features_t ipoib_fix_features(struct net_device *dev, netdev_features_t features)218{219 struct ipoib_dev_priv *priv = ipoib_priv(dev);220 221 if (test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags))222 features &= ~(NETIF_F_IP_CSUM | NETIF_F_TSO);223 224 return features;225}226 227static int ipoib_change_mtu(struct net_device *dev, int new_mtu)228{229 struct ipoib_dev_priv *priv = ipoib_priv(dev);230 int ret = 0;231 232 /* dev->mtu > 2K ==> connected mode */233 if (ipoib_cm_admin_enabled(dev)) {234 if (new_mtu > ipoib_cm_max_mtu(dev))235 return -EINVAL;236 237 if (new_mtu > priv->mcast_mtu)238 ipoib_warn(priv, "mtu > %d will cause multicast packet drops.\n",239 priv->mcast_mtu);240 241 WRITE_ONCE(dev->mtu, new_mtu);242 return 0;243 }244 245 if (new_mtu < (ETH_MIN_MTU + IPOIB_ENCAP_LEN) ||246 new_mtu > IPOIB_UD_MTU(priv->max_ib_mtu))247 return -EINVAL;248 249 priv->admin_mtu = new_mtu;250 251 if (priv->mcast_mtu < priv->admin_mtu)252 ipoib_dbg(priv, "MTU must be smaller than the underlying "253 "link layer MTU - 4 (%u)\n", priv->mcast_mtu);254 255 new_mtu = min(priv->mcast_mtu, priv->admin_mtu);256 257 if (priv->rn_ops->ndo_change_mtu) {258 bool carrier_status = netif_carrier_ok(dev);259 260 netif_carrier_off(dev);261 262 /* notify lower level on the real mtu */263 ret = priv->rn_ops->ndo_change_mtu(dev, new_mtu);264 265 if (carrier_status)266 netif_carrier_on(dev);267 } else {268 WRITE_ONCE(dev->mtu, new_mtu);269 }270 271 return ret;272}273 274static void ipoib_get_stats(struct net_device *dev,275 struct rtnl_link_stats64 *stats)276{277 struct ipoib_dev_priv *priv = ipoib_priv(dev);278 279 if (priv->rn_ops->ndo_get_stats64)280 priv->rn_ops->ndo_get_stats64(dev, stats);281 else282 netdev_stats_to_stats64(stats, &dev->stats);283}284 285/* Called with an RCU read lock taken */286static bool ipoib_is_dev_match_addr_rcu(const struct sockaddr *addr,287 struct net_device *dev)288{289 struct net *net = dev_net(dev);290 struct in_device *in_dev;291 struct sockaddr_in *addr_in = (struct sockaddr_in *)addr;292 struct sockaddr_in6 *addr_in6 = (struct sockaddr_in6 *)addr;293 __be32 ret_addr;294 295 switch (addr->sa_family) {296 case AF_INET:297 in_dev = in_dev_get(dev);298 if (!in_dev)299 return false;300 301 ret_addr = inet_confirm_addr(net, in_dev, 0,302 addr_in->sin_addr.s_addr,303 RT_SCOPE_HOST);304 in_dev_put(in_dev);305 if (ret_addr)306 return true;307 308 break;309 case AF_INET6:310 if (IS_ENABLED(CONFIG_IPV6) &&311 ipv6_chk_addr(net, &addr_in6->sin6_addr, dev, 1))312 return true;313 314 break;315 }316 return false;317}318 319/*320 * Find the master net_device on top of the given net_device.321 * @dev: base IPoIB net_device322 *323 * Returns the master net_device with a reference held, or the same net_device324 * if no master exists.325 */326static struct net_device *ipoib_get_master_net_dev(struct net_device *dev)327{328 struct net_device *master;329 330 rcu_read_lock();331 master = netdev_master_upper_dev_get_rcu(dev);332 dev_hold(master);333 rcu_read_unlock();334 335 if (master)336 return master;337 338 dev_hold(dev);339 return dev;340}341 342struct ipoib_walk_data {343 const struct sockaddr *addr;344 struct net_device *result;345};346 347static int ipoib_upper_walk(struct net_device *upper,348 struct netdev_nested_priv *priv)349{350 struct ipoib_walk_data *data = (struct ipoib_walk_data *)priv->data;351 int ret = 0;352 353 if (ipoib_is_dev_match_addr_rcu(data->addr, upper)) {354 dev_hold(upper);355 data->result = upper;356 ret = 1;357 }358 359 return ret;360}361 362/**363 * ipoib_get_net_dev_match_addr - Find a net_device matching364 * the given address, which is an upper device of the given net_device.365 *366 * @addr: IP address to look for.367 * @dev: base IPoIB net_device368 *369 * If found, returns the net_device with a reference held. Otherwise return370 * NULL.371 */372static struct net_device *ipoib_get_net_dev_match_addr(373 const struct sockaddr *addr, struct net_device *dev)374{375 struct netdev_nested_priv priv;376 struct ipoib_walk_data data = {377 .addr = addr,378 };379 380 priv.data = (void *)&data;381 rcu_read_lock();382 if (ipoib_is_dev_match_addr_rcu(addr, dev)) {383 dev_hold(dev);384 data.result = dev;385 goto out;386 }387 388 netdev_walk_all_upper_dev_rcu(dev, ipoib_upper_walk, &priv);389out:390 rcu_read_unlock();391 return data.result;392}393 394/* returns the number of IPoIB netdevs on top a given ipoib device matching a395 * pkey_index and address, if one exists.396 *397 * @found_net_dev: contains a matching net_device if the return value >= 1,398 * with a reference held. */399static int ipoib_match_gid_pkey_addr(struct ipoib_dev_priv *priv,400 const union ib_gid *gid,401 u16 pkey_index,402 const struct sockaddr *addr,403 int nesting,404 struct net_device **found_net_dev)405{406 struct ipoib_dev_priv *child_priv;407 struct net_device *net_dev = NULL;408 int matches = 0;409 410 if (priv->pkey_index == pkey_index &&411 (!gid || !memcmp(gid, &priv->local_gid, sizeof(*gid)))) {412 if (!addr) {413 net_dev = ipoib_get_master_net_dev(priv->dev);414 } else {415 /* Verify the net_device matches the IP address, as416 * IPoIB child devices currently share a GID. */417 net_dev = ipoib_get_net_dev_match_addr(addr, priv->dev);418 }419 if (net_dev) {420 if (!*found_net_dev)421 *found_net_dev = net_dev;422 else423 dev_put(net_dev);424 ++matches;425 }426 }427 428 /* Check child interfaces */429 down_read_nested(&priv->vlan_rwsem, nesting);430 list_for_each_entry(child_priv, &priv->child_intfs, list) {431 matches += ipoib_match_gid_pkey_addr(child_priv, gid,432 pkey_index, addr,433 nesting + 1,434 found_net_dev);435 if (matches > 1)436 break;437 }438 up_read(&priv->vlan_rwsem);439 440 return matches;441}442 443/* Returns the number of matching net_devs found (between 0 and 2). Also444 * return the matching net_device in the @net_dev parameter, holding a445 * reference to the net_device, if the number of matches >= 1 */446static int __ipoib_get_net_dev_by_params(struct list_head *dev_list, u32 port,447 u16 pkey_index,448 const union ib_gid *gid,449 const struct sockaddr *addr,450 struct net_device **net_dev)451{452 struct ipoib_dev_priv *priv;453 int matches = 0;454 455 *net_dev = NULL;456 457 list_for_each_entry(priv, dev_list, list) {458 if (priv->port != port)459 continue;460 461 matches += ipoib_match_gid_pkey_addr(priv, gid, pkey_index,462 addr, 0, net_dev);463 if (matches > 1)464 break;465 }466 467 return matches;468}469 470static struct net_device *ipoib_get_net_dev_by_params(471 struct ib_device *dev, u32 port, u16 pkey,472 const union ib_gid *gid, const struct sockaddr *addr,473 void *client_data)474{475 struct net_device *net_dev;476 struct list_head *dev_list = client_data;477 u16 pkey_index;478 int matches;479 int ret;480 481 if (!rdma_protocol_ib(dev, port))482 return NULL;483 484 ret = ib_find_cached_pkey(dev, port, pkey, &pkey_index);485 if (ret)486 return NULL;487 488 /* See if we can find a unique device matching the L2 parameters */489 matches = __ipoib_get_net_dev_by_params(dev_list, port, pkey_index,490 gid, NULL, &net_dev);491 492 switch (matches) {493 case 0:494 return NULL;495 case 1:496 return net_dev;497 }498 499 dev_put(net_dev);500 501 /* Couldn't find a unique device with L2 parameters only. Use L3502 * address to uniquely match the net device */503 matches = __ipoib_get_net_dev_by_params(dev_list, port, pkey_index,504 gid, addr, &net_dev);505 switch (matches) {506 case 0:507 return NULL;508 default:509 dev_warn_ratelimited(&dev->dev,510 "duplicate IP address detected\n");511 fallthrough;512 case 1:513 return net_dev;514 }515}516 517int ipoib_set_mode(struct net_device *dev, const char *buf)518{519 struct ipoib_dev_priv *priv = ipoib_priv(dev);520 521 if ((test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags) &&522 !strcmp(buf, "connected\n")) ||523 (!test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags) &&524 !strcmp(buf, "datagram\n"))) {525 return 0;526 }527 528 /* flush paths if we switch modes so that connections are restarted */529 if (IPOIB_CM_SUPPORTED(dev->dev_addr) && !strcmp(buf, "connected\n")) {530 set_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags);531 ipoib_warn(priv, "enabling connected mode "532 "will cause multicast packet drops\n");533 netdev_update_features(dev);534 dev_set_mtu(dev, ipoib_cm_max_mtu(dev));535 netif_set_real_num_tx_queues(dev, 1);536 rtnl_unlock();537 priv->tx_wr.wr.send_flags &= ~IB_SEND_IP_CSUM;538 539 ipoib_flush_paths(dev);540 return (!rtnl_trylock()) ? -EBUSY : 0;541 }542 543 if (!strcmp(buf, "datagram\n")) {544 clear_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags);545 netdev_update_features(dev);546 dev_set_mtu(dev, min(priv->mcast_mtu, dev->mtu));547 netif_set_real_num_tx_queues(dev, dev->num_tx_queues);548 rtnl_unlock();549 ipoib_flush_paths(dev);550 return (!rtnl_trylock()) ? -EBUSY : 0;551 }552 553 return -EINVAL;554}555 556struct ipoib_path *__path_find(struct net_device *dev, void *gid)557{558 struct ipoib_dev_priv *priv = ipoib_priv(dev);559 struct rb_node *n = priv->path_tree.rb_node;560 struct ipoib_path *path;561 int ret;562 563 while (n) {564 path = rb_entry(n, struct ipoib_path, rb_node);565 566 ret = memcmp(gid, path->pathrec.dgid.raw,567 sizeof (union ib_gid));568 569 if (ret < 0)570 n = n->rb_left;571 else if (ret > 0)572 n = n->rb_right;573 else574 return path;575 }576 577 return NULL;578}579 580static int __path_add(struct net_device *dev, struct ipoib_path *path)581{582 struct ipoib_dev_priv *priv = ipoib_priv(dev);583 struct rb_node **n = &priv->path_tree.rb_node;584 struct rb_node *pn = NULL;585 struct ipoib_path *tpath;586 int ret;587 588 while (*n) {589 pn = *n;590 tpath = rb_entry(pn, struct ipoib_path, rb_node);591 592 ret = memcmp(path->pathrec.dgid.raw, tpath->pathrec.dgid.raw,593 sizeof (union ib_gid));594 if (ret < 0)595 n = &pn->rb_left;596 else if (ret > 0)597 n = &pn->rb_right;598 else599 return -EEXIST;600 }601 602 rb_link_node(&path->rb_node, pn, n);603 rb_insert_color(&path->rb_node, &priv->path_tree);604 605 list_add_tail(&path->list, &priv->path_list);606 607 return 0;608}609 610static void path_free(struct net_device *dev, struct ipoib_path *path)611{612 struct sk_buff *skb;613 614 while ((skb = __skb_dequeue(&path->queue)))615 dev_kfree_skb_irq(skb);616 617 ipoib_dbg(ipoib_priv(dev), "%s\n", __func__);618 619 /* remove all neigh connected to this path */620 ipoib_del_neighs_by_gid(dev, path->pathrec.dgid.raw);621 622 if (path->ah)623 ipoib_put_ah(path->ah);624 625 kfree(path);626}627 628#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG629 630struct ipoib_path_iter *ipoib_path_iter_init(struct net_device *dev)631{632 struct ipoib_path_iter *iter;633 634 iter = kmalloc(sizeof(*iter), GFP_KERNEL);635 if (!iter)636 return NULL;637 638 iter->dev = dev;639 memset(iter->path.pathrec.dgid.raw, 0, 16);640 641 if (ipoib_path_iter_next(iter)) {642 kfree(iter);643 return NULL;644 }645 646 return iter;647}648 649int ipoib_path_iter_next(struct ipoib_path_iter *iter)650{651 struct ipoib_dev_priv *priv = ipoib_priv(iter->dev);652 struct rb_node *n;653 struct ipoib_path *path;654 int ret = 1;655 656 spin_lock_irq(&priv->lock);657 658 n = rb_first(&priv->path_tree);659 660 while (n) {661 path = rb_entry(n, struct ipoib_path, rb_node);662 663 if (memcmp(iter->path.pathrec.dgid.raw, path->pathrec.dgid.raw,664 sizeof (union ib_gid)) < 0) {665 iter->path = *path;666 ret = 0;667 break;668 }669 670 n = rb_next(n);671 }672 673 spin_unlock_irq(&priv->lock);674 675 return ret;676}677 678void ipoib_path_iter_read(struct ipoib_path_iter *iter,679 struct ipoib_path *path)680{681 *path = iter->path;682}683 684#endif /* CONFIG_INFINIBAND_IPOIB_DEBUG */685 686void ipoib_mark_paths_invalid(struct net_device *dev)687{688 struct ipoib_dev_priv *priv = ipoib_priv(dev);689 struct ipoib_path *path, *tp;690 691 spin_lock_irq(&priv->lock);692 693 list_for_each_entry_safe(path, tp, &priv->path_list, list) {694 ipoib_dbg(priv, "mark path LID 0x%08x GID %pI6 invalid\n",695 be32_to_cpu(sa_path_get_dlid(&path->pathrec)),696 path->pathrec.dgid.raw);697 if (path->ah)698 path->ah->valid = 0;699 }700 701 spin_unlock_irq(&priv->lock);702}703 704static void push_pseudo_header(struct sk_buff *skb, const char *daddr)705{706 struct ipoib_pseudo_header *phdr;707 708 phdr = skb_push(skb, sizeof(*phdr));709 memcpy(phdr->hwaddr, daddr, INFINIBAND_ALEN);710}711 712void ipoib_flush_paths(struct net_device *dev)713{714 struct ipoib_dev_priv *priv = ipoib_priv(dev);715 struct ipoib_path *path, *tp;716 LIST_HEAD(remove_list);717 unsigned long flags;718 719 netif_tx_lock_bh(dev);720 spin_lock_irqsave(&priv->lock, flags);721 722 list_splice_init(&priv->path_list, &remove_list);723 724 list_for_each_entry(path, &remove_list, list)725 rb_erase(&path->rb_node, &priv->path_tree);726 727 list_for_each_entry_safe(path, tp, &remove_list, list) {728 if (path->query)729 ib_sa_cancel_query(path->query_id, path->query);730 spin_unlock_irqrestore(&priv->lock, flags);731 netif_tx_unlock_bh(dev);732 wait_for_completion(&path->done);733 path_free(dev, path);734 netif_tx_lock_bh(dev);735 spin_lock_irqsave(&priv->lock, flags);736 }737 738 spin_unlock_irqrestore(&priv->lock, flags);739 netif_tx_unlock_bh(dev);740}741 742static void path_rec_completion(int status,743 struct sa_path_rec *pathrec,744 unsigned int num_prs, void *path_ptr)745{746 struct ipoib_path *path = path_ptr;747 struct net_device *dev = path->dev;748 struct ipoib_dev_priv *priv = ipoib_priv(dev);749 struct ipoib_ah *ah = NULL;750 struct ipoib_ah *old_ah = NULL;751 struct ipoib_neigh *neigh, *tn;752 struct sk_buff_head skqueue;753 struct sk_buff *skb;754 unsigned long flags;755 756 if (!status)757 ipoib_dbg(priv, "PathRec LID 0x%04x for GID %pI6\n",758 be32_to_cpu(sa_path_get_dlid(pathrec)),759 pathrec->dgid.raw);760 else761 ipoib_dbg(priv, "PathRec status %d for GID %pI6\n",762 status, path->pathrec.dgid.raw);763 764 skb_queue_head_init(&skqueue);765 766 if (!status) {767 struct rdma_ah_attr av;768 769 if (!ib_init_ah_attr_from_path(priv->ca, priv->port,770 pathrec, &av, NULL)) {771 ah = ipoib_create_ah(dev, priv->pd, &av);772 rdma_destroy_ah_attr(&av);773 }774 }775 776 spin_lock_irqsave(&priv->lock, flags);777 778 if (!IS_ERR_OR_NULL(ah)) {779 /*780 * pathrec.dgid is used as the database key from the LLADDR,781 * it must remain unchanged even if the SA returns a different782 * GID to use in the AH.783 */784 if (memcmp(pathrec->dgid.raw, path->pathrec.dgid.raw,785 sizeof(union ib_gid))) {786 ipoib_dbg(787 priv,788 "%s got PathRec for gid %pI6 while asked for %pI6\n",789 dev->name, pathrec->dgid.raw,790 path->pathrec.dgid.raw);791 memcpy(pathrec->dgid.raw, path->pathrec.dgid.raw,792 sizeof(union ib_gid));793 }794 795 path->pathrec = *pathrec;796 797 old_ah = path->ah;798 path->ah = ah;799 800 ipoib_dbg(priv, "created address handle %p for LID 0x%04x, SL %d\n",801 ah, be32_to_cpu(sa_path_get_dlid(pathrec)),802 pathrec->sl);803 804 while ((skb = __skb_dequeue(&path->queue)))805 __skb_queue_tail(&skqueue, skb);806 807 list_for_each_entry_safe(neigh, tn, &path->neigh_list, list) {808 if (neigh->ah) {809 WARN_ON(neigh->ah != old_ah);810 /*811 * Dropping the ah reference inside812 * priv->lock is safe here, because we813 * will hold one more reference from814 * the original value of path->ah (ie815 * old_ah).816 */817 ipoib_put_ah(neigh->ah);818 }819 kref_get(&path->ah->ref);820 neigh->ah = path->ah;821 822 if (ipoib_cm_enabled(dev, neigh->daddr)) {823 if (!ipoib_cm_get(neigh))824 ipoib_cm_set(neigh, ipoib_cm_create_tx(dev,825 path,826 neigh));827 if (!ipoib_cm_get(neigh)) {828 ipoib_neigh_free(neigh);829 continue;830 }831 }832 833 while ((skb = __skb_dequeue(&neigh->queue)))834 __skb_queue_tail(&skqueue, skb);835 }836 path->ah->valid = 1;837 }838 839 path->query = NULL;840 complete(&path->done);841 842 spin_unlock_irqrestore(&priv->lock, flags);843 844 if (IS_ERR_OR_NULL(ah))845 ipoib_del_neighs_by_gid(dev, path->pathrec.dgid.raw);846 847 if (old_ah)848 ipoib_put_ah(old_ah);849 850 while ((skb = __skb_dequeue(&skqueue))) {851 int ret;852 skb->dev = dev;853 ret = dev_queue_xmit(skb);854 if (ret)855 ipoib_warn(priv, "%s: dev_queue_xmit failed to re-queue packet, ret:%d\n",856 __func__, ret);857 }858}859 860static void init_path_rec(struct ipoib_dev_priv *priv, struct ipoib_path *path,861 void *gid)862{863 path->dev = priv->dev;864 865 if (rdma_cap_opa_ah(priv->ca, priv->port))866 path->pathrec.rec_type = SA_PATH_REC_TYPE_OPA;867 else868 path->pathrec.rec_type = SA_PATH_REC_TYPE_IB;869 870 memcpy(path->pathrec.dgid.raw, gid, sizeof(union ib_gid));871 path->pathrec.sgid = priv->local_gid;872 path->pathrec.pkey = cpu_to_be16(priv->pkey);873 path->pathrec.numb_path = 1;874 path->pathrec.traffic_class = priv->broadcast->mcmember.traffic_class;875}876 877static struct ipoib_path *path_rec_create(struct net_device *dev, void *gid)878{879 struct ipoib_dev_priv *priv = ipoib_priv(dev);880 struct ipoib_path *path;881 882 if (!priv->broadcast)883 return NULL;884 885 path = kzalloc(sizeof(*path), GFP_ATOMIC);886 if (!path)887 return NULL;888 889 skb_queue_head_init(&path->queue);890 891 INIT_LIST_HEAD(&path->neigh_list);892 893 init_path_rec(priv, path, gid);894 895 return path;896}897 898static int path_rec_start(struct net_device *dev,899 struct ipoib_path *path)900{901 struct ipoib_dev_priv *priv = ipoib_priv(dev);902 903 ipoib_dbg(priv, "Start path record lookup for %pI6\n",904 path->pathrec.dgid.raw);905 906 init_completion(&path->done);907 908 path->query_id =909 ib_sa_path_rec_get(&ipoib_sa_client, priv->ca, priv->port,910 &path->pathrec,911 IB_SA_PATH_REC_DGID |912 IB_SA_PATH_REC_SGID |913 IB_SA_PATH_REC_NUMB_PATH |914 IB_SA_PATH_REC_TRAFFIC_CLASS |915 IB_SA_PATH_REC_PKEY,916 1000, GFP_ATOMIC,917 path_rec_completion,918 path, &path->query);919 if (path->query_id < 0) {920 ipoib_warn(priv, "ib_sa_path_rec_get failed: %d\n", path->query_id);921 path->query = NULL;922 complete(&path->done);923 return path->query_id;924 }925 926 return 0;927}928 929static void neigh_refresh_path(struct ipoib_neigh *neigh, u8 *daddr,930 struct net_device *dev)931{932 struct ipoib_dev_priv *priv = ipoib_priv(dev);933 struct ipoib_path *path;934 unsigned long flags;935 936 spin_lock_irqsave(&priv->lock, flags);937 938 path = __path_find(dev, daddr + 4);939 if (!path)940 goto out;941 if (!path->query)942 path_rec_start(dev, path);943out:944 spin_unlock_irqrestore(&priv->lock, flags);945}946 947static struct ipoib_neigh *neigh_add_path(struct sk_buff *skb, u8 *daddr,948 struct net_device *dev)949{950 struct ipoib_dev_priv *priv = ipoib_priv(dev);951 struct rdma_netdev *rn = netdev_priv(dev);952 struct ipoib_path *path;953 struct ipoib_neigh *neigh;954 unsigned long flags;955 956 spin_lock_irqsave(&priv->lock, flags);957 neigh = ipoib_neigh_alloc(daddr, dev);958 if (!neigh) {959 spin_unlock_irqrestore(&priv->lock, flags);960 ++dev->stats.tx_dropped;961 dev_kfree_skb_any(skb);962 return NULL;963 }964 965 /* To avoid race condition, make sure that the966 * neigh will be added only once.967 */968 if (unlikely(!list_empty(&neigh->list))) {969 spin_unlock_irqrestore(&priv->lock, flags);970 return neigh;971 }972 973 path = __path_find(dev, daddr + 4);974 if (!path) {975 path = path_rec_create(dev, daddr + 4);976 if (!path)977 goto err_path;978 979 __path_add(dev, path);980 }981 982 list_add_tail(&neigh->list, &path->neigh_list);983 984 if (path->ah && path->ah->valid) {985 kref_get(&path->ah->ref);986 neigh->ah = path->ah;987 988 if (ipoib_cm_enabled(dev, neigh->daddr)) {989 if (!ipoib_cm_get(neigh))990 ipoib_cm_set(neigh, ipoib_cm_create_tx(dev, path, neigh));991 if (!ipoib_cm_get(neigh)) {992 ipoib_neigh_free(neigh);993 goto err_drop;994 }995 if (skb_queue_len(&neigh->queue) <996 IPOIB_MAX_PATH_REC_QUEUE) {997 push_pseudo_header(skb, neigh->daddr);998 __skb_queue_tail(&neigh->queue, skb);999 } else {1000 ipoib_warn(priv, "queue length limit %d. Packet drop.\n",1001 skb_queue_len(&neigh->queue));1002 goto err_drop;1003 }1004 } else {1005 spin_unlock_irqrestore(&priv->lock, flags);1006 path->ah->last_send = rn->send(dev, skb, path->ah->ah,1007 IPOIB_QPN(daddr));1008 ipoib_neigh_put(neigh);1009 return NULL;1010 }1011 } else {1012 neigh->ah = NULL;1013 1014 if (!path->query && path_rec_start(dev, path))1015 goto err_path;1016 if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) {1017 push_pseudo_header(skb, neigh->daddr);1018 __skb_queue_tail(&neigh->queue, skb);1019 } else {1020 goto err_drop;1021 }1022 }1023 1024 spin_unlock_irqrestore(&priv->lock, flags);1025 ipoib_neigh_put(neigh);1026 return NULL;1027 1028err_path:1029 ipoib_neigh_free(neigh);1030err_drop:1031 ++dev->stats.tx_dropped;1032 dev_kfree_skb_any(skb);1033 1034 spin_unlock_irqrestore(&priv->lock, flags);1035 ipoib_neigh_put(neigh);1036 1037 return NULL;1038}1039 1040static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev,1041 struct ipoib_pseudo_header *phdr)1042{1043 struct ipoib_dev_priv *priv = ipoib_priv(dev);1044 struct rdma_netdev *rn = netdev_priv(dev);1045 struct ipoib_path *path;1046 unsigned long flags;1047 1048 spin_lock_irqsave(&priv->lock, flags);1049 1050 /* no broadcast means that all paths are (going to be) not valid */1051 if (!priv->broadcast)1052 goto drop_and_unlock;1053 1054 path = __path_find(dev, phdr->hwaddr + 4);1055 if (!path || !path->ah || !path->ah->valid) {1056 if (!path) {1057 path = path_rec_create(dev, phdr->hwaddr + 4);1058 if (!path)1059 goto drop_and_unlock;1060 __path_add(dev, path);1061 } else {1062 /*1063 * make sure there are no changes in the existing1064 * path record1065 */1066 init_path_rec(priv, path, phdr->hwaddr + 4);1067 }1068 if (!path->query && path_rec_start(dev, path)) {1069 goto drop_and_unlock;1070 }1071 1072 if (skb_queue_len(&path->queue) < IPOIB_MAX_PATH_REC_QUEUE) {1073 push_pseudo_header(skb, phdr->hwaddr);1074 __skb_queue_tail(&path->queue, skb);1075 goto unlock;1076 } else {1077 goto drop_and_unlock;1078 }1079 }1080 1081 spin_unlock_irqrestore(&priv->lock, flags);1082 ipoib_dbg(priv, "Send unicast ARP to %08x\n",1083 be32_to_cpu(sa_path_get_dlid(&path->pathrec)));1084 path->ah->last_send = rn->send(dev, skb, path->ah->ah,1085 IPOIB_QPN(phdr->hwaddr));1086 return;1087 1088drop_and_unlock:1089 ++dev->stats.tx_dropped;1090 dev_kfree_skb_any(skb);1091unlock:1092 spin_unlock_irqrestore(&priv->lock, flags);1093}1094 1095static netdev_tx_t ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev)1096{1097 struct ipoib_dev_priv *priv = ipoib_priv(dev);1098 struct rdma_netdev *rn = netdev_priv(dev);1099 struct ipoib_neigh *neigh;1100 struct ipoib_pseudo_header *phdr;1101 struct ipoib_header *header;1102 unsigned long flags;1103 1104 phdr = (struct ipoib_pseudo_header *) skb->data;1105 skb_pull(skb, sizeof(*phdr));1106 header = (struct ipoib_header *) skb->data;1107 1108 if (unlikely(phdr->hwaddr[4] == 0xff)) {1109 /* multicast, arrange "if" according to probability */1110 if ((header->proto != htons(ETH_P_IP)) &&1111 (header->proto != htons(ETH_P_IPV6)) &&1112 (header->proto != htons(ETH_P_ARP)) &&1113 (header->proto != htons(ETH_P_RARP)) &&1114 (header->proto != htons(ETH_P_TIPC))) {1115 /* ethertype not supported by IPoIB */1116 ++dev->stats.tx_dropped;1117 dev_kfree_skb_any(skb);1118 return NETDEV_TX_OK;1119 }1120 /* Add in the P_Key for multicast*/1121 phdr->hwaddr[8] = (priv->pkey >> 8) & 0xff;1122 phdr->hwaddr[9] = priv->pkey & 0xff;1123 1124 neigh = ipoib_neigh_get(dev, phdr->hwaddr);1125 if (likely(neigh))1126 goto send_using_neigh;1127 ipoib_mcast_send(dev, phdr->hwaddr, skb);1128 return NETDEV_TX_OK;1129 }1130 1131 /* unicast, arrange "switch" according to probability */1132 switch (header->proto) {1133 case htons(ETH_P_IP):1134 case htons(ETH_P_IPV6):1135 case htons(ETH_P_TIPC):1136 neigh = ipoib_neigh_get(dev, phdr->hwaddr);1137 if (unlikely(!neigh)) {1138 neigh = neigh_add_path(skb, phdr->hwaddr, dev);1139 if (likely(!neigh))1140 return NETDEV_TX_OK;1141 }1142 break;1143 case htons(ETH_P_ARP):1144 case htons(ETH_P_RARP):1145 /* for unicast ARP and RARP should always perform path find */1146 unicast_arp_send(skb, dev, phdr);1147 return NETDEV_TX_OK;1148 default:1149 /* ethertype not supported by IPoIB */1150 ++dev->stats.tx_dropped;1151 dev_kfree_skb_any(skb);1152 return NETDEV_TX_OK;1153 }1154 1155send_using_neigh:1156 /* note we now hold a ref to neigh */1157 if (ipoib_cm_get(neigh)) {1158 if (ipoib_cm_up(neigh)) {1159 ipoib_cm_send(dev, skb, ipoib_cm_get(neigh));1160 goto unref;1161 }1162 } else if (neigh->ah && neigh->ah->valid) {1163 neigh->ah->last_send = rn->send(dev, skb, neigh->ah->ah,1164 IPOIB_QPN(phdr->hwaddr));1165 goto unref;1166 } else if (neigh->ah) {1167 neigh_refresh_path(neigh, phdr->hwaddr, dev);1168 }1169 1170 if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) {1171 push_pseudo_header(skb, phdr->hwaddr);1172 spin_lock_irqsave(&priv->lock, flags);1173 __skb_queue_tail(&neigh->queue, skb);1174 spin_unlock_irqrestore(&priv->lock, flags);1175 } else {1176 ++dev->stats.tx_dropped;1177 dev_kfree_skb_any(skb);1178 }1179 1180unref:1181 ipoib_neigh_put(neigh);1182 1183 return NETDEV_TX_OK;1184}1185 1186static void ipoib_timeout(struct net_device *dev, unsigned int txqueue)1187{1188 struct ipoib_dev_priv *priv = ipoib_priv(dev);1189 struct rdma_netdev *rn = netdev_priv(dev);1190 1191 if (rn->tx_timeout) {1192 rn->tx_timeout(dev, txqueue);1193 return;1194 }1195 ipoib_warn(priv, "transmit timeout: latency %d msecs\n",1196 jiffies_to_msecs(jiffies - dev_trans_start(dev)));1197 ipoib_warn(priv,1198 "queue stopped %d, tx_head %u, tx_tail %u, global_tx_head %u, global_tx_tail %u\n",1199 netif_queue_stopped(dev), priv->tx_head, priv->tx_tail,1200 priv->global_tx_head, priv->global_tx_tail);1201 1202 1203 schedule_work(&priv->tx_timeout_work);1204}1205 1206void ipoib_ib_tx_timeout_work(struct work_struct *work)1207{1208 struct ipoib_dev_priv *priv = container_of(work,1209 struct ipoib_dev_priv,1210 tx_timeout_work);1211 int err;1212 1213 rtnl_lock();1214 1215 if (!test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags))1216 goto unlock;1217 1218 ipoib_stop(priv->dev);1219 err = ipoib_open(priv->dev);1220 if (err) {1221 ipoib_warn(priv, "ipoib_open failed recovering from a tx_timeout, err(%d).\n",1222 err);1223 goto unlock;1224 }1225 1226 netif_tx_wake_all_queues(priv->dev);1227unlock:1228 rtnl_unlock();1229 1230}1231 1232static int ipoib_hard_header(struct sk_buff *skb,1233 struct net_device *dev,1234 unsigned short type,1235 const void *daddr,1236 const void *saddr,1237 unsigned int len)1238{1239 struct ipoib_header *header;1240 1241 header = skb_push(skb, sizeof(*header));1242 1243 header->proto = htons(type);1244 header->reserved = 0;1245 1246 /*1247 * we don't rely on dst_entry structure, always stuff the1248 * destination address into skb hard header so we can figure out where1249 * to send the packet later.1250 */1251 push_pseudo_header(skb, daddr);1252 1253 return IPOIB_HARD_LEN;1254}1255 1256static void ipoib_set_mcast_list(struct net_device *dev)1257{1258 struct ipoib_dev_priv *priv = ipoib_priv(dev);1259 1260 if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)) {1261 ipoib_dbg(priv, "IPOIB_FLAG_OPER_UP not set");1262 return;1263 }1264 1265 queue_work(priv->wq, &priv->restart_task);1266}1267 1268static int ipoib_get_iflink(const struct net_device *dev)1269{1270 struct ipoib_dev_priv *priv = ipoib_priv(dev);1271 1272 /* parent interface */1273 if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags))1274 return READ_ONCE(dev->ifindex);1275 1276 /* child/vlan interface */1277 return READ_ONCE(priv->parent->ifindex);1278}1279 1280static u32 ipoib_addr_hash(struct ipoib_neigh_hash *htbl, u8 *daddr)1281{1282 /*1283 * Use only the address parts that contributes to spreading1284 * The subnet prefix is not used as one can not connect to1285 * same remote port (GUID) using the same remote QPN via two1286 * different subnets.1287 */1288 /* qpn octets[1:4) & port GUID octets[12:20) */1289 u32 *d32 = (u32 *) daddr;1290 u32 hv;1291 1292 hv = jhash_3words(d32[3], d32[4], IPOIB_QPN_MASK & d32[0], 0);1293 return hv & htbl->mask;1294}1295 1296struct ipoib_neigh *ipoib_neigh_get(struct net_device *dev, u8 *daddr)1297{1298 struct ipoib_dev_priv *priv = ipoib_priv(dev);1299 struct ipoib_neigh_table *ntbl = &priv->ntbl;1300 struct ipoib_neigh_hash *htbl;1301 struct ipoib_neigh *neigh = NULL;1302 u32 hash_val;1303 1304 rcu_read_lock_bh();1305 1306 htbl = rcu_dereference_bh(ntbl->htbl);1307 1308 if (!htbl)1309 goto out_unlock;1310 1311 hash_val = ipoib_addr_hash(htbl, daddr);1312 for (neigh = rcu_dereference_bh(htbl->buckets[hash_val]);1313 neigh != NULL;1314 neigh = rcu_dereference_bh(neigh->hnext)) {1315 if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) {1316 /* found, take one ref on behalf of the caller */1317 if (!refcount_inc_not_zero(&neigh->refcnt)) {1318 /* deleted */1319 neigh = NULL;1320 goto out_unlock;1321 }1322 1323 if (likely(skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE))1324 neigh->alive = jiffies;1325 goto out_unlock;1326 }1327 }1328 1329out_unlock:1330 rcu_read_unlock_bh();1331 return neigh;1332}1333 1334static void __ipoib_reap_neigh(struct ipoib_dev_priv *priv)1335{1336 struct ipoib_neigh_table *ntbl = &priv->ntbl;1337 struct ipoib_neigh_hash *htbl;1338 unsigned long neigh_obsolete;1339 unsigned long dt;1340 unsigned long flags;1341 int i;1342 LIST_HEAD(remove_list);1343 1344 spin_lock_irqsave(&priv->lock, flags);1345 1346 htbl = rcu_dereference_protected(ntbl->htbl,1347 lockdep_is_held(&priv->lock));1348 1349 if (!htbl)1350 goto out_unlock;1351 1352 /* neigh is obsolete if it was idle for two GC periods */1353 dt = 2 * arp_tbl.gc_interval;1354 neigh_obsolete = jiffies - dt;1355 1356 for (i = 0; i < htbl->size; i++) {1357 struct ipoib_neigh *neigh;1358 struct ipoib_neigh __rcu **np = &htbl->buckets[i];1359 1360 while ((neigh = rcu_dereference_protected(*np,1361 lockdep_is_held(&priv->lock))) != NULL) {1362 /* was the neigh idle for two GC periods */1363 if (time_after(neigh_obsolete, neigh->alive)) {1364 1365 ipoib_check_and_add_mcast_sendonly(priv, neigh->daddr + 4, &remove_list);1366 1367 rcu_assign_pointer(*np,1368 rcu_dereference_protected(neigh->hnext,1369 lockdep_is_held(&priv->lock)));1370 /* remove from path/mc list */1371 list_del_init(&neigh->list);1372 call_rcu(&neigh->rcu, ipoib_neigh_reclaim);1373 } else {1374 np = &neigh->hnext;1375 }1376 1377 }1378 }1379 1380out_unlock:1381 spin_unlock_irqrestore(&priv->lock, flags);1382 ipoib_mcast_remove_list(&remove_list);1383}1384 1385static void ipoib_reap_neigh(struct work_struct *work)1386{1387 struct ipoib_dev_priv *priv =1388 container_of(work, struct ipoib_dev_priv, neigh_reap_task.work);1389 1390 __ipoib_reap_neigh(priv);1391 1392 queue_delayed_work(priv->wq, &priv->neigh_reap_task,1393 arp_tbl.gc_interval);1394}1395 1396 1397static struct ipoib_neigh *ipoib_neigh_ctor(u8 *daddr,1398 struct net_device *dev)1399{1400 struct ipoib_neigh *neigh;1401 1402 neigh = kzalloc(sizeof(*neigh), GFP_ATOMIC);1403 if (!neigh)1404 return NULL;1405 1406 neigh->dev = dev;1407 memcpy(&neigh->daddr, daddr, sizeof(neigh->daddr));1408 skb_queue_head_init(&neigh->queue);1409 INIT_LIST_HEAD(&neigh->list);1410 ipoib_cm_set(neigh, NULL);1411 /* one ref on behalf of the caller */1412 refcount_set(&neigh->refcnt, 1);1413 1414 return neigh;1415}1416 1417struct ipoib_neigh *ipoib_neigh_alloc(u8 *daddr,1418 struct net_device *dev)1419{1420 struct ipoib_dev_priv *priv = ipoib_priv(dev);1421 struct ipoib_neigh_table *ntbl = &priv->ntbl;1422 struct ipoib_neigh_hash *htbl;1423 struct ipoib_neigh *neigh;1424 u32 hash_val;1425 1426 htbl = rcu_dereference_protected(ntbl->htbl,1427 lockdep_is_held(&priv->lock));1428 if (!htbl) {1429 neigh = NULL;1430 goto out_unlock;1431 }1432 1433 /* need to add a new neigh, but maybe some other thread succeeded?1434 * recalc hash, maybe hash resize took place so we do a search1435 */1436 hash_val = ipoib_addr_hash(htbl, daddr);1437 for (neigh = rcu_dereference_protected(htbl->buckets[hash_val],1438 lockdep_is_held(&priv->lock));1439 neigh != NULL;1440 neigh = rcu_dereference_protected(neigh->hnext,1441 lockdep_is_held(&priv->lock))) {1442 if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) {1443 /* found, take one ref on behalf of the caller */1444 if (!refcount_inc_not_zero(&neigh->refcnt)) {1445 /* deleted */1446 neigh = NULL;1447 break;1448 }1449 neigh->alive = jiffies;1450 goto out_unlock;1451 }1452 }1453 1454 neigh = ipoib_neigh_ctor(daddr, dev);1455 if (!neigh)1456 goto out_unlock;1457 1458 /* one ref on behalf of the hash table */1459 refcount_inc(&neigh->refcnt);1460 neigh->alive = jiffies;1461 /* put in hash */1462 rcu_assign_pointer(neigh->hnext,1463 rcu_dereference_protected(htbl->buckets[hash_val],1464 lockdep_is_held(&priv->lock)));1465 rcu_assign_pointer(htbl->buckets[hash_val], neigh);1466 atomic_inc(&ntbl->entries);1467 1468out_unlock:1469 1470 return neigh;1471}1472 1473void ipoib_neigh_dtor(struct ipoib_neigh *neigh)1474{1475 /* neigh reference count was dropprd to zero */1476 struct net_device *dev = neigh->dev;1477 struct ipoib_dev_priv *priv = ipoib_priv(dev);1478 struct sk_buff *skb;1479 if (neigh->ah)1480 ipoib_put_ah(neigh->ah);1481 while ((skb = __skb_dequeue(&neigh->queue))) {1482 ++dev->stats.tx_dropped;1483 dev_kfree_skb_any(skb);1484 }1485 if (ipoib_cm_get(neigh))1486 ipoib_cm_destroy_tx(ipoib_cm_get(neigh));1487 ipoib_dbg(ipoib_priv(dev),1488 "neigh free for %06x %pI6\n",1489 IPOIB_QPN(neigh->daddr),1490 neigh->daddr + 4);1491 kfree(neigh);1492 if (atomic_dec_and_test(&priv->ntbl.entries)) {1493 if (test_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags))1494 complete(&priv->ntbl.flushed);1495 }1496}1497 1498static void ipoib_neigh_reclaim(struct rcu_head *rp)1499{1500 /* Called as a result of removal from hash table */1501 struct ipoib_neigh *neigh = container_of(rp, struct ipoib_neigh, rcu);1502 /* note TX context may hold another ref */1503 ipoib_neigh_put(neigh);1504}1505 1506void ipoib_neigh_free(struct ipoib_neigh *neigh)1507{1508 struct net_device *dev = neigh->dev;1509 struct ipoib_dev_priv *priv = ipoib_priv(dev);1510 struct ipoib_neigh_table *ntbl = &priv->ntbl;1511 struct ipoib_neigh_hash *htbl;1512 struct ipoib_neigh __rcu **np;1513 struct ipoib_neigh *n;1514 u32 hash_val;1515 1516 htbl = rcu_dereference_protected(ntbl->htbl,1517 lockdep_is_held(&priv->lock));1518 if (!htbl)1519 return;1520 1521 hash_val = ipoib_addr_hash(htbl, neigh->daddr);1522 np = &htbl->buckets[hash_val];1523 for (n = rcu_dereference_protected(*np,1524 lockdep_is_held(&priv->lock));1525 n != NULL;1526 n = rcu_dereference_protected(*np,1527 lockdep_is_held(&priv->lock))) {1528 if (n == neigh) {1529 /* found */1530 rcu_assign_pointer(*np,1531 rcu_dereference_protected(neigh->hnext,1532 lockdep_is_held(&priv->lock)));1533 /* remove from parent list */1534 list_del_init(&neigh->list);1535 call_rcu(&neigh->rcu, ipoib_neigh_reclaim);1536 return;1537 } else {1538 np = &n->hnext;1539 }1540 }1541}1542 1543static int ipoib_neigh_hash_init(struct ipoib_dev_priv *priv)1544{1545 struct ipoib_neigh_table *ntbl = &priv->ntbl;1546 struct ipoib_neigh_hash *htbl;1547 struct ipoib_neigh __rcu **buckets;1548 u32 size;1549 1550 clear_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags);1551 ntbl->htbl = NULL;1552 htbl = kzalloc(sizeof(*htbl), GFP_KERNEL);1553 if (!htbl)1554 return -ENOMEM;1555 size = roundup_pow_of_two(arp_tbl.gc_thresh3);1556 buckets = kvcalloc(size, sizeof(*buckets), GFP_KERNEL);1557 if (!buckets) {1558 kfree(htbl);1559 return -ENOMEM;1560 }1561 htbl->size = size;1562 htbl->mask = (size - 1);1563 htbl->buckets = buckets;1564 RCU_INIT_POINTER(ntbl->htbl, htbl);1565 htbl->ntbl = ntbl;1566 atomic_set(&ntbl->entries, 0);1567 1568 /* start garbage collection */1569 queue_delayed_work(priv->wq, &priv->neigh_reap_task,1570 arp_tbl.gc_interval);1571 1572 return 0;1573}1574 1575static void neigh_hash_free_rcu(struct rcu_head *head)1576{1577 struct ipoib_neigh_hash *htbl = container_of(head,1578 struct ipoib_neigh_hash,1579 rcu);1580 struct ipoib_neigh __rcu **buckets = htbl->buckets;1581 struct ipoib_neigh_table *ntbl = htbl->ntbl;1582 1583 kvfree(buckets);1584 kfree(htbl);1585 complete(&ntbl->deleted);1586}1587 1588void ipoib_del_neighs_by_gid(struct net_device *dev, u8 *gid)1589{1590 struct ipoib_dev_priv *priv = ipoib_priv(dev);1591 struct ipoib_neigh_table *ntbl = &priv->ntbl;1592 struct ipoib_neigh_hash *htbl;1593 unsigned long flags;1594 int i;1595 1596 /* remove all neigh connected to a given path or mcast */1597 spin_lock_irqsave(&priv->lock, flags);1598 1599 htbl = rcu_dereference_protected(ntbl->htbl,1600 lockdep_is_held(&priv->lock));1601 1602 if (!htbl)1603 goto out_unlock;1604 1605 for (i = 0; i < htbl->size; i++) {1606 struct ipoib_neigh *neigh;1607 struct ipoib_neigh __rcu **np = &htbl->buckets[i];1608 1609 while ((neigh = rcu_dereference_protected(*np,1610 lockdep_is_held(&priv->lock))) != NULL) {1611 /* delete neighs belong to this parent */1612 if (!memcmp(gid, neigh->daddr + 4, sizeof (union ib_gid))) {1613 rcu_assign_pointer(*np,1614 rcu_dereference_protected(neigh->hnext,1615 lockdep_is_held(&priv->lock)));1616 /* remove from parent list */1617 list_del_init(&neigh->list);1618 call_rcu(&neigh->rcu, ipoib_neigh_reclaim);1619 } else {1620 np = &neigh->hnext;1621 }1622 1623 }1624 }1625out_unlock:1626 spin_unlock_irqrestore(&priv->lock, flags);1627}1628 1629static void ipoib_flush_neighs(struct ipoib_dev_priv *priv)1630{1631 struct ipoib_neigh_table *ntbl = &priv->ntbl;1632 struct ipoib_neigh_hash *htbl;1633 unsigned long flags;1634 int i, wait_flushed = 0;1635 1636 init_completion(&priv->ntbl.flushed);1637 set_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags);1638 1639 spin_lock_irqsave(&priv->lock, flags);1640 1641 htbl = rcu_dereference_protected(ntbl->htbl,1642 lockdep_is_held(&priv->lock));1643 if (!htbl)1644 goto out_unlock;1645 1646 wait_flushed = atomic_read(&priv->ntbl.entries);1647 if (!wait_flushed)1648 goto free_htbl;1649 1650 for (i = 0; i < htbl->size; i++) {1651 struct ipoib_neigh *neigh;1652 struct ipoib_neigh __rcu **np = &htbl->buckets[i];1653 1654 while ((neigh = rcu_dereference_protected(*np,1655 lockdep_is_held(&priv->lock))) != NULL) {1656 rcu_assign_pointer(*np,1657 rcu_dereference_protected(neigh->hnext,1658 lockdep_is_held(&priv->lock)));1659 /* remove from path/mc list */1660 list_del_init(&neigh->list);1661 call_rcu(&neigh->rcu, ipoib_neigh_reclaim);1662 }1663 }1664 1665free_htbl:1666 rcu_assign_pointer(ntbl->htbl, NULL);1667 call_rcu(&htbl->rcu, neigh_hash_free_rcu);1668 1669out_unlock:1670 spin_unlock_irqrestore(&priv->lock, flags);1671 if (wait_flushed)1672 wait_for_completion(&priv->ntbl.flushed);1673}1674 1675static void ipoib_neigh_hash_uninit(struct net_device *dev)1676{1677 struct ipoib_dev_priv *priv = ipoib_priv(dev);1678 1679 ipoib_dbg(priv, "%s\n", __func__);1680 init_completion(&priv->ntbl.deleted);1681 1682 cancel_delayed_work_sync(&priv->neigh_reap_task);1683 1684 ipoib_flush_neighs(priv);1685 1686 wait_for_completion(&priv->ntbl.deleted);1687}1688 1689static void ipoib_napi_add(struct net_device *dev)1690{1691 struct ipoib_dev_priv *priv = ipoib_priv(dev);1692 1693 netif_napi_add_weight(dev, &priv->recv_napi, ipoib_rx_poll,1694 IPOIB_NUM_WC);1695 netif_napi_add_weight(dev, &priv->send_napi, ipoib_tx_poll,1696 MAX_SEND_CQE);1697}1698 1699static void ipoib_napi_del(struct net_device *dev)1700{1701 struct ipoib_dev_priv *priv = ipoib_priv(dev);1702 1703 netif_napi_del(&priv->recv_napi);1704 netif_napi_del(&priv->send_napi);1705}1706 1707static void ipoib_dev_uninit_default(struct net_device *dev)1708{1709 struct ipoib_dev_priv *priv = ipoib_priv(dev);1710 1711 ipoib_transport_dev_cleanup(dev);1712 1713 ipoib_napi_del(dev);1714 1715 ipoib_cm_dev_cleanup(dev);1716 1717 kfree(priv->rx_ring);1718 vfree(priv->tx_ring);1719 1720 priv->rx_ring = NULL;1721 priv->tx_ring = NULL;1722}1723 1724static int ipoib_dev_init_default(struct net_device *dev)1725{1726 struct ipoib_dev_priv *priv = ipoib_priv(dev);1727 u8 addr_mod[3];1728 1729 ipoib_napi_add(dev);1730 1731 /* Allocate RX/TX "rings" to hold queued skbs */1732 priv->rx_ring = kcalloc(ipoib_recvq_size,1733 sizeof(*priv->rx_ring),1734 GFP_KERNEL);1735 if (!priv->rx_ring)1736 goto out;1737 1738 priv->tx_ring = vzalloc(array_size(ipoib_sendq_size,1739 sizeof(*priv->tx_ring)));1740 if (!priv->tx_ring) {1741 pr_warn("%s: failed to allocate TX ring (%d entries)\n",1742 priv->ca->name, ipoib_sendq_size);1743 goto out_rx_ring_cleanup;1744 }1745 1746 /* priv->tx_head, tx_tail and global_tx_tail/head are already 0 */1747 1748 if (ipoib_transport_dev_init(dev, priv->ca)) {1749 pr_warn("%s: ipoib_transport_dev_init failed\n",1750 priv->ca->name);1751 goto out_tx_ring_cleanup;1752 }1753 1754 /* after qp created set dev address */1755 addr_mod[0] = (priv->qp->qp_num >> 16) & 0xff;1756 addr_mod[1] = (priv->qp->qp_num >> 8) & 0xff;1757 addr_mod[2] = (priv->qp->qp_num) & 0xff;1758 dev_addr_mod(priv->dev, 1, addr_mod, sizeof(addr_mod));1759 1760 return 0;1761 1762out_tx_ring_cleanup:1763 vfree(priv->tx_ring);1764 1765out_rx_ring_cleanup:1766 kfree(priv->rx_ring);1767 1768out:1769 ipoib_napi_del(dev);1770 return -ENOMEM;1771}1772 1773static int ipoib_ioctl(struct net_device *dev, struct ifreq *ifr,1774 int cmd)1775{1776 struct ipoib_dev_priv *priv = ipoib_priv(dev);1777 1778 if (!priv->rn_ops->ndo_eth_ioctl)1779 return -EOPNOTSUPP;1780 1781 return priv->rn_ops->ndo_eth_ioctl(dev, ifr, cmd);1782}1783 1784static int ipoib_dev_init(struct net_device *dev)1785{1786 struct ipoib_dev_priv *priv = ipoib_priv(dev);1787 int ret = -ENOMEM;1788 1789 priv->qp = NULL;1790 1791 /*1792 * the various IPoIB tasks assume they will never race against1793 * themselves, so always use a single thread workqueue1794 */1795 priv->wq = alloc_ordered_workqueue("ipoib_wq", WQ_MEM_RECLAIM);1796 if (!priv->wq) {1797 pr_warn("%s: failed to allocate device WQ\n", dev->name);1798 goto out;1799 }1800 1801 /* create pd, which used both for control and datapath*/1802 priv->pd = ib_alloc_pd(priv->ca, 0);1803 if (IS_ERR(priv->pd)) {1804 pr_warn("%s: failed to allocate PD\n", priv->ca->name);1805 goto clean_wq;1806 }1807 1808 ret = priv->rn_ops->ndo_init(dev);1809 if (ret) {1810 pr_warn("%s failed to init HW resource\n", dev->name);1811 goto out_free_pd;1812 }1813 1814 ret = ipoib_neigh_hash_init(priv);1815 if (ret) {1816 pr_warn("%s failed to init neigh hash\n", dev->name);1817 goto out_dev_uninit;1818 }1819 1820 if (dev->flags & IFF_UP) {1821 if (ipoib_ib_dev_open(dev)) {1822 pr_warn("%s failed to open device\n", dev->name);1823 ret = -ENODEV;1824 goto out_hash_uninit;1825 }1826 }1827 1828 return 0;1829 1830out_hash_uninit:1831 ipoib_neigh_hash_uninit(dev);1832 1833out_dev_uninit:1834 ipoib_ib_dev_cleanup(dev);1835 1836out_free_pd:1837 if (priv->pd) {1838 ib_dealloc_pd(priv->pd);1839 priv->pd = NULL;1840 }1841 1842clean_wq:1843 if (priv->wq) {1844 destroy_workqueue(priv->wq);1845 priv->wq = NULL;1846 }1847 1848out:1849 return ret;1850}1851 1852/*1853 * This must be called before doing an unregister_netdev on a parent device to1854 * shutdown the IB event handler.1855 */1856static void ipoib_parent_unregister_pre(struct net_device *ndev)1857{1858 struct ipoib_dev_priv *priv = ipoib_priv(ndev);1859 1860 /*1861 * ipoib_set_mac checks netif_running before pushing work, clearing1862 * running ensures the it will not add more work.1863 */1864 rtnl_lock();1865 dev_change_flags(priv->dev, priv->dev->flags & ~IFF_UP, NULL);1866 rtnl_unlock();1867 1868 /* ipoib_event() cannot be running once this returns */1869 ib_unregister_event_handler(&priv->event_handler);1870 1871 /*1872 * Work on the queue grabs the rtnl lock, so this cannot be done while1873 * also holding it.1874 */1875 flush_workqueue(ipoib_workqueue);1876}1877 1878static void ipoib_set_dev_features(struct ipoib_dev_priv *priv)1879{1880 priv->hca_caps = priv->ca->attrs.device_cap_flags;1881 priv->kernel_caps = priv->ca->attrs.kernel_cap_flags;1882 1883 if (priv->hca_caps & IB_DEVICE_UD_IP_CSUM) {1884 priv->dev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_RXCSUM;1885 1886 if (priv->kernel_caps & IBK_UD_TSO)1887 priv->dev->hw_features |= NETIF_F_TSO;1888 1889 priv->dev->features |= priv->dev->hw_features;1890 }1891}1892 1893static int ipoib_parent_init(struct net_device *ndev)1894{1895 struct ipoib_dev_priv *priv = ipoib_priv(ndev);1896 struct ib_port_attr attr;1897 int result;1898 1899 result = ib_query_port(priv->ca, priv->port, &attr);1900 if (result) {1901 pr_warn("%s: ib_query_port %d failed\n", priv->ca->name,1902 priv->port);1903 return result;1904 }1905 priv->max_ib_mtu = rdma_mtu_from_attr(priv->ca, priv->port, &attr);1906 1907 result = ib_query_pkey(priv->ca, priv->port, 0, &priv->pkey);1908 if (result) {1909 pr_warn("%s: ib_query_pkey port %d failed (ret = %d)\n",1910 priv->ca->name, priv->port, result);1911 return result;1912 }1913 1914 result = rdma_query_gid(priv->ca, priv->port, 0, &priv->local_gid);1915 if (result) {1916 pr_warn("%s: rdma_query_gid port %d failed (ret = %d)\n",1917 priv->ca->name, priv->port, result);1918 return result;1919 }1920 dev_addr_mod(priv->dev, 4, priv->local_gid.raw, sizeof(union ib_gid));1921 1922 SET_NETDEV_DEV(priv->dev, priv->ca->dev.parent);1923 priv->dev->dev_port = priv->port - 1;1924 /* Let's set this one too for backwards compatibility. */1925 priv->dev->dev_id = priv->port - 1;1926 1927 return 0;1928}1929 1930static void ipoib_child_init(struct net_device *ndev)1931{1932 struct ipoib_dev_priv *priv = ipoib_priv(ndev);1933 struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);1934 1935 priv->max_ib_mtu = ppriv->max_ib_mtu;1936 set_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags);1937 if (memchr_inv(priv->dev->dev_addr, 0, INFINIBAND_ALEN))1938 memcpy(&priv->local_gid, priv->dev->dev_addr + 4,1939 sizeof(priv->local_gid));1940 else {1941 __dev_addr_set(priv->dev, ppriv->dev->dev_addr,1942 INFINIBAND_ALEN);1943 memcpy(&priv->local_gid, &ppriv->local_gid,1944 sizeof(priv->local_gid));1945 }1946}1947 1948static int ipoib_ndo_init(struct net_device *ndev)1949{1950 struct ipoib_dev_priv *priv = ipoib_priv(ndev);1951 int rc;1952 struct rdma_netdev *rn = netdev_priv(ndev);1953 1954 if (priv->parent) {1955 ipoib_child_init(ndev);1956 } else {1957 rc = ipoib_parent_init(ndev);1958 if (rc)1959 return rc;1960 }1961 1962 /* MTU will be reset when mcast join happens */1963 ndev->mtu = IPOIB_UD_MTU(priv->max_ib_mtu);1964 priv->mcast_mtu = priv->admin_mtu = ndev->mtu;1965 rn->mtu = priv->mcast_mtu;1966 ndev->max_mtu = IPOIB_CM_MTU;1967 1968 ndev->neigh_priv_len = sizeof(struct ipoib_neigh);1969 1970 /*1971 * Set the full membership bit, so that we join the right1972 * broadcast group, etc.1973 */1974 priv->pkey |= 0x8000;1975 1976 ndev->broadcast[8] = priv->pkey >> 8;1977 ndev->broadcast[9] = priv->pkey & 0xff;1978 set_bit(IPOIB_FLAG_DEV_ADDR_SET, &priv->flags);1979 1980 ipoib_set_dev_features(priv);1981 1982 rc = ipoib_dev_init(ndev);1983 if (rc) {1984 pr_warn("%s: failed to initialize device: %s port %d (ret = %d)\n",1985 priv->ca->name, priv->dev->name, priv->port, rc);1986 return rc;1987 }1988 1989 if (priv->parent) {1990 struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);1991 1992 dev_hold(priv->parent);1993 1994 down_write(&ppriv->vlan_rwsem);1995 list_add_tail(&priv->list, &ppriv->child_intfs);1996 up_write(&ppriv->vlan_rwsem);1997 }1998 1999 return 0;2000}2001 2002static void ipoib_ndo_uninit(struct net_device *dev)2003{2004 struct ipoib_dev_priv *priv = ipoib_priv(dev);2005 2006 ASSERT_RTNL();2007 2008 /*2009 * ipoib_remove_one guarantees the children are removed before the2010 * parent, and that is the only place where a parent can be removed.2011 */2012 WARN_ON(!list_empty(&priv->child_intfs));2013 2014 if (priv->parent) {2015 struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);2016 2017 down_write(&ppriv->vlan_rwsem);2018 list_del(&priv->list);2019 up_write(&ppriv->vlan_rwsem);2020 }2021 2022 ipoib_neigh_hash_uninit(dev);2023 2024 ipoib_ib_dev_cleanup(dev);2025 2026 /* no more works over the priv->wq */2027 if (priv->wq) {2028 /* See ipoib_mcast_carrier_on_task() */2029 WARN_ON(test_bit(IPOIB_FLAG_OPER_UP, &priv->flags));2030 destroy_workqueue(priv->wq);2031 priv->wq = NULL;2032 }2033 2034 dev_put(priv->parent);2035}2036 2037static int ipoib_set_vf_link_state(struct net_device *dev, int vf, int link_state)2038{2039 struct ipoib_dev_priv *priv = ipoib_priv(dev);2040 2041 return ib_set_vf_link_state(priv->ca, vf, priv->port, link_state);2042}2043 2044static int ipoib_get_vf_config(struct net_device *dev, int vf,2045 struct ifla_vf_info *ivf)2046{2047 struct ipoib_dev_priv *priv = ipoib_priv(dev);2048 int err;2049 2050 err = ib_get_vf_config(priv->ca, vf, priv->port, ivf);2051 if (err)2052 return err;2053 2054 ivf->vf = vf;2055 memcpy(ivf->mac, dev->dev_addr, dev->addr_len);2056 2057 return 0;2058}2059 2060static int ipoib_set_vf_guid(struct net_device *dev, int vf, u64 guid, int type)2061{2062 struct ipoib_dev_priv *priv = ipoib_priv(dev);2063 2064 if (type != IFLA_VF_IB_NODE_GUID && type != IFLA_VF_IB_PORT_GUID)2065 return -EINVAL;2066 2067 return ib_set_vf_guid(priv->ca, vf, priv->port, guid, type);2068}2069 2070static int ipoib_get_vf_guid(struct net_device *dev, int vf,2071 struct ifla_vf_guid *node_guid,2072 struct ifla_vf_guid *port_guid)2073{2074 struct ipoib_dev_priv *priv = ipoib_priv(dev);2075 2076 return ib_get_vf_guid(priv->ca, vf, priv->port, node_guid, port_guid);2077}2078 2079static int ipoib_get_vf_stats(struct net_device *dev, int vf,2080 struct ifla_vf_stats *vf_stats)2081{2082 struct ipoib_dev_priv *priv = ipoib_priv(dev);2083 2084 return ib_get_vf_stats(priv->ca, vf, priv->port, vf_stats);2085}2086 2087static const struct header_ops ipoib_header_ops = {2088 .create = ipoib_hard_header,2089};2090 2091static const struct net_device_ops ipoib_netdev_ops_pf = {2092 .ndo_init = ipoib_ndo_init,2093 .ndo_uninit = ipoib_ndo_uninit,2094 .ndo_open = ipoib_open,2095 .ndo_stop = ipoib_stop,2096 .ndo_change_mtu = ipoib_change_mtu,2097 .ndo_fix_features = ipoib_fix_features,2098 .ndo_start_xmit = ipoib_start_xmit,2099 .ndo_tx_timeout = ipoib_timeout,2100 .ndo_set_rx_mode = ipoib_set_mcast_list,2101 .ndo_get_iflink = ipoib_get_iflink,2102 .ndo_set_vf_link_state = ipoib_set_vf_link_state,2103 .ndo_get_vf_config = ipoib_get_vf_config,2104 .ndo_get_vf_stats = ipoib_get_vf_stats,2105 .ndo_get_vf_guid = ipoib_get_vf_guid,2106 .ndo_set_vf_guid = ipoib_set_vf_guid,2107 .ndo_set_mac_address = ipoib_set_mac,2108 .ndo_get_stats64 = ipoib_get_stats,2109 .ndo_eth_ioctl = ipoib_ioctl,2110};2111 2112static const struct net_device_ops ipoib_netdev_ops_vf = {2113 .ndo_init = ipoib_ndo_init,2114 .ndo_uninit = ipoib_ndo_uninit,2115 .ndo_open = ipoib_open,2116 .ndo_stop = ipoib_stop,2117 .ndo_change_mtu = ipoib_change_mtu,2118 .ndo_fix_features = ipoib_fix_features,2119 .ndo_start_xmit = ipoib_start_xmit,2120 .ndo_tx_timeout = ipoib_timeout,2121 .ndo_set_rx_mode = ipoib_set_mcast_list,2122 .ndo_get_iflink = ipoib_get_iflink,2123 .ndo_get_stats64 = ipoib_get_stats,2124 .ndo_eth_ioctl = ipoib_ioctl,2125};2126 2127static const struct net_device_ops ipoib_netdev_default_pf = {2128 .ndo_init = ipoib_dev_init_default,2129 .ndo_uninit = ipoib_dev_uninit_default,2130 .ndo_open = ipoib_ib_dev_open_default,2131 .ndo_stop = ipoib_ib_dev_stop_default,2132};2133 2134void ipoib_setup_common(struct net_device *dev)2135{2136 dev->header_ops = &ipoib_header_ops;2137 dev->netdev_ops = &ipoib_netdev_default_pf;2138 2139 ipoib_set_ethtool_ops(dev);2140 2141 dev->watchdog_timeo = 10 * HZ;2142 2143 dev->flags |= IFF_BROADCAST | IFF_MULTICAST;2144 2145 dev->hard_header_len = IPOIB_HARD_LEN;2146 dev->addr_len = INFINIBAND_ALEN;2147 dev->type = ARPHRD_INFINIBAND;2148 dev->tx_queue_len = ipoib_sendq_size * 2;2149 dev->features = (NETIF_F_VLAN_CHALLENGED |2150 NETIF_F_HIGHDMA);2151 netif_keep_dst(dev);2152 2153 memcpy(dev->broadcast, ipv4_bcast_addr, INFINIBAND_ALEN);2154 2155 /*2156 * unregister_netdev always frees the netdev, we use this mode2157 * consistently to unify all the various unregister paths, including2158 * those connected to rtnl_link_ops which require it.2159 */2160 dev->needs_free_netdev = true;2161}2162 2163static void ipoib_build_priv(struct net_device *dev)2164{2165 struct ipoib_dev_priv *priv = ipoib_priv(dev);2166 2167 priv->dev = dev;2168 spin_lock_init(&priv->lock);2169 init_rwsem(&priv->vlan_rwsem);2170 mutex_init(&priv->mcast_mutex);2171 2172 INIT_LIST_HEAD(&priv->path_list);2173 INIT_LIST_HEAD(&priv->child_intfs);2174 INIT_LIST_HEAD(&priv->dead_ahs);2175 INIT_LIST_HEAD(&priv->multicast_list);2176 2177 INIT_DELAYED_WORK(&priv->mcast_task, ipoib_mcast_join_task);2178 INIT_WORK(&priv->carrier_on_task, ipoib_mcast_carrier_on_task);2179 INIT_WORK(&priv->reschedule_napi_work, ipoib_napi_schedule_work);2180 INIT_WORK(&priv->flush_light, ipoib_ib_dev_flush_light);2181 INIT_WORK(&priv->flush_normal, ipoib_ib_dev_flush_normal);2182 INIT_WORK(&priv->flush_heavy, ipoib_ib_dev_flush_heavy);2183 INIT_WORK(&priv->restart_task, ipoib_mcast_restart_task);2184 INIT_WORK(&priv->tx_timeout_work, ipoib_ib_tx_timeout_work);2185 INIT_DELAYED_WORK(&priv->ah_reap_task, ipoib_reap_ah);2186 INIT_DELAYED_WORK(&priv->neigh_reap_task, ipoib_reap_neigh);2187}2188 2189static struct net_device *ipoib_alloc_netdev(struct ib_device *hca, u32 port,2190 const char *name)2191{2192 struct net_device *dev;2193 2194 dev = rdma_alloc_netdev(hca, port, RDMA_NETDEV_IPOIB, name,2195 NET_NAME_UNKNOWN, ipoib_setup_common);2196 if (!IS_ERR(dev) || PTR_ERR(dev) != -EOPNOTSUPP)2197 return dev;2198 2199 dev = alloc_netdev(sizeof(struct rdma_netdev), name, NET_NAME_UNKNOWN,2200 ipoib_setup_common);2201 if (!dev)2202 return ERR_PTR(-ENOMEM);2203 return dev;2204}2205 2206int ipoib_intf_init(struct ib_device *hca, u32 port, const char *name,2207 struct net_device *dev)2208{2209 struct rdma_netdev *rn = netdev_priv(dev);2210 struct ipoib_dev_priv *priv;2211 int rc;2212 2213 priv = kzalloc(sizeof(*priv), GFP_KERNEL);2214 if (!priv)2215 return -ENOMEM;2216 2217 priv->ca = hca;2218 priv->port = port;2219 2220 rc = rdma_init_netdev(hca, port, RDMA_NETDEV_IPOIB, name,2221 NET_NAME_UNKNOWN, ipoib_setup_common, dev);2222 if (rc) {2223 if (rc != -EOPNOTSUPP)2224 goto out;2225 2226 rn->send = ipoib_send;2227 rn->attach_mcast = ipoib_mcast_attach;2228 rn->detach_mcast = ipoib_mcast_detach;2229 rn->hca = hca;2230 2231 rc = netif_set_real_num_tx_queues(dev, 1);2232 if (rc)2233 goto out;2234 2235 rc = netif_set_real_num_rx_queues(dev, 1);2236 if (rc)2237 goto out;2238 }2239 2240 priv->rn_ops = dev->netdev_ops;2241 2242 if (hca->attrs.kernel_cap_flags & IBK_VIRTUAL_FUNCTION)2243 dev->netdev_ops = &ipoib_netdev_ops_vf;2244 else2245 dev->netdev_ops = &ipoib_netdev_ops_pf;2246 2247 rn->clnt_priv = priv;2248 /*2249 * Only the child register_netdev flows can handle priv_destructor2250 * being set, so we force it to NULL here and handle manually until it2251 * is safe to turn on.2252 */2253 priv->next_priv_destructor = dev->priv_destructor;2254 dev->priv_destructor = NULL;2255 2256 ipoib_build_priv(dev);2257 2258 return 0;2259 2260out:2261 kfree(priv);2262 return rc;2263}2264 2265struct net_device *ipoib_intf_alloc(struct ib_device *hca, u32 port,2266 const char *name)2267{2268 struct net_device *dev;2269 int rc;2270 2271 dev = ipoib_alloc_netdev(hca, port, name);2272 if (IS_ERR(dev))2273 return dev;2274 2275 rc = ipoib_intf_init(hca, port, name, dev);2276 if (rc) {2277 free_netdev(dev);2278 return ERR_PTR(rc);2279 }2280 2281 /*2282 * Upon success the caller must ensure ipoib_intf_free is called or2283 * register_netdevice succeed'd and priv_destructor is set to2284 * ipoib_intf_free.2285 */2286 return dev;2287}2288 2289void ipoib_intf_free(struct net_device *dev)2290{2291 struct ipoib_dev_priv *priv = ipoib_priv(dev);2292 struct rdma_netdev *rn = netdev_priv(dev);2293 2294 dev->priv_destructor = priv->next_priv_destructor;2295 if (dev->priv_destructor)2296 dev->priv_destructor(dev);2297 2298 /*2299 * There are some error flows around register_netdev failing that may2300 * attempt to call priv_destructor twice, prevent that from happening.2301 */2302 dev->priv_destructor = NULL;2303 2304 /* unregister/destroy is very complicated. Make bugs more obvious. */2305 rn->clnt_priv = NULL;2306 2307 kfree(priv);2308}2309 2310static ssize_t pkey_show(struct device *dev, struct device_attribute *attr,2311 char *buf)2312{2313 struct net_device *ndev = to_net_dev(dev);2314 struct ipoib_dev_priv *priv = ipoib_priv(ndev);2315 2316 return sysfs_emit(buf, "0x%04x\n", priv->pkey);2317}2318static DEVICE_ATTR_RO(pkey);2319 2320static ssize_t umcast_show(struct device *dev, struct device_attribute *attr,2321 char *buf)2322{2323 struct net_device *ndev = to_net_dev(dev);2324 struct ipoib_dev_priv *priv = ipoib_priv(ndev);2325 2326 return sysfs_emit(buf, "%d\n",2327 test_bit(IPOIB_FLAG_UMCAST, &priv->flags));2328}2329 2330void ipoib_set_umcast(struct net_device *ndev, int umcast_val)2331{2332 struct ipoib_dev_priv *priv = ipoib_priv(ndev);2333 2334 if (umcast_val > 0) {2335 set_bit(IPOIB_FLAG_UMCAST, &priv->flags);2336 ipoib_warn(priv, "ignoring multicast groups joined directly "2337 "by userspace\n");2338 } else2339 clear_bit(IPOIB_FLAG_UMCAST, &priv->flags);2340}2341 2342static ssize_t umcast_store(struct device *dev, struct device_attribute *attr,2343 const char *buf, size_t count)2344{2345 unsigned long umcast_val = simple_strtoul(buf, NULL, 0);2346 2347 ipoib_set_umcast(to_net_dev(dev), umcast_val);2348 2349 return count;2350}2351static DEVICE_ATTR_RW(umcast);2352 2353int ipoib_add_umcast_attr(struct net_device *dev)2354{2355 return device_create_file(&dev->dev, &dev_attr_umcast);2356}2357 2358static void set_base_guid(struct ipoib_dev_priv *priv, union ib_gid *gid)2359{2360 struct ipoib_dev_priv *child_priv;2361 struct net_device *netdev = priv->dev;2362 2363 netif_addr_lock_bh(netdev);2364 2365 memcpy(&priv->local_gid.global.interface_id,2366 &gid->global.interface_id,2367 sizeof(gid->global.interface_id));2368 dev_addr_mod(netdev, 4, (u8 *)&priv->local_gid, sizeof(priv->local_gid));2369 clear_bit(IPOIB_FLAG_DEV_ADDR_SET, &priv->flags);2370 2371 netif_addr_unlock_bh(netdev);2372 2373 if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {2374 down_read(&priv->vlan_rwsem);2375 list_for_each_entry(child_priv, &priv->child_intfs, list)2376 set_base_guid(child_priv, gid);2377 up_read(&priv->vlan_rwsem);2378 }2379}2380 2381static int ipoib_check_lladdr(struct net_device *dev,2382 struct sockaddr_storage *ss)2383{2384 union ib_gid *gid = (union ib_gid *)(ss->__data + 4);2385 int ret = 0;2386 2387 netif_addr_lock_bh(dev);2388 2389 /* Make sure the QPN, reserved and subnet prefix match the current2390 * lladdr, it also makes sure the lladdr is unicast.2391 */2392 if (memcmp(dev->dev_addr, ss->__data,2393 4 + sizeof(gid->global.subnet_prefix)) ||2394 gid->global.interface_id == 0)2395 ret = -EINVAL;2396 2397 netif_addr_unlock_bh(dev);2398 2399 return ret;2400}2401 2402static int ipoib_set_mac(struct net_device *dev, void *addr)2403{2404 struct ipoib_dev_priv *priv = ipoib_priv(dev);2405 struct sockaddr_storage *ss = addr;2406 int ret;2407 2408 if (!(dev->priv_flags & IFF_LIVE_ADDR_CHANGE) && netif_running(dev))2409 return -EBUSY;2410 2411 ret = ipoib_check_lladdr(dev, ss);2412 if (ret)2413 return ret;2414 2415 set_base_guid(priv, (union ib_gid *)(ss->__data + 4));2416 2417 queue_work(ipoib_workqueue, &priv->flush_light);2418 2419 return 0;2420}2421 2422static ssize_t create_child_store(struct device *dev,2423 struct device_attribute *attr,2424 const char *buf, size_t count)2425{2426 int pkey;2427 int ret;2428 2429 if (sscanf(buf, "%i", &pkey) != 1)2430 return -EINVAL;2431 2432 if (pkey <= 0 || pkey > 0xffff || pkey == 0x8000)2433 return -EINVAL;2434 2435 ret = ipoib_vlan_add(to_net_dev(dev), pkey);2436 2437 return ret ? ret : count;2438}2439static DEVICE_ATTR_WO(create_child);2440 2441static ssize_t delete_child_store(struct device *dev,2442 struct device_attribute *attr,2443 const char *buf, size_t count)2444{2445 int pkey;2446 int ret;2447 2448 if (sscanf(buf, "%i", &pkey) != 1)2449 return -EINVAL;2450 2451 if (pkey < 0 || pkey > 0xffff)2452 return -EINVAL;2453 2454 ret = ipoib_vlan_delete(to_net_dev(dev), pkey);2455 2456 return ret ? ret : count;2457 2458}2459static DEVICE_ATTR_WO(delete_child);2460 2461int ipoib_add_pkey_attr(struct net_device *dev)2462{2463 return device_create_file(&dev->dev, &dev_attr_pkey);2464}2465 2466/*2467 * We erroneously exposed the iface's port number in the dev_id2468 * sysfs field long after dev_port was introduced for that purpose[1],2469 * and we need to stop everyone from relying on that.2470 * Let's overload the shower routine for the dev_id file here2471 * to gently bring the issue up.2472 *2473 * [1] https://www.spinics.net/lists/netdev/msg272123.html2474 */2475static ssize_t dev_id_show(struct device *dev,2476 struct device_attribute *attr, char *buf)2477{2478 struct net_device *ndev = to_net_dev(dev);2479 2480 /*2481 * ndev->dev_port will be equal to 0 in old kernel prior to commit2482 * 9b8b2a323008 ("IB/ipoib: Use dev_port to expose network interface2483 * port numbers") Zero was chosen as special case for user space2484 * applications to fallback and query dev_id to check if it has2485 * different value or not.2486 *2487 * Don't print warning in such scenario.2488 *2489 * https://github.com/systemd/systemd/blob/master/src/udev/udev-builtin-net_id.c#L3582490 */2491 if (ndev->dev_port && ndev->dev_id == ndev->dev_port)2492 netdev_info_once(ndev,2493 "\"%s\" wants to know my dev_id. Should it look at dev_port instead? See Documentation/ABI/testing/sysfs-class-net for more info.\n",2494 current->comm);2495 2496 return sysfs_emit(buf, "%#x\n", ndev->dev_id);2497}2498static DEVICE_ATTR_RO(dev_id);2499 2500static int ipoib_intercept_dev_id_attr(struct net_device *dev)2501{2502 device_remove_file(&dev->dev, &dev_attr_dev_id);2503 return device_create_file(&dev->dev, &dev_attr_dev_id);2504}2505 2506static struct net_device *ipoib_add_port(const char *format,2507 struct ib_device *hca, u32 port)2508{2509 struct rtnl_link_ops *ops = ipoib_get_link_ops();2510 struct rdma_netdev_alloc_params params;2511 struct ipoib_dev_priv *priv;2512 struct net_device *ndev;2513 int result;2514 2515 ndev = ipoib_intf_alloc(hca, port, format);2516 if (IS_ERR(ndev)) {2517 pr_warn("%s, %d: ipoib_intf_alloc failed %ld\n", hca->name, port,2518 PTR_ERR(ndev));2519 return ndev;2520 }2521 priv = ipoib_priv(ndev);2522 2523 INIT_IB_EVENT_HANDLER(&priv->event_handler,2524 priv->ca, ipoib_event);2525 ib_register_event_handler(&priv->event_handler);2526 2527 /* call event handler to ensure pkey in sync */2528 queue_work(ipoib_workqueue, &priv->flush_heavy);2529 2530 ndev->rtnl_link_ops = ipoib_get_link_ops();2531 2532 result = register_netdev(ndev);2533 if (result) {2534 pr_warn("%s: couldn't register ipoib port %d; error %d\n",2535 hca->name, port, result);2536 2537 ipoib_parent_unregister_pre(ndev);2538 ipoib_intf_free(ndev);2539 free_netdev(ndev);2540 2541 return ERR_PTR(result);2542 }2543 2544 if (hca->ops.rdma_netdev_get_params) {2545 int rc = hca->ops.rdma_netdev_get_params(hca, port,2546 RDMA_NETDEV_IPOIB,2547 ¶ms);2548 2549 if (!rc && ops->priv_size < params.sizeof_priv)2550 ops->priv_size = params.sizeof_priv;2551 }2552 /*2553 * We cannot set priv_destructor before register_netdev because we2554 * need priv to be always valid during the error flow to execute2555 * ipoib_parent_unregister_pre(). Instead handle it manually and only2556 * enter priv_destructor mode once we are completely registered.2557 */2558 ndev->priv_destructor = ipoib_intf_free;2559 2560 if (ipoib_intercept_dev_id_attr(ndev))2561 goto sysfs_failed;2562 if (ipoib_cm_add_mode_attr(ndev))2563 goto sysfs_failed;2564 if (ipoib_add_pkey_attr(ndev))2565 goto sysfs_failed;2566 if (ipoib_add_umcast_attr(ndev))2567 goto sysfs_failed;2568 if (device_create_file(&ndev->dev, &dev_attr_create_child))2569 goto sysfs_failed;2570 if (device_create_file(&ndev->dev, &dev_attr_delete_child))2571 goto sysfs_failed;2572 2573 return ndev;2574 2575sysfs_failed:2576 ipoib_parent_unregister_pre(ndev);2577 unregister_netdev(ndev);2578 return ERR_PTR(-ENOMEM);2579}2580 2581static int ipoib_add_one(struct ib_device *device)2582{2583 struct list_head *dev_list;2584 struct net_device *dev;2585 struct ipoib_dev_priv *priv;2586 unsigned int p;2587 int count = 0;2588 2589 dev_list = kmalloc(sizeof(*dev_list), GFP_KERNEL);2590 if (!dev_list)2591 return -ENOMEM;2592 2593 INIT_LIST_HEAD(dev_list);2594 2595 rdma_for_each_port (device, p) {2596 if (!rdma_protocol_ib(device, p))2597 continue;2598 dev = ipoib_add_port("ib%d", device, p);2599 if (!IS_ERR(dev)) {2600 priv = ipoib_priv(dev);2601 list_add_tail(&priv->list, dev_list);2602 count++;2603 }2604 }2605 2606 if (!count) {2607 kfree(dev_list);2608 return -EOPNOTSUPP;2609 }2610 2611 ib_set_client_data(device, &ipoib_client, dev_list);2612 return 0;2613}2614 2615static void ipoib_remove_one(struct ib_device *device, void *client_data)2616{2617 struct ipoib_dev_priv *priv, *tmp, *cpriv, *tcpriv;2618 struct list_head *dev_list = client_data;2619 2620 list_for_each_entry_safe(priv, tmp, dev_list, list) {2621 LIST_HEAD(head);2622 ipoib_parent_unregister_pre(priv->dev);2623 2624 rtnl_lock();2625 2626 list_for_each_entry_safe(cpriv, tcpriv, &priv->child_intfs,2627 list)2628 unregister_netdevice_queue(cpriv->dev, &head);2629 unregister_netdevice_queue(priv->dev, &head);2630 unregister_netdevice_many(&head);2631 2632 rtnl_unlock();2633 }2634 2635 kfree(dev_list);2636}2637 2638#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG2639static struct notifier_block ipoib_netdev_notifier = {2640 .notifier_call = ipoib_netdev_event,2641};2642#endif2643 2644static int __init ipoib_init_module(void)2645{2646 int ret;2647 2648 ipoib_recvq_size = roundup_pow_of_two(ipoib_recvq_size);2649 ipoib_recvq_size = min(ipoib_recvq_size, IPOIB_MAX_QUEUE_SIZE);2650 ipoib_recvq_size = max(ipoib_recvq_size, IPOIB_MIN_QUEUE_SIZE);2651 2652 ipoib_sendq_size = roundup_pow_of_two(ipoib_sendq_size);2653 ipoib_sendq_size = min(ipoib_sendq_size, IPOIB_MAX_QUEUE_SIZE);2654 ipoib_sendq_size = max3(ipoib_sendq_size, 2 * MAX_SEND_CQE, IPOIB_MIN_QUEUE_SIZE);2655#ifdef CONFIG_INFINIBAND_IPOIB_CM2656 ipoib_max_conn_qp = min(ipoib_max_conn_qp, IPOIB_CM_MAX_CONN_QP);2657 ipoib_max_conn_qp = max(ipoib_max_conn_qp, 0);2658#endif2659 2660 /*2661 * When copying small received packets, we only copy from the2662 * linear data part of the SKB, so we rely on this condition.2663 */2664 BUILD_BUG_ON(IPOIB_CM_COPYBREAK > IPOIB_CM_HEAD_SIZE);2665 2666 ipoib_register_debugfs();2667 2668 /*2669 * We create a global workqueue here that is used for all flush2670 * operations. However, if you attempt to flush a workqueue2671 * from a task on that same workqueue, it deadlocks the system.2672 * We want to be able to flush the tasks associated with a2673 * specific net device, so we also create a workqueue for each2674 * netdevice. We queue up the tasks for that device only on2675 * its private workqueue, and we only queue up flush events2676 * on our global flush workqueue. This avoids the deadlocks.2677 */2678 ipoib_workqueue = alloc_ordered_workqueue("ipoib_flush", 0);2679 if (!ipoib_workqueue) {2680 ret = -ENOMEM;2681 goto err_fs;2682 }2683 2684 ib_sa_register_client(&ipoib_sa_client);2685 2686 ret = ib_register_client(&ipoib_client);2687 if (ret)2688 goto err_sa;2689 2690 ret = ipoib_netlink_init();2691 if (ret)2692 goto err_client;2693 2694#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG2695 register_netdevice_notifier(&ipoib_netdev_notifier);2696#endif2697 return 0;2698 2699err_client:2700 ib_unregister_client(&ipoib_client);2701 2702err_sa:2703 ib_sa_unregister_client(&ipoib_sa_client);2704 destroy_workqueue(ipoib_workqueue);2705 2706err_fs:2707 ipoib_unregister_debugfs();2708 2709 return ret;2710}2711 2712static void __exit ipoib_cleanup_module(void)2713{2714#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG2715 unregister_netdevice_notifier(&ipoib_netdev_notifier);2716#endif2717 ipoib_netlink_fini();2718 ib_unregister_client(&ipoib_client);2719 ib_sa_unregister_client(&ipoib_sa_client);2720 ipoib_unregister_debugfs();2721 destroy_workqueue(ipoib_workqueue);2722}2723 2724module_init(ipoib_init_module);2725module_exit(ipoib_cleanup_module);2726