brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * drivers/net/team/team.c - Network team device driver4 * Copyright (c) 2011 Jiri Pirko <jpirko@redhat.com>5 */6 7#include <linux/ethtool.h>8#include <linux/kernel.h>9#include <linux/types.h>10#include <linux/module.h>11#include <linux/init.h>12#include <linux/slab.h>13#include <linux/rcupdate.h>14#include <linux/errno.h>15#include <linux/ctype.h>16#include <linux/notifier.h>17#include <linux/netdevice.h>18#include <linux/netpoll.h>19#include <linux/if_vlan.h>20#include <linux/if_arp.h>21#include <linux/socket.h>22#include <linux/etherdevice.h>23#include <linux/rtnetlink.h>24#include <net/rtnetlink.h>25#include <net/genetlink.h>26#include <net/netlink.h>27#include <net/sch_generic.h>28#include <linux/if_team.h>29 30#include "team_nl.h"31 32#define DRV_NAME "team"33 34 35/**********36 * Helpers37 **********/38 39static struct team_port *team_port_get_rtnl(const struct net_device *dev)40{41	struct team_port *port = rtnl_dereference(dev->rx_handler_data);42 43	return netif_is_team_port(dev) ? port : NULL;44}45 46/*47 * Since the ability to change device address for open port device is tested in48 * team_port_add, this function can be called without control of return value49 */50static int __set_port_dev_addr(struct net_device *port_dev,51			       const unsigned char *dev_addr)52{53	struct sockaddr_storage addr;54 55	memcpy(addr.__data, dev_addr, port_dev->addr_len);56	addr.ss_family = port_dev->type;57	return dev_set_mac_address(port_dev, (struct sockaddr *)&addr, NULL);58}59 60static int team_port_set_orig_dev_addr(struct team_port *port)61{62	return __set_port_dev_addr(port->dev, port->orig.dev_addr);63}64 65static int team_port_set_team_dev_addr(struct team *team,66				       struct team_port *port)67{68	return __set_port_dev_addr(port->dev, team->dev->dev_addr);69}70 71int team_modeop_port_enter(struct team *team, struct team_port *port)72{73	return team_port_set_team_dev_addr(team, port);74}75EXPORT_SYMBOL(team_modeop_port_enter);76 77void team_modeop_port_change_dev_addr(struct team *team,78				      struct team_port *port)79{80	team_port_set_team_dev_addr(team, port);81}82EXPORT_SYMBOL(team_modeop_port_change_dev_addr);83 84static void team_lower_state_changed(struct team_port *port)85{86	struct netdev_lag_lower_state_info info;87 88	info.link_up = port->linkup;89	info.tx_enabled = team_port_enabled(port);90	netdev_lower_state_changed(port->dev, &info);91}92 93static void team_refresh_port_linkup(struct team_port *port)94{95	bool new_linkup = port->user.linkup_enabled ? port->user.linkup :96						      port->state.linkup;97 98	if (port->linkup != new_linkup) {99		port->linkup = new_linkup;100		team_lower_state_changed(port);101	}102}103 104 105/*******************106 * Options handling107 *******************/108 109struct team_option_inst { /* One for each option instance */110	struct list_head list;111	struct list_head tmp_list;112	struct team_option *option;113	struct team_option_inst_info info;114	bool changed;115	bool removed;116};117 118static struct team_option *__team_find_option(struct team *team,119					      const char *opt_name)120{121	struct team_option *option;122 123	list_for_each_entry(option, &team->option_list, list) {124		if (strcmp(option->name, opt_name) == 0)125			return option;126	}127	return NULL;128}129 130static void __team_option_inst_del(struct team_option_inst *opt_inst)131{132	list_del(&opt_inst->list);133	kfree(opt_inst);134}135 136static void __team_option_inst_del_option(struct team *team,137					  struct team_option *option)138{139	struct team_option_inst *opt_inst, *tmp;140 141	list_for_each_entry_safe(opt_inst, tmp, &team->option_inst_list, list) {142		if (opt_inst->option == option)143			__team_option_inst_del(opt_inst);144	}145}146 147static int __team_option_inst_add(struct team *team, struct team_option *option,148				  struct team_port *port)149{150	struct team_option_inst *opt_inst;151	unsigned int array_size;152	unsigned int i;153 154	array_size = option->array_size;155	if (!array_size)156		array_size = 1; /* No array but still need one instance */157 158	for (i = 0; i < array_size; i++) {159		opt_inst = kmalloc(sizeof(*opt_inst), GFP_KERNEL);160		if (!opt_inst)161			return -ENOMEM;162		opt_inst->option = option;163		opt_inst->info.port = port;164		opt_inst->info.array_index = i;165		opt_inst->changed = true;166		opt_inst->removed = false;167		list_add_tail(&opt_inst->list, &team->option_inst_list);168		if (option->init)169			option->init(team, &opt_inst->info);170 171	}172	return 0;173}174 175static int __team_option_inst_add_option(struct team *team,176					 struct team_option *option)177{178	int err;179 180	if (!option->per_port) {181		err = __team_option_inst_add(team, option, NULL);182		if (err)183			goto inst_del_option;184	}185	return 0;186 187inst_del_option:188	__team_option_inst_del_option(team, option);189	return err;190}191 192static void __team_option_inst_mark_removed_option(struct team *team,193						   struct team_option *option)194{195	struct team_option_inst *opt_inst;196 197	list_for_each_entry(opt_inst, &team->option_inst_list, list) {198		if (opt_inst->option == option) {199			opt_inst->changed = true;200			opt_inst->removed = true;201		}202	}203}204 205static void __team_option_inst_del_port(struct team *team,206					struct team_port *port)207{208	struct team_option_inst *opt_inst, *tmp;209 210	list_for_each_entry_safe(opt_inst, tmp, &team->option_inst_list, list) {211		if (opt_inst->option->per_port &&212		    opt_inst->info.port == port)213			__team_option_inst_del(opt_inst);214	}215}216 217static int __team_option_inst_add_port(struct team *team,218				       struct team_port *port)219{220	struct team_option *option;221	int err;222 223	list_for_each_entry(option, &team->option_list, list) {224		if (!option->per_port)225			continue;226		err = __team_option_inst_add(team, option, port);227		if (err)228			goto inst_del_port;229	}230	return 0;231 232inst_del_port:233	__team_option_inst_del_port(team, port);234	return err;235}236 237static void __team_option_inst_mark_removed_port(struct team *team,238						 struct team_port *port)239{240	struct team_option_inst *opt_inst;241 242	list_for_each_entry(opt_inst, &team->option_inst_list, list) {243		if (opt_inst->info.port == port) {244			opt_inst->changed = true;245			opt_inst->removed = true;246		}247	}248}249 250static int __team_options_register(struct team *team,251				   const struct team_option *option,252				   size_t option_count)253{254	int i;255	struct team_option **dst_opts;256	int err;257 258	dst_opts = kcalloc(option_count, sizeof(struct team_option *),259			   GFP_KERNEL);260	if (!dst_opts)261		return -ENOMEM;262	for (i = 0; i < option_count; i++, option++) {263		if (__team_find_option(team, option->name)) {264			err = -EEXIST;265			goto alloc_rollback;266		}267		dst_opts[i] = kmemdup(option, sizeof(*option), GFP_KERNEL);268		if (!dst_opts[i]) {269			err = -ENOMEM;270			goto alloc_rollback;271		}272	}273 274	for (i = 0; i < option_count; i++) {275		err = __team_option_inst_add_option(team, dst_opts[i]);276		if (err)277			goto inst_rollback;278		list_add_tail(&dst_opts[i]->list, &team->option_list);279	}280 281	kfree(dst_opts);282	return 0;283 284inst_rollback:285	for (i--; i >= 0; i--) {286		__team_option_inst_del_option(team, dst_opts[i]);287		list_del(&dst_opts[i]->list);288	}289 290	i = option_count;291alloc_rollback:292	for (i--; i >= 0; i--)293		kfree(dst_opts[i]);294 295	kfree(dst_opts);296	return err;297}298 299static void __team_options_mark_removed(struct team *team,300					const struct team_option *option,301					size_t option_count)302{303	int i;304 305	for (i = 0; i < option_count; i++, option++) {306		struct team_option *del_opt;307 308		del_opt = __team_find_option(team, option->name);309		if (del_opt)310			__team_option_inst_mark_removed_option(team, del_opt);311	}312}313 314static void __team_options_unregister(struct team *team,315				      const struct team_option *option,316				      size_t option_count)317{318	int i;319 320	for (i = 0; i < option_count; i++, option++) {321		struct team_option *del_opt;322 323		del_opt = __team_find_option(team, option->name);324		if (del_opt) {325			__team_option_inst_del_option(team, del_opt);326			list_del(&del_opt->list);327			kfree(del_opt);328		}329	}330}331 332static void __team_options_change_check(struct team *team);333 334int team_options_register(struct team *team,335			  const struct team_option *option,336			  size_t option_count)337{338	int err;339 340	err = __team_options_register(team, option, option_count);341	if (err)342		return err;343	__team_options_change_check(team);344	return 0;345}346EXPORT_SYMBOL(team_options_register);347 348void team_options_unregister(struct team *team,349			     const struct team_option *option,350			     size_t option_count)351{352	__team_options_mark_removed(team, option, option_count);353	__team_options_change_check(team);354	__team_options_unregister(team, option, option_count);355}356EXPORT_SYMBOL(team_options_unregister);357 358static int team_option_get(struct team *team,359			   struct team_option_inst *opt_inst,360			   struct team_gsetter_ctx *ctx)361{362	if (!opt_inst->option->getter)363		return -EOPNOTSUPP;364 365	opt_inst->option->getter(team, ctx);366	return 0;367}368 369static int team_option_set(struct team *team,370			   struct team_option_inst *opt_inst,371			   struct team_gsetter_ctx *ctx)372{373	if (!opt_inst->option->setter)374		return -EOPNOTSUPP;375	return opt_inst->option->setter(team, ctx);376}377 378void team_option_inst_set_change(struct team_option_inst_info *opt_inst_info)379{380	struct team_option_inst *opt_inst;381 382	opt_inst = container_of(opt_inst_info, struct team_option_inst, info);383	opt_inst->changed = true;384}385EXPORT_SYMBOL(team_option_inst_set_change);386 387void team_options_change_check(struct team *team)388{389	__team_options_change_check(team);390}391EXPORT_SYMBOL(team_options_change_check);392 393 394/****************395 * Mode handling396 ****************/397 398static LIST_HEAD(mode_list);399static DEFINE_SPINLOCK(mode_list_lock);400 401struct team_mode_item {402	struct list_head list;403	const struct team_mode *mode;404};405 406static struct team_mode_item *__find_mode(const char *kind)407{408	struct team_mode_item *mitem;409 410	list_for_each_entry(mitem, &mode_list, list) {411		if (strcmp(mitem->mode->kind, kind) == 0)412			return mitem;413	}414	return NULL;415}416 417static bool is_good_mode_name(const char *name)418{419	while (*name != '\0') {420		if (!isalpha(*name) && !isdigit(*name) && *name != '_')421			return false;422		name++;423	}424	return true;425}426 427int team_mode_register(const struct team_mode *mode)428{429	int err = 0;430	struct team_mode_item *mitem;431 432	if (!is_good_mode_name(mode->kind) ||433	    mode->priv_size > TEAM_MODE_PRIV_SIZE)434		return -EINVAL;435 436	mitem = kmalloc(sizeof(*mitem), GFP_KERNEL);437	if (!mitem)438		return -ENOMEM;439 440	spin_lock(&mode_list_lock);441	if (__find_mode(mode->kind)) {442		err = -EEXIST;443		kfree(mitem);444		goto unlock;445	}446	mitem->mode = mode;447	list_add_tail(&mitem->list, &mode_list);448unlock:449	spin_unlock(&mode_list_lock);450	return err;451}452EXPORT_SYMBOL(team_mode_register);453 454void team_mode_unregister(const struct team_mode *mode)455{456	struct team_mode_item *mitem;457 458	spin_lock(&mode_list_lock);459	mitem = __find_mode(mode->kind);460	if (mitem) {461		list_del_init(&mitem->list);462		kfree(mitem);463	}464	spin_unlock(&mode_list_lock);465}466EXPORT_SYMBOL(team_mode_unregister);467 468static const struct team_mode *team_mode_get(const char *kind)469{470	struct team_mode_item *mitem;471	const struct team_mode *mode = NULL;472 473	if (!try_module_get(THIS_MODULE))474		return NULL;475 476	spin_lock(&mode_list_lock);477	mitem = __find_mode(kind);478	if (!mitem) {479		spin_unlock(&mode_list_lock);480		request_module("team-mode-%s", kind);481		spin_lock(&mode_list_lock);482		mitem = __find_mode(kind);483	}484	if (mitem) {485		mode = mitem->mode;486		if (!try_module_get(mode->owner))487			mode = NULL;488	}489 490	spin_unlock(&mode_list_lock);491	module_put(THIS_MODULE);492	return mode;493}494 495static void team_mode_put(const struct team_mode *mode)496{497	module_put(mode->owner);498}499 500static bool team_dummy_transmit(struct team *team, struct sk_buff *skb)501{502	dev_kfree_skb_any(skb);503	return false;504}505 506static rx_handler_result_t team_dummy_receive(struct team *team,507					      struct team_port *port,508					      struct sk_buff *skb)509{510	return RX_HANDLER_ANOTHER;511}512 513static const struct team_mode __team_no_mode = {514	.kind		= "*NOMODE*",515};516 517static bool team_is_mode_set(struct team *team)518{519	return team->mode != &__team_no_mode;520}521 522static void team_set_no_mode(struct team *team)523{524	team->user_carrier_enabled = false;525	team->mode = &__team_no_mode;526}527 528static void team_adjust_ops(struct team *team)529{530	/*531	 * To avoid checks in rx/tx skb paths, ensure here that non-null and532	 * correct ops are always set.533	 */534 535	if (!team->en_port_count || !team_is_mode_set(team) ||536	    !team->mode->ops->transmit)537		team->ops.transmit = team_dummy_transmit;538	else539		team->ops.transmit = team->mode->ops->transmit;540 541	if (!team->en_port_count || !team_is_mode_set(team) ||542	    !team->mode->ops->receive)543		team->ops.receive = team_dummy_receive;544	else545		team->ops.receive = team->mode->ops->receive;546}547 548/*549 * We can benefit from the fact that it's ensured no port is present550 * at the time of mode change. Therefore no packets are in fly so there's no551 * need to set mode operations in any special way.552 */553static int __team_change_mode(struct team *team,554			      const struct team_mode *new_mode)555{556	/* Check if mode was previously set and do cleanup if so */557	if (team_is_mode_set(team)) {558		void (*exit_op)(struct team *team) = team->ops.exit;559 560		/* Clear ops area so no callback is called any longer */561		memset(&team->ops, 0, sizeof(struct team_mode_ops));562		team_adjust_ops(team);563 564		if (exit_op)565			exit_op(team);566		team_mode_put(team->mode);567		team_set_no_mode(team);568		/* zero private data area */569		memset(&team->mode_priv, 0,570		       sizeof(struct team) - offsetof(struct team, mode_priv));571	}572 573	if (!new_mode)574		return 0;575 576	if (new_mode->ops->init) {577		int err;578 579		err = new_mode->ops->init(team);580		if (err)581			return err;582	}583 584	team->mode = new_mode;585	memcpy(&team->ops, new_mode->ops, sizeof(struct team_mode_ops));586	team_adjust_ops(team);587 588	return 0;589}590 591static int team_change_mode(struct team *team, const char *kind)592{593	const struct team_mode *new_mode;594	struct net_device *dev = team->dev;595	int err;596 597	if (!list_empty(&team->port_list)) {598		netdev_err(dev, "No ports can be present during mode change\n");599		return -EBUSY;600	}601 602	if (team_is_mode_set(team) && strcmp(team->mode->kind, kind) == 0) {603		netdev_err(dev, "Unable to change to the same mode the team is in\n");604		return -EINVAL;605	}606 607	new_mode = team_mode_get(kind);608	if (!new_mode) {609		netdev_err(dev, "Mode \"%s\" not found\n", kind);610		return -EINVAL;611	}612 613	err = __team_change_mode(team, new_mode);614	if (err) {615		netdev_err(dev, "Failed to change to mode \"%s\"\n", kind);616		team_mode_put(new_mode);617		return err;618	}619 620	netdev_info(dev, "Mode changed to \"%s\"\n", kind);621	return 0;622}623 624 625/*********************626 * Peers notification627 *********************/628 629static void team_notify_peers_work(struct work_struct *work)630{631	struct team *team;632	int val;633 634	team = container_of(work, struct team, notify_peers.dw.work);635 636	if (!rtnl_trylock()) {637		schedule_delayed_work(&team->notify_peers.dw, 0);638		return;639	}640	val = atomic_dec_if_positive(&team->notify_peers.count_pending);641	if (val < 0) {642		rtnl_unlock();643		return;644	}645	call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, team->dev);646	rtnl_unlock();647	if (val)648		schedule_delayed_work(&team->notify_peers.dw,649				      msecs_to_jiffies(team->notify_peers.interval));650}651 652static void team_notify_peers(struct team *team)653{654	if (!team->notify_peers.count || !netif_running(team->dev))655		return;656	atomic_add(team->notify_peers.count, &team->notify_peers.count_pending);657	schedule_delayed_work(&team->notify_peers.dw, 0);658}659 660static void team_notify_peers_init(struct team *team)661{662	INIT_DELAYED_WORK(&team->notify_peers.dw, team_notify_peers_work);663}664 665static void team_notify_peers_fini(struct team *team)666{667	cancel_delayed_work_sync(&team->notify_peers.dw);668}669 670 671/*******************************672 * Send multicast group rejoins673 *******************************/674 675static void team_mcast_rejoin_work(struct work_struct *work)676{677	struct team *team;678	int val;679 680	team = container_of(work, struct team, mcast_rejoin.dw.work);681 682	if (!rtnl_trylock()) {683		schedule_delayed_work(&team->mcast_rejoin.dw, 0);684		return;685	}686	val = atomic_dec_if_positive(&team->mcast_rejoin.count_pending);687	if (val < 0) {688		rtnl_unlock();689		return;690	}691	call_netdevice_notifiers(NETDEV_RESEND_IGMP, team->dev);692	rtnl_unlock();693	if (val)694		schedule_delayed_work(&team->mcast_rejoin.dw,695				      msecs_to_jiffies(team->mcast_rejoin.interval));696}697 698static void team_mcast_rejoin(struct team *team)699{700	if (!team->mcast_rejoin.count || !netif_running(team->dev))701		return;702	atomic_add(team->mcast_rejoin.count, &team->mcast_rejoin.count_pending);703	schedule_delayed_work(&team->mcast_rejoin.dw, 0);704}705 706static void team_mcast_rejoin_init(struct team *team)707{708	INIT_DELAYED_WORK(&team->mcast_rejoin.dw, team_mcast_rejoin_work);709}710 711static void team_mcast_rejoin_fini(struct team *team)712{713	cancel_delayed_work_sync(&team->mcast_rejoin.dw);714}715 716 717/************************718 * Rx path frame handler719 ************************/720 721/* note: already called with rcu_read_lock */722static rx_handler_result_t team_handle_frame(struct sk_buff **pskb)723{724	struct sk_buff *skb = *pskb;725	struct team_port *port;726	struct team *team;727	rx_handler_result_t res;728 729	skb = skb_share_check(skb, GFP_ATOMIC);730	if (!skb)731		return RX_HANDLER_CONSUMED;732 733	*pskb = skb;734 735	port = team_port_get_rcu(skb->dev);736	team = port->team;737	if (!team_port_enabled(port)) {738		if (is_link_local_ether_addr(eth_hdr(skb)->h_dest))739			/* link-local packets are mostly useful when stack receives them740			 * with the link they arrive on.741			 */742			return RX_HANDLER_PASS;743		/* allow exact match delivery for disabled ports */744		res = RX_HANDLER_EXACT;745	} else {746		res = team->ops.receive(team, port, skb);747	}748	if (res == RX_HANDLER_ANOTHER) {749		struct team_pcpu_stats *pcpu_stats;750 751		pcpu_stats = this_cpu_ptr(team->pcpu_stats);752		u64_stats_update_begin(&pcpu_stats->syncp);753		u64_stats_inc(&pcpu_stats->rx_packets);754		u64_stats_add(&pcpu_stats->rx_bytes, skb->len);755		if (skb->pkt_type == PACKET_MULTICAST)756			u64_stats_inc(&pcpu_stats->rx_multicast);757		u64_stats_update_end(&pcpu_stats->syncp);758 759		skb->dev = team->dev;760	} else if (res == RX_HANDLER_EXACT) {761		this_cpu_inc(team->pcpu_stats->rx_nohandler);762	} else {763		this_cpu_inc(team->pcpu_stats->rx_dropped);764	}765 766	return res;767}768 769 770/*************************************771 * Multiqueue Tx port select override772 *************************************/773 774static int team_queue_override_init(struct team *team)775{776	struct list_head *listarr;777	unsigned int queue_cnt = team->dev->num_tx_queues - 1;778	unsigned int i;779 780	if (!queue_cnt)781		return 0;782	listarr = kmalloc_array(queue_cnt, sizeof(struct list_head),783				GFP_KERNEL);784	if (!listarr)785		return -ENOMEM;786	team->qom_lists = listarr;787	for (i = 0; i < queue_cnt; i++)788		INIT_LIST_HEAD(listarr++);789	return 0;790}791 792static void team_queue_override_fini(struct team *team)793{794	kfree(team->qom_lists);795}796 797static struct list_head *__team_get_qom_list(struct team *team, u16 queue_id)798{799	return &team->qom_lists[queue_id - 1];800}801 802/*803 * note: already called with rcu_read_lock804 */805static bool team_queue_override_transmit(struct team *team, struct sk_buff *skb)806{807	struct list_head *qom_list;808	struct team_port *port;809 810	if (!team->queue_override_enabled || !skb->queue_mapping)811		return false;812	qom_list = __team_get_qom_list(team, skb->queue_mapping);813	list_for_each_entry_rcu(port, qom_list, qom_list) {814		if (!team_dev_queue_xmit(team, port, skb))815			return true;816	}817	return false;818}819 820static void __team_queue_override_port_del(struct team *team,821					   struct team_port *port)822{823	if (!port->queue_id)824		return;825	list_del_rcu(&port->qom_list);826}827 828static bool team_queue_override_port_has_gt_prio_than(struct team_port *port,829						      struct team_port *cur)830{831	if (port->priority < cur->priority)832		return true;833	if (port->priority > cur->priority)834		return false;835	if (port->index < cur->index)836		return true;837	return false;838}839 840static void __team_queue_override_port_add(struct team *team,841					   struct team_port *port)842{843	struct team_port *cur;844	struct list_head *qom_list;845	struct list_head *node;846 847	if (!port->queue_id)848		return;849	qom_list = __team_get_qom_list(team, port->queue_id);850	node = qom_list;851	list_for_each_entry(cur, qom_list, qom_list) {852		if (team_queue_override_port_has_gt_prio_than(port, cur))853			break;854		node = &cur->qom_list;855	}856	list_add_tail_rcu(&port->qom_list, node);857}858 859static void __team_queue_override_enabled_check(struct team *team)860{861	struct team_port *port;862	bool enabled = false;863 864	list_for_each_entry(port, &team->port_list, list) {865		if (port->queue_id) {866			enabled = true;867			break;868		}869	}870	if (enabled == team->queue_override_enabled)871		return;872	netdev_dbg(team->dev, "%s queue override\n",873		   enabled ? "Enabling" : "Disabling");874	team->queue_override_enabled = enabled;875}876 877static void team_queue_override_port_prio_changed(struct team *team,878						  struct team_port *port)879{880	if (!port->queue_id || team_port_enabled(port))881		return;882	__team_queue_override_port_del(team, port);883	__team_queue_override_port_add(team, port);884	__team_queue_override_enabled_check(team);885}886 887static void team_queue_override_port_change_queue_id(struct team *team,888						     struct team_port *port,889						     u16 new_queue_id)890{891	if (team_port_enabled(port)) {892		__team_queue_override_port_del(team, port);893		port->queue_id = new_queue_id;894		__team_queue_override_port_add(team, port);895		__team_queue_override_enabled_check(team);896	} else {897		port->queue_id = new_queue_id;898	}899}900 901static void team_queue_override_port_add(struct team *team,902					 struct team_port *port)903{904	__team_queue_override_port_add(team, port);905	__team_queue_override_enabled_check(team);906}907 908static void team_queue_override_port_del(struct team *team,909					 struct team_port *port)910{911	__team_queue_override_port_del(team, port);912	__team_queue_override_enabled_check(team);913}914 915 916/****************917 * Port handling918 ****************/919 920static bool team_port_find(const struct team *team,921			   const struct team_port *port)922{923	struct team_port *cur;924 925	list_for_each_entry(cur, &team->port_list, list)926		if (cur == port)927			return true;928	return false;929}930 931/*932 * Enable/disable port by adding to enabled port hashlist and setting933 * port->index (Might be racy so reader could see incorrect ifindex when934 * processing a flying packet, but that is not a problem). Write guarded935 * by team->lock.936 */937static void team_port_enable(struct team *team,938			     struct team_port *port)939{940	if (team_port_enabled(port))941		return;942	port->index = team->en_port_count++;943	hlist_add_head_rcu(&port->hlist,944			   team_port_index_hash(team, port->index));945	team_adjust_ops(team);946	team_queue_override_port_add(team, port);947	if (team->ops.port_enabled)948		team->ops.port_enabled(team, port);949	team_notify_peers(team);950	team_mcast_rejoin(team);951	team_lower_state_changed(port);952}953 954static void __reconstruct_port_hlist(struct team *team, int rm_index)955{956	int i;957	struct team_port *port;958 959	for (i = rm_index + 1; i < team->en_port_count; i++) {960		port = team_get_port_by_index(team, i);961		hlist_del_rcu(&port->hlist);962		port->index--;963		hlist_add_head_rcu(&port->hlist,964				   team_port_index_hash(team, port->index));965	}966}967 968static void team_port_disable(struct team *team,969			      struct team_port *port)970{971	if (!team_port_enabled(port))972		return;973	if (team->ops.port_disabled)974		team->ops.port_disabled(team, port);975	hlist_del_rcu(&port->hlist);976	__reconstruct_port_hlist(team, port->index);977	port->index = -1;978	team->en_port_count--;979	team_queue_override_port_del(team, port);980	team_adjust_ops(team);981	team_lower_state_changed(port);982}983 984#define TEAM_VLAN_FEATURES (NETIF_F_HW_CSUM | NETIF_F_SG | \985			    NETIF_F_FRAGLIST | NETIF_F_GSO_SOFTWARE | \986			    NETIF_F_HIGHDMA | NETIF_F_LRO)987 988#define TEAM_ENC_FEATURES	(NETIF_F_HW_CSUM | NETIF_F_SG | \989				 NETIF_F_RXCSUM | NETIF_F_GSO_SOFTWARE)990 991static void __team_compute_features(struct team *team)992{993	struct team_port *port;994	netdev_features_t vlan_features = TEAM_VLAN_FEATURES &995					  NETIF_F_ALL_FOR_ALL;996	netdev_features_t enc_features  = TEAM_ENC_FEATURES;997	unsigned short max_hard_header_len = ETH_HLEN;998	unsigned int dst_release_flag = IFF_XMIT_DST_RELEASE |999					IFF_XMIT_DST_RELEASE_PERM;1000 1001	rcu_read_lock();1002	list_for_each_entry_rcu(port, &team->port_list, list) {1003		vlan_features = netdev_increment_features(vlan_features,1004					port->dev->vlan_features,1005					TEAM_VLAN_FEATURES);1006		enc_features =1007			netdev_increment_features(enc_features,1008						  port->dev->hw_enc_features,1009						  TEAM_ENC_FEATURES);1010 1011 1012		dst_release_flag &= port->dev->priv_flags;1013		if (port->dev->hard_header_len > max_hard_header_len)1014			max_hard_header_len = port->dev->hard_header_len;1015	}1016	rcu_read_unlock();1017 1018	team->dev->vlan_features = vlan_features;1019	team->dev->hw_enc_features = enc_features | NETIF_F_GSO_ENCAP_ALL |1020				     NETIF_F_HW_VLAN_CTAG_TX |1021				     NETIF_F_HW_VLAN_STAG_TX;1022	team->dev->hard_header_len = max_hard_header_len;1023 1024	team->dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;1025	if (dst_release_flag == (IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM))1026		team->dev->priv_flags |= IFF_XMIT_DST_RELEASE;1027}1028 1029static void team_compute_features(struct team *team)1030{1031	__team_compute_features(team);1032	netdev_change_features(team->dev);1033}1034 1035static int team_port_enter(struct team *team, struct team_port *port)1036{1037	int err = 0;1038 1039	dev_hold(team->dev);1040	if (team->ops.port_enter) {1041		err = team->ops.port_enter(team, port);1042		if (err) {1043			netdev_err(team->dev, "Device %s failed to enter team mode\n",1044				   port->dev->name);1045			goto err_port_enter;1046		}1047	}1048 1049	return 0;1050 1051err_port_enter:1052	dev_put(team->dev);1053 1054	return err;1055}1056 1057static void team_port_leave(struct team *team, struct team_port *port)1058{1059	if (team->ops.port_leave)1060		team->ops.port_leave(team, port);1061	dev_put(team->dev);1062}1063 1064#ifdef CONFIG_NET_POLL_CONTROLLER1065static int __team_port_enable_netpoll(struct team_port *port)1066{1067	struct netpoll *np;1068	int err;1069 1070	np = kzalloc(sizeof(*np), GFP_KERNEL);1071	if (!np)1072		return -ENOMEM;1073 1074	err = __netpoll_setup(np, port->dev);1075	if (err) {1076		kfree(np);1077		return err;1078	}1079	port->np = np;1080	return err;1081}1082 1083static int team_port_enable_netpoll(struct team_port *port)1084{1085	if (!port->team->dev->npinfo)1086		return 0;1087 1088	return __team_port_enable_netpoll(port);1089}1090 1091static void team_port_disable_netpoll(struct team_port *port)1092{1093	struct netpoll *np = port->np;1094 1095	if (!np)1096		return;1097	port->np = NULL;1098 1099	__netpoll_free(np);1100}1101#else1102static int team_port_enable_netpoll(struct team_port *port)1103{1104	return 0;1105}1106static void team_port_disable_netpoll(struct team_port *port)1107{1108}1109#endif1110 1111static int team_upper_dev_link(struct team *team, struct team_port *port,1112			       struct netlink_ext_ack *extack)1113{1114	struct netdev_lag_upper_info lag_upper_info;1115	int err;1116 1117	lag_upper_info.tx_type = team->mode->lag_tx_type;1118	lag_upper_info.hash_type = NETDEV_LAG_HASH_UNKNOWN;1119	err = netdev_master_upper_dev_link(port->dev, team->dev, NULL,1120					   &lag_upper_info, extack);1121	if (err)1122		return err;1123	port->dev->priv_flags |= IFF_TEAM_PORT;1124	return 0;1125}1126 1127static void team_upper_dev_unlink(struct team *team, struct team_port *port)1128{1129	netdev_upper_dev_unlink(port->dev, team->dev);1130	port->dev->priv_flags &= ~IFF_TEAM_PORT;1131}1132 1133static void __team_port_change_port_added(struct team_port *port, bool linkup);1134static int team_dev_type_check_change(struct net_device *dev,1135				      struct net_device *port_dev);1136 1137static int team_port_add(struct team *team, struct net_device *port_dev,1138			 struct netlink_ext_ack *extack)1139{1140	struct net_device *dev = team->dev;1141	struct team_port *port;1142	char *portname = port_dev->name;1143	int err;1144 1145	if (port_dev->flags & IFF_LOOPBACK) {1146		NL_SET_ERR_MSG(extack, "Loopback device can't be added as a team port");1147		netdev_err(dev, "Device %s is loopback device. Loopback devices can't be added as a team port\n",1148			   portname);1149		return -EINVAL;1150	}1151 1152	if (netif_is_team_port(port_dev)) {1153		NL_SET_ERR_MSG(extack, "Device is already a port of a team device");1154		netdev_err(dev, "Device %s is already a port "1155				"of a team device\n", portname);1156		return -EBUSY;1157	}1158 1159	if (dev == port_dev) {1160		NL_SET_ERR_MSG(extack, "Cannot enslave team device to itself");1161		netdev_err(dev, "Cannot enslave team device to itself\n");1162		return -EINVAL;1163	}1164 1165	if (netdev_has_upper_dev(dev, port_dev)) {1166		NL_SET_ERR_MSG(extack, "Device is already an upper device of the team interface");1167		netdev_err(dev, "Device %s is already an upper device of the team interface\n",1168			   portname);1169		return -EBUSY;1170	}1171 1172	if (port_dev->features & NETIF_F_VLAN_CHALLENGED &&1173	    vlan_uses_dev(dev)) {1174		NL_SET_ERR_MSG(extack, "Device is VLAN challenged and team device has VLAN set up");1175		netdev_err(dev, "Device %s is VLAN challenged and team device has VLAN set up\n",1176			   portname);1177		return -EPERM;1178	}1179 1180	err = team_dev_type_check_change(dev, port_dev);1181	if (err)1182		return err;1183 1184	if (port_dev->flags & IFF_UP) {1185		NL_SET_ERR_MSG(extack, "Device is up. Set it down before adding it as a team port");1186		netdev_err(dev, "Device %s is up. Set it down before adding it as a team port\n",1187			   portname);1188		return -EBUSY;1189	}1190 1191	port = kzalloc(sizeof(struct team_port) + team->mode->port_priv_size,1192		       GFP_KERNEL);1193	if (!port)1194		return -ENOMEM;1195 1196	port->dev = port_dev;1197	port->team = team;1198	INIT_LIST_HEAD(&port->qom_list);1199 1200	port->orig.mtu = port_dev->mtu;1201	err = dev_set_mtu(port_dev, dev->mtu);1202	if (err) {1203		netdev_dbg(dev, "Error %d calling dev_set_mtu\n", err);1204		goto err_set_mtu;1205	}1206 1207	memcpy(port->orig.dev_addr, port_dev->dev_addr, port_dev->addr_len);1208 1209	err = team_port_enter(team, port);1210	if (err) {1211		netdev_err(dev, "Device %s failed to enter team mode\n",1212			   portname);1213		goto err_port_enter;1214	}1215 1216	err = dev_open(port_dev, extack);1217	if (err) {1218		netdev_dbg(dev, "Device %s opening failed\n",1219			   portname);1220		goto err_dev_open;1221	}1222 1223	err = vlan_vids_add_by_dev(port_dev, dev);1224	if (err) {1225		netdev_err(dev, "Failed to add vlan ids to device %s\n",1226				portname);1227		goto err_vids_add;1228	}1229 1230	err = team_port_enable_netpoll(port);1231	if (err) {1232		netdev_err(dev, "Failed to enable netpoll on device %s\n",1233			   portname);1234		goto err_enable_netpoll;1235	}1236 1237	if (!(dev->features & NETIF_F_LRO))1238		dev_disable_lro(port_dev);1239 1240	err = netdev_rx_handler_register(port_dev, team_handle_frame,1241					 port);1242	if (err) {1243		netdev_err(dev, "Device %s failed to register rx_handler\n",1244			   portname);1245		goto err_handler_register;1246	}1247 1248	err = team_upper_dev_link(team, port, extack);1249	if (err) {1250		netdev_err(dev, "Device %s failed to set upper link\n",1251			   portname);1252		goto err_set_upper_link;1253	}1254 1255	err = __team_option_inst_add_port(team, port);1256	if (err) {1257		netdev_err(dev, "Device %s failed to add per-port options\n",1258			   portname);1259		goto err_option_port_add;1260	}1261 1262	/* set promiscuity level to new slave */1263	if (dev->flags & IFF_PROMISC) {1264		err = dev_set_promiscuity(port_dev, 1);1265		if (err)1266			goto err_set_slave_promisc;1267	}1268 1269	/* set allmulti level to new slave */1270	if (dev->flags & IFF_ALLMULTI) {1271		err = dev_set_allmulti(port_dev, 1);1272		if (err) {1273			if (dev->flags & IFF_PROMISC)1274				dev_set_promiscuity(port_dev, -1);1275			goto err_set_slave_promisc;1276		}1277	}1278 1279	if (dev->flags & IFF_UP) {1280		netif_addr_lock_bh(dev);1281		dev_uc_sync_multiple(port_dev, dev);1282		dev_mc_sync_multiple(port_dev, dev);1283		netif_addr_unlock_bh(dev);1284	}1285 1286	port->index = -1;1287	list_add_tail_rcu(&port->list, &team->port_list);1288	team_port_enable(team, port);1289	__team_compute_features(team);1290	__team_port_change_port_added(port, !!netif_oper_up(port_dev));1291	__team_options_change_check(team);1292 1293	netdev_info(dev, "Port device %s added\n", portname);1294 1295	return 0;1296 1297err_set_slave_promisc:1298	__team_option_inst_del_port(team, port);1299 1300err_option_port_add:1301	team_upper_dev_unlink(team, port);1302 1303err_set_upper_link:1304	netdev_rx_handler_unregister(port_dev);1305 1306err_handler_register:1307	team_port_disable_netpoll(port);1308 1309err_enable_netpoll:1310	vlan_vids_del_by_dev(port_dev, dev);1311 1312err_vids_add:1313	dev_close(port_dev);1314 1315err_dev_open:1316	team_port_leave(team, port);1317	team_port_set_orig_dev_addr(port);1318 1319err_port_enter:1320	dev_set_mtu(port_dev, port->orig.mtu);1321 1322err_set_mtu:1323	kfree(port);1324 1325	return err;1326}1327 1328static void __team_port_change_port_removed(struct team_port *port);1329 1330static int team_port_del(struct team *team, struct net_device *port_dev)1331{1332	struct net_device *dev = team->dev;1333	struct team_port *port;1334	char *portname = port_dev->name;1335 1336	port = team_port_get_rtnl(port_dev);1337	if (!port || !team_port_find(team, port)) {1338		netdev_err(dev, "Device %s does not act as a port of this team\n",1339			   portname);1340		return -ENOENT;1341	}1342 1343	team_port_disable(team, port);1344	list_del_rcu(&port->list);1345 1346	if (dev->flags & IFF_PROMISC)1347		dev_set_promiscuity(port_dev, -1);1348	if (dev->flags & IFF_ALLMULTI)1349		dev_set_allmulti(port_dev, -1);1350 1351	team_upper_dev_unlink(team, port);1352	netdev_rx_handler_unregister(port_dev);1353	team_port_disable_netpoll(port);1354	vlan_vids_del_by_dev(port_dev, dev);1355	if (dev->flags & IFF_UP) {1356		dev_uc_unsync(port_dev, dev);1357		dev_mc_unsync(port_dev, dev);1358	}1359	dev_close(port_dev);1360	team_port_leave(team, port);1361 1362	__team_option_inst_mark_removed_port(team, port);1363	__team_options_change_check(team);1364	__team_option_inst_del_port(team, port);1365	__team_port_change_port_removed(port);1366 1367	team_port_set_orig_dev_addr(port);1368	dev_set_mtu(port_dev, port->orig.mtu);1369	kfree_rcu(port, rcu);1370	netdev_info(dev, "Port device %s removed\n", portname);1371	__team_compute_features(team);1372 1373	return 0;1374}1375 1376 1377/*****************1378 * Net device ops1379 *****************/1380 1381static void team_mode_option_get(struct team *team, struct team_gsetter_ctx *ctx)1382{1383	ctx->data.str_val = team->mode->kind;1384}1385 1386static int team_mode_option_set(struct team *team, struct team_gsetter_ctx *ctx)1387{1388	return team_change_mode(team, ctx->data.str_val);1389}1390 1391static void team_notify_peers_count_get(struct team *team,1392					struct team_gsetter_ctx *ctx)1393{1394	ctx->data.u32_val = team->notify_peers.count;1395}1396 1397static int team_notify_peers_count_set(struct team *team,1398				       struct team_gsetter_ctx *ctx)1399{1400	team->notify_peers.count = ctx->data.u32_val;1401	return 0;1402}1403 1404static void team_notify_peers_interval_get(struct team *team,1405					   struct team_gsetter_ctx *ctx)1406{1407	ctx->data.u32_val = team->notify_peers.interval;1408}1409 1410static int team_notify_peers_interval_set(struct team *team,1411					  struct team_gsetter_ctx *ctx)1412{1413	team->notify_peers.interval = ctx->data.u32_val;1414	return 0;1415}1416 1417static void team_mcast_rejoin_count_get(struct team *team,1418					struct team_gsetter_ctx *ctx)1419{1420	ctx->data.u32_val = team->mcast_rejoin.count;1421}1422 1423static int team_mcast_rejoin_count_set(struct team *team,1424				       struct team_gsetter_ctx *ctx)1425{1426	team->mcast_rejoin.count = ctx->data.u32_val;1427	return 0;1428}1429 1430static void team_mcast_rejoin_interval_get(struct team *team,1431					   struct team_gsetter_ctx *ctx)1432{1433	ctx->data.u32_val = team->mcast_rejoin.interval;1434}1435 1436static int team_mcast_rejoin_interval_set(struct team *team,1437					  struct team_gsetter_ctx *ctx)1438{1439	team->mcast_rejoin.interval = ctx->data.u32_val;1440	return 0;1441}1442 1443static void team_port_en_option_get(struct team *team,1444				    struct team_gsetter_ctx *ctx)1445{1446	struct team_port *port = ctx->info->port;1447 1448	ctx->data.bool_val = team_port_enabled(port);1449}1450 1451static int team_port_en_option_set(struct team *team,1452				   struct team_gsetter_ctx *ctx)1453{1454	struct team_port *port = ctx->info->port;1455 1456	if (ctx->data.bool_val)1457		team_port_enable(team, port);1458	else1459		team_port_disable(team, port);1460	return 0;1461}1462 1463static void team_user_linkup_option_get(struct team *team,1464					struct team_gsetter_ctx *ctx)1465{1466	struct team_port *port = ctx->info->port;1467 1468	ctx->data.bool_val = port->user.linkup;1469}1470 1471static void __team_carrier_check(struct team *team);1472 1473static int team_user_linkup_option_set(struct team *team,1474				       struct team_gsetter_ctx *ctx)1475{1476	struct team_port *port = ctx->info->port;1477 1478	port->user.linkup = ctx->data.bool_val;1479	team_refresh_port_linkup(port);1480	__team_carrier_check(port->team);1481	return 0;1482}1483 1484static void team_user_linkup_en_option_get(struct team *team,1485					   struct team_gsetter_ctx *ctx)1486{1487	struct team_port *port = ctx->info->port;1488 1489	ctx->data.bool_val = port->user.linkup_enabled;1490}1491 1492static int team_user_linkup_en_option_set(struct team *team,1493					  struct team_gsetter_ctx *ctx)1494{1495	struct team_port *port = ctx->info->port;1496 1497	port->user.linkup_enabled = ctx->data.bool_val;1498	team_refresh_port_linkup(port);1499	__team_carrier_check(port->team);1500	return 0;1501}1502 1503static void team_priority_option_get(struct team *team,1504				     struct team_gsetter_ctx *ctx)1505{1506	struct team_port *port = ctx->info->port;1507 1508	ctx->data.s32_val = port->priority;1509}1510 1511static int team_priority_option_set(struct team *team,1512				    struct team_gsetter_ctx *ctx)1513{1514	struct team_port *port = ctx->info->port;1515	s32 priority = ctx->data.s32_val;1516 1517	if (port->priority == priority)1518		return 0;1519	port->priority = priority;1520	team_queue_override_port_prio_changed(team, port);1521	return 0;1522}1523 1524static void team_queue_id_option_get(struct team *team,1525				     struct team_gsetter_ctx *ctx)1526{1527	struct team_port *port = ctx->info->port;1528 1529	ctx->data.u32_val = port->queue_id;1530}1531 1532static int team_queue_id_option_set(struct team *team,1533				    struct team_gsetter_ctx *ctx)1534{1535	struct team_port *port = ctx->info->port;1536	u16 new_queue_id = ctx->data.u32_val;1537 1538	if (port->queue_id == new_queue_id)1539		return 0;1540	if (new_queue_id >= team->dev->real_num_tx_queues)1541		return -EINVAL;1542	team_queue_override_port_change_queue_id(team, port, new_queue_id);1543	return 0;1544}1545 1546static const struct team_option team_options[] = {1547	{1548		.name = "mode",1549		.type = TEAM_OPTION_TYPE_STRING,1550		.getter = team_mode_option_get,1551		.setter = team_mode_option_set,1552	},1553	{1554		.name = "notify_peers_count",1555		.type = TEAM_OPTION_TYPE_U32,1556		.getter = team_notify_peers_count_get,1557		.setter = team_notify_peers_count_set,1558	},1559	{1560		.name = "notify_peers_interval",1561		.type = TEAM_OPTION_TYPE_U32,1562		.getter = team_notify_peers_interval_get,1563		.setter = team_notify_peers_interval_set,1564	},1565	{1566		.name = "mcast_rejoin_count",1567		.type = TEAM_OPTION_TYPE_U32,1568		.getter = team_mcast_rejoin_count_get,1569		.setter = team_mcast_rejoin_count_set,1570	},1571	{1572		.name = "mcast_rejoin_interval",1573		.type = TEAM_OPTION_TYPE_U32,1574		.getter = team_mcast_rejoin_interval_get,1575		.setter = team_mcast_rejoin_interval_set,1576	},1577	{1578		.name = "enabled",1579		.type = TEAM_OPTION_TYPE_BOOL,1580		.per_port = true,1581		.getter = team_port_en_option_get,1582		.setter = team_port_en_option_set,1583	},1584	{1585		.name = "user_linkup",1586		.type = TEAM_OPTION_TYPE_BOOL,1587		.per_port = true,1588		.getter = team_user_linkup_option_get,1589		.setter = team_user_linkup_option_set,1590	},1591	{1592		.name = "user_linkup_enabled",1593		.type = TEAM_OPTION_TYPE_BOOL,1594		.per_port = true,1595		.getter = team_user_linkup_en_option_get,1596		.setter = team_user_linkup_en_option_set,1597	},1598	{1599		.name = "priority",1600		.type = TEAM_OPTION_TYPE_S32,1601		.per_port = true,1602		.getter = team_priority_option_get,1603		.setter = team_priority_option_set,1604	},1605	{1606		.name = "queue_id",1607		.type = TEAM_OPTION_TYPE_U32,1608		.per_port = true,1609		.getter = team_queue_id_option_get,1610		.setter = team_queue_id_option_set,1611	},1612};1613 1614 1615static int team_init(struct net_device *dev)1616{1617	struct team *team = netdev_priv(dev);1618	int i;1619	int err;1620 1621	team->dev = dev;1622	team_set_no_mode(team);1623	team->notifier_ctx = false;1624 1625	team->pcpu_stats = netdev_alloc_pcpu_stats(struct team_pcpu_stats);1626	if (!team->pcpu_stats)1627		return -ENOMEM;1628 1629	for (i = 0; i < TEAM_PORT_HASHENTRIES; i++)1630		INIT_HLIST_HEAD(&team->en_port_hlist[i]);1631	INIT_LIST_HEAD(&team->port_list);1632	err = team_queue_override_init(team);1633	if (err)1634		goto err_team_queue_override_init;1635 1636	team_adjust_ops(team);1637 1638	INIT_LIST_HEAD(&team->option_list);1639	INIT_LIST_HEAD(&team->option_inst_list);1640 1641	team_notify_peers_init(team);1642	team_mcast_rejoin_init(team);1643 1644	err = team_options_register(team, team_options, ARRAY_SIZE(team_options));1645	if (err)1646		goto err_options_register;1647	netif_carrier_off(dev);1648 1649	lockdep_register_key(&team->team_lock_key);1650	__mutex_init(&team->lock, "team->team_lock_key", &team->team_lock_key);1651	netdev_lockdep_set_classes(dev);1652 1653	return 0;1654 1655err_options_register:1656	team_mcast_rejoin_fini(team);1657	team_notify_peers_fini(team);1658	team_queue_override_fini(team);1659err_team_queue_override_init:1660	free_percpu(team->pcpu_stats);1661 1662	return err;1663}1664 1665static void team_uninit(struct net_device *dev)1666{1667	struct team *team = netdev_priv(dev);1668	struct team_port *port;1669	struct team_port *tmp;1670 1671	mutex_lock(&team->lock);1672	list_for_each_entry_safe(port, tmp, &team->port_list, list)1673		team_port_del(team, port->dev);1674 1675	__team_change_mode(team, NULL); /* cleanup */1676	__team_options_unregister(team, team_options, ARRAY_SIZE(team_options));1677	team_mcast_rejoin_fini(team);1678	team_notify_peers_fini(team);1679	team_queue_override_fini(team);1680	mutex_unlock(&team->lock);1681	netdev_change_features(dev);1682	lockdep_unregister_key(&team->team_lock_key);1683}1684 1685static void team_destructor(struct net_device *dev)1686{1687	struct team *team = netdev_priv(dev);1688 1689	free_percpu(team->pcpu_stats);1690}1691 1692static int team_open(struct net_device *dev)1693{1694	return 0;1695}1696 1697static int team_close(struct net_device *dev)1698{1699	struct team *team = netdev_priv(dev);1700	struct team_port *port;1701 1702	list_for_each_entry(port, &team->port_list, list) {1703		dev_uc_unsync(port->dev, dev);1704		dev_mc_unsync(port->dev, dev);1705	}1706 1707	return 0;1708}1709 1710/*1711 * note: already called with rcu_read_lock1712 */1713static netdev_tx_t team_xmit(struct sk_buff *skb, struct net_device *dev)1714{1715	struct team *team = netdev_priv(dev);1716	bool tx_success;1717	unsigned int len = skb->len;1718 1719	tx_success = team_queue_override_transmit(team, skb);1720	if (!tx_success)1721		tx_success = team->ops.transmit(team, skb);1722	if (tx_success) {1723		struct team_pcpu_stats *pcpu_stats;1724 1725		pcpu_stats = this_cpu_ptr(team->pcpu_stats);1726		u64_stats_update_begin(&pcpu_stats->syncp);1727		u64_stats_inc(&pcpu_stats->tx_packets);1728		u64_stats_add(&pcpu_stats->tx_bytes, len);1729		u64_stats_update_end(&pcpu_stats->syncp);1730	} else {1731		this_cpu_inc(team->pcpu_stats->tx_dropped);1732	}1733 1734	return NETDEV_TX_OK;1735}1736 1737static u16 team_select_queue(struct net_device *dev, struct sk_buff *skb,1738			     struct net_device *sb_dev)1739{1740	/*1741	 * This helper function exists to help dev_pick_tx get the correct1742	 * destination queue.  Using a helper function skips a call to1743	 * skb_tx_hash and will put the skbs in the queue we expect on their1744	 * way down to the team driver.1745	 */1746	u16 txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) : 0;1747 1748	/*1749	 * Save the original txq to restore before passing to the driver1750	 */1751	qdisc_skb_cb(skb)->slave_dev_queue_mapping = skb->queue_mapping;1752 1753	if (unlikely(txq >= dev->real_num_tx_queues)) {1754		do {1755			txq -= dev->real_num_tx_queues;1756		} while (txq >= dev->real_num_tx_queues);1757	}1758	return txq;1759}1760 1761static void team_change_rx_flags(struct net_device *dev, int change)1762{1763	struct team *team = netdev_priv(dev);1764	struct team_port *port;1765	int inc;1766 1767	rcu_read_lock();1768	list_for_each_entry_rcu(port, &team->port_list, list) {1769		if (change & IFF_PROMISC) {1770			inc = dev->flags & IFF_PROMISC ? 1 : -1;1771			dev_set_promiscuity(port->dev, inc);1772		}1773		if (change & IFF_ALLMULTI) {1774			inc = dev->flags & IFF_ALLMULTI ? 1 : -1;1775			dev_set_allmulti(port->dev, inc);1776		}1777	}1778	rcu_read_unlock();1779}1780 1781static void team_set_rx_mode(struct net_device *dev)1782{1783	struct team *team = netdev_priv(dev);1784	struct team_port *port;1785 1786	rcu_read_lock();1787	list_for_each_entry_rcu(port, &team->port_list, list) {1788		dev_uc_sync_multiple(port->dev, dev);1789		dev_mc_sync_multiple(port->dev, dev);1790	}1791	rcu_read_unlock();1792}1793 1794static int team_set_mac_address(struct net_device *dev, void *p)1795{1796	struct sockaddr *addr = p;1797	struct team *team = netdev_priv(dev);1798	struct team_port *port;1799 1800	if (dev->type == ARPHRD_ETHER && !is_valid_ether_addr(addr->sa_data))1801		return -EADDRNOTAVAIL;1802	dev_addr_set(dev, addr->sa_data);1803	mutex_lock(&team->lock);1804	list_for_each_entry(port, &team->port_list, list)1805		if (team->ops.port_change_dev_addr)1806			team->ops.port_change_dev_addr(team, port);1807	mutex_unlock(&team->lock);1808	return 0;1809}1810 1811static int team_change_mtu(struct net_device *dev, int new_mtu)1812{1813	struct team *team = netdev_priv(dev);1814	struct team_port *port;1815	int err;1816 1817	/*1818	 * Alhough this is reader, it's guarded by team lock. It's not possible1819	 * to traverse list in reverse under rcu_read_lock1820	 */1821	mutex_lock(&team->lock);1822	team->port_mtu_change_allowed = true;1823	list_for_each_entry(port, &team->port_list, list) {1824		err = dev_set_mtu(port->dev, new_mtu);1825		if (err) {1826			netdev_err(dev, "Device %s failed to change mtu",1827				   port->dev->name);1828			goto unwind;1829		}1830	}1831	team->port_mtu_change_allowed = false;1832	mutex_unlock(&team->lock);1833 1834	WRITE_ONCE(dev->mtu, new_mtu);1835 1836	return 0;1837 1838unwind:1839	list_for_each_entry_continue_reverse(port, &team->port_list, list)1840		dev_set_mtu(port->dev, dev->mtu);1841	team->port_mtu_change_allowed = false;1842	mutex_unlock(&team->lock);1843 1844	return err;1845}1846 1847static void1848team_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)1849{1850	struct team *team = netdev_priv(dev);1851	struct team_pcpu_stats *p;1852	u64 rx_packets, rx_bytes, rx_multicast, tx_packets, tx_bytes;1853	u32 rx_dropped = 0, tx_dropped = 0, rx_nohandler = 0;1854	unsigned int start;1855	int i;1856 1857	for_each_possible_cpu(i) {1858		p = per_cpu_ptr(team->pcpu_stats, i);1859		do {1860			start = u64_stats_fetch_begin(&p->syncp);1861			rx_packets	= u64_stats_read(&p->rx_packets);1862			rx_bytes	= u64_stats_read(&p->rx_bytes);1863			rx_multicast	= u64_stats_read(&p->rx_multicast);1864			tx_packets	= u64_stats_read(&p->tx_packets);1865			tx_bytes	= u64_stats_read(&p->tx_bytes);1866		} while (u64_stats_fetch_retry(&p->syncp, start));1867 1868		stats->rx_packets	+= rx_packets;1869		stats->rx_bytes		+= rx_bytes;1870		stats->multicast	+= rx_multicast;1871		stats->tx_packets	+= tx_packets;1872		stats->tx_bytes		+= tx_bytes;1873		/*1874		 * rx_dropped, tx_dropped & rx_nohandler are u32,1875		 * updated without syncp protection.1876		 */1877		rx_dropped	+= READ_ONCE(p->rx_dropped);1878		tx_dropped	+= READ_ONCE(p->tx_dropped);1879		rx_nohandler	+= READ_ONCE(p->rx_nohandler);1880	}1881	stats->rx_dropped	= rx_dropped;1882	stats->tx_dropped	= tx_dropped;1883	stats->rx_nohandler	= rx_nohandler;1884}1885 1886static int team_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid)1887{1888	struct team *team = netdev_priv(dev);1889	struct team_port *port;1890	int err;1891 1892	/*1893	 * Alhough this is reader, it's guarded by team lock. It's not possible1894	 * to traverse list in reverse under rcu_read_lock1895	 */1896	mutex_lock(&team->lock);1897	list_for_each_entry(port, &team->port_list, list) {1898		err = vlan_vid_add(port->dev, proto, vid);1899		if (err)1900			goto unwind;1901	}1902	mutex_unlock(&team->lock);1903 1904	return 0;1905 1906unwind:1907	list_for_each_entry_continue_reverse(port, &team->port_list, list)1908		vlan_vid_del(port->dev, proto, vid);1909	mutex_unlock(&team->lock);1910 1911	return err;1912}1913 1914static int team_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid)1915{1916	struct team *team = netdev_priv(dev);1917	struct team_port *port;1918 1919	mutex_lock(&team->lock);1920	list_for_each_entry(port, &team->port_list, list)1921		vlan_vid_del(port->dev, proto, vid);1922	mutex_unlock(&team->lock);1923 1924	return 0;1925}1926 1927#ifdef CONFIG_NET_POLL_CONTROLLER1928static void team_poll_controller(struct net_device *dev)1929{1930}1931 1932static void __team_netpoll_cleanup(struct team *team)1933{1934	struct team_port *port;1935 1936	list_for_each_entry(port, &team->port_list, list)1937		team_port_disable_netpoll(port);1938}1939 1940static void team_netpoll_cleanup(struct net_device *dev)1941{1942	struct team *team = netdev_priv(dev);1943 1944	mutex_lock(&team->lock);1945	__team_netpoll_cleanup(team);1946	mutex_unlock(&team->lock);1947}1948 1949static int team_netpoll_setup(struct net_device *dev,1950			      struct netpoll_info *npifo)1951{1952	struct team *team = netdev_priv(dev);1953	struct team_port *port;1954	int err = 0;1955 1956	mutex_lock(&team->lock);1957	list_for_each_entry(port, &team->port_list, list) {1958		err = __team_port_enable_netpoll(port);1959		if (err) {1960			__team_netpoll_cleanup(team);1961			break;1962		}1963	}1964	mutex_unlock(&team->lock);1965	return err;1966}1967#endif1968 1969static int team_add_slave(struct net_device *dev, struct net_device *port_dev,1970			  struct netlink_ext_ack *extack)1971{1972	struct team *team = netdev_priv(dev);1973	int err;1974 1975	mutex_lock(&team->lock);1976	err = team_port_add(team, port_dev, extack);1977	mutex_unlock(&team->lock);1978 1979	if (!err)1980		netdev_change_features(dev);1981 1982	return err;1983}1984 1985static int team_del_slave(struct net_device *dev, struct net_device *port_dev)1986{1987	struct team *team = netdev_priv(dev);1988	int err;1989 1990	mutex_lock(&team->lock);1991	err = team_port_del(team, port_dev);1992	mutex_unlock(&team->lock);1993 1994	if (err)1995		return err;1996 1997	if (netif_is_team_master(port_dev)) {1998		lockdep_unregister_key(&team->team_lock_key);1999		lockdep_register_key(&team->team_lock_key);2000		lockdep_set_class(&team->lock, &team->team_lock_key);2001	}2002	netdev_change_features(dev);2003 2004	return err;2005}2006 2007static netdev_features_t team_fix_features(struct net_device *dev,2008					   netdev_features_t features)2009{2010	struct team_port *port;2011	struct team *team = netdev_priv(dev);2012	netdev_features_t mask;2013 2014	mask = features;2015	features &= ~NETIF_F_ONE_FOR_ALL;2016	features |= NETIF_F_ALL_FOR_ALL;2017 2018	rcu_read_lock();2019	list_for_each_entry_rcu(port, &team->port_list, list) {2020		features = netdev_increment_features(features,2021						     port->dev->features,2022						     mask);2023	}2024	rcu_read_unlock();2025 2026	features = netdev_add_tso_features(features, mask);2027 2028	return features;2029}2030 2031static int team_change_carrier(struct net_device *dev, bool new_carrier)2032{2033	struct team *team = netdev_priv(dev);2034 2035	team->user_carrier_enabled = true;2036 2037	if (new_carrier)2038		netif_carrier_on(dev);2039	else2040		netif_carrier_off(dev);2041	return 0;2042}2043 2044static const struct net_device_ops team_netdev_ops = {2045	.ndo_init		= team_init,2046	.ndo_uninit		= team_uninit,2047	.ndo_open		= team_open,2048	.ndo_stop		= team_close,2049	.ndo_start_xmit		= team_xmit,2050	.ndo_select_queue	= team_select_queue,2051	.ndo_change_rx_flags	= team_change_rx_flags,2052	.ndo_set_rx_mode	= team_set_rx_mode,2053	.ndo_set_mac_address	= team_set_mac_address,2054	.ndo_change_mtu		= team_change_mtu,2055	.ndo_get_stats64	= team_get_stats64,2056	.ndo_vlan_rx_add_vid	= team_vlan_rx_add_vid,2057	.ndo_vlan_rx_kill_vid	= team_vlan_rx_kill_vid,2058#ifdef CONFIG_NET_POLL_CONTROLLER2059	.ndo_poll_controller	= team_poll_controller,2060	.ndo_netpoll_setup	= team_netpoll_setup,2061	.ndo_netpoll_cleanup	= team_netpoll_cleanup,2062#endif2063	.ndo_add_slave		= team_add_slave,2064	.ndo_del_slave		= team_del_slave,2065	.ndo_fix_features	= team_fix_features,2066	.ndo_change_carrier     = team_change_carrier,2067	.ndo_features_check	= passthru_features_check,2068};2069 2070/***********************2071 * ethtool interface2072 ***********************/2073 2074static void team_ethtool_get_drvinfo(struct net_device *dev,2075				     struct ethtool_drvinfo *drvinfo)2076{2077	strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));2078}2079 2080static int team_ethtool_get_link_ksettings(struct net_device *dev,2081					   struct ethtool_link_ksettings *cmd)2082{2083	struct team *team= netdev_priv(dev);2084	unsigned long speed = 0;2085	struct team_port *port;2086 2087	cmd->base.duplex = DUPLEX_UNKNOWN;2088	cmd->base.port = PORT_OTHER;2089 2090	rcu_read_lock();2091	list_for_each_entry_rcu(port, &team->port_list, list) {2092		if (team_port_txable(port)) {2093			if (port->state.speed != SPEED_UNKNOWN)2094				speed += port->state.speed;2095			if (cmd->base.duplex == DUPLEX_UNKNOWN &&2096			    port->state.duplex != DUPLEX_UNKNOWN)2097				cmd->base.duplex = port->state.duplex;2098		}2099	}2100	rcu_read_unlock();2101 2102	cmd->base.speed = speed ? : SPEED_UNKNOWN;2103 2104	return 0;2105}2106 2107static const struct ethtool_ops team_ethtool_ops = {2108	.get_drvinfo		= team_ethtool_get_drvinfo,2109	.get_link		= ethtool_op_get_link,2110	.get_link_ksettings	= team_ethtool_get_link_ksettings,2111};2112 2113/***********************2114 * rt netlink interface2115 ***********************/2116 2117static void team_setup_by_port(struct net_device *dev,2118			       struct net_device *port_dev)2119{2120	struct team *team = netdev_priv(dev);2121 2122	if (port_dev->type == ARPHRD_ETHER)2123		dev->header_ops	= team->header_ops_cache;2124	else2125		dev->header_ops	= port_dev->header_ops;2126	dev->type = port_dev->type;2127	dev->hard_header_len = port_dev->hard_header_len;2128	dev->needed_headroom = port_dev->needed_headroom;2129	dev->addr_len = port_dev->addr_len;2130	dev->mtu = port_dev->mtu;2131	memcpy(dev->broadcast, port_dev->broadcast, port_dev->addr_len);2132	eth_hw_addr_inherit(dev, port_dev);2133 2134	if (port_dev->flags & IFF_POINTOPOINT) {2135		dev->flags &= ~(IFF_BROADCAST | IFF_MULTICAST);2136		dev->flags |= (IFF_POINTOPOINT | IFF_NOARP);2137	} else if ((port_dev->flags & (IFF_BROADCAST | IFF_MULTICAST)) ==2138		    (IFF_BROADCAST | IFF_MULTICAST)) {2139		dev->flags |= (IFF_BROADCAST | IFF_MULTICAST);2140		dev->flags &= ~(IFF_POINTOPOINT | IFF_NOARP);2141	}2142}2143 2144static int team_dev_type_check_change(struct net_device *dev,2145				      struct net_device *port_dev)2146{2147	struct team *team = netdev_priv(dev);2148	char *portname = port_dev->name;2149	int err;2150 2151	if (dev->type == port_dev->type)2152		return 0;2153	if (!list_empty(&team->port_list)) {2154		netdev_err(dev, "Device %s is of different type\n", portname);2155		return -EBUSY;2156	}2157	err = call_netdevice_notifiers(NETDEV_PRE_TYPE_CHANGE, dev);2158	err = notifier_to_errno(err);2159	if (err) {2160		netdev_err(dev, "Refused to change device type\n");2161		return err;2162	}2163	dev_uc_flush(dev);2164	dev_mc_flush(dev);2165	team_setup_by_port(dev, port_dev);2166	call_netdevice_notifiers(NETDEV_POST_TYPE_CHANGE, dev);2167	return 0;2168}2169 2170static void team_setup(struct net_device *dev)2171{2172	struct team *team = netdev_priv(dev);2173 2174	ether_setup(dev);2175	dev->max_mtu = ETH_MAX_MTU;2176	team->header_ops_cache = dev->header_ops;2177 2178	dev->netdev_ops = &team_netdev_ops;2179	dev->ethtool_ops = &team_ethtool_ops;2180	dev->needs_free_netdev = true;2181	dev->priv_destructor = team_destructor;2182	dev->priv_flags &= ~(IFF_XMIT_DST_RELEASE | IFF_TX_SKB_SHARING);2183	dev->priv_flags |= IFF_NO_QUEUE;2184	dev->priv_flags |= IFF_TEAM;2185 2186	/*2187	 * Indicate we support unicast address filtering. That way core won't2188	 * bring us to promisc mode in case a unicast addr is added.2189	 * Let this up to underlay drivers.2190	 */2191	dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE;2192	dev->lltx = true;2193 2194	/* Don't allow team devices to change network namespaces. */2195	dev->netns_local = true;2196 2197	dev->features |= NETIF_F_GRO;2198 2199	dev->hw_features = TEAM_VLAN_FEATURES |2200			   NETIF_F_HW_VLAN_CTAG_RX |2201			   NETIF_F_HW_VLAN_CTAG_FILTER |2202			   NETIF_F_HW_VLAN_STAG_RX |2203			   NETIF_F_HW_VLAN_STAG_FILTER;2204 2205	dev->hw_features |= NETIF_F_GSO_ENCAP_ALL;2206	dev->features |= dev->hw_features;2207	dev->features |= NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_STAG_TX;2208}2209 2210static int team_newlink(struct net *src_net, struct net_device *dev,2211			struct nlattr *tb[], struct nlattr *data[],2212			struct netlink_ext_ack *extack)2213{2214	if (tb[IFLA_ADDRESS] == NULL)2215		eth_hw_addr_random(dev);2216 2217	return register_netdevice(dev);2218}2219 2220static int team_validate(struct nlattr *tb[], struct nlattr *data[],2221			 struct netlink_ext_ack *extack)2222{2223	if (tb[IFLA_ADDRESS]) {2224		if (nla_len(tb[IFLA_ADDRESS]) != ETH_ALEN)2225			return -EINVAL;2226		if (!is_valid_ether_addr(nla_data(tb[IFLA_ADDRESS])))2227			return -EADDRNOTAVAIL;2228	}2229	return 0;2230}2231 2232static unsigned int team_get_num_tx_queues(void)2233{2234	return TEAM_DEFAULT_NUM_TX_QUEUES;2235}2236 2237static unsigned int team_get_num_rx_queues(void)2238{2239	return TEAM_DEFAULT_NUM_RX_QUEUES;2240}2241 2242static struct rtnl_link_ops team_link_ops __read_mostly = {2243	.kind			= DRV_NAME,2244	.priv_size		= sizeof(struct team),2245	.setup			= team_setup,2246	.newlink		= team_newlink,2247	.validate		= team_validate,2248	.get_num_tx_queues	= team_get_num_tx_queues,2249	.get_num_rx_queues	= team_get_num_rx_queues,2250};2251 2252 2253/***********************************2254 * Generic netlink custom interface2255 ***********************************/2256 2257static struct genl_family team_nl_family;2258 2259int team_nl_noop_doit(struct sk_buff *skb, struct genl_info *info)2260{2261	struct sk_buff *msg;2262	void *hdr;2263	int err;2264 2265	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);2266	if (!msg)2267		return -ENOMEM;2268 2269	hdr = genlmsg_put(msg, info->snd_portid, info->snd_seq,2270			  &team_nl_family, 0, TEAM_CMD_NOOP);2271	if (!hdr) {2272		err = -EMSGSIZE;2273		goto err_msg_put;2274	}2275 2276	genlmsg_end(msg, hdr);2277 2278	return genlmsg_unicast(genl_info_net(info), msg, info->snd_portid);2279 2280err_msg_put:2281	nlmsg_free(msg);2282 2283	return err;2284}2285 2286/*2287 * Netlink cmd functions should be locked by following two functions.2288 * Since dev gets held here, that ensures dev won't disappear in between.2289 */2290static struct team *team_nl_team_get(struct genl_info *info)2291{2292	struct net *net = genl_info_net(info);2293	int ifindex;2294	struct net_device *dev;2295	struct team *team;2296 2297	if (!info->attrs[TEAM_ATTR_TEAM_IFINDEX])2298		return NULL;2299 2300	ifindex = nla_get_u32(info->attrs[TEAM_ATTR_TEAM_IFINDEX]);2301	dev = dev_get_by_index(net, ifindex);2302	if (!dev || dev->netdev_ops != &team_netdev_ops) {2303		dev_put(dev);2304		return NULL;2305	}2306 2307	team = netdev_priv(dev);2308	mutex_lock(&team->lock);2309	return team;2310}2311 2312static void team_nl_team_put(struct team *team)2313{2314	mutex_unlock(&team->lock);2315	dev_put(team->dev);2316}2317 2318typedef int team_nl_send_func_t(struct sk_buff *skb,2319				struct team *team, u32 portid);2320 2321static int team_nl_send_unicast(struct sk_buff *skb, struct team *team, u32 portid)2322{2323	return genlmsg_unicast(dev_net(team->dev), skb, portid);2324}2325 2326static int team_nl_fill_one_option_get(struct sk_buff *skb, struct team *team,2327				       struct team_option_inst *opt_inst)2328{2329	struct nlattr *option_item;2330	struct team_option *option = opt_inst->option;2331	struct team_option_inst_info *opt_inst_info = &opt_inst->info;2332	struct team_gsetter_ctx ctx;2333	int err;2334 2335	ctx.info = opt_inst_info;2336	err = team_option_get(team, opt_inst, &ctx);2337	if (err)2338		return err;2339 2340	option_item = nla_nest_start_noflag(skb, TEAM_ATTR_ITEM_OPTION);2341	if (!option_item)2342		return -EMSGSIZE;2343 2344	if (nla_put_string(skb, TEAM_ATTR_OPTION_NAME, option->name))2345		goto nest_cancel;2346	if (opt_inst_info->port &&2347	    nla_put_u32(skb, TEAM_ATTR_OPTION_PORT_IFINDEX,2348			opt_inst_info->port->dev->ifindex))2349		goto nest_cancel;2350	if (opt_inst->option->array_size &&2351	    nla_put_u32(skb, TEAM_ATTR_OPTION_ARRAY_INDEX,2352			opt_inst_info->array_index))2353		goto nest_cancel;2354 2355	switch (option->type) {2356	case TEAM_OPTION_TYPE_U32:2357		if (nla_put_u8(skb, TEAM_ATTR_OPTION_TYPE, NLA_U32))2358			goto nest_cancel;2359		if (nla_put_u32(skb, TEAM_ATTR_OPTION_DATA, ctx.data.u32_val))2360			goto nest_cancel;2361		break;2362	case TEAM_OPTION_TYPE_STRING:2363		if (nla_put_u8(skb, TEAM_ATTR_OPTION_TYPE, NLA_STRING))2364			goto nest_cancel;2365		if (nla_put_string(skb, TEAM_ATTR_OPTION_DATA,2366				   ctx.data.str_val))2367			goto nest_cancel;2368		break;2369	case TEAM_OPTION_TYPE_BINARY:2370		if (nla_put_u8(skb, TEAM_ATTR_OPTION_TYPE, NLA_BINARY))2371			goto nest_cancel;2372		if (nla_put(skb, TEAM_ATTR_OPTION_DATA, ctx.data.bin_val.len,2373			    ctx.data.bin_val.ptr))2374			goto nest_cancel;2375		break;2376	case TEAM_OPTION_TYPE_BOOL:2377		if (nla_put_u8(skb, TEAM_ATTR_OPTION_TYPE, NLA_FLAG))2378			goto nest_cancel;2379		if (ctx.data.bool_val &&2380		    nla_put_flag(skb, TEAM_ATTR_OPTION_DATA))2381			goto nest_cancel;2382		break;2383	case TEAM_OPTION_TYPE_S32:2384		if (nla_put_u8(skb, TEAM_ATTR_OPTION_TYPE, NLA_S32))2385			goto nest_cancel;2386		if (nla_put_s32(skb, TEAM_ATTR_OPTION_DATA, ctx.data.s32_val))2387			goto nest_cancel;2388		break;2389	default:2390		BUG();2391	}2392	if (opt_inst->removed && nla_put_flag(skb, TEAM_ATTR_OPTION_REMOVED))2393		goto nest_cancel;2394	if (opt_inst->changed) {2395		if (nla_put_flag(skb, TEAM_ATTR_OPTION_CHANGED))2396			goto nest_cancel;2397		opt_inst->changed = false;2398	}2399	nla_nest_end(skb, option_item);2400	return 0;2401 2402nest_cancel:2403	nla_nest_cancel(skb, option_item);2404	return -EMSGSIZE;2405}2406 2407static int __send_and_alloc_skb(struct sk_buff **pskb,2408				struct team *team, u32 portid,2409				team_nl_send_func_t *send_func)2410{2411	int err;2412 2413	if (*pskb) {2414		err = send_func(*pskb, team, portid);2415		if (err)2416			return err;2417	}2418	*pskb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);2419	if (!*pskb)2420		return -ENOMEM;2421	return 0;2422}2423 2424static int team_nl_send_options_get(struct team *team, u32 portid, u32 seq,2425				    int flags, team_nl_send_func_t *send_func,2426				    struct list_head *sel_opt_inst_list)2427{2428	struct nlattr *option_list;2429	struct nlmsghdr *nlh;2430	void *hdr;2431	struct team_option_inst *opt_inst;2432	int err;2433	struct sk_buff *skb = NULL;2434	bool incomplete;2435	int i;2436 2437	opt_inst = list_first_entry(sel_opt_inst_list,2438				    struct team_option_inst, tmp_list);2439 2440start_again:2441	err = __send_and_alloc_skb(&skb, team, portid, send_func);2442	if (err)2443		return err;2444 2445	hdr = genlmsg_put(skb, portid, seq, &team_nl_family, flags | NLM_F_MULTI,2446			  TEAM_CMD_OPTIONS_GET);2447	if (!hdr) {2448		nlmsg_free(skb);2449		return -EMSGSIZE;2450	}2451 2452	if (nla_put_u32(skb, TEAM_ATTR_TEAM_IFINDEX, team->dev->ifindex))2453		goto nla_put_failure;2454	option_list = nla_nest_start_noflag(skb, TEAM_ATTR_LIST_OPTION);2455	if (!option_list)2456		goto nla_put_failure;2457 2458	i = 0;2459	incomplete = false;2460	list_for_each_entry_from(opt_inst, sel_opt_inst_list, tmp_list) {2461		err = team_nl_fill_one_option_get(skb, team, opt_inst);2462		if (err) {2463			if (err == -EMSGSIZE) {2464				if (!i)2465					goto errout;2466				incomplete = true;2467				break;2468			}2469			goto errout;2470		}2471		i++;2472	}2473 2474	nla_nest_end(skb, option_list);2475	genlmsg_end(skb, hdr);2476	if (incomplete)2477		goto start_again;2478 2479send_done:2480	nlh = nlmsg_put(skb, portid, seq, NLMSG_DONE, 0, flags | NLM_F_MULTI);2481	if (!nlh) {2482		err = __send_and_alloc_skb(&skb, team, portid, send_func);2483		if (err)2484			return err;2485		goto send_done;2486	}2487 2488	return send_func(skb, team, portid);2489 2490nla_put_failure:2491	err = -EMSGSIZE;2492errout:2493	nlmsg_free(skb);2494	return err;2495}2496 2497int team_nl_options_get_doit(struct sk_buff *skb, struct genl_info *info)2498{2499	struct team *team;2500	struct team_option_inst *opt_inst;2501	int err;2502	LIST_HEAD(sel_opt_inst_list);2503 2504	team = team_nl_team_get(info);2505	if (!team)2506		return -EINVAL;2507 2508	list_for_each_entry(opt_inst, &team->option_inst_list, list)2509		list_add_tail(&opt_inst->tmp_list, &sel_opt_inst_list);2510	err = team_nl_send_options_get(team, info->snd_portid, info->snd_seq,2511				       NLM_F_ACK, team_nl_send_unicast,2512				       &sel_opt_inst_list);2513 2514	team_nl_team_put(team);2515 2516	return err;2517}2518 2519static int team_nl_send_event_options_get(struct team *team,2520					  struct list_head *sel_opt_inst_list);2521 2522int team_nl_options_set_doit(struct sk_buff *skb, struct genl_info *info)2523{2524	struct team *team;2525	int err = 0;2526	int i;2527	struct nlattr *nl_option;2528 2529	rtnl_lock();2530 2531	team = team_nl_team_get(info);2532	if (!team) {2533		err = -EINVAL;2534		goto rtnl_unlock;2535	}2536 2537	err = -EINVAL;2538	if (!info->attrs[TEAM_ATTR_LIST_OPTION]) {2539		err = -EINVAL;2540		goto team_put;2541	}2542 2543	nla_for_each_nested(nl_option, info->attrs[TEAM_ATTR_LIST_OPTION], i) {2544		struct nlattr *opt_attrs[TEAM_ATTR_OPTION_MAX + 1];2545		struct nlattr *attr;2546		struct nlattr *attr_data;2547		LIST_HEAD(opt_inst_list);2548		enum team_option_type opt_type;2549		int opt_port_ifindex = 0; /* != 0 for per-port options */2550		u32 opt_array_index = 0;2551		bool opt_is_array = false;2552		struct team_option_inst *opt_inst;2553		char *opt_name;2554		bool opt_found = false;2555 2556		if (nla_type(nl_option) != TEAM_ATTR_ITEM_OPTION) {2557			err = -EINVAL;2558			goto team_put;2559		}2560		err = nla_parse_nested_deprecated(opt_attrs,2561						  TEAM_ATTR_OPTION_MAX,2562						  nl_option,2563						  team_attr_option_nl_policy,2564						  info->extack);2565		if (err)2566			goto team_put;2567		if (!opt_attrs[TEAM_ATTR_OPTION_NAME] ||2568		    !opt_attrs[TEAM_ATTR_OPTION_TYPE]) {2569			err = -EINVAL;2570			goto team_put;2571		}2572		switch (nla_get_u8(opt_attrs[TEAM_ATTR_OPTION_TYPE])) {2573		case NLA_U32:2574			opt_type = TEAM_OPTION_TYPE_U32;2575			break;2576		case NLA_STRING:2577			opt_type = TEAM_OPTION_TYPE_STRING;2578			break;2579		case NLA_BINARY:2580			opt_type = TEAM_OPTION_TYPE_BINARY;2581			break;2582		case NLA_FLAG:2583			opt_type = TEAM_OPTION_TYPE_BOOL;2584			break;2585		case NLA_S32:2586			opt_type = TEAM_OPTION_TYPE_S32;2587			break;2588		default:2589			goto team_put;2590		}2591 2592		attr_data = opt_attrs[TEAM_ATTR_OPTION_DATA];2593		if (opt_type != TEAM_OPTION_TYPE_BOOL && !attr_data) {2594			err = -EINVAL;2595			goto team_put;2596		}2597 2598		opt_name = nla_data(opt_attrs[TEAM_ATTR_OPTION_NAME]);2599		attr = opt_attrs[TEAM_ATTR_OPTION_PORT_IFINDEX];2600		if (attr)2601			opt_port_ifindex = nla_get_u32(attr);2602 2603		attr = opt_attrs[TEAM_ATTR_OPTION_ARRAY_INDEX];2604		if (attr) {2605			opt_is_array = true;2606			opt_array_index = nla_get_u32(attr);2607		}2608 2609		list_for_each_entry(opt_inst, &team->option_inst_list, list) {2610			struct team_option *option = opt_inst->option;2611			struct team_gsetter_ctx ctx;2612			struct team_option_inst_info *opt_inst_info;2613			int tmp_ifindex;2614 2615			opt_inst_info = &opt_inst->info;2616			tmp_ifindex = opt_inst_info->port ?2617				      opt_inst_info->port->dev->ifindex : 0;2618			if (option->type != opt_type ||2619			    strcmp(option->name, opt_name) ||2620			    tmp_ifindex != opt_port_ifindex ||2621			    (option->array_size && !opt_is_array) ||2622			    opt_inst_info->array_index != opt_array_index)2623				continue;2624			opt_found = true;2625			ctx.info = opt_inst_info;2626			switch (opt_type) {2627			case TEAM_OPTION_TYPE_U32:2628				ctx.data.u32_val = nla_get_u32(attr_data);2629				break;2630			case TEAM_OPTION_TYPE_STRING:2631				if (nla_len(attr_data) > TEAM_STRING_MAX_LEN) {2632					err = -EINVAL;2633					goto team_put;2634				}2635				ctx.data.str_val = nla_data(attr_data);2636				break;2637			case TEAM_OPTION_TYPE_BINARY:2638				ctx.data.bin_val.len = nla_len(attr_data);2639				ctx.data.bin_val.ptr = nla_data(attr_data);2640				break;2641			case TEAM_OPTION_TYPE_BOOL:2642				ctx.data.bool_val = attr_data ? true : false;2643				break;2644			case TEAM_OPTION_TYPE_S32:2645				ctx.data.s32_val = nla_get_s32(attr_data);2646				break;2647			default:2648				BUG();2649			}2650			err = team_option_set(team, opt_inst, &ctx);2651			if (err)2652				goto team_put;2653			opt_inst->changed = true;2654			list_add(&opt_inst->tmp_list, &opt_inst_list);2655		}2656		if (!opt_found) {2657			err = -ENOENT;2658			goto team_put;2659		}2660 2661		err = team_nl_send_event_options_get(team, &opt_inst_list);2662		if (err)2663			break;2664	}2665 2666team_put:2667	team_nl_team_put(team);2668rtnl_unlock:2669	rtnl_unlock();2670	return err;2671}2672 2673static int team_nl_fill_one_port_get(struct sk_buff *skb,2674				     struct team_port *port)2675{2676	struct nlattr *port_item;2677 2678	port_item = nla_nest_start_noflag(skb, TEAM_ATTR_ITEM_PORT);2679	if (!port_item)2680		goto nest_cancel;2681	if (nla_put_u32(skb, TEAM_ATTR_PORT_IFINDEX, port->dev->ifindex))2682		goto nest_cancel;2683	if (port->changed) {2684		if (nla_put_flag(skb, TEAM_ATTR_PORT_CHANGED))2685			goto nest_cancel;2686		port->changed = false;2687	}2688	if ((port->removed &&2689	     nla_put_flag(skb, TEAM_ATTR_PORT_REMOVED)) ||2690	    (port->state.linkup &&2691	     nla_put_flag(skb, TEAM_ATTR_PORT_LINKUP)) ||2692	    nla_put_u32(skb, TEAM_ATTR_PORT_SPEED, port->state.speed) ||2693	    nla_put_u8(skb, TEAM_ATTR_PORT_DUPLEX, port->state.duplex))2694		goto nest_cancel;2695	nla_nest_end(skb, port_item);2696	return 0;2697 2698nest_cancel:2699	nla_nest_cancel(skb, port_item);2700	return -EMSGSIZE;2701}2702 2703static int team_nl_send_port_list_get(struct team *team, u32 portid, u32 seq,2704				      int flags, team_nl_send_func_t *send_func,2705				      struct team_port *one_port)2706{2707	struct nlattr *port_list;2708	struct nlmsghdr *nlh;2709	void *hdr;2710	struct team_port *port;2711	int err;2712	struct sk_buff *skb = NULL;2713	bool incomplete;2714	int i;2715 2716	port = list_first_entry_or_null(&team->port_list,2717					struct team_port, list);2718 2719start_again:2720	err = __send_and_alloc_skb(&skb, team, portid, send_func);2721	if (err)2722		return err;2723 2724	hdr = genlmsg_put(skb, portid, seq, &team_nl_family, flags | NLM_F_MULTI,2725			  TEAM_CMD_PORT_LIST_GET);2726	if (!hdr) {2727		nlmsg_free(skb);2728		return -EMSGSIZE;2729	}2730 2731	if (nla_put_u32(skb, TEAM_ATTR_TEAM_IFINDEX, team->dev->ifindex))2732		goto nla_put_failure;2733	port_list = nla_nest_start_noflag(skb, TEAM_ATTR_LIST_PORT);2734	if (!port_list)2735		goto nla_put_failure;2736 2737	i = 0;2738	incomplete = false;2739 2740	/* If one port is selected, called wants to send port list containing2741	 * only this port. Otherwise go through all listed ports and send all2742	 */2743	if (one_port) {2744		err = team_nl_fill_one_port_get(skb, one_port);2745		if (err)2746			goto errout;2747	} else if (port) {2748		list_for_each_entry_from(port, &team->port_list, list) {2749			err = team_nl_fill_one_port_get(skb, port);2750			if (err) {2751				if (err == -EMSGSIZE) {2752					if (!i)2753						goto errout;2754					incomplete = true;2755					break;2756				}2757				goto errout;2758			}2759			i++;2760		}2761	}2762 2763	nla_nest_end(skb, port_list);2764	genlmsg_end(skb, hdr);2765	if (incomplete)2766		goto start_again;2767 2768send_done:2769	nlh = nlmsg_put(skb, portid, seq, NLMSG_DONE, 0, flags | NLM_F_MULTI);2770	if (!nlh) {2771		err = __send_and_alloc_skb(&skb, team, portid, send_func);2772		if (err)2773			return err;2774		goto send_done;2775	}2776 2777	return send_func(skb, team, portid);2778 2779nla_put_failure:2780	err = -EMSGSIZE;2781errout:2782	nlmsg_free(skb);2783	return err;2784}2785 2786int team_nl_port_list_get_doit(struct sk_buff *skb,2787			       struct genl_info *info)2788{2789	struct team *team;2790	int err;2791 2792	team = team_nl_team_get(info);2793	if (!team)2794		return -EINVAL;2795 2796	err = team_nl_send_port_list_get(team, info->snd_portid, info->snd_seq,2797					 NLM_F_ACK, team_nl_send_unicast, NULL);2798 2799	team_nl_team_put(team);2800 2801	return err;2802}2803 2804static const struct genl_multicast_group team_nl_mcgrps[] = {2805	{ .name = TEAM_GENL_CHANGE_EVENT_MC_GRP_NAME, },2806};2807 2808static struct genl_family team_nl_family __ro_after_init = {2809	.name		= TEAM_GENL_NAME,2810	.version	= TEAM_GENL_VERSION,2811	.maxattr	= ARRAY_SIZE(team_nl_policy) - 1,2812	.policy = team_nl_policy,2813	.netnsok	= true,2814	.module		= THIS_MODULE,2815	.small_ops	= team_nl_ops,2816	.n_small_ops	= ARRAY_SIZE(team_nl_ops),2817	.resv_start_op	= TEAM_CMD_PORT_LIST_GET + 1,2818	.mcgrps		= team_nl_mcgrps,2819	.n_mcgrps	= ARRAY_SIZE(team_nl_mcgrps),2820};2821 2822static int team_nl_send_multicast(struct sk_buff *skb,2823				  struct team *team, u32 portid)2824{2825	return genlmsg_multicast_netns(&team_nl_family, dev_net(team->dev),2826				       skb, 0, 0, GFP_KERNEL);2827}2828 2829static int team_nl_send_event_options_get(struct team *team,2830					  struct list_head *sel_opt_inst_list)2831{2832	return team_nl_send_options_get(team, 0, 0, 0, team_nl_send_multicast,2833					sel_opt_inst_list);2834}2835 2836static int team_nl_send_event_port_get(struct team *team,2837				       struct team_port *port)2838{2839	return team_nl_send_port_list_get(team, 0, 0, 0, team_nl_send_multicast,2840					  port);2841}2842 2843static int __init team_nl_init(void)2844{2845	return genl_register_family(&team_nl_family);2846}2847 2848static void __exit team_nl_fini(void)2849{2850	genl_unregister_family(&team_nl_family);2851}2852 2853 2854/******************2855 * Change checkers2856 ******************/2857 2858static void __team_options_change_check(struct team *team)2859{2860	int err;2861	struct team_option_inst *opt_inst;2862	LIST_HEAD(sel_opt_inst_list);2863 2864	list_for_each_entry(opt_inst, &team->option_inst_list, list) {2865		if (opt_inst->changed)2866			list_add_tail(&opt_inst->tmp_list, &sel_opt_inst_list);2867	}2868	err = team_nl_send_event_options_get(team, &sel_opt_inst_list);2869	if (err && err != -ESRCH)2870		netdev_warn(team->dev, "Failed to send options change via netlink (err %d)\n",2871			    err);2872}2873 2874/* rtnl lock is held */2875 2876static void __team_port_change_send(struct team_port *port, bool linkup)2877{2878	int err;2879 2880	port->changed = true;2881	port->state.linkup = linkup;2882	team_refresh_port_linkup(port);2883	if (linkup) {2884		struct ethtool_link_ksettings ecmd;2885 2886		err = __ethtool_get_link_ksettings(port->dev, &ecmd);2887		if (!err) {2888			port->state.speed = ecmd.base.speed;2889			port->state.duplex = ecmd.base.duplex;2890			goto send_event;2891		}2892	}2893	port->state.speed = 0;2894	port->state.duplex = 0;2895 2896send_event:2897	err = team_nl_send_event_port_get(port->team, port);2898	if (err && err != -ESRCH)2899		netdev_warn(port->team->dev, "Failed to send port change of device %s via netlink (err %d)\n",2900			    port->dev->name, err);2901 2902}2903 2904static void __team_carrier_check(struct team *team)2905{2906	struct team_port *port;2907	bool team_linkup;2908 2909	if (team->user_carrier_enabled)2910		return;2911 2912	team_linkup = false;2913	list_for_each_entry(port, &team->port_list, list) {2914		if (port->linkup) {2915			team_linkup = true;2916			break;2917		}2918	}2919 2920	if (team_linkup)2921		netif_carrier_on(team->dev);2922	else2923		netif_carrier_off(team->dev);2924}2925 2926static void __team_port_change_check(struct team_port *port, bool linkup)2927{2928	if (port->state.linkup != linkup)2929		__team_port_change_send(port, linkup);2930	__team_carrier_check(port->team);2931}2932 2933static void __team_port_change_port_added(struct team_port *port, bool linkup)2934{2935	__team_port_change_send(port, linkup);2936	__team_carrier_check(port->team);2937}2938 2939static void __team_port_change_port_removed(struct team_port *port)2940{2941	port->removed = true;2942	__team_port_change_send(port, false);2943	__team_carrier_check(port->team);2944}2945 2946static void team_port_change_check(struct team_port *port, bool linkup)2947{2948	struct team *team = port->team;2949 2950	mutex_lock(&team->lock);2951	__team_port_change_check(port, linkup);2952	mutex_unlock(&team->lock);2953}2954 2955 2956/************************************2957 * Net device notifier event handler2958 ************************************/2959 2960static int team_device_event(struct notifier_block *unused,2961			     unsigned long event, void *ptr)2962{2963	struct net_device *dev = netdev_notifier_info_to_dev(ptr);2964	struct team_port *port;2965 2966	port = team_port_get_rtnl(dev);2967	if (!port)2968		return NOTIFY_DONE;2969 2970	switch (event) {2971	case NETDEV_UP:2972		if (netif_oper_up(dev))2973			team_port_change_check(port, true);2974		break;2975	case NETDEV_DOWN:2976		team_port_change_check(port, false);2977		break;2978	case NETDEV_CHANGE:2979		if (netif_running(port->dev))2980			team_port_change_check(port,2981					       !!netif_oper_up(port->dev));2982		break;2983	case NETDEV_UNREGISTER:2984		team_del_slave(port->team->dev, dev);2985		break;2986	case NETDEV_FEAT_CHANGE:2987		if (!port->team->notifier_ctx) {2988			port->team->notifier_ctx = true;2989			team_compute_features(port->team);2990			port->team->notifier_ctx = false;2991		}2992		break;2993	case NETDEV_PRECHANGEMTU:2994		/* Forbid to change mtu of underlaying device */2995		if (!port->team->port_mtu_change_allowed)2996			return NOTIFY_BAD;2997		break;2998	case NETDEV_PRE_TYPE_CHANGE:2999		/* Forbid to change type of underlaying device */3000		return NOTIFY_BAD;3001	case NETDEV_RESEND_IGMP:3002		/* Propagate to master device */3003		call_netdevice_notifiers(event, port->team->dev);3004		break;3005	}3006	return NOTIFY_DONE;3007}3008 3009static struct notifier_block team_notifier_block __read_mostly = {3010	.notifier_call = team_device_event,3011};3012 3013 3014/***********************3015 * Module init and exit3016 ***********************/3017 3018static int __init team_module_init(void)3019{3020	int err;3021 3022	register_netdevice_notifier(&team_notifier_block);3023 3024	err = rtnl_link_register(&team_link_ops);3025	if (err)3026		goto err_rtnl_reg;3027 3028	err = team_nl_init();3029	if (err)3030		goto err_nl_init;3031 3032	return 0;3033 3034err_nl_init:3035	rtnl_link_unregister(&team_link_ops);3036 3037err_rtnl_reg:3038	unregister_netdevice_notifier(&team_notifier_block);3039 3040	return err;3041}3042 3043static void __exit team_module_exit(void)3044{3045	team_nl_fini();3046	rtnl_link_unregister(&team_link_ops);3047	unregister_netdevice_notifier(&team_notifier_block);3048}3049 3050module_init(team_module_init);3051module_exit(team_module_exit);3052 3053MODULE_LICENSE("GPL v2");3054MODULE_AUTHOR("Jiri Pirko <jpirko@redhat.com>");3055MODULE_DESCRIPTION("Ethernet team device driver");3056MODULE_ALIAS_RTNL_LINK(DRV_NAME);3057