702 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * drivers/net/team/team_mode_loadbalance.c - Load-balancing mode for team4 * Copyright (c) 2012 Jiri Pirko <jpirko@redhat.com>5 */6 7#include <linux/kernel.h>8#include <linux/types.h>9#include <linux/module.h>10#include <linux/init.h>11#include <linux/errno.h>12#include <linux/netdevice.h>13#include <linux/etherdevice.h>14#include <linux/filter.h>15#include <linux/if_team.h>16 17static rx_handler_result_t lb_receive(struct team *team, struct team_port *port,18 struct sk_buff *skb)19{20 if (unlikely(skb->protocol == htons(ETH_P_SLOW))) {21 /* LACPDU packets should go to exact delivery */22 const unsigned char *dest = eth_hdr(skb)->h_dest;23 24 if (is_link_local_ether_addr(dest) && dest[5] == 0x02)25 return RX_HANDLER_EXACT;26 }27 return RX_HANDLER_ANOTHER;28}29 30struct lb_priv;31 32typedef struct team_port *lb_select_tx_port_func_t(struct team *,33 unsigned char);34 35#define LB_TX_HASHTABLE_SIZE 256 /* hash is a char */36 37struct lb_stats {38 u64 tx_bytes;39};40 41struct lb_pcpu_stats {42 struct lb_stats hash_stats[LB_TX_HASHTABLE_SIZE];43 struct u64_stats_sync syncp;44};45 46struct lb_stats_info {47 struct lb_stats stats;48 struct lb_stats last_stats;49 struct team_option_inst_info *opt_inst_info;50};51 52struct lb_port_mapping {53 struct team_port __rcu *port;54 struct team_option_inst_info *opt_inst_info;55};56 57struct lb_priv_ex {58 struct team *team;59 struct lb_port_mapping tx_hash_to_port_mapping[LB_TX_HASHTABLE_SIZE];60 struct sock_fprog_kern *orig_fprog;61 struct {62 unsigned int refresh_interval; /* in tenths of second */63 struct delayed_work refresh_dw;64 struct lb_stats_info info[LB_TX_HASHTABLE_SIZE];65 } stats;66};67 68struct lb_priv {69 struct bpf_prog __rcu *fp;70 lb_select_tx_port_func_t __rcu *select_tx_port_func;71 struct lb_pcpu_stats __percpu *pcpu_stats;72 struct lb_priv_ex *ex; /* priv extension */73};74 75static struct lb_priv *get_lb_priv(struct team *team)76{77 return (struct lb_priv *) &team->mode_priv;78}79 80struct lb_port_priv {81 struct lb_stats __percpu *pcpu_stats;82 struct lb_stats_info stats_info;83};84 85static struct lb_port_priv *get_lb_port_priv(struct team_port *port)86{87 return (struct lb_port_priv *) &port->mode_priv;88}89 90#define LB_HTPM_PORT_BY_HASH(lp_priv, hash) \91 (lb_priv)->ex->tx_hash_to_port_mapping[hash].port92 93#define LB_HTPM_OPT_INST_INFO_BY_HASH(lp_priv, hash) \94 (lb_priv)->ex->tx_hash_to_port_mapping[hash].opt_inst_info95 96static void lb_tx_hash_to_port_mapping_null_port(struct team *team,97 struct team_port *port)98{99 struct lb_priv *lb_priv = get_lb_priv(team);100 bool changed = false;101 int i;102 103 for (i = 0; i < LB_TX_HASHTABLE_SIZE; i++) {104 struct lb_port_mapping *pm;105 106 pm = &lb_priv->ex->tx_hash_to_port_mapping[i];107 if (rcu_access_pointer(pm->port) == port) {108 RCU_INIT_POINTER(pm->port, NULL);109 team_option_inst_set_change(pm->opt_inst_info);110 changed = true;111 }112 }113 if (changed)114 team_options_change_check(team);115}116 117/* Basic tx selection based solely by hash */118static struct team_port *lb_hash_select_tx_port(struct team *team,119 unsigned char hash)120{121 int port_index = team_num_to_port_index(team, hash);122 123 return team_get_port_by_index_rcu(team, port_index);124}125 126/* Hash to port mapping select tx port */127static struct team_port *lb_htpm_select_tx_port(struct team *team,128 unsigned char hash)129{130 struct lb_priv *lb_priv = get_lb_priv(team);131 struct team_port *port;132 133 port = rcu_dereference_bh(LB_HTPM_PORT_BY_HASH(lb_priv, hash));134 if (likely(port))135 return port;136 /* If no valid port in the table, fall back to simple hash */137 return lb_hash_select_tx_port(team, hash);138}139 140struct lb_select_tx_port {141 char *name;142 lb_select_tx_port_func_t *func;143};144 145static const struct lb_select_tx_port lb_select_tx_port_list[] = {146 {147 .name = "hash",148 .func = lb_hash_select_tx_port,149 },150 {151 .name = "hash_to_port_mapping",152 .func = lb_htpm_select_tx_port,153 },154};155#define LB_SELECT_TX_PORT_LIST_COUNT ARRAY_SIZE(lb_select_tx_port_list)156 157static char *lb_select_tx_port_get_name(lb_select_tx_port_func_t *func)158{159 int i;160 161 for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) {162 const struct lb_select_tx_port *item;163 164 item = &lb_select_tx_port_list[i];165 if (item->func == func)166 return item->name;167 }168 return NULL;169}170 171static lb_select_tx_port_func_t *lb_select_tx_port_get_func(const char *name)172{173 int i;174 175 for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) {176 const struct lb_select_tx_port *item;177 178 item = &lb_select_tx_port_list[i];179 if (!strcmp(item->name, name))180 return item->func;181 }182 return NULL;183}184 185static unsigned int lb_get_skb_hash(struct lb_priv *lb_priv,186 struct sk_buff *skb)187{188 struct bpf_prog *fp;189 uint32_t lhash;190 unsigned char *c;191 192 fp = rcu_dereference_bh(lb_priv->fp);193 if (unlikely(!fp))194 return 0;195 lhash = bpf_prog_run(fp, skb);196 c = (char *) &lhash;197 return c[0] ^ c[1] ^ c[2] ^ c[3];198}199 200static void lb_update_tx_stats(unsigned int tx_bytes, struct lb_priv *lb_priv,201 struct lb_port_priv *lb_port_priv,202 unsigned char hash)203{204 struct lb_pcpu_stats *pcpu_stats;205 struct lb_stats *port_stats;206 struct lb_stats *hash_stats;207 208 pcpu_stats = this_cpu_ptr(lb_priv->pcpu_stats);209 port_stats = this_cpu_ptr(lb_port_priv->pcpu_stats);210 hash_stats = &pcpu_stats->hash_stats[hash];211 u64_stats_update_begin(&pcpu_stats->syncp);212 port_stats->tx_bytes += tx_bytes;213 hash_stats->tx_bytes += tx_bytes;214 u64_stats_update_end(&pcpu_stats->syncp);215}216 217static bool lb_transmit(struct team *team, struct sk_buff *skb)218{219 struct lb_priv *lb_priv = get_lb_priv(team);220 lb_select_tx_port_func_t *select_tx_port_func;221 struct team_port *port;222 unsigned char hash;223 unsigned int tx_bytes = skb->len;224 225 hash = lb_get_skb_hash(lb_priv, skb);226 select_tx_port_func = rcu_dereference_bh(lb_priv->select_tx_port_func);227 port = select_tx_port_func(team, hash);228 if (unlikely(!port))229 goto drop;230 if (team_dev_queue_xmit(team, port, skb))231 return false;232 lb_update_tx_stats(tx_bytes, lb_priv, get_lb_port_priv(port), hash);233 return true;234 235drop:236 dev_kfree_skb_any(skb);237 return false;238}239 240static void lb_bpf_func_get(struct team *team, struct team_gsetter_ctx *ctx)241{242 struct lb_priv *lb_priv = get_lb_priv(team);243 244 if (!lb_priv->ex->orig_fprog) {245 ctx->data.bin_val.len = 0;246 ctx->data.bin_val.ptr = NULL;247 return;248 }249 ctx->data.bin_val.len = lb_priv->ex->orig_fprog->len *250 sizeof(struct sock_filter);251 ctx->data.bin_val.ptr = lb_priv->ex->orig_fprog->filter;252}253 254static int __fprog_create(struct sock_fprog_kern **pfprog, u32 data_len,255 const void *data)256{257 struct sock_fprog_kern *fprog;258 struct sock_filter *filter = (struct sock_filter *) data;259 260 if (data_len % sizeof(struct sock_filter))261 return -EINVAL;262 fprog = kmalloc(sizeof(*fprog), GFP_KERNEL);263 if (!fprog)264 return -ENOMEM;265 fprog->filter = kmemdup(filter, data_len, GFP_KERNEL);266 if (!fprog->filter) {267 kfree(fprog);268 return -ENOMEM;269 }270 fprog->len = data_len / sizeof(struct sock_filter);271 *pfprog = fprog;272 return 0;273}274 275static void __fprog_destroy(struct sock_fprog_kern *fprog)276{277 kfree(fprog->filter);278 kfree(fprog);279}280 281static int lb_bpf_func_set(struct team *team, struct team_gsetter_ctx *ctx)282{283 struct lb_priv *lb_priv = get_lb_priv(team);284 struct bpf_prog *fp = NULL;285 struct bpf_prog *orig_fp = NULL;286 struct sock_fprog_kern *fprog = NULL;287 int err;288 289 if (ctx->data.bin_val.len) {290 err = __fprog_create(&fprog, ctx->data.bin_val.len,291 ctx->data.bin_val.ptr);292 if (err)293 return err;294 err = bpf_prog_create(&fp, fprog);295 if (err) {296 __fprog_destroy(fprog);297 return err;298 }299 }300 301 if (lb_priv->ex->orig_fprog) {302 /* Clear old filter data */303 __fprog_destroy(lb_priv->ex->orig_fprog);304 orig_fp = rcu_dereference_protected(lb_priv->fp,305 lockdep_is_held(&team->lock));306 }307 308 rcu_assign_pointer(lb_priv->fp, fp);309 lb_priv->ex->orig_fprog = fprog;310 311 if (orig_fp) {312 synchronize_rcu();313 bpf_prog_destroy(orig_fp);314 }315 return 0;316}317 318static void lb_bpf_func_free(struct team *team)319{320 struct lb_priv *lb_priv = get_lb_priv(team);321 struct bpf_prog *fp;322 323 if (!lb_priv->ex->orig_fprog)324 return;325 326 __fprog_destroy(lb_priv->ex->orig_fprog);327 fp = rcu_dereference_protected(lb_priv->fp,328 lockdep_is_held(&team->lock));329 bpf_prog_destroy(fp);330}331 332static void lb_tx_method_get(struct team *team, struct team_gsetter_ctx *ctx)333{334 struct lb_priv *lb_priv = get_lb_priv(team);335 lb_select_tx_port_func_t *func;336 char *name;337 338 func = rcu_dereference_protected(lb_priv->select_tx_port_func,339 lockdep_is_held(&team->lock));340 name = lb_select_tx_port_get_name(func);341 BUG_ON(!name);342 ctx->data.str_val = name;343}344 345static int lb_tx_method_set(struct team *team, struct team_gsetter_ctx *ctx)346{347 struct lb_priv *lb_priv = get_lb_priv(team);348 lb_select_tx_port_func_t *func;349 350 func = lb_select_tx_port_get_func(ctx->data.str_val);351 if (!func)352 return -EINVAL;353 rcu_assign_pointer(lb_priv->select_tx_port_func, func);354 return 0;355}356 357static void lb_tx_hash_to_port_mapping_init(struct team *team,358 struct team_option_inst_info *info)359{360 struct lb_priv *lb_priv = get_lb_priv(team);361 unsigned char hash = info->array_index;362 363 LB_HTPM_OPT_INST_INFO_BY_HASH(lb_priv, hash) = info;364}365 366static void lb_tx_hash_to_port_mapping_get(struct team *team,367 struct team_gsetter_ctx *ctx)368{369 struct lb_priv *lb_priv = get_lb_priv(team);370 struct team_port *port;371 unsigned char hash = ctx->info->array_index;372 373 port = LB_HTPM_PORT_BY_HASH(lb_priv, hash);374 ctx->data.u32_val = port ? port->dev->ifindex : 0;375}376 377static int lb_tx_hash_to_port_mapping_set(struct team *team,378 struct team_gsetter_ctx *ctx)379{380 struct lb_priv *lb_priv = get_lb_priv(team);381 struct team_port *port;382 unsigned char hash = ctx->info->array_index;383 384 list_for_each_entry(port, &team->port_list, list) {385 if (ctx->data.u32_val == port->dev->ifindex &&386 team_port_enabled(port)) {387 rcu_assign_pointer(LB_HTPM_PORT_BY_HASH(lb_priv, hash),388 port);389 return 0;390 }391 }392 return -ENODEV;393}394 395static void lb_hash_stats_init(struct team *team,396 struct team_option_inst_info *info)397{398 struct lb_priv *lb_priv = get_lb_priv(team);399 unsigned char hash = info->array_index;400 401 lb_priv->ex->stats.info[hash].opt_inst_info = info;402}403 404static void lb_hash_stats_get(struct team *team, struct team_gsetter_ctx *ctx)405{406 struct lb_priv *lb_priv = get_lb_priv(team);407 unsigned char hash = ctx->info->array_index;408 409 ctx->data.bin_val.ptr = &lb_priv->ex->stats.info[hash].stats;410 ctx->data.bin_val.len = sizeof(struct lb_stats);411}412 413static void lb_port_stats_init(struct team *team,414 struct team_option_inst_info *info)415{416 struct team_port *port = info->port;417 struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);418 419 lb_port_priv->stats_info.opt_inst_info = info;420}421 422static void lb_port_stats_get(struct team *team, struct team_gsetter_ctx *ctx)423{424 struct team_port *port = ctx->info->port;425 struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);426 427 ctx->data.bin_val.ptr = &lb_port_priv->stats_info.stats;428 ctx->data.bin_val.len = sizeof(struct lb_stats);429}430 431static void __lb_stats_info_refresh_prepare(struct lb_stats_info *s_info)432{433 memcpy(&s_info->last_stats, &s_info->stats, sizeof(struct lb_stats));434 memset(&s_info->stats, 0, sizeof(struct lb_stats));435}436 437static bool __lb_stats_info_refresh_check(struct lb_stats_info *s_info,438 struct team *team)439{440 if (memcmp(&s_info->last_stats, &s_info->stats,441 sizeof(struct lb_stats))) {442 team_option_inst_set_change(s_info->opt_inst_info);443 return true;444 }445 return false;446}447 448static void __lb_one_cpu_stats_add(struct lb_stats *acc_stats,449 struct lb_stats *cpu_stats,450 struct u64_stats_sync *syncp)451{452 unsigned int start;453 struct lb_stats tmp;454 455 do {456 start = u64_stats_fetch_begin(syncp);457 tmp.tx_bytes = cpu_stats->tx_bytes;458 } while (u64_stats_fetch_retry(syncp, start));459 acc_stats->tx_bytes += tmp.tx_bytes;460}461 462static void lb_stats_refresh(struct work_struct *work)463{464 struct team *team;465 struct lb_priv *lb_priv;466 struct lb_priv_ex *lb_priv_ex;467 struct lb_pcpu_stats *pcpu_stats;468 struct lb_stats *stats;469 struct lb_stats_info *s_info;470 struct team_port *port;471 bool changed = false;472 int i;473 int j;474 475 lb_priv_ex = container_of(work, struct lb_priv_ex,476 stats.refresh_dw.work);477 478 team = lb_priv_ex->team;479 lb_priv = get_lb_priv(team);480 481 if (!mutex_trylock(&team->lock)) {482 schedule_delayed_work(&lb_priv_ex->stats.refresh_dw, 0);483 return;484 }485 486 for (j = 0; j < LB_TX_HASHTABLE_SIZE; j++) {487 s_info = &lb_priv->ex->stats.info[j];488 __lb_stats_info_refresh_prepare(s_info);489 for_each_possible_cpu(i) {490 pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i);491 stats = &pcpu_stats->hash_stats[j];492 __lb_one_cpu_stats_add(&s_info->stats, stats,493 &pcpu_stats->syncp);494 }495 changed |= __lb_stats_info_refresh_check(s_info, team);496 }497 498 list_for_each_entry(port, &team->port_list, list) {499 struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);500 501 s_info = &lb_port_priv->stats_info;502 __lb_stats_info_refresh_prepare(s_info);503 for_each_possible_cpu(i) {504 pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i);505 stats = per_cpu_ptr(lb_port_priv->pcpu_stats, i);506 __lb_one_cpu_stats_add(&s_info->stats, stats,507 &pcpu_stats->syncp);508 }509 changed |= __lb_stats_info_refresh_check(s_info, team);510 }511 512 if (changed)513 team_options_change_check(team);514 515 schedule_delayed_work(&lb_priv_ex->stats.refresh_dw,516 (lb_priv_ex->stats.refresh_interval * HZ) / 10);517 518 mutex_unlock(&team->lock);519}520 521static void lb_stats_refresh_interval_get(struct team *team,522 struct team_gsetter_ctx *ctx)523{524 struct lb_priv *lb_priv = get_lb_priv(team);525 526 ctx->data.u32_val = lb_priv->ex->stats.refresh_interval;527}528 529static int lb_stats_refresh_interval_set(struct team *team,530 struct team_gsetter_ctx *ctx)531{532 struct lb_priv *lb_priv = get_lb_priv(team);533 unsigned int interval;534 535 interval = ctx->data.u32_val;536 if (lb_priv->ex->stats.refresh_interval == interval)537 return 0;538 lb_priv->ex->stats.refresh_interval = interval;539 if (interval)540 schedule_delayed_work(&lb_priv->ex->stats.refresh_dw, 0);541 else542 cancel_delayed_work(&lb_priv->ex->stats.refresh_dw);543 return 0;544}545 546static const struct team_option lb_options[] = {547 {548 .name = "bpf_hash_func",549 .type = TEAM_OPTION_TYPE_BINARY,550 .getter = lb_bpf_func_get,551 .setter = lb_bpf_func_set,552 },553 {554 .name = "lb_tx_method",555 .type = TEAM_OPTION_TYPE_STRING,556 .getter = lb_tx_method_get,557 .setter = lb_tx_method_set,558 },559 {560 .name = "lb_tx_hash_to_port_mapping",561 .array_size = LB_TX_HASHTABLE_SIZE,562 .type = TEAM_OPTION_TYPE_U32,563 .init = lb_tx_hash_to_port_mapping_init,564 .getter = lb_tx_hash_to_port_mapping_get,565 .setter = lb_tx_hash_to_port_mapping_set,566 },567 {568 .name = "lb_hash_stats",569 .array_size = LB_TX_HASHTABLE_SIZE,570 .type = TEAM_OPTION_TYPE_BINARY,571 .init = lb_hash_stats_init,572 .getter = lb_hash_stats_get,573 },574 {575 .name = "lb_port_stats",576 .per_port = true,577 .type = TEAM_OPTION_TYPE_BINARY,578 .init = lb_port_stats_init,579 .getter = lb_port_stats_get,580 },581 {582 .name = "lb_stats_refresh_interval",583 .type = TEAM_OPTION_TYPE_U32,584 .getter = lb_stats_refresh_interval_get,585 .setter = lb_stats_refresh_interval_set,586 },587};588 589static int lb_init(struct team *team)590{591 struct lb_priv *lb_priv = get_lb_priv(team);592 lb_select_tx_port_func_t *func;593 int i, err;594 595 /* set default tx port selector */596 func = lb_select_tx_port_get_func("hash");597 BUG_ON(!func);598 rcu_assign_pointer(lb_priv->select_tx_port_func, func);599 600 lb_priv->ex = kzalloc(sizeof(*lb_priv->ex), GFP_KERNEL);601 if (!lb_priv->ex)602 return -ENOMEM;603 lb_priv->ex->team = team;604 605 lb_priv->pcpu_stats = alloc_percpu(struct lb_pcpu_stats);606 if (!lb_priv->pcpu_stats) {607 err = -ENOMEM;608 goto err_alloc_pcpu_stats;609 }610 611 for_each_possible_cpu(i) {612 struct lb_pcpu_stats *team_lb_stats;613 team_lb_stats = per_cpu_ptr(lb_priv->pcpu_stats, i);614 u64_stats_init(&team_lb_stats->syncp);615 }616 617 618 INIT_DELAYED_WORK(&lb_priv->ex->stats.refresh_dw, lb_stats_refresh);619 620 err = team_options_register(team, lb_options, ARRAY_SIZE(lb_options));621 if (err)622 goto err_options_register;623 return 0;624 625err_options_register:626 free_percpu(lb_priv->pcpu_stats);627err_alloc_pcpu_stats:628 kfree(lb_priv->ex);629 return err;630}631 632static void lb_exit(struct team *team)633{634 struct lb_priv *lb_priv = get_lb_priv(team);635 636 team_options_unregister(team, lb_options,637 ARRAY_SIZE(lb_options));638 lb_bpf_func_free(team);639 cancel_delayed_work_sync(&lb_priv->ex->stats.refresh_dw);640 free_percpu(lb_priv->pcpu_stats);641 kfree(lb_priv->ex);642}643 644static int lb_port_enter(struct team *team, struct team_port *port)645{646 struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);647 648 lb_port_priv->pcpu_stats = alloc_percpu(struct lb_stats);649 if (!lb_port_priv->pcpu_stats)650 return -ENOMEM;651 return 0;652}653 654static void lb_port_leave(struct team *team, struct team_port *port)655{656 struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);657 658 free_percpu(lb_port_priv->pcpu_stats);659}660 661static void lb_port_disabled(struct team *team, struct team_port *port)662{663 lb_tx_hash_to_port_mapping_null_port(team, port);664}665 666static const struct team_mode_ops lb_mode_ops = {667 .init = lb_init,668 .exit = lb_exit,669 .port_enter = lb_port_enter,670 .port_leave = lb_port_leave,671 .port_disabled = lb_port_disabled,672 .receive = lb_receive,673 .transmit = lb_transmit,674};675 676static const struct team_mode lb_mode = {677 .kind = "loadbalance",678 .owner = THIS_MODULE,679 .priv_size = sizeof(struct lb_priv),680 .port_priv_size = sizeof(struct lb_port_priv),681 .ops = &lb_mode_ops,682 .lag_tx_type = NETDEV_LAG_TX_TYPE_HASH,683};684 685static int __init lb_init_module(void)686{687 return team_mode_register(&lb_mode);688}689 690static void __exit lb_cleanup_module(void)691{692 team_mode_unregister(&lb_mode);693}694 695module_init(lb_init_module);696module_exit(lb_cleanup_module);697 698MODULE_LICENSE("GPL v2");699MODULE_AUTHOR("Jiri Pirko <jpirko@redhat.com>");700MODULE_DESCRIPTION("Load-balancing mode for team");701MODULE_ALIAS_TEAM_MODE("loadbalance");702