627 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) Meta Platforms, Inc. and affiliates. */3 4#include <linux/etherdevice.h>5#include <linux/ipv6.h>6#include <linux/types.h>7#include <net/netdev_queues.h>8 9#include "fbnic.h"10#include "fbnic_netdev.h"11#include "fbnic_txrx.h"12 13int __fbnic_open(struct fbnic_net *fbn)14{15 struct fbnic_dev *fbd = fbn->fbd;16 int err;17 18 err = fbnic_alloc_napi_vectors(fbn);19 if (err)20 return err;21 22 err = fbnic_alloc_resources(fbn);23 if (err)24 goto free_napi_vectors;25 26 err = netif_set_real_num_tx_queues(fbn->netdev,27 fbn->num_tx_queues);28 if (err)29 goto free_resources;30 31 err = netif_set_real_num_rx_queues(fbn->netdev,32 fbn->num_rx_queues);33 if (err)34 goto free_resources;35 36 /* Send ownership message and flush to verify FW has seen it */37 err = fbnic_fw_xmit_ownership_msg(fbd, true);38 if (err) {39 dev_warn(fbd->dev,40 "Error %d sending host ownership message to the firmware\n",41 err);42 goto free_resources;43 }44 45 err = fbnic_fw_init_heartbeat(fbd, false);46 if (err)47 goto release_ownership;48 49 err = fbnic_pcs_irq_enable(fbd);50 if (err)51 goto release_ownership;52 /* Pull the BMC config and initialize the RPC */53 fbnic_bmc_rpc_init(fbd);54 fbnic_rss_reinit(fbd, fbn);55 56 return 0;57release_ownership:58 fbnic_fw_xmit_ownership_msg(fbn->fbd, false);59free_resources:60 fbnic_free_resources(fbn);61free_napi_vectors:62 fbnic_free_napi_vectors(fbn);63 return err;64}65 66static int fbnic_open(struct net_device *netdev)67{68 struct fbnic_net *fbn = netdev_priv(netdev);69 int err;70 71 err = __fbnic_open(fbn);72 if (!err)73 fbnic_up(fbn);74 75 return err;76}77 78static int fbnic_stop(struct net_device *netdev)79{80 struct fbnic_net *fbn = netdev_priv(netdev);81 82 fbnic_down(fbn);83 fbnic_pcs_irq_disable(fbn->fbd);84 85 fbnic_fw_xmit_ownership_msg(fbn->fbd, false);86 87 fbnic_free_resources(fbn);88 fbnic_free_napi_vectors(fbn);89 90 return 0;91}92 93static int fbnic_uc_sync(struct net_device *netdev, const unsigned char *addr)94{95 struct fbnic_net *fbn = netdev_priv(netdev);96 struct fbnic_mac_addr *avail_addr;97 98 if (WARN_ON(!is_valid_ether_addr(addr)))99 return -EADDRNOTAVAIL;100 101 avail_addr = __fbnic_uc_sync(fbn->fbd, addr);102 if (!avail_addr)103 return -ENOSPC;104 105 /* Add type flag indicating this address is in use by the host */106 set_bit(FBNIC_MAC_ADDR_T_UNICAST, avail_addr->act_tcam);107 108 return 0;109}110 111static int fbnic_uc_unsync(struct net_device *netdev, const unsigned char *addr)112{113 struct fbnic_net *fbn = netdev_priv(netdev);114 struct fbnic_dev *fbd = fbn->fbd;115 int i, ret;116 117 /* Scan from middle of list to bottom, filling bottom up.118 * Skip the first entry which is reserved for dev_addr and119 * leave the last entry to use for promiscuous filtering.120 */121 for (i = fbd->mac_addr_boundary, ret = -ENOENT;122 i < FBNIC_RPC_TCAM_MACDA_HOST_ADDR_IDX && ret; i++) {123 struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i];124 125 if (!ether_addr_equal(mac_addr->value.addr8, addr))126 continue;127 128 ret = __fbnic_uc_unsync(mac_addr);129 }130 131 return ret;132}133 134static int fbnic_mc_sync(struct net_device *netdev, const unsigned char *addr)135{136 struct fbnic_net *fbn = netdev_priv(netdev);137 struct fbnic_mac_addr *avail_addr;138 139 if (WARN_ON(!is_multicast_ether_addr(addr)))140 return -EADDRNOTAVAIL;141 142 avail_addr = __fbnic_mc_sync(fbn->fbd, addr);143 if (!avail_addr)144 return -ENOSPC;145 146 /* Add type flag indicating this address is in use by the host */147 set_bit(FBNIC_MAC_ADDR_T_MULTICAST, avail_addr->act_tcam);148 149 return 0;150}151 152static int fbnic_mc_unsync(struct net_device *netdev, const unsigned char *addr)153{154 struct fbnic_net *fbn = netdev_priv(netdev);155 struct fbnic_dev *fbd = fbn->fbd;156 int i, ret;157 158 /* Scan from middle of list to top, filling top down.159 * Skip over the address reserved for the BMC MAC and160 * exclude index 0 as that belongs to the broadcast address161 */162 for (i = fbd->mac_addr_boundary, ret = -ENOENT;163 --i > FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX && ret;) {164 struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i];165 166 if (!ether_addr_equal(mac_addr->value.addr8, addr))167 continue;168 169 ret = __fbnic_mc_unsync(mac_addr);170 }171 172 return ret;173}174 175void __fbnic_set_rx_mode(struct net_device *netdev)176{177 struct fbnic_net *fbn = netdev_priv(netdev);178 bool uc_promisc = false, mc_promisc = false;179 struct fbnic_dev *fbd = fbn->fbd;180 struct fbnic_mac_addr *mac_addr;181 int err;182 183 /* Populate host address from dev_addr */184 mac_addr = &fbd->mac_addr[FBNIC_RPC_TCAM_MACDA_HOST_ADDR_IDX];185 if (!ether_addr_equal(mac_addr->value.addr8, netdev->dev_addr) ||186 mac_addr->state != FBNIC_TCAM_S_VALID) {187 ether_addr_copy(mac_addr->value.addr8, netdev->dev_addr);188 mac_addr->state = FBNIC_TCAM_S_UPDATE;189 set_bit(FBNIC_MAC_ADDR_T_UNICAST, mac_addr->act_tcam);190 }191 192 /* Populate broadcast address if broadcast is enabled */193 mac_addr = &fbd->mac_addr[FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX];194 if (netdev->flags & IFF_BROADCAST) {195 if (!is_broadcast_ether_addr(mac_addr->value.addr8) ||196 mac_addr->state != FBNIC_TCAM_S_VALID) {197 eth_broadcast_addr(mac_addr->value.addr8);198 mac_addr->state = FBNIC_TCAM_S_ADD;199 }200 set_bit(FBNIC_MAC_ADDR_T_BROADCAST, mac_addr->act_tcam);201 } else if (mac_addr->state == FBNIC_TCAM_S_VALID) {202 __fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_BROADCAST);203 }204 205 /* Synchronize unicast and multicast address lists */206 err = __dev_uc_sync(netdev, fbnic_uc_sync, fbnic_uc_unsync);207 if (err == -ENOSPC)208 uc_promisc = true;209 err = __dev_mc_sync(netdev, fbnic_mc_sync, fbnic_mc_unsync);210 if (err == -ENOSPC)211 mc_promisc = true;212 213 uc_promisc |= !!(netdev->flags & IFF_PROMISC);214 mc_promisc |= !!(netdev->flags & IFF_ALLMULTI) || uc_promisc;215 216 /* Populate last TCAM entry with promiscuous entry and 0/1 bit mask */217 mac_addr = &fbd->mac_addr[FBNIC_RPC_TCAM_MACDA_PROMISC_IDX];218 if (uc_promisc) {219 if (!is_zero_ether_addr(mac_addr->value.addr8) ||220 mac_addr->state != FBNIC_TCAM_S_VALID) {221 eth_zero_addr(mac_addr->value.addr8);222 eth_broadcast_addr(mac_addr->mask.addr8);223 clear_bit(FBNIC_MAC_ADDR_T_ALLMULTI,224 mac_addr->act_tcam);225 set_bit(FBNIC_MAC_ADDR_T_PROMISC,226 mac_addr->act_tcam);227 mac_addr->state = FBNIC_TCAM_S_ADD;228 }229 } else if (mc_promisc &&230 (!fbnic_bmc_present(fbd) || !fbd->fw_cap.all_multi)) {231 /* We have to add a special handler for multicast as the232 * BMC may have an all-multi rule already in place. As such233 * adding a rule ourselves won't do any good so we will have234 * to modify the rules for the ALL MULTI below if the BMC235 * already has the rule in place.236 */237 if (!is_multicast_ether_addr(mac_addr->value.addr8) ||238 mac_addr->state != FBNIC_TCAM_S_VALID) {239 eth_zero_addr(mac_addr->value.addr8);240 eth_broadcast_addr(mac_addr->mask.addr8);241 mac_addr->value.addr8[0] ^= 1;242 mac_addr->mask.addr8[0] ^= 1;243 set_bit(FBNIC_MAC_ADDR_T_ALLMULTI,244 mac_addr->act_tcam);245 clear_bit(FBNIC_MAC_ADDR_T_PROMISC,246 mac_addr->act_tcam);247 mac_addr->state = FBNIC_TCAM_S_ADD;248 }249 } else if (mac_addr->state == FBNIC_TCAM_S_VALID) {250 if (test_bit(FBNIC_MAC_ADDR_T_BMC, mac_addr->act_tcam)) {251 clear_bit(FBNIC_MAC_ADDR_T_ALLMULTI,252 mac_addr->act_tcam);253 clear_bit(FBNIC_MAC_ADDR_T_PROMISC,254 mac_addr->act_tcam);255 } else {256 mac_addr->state = FBNIC_TCAM_S_DELETE;257 }258 }259 260 /* Add rules for BMC all multicast if it is enabled */261 fbnic_bmc_rpc_all_multi_config(fbd, mc_promisc);262 263 /* Sift out any unshared BMC rules and place them in BMC only section */264 fbnic_sift_macda(fbd);265 266 /* Write updates to hardware */267 fbnic_write_rules(fbd);268 fbnic_write_macda(fbd);269}270 271static void fbnic_set_rx_mode(struct net_device *netdev)272{273 /* No need to update the hardware if we are not running */274 if (netif_running(netdev))275 __fbnic_set_rx_mode(netdev);276}277 278static int fbnic_set_mac(struct net_device *netdev, void *p)279{280 struct sockaddr *addr = p;281 282 if (!is_valid_ether_addr(addr->sa_data))283 return -EADDRNOTAVAIL;284 285 eth_hw_addr_set(netdev, addr->sa_data);286 287 fbnic_set_rx_mode(netdev);288 289 return 0;290}291 292void fbnic_clear_rx_mode(struct net_device *netdev)293{294 struct fbnic_net *fbn = netdev_priv(netdev);295 struct fbnic_dev *fbd = fbn->fbd;296 int idx;297 298 for (idx = ARRAY_SIZE(fbd->mac_addr); idx--;) {299 struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[idx];300 301 if (mac_addr->state != FBNIC_TCAM_S_VALID)302 continue;303 304 bitmap_clear(mac_addr->act_tcam,305 FBNIC_MAC_ADDR_T_HOST_START,306 FBNIC_MAC_ADDR_T_HOST_LEN);307 308 if (bitmap_empty(mac_addr->act_tcam,309 FBNIC_RPC_TCAM_ACT_NUM_ENTRIES))310 mac_addr->state = FBNIC_TCAM_S_DELETE;311 }312 313 /* Write updates to hardware */314 fbnic_write_macda(fbd);315 316 __dev_uc_unsync(netdev, NULL);317 __dev_mc_unsync(netdev, NULL);318}319 320static void fbnic_get_stats64(struct net_device *dev,321 struct rtnl_link_stats64 *stats64)322{323 u64 tx_bytes, tx_packets, tx_dropped = 0;324 u64 rx_bytes, rx_packets, rx_dropped = 0;325 struct fbnic_net *fbn = netdev_priv(dev);326 struct fbnic_queue_stats *stats;327 unsigned int start, i;328 329 stats = &fbn->tx_stats;330 331 tx_bytes = stats->bytes;332 tx_packets = stats->packets;333 tx_dropped = stats->dropped;334 335 stats64->tx_bytes = tx_bytes;336 stats64->tx_packets = tx_packets;337 stats64->tx_dropped = tx_dropped;338 339 for (i = 0; i < fbn->num_tx_queues; i++) {340 struct fbnic_ring *txr = fbn->tx[i];341 342 if (!txr)343 continue;344 345 stats = &txr->stats;346 do {347 start = u64_stats_fetch_begin(&stats->syncp);348 tx_bytes = stats->bytes;349 tx_packets = stats->packets;350 tx_dropped = stats->dropped;351 } while (u64_stats_fetch_retry(&stats->syncp, start));352 353 stats64->tx_bytes += tx_bytes;354 stats64->tx_packets += tx_packets;355 stats64->tx_dropped += tx_dropped;356 }357 358 stats = &fbn->rx_stats;359 360 rx_bytes = stats->bytes;361 rx_packets = stats->packets;362 rx_dropped = stats->dropped;363 364 stats64->rx_bytes = rx_bytes;365 stats64->rx_packets = rx_packets;366 stats64->rx_dropped = rx_dropped;367 368 for (i = 0; i < fbn->num_rx_queues; i++) {369 struct fbnic_ring *rxr = fbn->rx[i];370 371 if (!rxr)372 continue;373 374 stats = &rxr->stats;375 do {376 start = u64_stats_fetch_begin(&stats->syncp);377 rx_bytes = stats->bytes;378 rx_packets = stats->packets;379 rx_dropped = stats->dropped;380 } while (u64_stats_fetch_retry(&stats->syncp, start));381 382 stats64->rx_bytes += rx_bytes;383 stats64->rx_packets += rx_packets;384 stats64->rx_dropped += rx_dropped;385 }386}387 388static const struct net_device_ops fbnic_netdev_ops = {389 .ndo_open = fbnic_open,390 .ndo_stop = fbnic_stop,391 .ndo_validate_addr = eth_validate_addr,392 .ndo_start_xmit = fbnic_xmit_frame,393 .ndo_features_check = fbnic_features_check,394 .ndo_set_mac_address = fbnic_set_mac,395 .ndo_set_rx_mode = fbnic_set_rx_mode,396 .ndo_get_stats64 = fbnic_get_stats64,397};398 399static void fbnic_get_queue_stats_rx(struct net_device *dev, int idx,400 struct netdev_queue_stats_rx *rx)401{402 struct fbnic_net *fbn = netdev_priv(dev);403 struct fbnic_ring *rxr = fbn->rx[idx];404 struct fbnic_queue_stats *stats;405 unsigned int start;406 u64 bytes, packets;407 408 if (!rxr)409 return;410 411 stats = &rxr->stats;412 do {413 start = u64_stats_fetch_begin(&stats->syncp);414 bytes = stats->bytes;415 packets = stats->packets;416 } while (u64_stats_fetch_retry(&stats->syncp, start));417 418 rx->bytes = bytes;419 rx->packets = packets;420}421 422static void fbnic_get_queue_stats_tx(struct net_device *dev, int idx,423 struct netdev_queue_stats_tx *tx)424{425 struct fbnic_net *fbn = netdev_priv(dev);426 struct fbnic_ring *txr = fbn->tx[idx];427 struct fbnic_queue_stats *stats;428 unsigned int start;429 u64 bytes, packets;430 431 if (!txr)432 return;433 434 stats = &txr->stats;435 do {436 start = u64_stats_fetch_begin(&stats->syncp);437 bytes = stats->bytes;438 packets = stats->packets;439 } while (u64_stats_fetch_retry(&stats->syncp, start));440 441 tx->bytes = bytes;442 tx->packets = packets;443}444 445static void fbnic_get_base_stats(struct net_device *dev,446 struct netdev_queue_stats_rx *rx,447 struct netdev_queue_stats_tx *tx)448{449 struct fbnic_net *fbn = netdev_priv(dev);450 451 tx->bytes = fbn->tx_stats.bytes;452 tx->packets = fbn->tx_stats.packets;453 454 rx->bytes = fbn->rx_stats.bytes;455 rx->packets = fbn->rx_stats.packets;456}457 458static const struct netdev_stat_ops fbnic_stat_ops = {459 .get_queue_stats_rx = fbnic_get_queue_stats_rx,460 .get_queue_stats_tx = fbnic_get_queue_stats_tx,461 .get_base_stats = fbnic_get_base_stats,462};463 464void fbnic_reset_queues(struct fbnic_net *fbn,465 unsigned int tx, unsigned int rx)466{467 struct fbnic_dev *fbd = fbn->fbd;468 unsigned int max_napis;469 470 max_napis = fbd->num_irqs - FBNIC_NON_NAPI_VECTORS;471 472 tx = min(tx, max_napis);473 fbn->num_tx_queues = tx;474 475 rx = min(rx, max_napis);476 fbn->num_rx_queues = rx;477 478 fbn->num_napi = max(tx, rx);479}480 481/**482 * fbnic_netdev_free - Free the netdev associate with fbnic483 * @fbd: Driver specific structure to free netdev from484 *485 * Allocate and initialize the netdev and netdev private structure. Bind486 * together the hardware, netdev, and pci data structures.487 **/488void fbnic_netdev_free(struct fbnic_dev *fbd)489{490 struct fbnic_net *fbn = netdev_priv(fbd->netdev);491 492 if (fbn->phylink)493 phylink_destroy(fbn->phylink);494 495 free_netdev(fbd->netdev);496 fbd->netdev = NULL;497}498 499/**500 * fbnic_netdev_alloc - Allocate a netdev and associate with fbnic501 * @fbd: Driver specific structure to associate netdev with502 *503 * Allocate and initialize the netdev and netdev private structure. Bind504 * together the hardware, netdev, and pci data structures.505 *506 * Return: 0 on success, negative on failure507 **/508struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd)509{510 struct net_device *netdev;511 struct fbnic_net *fbn;512 int default_queues;513 514 netdev = alloc_etherdev_mq(sizeof(*fbn), FBNIC_MAX_RXQS);515 if (!netdev)516 return NULL;517 518 SET_NETDEV_DEV(netdev, fbd->dev);519 fbd->netdev = netdev;520 521 netdev->netdev_ops = &fbnic_netdev_ops;522 netdev->stat_ops = &fbnic_stat_ops;523 524 fbnic_set_ethtool_ops(netdev);525 526 fbn = netdev_priv(netdev);527 528 fbn->netdev = netdev;529 fbn->fbd = fbd;530 INIT_LIST_HEAD(&fbn->napis);531 532 fbn->txq_size = FBNIC_TXQ_SIZE_DEFAULT;533 fbn->hpq_size = FBNIC_HPQ_SIZE_DEFAULT;534 fbn->ppq_size = FBNIC_PPQ_SIZE_DEFAULT;535 fbn->rcq_size = FBNIC_RCQ_SIZE_DEFAULT;536 537 default_queues = netif_get_num_default_rss_queues();538 if (default_queues > fbd->max_num_queues)539 default_queues = fbd->max_num_queues;540 541 fbnic_reset_queues(fbn, default_queues, default_queues);542 543 fbnic_reset_indir_tbl(fbn);544 fbnic_rss_key_fill(fbn->rss_key);545 fbnic_rss_init_en_mask(fbn);546 547 netdev->features |=548 NETIF_F_RXHASH |549 NETIF_F_SG |550 NETIF_F_HW_CSUM |551 NETIF_F_RXCSUM;552 553 netdev->hw_features |= netdev->features;554 netdev->vlan_features |= netdev->features;555 netdev->hw_enc_features |= netdev->features;556 557 netdev->min_mtu = IPV6_MIN_MTU;558 netdev->max_mtu = FBNIC_MAX_JUMBO_FRAME_SIZE - ETH_HLEN;559 560 /* TBD: This is workaround for BMC as phylink doesn't have support561 * for leavling the link enabled if a BMC is present.562 */563 netdev->ethtool->wol_enabled = true;564 565 fbn->fec = FBNIC_FEC_AUTO | FBNIC_FEC_RS;566 fbn->link_mode = FBNIC_LINK_AUTO | FBNIC_LINK_50R2;567 netif_carrier_off(netdev);568 569 netif_tx_stop_all_queues(netdev);570 571 if (fbnic_phylink_init(netdev)) {572 fbnic_netdev_free(fbd);573 return NULL;574 }575 576 return netdev;577}578 579static int fbnic_dsn_to_mac_addr(u64 dsn, char *addr)580{581 addr[0] = (dsn >> 56) & 0xFF;582 addr[1] = (dsn >> 48) & 0xFF;583 addr[2] = (dsn >> 40) & 0xFF;584 addr[3] = (dsn >> 16) & 0xFF;585 addr[4] = (dsn >> 8) & 0xFF;586 addr[5] = dsn & 0xFF;587 588 return is_valid_ether_addr(addr) ? 0 : -EINVAL;589}590 591/**592 * fbnic_netdev_register - Initialize general software structures593 * @netdev: Netdev containing structure to initialize and register594 *595 * Initialize the MAC address for the netdev and register it.596 *597 * Return: 0 on success, negative on failure598 **/599int fbnic_netdev_register(struct net_device *netdev)600{601 struct fbnic_net *fbn = netdev_priv(netdev);602 struct fbnic_dev *fbd = fbn->fbd;603 u64 dsn = fbd->dsn;604 u8 addr[ETH_ALEN];605 int err;606 607 err = fbnic_dsn_to_mac_addr(dsn, addr);608 if (!err) {609 ether_addr_copy(netdev->perm_addr, addr);610 eth_hw_addr_set(netdev, addr);611 } else {612 /* A randomly assigned MAC address will cause provisioning613 * issues so instead just fail to spawn the netdev and614 * avoid any confusion.615 */616 dev_err(fbd->dev, "MAC addr %pM invalid\n", addr);617 return err;618 }619 620 return register_netdev(netdev);621}622 623void fbnic_netdev_unregister(struct net_device *netdev)624{625 unregister_netdev(netdev);626}627