16763 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright(c) 2013 - 2021 Intel Corporation. */3 4#include <generated/utsrelease.h>5#include <linux/crash_dump.h>6#include <linux/if_bridge.h>7#include <linux/if_macvlan.h>8#include <linux/module.h>9#include <net/pkt_cls.h>10#include <net/xdp_sock_drv.h>11 12/* Local includes */13#include "i40e.h"14#include "i40e_devids.h"15#include "i40e_diag.h"16#include "i40e_lan_hmc.h"17#include "i40e_virtchnl_pf.h"18#include "i40e_xsk.h"19 20/* All i40e tracepoints are defined by the include below, which21 * must be included exactly once across the whole kernel with22 * CREATE_TRACE_POINTS defined23 */24#define CREATE_TRACE_POINTS25#include "i40e_trace.h"26 27const char i40e_driver_name[] = "i40e";28static const char i40e_driver_string[] =29 "Intel(R) Ethernet Connection XL710 Network Driver";30 31static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation.";32 33/* a bit of forward declarations */34static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);35static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);36static int i40e_add_vsi(struct i40e_vsi *vsi);37static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);38static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired);39static int i40e_setup_misc_vector(struct i40e_pf *pf);40static void i40e_determine_queue_usage(struct i40e_pf *pf);41static int i40e_setup_pf_filter_control(struct i40e_pf *pf);42static void i40e_prep_for_reset(struct i40e_pf *pf);43static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,44 bool lock_acquired);45static int i40e_reset(struct i40e_pf *pf);46static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);47static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf);48static int i40e_restore_interrupt_scheme(struct i40e_pf *pf);49static bool i40e_check_recovery_mode(struct i40e_pf *pf);50static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw);51static void i40e_fdir_sb_setup(struct i40e_pf *pf);52static int i40e_veb_get_bw_info(struct i40e_veb *veb);53static int i40e_get_capabilities(struct i40e_pf *pf,54 enum i40e_admin_queue_opc list_type);55static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf);56 57/* i40e_pci_tbl - PCI Device ID Table58 *59 * Last entry must be all 0s60 *61 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,62 * Class, Class Mask, private data (not used) }63 */64static const struct pci_device_id i40e_pci_tbl[] = {65 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},66 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},67 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},68 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},69 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},70 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},71 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},72 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_BC), 0},73 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},74 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},75 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0},76 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0},77 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0},78 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},79 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},80 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},81 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},82 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},83 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},84 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722_A), 0},85 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},86 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},87 {PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0},88 {PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0},89 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},90 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},91 /* required last entry */92 {0, }93};94MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);95 96#define I40E_MAX_VF_COUNT 12897static int debug = -1;98module_param(debug, uint, 0);99MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");100 101MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");102MODULE_IMPORT_NS(LIBIE);103MODULE_LICENSE("GPL v2");104 105static struct workqueue_struct *i40e_wq;106 107static void netdev_hw_addr_refcnt(struct i40e_mac_filter *f,108 struct net_device *netdev, int delta)109{110 struct netdev_hw_addr_list *ha_list;111 struct netdev_hw_addr *ha;112 113 if (!f || !netdev)114 return;115 116 if (is_unicast_ether_addr(f->macaddr) || is_link_local_ether_addr(f->macaddr))117 ha_list = &netdev->uc;118 else119 ha_list = &netdev->mc;120 121 netdev_hw_addr_list_for_each(ha, ha_list) {122 if (ether_addr_equal(ha->addr, f->macaddr)) {123 ha->refcount += delta;124 if (ha->refcount <= 0)125 ha->refcount = 1;126 break;127 }128 }129}130 131/**132 * i40e_hw_to_dev - get device pointer from the hardware structure133 * @hw: pointer to the device HW structure134 **/135struct device *i40e_hw_to_dev(struct i40e_hw *hw)136{137 struct i40e_pf *pf = i40e_hw_to_pf(hw);138 139 return &pf->pdev->dev;140}141 142/**143 * i40e_allocate_dma_mem - OS specific memory alloc for shared code144 * @hw: pointer to the HW structure145 * @mem: ptr to mem struct to fill out146 * @size: size of memory requested147 * @alignment: what to align the allocation to148 **/149int i40e_allocate_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem,150 u64 size, u32 alignment)151{152 struct i40e_pf *pf = i40e_hw_to_pf(hw);153 154 mem->size = ALIGN(size, alignment);155 mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa,156 GFP_KERNEL);157 if (!mem->va)158 return -ENOMEM;159 160 return 0;161}162 163/**164 * i40e_free_dma_mem - OS specific memory free for shared code165 * @hw: pointer to the HW structure166 * @mem: ptr to mem struct to free167 **/168int i40e_free_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem)169{170 struct i40e_pf *pf = i40e_hw_to_pf(hw);171 172 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);173 mem->va = NULL;174 mem->pa = 0;175 mem->size = 0;176 177 return 0;178}179 180/**181 * i40e_allocate_virt_mem - OS specific memory alloc for shared code182 * @hw: pointer to the HW structure183 * @mem: ptr to mem struct to fill out184 * @size: size of memory requested185 **/186int i40e_allocate_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem,187 u32 size)188{189 mem->size = size;190 mem->va = kzalloc(size, GFP_KERNEL);191 192 if (!mem->va)193 return -ENOMEM;194 195 return 0;196}197 198/**199 * i40e_free_virt_mem - OS specific memory free for shared code200 * @hw: pointer to the HW structure201 * @mem: ptr to mem struct to free202 **/203int i40e_free_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem)204{205 /* it's ok to kfree a NULL pointer */206 kfree(mem->va);207 mem->va = NULL;208 mem->size = 0;209 210 return 0;211}212 213/**214 * i40e_get_lump - find a lump of free generic resource215 * @pf: board private structure216 * @pile: the pile of resource to search217 * @needed: the number of items needed218 * @id: an owner id to stick on the items assigned219 *220 * Returns the base item index of the lump, or negative for error221 **/222static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,223 u16 needed, u16 id)224{225 int ret = -ENOMEM;226 int i, j;227 228 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {229 dev_info(&pf->pdev->dev,230 "param err: pile=%s needed=%d id=0x%04x\n",231 pile ? "<valid>" : "<null>", needed, id);232 return -EINVAL;233 }234 235 /* Allocate last queue in the pile for FDIR VSI queue236 * so it doesn't fragment the qp_pile237 */238 if (pile == pf->qp_pile && pf->vsi[id]->type == I40E_VSI_FDIR) {239 if (pile->list[pile->num_entries - 1] & I40E_PILE_VALID_BIT) {240 dev_err(&pf->pdev->dev,241 "Cannot allocate queue %d for I40E_VSI_FDIR\n",242 pile->num_entries - 1);243 return -ENOMEM;244 }245 pile->list[pile->num_entries - 1] = id | I40E_PILE_VALID_BIT;246 return pile->num_entries - 1;247 }248 249 i = 0;250 while (i < pile->num_entries) {251 /* skip already allocated entries */252 if (pile->list[i] & I40E_PILE_VALID_BIT) {253 i++;254 continue;255 }256 257 /* do we have enough in this lump? */258 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {259 if (pile->list[i+j] & I40E_PILE_VALID_BIT)260 break;261 }262 263 if (j == needed) {264 /* there was enough, so assign it to the requestor */265 for (j = 0; j < needed; j++)266 pile->list[i+j] = id | I40E_PILE_VALID_BIT;267 ret = i;268 break;269 }270 271 /* not enough, so skip over it and continue looking */272 i += j;273 }274 275 return ret;276}277 278/**279 * i40e_put_lump - return a lump of generic resource280 * @pile: the pile of resource to search281 * @index: the base item index282 * @id: the owner id of the items assigned283 *284 * Returns the count of items in the lump285 **/286static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)287{288 int valid_id = (id | I40E_PILE_VALID_BIT);289 int count = 0;290 u16 i;291 292 if (!pile || index >= pile->num_entries)293 return -EINVAL;294 295 for (i = index;296 i < pile->num_entries && pile->list[i] == valid_id;297 i++) {298 pile->list[i] = 0;299 count++;300 }301 302 303 return count;304}305 306/**307 * i40e_find_vsi_from_id - searches for the vsi with the given id308 * @pf: the pf structure to search for the vsi309 * @id: id of the vsi it is searching for310 **/311struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)312{313 struct i40e_vsi *vsi;314 int i;315 316 i40e_pf_for_each_vsi(pf, i, vsi)317 if (vsi->id == id)318 return vsi;319 320 return NULL;321}322 323/**324 * i40e_service_event_schedule - Schedule the service task to wake up325 * @pf: board private structure326 *327 * If not already scheduled, this puts the task into the work queue328 **/329void i40e_service_event_schedule(struct i40e_pf *pf)330{331 if ((!test_bit(__I40E_DOWN, pf->state) &&332 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) ||333 test_bit(__I40E_RECOVERY_MODE, pf->state))334 queue_work(i40e_wq, &pf->service_task);335}336 337/**338 * i40e_tx_timeout - Respond to a Tx Hang339 * @netdev: network interface device structure340 * @txqueue: queue number timing out341 *342 * If any port has noticed a Tx timeout, it is likely that the whole343 * device is munged, not just the one netdev port, so go for the full344 * reset.345 **/346static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue)347{348 struct i40e_netdev_priv *np = netdev_priv(netdev);349 struct i40e_vsi *vsi = np->vsi;350 struct i40e_pf *pf = vsi->back;351 struct i40e_ring *tx_ring = NULL;352 unsigned int i;353 u32 head, val;354 355 pf->tx_timeout_count++;356 357 /* with txqueue index, find the tx_ring struct */358 for (i = 0; i < vsi->num_queue_pairs; i++) {359 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {360 if (txqueue ==361 vsi->tx_rings[i]->queue_index) {362 tx_ring = vsi->tx_rings[i];363 break;364 }365 }366 }367 368 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))369 pf->tx_timeout_recovery_level = 1; /* reset after some time */370 else if (time_before(jiffies,371 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))372 return; /* don't do any new action before the next timeout */373 374 /* don't kick off another recovery if one is already pending */375 if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state))376 return;377 378 if (tx_ring) {379 head = i40e_get_head(tx_ring);380 /* Read interrupt register */381 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))382 val = rd32(&pf->hw,383 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +384 tx_ring->vsi->base_vector - 1));385 else386 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);387 388 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",389 vsi->seid, txqueue, tx_ring->next_to_clean,390 head, tx_ring->next_to_use,391 readl(tx_ring->tail), val);392 }393 394 pf->tx_timeout_last_recovery = jiffies;395 netdev_info(netdev, "tx_timeout recovery level %d, txqueue %d\n",396 pf->tx_timeout_recovery_level, txqueue);397 398 switch (pf->tx_timeout_recovery_level) {399 case 1:400 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);401 break;402 case 2:403 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);404 break;405 case 3:406 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);407 break;408 default:409 netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in non-recoverable state.\n");410 set_bit(__I40E_DOWN_REQUESTED, pf->state);411 set_bit(__I40E_VSI_DOWN_REQUESTED, vsi->state);412 break;413 }414 415 i40e_service_event_schedule(pf);416 pf->tx_timeout_recovery_level++;417}418 419/**420 * i40e_get_vsi_stats_struct - Get System Network Statistics421 * @vsi: the VSI we care about422 *423 * Returns the address of the device statistics structure.424 * The statistics are actually updated from the service task.425 **/426struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)427{428 return &vsi->net_stats;429}430 431/**432 * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring433 * @ring: Tx ring to get statistics from434 * @stats: statistics entry to be updated435 **/436static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,437 struct rtnl_link_stats64 *stats)438{439 u64 bytes, packets;440 unsigned int start;441 442 do {443 start = u64_stats_fetch_begin(&ring->syncp);444 packets = ring->stats.packets;445 bytes = ring->stats.bytes;446 } while (u64_stats_fetch_retry(&ring->syncp, start));447 448 stats->tx_packets += packets;449 stats->tx_bytes += bytes;450}451 452/**453 * i40e_get_netdev_stats_struct - Get statistics for netdev interface454 * @netdev: network interface device structure455 * @stats: data structure to store statistics456 *457 * Returns the address of the device statistics structure.458 * The statistics are actually updated from the service task.459 **/460static void i40e_get_netdev_stats_struct(struct net_device *netdev,461 struct rtnl_link_stats64 *stats)462{463 struct i40e_netdev_priv *np = netdev_priv(netdev);464 struct i40e_vsi *vsi = np->vsi;465 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);466 struct i40e_ring *ring;467 int i;468 469 if (test_bit(__I40E_VSI_DOWN, vsi->state))470 return;471 472 if (!vsi->tx_rings)473 return;474 475 rcu_read_lock();476 for (i = 0; i < vsi->num_queue_pairs; i++) {477 u64 bytes, packets;478 unsigned int start;479 480 ring = READ_ONCE(vsi->tx_rings[i]);481 if (!ring)482 continue;483 i40e_get_netdev_stats_struct_tx(ring, stats);484 485 if (i40e_enabled_xdp_vsi(vsi)) {486 ring = READ_ONCE(vsi->xdp_rings[i]);487 if (!ring)488 continue;489 i40e_get_netdev_stats_struct_tx(ring, stats);490 }491 492 ring = READ_ONCE(vsi->rx_rings[i]);493 if (!ring)494 continue;495 do {496 start = u64_stats_fetch_begin(&ring->syncp);497 packets = ring->stats.packets;498 bytes = ring->stats.bytes;499 } while (u64_stats_fetch_retry(&ring->syncp, start));500 501 stats->rx_packets += packets;502 stats->rx_bytes += bytes;503 504 }505 rcu_read_unlock();506 507 /* following stats updated by i40e_watchdog_subtask() */508 stats->multicast = vsi_stats->multicast;509 stats->tx_errors = vsi_stats->tx_errors;510 stats->tx_dropped = vsi_stats->tx_dropped;511 stats->rx_errors = vsi_stats->rx_errors;512 stats->rx_dropped = vsi_stats->rx_dropped;513 stats->rx_missed_errors = vsi_stats->rx_missed_errors;514 stats->rx_crc_errors = vsi_stats->rx_crc_errors;515 stats->rx_length_errors = vsi_stats->rx_length_errors;516}517 518/**519 * i40e_vsi_reset_stats - Resets all stats of the given vsi520 * @vsi: the VSI to have its stats reset521 **/522void i40e_vsi_reset_stats(struct i40e_vsi *vsi)523{524 struct rtnl_link_stats64 *ns;525 int i;526 527 if (!vsi)528 return;529 530 ns = i40e_get_vsi_stats_struct(vsi);531 memset(ns, 0, sizeof(*ns));532 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));533 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));534 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));535 if (vsi->rx_rings && vsi->rx_rings[0]) {536 for (i = 0; i < vsi->num_queue_pairs; i++) {537 memset(&vsi->rx_rings[i]->stats, 0,538 sizeof(vsi->rx_rings[i]->stats));539 memset(&vsi->rx_rings[i]->rx_stats, 0,540 sizeof(vsi->rx_rings[i]->rx_stats));541 memset(&vsi->tx_rings[i]->stats, 0,542 sizeof(vsi->tx_rings[i]->stats));543 memset(&vsi->tx_rings[i]->tx_stats, 0,544 sizeof(vsi->tx_rings[i]->tx_stats));545 }546 }547 vsi->stat_offsets_loaded = false;548}549 550/**551 * i40e_pf_reset_stats - Reset all of the stats for the given PF552 * @pf: the PF to be reset553 **/554void i40e_pf_reset_stats(struct i40e_pf *pf)555{556 struct i40e_veb *veb;557 int i;558 559 memset(&pf->stats, 0, sizeof(pf->stats));560 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));561 pf->stat_offsets_loaded = false;562 563 i40e_pf_for_each_veb(pf, i, veb) {564 memset(&veb->stats, 0, sizeof(veb->stats));565 memset(&veb->stats_offsets, 0, sizeof(veb->stats_offsets));566 memset(&veb->tc_stats, 0, sizeof(veb->tc_stats));567 memset(&veb->tc_stats_offsets, 0, sizeof(veb->tc_stats_offsets));568 veb->stat_offsets_loaded = false;569 }570 pf->hw_csum_rx_error = 0;571}572 573/**574 * i40e_compute_pci_to_hw_id - compute index form PCI function.575 * @vsi: ptr to the VSI to read from.576 * @hw: ptr to the hardware info.577 **/578static u32 i40e_compute_pci_to_hw_id(struct i40e_vsi *vsi, struct i40e_hw *hw)579{580 int pf_count = i40e_get_pf_count(hw);581 582 if (vsi->type == I40E_VSI_SRIOV)583 return (hw->port * BIT(7)) / pf_count + vsi->vf_id;584 585 return hw->port + BIT(7);586}587 588/**589 * i40e_stat_update64 - read and update a 64 bit stat from the chip.590 * @hw: ptr to the hardware info.591 * @hireg: the high 32 bit reg to read.592 * @loreg: the low 32 bit reg to read.593 * @offset_loaded: has the initial offset been loaded yet.594 * @offset: ptr to current offset value.595 * @stat: ptr to the stat.596 *597 * Since the device stats are not reset at PFReset, they will not598 * be zeroed when the driver starts. We'll save the first values read599 * and use them as offsets to be subtracted from the raw values in order600 * to report stats that count from zero.601 **/602static void i40e_stat_update64(struct i40e_hw *hw, u32 hireg, u32 loreg,603 bool offset_loaded, u64 *offset, u64 *stat)604{605 u64 new_data;606 607 new_data = rd64(hw, loreg);608 609 if (!offset_loaded || new_data < *offset)610 *offset = new_data;611 *stat = new_data - *offset;612}613 614/**615 * i40e_stat_update48 - read and update a 48 bit stat from the chip616 * @hw: ptr to the hardware info617 * @hireg: the high 32 bit reg to read618 * @loreg: the low 32 bit reg to read619 * @offset_loaded: has the initial offset been loaded yet620 * @offset: ptr to current offset value621 * @stat: ptr to the stat622 *623 * Since the device stats are not reset at PFReset, they likely will not624 * be zeroed when the driver starts. We'll save the first values read625 * and use them as offsets to be subtracted from the raw values in order626 * to report stats that count from zero. In the process, we also manage627 * the potential roll-over.628 **/629static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,630 bool offset_loaded, u64 *offset, u64 *stat)631{632 u64 new_data;633 634 if (hw->device_id == I40E_DEV_ID_QEMU) {635 new_data = rd32(hw, loreg);636 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;637 } else {638 new_data = rd64(hw, loreg);639 }640 if (!offset_loaded)641 *offset = new_data;642 if (likely(new_data >= *offset))643 *stat = new_data - *offset;644 else645 *stat = (new_data + BIT_ULL(48)) - *offset;646 *stat &= 0xFFFFFFFFFFFFULL;647}648 649/**650 * i40e_stat_update32 - read and update a 32 bit stat from the chip651 * @hw: ptr to the hardware info652 * @reg: the hw reg to read653 * @offset_loaded: has the initial offset been loaded yet654 * @offset: ptr to current offset value655 * @stat: ptr to the stat656 **/657static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,658 bool offset_loaded, u64 *offset, u64 *stat)659{660 u32 new_data;661 662 new_data = rd32(hw, reg);663 if (!offset_loaded)664 *offset = new_data;665 if (likely(new_data >= *offset))666 *stat = (u32)(new_data - *offset);667 else668 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);669}670 671/**672 * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat673 * @hw: ptr to the hardware info674 * @reg: the hw reg to read and clear675 * @stat: ptr to the stat676 **/677static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat)678{679 u32 new_data = rd32(hw, reg);680 681 wr32(hw, reg, 1); /* must write a nonzero value to clear register */682 *stat += new_data;683}684 685/**686 * i40e_stats_update_rx_discards - update rx_discards.687 * @vsi: ptr to the VSI to be updated.688 * @hw: ptr to the hardware info.689 * @stat_idx: VSI's stat_counter_idx.690 * @offset_loaded: ptr to the VSI's stat_offsets_loaded.691 * @stat_offset: ptr to stat_offset to store first read of specific register.692 * @stat: ptr to VSI's stat to be updated.693 **/694static void695i40e_stats_update_rx_discards(struct i40e_vsi *vsi, struct i40e_hw *hw,696 int stat_idx, bool offset_loaded,697 struct i40e_eth_stats *stat_offset,698 struct i40e_eth_stats *stat)699{700 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), offset_loaded,701 &stat_offset->rx_discards, &stat->rx_discards);702 i40e_stat_update64(hw,703 I40E_GL_RXERR1H(i40e_compute_pci_to_hw_id(vsi, hw)),704 I40E_GL_RXERR1L(i40e_compute_pci_to_hw_id(vsi, hw)),705 offset_loaded, &stat_offset->rx_discards_other,706 &stat->rx_discards_other);707}708 709/**710 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.711 * @vsi: the VSI to be updated712 **/713void i40e_update_eth_stats(struct i40e_vsi *vsi)714{715 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);716 struct i40e_pf *pf = vsi->back;717 struct i40e_hw *hw = &pf->hw;718 struct i40e_eth_stats *oes;719 struct i40e_eth_stats *es; /* device's eth stats */720 721 es = &vsi->eth_stats;722 oes = &vsi->eth_stats_offsets;723 724 /* Gather up the stats that the hw collects */725 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),726 vsi->stat_offsets_loaded,727 &oes->tx_errors, &es->tx_errors);728 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),729 vsi->stat_offsets_loaded,730 &oes->rx_unknown_protocol, &es->rx_unknown_protocol);731 732 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),733 I40E_GLV_GORCL(stat_idx),734 vsi->stat_offsets_loaded,735 &oes->rx_bytes, &es->rx_bytes);736 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),737 I40E_GLV_UPRCL(stat_idx),738 vsi->stat_offsets_loaded,739 &oes->rx_unicast, &es->rx_unicast);740 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),741 I40E_GLV_MPRCL(stat_idx),742 vsi->stat_offsets_loaded,743 &oes->rx_multicast, &es->rx_multicast);744 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),745 I40E_GLV_BPRCL(stat_idx),746 vsi->stat_offsets_loaded,747 &oes->rx_broadcast, &es->rx_broadcast);748 749 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),750 I40E_GLV_GOTCL(stat_idx),751 vsi->stat_offsets_loaded,752 &oes->tx_bytes, &es->tx_bytes);753 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),754 I40E_GLV_UPTCL(stat_idx),755 vsi->stat_offsets_loaded,756 &oes->tx_unicast, &es->tx_unicast);757 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),758 I40E_GLV_MPTCL(stat_idx),759 vsi->stat_offsets_loaded,760 &oes->tx_multicast, &es->tx_multicast);761 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),762 I40E_GLV_BPTCL(stat_idx),763 vsi->stat_offsets_loaded,764 &oes->tx_broadcast, &es->tx_broadcast);765 766 i40e_stats_update_rx_discards(vsi, hw, stat_idx,767 vsi->stat_offsets_loaded, oes, es);768 769 vsi->stat_offsets_loaded = true;770}771 772/**773 * i40e_update_veb_stats - Update Switch component statistics774 * @veb: the VEB being updated775 **/776void i40e_update_veb_stats(struct i40e_veb *veb)777{778 struct i40e_pf *pf = veb->pf;779 struct i40e_hw *hw = &pf->hw;780 struct i40e_eth_stats *oes;781 struct i40e_eth_stats *es; /* device's eth stats */782 struct i40e_veb_tc_stats *veb_oes;783 struct i40e_veb_tc_stats *veb_es;784 int i, idx = 0;785 786 idx = veb->stats_idx;787 es = &veb->stats;788 oes = &veb->stats_offsets;789 veb_es = &veb->tc_stats;790 veb_oes = &veb->tc_stats_offsets;791 792 /* Gather up the stats that the hw collects */793 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),794 veb->stat_offsets_loaded,795 &oes->tx_discards, &es->tx_discards);796 if (hw->revision_id > 0)797 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),798 veb->stat_offsets_loaded,799 &oes->rx_unknown_protocol,800 &es->rx_unknown_protocol);801 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),802 veb->stat_offsets_loaded,803 &oes->rx_bytes, &es->rx_bytes);804 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),805 veb->stat_offsets_loaded,806 &oes->rx_unicast, &es->rx_unicast);807 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),808 veb->stat_offsets_loaded,809 &oes->rx_multicast, &es->rx_multicast);810 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),811 veb->stat_offsets_loaded,812 &oes->rx_broadcast, &es->rx_broadcast);813 814 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),815 veb->stat_offsets_loaded,816 &oes->tx_bytes, &es->tx_bytes);817 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),818 veb->stat_offsets_loaded,819 &oes->tx_unicast, &es->tx_unicast);820 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),821 veb->stat_offsets_loaded,822 &oes->tx_multicast, &es->tx_multicast);823 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),824 veb->stat_offsets_loaded,825 &oes->tx_broadcast, &es->tx_broadcast);826 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {827 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),828 I40E_GLVEBTC_RPCL(i, idx),829 veb->stat_offsets_loaded,830 &veb_oes->tc_rx_packets[i],831 &veb_es->tc_rx_packets[i]);832 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),833 I40E_GLVEBTC_RBCL(i, idx),834 veb->stat_offsets_loaded,835 &veb_oes->tc_rx_bytes[i],836 &veb_es->tc_rx_bytes[i]);837 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),838 I40E_GLVEBTC_TPCL(i, idx),839 veb->stat_offsets_loaded,840 &veb_oes->tc_tx_packets[i],841 &veb_es->tc_tx_packets[i]);842 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),843 I40E_GLVEBTC_TBCL(i, idx),844 veb->stat_offsets_loaded,845 &veb_oes->tc_tx_bytes[i],846 &veb_es->tc_tx_bytes[i]);847 }848 veb->stat_offsets_loaded = true;849}850 851/**852 * i40e_update_vsi_stats - Update the vsi statistics counters.853 * @vsi: the VSI to be updated854 *855 * There are a few instances where we store the same stat in a856 * couple of different structs. This is partly because we have857 * the netdev stats that need to be filled out, which is slightly858 * different from the "eth_stats" defined by the chip and used in859 * VF communications. We sort it out here.860 **/861static void i40e_update_vsi_stats(struct i40e_vsi *vsi)862{863 u64 rx_page, rx_buf, rx_reuse, rx_alloc, rx_waive, rx_busy;864 struct i40e_pf *pf = vsi->back;865 struct rtnl_link_stats64 *ons;866 struct rtnl_link_stats64 *ns; /* netdev stats */867 struct i40e_eth_stats *oes;868 struct i40e_eth_stats *es; /* device's eth stats */869 u64 tx_restart, tx_busy;870 struct i40e_ring *p;871 u64 bytes, packets;872 unsigned int start;873 u64 tx_linearize;874 u64 tx_force_wb;875 u64 tx_stopped;876 u64 rx_p, rx_b;877 u64 tx_p, tx_b;878 u16 q;879 880 if (test_bit(__I40E_VSI_DOWN, vsi->state) ||881 test_bit(__I40E_CONFIG_BUSY, pf->state))882 return;883 884 ns = i40e_get_vsi_stats_struct(vsi);885 ons = &vsi->net_stats_offsets;886 es = &vsi->eth_stats;887 oes = &vsi->eth_stats_offsets;888 889 /* Gather up the netdev and vsi stats that the driver collects890 * on the fly during packet processing891 */892 rx_b = rx_p = 0;893 tx_b = tx_p = 0;894 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;895 tx_stopped = 0;896 rx_page = 0;897 rx_buf = 0;898 rx_reuse = 0;899 rx_alloc = 0;900 rx_waive = 0;901 rx_busy = 0;902 rcu_read_lock();903 for (q = 0; q < vsi->num_queue_pairs; q++) {904 /* locate Tx ring */905 p = READ_ONCE(vsi->tx_rings[q]);906 if (!p)907 continue;908 909 do {910 start = u64_stats_fetch_begin(&p->syncp);911 packets = p->stats.packets;912 bytes = p->stats.bytes;913 } while (u64_stats_fetch_retry(&p->syncp, start));914 tx_b += bytes;915 tx_p += packets;916 tx_restart += p->tx_stats.restart_queue;917 tx_busy += p->tx_stats.tx_busy;918 tx_linearize += p->tx_stats.tx_linearize;919 tx_force_wb += p->tx_stats.tx_force_wb;920 tx_stopped += p->tx_stats.tx_stopped;921 922 /* locate Rx ring */923 p = READ_ONCE(vsi->rx_rings[q]);924 if (!p)925 continue;926 927 do {928 start = u64_stats_fetch_begin(&p->syncp);929 packets = p->stats.packets;930 bytes = p->stats.bytes;931 } while (u64_stats_fetch_retry(&p->syncp, start));932 rx_b += bytes;933 rx_p += packets;934 rx_buf += p->rx_stats.alloc_buff_failed;935 rx_page += p->rx_stats.alloc_page_failed;936 rx_reuse += p->rx_stats.page_reuse_count;937 rx_alloc += p->rx_stats.page_alloc_count;938 rx_waive += p->rx_stats.page_waive_count;939 rx_busy += p->rx_stats.page_busy_count;940 941 if (i40e_enabled_xdp_vsi(vsi)) {942 /* locate XDP ring */943 p = READ_ONCE(vsi->xdp_rings[q]);944 if (!p)945 continue;946 947 do {948 start = u64_stats_fetch_begin(&p->syncp);949 packets = p->stats.packets;950 bytes = p->stats.bytes;951 } while (u64_stats_fetch_retry(&p->syncp, start));952 tx_b += bytes;953 tx_p += packets;954 tx_restart += p->tx_stats.restart_queue;955 tx_busy += p->tx_stats.tx_busy;956 tx_linearize += p->tx_stats.tx_linearize;957 tx_force_wb += p->tx_stats.tx_force_wb;958 }959 }960 rcu_read_unlock();961 vsi->tx_restart = tx_restart;962 vsi->tx_busy = tx_busy;963 vsi->tx_linearize = tx_linearize;964 vsi->tx_force_wb = tx_force_wb;965 vsi->tx_stopped = tx_stopped;966 vsi->rx_page_failed = rx_page;967 vsi->rx_buf_failed = rx_buf;968 vsi->rx_page_reuse = rx_reuse;969 vsi->rx_page_alloc = rx_alloc;970 vsi->rx_page_waive = rx_waive;971 vsi->rx_page_busy = rx_busy;972 973 ns->rx_packets = rx_p;974 ns->rx_bytes = rx_b;975 ns->tx_packets = tx_p;976 ns->tx_bytes = tx_b;977 978 /* update netdev stats from eth stats */979 i40e_update_eth_stats(vsi);980 ons->tx_errors = oes->tx_errors;981 ns->tx_errors = es->tx_errors;982 ons->multicast = oes->rx_multicast;983 ns->multicast = es->rx_multicast;984 ons->rx_dropped = oes->rx_discards_other;985 ns->rx_dropped = es->rx_discards_other;986 ons->rx_missed_errors = oes->rx_discards;987 ns->rx_missed_errors = es->rx_discards;988 ons->tx_dropped = oes->tx_discards;989 ns->tx_dropped = es->tx_discards;990 991 /* pull in a couple PF stats if this is the main vsi */992 if (vsi->type == I40E_VSI_MAIN) {993 ns->rx_crc_errors = pf->stats.crc_errors;994 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;995 ns->rx_length_errors = pf->stats.rx_length_errors;996 }997}998 999/**1000 * i40e_update_pf_stats - Update the PF statistics counters.1001 * @pf: the PF to be updated1002 **/1003static void i40e_update_pf_stats(struct i40e_pf *pf)1004{1005 struct i40e_hw_port_stats *osd = &pf->stats_offsets;1006 struct i40e_hw_port_stats *nsd = &pf->stats;1007 struct i40e_hw *hw = &pf->hw;1008 u32 val;1009 int i;1010 1011 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),1012 I40E_GLPRT_GORCL(hw->port),1013 pf->stat_offsets_loaded,1014 &osd->eth.rx_bytes, &nsd->eth.rx_bytes);1015 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),1016 I40E_GLPRT_GOTCL(hw->port),1017 pf->stat_offsets_loaded,1018 &osd->eth.tx_bytes, &nsd->eth.tx_bytes);1019 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),1020 pf->stat_offsets_loaded,1021 &osd->eth.rx_discards,1022 &nsd->eth.rx_discards);1023 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),1024 I40E_GLPRT_UPRCL(hw->port),1025 pf->stat_offsets_loaded,1026 &osd->eth.rx_unicast,1027 &nsd->eth.rx_unicast);1028 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),1029 I40E_GLPRT_MPRCL(hw->port),1030 pf->stat_offsets_loaded,1031 &osd->eth.rx_multicast,1032 &nsd->eth.rx_multicast);1033 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),1034 I40E_GLPRT_BPRCL(hw->port),1035 pf->stat_offsets_loaded,1036 &osd->eth.rx_broadcast,1037 &nsd->eth.rx_broadcast);1038 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),1039 I40E_GLPRT_UPTCL(hw->port),1040 pf->stat_offsets_loaded,1041 &osd->eth.tx_unicast,1042 &nsd->eth.tx_unicast);1043 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),1044 I40E_GLPRT_MPTCL(hw->port),1045 pf->stat_offsets_loaded,1046 &osd->eth.tx_multicast,1047 &nsd->eth.tx_multicast);1048 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),1049 I40E_GLPRT_BPTCL(hw->port),1050 pf->stat_offsets_loaded,1051 &osd->eth.tx_broadcast,1052 &nsd->eth.tx_broadcast);1053 1054 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),1055 pf->stat_offsets_loaded,1056 &osd->tx_dropped_link_down,1057 &nsd->tx_dropped_link_down);1058 1059 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),1060 pf->stat_offsets_loaded,1061 &osd->crc_errors, &nsd->crc_errors);1062 1063 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),1064 pf->stat_offsets_loaded,1065 &osd->illegal_bytes, &nsd->illegal_bytes);1066 1067 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),1068 pf->stat_offsets_loaded,1069 &osd->mac_local_faults,1070 &nsd->mac_local_faults);1071 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),1072 pf->stat_offsets_loaded,1073 &osd->mac_remote_faults,1074 &nsd->mac_remote_faults);1075 1076 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),1077 pf->stat_offsets_loaded,1078 &osd->rx_length_errors,1079 &nsd->rx_length_errors);1080 1081 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),1082 pf->stat_offsets_loaded,1083 &osd->link_xon_rx, &nsd->link_xon_rx);1084 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),1085 pf->stat_offsets_loaded,1086 &osd->link_xon_tx, &nsd->link_xon_tx);1087 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),1088 pf->stat_offsets_loaded,1089 &osd->link_xoff_rx, &nsd->link_xoff_rx);1090 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),1091 pf->stat_offsets_loaded,1092 &osd->link_xoff_tx, &nsd->link_xoff_tx);1093 1094 for (i = 0; i < 8; i++) {1095 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),1096 pf->stat_offsets_loaded,1097 &osd->priority_xoff_rx[i],1098 &nsd->priority_xoff_rx[i]);1099 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),1100 pf->stat_offsets_loaded,1101 &osd->priority_xon_rx[i],1102 &nsd->priority_xon_rx[i]);1103 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),1104 pf->stat_offsets_loaded,1105 &osd->priority_xon_tx[i],1106 &nsd->priority_xon_tx[i]);1107 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),1108 pf->stat_offsets_loaded,1109 &osd->priority_xoff_tx[i],1110 &nsd->priority_xoff_tx[i]);1111 i40e_stat_update32(hw,1112 I40E_GLPRT_RXON2OFFCNT(hw->port, i),1113 pf->stat_offsets_loaded,1114 &osd->priority_xon_2_xoff[i],1115 &nsd->priority_xon_2_xoff[i]);1116 }1117 1118 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),1119 I40E_GLPRT_PRC64L(hw->port),1120 pf->stat_offsets_loaded,1121 &osd->rx_size_64, &nsd->rx_size_64);1122 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),1123 I40E_GLPRT_PRC127L(hw->port),1124 pf->stat_offsets_loaded,1125 &osd->rx_size_127, &nsd->rx_size_127);1126 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),1127 I40E_GLPRT_PRC255L(hw->port),1128 pf->stat_offsets_loaded,1129 &osd->rx_size_255, &nsd->rx_size_255);1130 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),1131 I40E_GLPRT_PRC511L(hw->port),1132 pf->stat_offsets_loaded,1133 &osd->rx_size_511, &nsd->rx_size_511);1134 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),1135 I40E_GLPRT_PRC1023L(hw->port),1136 pf->stat_offsets_loaded,1137 &osd->rx_size_1023, &nsd->rx_size_1023);1138 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),1139 I40E_GLPRT_PRC1522L(hw->port),1140 pf->stat_offsets_loaded,1141 &osd->rx_size_1522, &nsd->rx_size_1522);1142 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),1143 I40E_GLPRT_PRC9522L(hw->port),1144 pf->stat_offsets_loaded,1145 &osd->rx_size_big, &nsd->rx_size_big);1146 1147 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),1148 I40E_GLPRT_PTC64L(hw->port),1149 pf->stat_offsets_loaded,1150 &osd->tx_size_64, &nsd->tx_size_64);1151 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),1152 I40E_GLPRT_PTC127L(hw->port),1153 pf->stat_offsets_loaded,1154 &osd->tx_size_127, &nsd->tx_size_127);1155 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),1156 I40E_GLPRT_PTC255L(hw->port),1157 pf->stat_offsets_loaded,1158 &osd->tx_size_255, &nsd->tx_size_255);1159 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),1160 I40E_GLPRT_PTC511L(hw->port),1161 pf->stat_offsets_loaded,1162 &osd->tx_size_511, &nsd->tx_size_511);1163 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),1164 I40E_GLPRT_PTC1023L(hw->port),1165 pf->stat_offsets_loaded,1166 &osd->tx_size_1023, &nsd->tx_size_1023);1167 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),1168 I40E_GLPRT_PTC1522L(hw->port),1169 pf->stat_offsets_loaded,1170 &osd->tx_size_1522, &nsd->tx_size_1522);1171 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),1172 I40E_GLPRT_PTC9522L(hw->port),1173 pf->stat_offsets_loaded,1174 &osd->tx_size_big, &nsd->tx_size_big);1175 1176 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),1177 pf->stat_offsets_loaded,1178 &osd->rx_undersize, &nsd->rx_undersize);1179 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),1180 pf->stat_offsets_loaded,1181 &osd->rx_fragments, &nsd->rx_fragments);1182 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),1183 pf->stat_offsets_loaded,1184 &osd->rx_oversize, &nsd->rx_oversize);1185 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),1186 pf->stat_offsets_loaded,1187 &osd->rx_jabber, &nsd->rx_jabber);1188 1189 /* FDIR stats */1190 i40e_stat_update_and_clear32(hw,1191 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),1192 &nsd->fd_atr_match);1193 i40e_stat_update_and_clear32(hw,1194 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),1195 &nsd->fd_sb_match);1196 i40e_stat_update_and_clear32(hw,1197 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),1198 &nsd->fd_atr_tunnel_match);1199 1200 val = rd32(hw, I40E_PRTPM_EEE_STAT);1201 nsd->tx_lpi_status =1202 FIELD_GET(I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK, val);1203 nsd->rx_lpi_status =1204 FIELD_GET(I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK, val);1205 i40e_stat_update32(hw, I40E_PRTPM_TLPIC,1206 pf->stat_offsets_loaded,1207 &osd->tx_lpi_count, &nsd->tx_lpi_count);1208 i40e_stat_update32(hw, I40E_PRTPM_RLPIC,1209 pf->stat_offsets_loaded,1210 &osd->rx_lpi_count, &nsd->rx_lpi_count);1211 1212 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&1213 !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))1214 nsd->fd_sb_status = true;1215 else1216 nsd->fd_sb_status = false;1217 1218 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&1219 !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))1220 nsd->fd_atr_status = true;1221 else1222 nsd->fd_atr_status = false;1223 1224 pf->stat_offsets_loaded = true;1225}1226 1227/**1228 * i40e_update_stats - Update the various statistics counters.1229 * @vsi: the VSI to be updated1230 *1231 * Update the various stats for this VSI and its related entities.1232 **/1233void i40e_update_stats(struct i40e_vsi *vsi)1234{1235 struct i40e_pf *pf = vsi->back;1236 1237 if (vsi->type == I40E_VSI_MAIN)1238 i40e_update_pf_stats(pf);1239 1240 i40e_update_vsi_stats(vsi);1241}1242 1243/**1244 * i40e_count_filters - counts VSI mac filters1245 * @vsi: the VSI to be searched1246 *1247 * Returns count of mac filters1248 **/1249int i40e_count_filters(struct i40e_vsi *vsi)1250{1251 struct i40e_mac_filter *f;1252 struct hlist_node *h;1253 int bkt;1254 int cnt = 0;1255 1256 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {1257 if (f->state == I40E_FILTER_NEW ||1258 f->state == I40E_FILTER_NEW_SYNC ||1259 f->state == I40E_FILTER_ACTIVE)1260 ++cnt;1261 }1262 1263 return cnt;1264}1265 1266/**1267 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter1268 * @vsi: the VSI to be searched1269 * @macaddr: the MAC address1270 * @vlan: the vlan1271 *1272 * Returns ptr to the filter object or NULL1273 **/1274static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,1275 const u8 *macaddr, s16 vlan)1276{1277 struct i40e_mac_filter *f;1278 u64 key;1279 1280 if (!vsi || !macaddr)1281 return NULL;1282 1283 key = i40e_addr_to_hkey(macaddr);1284 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {1285 if ((ether_addr_equal(macaddr, f->macaddr)) &&1286 (vlan == f->vlan))1287 return f;1288 }1289 return NULL;1290}1291 1292/**1293 * i40e_find_mac - Find a mac addr in the macvlan filters list1294 * @vsi: the VSI to be searched1295 * @macaddr: the MAC address we are searching for1296 *1297 * Returns the first filter with the provided MAC address or NULL if1298 * MAC address was not found1299 **/1300struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)1301{1302 struct i40e_mac_filter *f;1303 u64 key;1304 1305 if (!vsi || !macaddr)1306 return NULL;1307 1308 key = i40e_addr_to_hkey(macaddr);1309 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {1310 if ((ether_addr_equal(macaddr, f->macaddr)))1311 return f;1312 }1313 return NULL;1314}1315 1316/**1317 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode1318 * @vsi: the VSI to be searched1319 *1320 * Returns true if VSI is in vlan mode or false otherwise1321 **/1322bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)1323{1324 /* If we have a PVID, always operate in VLAN mode */1325 if (vsi->info.pvid)1326 return true;1327 1328 /* We need to operate in VLAN mode whenever we have any filters with1329 * a VLAN other than I40E_VLAN_ALL. We could check the table each1330 * time, incurring search cost repeatedly. However, we can notice two1331 * things:1332 *1333 * 1) the only place where we can gain a VLAN filter is in1334 * i40e_add_filter.1335 *1336 * 2) the only place where filters are actually removed is in1337 * i40e_sync_filters_subtask.1338 *1339 * Thus, we can simply use a boolean value, has_vlan_filters which we1340 * will set to true when we add a VLAN filter in i40e_add_filter. Then1341 * we have to perform the full search after deleting filters in1342 * i40e_sync_filters_subtask, but we already have to search1343 * filters here and can perform the check at the same time. This1344 * results in avoiding embedding a loop for VLAN mode inside another1345 * loop over all the filters, and should maintain correctness as noted1346 * above.1347 */1348 return vsi->has_vlan_filter;1349}1350 1351/**1352 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary1353 * @vsi: the VSI to configure1354 * @tmp_add_list: list of filters ready to be added1355 * @tmp_del_list: list of filters ready to be deleted1356 * @vlan_filters: the number of active VLAN filters1357 *1358 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they1359 * behave as expected. If we have any active VLAN filters remaining or about1360 * to be added then we need to update non-VLAN filters to be marked as VLAN=01361 * so that they only match against untagged traffic. If we no longer have any1362 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-11363 * so that they match against both tagged and untagged traffic. In this way,1364 * we ensure that we correctly receive the desired traffic. This ensures that1365 * when we have an active VLAN we will receive only untagged traffic and1366 * traffic matching active VLANs. If we have no active VLANs then we will1367 * operate in non-VLAN mode and receive all traffic, tagged or untagged.1368 *1369 * Finally, in a similar fashion, this function also corrects filters when1370 * there is an active PVID assigned to this VSI.1371 *1372 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.1373 *1374 * This function is only expected to be called from within1375 * i40e_sync_vsi_filters.1376 *1377 * NOTE: This function expects to be called while under the1378 * mac_filter_hash_lock1379 */1380static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,1381 struct hlist_head *tmp_add_list,1382 struct hlist_head *tmp_del_list,1383 int vlan_filters)1384{1385 s16 pvid = le16_to_cpu(vsi->info.pvid);1386 struct i40e_mac_filter *f, *add_head;1387 struct i40e_new_mac_filter *new;1388 struct hlist_node *h;1389 int bkt, new_vlan;1390 1391 /* To determine if a particular filter needs to be replaced we1392 * have the three following conditions:1393 *1394 * a) if we have a PVID assigned, then all filters which are1395 * not marked as VLAN=PVID must be replaced with filters that1396 * are.1397 * b) otherwise, if we have any active VLANS, all filters1398 * which are marked as VLAN=-1 must be replaced with1399 * filters marked as VLAN=01400 * c) finally, if we do not have any active VLANS, all filters1401 * which are marked as VLAN=0 must be replaced with filters1402 * marked as VLAN=-11403 */1404 1405 /* Update the filters about to be added in place */1406 hlist_for_each_entry(new, tmp_add_list, hlist) {1407 if (pvid && new->f->vlan != pvid)1408 new->f->vlan = pvid;1409 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)1410 new->f->vlan = 0;1411 else if (!vlan_filters && new->f->vlan == 0)1412 new->f->vlan = I40E_VLAN_ANY;1413 }1414 1415 /* Update the remaining active filters */1416 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {1417 /* Combine the checks for whether a filter needs to be changed1418 * and then determine the new VLAN inside the if block, in1419 * order to avoid duplicating code for adding the new filter1420 * then deleting the old filter.1421 */1422 if ((pvid && f->vlan != pvid) ||1423 (vlan_filters && f->vlan == I40E_VLAN_ANY) ||1424 (!vlan_filters && f->vlan == 0)) {1425 /* Determine the new vlan we will be adding */1426 if (pvid)1427 new_vlan = pvid;1428 else if (vlan_filters)1429 new_vlan = 0;1430 else1431 new_vlan = I40E_VLAN_ANY;1432 1433 /* Create the new filter */1434 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);1435 if (!add_head)1436 return -ENOMEM;1437 1438 /* Create a temporary i40e_new_mac_filter */1439 new = kzalloc(sizeof(*new), GFP_ATOMIC);1440 if (!new)1441 return -ENOMEM;1442 1443 new->f = add_head;1444 new->state = add_head->state;1445 if (add_head->state == I40E_FILTER_NEW)1446 add_head->state = I40E_FILTER_NEW_SYNC;1447 1448 /* Add the new filter to the tmp list */1449 hlist_add_head(&new->hlist, tmp_add_list);1450 1451 /* Put the original filter into the delete list */1452 f->state = I40E_FILTER_REMOVE;1453 hash_del(&f->hlist);1454 hlist_add_head(&f->hlist, tmp_del_list);1455 }1456 }1457 1458 vsi->has_vlan_filter = !!vlan_filters;1459 1460 return 0;1461}1462 1463/**1464 * i40e_get_vf_new_vlan - Get new vlan id on a vf1465 * @vsi: the vsi to configure1466 * @new_mac: new mac filter to be added1467 * @f: existing mac filter, replaced with new_mac->f if new_mac is not NULL1468 * @vlan_filters: the number of active VLAN filters1469 * @trusted: flag if the VF is trusted1470 *1471 * Get new VLAN id based on current VLAN filters, trust, PVID1472 * and vf-vlan-prune-disable flag.1473 *1474 * Returns the value of the new vlan filter or1475 * the old value if no new filter is needed.1476 */1477static s16 i40e_get_vf_new_vlan(struct i40e_vsi *vsi,1478 struct i40e_new_mac_filter *new_mac,1479 struct i40e_mac_filter *f,1480 int vlan_filters,1481 bool trusted)1482{1483 s16 pvid = le16_to_cpu(vsi->info.pvid);1484 struct i40e_pf *pf = vsi->back;1485 bool is_any;1486 1487 if (new_mac)1488 f = new_mac->f;1489 1490 if (pvid && f->vlan != pvid)1491 return pvid;1492 1493 is_any = (trusted ||1494 !test_bit(I40E_FLAG_VF_VLAN_PRUNING_ENA, pf->flags));1495 1496 if ((vlan_filters && f->vlan == I40E_VLAN_ANY) ||1497 (!is_any && !vlan_filters && f->vlan == I40E_VLAN_ANY) ||1498 (is_any && !vlan_filters && f->vlan == 0)) {1499 if (is_any)1500 return I40E_VLAN_ANY;1501 else1502 return 0;1503 }1504 1505 return f->vlan;1506}1507 1508/**1509 * i40e_correct_vf_mac_vlan_filters - Correct non-VLAN VF filters if necessary1510 * @vsi: the vsi to configure1511 * @tmp_add_list: list of filters ready to be added1512 * @tmp_del_list: list of filters ready to be deleted1513 * @vlan_filters: the number of active VLAN filters1514 * @trusted: flag if the VF is trusted1515 *1516 * Correct VF VLAN filters based on current VLAN filters, trust, PVID1517 * and vf-vlan-prune-disable flag.1518 *1519 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.1520 *1521 * This function is only expected to be called from within1522 * i40e_sync_vsi_filters.1523 *1524 * NOTE: This function expects to be called while under the1525 * mac_filter_hash_lock1526 */1527static int i40e_correct_vf_mac_vlan_filters(struct i40e_vsi *vsi,1528 struct hlist_head *tmp_add_list,1529 struct hlist_head *tmp_del_list,1530 int vlan_filters,1531 bool trusted)1532{1533 struct i40e_mac_filter *f, *add_head;1534 struct i40e_new_mac_filter *new_mac;1535 struct hlist_node *h;1536 int bkt, new_vlan;1537 1538 hlist_for_each_entry(new_mac, tmp_add_list, hlist) {1539 new_mac->f->vlan = i40e_get_vf_new_vlan(vsi, new_mac, NULL,1540 vlan_filters, trusted);1541 }1542 1543 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {1544 new_vlan = i40e_get_vf_new_vlan(vsi, NULL, f, vlan_filters,1545 trusted);1546 if (new_vlan != f->vlan) {1547 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);1548 if (!add_head)1549 return -ENOMEM;1550 /* Create a temporary i40e_new_mac_filter */1551 new_mac = kzalloc(sizeof(*new_mac), GFP_ATOMIC);1552 if (!new_mac)1553 return -ENOMEM;1554 new_mac->f = add_head;1555 new_mac->state = add_head->state;1556 if (add_head->state == I40E_FILTER_NEW)1557 add_head->state = I40E_FILTER_NEW_SYNC;1558 1559 /* Add the new filter to the tmp list */1560 hlist_add_head(&new_mac->hlist, tmp_add_list);1561 1562 /* Put the original filter into the delete list */1563 f->state = I40E_FILTER_REMOVE;1564 hash_del(&f->hlist);1565 hlist_add_head(&f->hlist, tmp_del_list);1566 }1567 }1568 1569 vsi->has_vlan_filter = !!vlan_filters;1570 return 0;1571}1572 1573/**1574 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM1575 * @vsi: the PF Main VSI - inappropriate for any other VSI1576 * @macaddr: the MAC address1577 *1578 * Remove whatever filter the firmware set up so the driver can manage1579 * its own filtering intelligently.1580 **/1581static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)1582{1583 struct i40e_aqc_remove_macvlan_element_data element;1584 struct i40e_pf *pf = vsi->back;1585 1586 /* Only appropriate for the PF main VSI */1587 if (vsi->type != I40E_VSI_MAIN)1588 return;1589 1590 memset(&element, 0, sizeof(element));1591 ether_addr_copy(element.mac_addr, macaddr);1592 element.vlan_tag = 0;1593 /* Ignore error returns, some firmware does it this way... */1594 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;1595 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);1596 1597 memset(&element, 0, sizeof(element));1598 ether_addr_copy(element.mac_addr, macaddr);1599 element.vlan_tag = 0;1600 /* ...and some firmware does it this way. */1601 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |1602 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;1603 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);1604}1605 1606/**1607 * i40e_add_filter - Add a mac/vlan filter to the VSI1608 * @vsi: the VSI to be searched1609 * @macaddr: the MAC address1610 * @vlan: the vlan1611 *1612 * Returns ptr to the filter object or NULL when no memory available.1613 *1614 * NOTE: This function is expected to be called with mac_filter_hash_lock1615 * being held.1616 **/1617struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,1618 const u8 *macaddr, s16 vlan)1619{1620 struct i40e_mac_filter *f;1621 u64 key;1622 1623 if (!vsi || !macaddr)1624 return NULL;1625 1626 f = i40e_find_filter(vsi, macaddr, vlan);1627 if (!f) {1628 f = kzalloc(sizeof(*f), GFP_ATOMIC);1629 if (!f)1630 return NULL;1631 1632 /* Update the boolean indicating if we need to function in1633 * VLAN mode.1634 */1635 if (vlan >= 0)1636 vsi->has_vlan_filter = true;1637 1638 ether_addr_copy(f->macaddr, macaddr);1639 f->vlan = vlan;1640 f->state = I40E_FILTER_NEW;1641 INIT_HLIST_NODE(&f->hlist);1642 1643 key = i40e_addr_to_hkey(macaddr);1644 hash_add(vsi->mac_filter_hash, &f->hlist, key);1645 1646 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;1647 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);1648 }1649 1650 /* If we're asked to add a filter that has been marked for removal, it1651 * is safe to simply restore it to active state. __i40e_del_filter1652 * will have simply deleted any filters which were previously marked1653 * NEW or FAILED, so if it is currently marked REMOVE it must have1654 * previously been ACTIVE. Since we haven't yet run the sync filters1655 * task, just restore this filter to the ACTIVE state so that the1656 * sync task leaves it in place1657 */1658 if (f->state == I40E_FILTER_REMOVE)1659 f->state = I40E_FILTER_ACTIVE;1660 1661 return f;1662}1663 1664/**1665 * __i40e_del_filter - Remove a specific filter from the VSI1666 * @vsi: VSI to remove from1667 * @f: the filter to remove from the list1668 *1669 * This function should be called instead of i40e_del_filter only if you know1670 * the exact filter you will remove already, such as via i40e_find_filter or1671 * i40e_find_mac.1672 *1673 * NOTE: This function is expected to be called with mac_filter_hash_lock1674 * being held.1675 * ANOTHER NOTE: This function MUST be called from within the context of1676 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()1677 * instead of list_for_each_entry().1678 **/1679void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)1680{1681 if (!f)1682 return;1683 1684 /* If the filter was never added to firmware then we can just delete it1685 * directly and we don't want to set the status to remove or else an1686 * admin queue command will unnecessarily fire.1687 */1688 if ((f->state == I40E_FILTER_FAILED) ||1689 (f->state == I40E_FILTER_NEW)) {1690 hash_del(&f->hlist);1691 kfree(f);1692 } else {1693 f->state = I40E_FILTER_REMOVE;1694 }1695 1696 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;1697 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);1698}1699 1700/**1701 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI1702 * @vsi: the VSI to be searched1703 * @macaddr: the MAC address1704 * @vlan: the VLAN1705 *1706 * NOTE: This function is expected to be called with mac_filter_hash_lock1707 * being held.1708 * ANOTHER NOTE: This function MUST be called from within the context of1709 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()1710 * instead of list_for_each_entry().1711 **/1712void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)1713{1714 struct i40e_mac_filter *f;1715 1716 if (!vsi || !macaddr)1717 return;1718 1719 f = i40e_find_filter(vsi, macaddr, vlan);1720 __i40e_del_filter(vsi, f);1721}1722 1723/**1724 * i40e_add_mac_filter - Add a MAC filter for all active VLANs1725 * @vsi: the VSI to be searched1726 * @macaddr: the mac address to be filtered1727 *1728 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,1729 * go through all the macvlan filters and add a macvlan filter for each1730 * unique vlan that already exists. If a PVID has been assigned, instead only1731 * add the macaddr to that VLAN.1732 *1733 * Returns last filter added on success, else NULL1734 **/1735struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,1736 const u8 *macaddr)1737{1738 struct i40e_mac_filter *f, *add = NULL;1739 struct hlist_node *h;1740 int bkt;1741 1742 lockdep_assert_held(&vsi->mac_filter_hash_lock);1743 if (vsi->info.pvid)1744 return i40e_add_filter(vsi, macaddr,1745 le16_to_cpu(vsi->info.pvid));1746 1747 if (!i40e_is_vsi_in_vlan(vsi))1748 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);1749 1750 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {1751 if (f->state == I40E_FILTER_REMOVE)1752 continue;1753 add = i40e_add_filter(vsi, macaddr, f->vlan);1754 if (!add)1755 return NULL;1756 }1757 1758 return add;1759}1760 1761/**1762 * i40e_del_mac_filter - Remove a MAC filter from all VLANs1763 * @vsi: the VSI to be searched1764 * @macaddr: the mac address to be removed1765 *1766 * Removes a given MAC address from a VSI regardless of what VLAN it has been1767 * associated with.1768 *1769 * Returns 0 for success, or error1770 **/1771int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)1772{1773 struct i40e_mac_filter *f;1774 struct hlist_node *h;1775 bool found = false;1776 int bkt;1777 1778 lockdep_assert_held(&vsi->mac_filter_hash_lock);1779 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {1780 if (ether_addr_equal(macaddr, f->macaddr)) {1781 __i40e_del_filter(vsi, f);1782 found = true;1783 }1784 }1785 1786 if (found)1787 return 0;1788 else1789 return -ENOENT;1790}1791 1792/**1793 * i40e_set_mac - NDO callback to set mac address1794 * @netdev: network interface device structure1795 * @p: pointer to an address structure1796 *1797 * Returns 0 on success, negative on failure1798 **/1799static int i40e_set_mac(struct net_device *netdev, void *p)1800{1801 struct i40e_netdev_priv *np = netdev_priv(netdev);1802 struct i40e_vsi *vsi = np->vsi;1803 struct i40e_pf *pf = vsi->back;1804 struct i40e_hw *hw = &pf->hw;1805 struct sockaddr *addr = p;1806 1807 if (!is_valid_ether_addr(addr->sa_data))1808 return -EADDRNOTAVAIL;1809 1810 if (test_bit(__I40E_DOWN, pf->state) ||1811 test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))1812 return -EADDRNOTAVAIL;1813 1814 if (ether_addr_equal(hw->mac.addr, addr->sa_data))1815 netdev_info(netdev, "returning to hw mac address %pM\n",1816 hw->mac.addr);1817 else1818 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);1819 1820 /* Copy the address first, so that we avoid a possible race with1821 * .set_rx_mode().1822 * - Remove old address from MAC filter1823 * - Copy new address1824 * - Add new address to MAC filter1825 */1826 spin_lock_bh(&vsi->mac_filter_hash_lock);1827 i40e_del_mac_filter(vsi, netdev->dev_addr);1828 eth_hw_addr_set(netdev, addr->sa_data);1829 i40e_add_mac_filter(vsi, netdev->dev_addr);1830 spin_unlock_bh(&vsi->mac_filter_hash_lock);1831 1832 if (vsi->type == I40E_VSI_MAIN) {1833 int ret;1834 1835 ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL,1836 addr->sa_data, NULL);1837 if (ret)1838 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %pe, AQ ret %s\n",1839 ERR_PTR(ret),1840 i40e_aq_str(hw, hw->aq.asq_last_status));1841 }1842 1843 /* schedule our worker thread which will take care of1844 * applying the new filter changes1845 */1846 i40e_service_event_schedule(pf);1847 return 0;1848}1849 1850/**1851 * i40e_config_rss_aq - Prepare for RSS using AQ commands1852 * @vsi: vsi structure1853 * @seed: RSS hash seed1854 * @lut: pointer to lookup table of lut_size1855 * @lut_size: size of the lookup table1856 **/1857static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,1858 u8 *lut, u16 lut_size)1859{1860 struct i40e_pf *pf = vsi->back;1861 struct i40e_hw *hw = &pf->hw;1862 int ret = 0;1863 1864 if (seed) {1865 struct i40e_aqc_get_set_rss_key_data *seed_dw =1866 (struct i40e_aqc_get_set_rss_key_data *)seed;1867 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);1868 if (ret) {1869 dev_info(&pf->pdev->dev,1870 "Cannot set RSS key, err %pe aq_err %s\n",1871 ERR_PTR(ret),1872 i40e_aq_str(hw, hw->aq.asq_last_status));1873 return ret;1874 }1875 }1876 if (lut) {1877 bool pf_lut = vsi->type == I40E_VSI_MAIN;1878 1879 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);1880 if (ret) {1881 dev_info(&pf->pdev->dev,1882 "Cannot set RSS lut, err %pe aq_err %s\n",1883 ERR_PTR(ret),1884 i40e_aq_str(hw, hw->aq.asq_last_status));1885 return ret;1886 }1887 }1888 return ret;1889}1890 1891/**1892 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used1893 * @vsi: VSI structure1894 **/1895static int i40e_vsi_config_rss(struct i40e_vsi *vsi)1896{1897 struct i40e_pf *pf = vsi->back;1898 u8 seed[I40E_HKEY_ARRAY_SIZE];1899 u8 *lut;1900 int ret;1901 1902 if (!test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))1903 return 0;1904 if (!vsi->rss_size)1905 vsi->rss_size = min_t(int, pf->alloc_rss_size,1906 vsi->num_queue_pairs);1907 if (!vsi->rss_size)1908 return -EINVAL;1909 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);1910 if (!lut)1911 return -ENOMEM;1912 1913 /* Use the user configured hash keys and lookup table if there is one,1914 * otherwise use default1915 */1916 if (vsi->rss_lut_user)1917 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);1918 else1919 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);1920 if (vsi->rss_hkey_user)1921 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);1922 else1923 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);1924 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);1925 kfree(lut);1926 return ret;1927}1928 1929/**1930 * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config1931 * @vsi: the VSI being configured,1932 * @ctxt: VSI context structure1933 * @enabled_tc: number of traffic classes to enable1934 *1935 * Prepares VSI tc_config to have queue configurations based on MQPRIO options.1936 **/1937static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi,1938 struct i40e_vsi_context *ctxt,1939 u8 enabled_tc)1940{1941 u16 qcount = 0, max_qcount, qmap, sections = 0;1942 int i, override_q, pow, num_qps, ret;1943 u8 netdev_tc = 0, offset = 0;1944 1945 if (vsi->type != I40E_VSI_MAIN)1946 return -EINVAL;1947 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;1948 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;1949 vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc;1950 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;1951 num_qps = vsi->mqprio_qopt.qopt.count[0];1952 1953 /* find the next higher power-of-2 of num queue pairs */1954 pow = ilog2(num_qps);1955 if (!is_power_of_2(num_qps))1956 pow++;1957 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |1958 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);1959 1960 /* Setup queue offset/count for all TCs for given VSI */1961 max_qcount = vsi->mqprio_qopt.qopt.count[0];1962 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {1963 /* See if the given TC is enabled for the given VSI */1964 if (vsi->tc_config.enabled_tc & BIT(i)) {1965 offset = vsi->mqprio_qopt.qopt.offset[i];1966 qcount = vsi->mqprio_qopt.qopt.count[i];1967 if (qcount > max_qcount)1968 max_qcount = qcount;1969 vsi->tc_config.tc_info[i].qoffset = offset;1970 vsi->tc_config.tc_info[i].qcount = qcount;1971 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;1972 } else {1973 /* TC is not enabled so set the offset to1974 * default queue and allocate one queue1975 * for the given TC.1976 */1977 vsi->tc_config.tc_info[i].qoffset = 0;1978 vsi->tc_config.tc_info[i].qcount = 1;1979 vsi->tc_config.tc_info[i].netdev_tc = 0;1980 }1981 }1982 1983 /* Set actual Tx/Rx queue pairs */1984 vsi->num_queue_pairs = offset + qcount;1985 1986 /* Setup queue TC[0].qmap for given VSI context */1987 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);1988 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);1989 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);1990 ctxt->info.valid_sections |= cpu_to_le16(sections);1991 1992 /* Reconfigure RSS for main VSI with max queue count */1993 vsi->rss_size = max_qcount;1994 ret = i40e_vsi_config_rss(vsi);1995 if (ret) {1996 dev_info(&vsi->back->pdev->dev,1997 "Failed to reconfig rss for num_queues (%u)\n",1998 max_qcount);1999 return ret;2000 }2001 vsi->reconfig_rss = true;2002 dev_dbg(&vsi->back->pdev->dev,2003 "Reconfigured rss with num_queues (%u)\n", max_qcount);2004 2005 /* Find queue count available for channel VSIs and starting offset2006 * for channel VSIs2007 */2008 override_q = vsi->mqprio_qopt.qopt.count[0];2009 if (override_q && override_q < vsi->num_queue_pairs) {2010 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q;2011 vsi->next_base_queue = override_q;2012 }2013 return 0;2014}2015 2016/**2017 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc2018 * @vsi: the VSI being setup2019 * @ctxt: VSI context structure2020 * @enabled_tc: Enabled TCs bitmap2021 * @is_add: True if called before Add VSI2022 *2023 * Setup VSI queue mapping for enabled traffic classes.2024 **/2025static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,2026 struct i40e_vsi_context *ctxt,2027 u8 enabled_tc,2028 bool is_add)2029{2030 struct i40e_pf *pf = vsi->back;2031 u16 num_tc_qps = 0;2032 u16 sections = 0;2033 u8 netdev_tc = 0;2034 u16 numtc = 1;2035 u16 qcount;2036 u8 offset;2037 u16 qmap;2038 int i;2039 2040 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;2041 offset = 0;2042 /* zero out queue mapping, it will get updated on the end of the function */2043 memset(ctxt->info.queue_mapping, 0, sizeof(ctxt->info.queue_mapping));2044 2045 if (vsi->type == I40E_VSI_MAIN) {2046 /* This code helps add more queue to the VSI if we have2047 * more cores than RSS can support, the higher cores will2048 * be served by ATR or other filters. Furthermore, the2049 * non-zero req_queue_pairs says that user requested a new2050 * queue count via ethtool's set_channels, so use this2051 * value for queues distribution across traffic classes2052 * We need at least one queue pair for the interface2053 * to be usable as we see in else statement.2054 */2055 if (vsi->req_queue_pairs > 0)2056 vsi->num_queue_pairs = vsi->req_queue_pairs;2057 else if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))2058 vsi->num_queue_pairs = pf->num_lan_msix;2059 else2060 vsi->num_queue_pairs = 1;2061 }2062 2063 /* Number of queues per enabled TC */2064 if (vsi->type == I40E_VSI_MAIN ||2065 (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs != 0))2066 num_tc_qps = vsi->num_queue_pairs;2067 else2068 num_tc_qps = vsi->alloc_queue_pairs;2069 2070 if (enabled_tc && test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) {2071 /* Find numtc from enabled TC bitmap */2072 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {2073 if (enabled_tc & BIT(i)) /* TC is enabled */2074 numtc++;2075 }2076 if (!numtc) {2077 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");2078 numtc = 1;2079 }2080 num_tc_qps = num_tc_qps / numtc;2081 num_tc_qps = min_t(int, num_tc_qps,2082 i40e_pf_get_max_q_per_tc(pf));2083 }2084 2085 vsi->tc_config.numtc = numtc;2086 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;2087 2088 /* Do not allow use more TC queue pairs than MSI-X vectors exist */2089 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))2090 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix);2091 2092 /* Setup queue offset/count for all TCs for given VSI */2093 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {2094 /* See if the given TC is enabled for the given VSI */2095 if (vsi->tc_config.enabled_tc & BIT(i)) {2096 /* TC is enabled */2097 int pow, num_qps;2098 2099 switch (vsi->type) {2100 case I40E_VSI_MAIN:2101 if ((!test_bit(I40E_FLAG_FD_SB_ENA,2102 pf->flags) &&2103 !test_bit(I40E_FLAG_FD_ATR_ENA,2104 pf->flags)) ||2105 vsi->tc_config.enabled_tc != 1) {2106 qcount = min_t(int, pf->alloc_rss_size,2107 num_tc_qps);2108 break;2109 }2110 fallthrough;2111 case I40E_VSI_FDIR:2112 case I40E_VSI_SRIOV:2113 case I40E_VSI_VMDQ2:2114 default:2115 qcount = num_tc_qps;2116 WARN_ON(i != 0);2117 break;2118 }2119 vsi->tc_config.tc_info[i].qoffset = offset;2120 vsi->tc_config.tc_info[i].qcount = qcount;2121 2122 /* find the next higher power-of-2 of num queue pairs */2123 num_qps = qcount;2124 pow = 0;2125 while (num_qps && (BIT_ULL(pow) < qcount)) {2126 pow++;2127 num_qps >>= 1;2128 }2129 2130 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;2131 qmap =2132 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |2133 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);2134 2135 offset += qcount;2136 } else {2137 /* TC is not enabled so set the offset to2138 * default queue and allocate one queue2139 * for the given TC.2140 */2141 vsi->tc_config.tc_info[i].qoffset = 0;2142 vsi->tc_config.tc_info[i].qcount = 1;2143 vsi->tc_config.tc_info[i].netdev_tc = 0;2144 2145 qmap = 0;2146 }2147 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);2148 }2149 /* Do not change previously set num_queue_pairs for PFs and VFs*/2150 if ((vsi->type == I40E_VSI_MAIN && numtc != 1) ||2151 (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs == 0) ||2152 (vsi->type != I40E_VSI_MAIN && vsi->type != I40E_VSI_SRIOV))2153 vsi->num_queue_pairs = offset;2154 2155 /* Scheduler section valid can only be set for ADD VSI */2156 if (is_add) {2157 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;2158 2159 ctxt->info.up_enable_bits = enabled_tc;2160 }2161 if (vsi->type == I40E_VSI_SRIOV) {2162 ctxt->info.mapping_flags |=2163 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);2164 for (i = 0; i < vsi->num_queue_pairs; i++)2165 ctxt->info.queue_mapping[i] =2166 cpu_to_le16(vsi->base_queue + i);2167 } else {2168 ctxt->info.mapping_flags |=2169 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);2170 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);2171 }2172 ctxt->info.valid_sections |= cpu_to_le16(sections);2173}2174 2175/**2176 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address2177 * @netdev: the netdevice2178 * @addr: address to add2179 *2180 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call2181 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.2182 */2183static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)2184{2185 struct i40e_netdev_priv *np = netdev_priv(netdev);2186 struct i40e_vsi *vsi = np->vsi;2187 2188 if (i40e_add_mac_filter(vsi, addr))2189 return 0;2190 else2191 return -ENOMEM;2192}2193 2194/**2195 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address2196 * @netdev: the netdevice2197 * @addr: address to add2198 *2199 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call2200 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.2201 */2202static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)2203{2204 struct i40e_netdev_priv *np = netdev_priv(netdev);2205 struct i40e_vsi *vsi = np->vsi;2206 2207 /* Under some circumstances, we might receive a request to delete2208 * our own device address from our uc list. Because we store the2209 * device address in the VSI's MAC/VLAN filter list, we need to ignore2210 * such requests and not delete our device address from this list.2211 */2212 if (ether_addr_equal(addr, netdev->dev_addr))2213 return 0;2214 2215 i40e_del_mac_filter(vsi, addr);2216 2217 return 0;2218}2219 2220/**2221 * i40e_set_rx_mode - NDO callback to set the netdev filters2222 * @netdev: network interface device structure2223 **/2224static void i40e_set_rx_mode(struct net_device *netdev)2225{2226 struct i40e_netdev_priv *np = netdev_priv(netdev);2227 struct i40e_vsi *vsi = np->vsi;2228 2229 spin_lock_bh(&vsi->mac_filter_hash_lock);2230 2231 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);2232 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);2233 2234 spin_unlock_bh(&vsi->mac_filter_hash_lock);2235 2236 /* check for other flag changes */2237 if (vsi->current_netdev_flags != vsi->netdev->flags) {2238 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;2239 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);2240 }2241}2242 2243/**2244 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries2245 * @vsi: Pointer to VSI struct2246 * @from: Pointer to list which contains MAC filter entries - changes to2247 * those entries needs to be undone.2248 *2249 * MAC filter entries from this list were slated for deletion.2250 **/2251static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,2252 struct hlist_head *from)2253{2254 struct i40e_mac_filter *f;2255 struct hlist_node *h;2256 2257 hlist_for_each_entry_safe(f, h, from, hlist) {2258 u64 key = i40e_addr_to_hkey(f->macaddr);2259 2260 /* Move the element back into MAC filter list*/2261 hlist_del(&f->hlist);2262 hash_add(vsi->mac_filter_hash, &f->hlist, key);2263 }2264}2265 2266/**2267 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries2268 * @vsi: Pointer to vsi struct2269 * @from: Pointer to list which contains MAC filter entries - changes to2270 * those entries needs to be undone.2271 *2272 * MAC filter entries from this list were slated for addition.2273 **/2274static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,2275 struct hlist_head *from)2276{2277 struct i40e_new_mac_filter *new;2278 struct hlist_node *h;2279 2280 hlist_for_each_entry_safe(new, h, from, hlist) {2281 /* We can simply free the wrapper structure */2282 hlist_del(&new->hlist);2283 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);2284 kfree(new);2285 }2286}2287 2288/**2289 * i40e_next_filter - Get the next non-broadcast filter from a list2290 * @next: pointer to filter in list2291 *2292 * Returns the next non-broadcast filter in the list. Required so that we2293 * ignore broadcast filters within the list, since these are not handled via2294 * the normal firmware update path.2295 */2296static2297struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)2298{2299 hlist_for_each_entry_continue(next, hlist) {2300 if (!is_broadcast_ether_addr(next->f->macaddr))2301 return next;2302 }2303 2304 return NULL;2305}2306 2307/**2308 * i40e_update_filter_state - Update filter state based on return data2309 * from firmware2310 * @count: Number of filters added2311 * @add_list: return data from fw2312 * @add_head: pointer to first filter in current batch2313 *2314 * MAC filter entries from list were slated to be added to device. Returns2315 * number of successful filters. Note that 0 does NOT mean success!2316 **/2317static int2318i40e_update_filter_state(int count,2319 struct i40e_aqc_add_macvlan_element_data *add_list,2320 struct i40e_new_mac_filter *add_head)2321{2322 int retval = 0;2323 int i;2324 2325 for (i = 0; i < count; i++) {2326 /* Always check status of each filter. We don't need to check2327 * the firmware return status because we pre-set the filter2328 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter2329 * request to the adminq. Thus, if it no longer matches then2330 * we know the filter is active.2331 */2332 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {2333 add_head->state = I40E_FILTER_FAILED;2334 } else {2335 add_head->state = I40E_FILTER_ACTIVE;2336 retval++;2337 }2338 2339 add_head = i40e_next_filter(add_head);2340 if (!add_head)2341 break;2342 }2343 2344 return retval;2345}2346 2347/**2348 * i40e_aqc_del_filters - Request firmware to delete a set of filters2349 * @vsi: ptr to the VSI2350 * @vsi_name: name to display in messages2351 * @list: the list of filters to send to firmware2352 * @num_del: the number of filters to delete2353 * @retval: Set to -EIO on failure to delete2354 *2355 * Send a request to firmware via AdminQ to delete a set of filters. Uses2356 * *retval instead of a return value so that success does not force ret_val to2357 * be set to 0. This ensures that a sequence of calls to this function2358 * preserve the previous value of *retval on successful delete.2359 */2360static2361void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,2362 struct i40e_aqc_remove_macvlan_element_data *list,2363 int num_del, int *retval)2364{2365 struct i40e_hw *hw = &vsi->back->hw;2366 enum i40e_admin_queue_err aq_status;2367 int aq_ret;2368 2369 aq_ret = i40e_aq_remove_macvlan_v2(hw, vsi->seid, list, num_del, NULL,2370 &aq_status);2371 2372 /* Explicitly ignore and do not report when firmware returns ENOENT */2373 if (aq_ret && !(aq_status == I40E_AQ_RC_ENOENT)) {2374 *retval = -EIO;2375 dev_info(&vsi->back->pdev->dev,2376 "ignoring delete macvlan error on %s, err %pe, aq_err %s\n",2377 vsi_name, ERR_PTR(aq_ret),2378 i40e_aq_str(hw, aq_status));2379 }2380}2381 2382/**2383 * i40e_aqc_add_filters - Request firmware to add a set of filters2384 * @vsi: ptr to the VSI2385 * @vsi_name: name to display in messages2386 * @list: the list of filters to send to firmware2387 * @add_head: Position in the add hlist2388 * @num_add: the number of filters to add2389 *2390 * Send a request to firmware via AdminQ to add a chunk of filters. Will set2391 * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of2392 * space for more filters.2393 */2394static2395void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,2396 struct i40e_aqc_add_macvlan_element_data *list,2397 struct i40e_new_mac_filter *add_head,2398 int num_add)2399{2400 struct i40e_hw *hw = &vsi->back->hw;2401 enum i40e_admin_queue_err aq_status;2402 int fcnt;2403 2404 i40e_aq_add_macvlan_v2(hw, vsi->seid, list, num_add, NULL, &aq_status);2405 fcnt = i40e_update_filter_state(num_add, list, add_head);2406 2407 if (fcnt != num_add) {2408 if (vsi->type == I40E_VSI_MAIN) {2409 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);2410 dev_warn(&vsi->back->pdev->dev,2411 "Error %s adding RX filters on %s, promiscuous mode forced on\n",2412 i40e_aq_str(hw, aq_status), vsi_name);2413 } else if (vsi->type == I40E_VSI_SRIOV ||2414 vsi->type == I40E_VSI_VMDQ1 ||2415 vsi->type == I40E_VSI_VMDQ2) {2416 dev_warn(&vsi->back->pdev->dev,2417 "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n",2418 i40e_aq_str(hw, aq_status), vsi_name,2419 vsi_name);2420 } else {2421 dev_warn(&vsi->back->pdev->dev,2422 "Error %s adding RX filters on %s, incorrect VSI type: %i.\n",2423 i40e_aq_str(hw, aq_status), vsi_name,2424 vsi->type);2425 }2426 }2427}2428 2429/**2430 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags2431 * @vsi: pointer to the VSI2432 * @vsi_name: the VSI name2433 * @f: filter data2434 *2435 * This function sets or clears the promiscuous broadcast flags for VLAN2436 * filters in order to properly receive broadcast frames. Assumes that only2437 * broadcast filters are passed.2438 *2439 * Returns status indicating success or failure;2440 **/2441static int2442i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,2443 struct i40e_mac_filter *f)2444{2445 bool enable = f->state == I40E_FILTER_NEW ||2446 f->state == I40E_FILTER_NEW_SYNC;2447 struct i40e_hw *hw = &vsi->back->hw;2448 int aq_ret;2449 2450 if (f->vlan == I40E_VLAN_ANY) {2451 aq_ret = i40e_aq_set_vsi_broadcast(hw,2452 vsi->seid,2453 enable,2454 NULL);2455 } else {2456 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,2457 vsi->seid,2458 enable,2459 f->vlan,2460 NULL);2461 }2462 2463 if (aq_ret) {2464 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);2465 dev_warn(&vsi->back->pdev->dev,2466 "Error %s, forcing overflow promiscuous on %s\n",2467 i40e_aq_str(hw, hw->aq.asq_last_status),2468 vsi_name);2469 }2470 2471 return aq_ret;2472}2473 2474/**2475 * i40e_set_promiscuous - set promiscuous mode2476 * @pf: board private structure2477 * @promisc: promisc on or off2478 *2479 * There are different ways of setting promiscuous mode on a PF depending on2480 * what state/environment we're in. This identifies and sets it appropriately.2481 * Returns 0 on success.2482 **/2483static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc)2484{2485 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);2486 struct i40e_hw *hw = &pf->hw;2487 int aq_ret;2488 2489 if (vsi->type == I40E_VSI_MAIN &&2490 i40e_pf_get_main_veb(pf) &&2491 !test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {2492 /* set defport ON for Main VSI instead of true promisc2493 * this way we will get all unicast/multicast and VLAN2494 * promisc behavior but will not get VF or VMDq traffic2495 * replicated on the Main VSI.2496 */2497 if (promisc)2498 aq_ret = i40e_aq_set_default_vsi(hw,2499 vsi->seid,2500 NULL);2501 else2502 aq_ret = i40e_aq_clear_default_vsi(hw,2503 vsi->seid,2504 NULL);2505 if (aq_ret) {2506 dev_info(&pf->pdev->dev,2507 "Set default VSI failed, err %pe, aq_err %s\n",2508 ERR_PTR(aq_ret),2509 i40e_aq_str(hw, hw->aq.asq_last_status));2510 }2511 } else {2512 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(2513 hw,2514 vsi->seid,2515 promisc, NULL,2516 true);2517 if (aq_ret) {2518 dev_info(&pf->pdev->dev,2519 "set unicast promisc failed, err %pe, aq_err %s\n",2520 ERR_PTR(aq_ret),2521 i40e_aq_str(hw, hw->aq.asq_last_status));2522 }2523 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(2524 hw,2525 vsi->seid,2526 promisc, NULL);2527 if (aq_ret) {2528 dev_info(&pf->pdev->dev,2529 "set multicast promisc failed, err %pe, aq_err %s\n",2530 ERR_PTR(aq_ret),2531 i40e_aq_str(hw, hw->aq.asq_last_status));2532 }2533 }2534 2535 if (!aq_ret)2536 pf->cur_promisc = promisc;2537 2538 return aq_ret;2539}2540 2541/**2542 * i40e_sync_vsi_filters - Update the VSI filter list to the HW2543 * @vsi: ptr to the VSI2544 *2545 * Push any outstanding VSI filter changes through the AdminQ.2546 *2547 * Returns 0 or error value2548 **/2549int i40e_sync_vsi_filters(struct i40e_vsi *vsi)2550{2551 struct hlist_head tmp_add_list, tmp_del_list;2552 struct i40e_mac_filter *f;2553 struct i40e_new_mac_filter *new, *add_head = NULL;2554 struct i40e_hw *hw = &vsi->back->hw;2555 bool old_overflow, new_overflow;2556 unsigned int failed_filters = 0;2557 unsigned int vlan_filters = 0;2558 char vsi_name[16] = "PF";2559 int filter_list_len = 0;2560 u32 changed_flags = 0;2561 struct hlist_node *h;2562 struct i40e_pf *pf;2563 int num_add = 0;2564 int num_del = 0;2565 int aq_ret = 0;2566 int retval = 0;2567 u16 cmd_flags;2568 int list_size;2569 int bkt;2570 2571 /* empty array typed pointers, kcalloc later */2572 struct i40e_aqc_add_macvlan_element_data *add_list;2573 struct i40e_aqc_remove_macvlan_element_data *del_list;2574 2575 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))2576 usleep_range(1000, 2000);2577 pf = vsi->back;2578 2579 old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);2580 2581 if (vsi->netdev) {2582 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;2583 vsi->current_netdev_flags = vsi->netdev->flags;2584 }2585 2586 INIT_HLIST_HEAD(&tmp_add_list);2587 INIT_HLIST_HEAD(&tmp_del_list);2588 2589 if (vsi->type == I40E_VSI_SRIOV)2590 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);2591 else if (vsi->type != I40E_VSI_MAIN)2592 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);2593 2594 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {2595 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;2596 2597 spin_lock_bh(&vsi->mac_filter_hash_lock);2598 /* Create a list of filters to delete. */2599 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {2600 if (f->state == I40E_FILTER_REMOVE) {2601 /* Move the element into temporary del_list */2602 hash_del(&f->hlist);2603 hlist_add_head(&f->hlist, &tmp_del_list);2604 2605 /* Avoid counting removed filters */2606 continue;2607 }2608 if (f->state == I40E_FILTER_NEW) {2609 /* Create a temporary i40e_new_mac_filter */2610 new = kzalloc(sizeof(*new), GFP_ATOMIC);2611 if (!new)2612 goto err_no_memory_locked;2613 2614 /* Store pointer to the real filter */2615 new->f = f;2616 new->state = f->state;2617 2618 /* Add it to the hash list */2619 hlist_add_head(&new->hlist, &tmp_add_list);2620 f->state = I40E_FILTER_NEW_SYNC;2621 }2622 2623 /* Count the number of active (current and new) VLAN2624 * filters we have now. Does not count filters which2625 * are marked for deletion.2626 */2627 if (f->vlan > 0)2628 vlan_filters++;2629 }2630 2631 if (vsi->type != I40E_VSI_SRIOV)2632 retval = i40e_correct_mac_vlan_filters2633 (vsi, &tmp_add_list, &tmp_del_list,2634 vlan_filters);2635 else if (pf->vf)2636 retval = i40e_correct_vf_mac_vlan_filters2637 (vsi, &tmp_add_list, &tmp_del_list,2638 vlan_filters, pf->vf[vsi->vf_id].trusted);2639 2640 hlist_for_each_entry(new, &tmp_add_list, hlist)2641 netdev_hw_addr_refcnt(new->f, vsi->netdev, 1);2642 2643 if (retval)2644 goto err_no_memory_locked;2645 2646 spin_unlock_bh(&vsi->mac_filter_hash_lock);2647 }2648 2649 /* Now process 'del_list' outside the lock */2650 if (!hlist_empty(&tmp_del_list)) {2651 filter_list_len = hw->aq.asq_buf_size /2652 sizeof(struct i40e_aqc_remove_macvlan_element_data);2653 list_size = filter_list_len *2654 sizeof(struct i40e_aqc_remove_macvlan_element_data);2655 del_list = kzalloc(list_size, GFP_ATOMIC);2656 if (!del_list)2657 goto err_no_memory;2658 2659 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {2660 cmd_flags = 0;2661 2662 /* handle broadcast filters by updating the broadcast2663 * promiscuous flag and release filter list.2664 */2665 if (is_broadcast_ether_addr(f->macaddr)) {2666 i40e_aqc_broadcast_filter(vsi, vsi_name, f);2667 2668 hlist_del(&f->hlist);2669 kfree(f);2670 continue;2671 }2672 2673 /* add to delete list */2674 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);2675 if (f->vlan == I40E_VLAN_ANY) {2676 del_list[num_del].vlan_tag = 0;2677 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;2678 } else {2679 del_list[num_del].vlan_tag =2680 cpu_to_le16((u16)(f->vlan));2681 }2682 2683 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;2684 del_list[num_del].flags = cmd_flags;2685 num_del++;2686 2687 /* flush a full buffer */2688 if (num_del == filter_list_len) {2689 i40e_aqc_del_filters(vsi, vsi_name, del_list,2690 num_del, &retval);2691 memset(del_list, 0, list_size);2692 num_del = 0;2693 }2694 /* Release memory for MAC filter entries which were2695 * synced up with HW.2696 */2697 hlist_del(&f->hlist);2698 kfree(f);2699 }2700 2701 if (num_del) {2702 i40e_aqc_del_filters(vsi, vsi_name, del_list,2703 num_del, &retval);2704 }2705 2706 kfree(del_list);2707 del_list = NULL;2708 }2709 2710 if (!hlist_empty(&tmp_add_list)) {2711 /* Do all the adds now. */2712 filter_list_len = hw->aq.asq_buf_size /2713 sizeof(struct i40e_aqc_add_macvlan_element_data);2714 list_size = filter_list_len *2715 sizeof(struct i40e_aqc_add_macvlan_element_data);2716 add_list = kzalloc(list_size, GFP_ATOMIC);2717 if (!add_list)2718 goto err_no_memory;2719 2720 num_add = 0;2721 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {2722 /* handle broadcast filters by updating the broadcast2723 * promiscuous flag instead of adding a MAC filter.2724 */2725 if (is_broadcast_ether_addr(new->f->macaddr)) {2726 if (i40e_aqc_broadcast_filter(vsi, vsi_name,2727 new->f))2728 new->state = I40E_FILTER_FAILED;2729 else2730 new->state = I40E_FILTER_ACTIVE;2731 continue;2732 }2733 2734 /* add to add array */2735 if (num_add == 0)2736 add_head = new;2737 cmd_flags = 0;2738 ether_addr_copy(add_list[num_add].mac_addr,2739 new->f->macaddr);2740 if (new->f->vlan == I40E_VLAN_ANY) {2741 add_list[num_add].vlan_tag = 0;2742 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;2743 } else {2744 add_list[num_add].vlan_tag =2745 cpu_to_le16((u16)(new->f->vlan));2746 }2747 add_list[num_add].queue_number = 0;2748 /* set invalid match method for later detection */2749 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;2750 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;2751 add_list[num_add].flags = cpu_to_le16(cmd_flags);2752 num_add++;2753 2754 /* flush a full buffer */2755 if (num_add == filter_list_len) {2756 i40e_aqc_add_filters(vsi, vsi_name, add_list,2757 add_head, num_add);2758 memset(add_list, 0, list_size);2759 num_add = 0;2760 }2761 }2762 if (num_add) {2763 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,2764 num_add);2765 }2766 /* Now move all of the filters from the temp add list back to2767 * the VSI's list.2768 */2769 spin_lock_bh(&vsi->mac_filter_hash_lock);2770 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {2771 /* Only update the state if we're still NEW */2772 if (new->f->state == I40E_FILTER_NEW ||2773 new->f->state == I40E_FILTER_NEW_SYNC)2774 new->f->state = new->state;2775 hlist_del(&new->hlist);2776 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);2777 kfree(new);2778 }2779 spin_unlock_bh(&vsi->mac_filter_hash_lock);2780 kfree(add_list);2781 add_list = NULL;2782 }2783 2784 /* Determine the number of active and failed filters. */2785 spin_lock_bh(&vsi->mac_filter_hash_lock);2786 vsi->active_filters = 0;2787 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {2788 if (f->state == I40E_FILTER_ACTIVE)2789 vsi->active_filters++;2790 else if (f->state == I40E_FILTER_FAILED)2791 failed_filters++;2792 }2793 spin_unlock_bh(&vsi->mac_filter_hash_lock);2794 2795 /* Check if we are able to exit overflow promiscuous mode. We can2796 * safely exit if we didn't just enter, we no longer have any failed2797 * filters, and we have reduced filters below the threshold value.2798 */2799 if (old_overflow && !failed_filters &&2800 vsi->active_filters < vsi->promisc_threshold) {2801 dev_info(&pf->pdev->dev,2802 "filter logjam cleared on %s, leaving overflow promiscuous mode\n",2803 vsi_name);2804 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);2805 vsi->promisc_threshold = 0;2806 }2807 2808 /* if the VF is not trusted do not do promisc */2809 if (vsi->type == I40E_VSI_SRIOV && pf->vf &&2810 !pf->vf[vsi->vf_id].trusted) {2811 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);2812 goto out;2813 }2814 2815 new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);2816 2817 /* If we are entering overflow promiscuous, we need to calculate a new2818 * threshold for when we are safe to exit2819 */2820 if (!old_overflow && new_overflow)2821 vsi->promisc_threshold = (vsi->active_filters * 3) / 4;2822 2823 /* check for changes in promiscuous modes */2824 if (changed_flags & IFF_ALLMULTI) {2825 bool cur_multipromisc;2826 2827 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);2828 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,2829 vsi->seid,2830 cur_multipromisc,2831 NULL);2832 if (aq_ret) {2833 retval = i40e_aq_rc_to_posix(aq_ret,2834 hw->aq.asq_last_status);2835 dev_info(&pf->pdev->dev,2836 "set multi promisc failed on %s, err %pe aq_err %s\n",2837 vsi_name,2838 ERR_PTR(aq_ret),2839 i40e_aq_str(hw, hw->aq.asq_last_status));2840 } else {2841 dev_info(&pf->pdev->dev, "%s allmulti mode.\n",2842 cur_multipromisc ? "entering" : "leaving");2843 }2844 }2845 2846 if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) {2847 bool cur_promisc;2848 2849 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||2850 new_overflow);2851 aq_ret = i40e_set_promiscuous(pf, cur_promisc);2852 if (aq_ret) {2853 retval = i40e_aq_rc_to_posix(aq_ret,2854 hw->aq.asq_last_status);2855 dev_info(&pf->pdev->dev,2856 "Setting promiscuous %s failed on %s, err %pe aq_err %s\n",2857 cur_promisc ? "on" : "off",2858 vsi_name,2859 ERR_PTR(aq_ret),2860 i40e_aq_str(hw, hw->aq.asq_last_status));2861 }2862 }2863out:2864 /* if something went wrong then set the changed flag so we try again */2865 if (retval)2866 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;2867 2868 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);2869 return retval;2870 2871err_no_memory:2872 /* Restore elements on the temporary add and delete lists */2873 spin_lock_bh(&vsi->mac_filter_hash_lock);2874err_no_memory_locked:2875 i40e_undo_del_filter_entries(vsi, &tmp_del_list);2876 i40e_undo_add_filter_entries(vsi, &tmp_add_list);2877 spin_unlock_bh(&vsi->mac_filter_hash_lock);2878 2879 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;2880 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);2881 return -ENOMEM;2882}2883 2884/**2885 * i40e_sync_filters_subtask - Sync the VSI filter list with HW2886 * @pf: board private structure2887 **/2888static void i40e_sync_filters_subtask(struct i40e_pf *pf)2889{2890 struct i40e_vsi *vsi;2891 int v;2892 2893 if (!pf)2894 return;2895 if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state))2896 return;2897 if (test_bit(__I40E_VF_DISABLE, pf->state)) {2898 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);2899 return;2900 }2901 2902 i40e_pf_for_each_vsi(pf, v, vsi) {2903 if ((vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) &&2904 !test_bit(__I40E_VSI_RELEASING, vsi->state)) {2905 int ret = i40e_sync_vsi_filters(vsi);2906 2907 if (ret) {2908 /* come back and try again later */2909 set_bit(__I40E_MACVLAN_SYNC_PENDING,2910 pf->state);2911 break;2912 }2913 }2914 }2915}2916 2917/**2918 * i40e_calculate_vsi_rx_buf_len - Calculates buffer length2919 *2920 * @vsi: VSI to calculate rx_buf_len from2921 */2922static u16 i40e_calculate_vsi_rx_buf_len(struct i40e_vsi *vsi)2923{2924 if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags))2925 return SKB_WITH_OVERHEAD(I40E_RXBUFFER_2048);2926 2927 return PAGE_SIZE < 8192 ? I40E_RXBUFFER_3072 : I40E_RXBUFFER_2048;2928}2929 2930/**2931 * i40e_max_vsi_frame_size - returns the maximum allowed frame size for VSI2932 * @vsi: the vsi2933 * @xdp_prog: XDP program2934 **/2935static int i40e_max_vsi_frame_size(struct i40e_vsi *vsi,2936 struct bpf_prog *xdp_prog)2937{2938 u16 rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi);2939 u16 chain_len;2940 2941 if (xdp_prog && !xdp_prog->aux->xdp_has_frags)2942 chain_len = 1;2943 else2944 chain_len = I40E_MAX_CHAINED_RX_BUFFERS;2945 2946 return min_t(u16, rx_buf_len * chain_len, I40E_MAX_RXBUFFER);2947}2948 2949/**2950 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit2951 * @netdev: network interface device structure2952 * @new_mtu: new value for maximum frame size2953 *2954 * Returns 0 on success, negative on failure2955 **/2956static int i40e_change_mtu(struct net_device *netdev, int new_mtu)2957{2958 struct i40e_netdev_priv *np = netdev_priv(netdev);2959 struct i40e_vsi *vsi = np->vsi;2960 struct i40e_pf *pf = vsi->back;2961 int frame_size;2962 2963 frame_size = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog);2964 if (new_mtu > frame_size - I40E_PACKET_HDR_PAD) {2965 netdev_err(netdev, "Error changing mtu to %d, Max is %d\n",2966 new_mtu, frame_size - I40E_PACKET_HDR_PAD);2967 return -EINVAL;2968 }2969 2970 netdev_dbg(netdev, "changing MTU from %d to %d\n",2971 netdev->mtu, new_mtu);2972 WRITE_ONCE(netdev->mtu, new_mtu);2973 if (netif_running(netdev))2974 i40e_vsi_reinit_locked(vsi);2975 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);2976 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);2977 return 0;2978}2979 2980/**2981 * i40e_ioctl - Access the hwtstamp interface2982 * @netdev: network interface device structure2983 * @ifr: interface request data2984 * @cmd: ioctl command2985 **/2986int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)2987{2988 struct i40e_netdev_priv *np = netdev_priv(netdev);2989 struct i40e_pf *pf = np->vsi->back;2990 2991 switch (cmd) {2992 case SIOCGHWTSTAMP:2993 return i40e_ptp_get_ts_config(pf, ifr);2994 case SIOCSHWTSTAMP:2995 return i40e_ptp_set_ts_config(pf, ifr);2996 default:2997 return -EOPNOTSUPP;2998 }2999}3000 3001/**3002 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI3003 * @vsi: the vsi being adjusted3004 **/3005void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)3006{3007 struct i40e_vsi_context ctxt;3008 int ret;3009 3010 /* Don't modify stripping options if a port VLAN is active */3011 if (vsi->info.pvid)3012 return;3013 3014 if ((vsi->info.valid_sections &3015 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&3016 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))3017 return; /* already enabled */3018 3019 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);3020 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |3021 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;3022 3023 ctxt.seid = vsi->seid;3024 ctxt.info = vsi->info;3025 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);3026 if (ret) {3027 dev_info(&vsi->back->pdev->dev,3028 "update vlan stripping failed, err %pe aq_err %s\n",3029 ERR_PTR(ret),3030 i40e_aq_str(&vsi->back->hw,3031 vsi->back->hw.aq.asq_last_status));3032 }3033}3034 3035/**3036 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI3037 * @vsi: the vsi being adjusted3038 **/3039void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)3040{3041 struct i40e_vsi_context ctxt;3042 int ret;3043 3044 /* Don't modify stripping options if a port VLAN is active */3045 if (vsi->info.pvid)3046 return;3047 3048 if ((vsi->info.valid_sections &3049 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&3050 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==3051 I40E_AQ_VSI_PVLAN_EMOD_MASK))3052 return; /* already disabled */3053 3054 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);3055 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |3056 I40E_AQ_VSI_PVLAN_EMOD_NOTHING;3057 3058 ctxt.seid = vsi->seid;3059 ctxt.info = vsi->info;3060 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);3061 if (ret) {3062 dev_info(&vsi->back->pdev->dev,3063 "update vlan stripping failed, err %pe aq_err %s\n",3064 ERR_PTR(ret),3065 i40e_aq_str(&vsi->back->hw,3066 vsi->back->hw.aq.asq_last_status));3067 }3068}3069 3070/**3071 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address3072 * @vsi: the vsi being configured3073 * @vid: vlan id to be added (0 = untagged only , -1 = any)3074 *3075 * This is a helper function for adding a new MAC/VLAN filter with the3076 * specified VLAN for each existing MAC address already in the hash table.3077 * This function does *not* perform any accounting to update filters based on3078 * VLAN mode.3079 *3080 * NOTE: this function expects to be called while under the3081 * mac_filter_hash_lock3082 **/3083int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)3084{3085 struct i40e_mac_filter *f, *add_f;3086 struct hlist_node *h;3087 int bkt;3088 3089 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {3090 /* If we're asked to add a filter that has been marked for3091 * removal, it is safe to simply restore it to active state.3092 * __i40e_del_filter will have simply deleted any filters which3093 * were previously marked NEW or FAILED, so if it is currently3094 * marked REMOVE it must have previously been ACTIVE. Since we3095 * haven't yet run the sync filters task, just restore this3096 * filter to the ACTIVE state so that the sync task leaves it3097 * in place.3098 */3099 if (f->state == I40E_FILTER_REMOVE && f->vlan == vid) {3100 f->state = I40E_FILTER_ACTIVE;3101 continue;3102 } else if (f->state == I40E_FILTER_REMOVE) {3103 continue;3104 }3105 add_f = i40e_add_filter(vsi, f->macaddr, vid);3106 if (!add_f) {3107 dev_info(&vsi->back->pdev->dev,3108 "Could not add vlan filter %d for %pM\n",3109 vid, f->macaddr);3110 return -ENOMEM;3111 }3112 }3113 3114 return 0;3115}3116 3117/**3118 * i40e_vsi_add_vlan - Add VSI membership for given VLAN3119 * @vsi: the VSI being configured3120 * @vid: VLAN id to be added3121 **/3122int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)3123{3124 int err;3125 3126 if (vsi->info.pvid)3127 return -EINVAL;3128 3129 /* The network stack will attempt to add VID=0, with the intention to3130 * receive priority tagged packets with a VLAN of 0. Our HW receives3131 * these packets by default when configured to receive untagged3132 * packets, so we don't need to add a filter for this case.3133 * Additionally, HW interprets adding a VID=0 filter as meaning to3134 * receive *only* tagged traffic and stops receiving untagged traffic.3135 * Thus, we do not want to actually add a filter for VID=03136 */3137 if (!vid)3138 return 0;3139 3140 /* Locked once because all functions invoked below iterates list*/3141 spin_lock_bh(&vsi->mac_filter_hash_lock);3142 err = i40e_add_vlan_all_mac(vsi, vid);3143 spin_unlock_bh(&vsi->mac_filter_hash_lock);3144 if (err)3145 return err;3146 3147 /* schedule our worker thread which will take care of3148 * applying the new filter changes3149 */3150 i40e_service_event_schedule(vsi->back);3151 return 0;3152}3153 3154/**3155 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN3156 * @vsi: the vsi being configured3157 * @vid: vlan id to be removed (0 = untagged only , -1 = any)3158 *3159 * This function should be used to remove all VLAN filters which match the3160 * given VID. It does not schedule the service event and does not take the3161 * mac_filter_hash_lock so it may be combined with other operations under3162 * a single invocation of the mac_filter_hash_lock.3163 *3164 * NOTE: this function expects to be called while under the3165 * mac_filter_hash_lock3166 */3167void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)3168{3169 struct i40e_mac_filter *f;3170 struct hlist_node *h;3171 int bkt;3172 3173 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {3174 if (f->vlan == vid)3175 __i40e_del_filter(vsi, f);3176 }3177}3178 3179/**3180 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN3181 * @vsi: the VSI being configured3182 * @vid: VLAN id to be removed3183 **/3184void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)3185{3186 if (!vid || vsi->info.pvid)3187 return;3188 3189 spin_lock_bh(&vsi->mac_filter_hash_lock);3190 i40e_rm_vlan_all_mac(vsi, vid);3191 spin_unlock_bh(&vsi->mac_filter_hash_lock);3192 3193 /* schedule our worker thread which will take care of3194 * applying the new filter changes3195 */3196 i40e_service_event_schedule(vsi->back);3197}3198 3199/**3200 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload3201 * @netdev: network interface to be adjusted3202 * @proto: unused protocol value3203 * @vid: vlan id to be added3204 *3205 * net_device_ops implementation for adding vlan ids3206 **/3207static int i40e_vlan_rx_add_vid(struct net_device *netdev,3208 __always_unused __be16 proto, u16 vid)3209{3210 struct i40e_netdev_priv *np = netdev_priv(netdev);3211 struct i40e_vsi *vsi = np->vsi;3212 int ret = 0;3213 3214 if (vid >= VLAN_N_VID)3215 return -EINVAL;3216 3217 ret = i40e_vsi_add_vlan(vsi, vid);3218 if (!ret)3219 set_bit(vid, vsi->active_vlans);3220 3221 return ret;3222}3223 3224/**3225 * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path3226 * @netdev: network interface to be adjusted3227 * @proto: unused protocol value3228 * @vid: vlan id to be added3229 **/3230static void i40e_vlan_rx_add_vid_up(struct net_device *netdev,3231 __always_unused __be16 proto, u16 vid)3232{3233 struct i40e_netdev_priv *np = netdev_priv(netdev);3234 struct i40e_vsi *vsi = np->vsi;3235 3236 if (vid >= VLAN_N_VID)3237 return;3238 set_bit(vid, vsi->active_vlans);3239}3240 3241/**3242 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload3243 * @netdev: network interface to be adjusted3244 * @proto: unused protocol value3245 * @vid: vlan id to be removed3246 *3247 * net_device_ops implementation for removing vlan ids3248 **/3249static int i40e_vlan_rx_kill_vid(struct net_device *netdev,3250 __always_unused __be16 proto, u16 vid)3251{3252 struct i40e_netdev_priv *np = netdev_priv(netdev);3253 struct i40e_vsi *vsi = np->vsi;3254 3255 /* return code is ignored as there is nothing a user3256 * can do about failure to remove and a log message was3257 * already printed from the other function3258 */3259 i40e_vsi_kill_vlan(vsi, vid);3260 3261 clear_bit(vid, vsi->active_vlans);3262 3263 return 0;3264}3265 3266/**3267 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up3268 * @vsi: the vsi being brought back up3269 **/3270static void i40e_restore_vlan(struct i40e_vsi *vsi)3271{3272 u16 vid;3273 3274 if (!vsi->netdev)3275 return;3276 3277 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)3278 i40e_vlan_stripping_enable(vsi);3279 else3280 i40e_vlan_stripping_disable(vsi);3281 3282 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)3283 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q),3284 vid);3285}3286 3287/**3288 * i40e_vsi_add_pvid - Add pvid for the VSI3289 * @vsi: the vsi being adjusted3290 * @vid: the vlan id to set as a PVID3291 **/3292int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)3293{3294 struct i40e_vsi_context ctxt;3295 int ret;3296 3297 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);3298 vsi->info.pvid = cpu_to_le16(vid);3299 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |3300 I40E_AQ_VSI_PVLAN_INSERT_PVID |3301 I40E_AQ_VSI_PVLAN_EMOD_STR;3302 3303 ctxt.seid = vsi->seid;3304 ctxt.info = vsi->info;3305 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);3306 if (ret) {3307 dev_info(&vsi->back->pdev->dev,3308 "add pvid failed, err %pe aq_err %s\n",3309 ERR_PTR(ret),3310 i40e_aq_str(&vsi->back->hw,3311 vsi->back->hw.aq.asq_last_status));3312 return -ENOENT;3313 }3314 3315 return 0;3316}3317 3318/**3319 * i40e_vsi_remove_pvid - Remove the pvid from the VSI3320 * @vsi: the vsi being adjusted3321 *3322 * Just use the vlan_rx_register() service to put it back to normal3323 **/3324void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)3325{3326 vsi->info.pvid = 0;3327 3328 i40e_vlan_stripping_disable(vsi);3329}3330 3331/**3332 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources3333 * @vsi: ptr to the VSI3334 *3335 * If this function returns with an error, then it's possible one or3336 * more of the rings is populated (while the rest are not). It is the3337 * callers duty to clean those orphaned rings.3338 *3339 * Return 0 on success, negative on failure3340 **/3341static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)3342{3343 int i, err = 0;3344 3345 for (i = 0; i < vsi->num_queue_pairs && !err; i++)3346 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);3347 3348 if (!i40e_enabled_xdp_vsi(vsi))3349 return err;3350 3351 for (i = 0; i < vsi->num_queue_pairs && !err; i++)3352 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);3353 3354 return err;3355}3356 3357/**3358 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues3359 * @vsi: ptr to the VSI3360 *3361 * Free VSI's transmit software resources3362 **/3363static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)3364{3365 int i;3366 3367 if (vsi->tx_rings) {3368 for (i = 0; i < vsi->num_queue_pairs; i++)3369 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)3370 i40e_free_tx_resources(vsi->tx_rings[i]);3371 }3372 3373 if (vsi->xdp_rings) {3374 for (i = 0; i < vsi->num_queue_pairs; i++)3375 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)3376 i40e_free_tx_resources(vsi->xdp_rings[i]);3377 }3378}3379 3380/**3381 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources3382 * @vsi: ptr to the VSI3383 *3384 * If this function returns with an error, then it's possible one or3385 * more of the rings is populated (while the rest are not). It is the3386 * callers duty to clean those orphaned rings.3387 *3388 * Return 0 on success, negative on failure3389 **/3390static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)3391{3392 int i, err = 0;3393 3394 for (i = 0; i < vsi->num_queue_pairs && !err; i++)3395 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);3396 return err;3397}3398 3399/**3400 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues3401 * @vsi: ptr to the VSI3402 *3403 * Free all receive software resources3404 **/3405static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)3406{3407 int i;3408 3409 if (!vsi->rx_rings)3410 return;3411 3412 for (i = 0; i < vsi->num_queue_pairs; i++)3413 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)3414 i40e_free_rx_resources(vsi->rx_rings[i]);3415}3416 3417/**3418 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring3419 * @ring: The Tx ring to configure3420 *3421 * This enables/disables XPS for a given Tx descriptor ring3422 * based on the TCs enabled for the VSI that ring belongs to.3423 **/3424static void i40e_config_xps_tx_ring(struct i40e_ring *ring)3425{3426 int cpu;3427 3428 if (!ring->q_vector || !ring->netdev || ring->ch)3429 return;3430 3431 /* We only initialize XPS once, so as not to overwrite user settings */3432 if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state))3433 return;3434 3435 cpu = cpumask_local_spread(ring->q_vector->v_idx, -1);3436 netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu),3437 ring->queue_index);3438}3439 3440/**3441 * i40e_xsk_pool - Retrieve the AF_XDP buffer pool if XDP and ZC is enabled3442 * @ring: The Tx or Rx ring3443 *3444 * Returns the AF_XDP buffer pool or NULL.3445 **/3446static struct xsk_buff_pool *i40e_xsk_pool(struct i40e_ring *ring)3447{3448 bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi);3449 int qid = ring->queue_index;3450 3451 if (ring_is_xdp(ring))3452 qid -= ring->vsi->alloc_queue_pairs;3453 3454 if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps))3455 return NULL;3456 3457 return xsk_get_pool_from_qid(ring->vsi->netdev, qid);3458}3459 3460/**3461 * i40e_configure_tx_ring - Configure a transmit ring context and rest3462 * @ring: The Tx ring to configure3463 *3464 * Configure the Tx descriptor ring in the HMC context.3465 **/3466static int i40e_configure_tx_ring(struct i40e_ring *ring)3467{3468 struct i40e_vsi *vsi = ring->vsi;3469 u16 pf_q = vsi->base_queue + ring->queue_index;3470 struct i40e_hw *hw = &vsi->back->hw;3471 struct i40e_hmc_obj_txq tx_ctx;3472 u32 qtx_ctl = 0;3473 int err = 0;3474 3475 if (ring_is_xdp(ring))3476 ring->xsk_pool = i40e_xsk_pool(ring);3477 3478 /* some ATR related tx ring init */3479 if (test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags)) {3480 ring->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;3481 ring->atr_count = 0;3482 } else {3483 ring->atr_sample_rate = 0;3484 }3485 3486 /* configure XPS */3487 i40e_config_xps_tx_ring(ring);3488 3489 /* clear the context structure first */3490 memset(&tx_ctx, 0, sizeof(tx_ctx));3491 3492 tx_ctx.new_context = 1;3493 tx_ctx.base = (ring->dma / 128);3494 tx_ctx.qlen = ring->count;3495 if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags) ||3496 test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags))3497 tx_ctx.fd_ena = 1;3498 if (test_bit(I40E_FLAG_PTP_ENA, vsi->back->flags))3499 tx_ctx.timesync_ena = 1;3500 /* FDIR VSI tx ring can still use RS bit and writebacks */3501 if (vsi->type != I40E_VSI_FDIR)3502 tx_ctx.head_wb_ena = 1;3503 tx_ctx.head_wb_addr = ring->dma +3504 (ring->count * sizeof(struct i40e_tx_desc));3505 3506 /* As part of VSI creation/update, FW allocates certain3507 * Tx arbitration queue sets for each TC enabled for3508 * the VSI. The FW returns the handles to these queue3509 * sets as part of the response buffer to Add VSI,3510 * Update VSI, etc. AQ commands. It is expected that3511 * these queue set handles be associated with the Tx3512 * queues by the driver as part of the TX queue context3513 * initialization. This has to be done regardless of3514 * DCB as by default everything is mapped to TC0.3515 */3516 3517 if (ring->ch)3518 tx_ctx.rdylist =3519 le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]);3520 3521 else3522 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);3523 3524 tx_ctx.rdylist_act = 0;3525 3526 /* clear the context in the HMC */3527 err = i40e_clear_lan_tx_queue_context(hw, pf_q);3528 if (err) {3529 dev_info(&vsi->back->pdev->dev,3530 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",3531 ring->queue_index, pf_q, err);3532 return -ENOMEM;3533 }3534 3535 /* set the context in the HMC */3536 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);3537 if (err) {3538 dev_info(&vsi->back->pdev->dev,3539 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",3540 ring->queue_index, pf_q, err);3541 return -ENOMEM;3542 }3543 3544 /* Now associate this queue with this PCI function */3545 if (ring->ch) {3546 if (ring->ch->type == I40E_VSI_VMDQ2)3547 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;3548 else3549 return -EINVAL;3550 3551 qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_VFVM_INDX_MASK,3552 ring->ch->vsi_number);3553 } else {3554 if (vsi->type == I40E_VSI_VMDQ2) {3555 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;3556 qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_VFVM_INDX_MASK,3557 vsi->id);3558 } else {3559 qtx_ctl = I40E_QTX_CTL_PF_QUEUE;3560 }3561 }3562 3563 qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_PF_INDX_MASK, hw->pf_id);3564 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);3565 i40e_flush(hw);3566 3567 /* cache tail off for easier writes later */3568 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);3569 3570 return 0;3571}3572 3573/**3574 * i40e_rx_offset - Return expected offset into page to access data3575 * @rx_ring: Ring we are requesting offset of3576 *3577 * Returns the offset value for ring into the data buffer.3578 */3579static unsigned int i40e_rx_offset(struct i40e_ring *rx_ring)3580{3581 return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0;3582}3583 3584/**3585 * i40e_configure_rx_ring - Configure a receive ring context3586 * @ring: The Rx ring to configure3587 *3588 * Configure the Rx descriptor ring in the HMC context.3589 **/3590static int i40e_configure_rx_ring(struct i40e_ring *ring)3591{3592 struct i40e_vsi *vsi = ring->vsi;3593 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;3594 u16 pf_q = vsi->base_queue + ring->queue_index;3595 struct i40e_hw *hw = &vsi->back->hw;3596 struct i40e_hmc_obj_rxq rx_ctx;3597 int err = 0;3598 bool ok;3599 3600 bitmap_zero(ring->state, __I40E_RING_STATE_NBITS);3601 3602 /* clear the context structure first */3603 memset(&rx_ctx, 0, sizeof(rx_ctx));3604 3605 ring->rx_buf_len = vsi->rx_buf_len;3606 3607 /* XDP RX-queue info only needed for RX rings exposed to XDP */3608 if (ring->vsi->type != I40E_VSI_MAIN)3609 goto skip;3610 3611 if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) {3612 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,3613 ring->queue_index,3614 ring->q_vector->napi.napi_id,3615 ring->rx_buf_len);3616 if (err)3617 return err;3618 }3619 3620 ring->xsk_pool = i40e_xsk_pool(ring);3621 if (ring->xsk_pool) {3622 xdp_rxq_info_unreg(&ring->xdp_rxq);3623 ring->rx_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool);3624 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,3625 ring->queue_index,3626 ring->q_vector->napi.napi_id,3627 ring->rx_buf_len);3628 if (err)3629 return err;3630 err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,3631 MEM_TYPE_XSK_BUFF_POOL,3632 NULL);3633 if (err)3634 return err;3635 dev_info(&vsi->back->pdev->dev,3636 "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n",3637 ring->queue_index);3638 3639 } else {3640 err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,3641 MEM_TYPE_PAGE_SHARED,3642 NULL);3643 if (err)3644 return err;3645 }3646 3647skip:3648 xdp_init_buff(&ring->xdp, i40e_rx_pg_size(ring) / 2, &ring->xdp_rxq);3649 3650 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,3651 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));3652 3653 rx_ctx.base = (ring->dma / 128);3654 rx_ctx.qlen = ring->count;3655 3656 /* use 16 byte descriptors */3657 rx_ctx.dsize = 0;3658 3659 /* descriptor type is always zero3660 * rx_ctx.dtype = 0;3661 */3662 rx_ctx.hsplit_0 = 0;3663 3664 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);3665 if (hw->revision_id == 0)3666 rx_ctx.lrxqthresh = 0;3667 else3668 rx_ctx.lrxqthresh = 1;3669 rx_ctx.crcstrip = 1;3670 rx_ctx.l2tsel = 1;3671 /* this controls whether VLAN is stripped from inner headers */3672 rx_ctx.showiv = 0;3673 /* set the prefena field to 1 because the manual says to */3674 rx_ctx.prefena = 1;3675 3676 /* clear the context in the HMC */3677 err = i40e_clear_lan_rx_queue_context(hw, pf_q);3678 if (err) {3679 dev_info(&vsi->back->pdev->dev,3680 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",3681 ring->queue_index, pf_q, err);3682 return -ENOMEM;3683 }3684 3685 /* set the context in the HMC */3686 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);3687 if (err) {3688 dev_info(&vsi->back->pdev->dev,3689 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",3690 ring->queue_index, pf_q, err);3691 return -ENOMEM;3692 }3693 3694 /* configure Rx buffer alignment */3695 if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags)) {3696 if (I40E_2K_TOO_SMALL_WITH_PADDING) {3697 dev_info(&vsi->back->pdev->dev,3698 "2k Rx buffer is too small to fit standard MTU and skb_shared_info\n");3699 return -EOPNOTSUPP;3700 }3701 clear_ring_build_skb_enabled(ring);3702 } else {3703 set_ring_build_skb_enabled(ring);3704 }3705 3706 ring->rx_offset = i40e_rx_offset(ring);3707 3708 /* cache tail for quicker writes, and clear the reg before use */3709 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);3710 writel(0, ring->tail);3711 3712 if (ring->xsk_pool) {3713 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq);3714 ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring));3715 } else {3716 ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));3717 }3718 if (!ok) {3719 /* Log this in case the user has forgotten to give the kernel3720 * any buffers, even later in the application.3721 */3722 dev_info(&vsi->back->pdev->dev,3723 "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n",3724 ring->xsk_pool ? "AF_XDP ZC enabled " : "",3725 ring->queue_index, pf_q);3726 }3727 3728 return 0;3729}3730 3731/**3732 * i40e_vsi_configure_tx - Configure the VSI for Tx3733 * @vsi: VSI structure describing this set of rings and resources3734 *3735 * Configure the Tx VSI for operation.3736 **/3737static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)3738{3739 int err = 0;3740 u16 i;3741 3742 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)3743 err = i40e_configure_tx_ring(vsi->tx_rings[i]);3744 3745 if (err || !i40e_enabled_xdp_vsi(vsi))3746 return err;3747 3748 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)3749 err = i40e_configure_tx_ring(vsi->xdp_rings[i]);3750 3751 return err;3752}3753 3754/**3755 * i40e_vsi_configure_rx - Configure the VSI for Rx3756 * @vsi: the VSI being configured3757 *3758 * Configure the Rx VSI for operation.3759 **/3760static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)3761{3762 int err = 0;3763 u16 i;3764 3765 vsi->max_frame = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog);3766 vsi->rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi);3767 3768#if (PAGE_SIZE < 8192)3769 if (vsi->netdev && !I40E_2K_TOO_SMALL_WITH_PADDING &&3770 vsi->netdev->mtu <= ETH_DATA_LEN) {3771 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;3772 vsi->max_frame = vsi->rx_buf_len;3773 }3774#endif3775 3776 /* set up individual rings */3777 for (i = 0; i < vsi->num_queue_pairs && !err; i++)3778 err = i40e_configure_rx_ring(vsi->rx_rings[i]);3779 3780 return err;3781}3782 3783/**3784 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC3785 * @vsi: ptr to the VSI3786 **/3787static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)3788{3789 struct i40e_ring *tx_ring, *rx_ring;3790 u16 qoffset, qcount;3791 int i, n;3792 3793 if (!test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) {3794 /* Reset the TC information */3795 for (i = 0; i < vsi->num_queue_pairs; i++) {3796 rx_ring = vsi->rx_rings[i];3797 tx_ring = vsi->tx_rings[i];3798 rx_ring->dcb_tc = 0;3799 tx_ring->dcb_tc = 0;3800 }3801 return;3802 }3803 3804 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {3805 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))3806 continue;3807 3808 qoffset = vsi->tc_config.tc_info[n].qoffset;3809 qcount = vsi->tc_config.tc_info[n].qcount;3810 for (i = qoffset; i < (qoffset + qcount); i++) {3811 rx_ring = vsi->rx_rings[i];3812 tx_ring = vsi->tx_rings[i];3813 rx_ring->dcb_tc = n;3814 tx_ring->dcb_tc = n;3815 }3816 }3817}3818 3819/**3820 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI3821 * @vsi: ptr to the VSI3822 **/3823static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)3824{3825 if (vsi->netdev)3826 i40e_set_rx_mode(vsi->netdev);3827}3828 3829/**3830 * i40e_reset_fdir_filter_cnt - Reset flow director filter counters3831 * @pf: Pointer to the targeted PF3832 *3833 * Set all flow director counters to 0.3834 */3835static void i40e_reset_fdir_filter_cnt(struct i40e_pf *pf)3836{3837 pf->fd_tcp4_filter_cnt = 0;3838 pf->fd_udp4_filter_cnt = 0;3839 pf->fd_sctp4_filter_cnt = 0;3840 pf->fd_ip4_filter_cnt = 0;3841 pf->fd_tcp6_filter_cnt = 0;3842 pf->fd_udp6_filter_cnt = 0;3843 pf->fd_sctp6_filter_cnt = 0;3844 pf->fd_ip6_filter_cnt = 0;3845}3846 3847/**3848 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters3849 * @vsi: Pointer to the targeted VSI3850 *3851 * This function replays the hlist on the hw where all the SB Flow Director3852 * filters were saved.3853 **/3854static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)3855{3856 struct i40e_fdir_filter *filter;3857 struct i40e_pf *pf = vsi->back;3858 struct hlist_node *node;3859 3860 if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))3861 return;3862 3863 /* Reset FDir counters as we're replaying all existing filters */3864 i40e_reset_fdir_filter_cnt(pf);3865 3866 hlist_for_each_entry_safe(filter, node,3867 &pf->fdir_filter_list, fdir_node) {3868 i40e_add_del_fdir(vsi, filter, true);3869 }3870}3871 3872/**3873 * i40e_vsi_configure - Set up the VSI for action3874 * @vsi: the VSI being configured3875 **/3876static int i40e_vsi_configure(struct i40e_vsi *vsi)3877{3878 int err;3879 3880 i40e_set_vsi_rx_mode(vsi);3881 i40e_restore_vlan(vsi);3882 i40e_vsi_config_dcb_rings(vsi);3883 err = i40e_vsi_configure_tx(vsi);3884 if (!err)3885 err = i40e_vsi_configure_rx(vsi);3886 3887 return err;3888}3889 3890/**3891 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW3892 * @vsi: the VSI being configured3893 **/3894static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)3895{3896 bool has_xdp = i40e_enabled_xdp_vsi(vsi);3897 struct i40e_pf *pf = vsi->back;3898 struct i40e_hw *hw = &pf->hw;3899 u16 vector;3900 int i, q;3901 u32 qp;3902 3903 /* The interrupt indexing is offset by 1 in the PFINT_ITRn3904 * and PFINT_LNKLSTn registers, e.g.:3905 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts)3906 */3907 qp = vsi->base_queue;3908 vector = vsi->base_vector;3909 for (i = 0; i < vsi->num_q_vectors; i++, vector++) {3910 struct i40e_q_vector *q_vector = vsi->q_vectors[i];3911 3912 q_vector->rx.next_update = jiffies + 1;3913 q_vector->rx.target_itr =3914 ITR_TO_REG(vsi->rx_rings[i]->itr_setting);3915 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),3916 q_vector->rx.target_itr >> 1);3917 q_vector->rx.current_itr = q_vector->rx.target_itr;3918 3919 q_vector->tx.next_update = jiffies + 1;3920 q_vector->tx.target_itr =3921 ITR_TO_REG(vsi->tx_rings[i]->itr_setting);3922 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),3923 q_vector->tx.target_itr >> 1);3924 q_vector->tx.current_itr = q_vector->tx.target_itr;3925 3926 /* Set ITR for software interrupts triggered after exiting3927 * busy-loop polling.3928 */3929 wr32(hw, I40E_PFINT_ITRN(I40E_SW_ITR, vector - 1),3930 I40E_ITR_20K);3931 3932 wr32(hw, I40E_PFINT_RATEN(vector - 1),3933 i40e_intrl_usec_to_reg(vsi->int_rate_limit));3934 3935 /* begin of linked list for RX queue assigned to this vector */3936 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);3937 for (q = 0; q < q_vector->num_ringpairs; q++) {3938 u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;3939 u32 val;3940 3941 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |3942 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |3943 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |3944 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |3945 (I40E_QUEUE_TYPE_TX <<3946 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);3947 3948 wr32(hw, I40E_QINT_RQCTL(qp), val);3949 3950 if (has_xdp) {3951 /* TX queue with next queue set to TX */3952 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |3953 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |3954 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |3955 (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |3956 (I40E_QUEUE_TYPE_TX <<3957 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);3958 3959 wr32(hw, I40E_QINT_TQCTL(nextqp), val);3960 }3961 /* TX queue with next RX or end of linked list */3962 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |3963 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |3964 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |3965 ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |3966 (I40E_QUEUE_TYPE_RX <<3967 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);3968 3969 /* Terminate the linked list */3970 if (q == (q_vector->num_ringpairs - 1))3971 val |= (I40E_QUEUE_END_OF_LIST <<3972 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);3973 3974 wr32(hw, I40E_QINT_TQCTL(qp), val);3975 qp++;3976 }3977 }3978 3979 i40e_flush(hw);3980}3981 3982/**3983 * i40e_enable_misc_int_causes - enable the non-queue interrupts3984 * @pf: pointer to private device data structure3985 **/3986static void i40e_enable_misc_int_causes(struct i40e_pf *pf)3987{3988 struct i40e_hw *hw = &pf->hw;3989 u32 val;3990 3991 /* clear things first */3992 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */3993 rd32(hw, I40E_PFINT_ICR0); /* read to clear */3994 3995 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |3996 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |3997 I40E_PFINT_ICR0_ENA_GRST_MASK |3998 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |3999 I40E_PFINT_ICR0_ENA_GPIO_MASK |4000 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK |4001 I40E_PFINT_ICR0_ENA_VFLR_MASK |4002 I40E_PFINT_ICR0_ENA_ADMINQ_MASK;4003 4004 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags))4005 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;4006 4007 if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))4008 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;4009 4010 wr32(hw, I40E_PFINT_ICR0_ENA, val);4011 4012 /* SW_ITR_IDX = 0, but don't change INTENA */4013 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |4014 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);4015 4016 /* OTHER_ITR_IDX = 0 */4017 wr32(hw, I40E_PFINT_STAT_CTL0, 0);4018}4019 4020/**4021 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW4022 * @vsi: the VSI being configured4023 **/4024static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)4025{4026 u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;4027 struct i40e_q_vector *q_vector = vsi->q_vectors[0];4028 struct i40e_pf *pf = vsi->back;4029 struct i40e_hw *hw = &pf->hw;4030 4031 /* set the ITR configuration */4032 q_vector->rx.next_update = jiffies + 1;4033 q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting);4034 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1);4035 q_vector->rx.current_itr = q_vector->rx.target_itr;4036 q_vector->tx.next_update = jiffies + 1;4037 q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting);4038 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1);4039 q_vector->tx.current_itr = q_vector->tx.target_itr;4040 4041 i40e_enable_misc_int_causes(pf);4042 4043 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */4044 wr32(hw, I40E_PFINT_LNKLST0, 0);4045 4046 /* Associate the queue pair to the vector and enable the queue4047 * interrupt RX queue in linked list with next queue set to TX4048 */4049 wr32(hw, I40E_QINT_RQCTL(0), I40E_QINT_RQCTL_VAL(nextqp, 0, TX));4050 4051 if (i40e_enabled_xdp_vsi(vsi)) {4052 /* TX queue in linked list with next queue set to TX */4053 wr32(hw, I40E_QINT_TQCTL(nextqp),4054 I40E_QINT_TQCTL_VAL(nextqp, 0, TX));4055 }4056 4057 /* last TX queue so the next RX queue doesn't matter */4058 wr32(hw, I40E_QINT_TQCTL(0),4059 I40E_QINT_TQCTL_VAL(I40E_QUEUE_END_OF_LIST, 0, RX));4060 i40e_flush(hw);4061}4062 4063/**4064 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr04065 * @pf: board private structure4066 **/4067void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)4068{4069 struct i40e_hw *hw = &pf->hw;4070 4071 wr32(hw, I40E_PFINT_DYN_CTL0,4072 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);4073 i40e_flush(hw);4074}4075 4076/**4077 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr04078 * @pf: board private structure4079 **/4080void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)4081{4082 struct i40e_hw *hw = &pf->hw;4083 u32 val;4084 4085 val = I40E_PFINT_DYN_CTL0_INTENA_MASK |4086 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |4087 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);4088 4089 wr32(hw, I40E_PFINT_DYN_CTL0, val);4090 i40e_flush(hw);4091}4092 4093/**4094 * i40e_msix_clean_rings - MSIX mode Interrupt Handler4095 * @irq: interrupt number4096 * @data: pointer to a q_vector4097 **/4098static irqreturn_t i40e_msix_clean_rings(int irq, void *data)4099{4100 struct i40e_q_vector *q_vector = data;4101 4102 if (!q_vector->tx.ring && !q_vector->rx.ring)4103 return IRQ_HANDLED;4104 4105 napi_schedule_irqoff(&q_vector->napi);4106 4107 return IRQ_HANDLED;4108}4109 4110/**4111 * i40e_irq_affinity_notify - Callback for affinity changes4112 * @notify: context as to what irq was changed4113 * @mask: the new affinity mask4114 *4115 * This is a callback function used by the irq_set_affinity_notifier function4116 * so that we may register to receive changes to the irq affinity masks.4117 **/4118static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,4119 const cpumask_t *mask)4120{4121 struct i40e_q_vector *q_vector =4122 container_of(notify, struct i40e_q_vector, affinity_notify);4123 4124 cpumask_copy(&q_vector->affinity_mask, mask);4125}4126 4127/**4128 * i40e_irq_affinity_release - Callback for affinity notifier release4129 * @ref: internal core kernel usage4130 *4131 * This is a callback function used by the irq_set_affinity_notifier function4132 * to inform the current notification subscriber that they will no longer4133 * receive notifications.4134 **/4135static void i40e_irq_affinity_release(struct kref *ref) {}4136 4137/**4138 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts4139 * @vsi: the VSI being configured4140 * @basename: name for the vector4141 *4142 * Allocates MSI-X vectors and requests interrupts from the kernel.4143 **/4144static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)4145{4146 int q_vectors = vsi->num_q_vectors;4147 struct i40e_pf *pf = vsi->back;4148 int base = vsi->base_vector;4149 int rx_int_idx = 0;4150 int tx_int_idx = 0;4151 int vector, err;4152 int irq_num;4153 int cpu;4154 4155 for (vector = 0; vector < q_vectors; vector++) {4156 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];4157 4158 irq_num = pf->msix_entries[base + vector].vector;4159 4160 if (q_vector->tx.ring && q_vector->rx.ring) {4161 snprintf(q_vector->name, sizeof(q_vector->name) - 1,4162 "%s-%s-%d", basename, "TxRx", rx_int_idx++);4163 tx_int_idx++;4164 } else if (q_vector->rx.ring) {4165 snprintf(q_vector->name, sizeof(q_vector->name) - 1,4166 "%s-%s-%d", basename, "rx", rx_int_idx++);4167 } else if (q_vector->tx.ring) {4168 snprintf(q_vector->name, sizeof(q_vector->name) - 1,4169 "%s-%s-%d", basename, "tx", tx_int_idx++);4170 } else {4171 /* skip this unused q_vector */4172 continue;4173 }4174 err = request_irq(irq_num,4175 vsi->irq_handler,4176 0,4177 q_vector->name,4178 q_vector);4179 if (err) {4180 dev_info(&pf->pdev->dev,4181 "MSIX request_irq failed, error: %d\n", err);4182 goto free_queue_irqs;4183 }4184 4185 /* register for affinity change notifications */4186 q_vector->irq_num = irq_num;4187 q_vector->affinity_notify.notify = i40e_irq_affinity_notify;4188 q_vector->affinity_notify.release = i40e_irq_affinity_release;4189 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);4190 /* Spread affinity hints out across online CPUs.4191 *4192 * get_cpu_mask returns a static constant mask with4193 * a permanent lifetime so it's ok to pass to4194 * irq_update_affinity_hint without making a copy.4195 */4196 cpu = cpumask_local_spread(q_vector->v_idx, -1);4197 irq_update_affinity_hint(irq_num, get_cpu_mask(cpu));4198 }4199 4200 vsi->irqs_ready = true;4201 return 0;4202 4203free_queue_irqs:4204 while (vector) {4205 vector--;4206 irq_num = pf->msix_entries[base + vector].vector;4207 irq_set_affinity_notifier(irq_num, NULL);4208 irq_update_affinity_hint(irq_num, NULL);4209 free_irq(irq_num, &vsi->q_vectors[vector]);4210 }4211 return err;4212}4213 4214/**4215 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI4216 * @vsi: the VSI being un-configured4217 **/4218static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)4219{4220 struct i40e_pf *pf = vsi->back;4221 struct i40e_hw *hw = &pf->hw;4222 int base = vsi->base_vector;4223 int i;4224 4225 /* disable interrupt causation from each queue */4226 for (i = 0; i < vsi->num_queue_pairs; i++) {4227 u32 val;4228 4229 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));4230 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;4231 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);4232 4233 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));4234 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;4235 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);4236 4237 if (!i40e_enabled_xdp_vsi(vsi))4238 continue;4239 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);4240 }4241 4242 /* disable each interrupt */4243 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {4244 for (i = vsi->base_vector;4245 i < (vsi->num_q_vectors + vsi->base_vector); i++)4246 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);4247 4248 i40e_flush(hw);4249 for (i = 0; i < vsi->num_q_vectors; i++)4250 synchronize_irq(pf->msix_entries[i + base].vector);4251 } else {4252 /* Legacy and MSI mode - this stops all interrupt handling */4253 wr32(hw, I40E_PFINT_ICR0_ENA, 0);4254 wr32(hw, I40E_PFINT_DYN_CTL0, 0);4255 i40e_flush(hw);4256 synchronize_irq(pf->pdev->irq);4257 }4258}4259 4260/**4261 * i40e_vsi_enable_irq - Enable IRQ for the given VSI4262 * @vsi: the VSI being configured4263 **/4264static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)4265{4266 struct i40e_pf *pf = vsi->back;4267 int i;4268 4269 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {4270 for (i = 0; i < vsi->num_q_vectors; i++)4271 i40e_irq_dynamic_enable(vsi, i);4272 } else {4273 i40e_irq_dynamic_enable_icr0(pf);4274 }4275 4276 i40e_flush(&pf->hw);4277 return 0;4278}4279 4280/**4281 * i40e_free_misc_vector - Free the vector that handles non-queue events4282 * @pf: board private structure4283 **/4284static void i40e_free_misc_vector(struct i40e_pf *pf)4285{4286 /* Disable ICR 0 */4287 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);4288 i40e_flush(&pf->hw);4289 4290 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && pf->msix_entries) {4291 free_irq(pf->msix_entries[0].vector, pf);4292 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);4293 }4294}4295 4296/**4297 * i40e_intr - MSI/Legacy and non-queue interrupt handler4298 * @irq: interrupt number4299 * @data: pointer to a q_vector4300 *4301 * This is the handler used for all MSI/Legacy interrupts, and deals4302 * with both queue and non-queue interrupts. This is also used in4303 * MSIX mode to handle the non-queue interrupts.4304 **/4305static irqreturn_t i40e_intr(int irq, void *data)4306{4307 struct i40e_pf *pf = (struct i40e_pf *)data;4308 struct i40e_hw *hw = &pf->hw;4309 irqreturn_t ret = IRQ_NONE;4310 u32 icr0, icr0_remaining;4311 u32 val, ena_mask;4312 4313 icr0 = rd32(hw, I40E_PFINT_ICR0);4314 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);4315 4316 /* if sharing a legacy IRQ, we might get called w/o an intr pending */4317 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)4318 goto enable_intr;4319 4320 /* if interrupt but no bits showing, must be SWINT */4321 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||4322 (icr0 & I40E_PFINT_ICR0_SWINT_MASK))4323 pf->sw_int_count++;4324 4325 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) &&4326 (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {4327 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;4328 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");4329 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);4330 }4331 4332 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */4333 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {4334 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);4335 struct i40e_q_vector *q_vector = vsi->q_vectors[0];4336 4337 /* We do not have a way to disarm Queue causes while leaving4338 * interrupt enabled for all other causes, ideally4339 * interrupt should be disabled while we are in NAPI but4340 * this is not a performance path and napi_schedule()4341 * can deal with rescheduling.4342 */4343 if (!test_bit(__I40E_DOWN, pf->state))4344 napi_schedule_irqoff(&q_vector->napi);4345 }4346 4347 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {4348 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;4349 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);4350 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");4351 }4352 4353 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {4354 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;4355 set_bit(__I40E_MDD_EVENT_PENDING, pf->state);4356 }4357 4358 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {4359 /* disable any further VFLR event notifications */4360 if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) {4361 u32 reg = rd32(hw, I40E_PFINT_ICR0_ENA);4362 4363 reg &= ~I40E_PFINT_ICR0_VFLR_MASK;4364 wr32(hw, I40E_PFINT_ICR0_ENA, reg);4365 } else {4366 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;4367 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);4368 }4369 }4370 4371 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {4372 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))4373 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);4374 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;4375 val = rd32(hw, I40E_GLGEN_RSTAT);4376 val = FIELD_GET(I40E_GLGEN_RSTAT_RESET_TYPE_MASK, val);4377 if (val == I40E_RESET_CORER) {4378 pf->corer_count++;4379 } else if (val == I40E_RESET_GLOBR) {4380 pf->globr_count++;4381 } else if (val == I40E_RESET_EMPR) {4382 pf->empr_count++;4383 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);4384 }4385 }4386 4387 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {4388 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;4389 dev_info(&pf->pdev->dev, "HMC error interrupt\n");4390 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",4391 rd32(hw, I40E_PFHMC_ERRORINFO),4392 rd32(hw, I40E_PFHMC_ERRORDATA));4393 }4394 4395 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {4396 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);4397 4398 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_EVENT0_MASK)4399 schedule_work(&pf->ptp_extts0_work);4400 4401 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK)4402 i40e_ptp_tx_hwtstamp(pf);4403 4404 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;4405 }4406 4407 /* If a critical error is pending we have no choice but to reset the4408 * device.4409 * Report and mask out any remaining unexpected interrupts.4410 */4411 icr0_remaining = icr0 & ena_mask;4412 if (icr0_remaining) {4413 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",4414 icr0_remaining);4415 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||4416 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||4417 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {4418 dev_info(&pf->pdev->dev, "device will be reset\n");4419 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);4420 i40e_service_event_schedule(pf);4421 }4422 ena_mask &= ~icr0_remaining;4423 }4424 ret = IRQ_HANDLED;4425 4426enable_intr:4427 /* re-enable interrupt causes */4428 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);4429 if (!test_bit(__I40E_DOWN, pf->state) ||4430 test_bit(__I40E_RECOVERY_MODE, pf->state)) {4431 i40e_service_event_schedule(pf);4432 i40e_irq_dynamic_enable_icr0(pf);4433 }4434 4435 return ret;4436}4437 4438/**4439 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes4440 * @tx_ring: tx ring to clean4441 * @budget: how many cleans we're allowed4442 *4443 * Returns true if there's any budget left (e.g. the clean is finished)4444 **/4445static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)4446{4447 struct i40e_vsi *vsi = tx_ring->vsi;4448 u16 i = tx_ring->next_to_clean;4449 struct i40e_tx_buffer *tx_buf;4450 struct i40e_tx_desc *tx_desc;4451 4452 tx_buf = &tx_ring->tx_bi[i];4453 tx_desc = I40E_TX_DESC(tx_ring, i);4454 i -= tx_ring->count;4455 4456 do {4457 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;4458 4459 /* if next_to_watch is not set then there is no work pending */4460 if (!eop_desc)4461 break;4462 4463 /* prevent any other reads prior to eop_desc */4464 smp_rmb();4465 4466 /* if the descriptor isn't done, no work yet to do */4467 if (!(eop_desc->cmd_type_offset_bsz &4468 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))4469 break;4470 4471 /* clear next_to_watch to prevent false hangs */4472 tx_buf->next_to_watch = NULL;4473 4474 tx_desc->buffer_addr = 0;4475 tx_desc->cmd_type_offset_bsz = 0;4476 /* move past filter desc */4477 tx_buf++;4478 tx_desc++;4479 i++;4480 if (unlikely(!i)) {4481 i -= tx_ring->count;4482 tx_buf = tx_ring->tx_bi;4483 tx_desc = I40E_TX_DESC(tx_ring, 0);4484 }4485 /* unmap skb header data */4486 dma_unmap_single(tx_ring->dev,4487 dma_unmap_addr(tx_buf, dma),4488 dma_unmap_len(tx_buf, len),4489 DMA_TO_DEVICE);4490 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)4491 kfree(tx_buf->raw_buf);4492 4493 tx_buf->raw_buf = NULL;4494 tx_buf->tx_flags = 0;4495 tx_buf->next_to_watch = NULL;4496 dma_unmap_len_set(tx_buf, len, 0);4497 tx_desc->buffer_addr = 0;4498 tx_desc->cmd_type_offset_bsz = 0;4499 4500 /* move us past the eop_desc for start of next FD desc */4501 tx_buf++;4502 tx_desc++;4503 i++;4504 if (unlikely(!i)) {4505 i -= tx_ring->count;4506 tx_buf = tx_ring->tx_bi;4507 tx_desc = I40E_TX_DESC(tx_ring, 0);4508 }4509 4510 /* update budget accounting */4511 budget--;4512 } while (likely(budget));4513 4514 i += tx_ring->count;4515 tx_ring->next_to_clean = i;4516 4517 if (test_bit(I40E_FLAG_MSIX_ENA, vsi->back->flags))4518 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);4519 4520 return budget > 0;4521}4522 4523/**4524 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring4525 * @irq: interrupt number4526 * @data: pointer to a q_vector4527 **/4528static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)4529{4530 struct i40e_q_vector *q_vector = data;4531 struct i40e_vsi *vsi;4532 4533 if (!q_vector->tx.ring)4534 return IRQ_HANDLED;4535 4536 vsi = q_vector->tx.ring->vsi;4537 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);4538 4539 return IRQ_HANDLED;4540}4541 4542/**4543 * i40e_map_vector_to_qp - Assigns the queue pair to the vector4544 * @vsi: the VSI being configured4545 * @v_idx: vector index4546 * @qp_idx: queue pair index4547 **/4548static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)4549{4550 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];4551 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];4552 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];4553 4554 tx_ring->q_vector = q_vector;4555 tx_ring->next = q_vector->tx.ring;4556 q_vector->tx.ring = tx_ring;4557 q_vector->tx.count++;4558 4559 /* Place XDP Tx ring in the same q_vector ring list as regular Tx */4560 if (i40e_enabled_xdp_vsi(vsi)) {4561 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];4562 4563 xdp_ring->q_vector = q_vector;4564 xdp_ring->next = q_vector->tx.ring;4565 q_vector->tx.ring = xdp_ring;4566 q_vector->tx.count++;4567 }4568 4569 rx_ring->q_vector = q_vector;4570 rx_ring->next = q_vector->rx.ring;4571 q_vector->rx.ring = rx_ring;4572 q_vector->rx.count++;4573}4574 4575/**4576 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors4577 * @vsi: the VSI being configured4578 *4579 * This function maps descriptor rings to the queue-specific vectors4580 * we were allotted through the MSI-X enabling code. Ideally, we'd have4581 * one vector per queue pair, but on a constrained vector budget, we4582 * group the queue pairs as "efficiently" as possible.4583 **/4584static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)4585{4586 int qp_remaining = vsi->num_queue_pairs;4587 int q_vectors = vsi->num_q_vectors;4588 int num_ringpairs;4589 int v_start = 0;4590 int qp_idx = 0;4591 4592 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to4593 * group them so there are multiple queues per vector.4594 * It is also important to go through all the vectors available to be4595 * sure that if we don't use all the vectors, that the remaining vectors4596 * are cleared. This is especially important when decreasing the4597 * number of queues in use.4598 */4599 for (; v_start < q_vectors; v_start++) {4600 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];4601 4602 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);4603 4604 q_vector->num_ringpairs = num_ringpairs;4605 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1;4606 4607 q_vector->rx.count = 0;4608 q_vector->tx.count = 0;4609 q_vector->rx.ring = NULL;4610 q_vector->tx.ring = NULL;4611 4612 while (num_ringpairs--) {4613 i40e_map_vector_to_qp(vsi, v_start, qp_idx);4614 qp_idx++;4615 qp_remaining--;4616 }4617 }4618}4619 4620/**4621 * i40e_vsi_request_irq - Request IRQ from the OS4622 * @vsi: the VSI being configured4623 * @basename: name for the vector4624 **/4625static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)4626{4627 struct i40e_pf *pf = vsi->back;4628 int err;4629 4630 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))4631 err = i40e_vsi_request_irq_msix(vsi, basename);4632 else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags))4633 err = request_irq(pf->pdev->irq, i40e_intr, 0,4634 pf->int_name, pf);4635 else4636 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,4637 pf->int_name, pf);4638 4639 if (err)4640 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);4641 4642 return err;4643}4644 4645#ifdef CONFIG_NET_POLL_CONTROLLER4646/**4647 * i40e_netpoll - A Polling 'interrupt' handler4648 * @netdev: network interface device structure4649 *4650 * This is used by netconsole to send skbs without having to re-enable4651 * interrupts. It's not called while the normal interrupt routine is executing.4652 **/4653static void i40e_netpoll(struct net_device *netdev)4654{4655 struct i40e_netdev_priv *np = netdev_priv(netdev);4656 struct i40e_vsi *vsi = np->vsi;4657 struct i40e_pf *pf = vsi->back;4658 int i;4659 4660 /* if interface is down do nothing */4661 if (test_bit(__I40E_VSI_DOWN, vsi->state))4662 return;4663 4664 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {4665 for (i = 0; i < vsi->num_q_vectors; i++)4666 i40e_msix_clean_rings(0, vsi->q_vectors[i]);4667 } else {4668 i40e_intr(pf->pdev->irq, netdev);4669 }4670}4671#endif4672 4673#define I40E_QTX_ENA_WAIT_COUNT 504674 4675/**4676 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled4677 * @pf: the PF being configured4678 * @pf_q: the PF queue4679 * @enable: enable or disable state of the queue4680 *4681 * This routine will wait for the given Tx queue of the PF to reach the4682 * enabled or disabled state.4683 * Returns -ETIMEDOUT in case of failing to reach the requested state after4684 * multiple retries; else will return 0 in case of success.4685 **/4686static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)4687{4688 int i;4689 u32 tx_reg;4690 4691 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {4692 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));4693 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))4694 break;4695 4696 usleep_range(10, 20);4697 }4698 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)4699 return -ETIMEDOUT;4700 4701 return 0;4702}4703 4704/**4705 * i40e_control_tx_q - Start or stop a particular Tx queue4706 * @pf: the PF structure4707 * @pf_q: the PF queue to configure4708 * @enable: start or stop the queue4709 *4710 * This function enables or disables a single queue. Note that any delay4711 * required after the operation is expected to be handled by the caller of4712 * this function.4713 **/4714static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)4715{4716 struct i40e_hw *hw = &pf->hw;4717 u32 tx_reg;4718 int i;4719 4720 /* warn the TX unit of coming changes */4721 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);4722 if (!enable)4723 usleep_range(10, 20);4724 4725 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {4726 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));4727 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==4728 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))4729 break;4730 usleep_range(1000, 2000);4731 }4732 4733 /* Skip if the queue is already in the requested state */4734 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))4735 return;4736 4737 /* turn on/off the queue */4738 if (enable) {4739 wr32(hw, I40E_QTX_HEAD(pf_q), 0);4740 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;4741 } else {4742 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;4743 }4744 4745 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);4746}4747 4748/**4749 * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion4750 * @seid: VSI SEID4751 * @pf: the PF structure4752 * @pf_q: the PF queue to configure4753 * @is_xdp: true if the queue is used for XDP4754 * @enable: start or stop the queue4755 **/4756int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,4757 bool is_xdp, bool enable)4758{4759 int ret;4760 4761 i40e_control_tx_q(pf, pf_q, enable);4762 4763 /* wait for the change to finish */4764 ret = i40e_pf_txq_wait(pf, pf_q, enable);4765 if (ret) {4766 dev_info(&pf->pdev->dev,4767 "VSI seid %d %sTx ring %d %sable timeout\n",4768 seid, (is_xdp ? "XDP " : ""), pf_q,4769 (enable ? "en" : "dis"));4770 }4771 4772 return ret;4773}4774 4775/**4776 * i40e_vsi_enable_tx - Start a VSI's rings4777 * @vsi: the VSI being configured4778 **/4779static int i40e_vsi_enable_tx(struct i40e_vsi *vsi)4780{4781 struct i40e_pf *pf = vsi->back;4782 int i, pf_q, ret = 0;4783 4784 pf_q = vsi->base_queue;4785 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {4786 ret = i40e_control_wait_tx_q(vsi->seid, pf,4787 pf_q,4788 false /*is xdp*/, true);4789 if (ret)4790 break;4791 4792 if (!i40e_enabled_xdp_vsi(vsi))4793 continue;4794 4795 ret = i40e_control_wait_tx_q(vsi->seid, pf,4796 pf_q + vsi->alloc_queue_pairs,4797 true /*is xdp*/, true);4798 if (ret)4799 break;4800 }4801 return ret;4802}4803 4804/**4805 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled4806 * @pf: the PF being configured4807 * @pf_q: the PF queue4808 * @enable: enable or disable state of the queue4809 *4810 * This routine will wait for the given Rx queue of the PF to reach the4811 * enabled or disabled state.4812 * Returns -ETIMEDOUT in case of failing to reach the requested state after4813 * multiple retries; else will return 0 in case of success.4814 **/4815static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)4816{4817 int i;4818 u32 rx_reg;4819 4820 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {4821 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));4822 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))4823 break;4824 4825 usleep_range(10, 20);4826 }4827 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)4828 return -ETIMEDOUT;4829 4830 return 0;4831}4832 4833/**4834 * i40e_control_rx_q - Start or stop a particular Rx queue4835 * @pf: the PF structure4836 * @pf_q: the PF queue to configure4837 * @enable: start or stop the queue4838 *4839 * This function enables or disables a single queue. Note that4840 * any delay required after the operation is expected to be4841 * handled by the caller of this function.4842 **/4843static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)4844{4845 struct i40e_hw *hw = &pf->hw;4846 u32 rx_reg;4847 int i;4848 4849 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {4850 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));4851 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==4852 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))4853 break;4854 usleep_range(1000, 2000);4855 }4856 4857 /* Skip if the queue is already in the requested state */4858 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))4859 return;4860 4861 /* turn on/off the queue */4862 if (enable)4863 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;4864 else4865 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;4866 4867 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);4868}4869 4870/**4871 * i40e_control_wait_rx_q4872 * @pf: the PF structure4873 * @pf_q: queue being configured4874 * @enable: start or stop the rings4875 *4876 * This function enables or disables a single queue along with waiting4877 * for the change to finish. The caller of this function should handle4878 * the delays needed in the case of disabling queues.4879 **/4880int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable)4881{4882 int ret = 0;4883 4884 i40e_control_rx_q(pf, pf_q, enable);4885 4886 /* wait for the change to finish */4887 ret = i40e_pf_rxq_wait(pf, pf_q, enable);4888 if (ret)4889 return ret;4890 4891 return ret;4892}4893 4894/**4895 * i40e_vsi_enable_rx - Start a VSI's rings4896 * @vsi: the VSI being configured4897 **/4898static int i40e_vsi_enable_rx(struct i40e_vsi *vsi)4899{4900 struct i40e_pf *pf = vsi->back;4901 int i, pf_q, ret = 0;4902 4903 pf_q = vsi->base_queue;4904 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {4905 ret = i40e_control_wait_rx_q(pf, pf_q, true);4906 if (ret) {4907 dev_info(&pf->pdev->dev,4908 "VSI seid %d Rx ring %d enable timeout\n",4909 vsi->seid, pf_q);4910 break;4911 }4912 }4913 4914 return ret;4915}4916 4917/**4918 * i40e_vsi_start_rings - Start a VSI's rings4919 * @vsi: the VSI being configured4920 **/4921int i40e_vsi_start_rings(struct i40e_vsi *vsi)4922{4923 int ret = 0;4924 4925 /* do rx first for enable and last for disable */4926 ret = i40e_vsi_enable_rx(vsi);4927 if (ret)4928 return ret;4929 ret = i40e_vsi_enable_tx(vsi);4930 4931 return ret;4932}4933 4934#define I40E_DISABLE_TX_GAP_MSEC 504935 4936/**4937 * i40e_vsi_stop_rings - Stop a VSI's rings4938 * @vsi: the VSI being configured4939 **/4940void i40e_vsi_stop_rings(struct i40e_vsi *vsi)4941{4942 struct i40e_pf *pf = vsi->back;4943 u32 pf_q, tx_q_end, rx_q_end;4944 4945 /* When port TX is suspended, don't wait */4946 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))4947 return i40e_vsi_stop_rings_no_wait(vsi);4948 4949 tx_q_end = vsi->base_queue +4950 vsi->alloc_queue_pairs * (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);4951 for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++)4952 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, false);4953 4954 rx_q_end = vsi->base_queue + vsi->num_queue_pairs;4955 for (pf_q = vsi->base_queue; pf_q < rx_q_end; pf_q++)4956 i40e_control_rx_q(pf, pf_q, false);4957 4958 msleep(I40E_DISABLE_TX_GAP_MSEC);4959 for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++)4960 wr32(&pf->hw, I40E_QTX_ENA(pf_q), 0);4961 4962 i40e_vsi_wait_queues_disabled(vsi);4963}4964 4965/**4966 * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay4967 * @vsi: the VSI being shutdown4968 *4969 * This function stops all the rings for a VSI but does not delay to verify4970 * that rings have been disabled. It is expected that the caller is shutting4971 * down multiple VSIs at once and will delay together for all the VSIs after4972 * initiating the shutdown. This is particularly useful for shutting down lots4973 * of VFs together. Otherwise, a large delay can be incurred while configuring4974 * each VSI in serial.4975 **/4976void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)4977{4978 struct i40e_pf *pf = vsi->back;4979 int i, pf_q;4980 4981 pf_q = vsi->base_queue;4982 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {4983 i40e_control_tx_q(pf, pf_q, false);4984 i40e_control_rx_q(pf, pf_q, false);4985 }4986}4987 4988/**4989 * i40e_vsi_free_irq - Free the irq association with the OS4990 * @vsi: the VSI being configured4991 **/4992static void i40e_vsi_free_irq(struct i40e_vsi *vsi)4993{4994 struct i40e_pf *pf = vsi->back;4995 struct i40e_hw *hw = &pf->hw;4996 int base = vsi->base_vector;4997 u32 val, qp;4998 int i;4999 5000 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {5001 if (!vsi->q_vectors)5002 return;5003 5004 if (!vsi->irqs_ready)5005 return;5006 5007 vsi->irqs_ready = false;5008 for (i = 0; i < vsi->num_q_vectors; i++) {5009 int irq_num;5010 u16 vector;5011 5012 vector = i + base;5013 irq_num = pf->msix_entries[vector].vector;5014 5015 /* free only the irqs that were actually requested */5016 if (!vsi->q_vectors[i] ||5017 !vsi->q_vectors[i]->num_ringpairs)5018 continue;5019 5020 /* clear the affinity notifier in the IRQ descriptor */5021 irq_set_affinity_notifier(irq_num, NULL);5022 /* remove our suggested affinity mask for this IRQ */5023 irq_update_affinity_hint(irq_num, NULL);5024 free_irq(irq_num, vsi->q_vectors[i]);5025 5026 /* Tear down the interrupt queue link list5027 *5028 * We know that they come in pairs and always5029 * the Rx first, then the Tx. To clear the5030 * link list, stick the EOL value into the5031 * next_q field of the registers.5032 */5033 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));5034 qp = FIELD_GET(I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK,5035 val);5036 val |= I40E_QUEUE_END_OF_LIST5037 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;5038 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);5039 5040 while (qp != I40E_QUEUE_END_OF_LIST) {5041 u32 next;5042 5043 val = rd32(hw, I40E_QINT_RQCTL(qp));5044 5045 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK |5046 I40E_QINT_RQCTL_MSIX0_INDX_MASK |5047 I40E_QINT_RQCTL_CAUSE_ENA_MASK |5048 I40E_QINT_RQCTL_INTEVENT_MASK);5049 5050 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |5051 I40E_QINT_RQCTL_NEXTQ_INDX_MASK);5052 5053 wr32(hw, I40E_QINT_RQCTL(qp), val);5054 5055 val = rd32(hw, I40E_QINT_TQCTL(qp));5056 5057 next = FIELD_GET(I40E_QINT_TQCTL_NEXTQ_INDX_MASK,5058 val);5059 5060 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK |5061 I40E_QINT_TQCTL_MSIX0_INDX_MASK |5062 I40E_QINT_TQCTL_CAUSE_ENA_MASK |5063 I40E_QINT_TQCTL_INTEVENT_MASK);5064 5065 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |5066 I40E_QINT_TQCTL_NEXTQ_INDX_MASK);5067 5068 wr32(hw, I40E_QINT_TQCTL(qp), val);5069 qp = next;5070 }5071 }5072 } else {5073 free_irq(pf->pdev->irq, pf);5074 5075 val = rd32(hw, I40E_PFINT_LNKLST0);5076 qp = FIELD_GET(I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK, val);5077 val |= I40E_QUEUE_END_OF_LIST5078 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;5079 wr32(hw, I40E_PFINT_LNKLST0, val);5080 5081 val = rd32(hw, I40E_QINT_RQCTL(qp));5082 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK |5083 I40E_QINT_RQCTL_MSIX0_INDX_MASK |5084 I40E_QINT_RQCTL_CAUSE_ENA_MASK |5085 I40E_QINT_RQCTL_INTEVENT_MASK);5086 5087 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |5088 I40E_QINT_RQCTL_NEXTQ_INDX_MASK);5089 5090 wr32(hw, I40E_QINT_RQCTL(qp), val);5091 5092 val = rd32(hw, I40E_QINT_TQCTL(qp));5093 5094 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK |5095 I40E_QINT_TQCTL_MSIX0_INDX_MASK |5096 I40E_QINT_TQCTL_CAUSE_ENA_MASK |5097 I40E_QINT_TQCTL_INTEVENT_MASK);5098 5099 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |5100 I40E_QINT_TQCTL_NEXTQ_INDX_MASK);5101 5102 wr32(hw, I40E_QINT_TQCTL(qp), val);5103 }5104}5105 5106/**5107 * i40e_free_q_vector - Free memory allocated for specific interrupt vector5108 * @vsi: the VSI being configured5109 * @v_idx: Index of vector to be freed5110 *5111 * This function frees the memory allocated to the q_vector. In addition if5112 * NAPI is enabled it will delete any references to the NAPI struct prior5113 * to freeing the q_vector.5114 **/5115static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)5116{5117 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];5118 struct i40e_ring *ring;5119 5120 if (!q_vector)5121 return;5122 5123 /* disassociate q_vector from rings */5124 i40e_for_each_ring(ring, q_vector->tx)5125 ring->q_vector = NULL;5126 5127 i40e_for_each_ring(ring, q_vector->rx)5128 ring->q_vector = NULL;5129 5130 /* only VSI w/ an associated netdev is set up w/ NAPI */5131 if (vsi->netdev)5132 netif_napi_del(&q_vector->napi);5133 5134 vsi->q_vectors[v_idx] = NULL;5135 5136 kfree_rcu(q_vector, rcu);5137}5138 5139/**5140 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors5141 * @vsi: the VSI being un-configured5142 *5143 * This frees the memory allocated to the q_vectors and5144 * deletes references to the NAPI struct.5145 **/5146static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)5147{5148 int v_idx;5149 5150 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)5151 i40e_free_q_vector(vsi, v_idx);5152}5153 5154/**5155 * i40e_reset_interrupt_capability - Disable interrupt setup in OS5156 * @pf: board private structure5157 **/5158static void i40e_reset_interrupt_capability(struct i40e_pf *pf)5159{5160 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */5161 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {5162 pci_disable_msix(pf->pdev);5163 kfree(pf->msix_entries);5164 pf->msix_entries = NULL;5165 kfree(pf->irq_pile);5166 pf->irq_pile = NULL;5167 } else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags)) {5168 pci_disable_msi(pf->pdev);5169 }5170 clear_bit(I40E_FLAG_MSI_ENA, pf->flags);5171 clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);5172}5173 5174/**5175 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings5176 * @pf: board private structure5177 *5178 * We go through and clear interrupt specific resources and reset the structure5179 * to pre-load conditions5180 **/5181static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)5182{5183 struct i40e_vsi *vsi;5184 int i;5185 5186 if (test_bit(__I40E_MISC_IRQ_REQUESTED, pf->state))5187 i40e_free_misc_vector(pf);5188 5189 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,5190 I40E_IWARP_IRQ_PILE_ID);5191 5192 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);5193 5194 i40e_pf_for_each_vsi(pf, i, vsi)5195 i40e_vsi_free_q_vectors(vsi);5196 5197 i40e_reset_interrupt_capability(pf);5198}5199 5200/**5201 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI5202 * @vsi: the VSI being configured5203 **/5204static void i40e_napi_enable_all(struct i40e_vsi *vsi)5205{5206 int q_idx;5207 5208 if (!vsi->netdev)5209 return;5210 5211 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {5212 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];5213 5214 if (q_vector->rx.ring || q_vector->tx.ring)5215 napi_enable(&q_vector->napi);5216 }5217}5218 5219/**5220 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI5221 * @vsi: the VSI being configured5222 **/5223static void i40e_napi_disable_all(struct i40e_vsi *vsi)5224{5225 int q_idx;5226 5227 if (!vsi->netdev)5228 return;5229 5230 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {5231 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];5232 5233 if (q_vector->rx.ring || q_vector->tx.ring)5234 napi_disable(&q_vector->napi);5235 }5236}5237 5238/**5239 * i40e_vsi_close - Shut down a VSI5240 * @vsi: the vsi to be quelled5241 **/5242static void i40e_vsi_close(struct i40e_vsi *vsi)5243{5244 struct i40e_pf *pf = vsi->back;5245 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))5246 i40e_down(vsi);5247 i40e_vsi_free_irq(vsi);5248 i40e_vsi_free_tx_resources(vsi);5249 i40e_vsi_free_rx_resources(vsi);5250 vsi->current_netdev_flags = 0;5251 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);5252 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))5253 set_bit(__I40E_CLIENT_RESET, pf->state);5254}5255 5256/**5257 * i40e_quiesce_vsi - Pause a given VSI5258 * @vsi: the VSI being paused5259 **/5260static void i40e_quiesce_vsi(struct i40e_vsi *vsi)5261{5262 if (test_bit(__I40E_VSI_DOWN, vsi->state))5263 return;5264 5265 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);5266 if (vsi->netdev && netif_running(vsi->netdev))5267 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);5268 else5269 i40e_vsi_close(vsi);5270}5271 5272/**5273 * i40e_unquiesce_vsi - Resume a given VSI5274 * @vsi: the VSI being resumed5275 **/5276static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)5277{5278 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))5279 return;5280 5281 if (vsi->netdev && netif_running(vsi->netdev))5282 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);5283 else5284 i40e_vsi_open(vsi); /* this clears the DOWN bit */5285}5286 5287/**5288 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF5289 * @pf: the PF5290 **/5291static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)5292{5293 struct i40e_vsi *vsi;5294 int v;5295 5296 i40e_pf_for_each_vsi(pf, v, vsi)5297 i40e_quiesce_vsi(vsi);5298}5299 5300/**5301 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF5302 * @pf: the PF5303 **/5304static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)5305{5306 struct i40e_vsi *vsi;5307 int v;5308 5309 i40e_pf_for_each_vsi(pf, v, vsi)5310 i40e_unquiesce_vsi(vsi);5311}5312 5313/**5314 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled5315 * @vsi: the VSI being configured5316 *5317 * Wait until all queues on a given VSI have been disabled.5318 **/5319int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)5320{5321 struct i40e_pf *pf = vsi->back;5322 int i, pf_q, ret;5323 5324 pf_q = vsi->base_queue;5325 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {5326 /* Check and wait for the Tx queue */5327 ret = i40e_pf_txq_wait(pf, pf_q, false);5328 if (ret) {5329 dev_info(&pf->pdev->dev,5330 "VSI seid %d Tx ring %d disable timeout\n",5331 vsi->seid, pf_q);5332 return ret;5333 }5334 5335 if (!i40e_enabled_xdp_vsi(vsi))5336 goto wait_rx;5337 5338 /* Check and wait for the XDP Tx queue */5339 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,5340 false);5341 if (ret) {5342 dev_info(&pf->pdev->dev,5343 "VSI seid %d XDP Tx ring %d disable timeout\n",5344 vsi->seid, pf_q);5345 return ret;5346 }5347wait_rx:5348 /* Check and wait for the Rx queue */5349 ret = i40e_pf_rxq_wait(pf, pf_q, false);5350 if (ret) {5351 dev_info(&pf->pdev->dev,5352 "VSI seid %d Rx ring %d disable timeout\n",5353 vsi->seid, pf_q);5354 return ret;5355 }5356 }5357 5358 return 0;5359}5360 5361#ifdef CONFIG_I40E_DCB5362/**5363 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled5364 * @pf: the PF5365 *5366 * This function waits for the queues to be in disabled state for all the5367 * VSIs that are managed by this PF.5368 **/5369static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)5370{5371 struct i40e_vsi *vsi;5372 int v, ret = 0;5373 5374 i40e_pf_for_each_vsi(pf, v, vsi) {5375 ret = i40e_vsi_wait_queues_disabled(vsi);5376 if (ret)5377 break;5378 }5379 5380 return ret;5381}5382 5383#endif5384 5385/**5386 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP5387 * @pf: pointer to PF5388 *5389 * Get TC map for ISCSI PF type that will include iSCSI TC5390 * and LAN TC.5391 **/5392static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)5393{5394 struct i40e_dcb_app_priority_table app;5395 struct i40e_hw *hw = &pf->hw;5396 u8 enabled_tc = 1; /* TC0 is always enabled */5397 u8 tc, i;5398 /* Get the iSCSI APP TLV */5399 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;5400 5401 for (i = 0; i < dcbcfg->numapps; i++) {5402 app = dcbcfg->app[i];5403 if (app.selector == I40E_APP_SEL_TCPIP &&5404 app.protocolid == I40E_APP_PROTOID_ISCSI) {5405 tc = dcbcfg->etscfg.prioritytable[app.priority];5406 enabled_tc |= BIT(tc);5407 break;5408 }5409 }5410 5411 return enabled_tc;5412}5413 5414/**5415 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config5416 * @dcbcfg: the corresponding DCBx configuration structure5417 *5418 * Return the number of TCs from given DCBx configuration5419 **/5420static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)5421{5422 int i, tc_unused = 0;5423 u8 num_tc = 0;5424 u8 ret = 0;5425 5426 /* Scan the ETS Config Priority Table to find5427 * traffic class enabled for a given priority5428 * and create a bitmask of enabled TCs5429 */5430 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)5431 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);5432 5433 /* Now scan the bitmask to check for5434 * contiguous TCs starting with TC05435 */5436 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {5437 if (num_tc & BIT(i)) {5438 if (!tc_unused) {5439 ret++;5440 } else {5441 pr_err("Non-contiguous TC - Disabling DCB\n");5442 return 1;5443 }5444 } else {5445 tc_unused = 1;5446 }5447 }5448 5449 /* There is always at least TC0 */5450 if (!ret)5451 ret = 1;5452 5453 return ret;5454}5455 5456/**5457 * i40e_dcb_get_enabled_tc - Get enabled traffic classes5458 * @dcbcfg: the corresponding DCBx configuration structure5459 *5460 * Query the current DCB configuration and return the number of5461 * traffic classes enabled from the given DCBX config5462 **/5463static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)5464{5465 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);5466 u8 enabled_tc = 1;5467 u8 i;5468 5469 for (i = 0; i < num_tc; i++)5470 enabled_tc |= BIT(i);5471 5472 return enabled_tc;5473}5474 5475/**5476 * i40e_mqprio_get_enabled_tc - Get enabled traffic classes5477 * @pf: PF being queried5478 *5479 * Query the current MQPRIO configuration and return the number of5480 * traffic classes enabled.5481 **/5482static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf)5483{5484 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);5485 u8 num_tc = vsi->mqprio_qopt.qopt.num_tc;5486 u8 enabled_tc = 1, i;5487 5488 for (i = 1; i < num_tc; i++)5489 enabled_tc |= BIT(i);5490 return enabled_tc;5491}5492 5493/**5494 * i40e_pf_get_num_tc - Get enabled traffic classes for PF5495 * @pf: PF being queried5496 *5497 * Return number of traffic classes enabled for the given PF5498 **/5499static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)5500{5501 u8 i, enabled_tc = 1;5502 u8 num_tc = 0;5503 5504 if (i40e_is_tc_mqprio_enabled(pf)) {5505 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);5506 5507 return vsi->mqprio_qopt.qopt.num_tc;5508 }5509 5510 /* If neither MQPRIO nor DCB is enabled, then always use single TC */5511 if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags))5512 return 1;5513 5514 /* SFP mode will be enabled for all TCs on port */5515 if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))5516 return i40e_dcb_get_num_tc(&pf->hw.local_dcbx_config);5517 5518 /* MFP mode return count of enabled TCs for this PF */5519 if (pf->hw.func_caps.iscsi)5520 enabled_tc = i40e_get_iscsi_tc_map(pf);5521 else5522 return 1; /* Only TC0 */5523 5524 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {5525 if (enabled_tc & BIT(i))5526 num_tc++;5527 }5528 return num_tc;5529}5530 5531/**5532 * i40e_pf_get_tc_map - Get bitmap for enabled traffic classes5533 * @pf: PF being queried5534 *5535 * Return a bitmap for enabled traffic classes for this PF.5536 **/5537static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)5538{5539 if (i40e_is_tc_mqprio_enabled(pf))5540 return i40e_mqprio_get_enabled_tc(pf);5541 5542 /* If neither MQPRIO nor DCB is enabled for this PF then just return5543 * default TC5544 */5545 if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags))5546 return I40E_DEFAULT_TRAFFIC_CLASS;5547 5548 /* SFP mode we want PF to be enabled for all TCs */5549 if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))5550 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);5551 5552 /* MFP enabled and iSCSI PF type */5553 if (pf->hw.func_caps.iscsi)5554 return i40e_get_iscsi_tc_map(pf);5555 else5556 return I40E_DEFAULT_TRAFFIC_CLASS;5557}5558 5559/**5560 * i40e_vsi_get_bw_info - Query VSI BW Information5561 * @vsi: the VSI being queried5562 *5563 * Returns 0 on success, negative value on failure5564 **/5565static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)5566{5567 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};5568 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};5569 struct i40e_pf *pf = vsi->back;5570 struct i40e_hw *hw = &pf->hw;5571 u32 tc_bw_max;5572 int ret;5573 int i;5574 5575 /* Get the VSI level BW configuration */5576 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);5577 if (ret) {5578 dev_info(&pf->pdev->dev,5579 "couldn't get PF vsi bw config, err %pe aq_err %s\n",5580 ERR_PTR(ret),5581 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));5582 return -EINVAL;5583 }5584 5585 /* Get the VSI level BW configuration per TC */5586 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,5587 NULL);5588 if (ret) {5589 dev_info(&pf->pdev->dev,5590 "couldn't get PF vsi ets bw config, err %pe aq_err %s\n",5591 ERR_PTR(ret),5592 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));5593 return -EINVAL;5594 }5595 5596 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {5597 dev_info(&pf->pdev->dev,5598 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",5599 bw_config.tc_valid_bits,5600 bw_ets_config.tc_valid_bits);5601 /* Still continuing */5602 }5603 5604 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);5605 vsi->bw_max_quanta = bw_config.max_bw;5606 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |5607 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);5608 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {5609 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];5610 vsi->bw_ets_limit_credits[i] =5611 le16_to_cpu(bw_ets_config.credits[i]);5612 /* 3 bits out of 4 for each TC */5613 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);5614 }5615 5616 return 0;5617}5618 5619/**5620 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC5621 * @vsi: the VSI being configured5622 * @enabled_tc: TC bitmap5623 * @bw_share: BW shared credits per TC5624 *5625 * Returns 0 on success, negative value on failure5626 **/5627static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,5628 u8 *bw_share)5629{5630 struct i40e_aqc_configure_vsi_tc_bw_data bw_data;5631 struct i40e_pf *pf = vsi->back;5632 int ret;5633 int i;5634 5635 /* There is no need to reset BW when mqprio mode is on. */5636 if (i40e_is_tc_mqprio_enabled(pf))5637 return 0;5638 if (!vsi->mqprio_qopt.qopt.hw && !test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {5639 ret = i40e_set_bw_limit(vsi, vsi->seid, 0);5640 if (ret)5641 dev_info(&pf->pdev->dev,5642 "Failed to reset tx rate for vsi->seid %u\n",5643 vsi->seid);5644 return ret;5645 }5646 memset(&bw_data, 0, sizeof(bw_data));5647 bw_data.tc_valid_bits = enabled_tc;5648 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)5649 bw_data.tc_bw_credits[i] = bw_share[i];5650 5651 ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL);5652 if (ret) {5653 dev_info(&pf->pdev->dev,5654 "AQ command Config VSI BW allocation per TC failed = %d\n",5655 pf->hw.aq.asq_last_status);5656 return -EINVAL;5657 }5658 5659 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)5660 vsi->info.qs_handle[i] = bw_data.qs_handles[i];5661 5662 return 0;5663}5664 5665/**5666 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration5667 * @vsi: the VSI being configured5668 * @enabled_tc: TC map to be enabled5669 *5670 **/5671static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)5672{5673 struct net_device *netdev = vsi->netdev;5674 struct i40e_pf *pf = vsi->back;5675 struct i40e_hw *hw = &pf->hw;5676 u8 netdev_tc = 0;5677 int i;5678 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;5679 5680 if (!netdev)5681 return;5682 5683 if (!enabled_tc) {5684 netdev_reset_tc(netdev);5685 return;5686 }5687 5688 /* Set up actual enabled TCs on the VSI */5689 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))5690 return;5691 5692 /* set per TC queues for the VSI */5693 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {5694 /* Only set TC queues for enabled tcs5695 *5696 * e.g. For a VSI that has TC0 and TC3 enabled the5697 * enabled_tc bitmap would be 0x00001001; the driver5698 * will set the numtc for netdev as 2 that will be5699 * referenced by the netdev layer as TC 0 and 1.5700 */5701 if (vsi->tc_config.enabled_tc & BIT(i))5702 netdev_set_tc_queue(netdev,5703 vsi->tc_config.tc_info[i].netdev_tc,5704 vsi->tc_config.tc_info[i].qcount,5705 vsi->tc_config.tc_info[i].qoffset);5706 }5707 5708 if (i40e_is_tc_mqprio_enabled(pf))5709 return;5710 5711 /* Assign UP2TC map for the VSI */5712 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {5713 /* Get the actual TC# for the UP */5714 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];5715 /* Get the mapped netdev TC# for the UP */5716 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc;5717 netdev_set_prio_tc_map(netdev, i, netdev_tc);5718 }5719}5720 5721/**5722 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map5723 * @vsi: the VSI being configured5724 * @ctxt: the ctxt buffer returned from AQ VSI update param command5725 **/5726static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,5727 struct i40e_vsi_context *ctxt)5728{5729 /* copy just the sections touched not the entire info5730 * since not all sections are valid as returned by5731 * update vsi params5732 */5733 vsi->info.mapping_flags = ctxt->info.mapping_flags;5734 memcpy(&vsi->info.queue_mapping,5735 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));5736 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,5737 sizeof(vsi->info.tc_mapping));5738}5739 5740/**5741 * i40e_update_adq_vsi_queues - update queue mapping for ADq VSI5742 * @vsi: the VSI being reconfigured5743 * @vsi_offset: offset from main VF VSI5744 */5745int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset)5746{5747 struct i40e_vsi_context ctxt = {};5748 struct i40e_pf *pf;5749 struct i40e_hw *hw;5750 int ret;5751 5752 if (!vsi)5753 return -EINVAL;5754 pf = vsi->back;5755 hw = &pf->hw;5756 5757 ctxt.seid = vsi->seid;5758 ctxt.pf_num = hw->pf_id;5759 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id + vsi_offset;5760 ctxt.uplink_seid = vsi->uplink_seid;5761 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;5762 ctxt.flags = I40E_AQ_VSI_TYPE_VF;5763 ctxt.info = vsi->info;5764 5765 i40e_vsi_setup_queue_map(vsi, &ctxt, vsi->tc_config.enabled_tc,5766 false);5767 if (vsi->reconfig_rss) {5768 vsi->rss_size = min_t(int, pf->alloc_rss_size,5769 vsi->num_queue_pairs);5770 ret = i40e_vsi_config_rss(vsi);5771 if (ret) {5772 dev_info(&pf->pdev->dev, "Failed to reconfig rss for num_queues\n");5773 return ret;5774 }5775 vsi->reconfig_rss = false;5776 }5777 5778 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);5779 if (ret) {5780 dev_info(&pf->pdev->dev, "Update vsi config failed, err %pe aq_err %s\n",5781 ERR_PTR(ret),5782 i40e_aq_str(hw, hw->aq.asq_last_status));5783 return ret;5784 }5785 /* update the local VSI info with updated queue map */5786 i40e_vsi_update_queue_map(vsi, &ctxt);5787 vsi->info.valid_sections = 0;5788 5789 return ret;5790}5791 5792/**5793 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map5794 * @vsi: VSI to be configured5795 * @enabled_tc: TC bitmap5796 *5797 * This configures a particular VSI for TCs that are mapped to the5798 * given TC bitmap. It uses default bandwidth share for TCs across5799 * VSIs to configure TC for a particular VSI.5800 *5801 * NOTE:5802 * It is expected that the VSI queues have been quisced before calling5803 * this function.5804 **/5805static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)5806{5807 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};5808 struct i40e_pf *pf = vsi->back;5809 struct i40e_hw *hw = &pf->hw;5810 struct i40e_vsi_context ctxt;5811 int ret = 0;5812 int i;5813 5814 /* Check if enabled_tc is same as existing or new TCs */5815 if (vsi->tc_config.enabled_tc == enabled_tc &&5816 vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL)5817 return ret;5818 5819 /* Enable ETS TCs with equal BW Share for now across all VSIs */5820 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {5821 if (enabled_tc & BIT(i))5822 bw_share[i] = 1;5823 }5824 5825 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);5826 if (ret) {5827 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};5828 5829 dev_info(&pf->pdev->dev,5830 "Failed configuring TC map %d for VSI %d\n",5831 enabled_tc, vsi->seid);5832 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid,5833 &bw_config, NULL);5834 if (ret) {5835 dev_info(&pf->pdev->dev,5836 "Failed querying vsi bw info, err %pe aq_err %s\n",5837 ERR_PTR(ret),5838 i40e_aq_str(hw, hw->aq.asq_last_status));5839 goto out;5840 }5841 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) {5842 u8 valid_tc = bw_config.tc_valid_bits & enabled_tc;5843 5844 if (!valid_tc)5845 valid_tc = bw_config.tc_valid_bits;5846 /* Always enable TC0, no matter what */5847 valid_tc |= 1;5848 dev_info(&pf->pdev->dev,5849 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n",5850 enabled_tc, bw_config.tc_valid_bits, valid_tc);5851 enabled_tc = valid_tc;5852 }5853 5854 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);5855 if (ret) {5856 dev_err(&pf->pdev->dev,5857 "Unable to configure TC map %d for VSI %d\n",5858 enabled_tc, vsi->seid);5859 goto out;5860 }5861 }5862 5863 /* Update Queue Pairs Mapping for currently enabled UPs */5864 ctxt.seid = vsi->seid;5865 ctxt.pf_num = vsi->back->hw.pf_id;5866 ctxt.vf_num = 0;5867 ctxt.uplink_seid = vsi->uplink_seid;5868 ctxt.info = vsi->info;5869 if (i40e_is_tc_mqprio_enabled(pf)) {5870 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc);5871 if (ret)5872 goto out;5873 } else {5874 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);5875 }5876 5877 /* On destroying the qdisc, reset vsi->rss_size, as number of enabled5878 * queues changed.5879 */5880 if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) {5881 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size,5882 vsi->num_queue_pairs);5883 ret = i40e_vsi_config_rss(vsi);5884 if (ret) {5885 dev_info(&vsi->back->pdev->dev,5886 "Failed to reconfig rss for num_queues\n");5887 return ret;5888 }5889 vsi->reconfig_rss = false;5890 }5891 if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) {5892 ctxt.info.valid_sections |=5893 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);5894 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;5895 }5896 5897 /* Update the VSI after updating the VSI queue-mapping5898 * information5899 */5900 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);5901 if (ret) {5902 dev_info(&pf->pdev->dev,5903 "Update vsi tc config failed, err %pe aq_err %s\n",5904 ERR_PTR(ret),5905 i40e_aq_str(hw, hw->aq.asq_last_status));5906 goto out;5907 }5908 /* update the local VSI info with updated queue map */5909 i40e_vsi_update_queue_map(vsi, &ctxt);5910 vsi->info.valid_sections = 0;5911 5912 /* Update current VSI BW information */5913 ret = i40e_vsi_get_bw_info(vsi);5914 if (ret) {5915 dev_info(&pf->pdev->dev,5916 "Failed updating vsi bw info, err %pe aq_err %s\n",5917 ERR_PTR(ret),5918 i40e_aq_str(hw, hw->aq.asq_last_status));5919 goto out;5920 }5921 5922 /* Update the netdev TC setup */5923 i40e_vsi_config_netdev_tc(vsi, enabled_tc);5924out:5925 return ret;5926}5927 5928/**5929 * i40e_vsi_reconfig_tc - Reconfigure VSI Tx Scheduler for stored TC map5930 * @vsi: VSI to be reconfigured5931 *5932 * This reconfigures a particular VSI for TCs that are mapped to the5933 * TC bitmap stored previously for the VSI.5934 *5935 * Context: It is expected that the VSI queues have been quisced before5936 * calling this function.5937 *5938 * Return: 0 on success, negative value on failure5939 **/5940static int i40e_vsi_reconfig_tc(struct i40e_vsi *vsi)5941{5942 u8 enabled_tc;5943 5944 enabled_tc = vsi->tc_config.enabled_tc;5945 vsi->tc_config.enabled_tc = 0;5946 5947 return i40e_vsi_config_tc(vsi, enabled_tc);5948}5949 5950/**5951 * i40e_get_link_speed - Returns link speed for the interface5952 * @vsi: VSI to be configured5953 *5954 **/5955static int i40e_get_link_speed(struct i40e_vsi *vsi)5956{5957 struct i40e_pf *pf = vsi->back;5958 5959 switch (pf->hw.phy.link_info.link_speed) {5960 case I40E_LINK_SPEED_40GB:5961 return 40000;5962 case I40E_LINK_SPEED_25GB:5963 return 25000;5964 case I40E_LINK_SPEED_20GB:5965 return 20000;5966 case I40E_LINK_SPEED_10GB:5967 return 10000;5968 case I40E_LINK_SPEED_1GB:5969 return 1000;5970 default:5971 return -EINVAL;5972 }5973}5974 5975/**5976 * i40e_bw_bytes_to_mbits - Convert max_tx_rate from bytes to mbits5977 * @vsi: Pointer to vsi structure5978 * @max_tx_rate: max TX rate in bytes to be converted into Mbits5979 *5980 * Helper function to convert units before send to set BW limit5981 **/5982static u64 i40e_bw_bytes_to_mbits(struct i40e_vsi *vsi, u64 max_tx_rate)5983{5984 if (max_tx_rate < I40E_BW_MBPS_DIVISOR) {5985 dev_warn(&vsi->back->pdev->dev,5986 "Setting max tx rate to minimum usable value of 50Mbps.\n");5987 max_tx_rate = I40E_BW_CREDIT_DIVISOR;5988 } else {5989 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);5990 }5991 5992 return max_tx_rate;5993}5994 5995/**5996 * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate5997 * @vsi: VSI to be configured5998 * @seid: seid of the channel/VSI5999 * @max_tx_rate: max TX rate to be configured as BW limit6000 *6001 * Helper function to set BW limit for a given VSI6002 **/6003int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate)6004{6005 struct i40e_pf *pf = vsi->back;6006 u64 credits = 0;6007 int speed = 0;6008 int ret = 0;6009 6010 speed = i40e_get_link_speed(vsi);6011 if (max_tx_rate > speed) {6012 dev_err(&pf->pdev->dev,6013 "Invalid max tx rate %llu specified for VSI seid %d.",6014 max_tx_rate, seid);6015 return -EINVAL;6016 }6017 if (max_tx_rate && max_tx_rate < I40E_BW_CREDIT_DIVISOR) {6018 dev_warn(&pf->pdev->dev,6019 "Setting max tx rate to minimum usable value of 50Mbps.\n");6020 max_tx_rate = I40E_BW_CREDIT_DIVISOR;6021 }6022 6023 /* Tx rate credits are in values of 50Mbps, 0 is disabled */6024 credits = max_tx_rate;6025 do_div(credits, I40E_BW_CREDIT_DIVISOR);6026 ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits,6027 I40E_MAX_BW_INACTIVE_ACCUM, NULL);6028 if (ret)6029 dev_err(&pf->pdev->dev,6030 "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %pe aq_err %s\n",6031 max_tx_rate, seid, ERR_PTR(ret),6032 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));6033 return ret;6034}6035 6036/**6037 * i40e_remove_queue_channels - Remove queue channels for the TCs6038 * @vsi: VSI to be configured6039 *6040 * Remove queue channels for the TCs6041 **/6042static void i40e_remove_queue_channels(struct i40e_vsi *vsi)6043{6044 enum i40e_admin_queue_err last_aq_status;6045 struct i40e_cloud_filter *cfilter;6046 struct i40e_channel *ch, *ch_tmp;6047 struct i40e_pf *pf = vsi->back;6048 struct hlist_node *node;6049 int ret, i;6050 6051 /* Reset rss size that was stored when reconfiguring rss for6052 * channel VSIs with non-power-of-2 queue count.6053 */6054 vsi->current_rss_size = 0;6055 6056 /* perform cleanup for channels if they exist */6057 if (list_empty(&vsi->ch_list))6058 return;6059 6060 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {6061 struct i40e_vsi *p_vsi;6062 6063 list_del(&ch->list);6064 p_vsi = ch->parent_vsi;6065 if (!p_vsi || !ch->initialized) {6066 kfree(ch);6067 continue;6068 }6069 /* Reset queue contexts */6070 for (i = 0; i < ch->num_queue_pairs; i++) {6071 struct i40e_ring *tx_ring, *rx_ring;6072 u16 pf_q;6073 6074 pf_q = ch->base_queue + i;6075 tx_ring = vsi->tx_rings[pf_q];6076 tx_ring->ch = NULL;6077 6078 rx_ring = vsi->rx_rings[pf_q];6079 rx_ring->ch = NULL;6080 }6081 6082 /* Reset BW configured for this VSI via mqprio */6083 ret = i40e_set_bw_limit(vsi, ch->seid, 0);6084 if (ret)6085 dev_info(&vsi->back->pdev->dev,6086 "Failed to reset tx rate for ch->seid %u\n",6087 ch->seid);6088 6089 /* delete cloud filters associated with this channel */6090 hlist_for_each_entry_safe(cfilter, node,6091 &pf->cloud_filter_list, cloud_node) {6092 if (cfilter->seid != ch->seid)6093 continue;6094 6095 hash_del(&cfilter->cloud_node);6096 if (cfilter->dst_port)6097 ret = i40e_add_del_cloud_filter_big_buf(vsi,6098 cfilter,6099 false);6100 else6101 ret = i40e_add_del_cloud_filter(vsi, cfilter,6102 false);6103 last_aq_status = pf->hw.aq.asq_last_status;6104 if (ret)6105 dev_info(&pf->pdev->dev,6106 "Failed to delete cloud filter, err %pe aq_err %s\n",6107 ERR_PTR(ret),6108 i40e_aq_str(&pf->hw, last_aq_status));6109 kfree(cfilter);6110 }6111 6112 /* delete VSI from FW */6113 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,6114 NULL);6115 if (ret)6116 dev_err(&vsi->back->pdev->dev,6117 "unable to remove channel (%d) for parent VSI(%d)\n",6118 ch->seid, p_vsi->seid);6119 kfree(ch);6120 }6121 INIT_LIST_HEAD(&vsi->ch_list);6122}6123 6124/**6125 * i40e_get_max_queues_for_channel6126 * @vsi: ptr to VSI to which channels are associated with6127 *6128 * Helper function which returns max value among the queue counts set on the6129 * channels/TCs created.6130 **/6131static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi)6132{6133 struct i40e_channel *ch, *ch_tmp;6134 int max = 0;6135 6136 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {6137 if (!ch->initialized)6138 continue;6139 if (ch->num_queue_pairs > max)6140 max = ch->num_queue_pairs;6141 }6142 6143 return max;6144}6145 6146/**6147 * i40e_validate_num_queues - validate num_queues w.r.t channel6148 * @pf: ptr to PF device6149 * @num_queues: number of queues6150 * @vsi: the parent VSI6151 * @reconfig_rss: indicates should the RSS be reconfigured or not6152 *6153 * This function validates number of queues in the context of new channel6154 * which is being established and determines if RSS should be reconfigured6155 * or not for parent VSI.6156 **/6157static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues,6158 struct i40e_vsi *vsi, bool *reconfig_rss)6159{6160 int max_ch_queues;6161 6162 if (!reconfig_rss)6163 return -EINVAL;6164 6165 *reconfig_rss = false;6166 if (vsi->current_rss_size) {6167 if (num_queues > vsi->current_rss_size) {6168 dev_dbg(&pf->pdev->dev,6169 "Error: num_queues (%d) > vsi's current_size(%d)\n",6170 num_queues, vsi->current_rss_size);6171 return -EINVAL;6172 } else if ((num_queues < vsi->current_rss_size) &&6173 (!is_power_of_2(num_queues))) {6174 dev_dbg(&pf->pdev->dev,6175 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n",6176 num_queues, vsi->current_rss_size);6177 return -EINVAL;6178 }6179 }6180 6181 if (!is_power_of_2(num_queues)) {6182 /* Find the max num_queues configured for channel if channel6183 * exist.6184 * if channel exist, then enforce 'num_queues' to be more than6185 * max ever queues configured for channel.6186 */6187 max_ch_queues = i40e_get_max_queues_for_channel(vsi);6188 if (num_queues < max_ch_queues) {6189 dev_dbg(&pf->pdev->dev,6190 "Error: num_queues (%d) < max queues configured for channel(%d)\n",6191 num_queues, max_ch_queues);6192 return -EINVAL;6193 }6194 *reconfig_rss = true;6195 }6196 6197 return 0;6198}6199 6200/**6201 * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size6202 * @vsi: the VSI being setup6203 * @rss_size: size of RSS, accordingly LUT gets reprogrammed6204 *6205 * This function reconfigures RSS by reprogramming LUTs using 'rss_size'6206 **/6207static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size)6208{6209 struct i40e_pf *pf = vsi->back;6210 u8 seed[I40E_HKEY_ARRAY_SIZE];6211 struct i40e_hw *hw = &pf->hw;6212 int local_rss_size;6213 u8 *lut;6214 int ret;6215 6216 if (!vsi->rss_size)6217 return -EINVAL;6218 6219 if (rss_size > vsi->rss_size)6220 return -EINVAL;6221 6222 local_rss_size = min_t(int, vsi->rss_size, rss_size);6223 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);6224 if (!lut)6225 return -ENOMEM;6226 6227 /* Ignoring user configured lut if there is one */6228 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size);6229 6230 /* Use user configured hash key if there is one, otherwise6231 * use default.6232 */6233 if (vsi->rss_hkey_user)6234 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);6235 else6236 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);6237 6238 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);6239 if (ret) {6240 dev_info(&pf->pdev->dev,6241 "Cannot set RSS lut, err %pe aq_err %s\n",6242 ERR_PTR(ret),6243 i40e_aq_str(hw, hw->aq.asq_last_status));6244 kfree(lut);6245 return ret;6246 }6247 kfree(lut);6248 6249 /* Do the update w.r.t. storing rss_size */6250 if (!vsi->orig_rss_size)6251 vsi->orig_rss_size = vsi->rss_size;6252 vsi->current_rss_size = local_rss_size;6253 6254 return ret;6255}6256 6257/**6258 * i40e_channel_setup_queue_map - Setup a channel queue map6259 * @pf: ptr to PF device6260 * @ctxt: VSI context structure6261 * @ch: ptr to channel structure6262 *6263 * Setup queue map for a specific channel6264 **/6265static void i40e_channel_setup_queue_map(struct i40e_pf *pf,6266 struct i40e_vsi_context *ctxt,6267 struct i40e_channel *ch)6268{6269 u16 qcount, qmap, sections = 0;6270 u8 offset = 0;6271 int pow;6272 6273 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;6274 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;6275 6276 qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix);6277 ch->num_queue_pairs = qcount;6278 6279 /* find the next higher power-of-2 of num queue pairs */6280 pow = ilog2(qcount);6281 if (!is_power_of_2(qcount))6282 pow++;6283 6284 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |6285 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);6286 6287 /* Setup queue TC[0].qmap for given VSI context */6288 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);6289 6290 ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */6291 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);6292 ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue);6293 ctxt->info.valid_sections |= cpu_to_le16(sections);6294}6295 6296/**6297 * i40e_add_channel - add a channel by adding VSI6298 * @pf: ptr to PF device6299 * @uplink_seid: underlying HW switching element (VEB) ID6300 * @ch: ptr to channel structure6301 *6302 * Add a channel (VSI) using add_vsi and queue_map6303 **/6304static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid,6305 struct i40e_channel *ch)6306{6307 struct i40e_hw *hw = &pf->hw;6308 struct i40e_vsi_context ctxt;6309 u8 enabled_tc = 0x1; /* TC0 enabled */6310 int ret;6311 6312 if (ch->type != I40E_VSI_VMDQ2) {6313 dev_info(&pf->pdev->dev,6314 "add new vsi failed, ch->type %d\n", ch->type);6315 return -EINVAL;6316 }6317 6318 memset(&ctxt, 0, sizeof(ctxt));6319 ctxt.pf_num = hw->pf_id;6320 ctxt.vf_num = 0;6321 ctxt.uplink_seid = uplink_seid;6322 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;6323 if (ch->type == I40E_VSI_VMDQ2)6324 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;6325 6326 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {6327 ctxt.info.valid_sections |=6328 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);6329 ctxt.info.switch_id =6330 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);6331 }6332 6333 /* Set queue map for a given VSI context */6334 i40e_channel_setup_queue_map(pf, &ctxt, ch);6335 6336 /* Now time to create VSI */6337 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);6338 if (ret) {6339 dev_info(&pf->pdev->dev,6340 "add new vsi failed, err %pe aq_err %s\n",6341 ERR_PTR(ret),6342 i40e_aq_str(&pf->hw,6343 pf->hw.aq.asq_last_status));6344 return -ENOENT;6345 }6346 6347 /* Success, update channel, set enabled_tc only if the channel6348 * is not a macvlan6349 */6350 ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc;6351 ch->seid = ctxt.seid;6352 ch->vsi_number = ctxt.vsi_number;6353 ch->stat_counter_idx = le16_to_cpu(ctxt.info.stat_counter_idx);6354 6355 /* copy just the sections touched not the entire info6356 * since not all sections are valid as returned by6357 * update vsi params6358 */6359 ch->info.mapping_flags = ctxt.info.mapping_flags;6360 memcpy(&ch->info.queue_mapping,6361 &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping));6362 memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping,6363 sizeof(ctxt.info.tc_mapping));6364 6365 return 0;6366}6367 6368static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch,6369 u8 *bw_share)6370{6371 struct i40e_aqc_configure_vsi_tc_bw_data bw_data;6372 int ret;6373 int i;6374 6375 memset(&bw_data, 0, sizeof(bw_data));6376 bw_data.tc_valid_bits = ch->enabled_tc;6377 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)6378 bw_data.tc_bw_credits[i] = bw_share[i];6379 6380 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid,6381 &bw_data, NULL);6382 if (ret) {6383 dev_info(&vsi->back->pdev->dev,6384 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n",6385 vsi->back->hw.aq.asq_last_status, ch->seid);6386 return -EINVAL;6387 }6388 6389 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)6390 ch->info.qs_handle[i] = bw_data.qs_handles[i];6391 6392 return 0;6393}6394 6395/**6396 * i40e_channel_config_tx_ring - config TX ring associated with new channel6397 * @pf: ptr to PF device6398 * @vsi: the VSI being setup6399 * @ch: ptr to channel structure6400 *6401 * Configure TX rings associated with channel (VSI) since queues are being6402 * from parent VSI.6403 **/6404static int i40e_channel_config_tx_ring(struct i40e_pf *pf,6405 struct i40e_vsi *vsi,6406 struct i40e_channel *ch)6407{6408 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};6409 int ret;6410 int i;6411 6412 /* Enable ETS TCs with equal BW Share for now across all VSIs */6413 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {6414 if (ch->enabled_tc & BIT(i))6415 bw_share[i] = 1;6416 }6417 6418 /* configure BW for new VSI */6419 ret = i40e_channel_config_bw(vsi, ch, bw_share);6420 if (ret) {6421 dev_info(&vsi->back->pdev->dev,6422 "Failed configuring TC map %d for channel (seid %u)\n",6423 ch->enabled_tc, ch->seid);6424 return ret;6425 }6426 6427 for (i = 0; i < ch->num_queue_pairs; i++) {6428 struct i40e_ring *tx_ring, *rx_ring;6429 u16 pf_q;6430 6431 pf_q = ch->base_queue + i;6432 6433 /* Get to TX ring ptr of main VSI, for re-setup TX queue6434 * context6435 */6436 tx_ring = vsi->tx_rings[pf_q];6437 tx_ring->ch = ch;6438 6439 /* Get the RX ring ptr */6440 rx_ring = vsi->rx_rings[pf_q];6441 rx_ring->ch = ch;6442 }6443 6444 return 0;6445}6446 6447/**6448 * i40e_setup_hw_channel - setup new channel6449 * @pf: ptr to PF device6450 * @vsi: the VSI being setup6451 * @ch: ptr to channel structure6452 * @uplink_seid: underlying HW switching element (VEB) ID6453 * @type: type of channel to be created (VMDq2/VF)6454 *6455 * Setup new channel (VSI) based on specified type (VMDq2/VF)6456 * and configures TX rings accordingly6457 **/6458static inline int i40e_setup_hw_channel(struct i40e_pf *pf,6459 struct i40e_vsi *vsi,6460 struct i40e_channel *ch,6461 u16 uplink_seid, u8 type)6462{6463 int ret;6464 6465 ch->initialized = false;6466 ch->base_queue = vsi->next_base_queue;6467 ch->type = type;6468 6469 /* Proceed with creation of channel (VMDq2) VSI */6470 ret = i40e_add_channel(pf, uplink_seid, ch);6471 if (ret) {6472 dev_info(&pf->pdev->dev,6473 "failed to add_channel using uplink_seid %u\n",6474 uplink_seid);6475 return ret;6476 }6477 6478 /* Mark the successful creation of channel */6479 ch->initialized = true;6480 6481 /* Reconfigure TX queues using QTX_CTL register */6482 ret = i40e_channel_config_tx_ring(pf, vsi, ch);6483 if (ret) {6484 dev_info(&pf->pdev->dev,6485 "failed to configure TX rings for channel %u\n",6486 ch->seid);6487 return ret;6488 }6489 6490 /* update 'next_base_queue' */6491 vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs;6492 dev_dbg(&pf->pdev->dev,6493 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n",6494 ch->seid, ch->vsi_number, ch->stat_counter_idx,6495 ch->num_queue_pairs,6496 vsi->next_base_queue);6497 return ret;6498}6499 6500/**6501 * i40e_setup_channel - setup new channel using uplink element6502 * @pf: ptr to PF device6503 * @vsi: pointer to the VSI to set up the channel within6504 * @ch: ptr to channel structure6505 *6506 * Setup new channel (VSI) based on specified type (VMDq2/VF)6507 * and uplink switching element (uplink_seid)6508 **/6509static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi,6510 struct i40e_channel *ch)6511{6512 struct i40e_vsi *main_vsi;6513 u8 vsi_type;6514 u16 seid;6515 int ret;6516 6517 if (vsi->type == I40E_VSI_MAIN) {6518 vsi_type = I40E_VSI_VMDQ2;6519 } else {6520 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n",6521 vsi->type);6522 return false;6523 }6524 6525 /* underlying switching element */6526 main_vsi = i40e_pf_get_main_vsi(pf);6527 seid = main_vsi->uplink_seid;6528 6529 /* create channel (VSI), configure TX rings */6530 ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type);6531 if (ret) {6532 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n");6533 return false;6534 }6535 6536 return ch->initialized ? true : false;6537}6538 6539/**6540 * i40e_validate_and_set_switch_mode - sets up switch mode correctly6541 * @vsi: ptr to VSI which has PF backing6542 *6543 * Sets up switch mode correctly if it needs to be changed and perform6544 * what are allowed modes.6545 **/6546static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi)6547{6548 u8 mode;6549 struct i40e_pf *pf = vsi->back;6550 struct i40e_hw *hw = &pf->hw;6551 int ret;6552 6553 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities);6554 if (ret)6555 return -EINVAL;6556 6557 if (hw->dev_caps.switch_mode) {6558 /* if switch mode is set, support mode2 (non-tunneled for6559 * cloud filter) for now6560 */6561 u32 switch_mode = hw->dev_caps.switch_mode &6562 I40E_SWITCH_MODE_MASK;6563 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) {6564 if (switch_mode == I40E_CLOUD_FILTER_MODE2)6565 return 0;6566 dev_err(&pf->pdev->dev,6567 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n",6568 hw->dev_caps.switch_mode);6569 return -EINVAL;6570 }6571 }6572 6573 /* Set Bit 7 to be valid */6574 mode = I40E_AQ_SET_SWITCH_BIT7_VALID;6575 6576 /* Set L4type for TCP support */6577 mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP;6578 6579 /* Set cloud filter mode */6580 mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL;6581 6582 /* Prep mode field for set_switch_config */6583 ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags,6584 pf->last_sw_conf_valid_flags,6585 mode, NULL);6586 if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH)6587 dev_err(&pf->pdev->dev,6588 "couldn't set switch config bits, err %pe aq_err %s\n",6589 ERR_PTR(ret),6590 i40e_aq_str(hw,6591 hw->aq.asq_last_status));6592 6593 return ret;6594}6595 6596/**6597 * i40e_create_queue_channel - function to create channel6598 * @vsi: VSI to be configured6599 * @ch: ptr to channel (it contains channel specific params)6600 *6601 * This function creates channel (VSI) using num_queues specified by user,6602 * reconfigs RSS if needed.6603 **/6604int i40e_create_queue_channel(struct i40e_vsi *vsi,6605 struct i40e_channel *ch)6606{6607 struct i40e_pf *pf = vsi->back;6608 bool reconfig_rss;6609 int err;6610 6611 if (!ch)6612 return -EINVAL;6613 6614 if (!ch->num_queue_pairs) {6615 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n",6616 ch->num_queue_pairs);6617 return -EINVAL;6618 }6619 6620 /* validate user requested num_queues for channel */6621 err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi,6622 &reconfig_rss);6623 if (err) {6624 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n",6625 ch->num_queue_pairs);6626 return -EINVAL;6627 }6628 6629 /* By default we are in VEPA mode, if this is the first VF/VMDq6630 * VSI to be added switch to VEB mode.6631 */6632 6633 if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {6634 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);6635 6636 if (vsi->type == I40E_VSI_MAIN) {6637 if (i40e_is_tc_mqprio_enabled(pf))6638 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);6639 else6640 i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG);6641 }6642 /* now onwards for main VSI, number of queues will be value6643 * of TC0's queue count6644 */6645 }6646 6647 /* By this time, vsi->cnt_q_avail shall be set to non-zero and6648 * it should be more than num_queues6649 */6650 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) {6651 dev_dbg(&pf->pdev->dev,6652 "Error: cnt_q_avail (%u) less than num_queues %d\n",6653 vsi->cnt_q_avail, ch->num_queue_pairs);6654 return -EINVAL;6655 }6656 6657 /* reconfig_rss only if vsi type is MAIN_VSI */6658 if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) {6659 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs);6660 if (err) {6661 dev_info(&pf->pdev->dev,6662 "Error: unable to reconfig rss for num_queues (%u)\n",6663 ch->num_queue_pairs);6664 return -EINVAL;6665 }6666 }6667 6668 if (!i40e_setup_channel(pf, vsi, ch)) {6669 dev_info(&pf->pdev->dev, "Failed to setup channel\n");6670 return -EINVAL;6671 }6672 6673 dev_info(&pf->pdev->dev,6674 "Setup channel (id:%u) utilizing num_queues %d\n",6675 ch->seid, ch->num_queue_pairs);6676 6677 /* configure VSI for BW limit */6678 if (ch->max_tx_rate) {6679 u64 credits = ch->max_tx_rate;6680 6681 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate))6682 return -EINVAL;6683 6684 do_div(credits, I40E_BW_CREDIT_DIVISOR);6685 dev_dbg(&pf->pdev->dev,6686 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",6687 ch->max_tx_rate,6688 credits,6689 ch->seid);6690 }6691 6692 /* in case of VF, this will be main SRIOV VSI */6693 ch->parent_vsi = vsi;6694 6695 /* and update main_vsi's count for queue_available to use */6696 vsi->cnt_q_avail -= ch->num_queue_pairs;6697 6698 return 0;6699}6700 6701/**6702 * i40e_configure_queue_channels - Add queue channel for the given TCs6703 * @vsi: VSI to be configured6704 *6705 * Configures queue channel mapping to the given TCs6706 **/6707static int i40e_configure_queue_channels(struct i40e_vsi *vsi)6708{6709 struct i40e_channel *ch;6710 u64 max_rate = 0;6711 int ret = 0, i;6712 6713 /* Create app vsi with the TCs. Main VSI with TC0 is already set up */6714 vsi->tc_seid_map[0] = vsi->seid;6715 for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) {6716 if (vsi->tc_config.enabled_tc & BIT(i)) {6717 ch = kzalloc(sizeof(*ch), GFP_KERNEL);6718 if (!ch) {6719 ret = -ENOMEM;6720 goto err_free;6721 }6722 6723 INIT_LIST_HEAD(&ch->list);6724 ch->num_queue_pairs =6725 vsi->tc_config.tc_info[i].qcount;6726 ch->base_queue =6727 vsi->tc_config.tc_info[i].qoffset;6728 6729 /* Bandwidth limit through tc interface is in bytes/s,6730 * change to Mbit/s6731 */6732 max_rate = vsi->mqprio_qopt.max_rate[i];6733 do_div(max_rate, I40E_BW_MBPS_DIVISOR);6734 ch->max_tx_rate = max_rate;6735 6736 list_add_tail(&ch->list, &vsi->ch_list);6737 6738 ret = i40e_create_queue_channel(vsi, ch);6739 if (ret) {6740 dev_err(&vsi->back->pdev->dev,6741 "Failed creating queue channel with TC%d: queues %d\n",6742 i, ch->num_queue_pairs);6743 goto err_free;6744 }6745 vsi->tc_seid_map[i] = ch->seid;6746 }6747 }6748 6749 /* reset to reconfigure TX queue contexts */6750 i40e_do_reset(vsi->back, I40E_PF_RESET_FLAG, true);6751 return ret;6752 6753err_free:6754 i40e_remove_queue_channels(vsi);6755 return ret;6756}6757 6758/**6759 * i40e_veb_config_tc - Configure TCs for given VEB6760 * @veb: given VEB6761 * @enabled_tc: TC bitmap6762 *6763 * Configures given TC bitmap for VEB (switching) element6764 **/6765int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)6766{6767 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};6768 struct i40e_pf *pf = veb->pf;6769 int ret = 0;6770 int i;6771 6772 /* No TCs or already enabled TCs just return */6773 if (!enabled_tc || veb->enabled_tc == enabled_tc)6774 return ret;6775 6776 bw_data.tc_valid_bits = enabled_tc;6777 /* bw_data.absolute_credits is not set (relative) */6778 6779 /* Enable ETS TCs with equal BW Share for now */6780 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {6781 if (enabled_tc & BIT(i))6782 bw_data.tc_bw_share_credits[i] = 1;6783 }6784 6785 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,6786 &bw_data, NULL);6787 if (ret) {6788 dev_info(&pf->pdev->dev,6789 "VEB bw config failed, err %pe aq_err %s\n",6790 ERR_PTR(ret),6791 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));6792 goto out;6793 }6794 6795 /* Update the BW information */6796 ret = i40e_veb_get_bw_info(veb);6797 if (ret) {6798 dev_info(&pf->pdev->dev,6799 "Failed getting veb bw config, err %pe aq_err %s\n",6800 ERR_PTR(ret),6801 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));6802 }6803 6804out:6805 return ret;6806}6807 6808#ifdef CONFIG_I40E_DCB6809/**6810 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs6811 * @pf: PF struct6812 *6813 * Reconfigure VEB/VSIs on a given PF; it is assumed that6814 * the caller would've quiesce all the VSIs before calling6815 * this function6816 **/6817static void i40e_dcb_reconfigure(struct i40e_pf *pf)6818{6819 struct i40e_vsi *vsi;6820 struct i40e_veb *veb;6821 u8 tc_map = 0;6822 int ret;6823 int v;6824 6825 /* Enable the TCs available on PF to all VEBs */6826 tc_map = i40e_pf_get_tc_map(pf);6827 if (tc_map == I40E_DEFAULT_TRAFFIC_CLASS)6828 return;6829 6830 i40e_pf_for_each_veb(pf, v, veb) {6831 ret = i40e_veb_config_tc(veb, tc_map);6832 if (ret) {6833 dev_info(&pf->pdev->dev,6834 "Failed configuring TC for VEB seid=%d\n",6835 veb->seid);6836 /* Will try to configure as many components */6837 }6838 }6839 6840 /* Update each VSI */6841 i40e_pf_for_each_vsi(pf, v, vsi) {6842 /* - Enable all TCs for the LAN VSI6843 * - For all others keep them at TC0 for now6844 */6845 if (vsi->type == I40E_VSI_MAIN)6846 tc_map = i40e_pf_get_tc_map(pf);6847 else6848 tc_map = I40E_DEFAULT_TRAFFIC_CLASS;6849 6850 ret = i40e_vsi_config_tc(vsi, tc_map);6851 if (ret) {6852 dev_info(&pf->pdev->dev,6853 "Failed configuring TC for VSI seid=%d\n",6854 vsi->seid);6855 /* Will try to configure as many components */6856 } else {6857 /* Re-configure VSI vectors based on updated TC map */6858 i40e_vsi_map_rings_to_vectors(vsi);6859 if (vsi->netdev)6860 i40e_dcbnl_set_all(vsi);6861 }6862 }6863}6864 6865/**6866 * i40e_resume_port_tx - Resume port Tx6867 * @pf: PF struct6868 *6869 * Resume a port's Tx and issue a PF reset in case of failure to6870 * resume.6871 **/6872static int i40e_resume_port_tx(struct i40e_pf *pf)6873{6874 struct i40e_hw *hw = &pf->hw;6875 int ret;6876 6877 ret = i40e_aq_resume_port_tx(hw, NULL);6878 if (ret) {6879 dev_info(&pf->pdev->dev,6880 "Resume Port Tx failed, err %pe aq_err %s\n",6881 ERR_PTR(ret),6882 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));6883 /* Schedule PF reset to recover */6884 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);6885 i40e_service_event_schedule(pf);6886 }6887 6888 return ret;6889}6890 6891/**6892 * i40e_suspend_port_tx - Suspend port Tx6893 * @pf: PF struct6894 *6895 * Suspend a port's Tx and issue a PF reset in case of failure.6896 **/6897static int i40e_suspend_port_tx(struct i40e_pf *pf)6898{6899 struct i40e_hw *hw = &pf->hw;6900 int ret;6901 6902 ret = i40e_aq_suspend_port_tx(hw, pf->mac_seid, NULL);6903 if (ret) {6904 dev_info(&pf->pdev->dev,6905 "Suspend Port Tx failed, err %pe aq_err %s\n",6906 ERR_PTR(ret),6907 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));6908 /* Schedule PF reset to recover */6909 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);6910 i40e_service_event_schedule(pf);6911 }6912 6913 return ret;6914}6915 6916/**6917 * i40e_hw_set_dcb_config - Program new DCBX settings into HW6918 * @pf: PF being configured6919 * @new_cfg: New DCBX configuration6920 *6921 * Program DCB settings into HW and reconfigure VEB/VSIs on6922 * given PF. Uses "Set LLDP MIB" AQC to program the hardware.6923 **/6924static int i40e_hw_set_dcb_config(struct i40e_pf *pf,6925 struct i40e_dcbx_config *new_cfg)6926{6927 struct i40e_dcbx_config *old_cfg = &pf->hw.local_dcbx_config;6928 int ret;6929 6930 /* Check if need reconfiguration */6931 if (!memcmp(&new_cfg, &old_cfg, sizeof(new_cfg))) {6932 dev_dbg(&pf->pdev->dev, "No Change in DCB Config required.\n");6933 return 0;6934 }6935 6936 /* Config change disable all VSIs */6937 i40e_pf_quiesce_all_vsi(pf);6938 6939 /* Copy the new config to the current config */6940 *old_cfg = *new_cfg;6941 old_cfg->etsrec = old_cfg->etscfg;6942 ret = i40e_set_dcb_config(&pf->hw);6943 if (ret) {6944 dev_info(&pf->pdev->dev,6945 "Set DCB Config failed, err %pe aq_err %s\n",6946 ERR_PTR(ret),6947 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));6948 goto out;6949 }6950 6951 /* Changes in configuration update VEB/VSI */6952 i40e_dcb_reconfigure(pf);6953out:6954 /* In case of reset do not try to resume anything */6955 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) {6956 /* Re-start the VSIs if disabled */6957 ret = i40e_resume_port_tx(pf);6958 /* In case of error no point in resuming VSIs */6959 if (ret)6960 goto err;6961 i40e_pf_unquiesce_all_vsi(pf);6962 }6963err:6964 return ret;6965}6966 6967/**6968 * i40e_hw_dcb_config - Program new DCBX settings into HW6969 * @pf: PF being configured6970 * @new_cfg: New DCBX configuration6971 *6972 * Program DCB settings into HW and reconfigure VEB/VSIs on6973 * given PF6974 **/6975int i40e_hw_dcb_config(struct i40e_pf *pf, struct i40e_dcbx_config *new_cfg)6976{6977 struct i40e_aqc_configure_switching_comp_ets_data ets_data;6978 u8 prio_type[I40E_MAX_TRAFFIC_CLASS] = {0};6979 u32 mfs_tc[I40E_MAX_TRAFFIC_CLASS];6980 struct i40e_dcbx_config *old_cfg;6981 u8 mode[I40E_MAX_TRAFFIC_CLASS];6982 struct i40e_rx_pb_config pb_cfg;6983 struct i40e_hw *hw = &pf->hw;6984 u8 num_ports = hw->num_ports;6985 bool need_reconfig;6986 int ret = -EINVAL;6987 u8 lltc_map = 0;6988 u8 tc_map = 0;6989 u8 new_numtc;6990 u8 i;6991 6992 dev_dbg(&pf->pdev->dev, "Configuring DCB registers directly\n");6993 /* Un-pack information to Program ETS HW via shared API6994 * numtc, tcmap6995 * LLTC map6996 * ETS/NON-ETS arbiter mode6997 * max exponent (credit refills)6998 * Total number of ports6999 * PFC priority bit-map7000 * Priority Table7001 * BW % per TC7002 * Arbiter mode between UPs sharing same TC7003 * TSA table (ETS or non-ETS)7004 * EEE enabled or not7005 * MFS TC table7006 */7007 7008 new_numtc = i40e_dcb_get_num_tc(new_cfg);7009 7010 memset(&ets_data, 0, sizeof(ets_data));7011 for (i = 0; i < new_numtc; i++) {7012 tc_map |= BIT(i);7013 switch (new_cfg->etscfg.tsatable[i]) {7014 case I40E_IEEE_TSA_ETS:7015 prio_type[i] = I40E_DCB_PRIO_TYPE_ETS;7016 ets_data.tc_bw_share_credits[i] =7017 new_cfg->etscfg.tcbwtable[i];7018 break;7019 case I40E_IEEE_TSA_STRICT:7020 prio_type[i] = I40E_DCB_PRIO_TYPE_STRICT;7021 lltc_map |= BIT(i);7022 ets_data.tc_bw_share_credits[i] =7023 I40E_DCB_STRICT_PRIO_CREDITS;7024 break;7025 default:7026 /* Invalid TSA type */7027 need_reconfig = false;7028 goto out;7029 }7030 }7031 7032 old_cfg = &hw->local_dcbx_config;7033 /* Check if need reconfiguration */7034 need_reconfig = i40e_dcb_need_reconfig(pf, old_cfg, new_cfg);7035 7036 /* If needed, enable/disable frame tagging, disable all VSIs7037 * and suspend port tx7038 */7039 if (need_reconfig) {7040 /* Enable DCB tagging only when more than one TC */7041 if (new_numtc > 1)7042 set_bit(I40E_FLAG_DCB_ENA, pf->flags);7043 else7044 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);7045 7046 set_bit(__I40E_PORT_SUSPENDED, pf->state);7047 /* Reconfiguration needed quiesce all VSIs */7048 i40e_pf_quiesce_all_vsi(pf);7049 ret = i40e_suspend_port_tx(pf);7050 if (ret)7051 goto err;7052 }7053 7054 /* Configure Port ETS Tx Scheduler */7055 ets_data.tc_valid_bits = tc_map;7056 ets_data.tc_strict_priority_flags = lltc_map;7057 ret = i40e_aq_config_switch_comp_ets7058 (hw, pf->mac_seid, &ets_data,7059 i40e_aqc_opc_modify_switching_comp_ets, NULL);7060 if (ret) {7061 dev_info(&pf->pdev->dev,7062 "Modify Port ETS failed, err %pe aq_err %s\n",7063 ERR_PTR(ret),7064 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));7065 goto out;7066 }7067 7068 /* Configure Rx ETS HW */7069 memset(&mode, I40E_DCB_ARB_MODE_ROUND_ROBIN, sizeof(mode));7070 i40e_dcb_hw_set_num_tc(hw, new_numtc);7071 i40e_dcb_hw_rx_fifo_config(hw, I40E_DCB_ARB_MODE_ROUND_ROBIN,7072 I40E_DCB_ARB_MODE_STRICT_PRIORITY,7073 I40E_DCB_DEFAULT_MAX_EXPONENT,7074 lltc_map);7075 i40e_dcb_hw_rx_cmd_monitor_config(hw, new_numtc, num_ports);7076 i40e_dcb_hw_rx_ets_bw_config(hw, new_cfg->etscfg.tcbwtable, mode,7077 prio_type);7078 i40e_dcb_hw_pfc_config(hw, new_cfg->pfc.pfcenable,7079 new_cfg->etscfg.prioritytable);7080 i40e_dcb_hw_rx_up2tc_config(hw, new_cfg->etscfg.prioritytable);7081 7082 /* Configure Rx Packet Buffers in HW */7083 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {7084 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf);7085 7086 mfs_tc[i] = main_vsi->netdev->mtu;7087 mfs_tc[i] += I40E_PACKET_HDR_PAD;7088 }7089 7090 i40e_dcb_hw_calculate_pool_sizes(hw, num_ports,7091 false, new_cfg->pfc.pfcenable,7092 mfs_tc, &pb_cfg);7093 i40e_dcb_hw_rx_pb_config(hw, &pf->pb_cfg, &pb_cfg);7094 7095 /* Update the local Rx Packet buffer config */7096 pf->pb_cfg = pb_cfg;7097 7098 /* Inform the FW about changes to DCB configuration */7099 ret = i40e_aq_dcb_updated(&pf->hw, NULL);7100 if (ret) {7101 dev_info(&pf->pdev->dev,7102 "DCB Updated failed, err %pe aq_err %s\n",7103 ERR_PTR(ret),7104 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));7105 goto out;7106 }7107 7108 /* Update the port DCBx configuration */7109 *old_cfg = *new_cfg;7110 7111 /* Changes in configuration update VEB/VSI */7112 i40e_dcb_reconfigure(pf);7113out:7114 /* Re-start the VSIs if disabled */7115 if (need_reconfig) {7116 ret = i40e_resume_port_tx(pf);7117 7118 clear_bit(__I40E_PORT_SUSPENDED, pf->state);7119 /* In case of error no point in resuming VSIs */7120 if (ret)7121 goto err;7122 7123 /* Wait for the PF's queues to be disabled */7124 ret = i40e_pf_wait_queues_disabled(pf);7125 if (ret) {7126 /* Schedule PF reset to recover */7127 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);7128 i40e_service_event_schedule(pf);7129 goto err;7130 } else {7131 i40e_pf_unquiesce_all_vsi(pf);7132 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);7133 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);7134 }7135 /* registers are set, lets apply */7136 if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps))7137 ret = i40e_hw_set_dcb_config(pf, new_cfg);7138 }7139 7140err:7141 return ret;7142}7143 7144/**7145 * i40e_dcb_sw_default_config - Set default DCB configuration when DCB in SW7146 * @pf: PF being queried7147 *7148 * Set default DCB configuration in case DCB is to be done in SW.7149 **/7150int i40e_dcb_sw_default_config(struct i40e_pf *pf)7151{7152 struct i40e_dcbx_config *dcb_cfg = &pf->hw.local_dcbx_config;7153 struct i40e_aqc_configure_switching_comp_ets_data ets_data;7154 struct i40e_hw *hw = &pf->hw;7155 int err;7156 7157 if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps)) {7158 /* Update the local cached instance with TC0 ETS */7159 memset(&pf->tmp_cfg, 0, sizeof(struct i40e_dcbx_config));7160 pf->tmp_cfg.etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;7161 pf->tmp_cfg.etscfg.maxtcs = 0;7162 pf->tmp_cfg.etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;7163 pf->tmp_cfg.etscfg.tsatable[0] = I40E_IEEE_TSA_ETS;7164 pf->tmp_cfg.pfc.willing = I40E_IEEE_DEFAULT_PFC_WILLING;7165 pf->tmp_cfg.pfc.pfccap = I40E_MAX_TRAFFIC_CLASS;7166 /* FW needs one App to configure HW */7167 pf->tmp_cfg.numapps = I40E_IEEE_DEFAULT_NUM_APPS;7168 pf->tmp_cfg.app[0].selector = I40E_APP_SEL_ETHTYPE;7169 pf->tmp_cfg.app[0].priority = I40E_IEEE_DEFAULT_APP_PRIO;7170 pf->tmp_cfg.app[0].protocolid = I40E_APP_PROTOID_FCOE;7171 7172 return i40e_hw_set_dcb_config(pf, &pf->tmp_cfg);7173 }7174 7175 memset(&ets_data, 0, sizeof(ets_data));7176 ets_data.tc_valid_bits = I40E_DEFAULT_TRAFFIC_CLASS; /* TC0 only */7177 ets_data.tc_strict_priority_flags = 0; /* ETS */7178 ets_data.tc_bw_share_credits[0] = I40E_IEEE_DEFAULT_ETS_TCBW; /* 100% to TC0 */7179 7180 /* Enable ETS on the Physical port */7181 err = i40e_aq_config_switch_comp_ets7182 (hw, pf->mac_seid, &ets_data,7183 i40e_aqc_opc_enable_switching_comp_ets, NULL);7184 if (err) {7185 dev_info(&pf->pdev->dev,7186 "Enable Port ETS failed, err %pe aq_err %s\n",7187 ERR_PTR(err),7188 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));7189 err = -ENOENT;7190 goto out;7191 }7192 7193 /* Update the local cached instance with TC0 ETS */7194 dcb_cfg->etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;7195 dcb_cfg->etscfg.cbs = 0;7196 dcb_cfg->etscfg.maxtcs = I40E_MAX_TRAFFIC_CLASS;7197 dcb_cfg->etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;7198 7199out:7200 return err;7201}7202 7203/**7204 * i40e_init_pf_dcb - Initialize DCB configuration7205 * @pf: PF being configured7206 *7207 * Query the current DCB configuration and cache it7208 * in the hardware structure7209 **/7210static int i40e_init_pf_dcb(struct i40e_pf *pf)7211{7212 struct i40e_hw *hw = &pf->hw;7213 int err;7214 7215 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable7216 * Also do not enable DCBx if FW LLDP agent is disabled7217 */7218 if (test_bit(I40E_HW_CAP_NO_DCB_SUPPORT, pf->hw.caps)) {7219 dev_info(&pf->pdev->dev, "DCB is not supported.\n");7220 err = -EOPNOTSUPP;7221 goto out;7222 }7223 if (test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) {7224 dev_info(&pf->pdev->dev, "FW LLDP is disabled, attempting SW DCB\n");7225 err = i40e_dcb_sw_default_config(pf);7226 if (err) {7227 dev_info(&pf->pdev->dev, "Could not initialize SW DCB\n");7228 goto out;7229 }7230 dev_info(&pf->pdev->dev, "SW DCB initialization succeeded.\n");7231 pf->dcbx_cap = DCB_CAP_DCBX_HOST |7232 DCB_CAP_DCBX_VER_IEEE;7233 /* at init capable but disabled */7234 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);7235 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);7236 goto out;7237 }7238 err = i40e_init_dcb(hw, true);7239 if (!err) {7240 /* Device/Function is not DCBX capable */7241 if ((!hw->func_caps.dcb) ||7242 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {7243 dev_info(&pf->pdev->dev,7244 "DCBX offload is not supported or is disabled for this PF.\n");7245 } else {7246 /* When status is not DISABLED then DCBX in FW */7247 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |7248 DCB_CAP_DCBX_VER_IEEE;7249 7250 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);7251 /* Enable DCB tagging only when more than one TC7252 * or explicitly disable if only one TC7253 */7254 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)7255 set_bit(I40E_FLAG_DCB_ENA, pf->flags);7256 else7257 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);7258 dev_dbg(&pf->pdev->dev,7259 "DCBX offload is supported for this PF.\n");7260 }7261 } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) {7262 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n");7263 set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags);7264 } else {7265 dev_info(&pf->pdev->dev,7266 "Query for DCB configuration failed, err %pe aq_err %s\n",7267 ERR_PTR(err),7268 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));7269 }7270 7271out:7272 return err;7273}7274#endif /* CONFIG_I40E_DCB */7275 7276static void i40e_print_link_message_eee(struct i40e_vsi *vsi,7277 const char *speed, const char *fc)7278{7279 struct ethtool_keee kedata;7280 7281 memzero_explicit(&kedata, sizeof(kedata));7282 if (vsi->netdev->ethtool_ops->get_eee)7283 vsi->netdev->ethtool_ops->get_eee(vsi->netdev, &kedata);7284 7285 if (!linkmode_empty(kedata.supported))7286 netdev_info(vsi->netdev,7287 "NIC Link is Up, %sbps Full Duplex, Flow Control: %s, EEE: %s\n",7288 speed, fc,7289 kedata.eee_enabled ? "Enabled" : "Disabled");7290 else7291 netdev_info(vsi->netdev,7292 "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n",7293 speed, fc);7294}7295 7296/**7297 * i40e_print_link_message - print link up or down7298 * @vsi: the VSI for which link needs a message7299 * @isup: true of link is up, false otherwise7300 */7301void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)7302{7303 enum i40e_aq_link_speed new_speed;7304 struct i40e_pf *pf = vsi->back;7305 char *speed = "Unknown";7306 char *fc = "Unknown";7307 char *fec = "";7308 char *req_fec = "";7309 char *an = "";7310 7311 if (isup)7312 new_speed = pf->hw.phy.link_info.link_speed;7313 else7314 new_speed = I40E_LINK_SPEED_UNKNOWN;7315 7316 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))7317 return;7318 vsi->current_isup = isup;7319 vsi->current_speed = new_speed;7320 if (!isup) {7321 netdev_info(vsi->netdev, "NIC Link is Down\n");7322 return;7323 }7324 7325 /* Warn user if link speed on NPAR enabled partition is not at7326 * least 10GB7327 */7328 if (pf->hw.func_caps.npar_enable &&7329 (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||7330 pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))7331 netdev_warn(vsi->netdev,7332 "The partition detected link speed that is less than 10Gbps\n");7333 7334 switch (pf->hw.phy.link_info.link_speed) {7335 case I40E_LINK_SPEED_40GB:7336 speed = "40 G";7337 break;7338 case I40E_LINK_SPEED_20GB:7339 speed = "20 G";7340 break;7341 case I40E_LINK_SPEED_25GB:7342 speed = "25 G";7343 break;7344 case I40E_LINK_SPEED_10GB:7345 speed = "10 G";7346 break;7347 case I40E_LINK_SPEED_5GB:7348 speed = "5 G";7349 break;7350 case I40E_LINK_SPEED_2_5GB:7351 speed = "2.5 G";7352 break;7353 case I40E_LINK_SPEED_1GB:7354 speed = "1000 M";7355 break;7356 case I40E_LINK_SPEED_100MB:7357 speed = "100 M";7358 break;7359 default:7360 break;7361 }7362 7363 switch (pf->hw.fc.current_mode) {7364 case I40E_FC_FULL:7365 fc = "RX/TX";7366 break;7367 case I40E_FC_TX_PAUSE:7368 fc = "TX";7369 break;7370 case I40E_FC_RX_PAUSE:7371 fc = "RX";7372 break;7373 default:7374 fc = "None";7375 break;7376 }7377 7378 if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {7379 req_fec = "None";7380 fec = "None";7381 an = "False";7382 7383 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)7384 an = "True";7385 7386 if (pf->hw.phy.link_info.fec_info &7387 I40E_AQ_CONFIG_FEC_KR_ENA)7388 fec = "CL74 FC-FEC/BASE-R";7389 else if (pf->hw.phy.link_info.fec_info &7390 I40E_AQ_CONFIG_FEC_RS_ENA)7391 fec = "CL108 RS-FEC";7392 7393 /* 'CL108 RS-FEC' should be displayed when RS is requested, or7394 * both RS and FC are requested7395 */7396 if (vsi->back->hw.phy.link_info.req_fec_info &7397 (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {7398 if (vsi->back->hw.phy.link_info.req_fec_info &7399 I40E_AQ_REQUEST_FEC_RS)7400 req_fec = "CL108 RS-FEC";7401 else7402 req_fec = "CL74 FC-FEC/BASE-R";7403 }7404 netdev_info(vsi->netdev,7405 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",7406 speed, req_fec, fec, an, fc);7407 } else if (pf->hw.device_id == I40E_DEV_ID_KX_X722) {7408 req_fec = "None";7409 fec = "None";7410 an = "False";7411 7412 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)7413 an = "True";7414 7415 if (pf->hw.phy.link_info.fec_info &7416 I40E_AQ_CONFIG_FEC_KR_ENA)7417 fec = "CL74 FC-FEC/BASE-R";7418 7419 if (pf->hw.phy.link_info.req_fec_info &7420 I40E_AQ_REQUEST_FEC_KR)7421 req_fec = "CL74 FC-FEC/BASE-R";7422 7423 netdev_info(vsi->netdev,7424 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",7425 speed, req_fec, fec, an, fc);7426 } else {7427 i40e_print_link_message_eee(vsi, speed, fc);7428 }7429 7430}7431 7432/**7433 * i40e_up_complete - Finish the last steps of bringing up a connection7434 * @vsi: the VSI being configured7435 **/7436static int i40e_up_complete(struct i40e_vsi *vsi)7437{7438 struct i40e_pf *pf = vsi->back;7439 int err;7440 7441 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))7442 i40e_vsi_configure_msix(vsi);7443 else7444 i40e_configure_msi_and_legacy(vsi);7445 7446 /* start rings */7447 err = i40e_vsi_start_rings(vsi);7448 if (err)7449 return err;7450 7451 clear_bit(__I40E_VSI_DOWN, vsi->state);7452 i40e_napi_enable_all(vsi);7453 i40e_vsi_enable_irq(vsi);7454 7455 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&7456 (vsi->netdev)) {7457 i40e_print_link_message(vsi, true);7458 netif_tx_start_all_queues(vsi->netdev);7459 netif_carrier_on(vsi->netdev);7460 }7461 7462 /* replay FDIR SB filters */7463 if (vsi->type == I40E_VSI_FDIR) {7464 /* reset fd counters */7465 pf->fd_add_err = 0;7466 pf->fd_atr_cnt = 0;7467 i40e_fdir_filter_restore(vsi);7468 }7469 7470 /* On the next run of the service_task, notify any clients of the new7471 * opened netdev7472 */7473 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);7474 i40e_service_event_schedule(pf);7475 7476 return 0;7477}7478 7479/**7480 * i40e_vsi_reinit_locked - Reset the VSI7481 * @vsi: the VSI being configured7482 *7483 * Rebuild the ring structs after some configuration7484 * has changed, e.g. MTU size.7485 **/7486static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)7487{7488 struct i40e_pf *pf = vsi->back;7489 7490 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))7491 usleep_range(1000, 2000);7492 i40e_down(vsi);7493 7494 i40e_up(vsi);7495 clear_bit(__I40E_CONFIG_BUSY, pf->state);7496}7497 7498/**7499 * i40e_force_link_state - Force the link status7500 * @pf: board private structure7501 * @is_up: whether the link state should be forced up or down7502 **/7503static int i40e_force_link_state(struct i40e_pf *pf, bool is_up)7504{7505 struct i40e_aq_get_phy_abilities_resp abilities;7506 struct i40e_aq_set_phy_config config = {0};7507 bool non_zero_phy_type = is_up;7508 struct i40e_hw *hw = &pf->hw;7509 u64 mask;7510 u8 speed;7511 int err;7512 7513 /* Card might've been put in an unstable state by other drivers7514 * and applications, which causes incorrect speed values being7515 * set on startup. In order to clear speed registers, we call7516 * get_phy_capabilities twice, once to get initial state of7517 * available speeds, and once to get current PHY config.7518 */7519 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities,7520 NULL);7521 if (err) {7522 dev_err(&pf->pdev->dev,7523 "failed to get phy cap., ret = %pe last_status = %s\n",7524 ERR_PTR(err),7525 i40e_aq_str(hw, hw->aq.asq_last_status));7526 return err;7527 }7528 speed = abilities.link_speed;7529 7530 /* Get the current phy config */7531 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities,7532 NULL);7533 if (err) {7534 dev_err(&pf->pdev->dev,7535 "failed to get phy cap., ret = %pe last_status = %s\n",7536 ERR_PTR(err),7537 i40e_aq_str(hw, hw->aq.asq_last_status));7538 return err;7539 }7540 7541 /* If link needs to go up, but was not forced to go down,7542 * and its speed values are OK, no need for a flap7543 * if non_zero_phy_type was set, still need to force up7544 */7545 if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags))7546 non_zero_phy_type = true;7547 else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0)7548 return 0;7549 7550 /* To force link we need to set bits for all supported PHY types,7551 * but there are now more than 32, so we need to split the bitmap7552 * across two fields.7553 */7554 mask = I40E_PHY_TYPES_BITMASK;7555 config.phy_type =7556 non_zero_phy_type ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0;7557 config.phy_type_ext =7558 non_zero_phy_type ? (u8)((mask >> 32) & 0xff) : 0;7559 /* Copy the old settings, except of phy_type */7560 config.abilities = abilities.abilities;7561 if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags)) {7562 if (is_up)7563 config.abilities |= I40E_AQ_PHY_ENABLE_LINK;7564 else7565 config.abilities &= ~(I40E_AQ_PHY_ENABLE_LINK);7566 }7567 if (abilities.link_speed != 0)7568 config.link_speed = abilities.link_speed;7569 else7570 config.link_speed = speed;7571 config.eee_capability = abilities.eee_capability;7572 config.eeer = abilities.eeer_val;7573 config.low_power_ctrl = abilities.d3_lpan;7574 config.fec_config = abilities.fec_cfg_curr_mod_ext_info &7575 I40E_AQ_PHY_FEC_CONFIG_MASK;7576 err = i40e_aq_set_phy_config(hw, &config, NULL);7577 7578 if (err) {7579 dev_err(&pf->pdev->dev,7580 "set phy config ret = %pe last_status = %s\n",7581 ERR_PTR(err),7582 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));7583 return err;7584 }7585 7586 /* Update the link info */7587 err = i40e_update_link_info(hw);7588 if (err) {7589 /* Wait a little bit (on 40G cards it sometimes takes a really7590 * long time for link to come back from the atomic reset)7591 * and try once more7592 */7593 msleep(1000);7594 i40e_update_link_info(hw);7595 }7596 7597 i40e_aq_set_link_restart_an(hw, is_up, NULL);7598 7599 return 0;7600}7601 7602/**7603 * i40e_up - Bring the connection back up after being down7604 * @vsi: the VSI being configured7605 **/7606int i40e_up(struct i40e_vsi *vsi)7607{7608 int err;7609 7610 if (vsi->type == I40E_VSI_MAIN &&7611 (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) ||7612 test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags)))7613 i40e_force_link_state(vsi->back, true);7614 7615 err = i40e_vsi_configure(vsi);7616 if (!err)7617 err = i40e_up_complete(vsi);7618 7619 return err;7620}7621 7622/**7623 * i40e_down - Shutdown the connection processing7624 * @vsi: the VSI being stopped7625 **/7626void i40e_down(struct i40e_vsi *vsi)7627{7628 int i;7629 7630 /* It is assumed that the caller of this function7631 * sets the vsi->state __I40E_VSI_DOWN bit.7632 */7633 if (vsi->netdev) {7634 netif_carrier_off(vsi->netdev);7635 netif_tx_disable(vsi->netdev);7636 }7637 i40e_vsi_disable_irq(vsi);7638 i40e_vsi_stop_rings(vsi);7639 if (vsi->type == I40E_VSI_MAIN &&7640 (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) ||7641 test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags)))7642 i40e_force_link_state(vsi->back, false);7643 i40e_napi_disable_all(vsi);7644 7645 for (i = 0; i < vsi->num_queue_pairs; i++) {7646 i40e_clean_tx_ring(vsi->tx_rings[i]);7647 if (i40e_enabled_xdp_vsi(vsi)) {7648 /* Make sure that in-progress ndo_xdp_xmit and7649 * ndo_xsk_wakeup calls are completed.7650 */7651 synchronize_rcu();7652 i40e_clean_tx_ring(vsi->xdp_rings[i]);7653 }7654 i40e_clean_rx_ring(vsi->rx_rings[i]);7655 }7656 7657}7658 7659/**7660 * i40e_validate_mqprio_qopt- validate queue mapping info7661 * @vsi: the VSI being configured7662 * @mqprio_qopt: queue parametrs7663 **/7664static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi,7665 struct tc_mqprio_qopt_offload *mqprio_qopt)7666{7667 u64 sum_max_rate = 0;7668 u64 max_rate = 0;7669 int i;7670 7671 if (mqprio_qopt->qopt.offset[0] != 0 ||7672 mqprio_qopt->qopt.num_tc < 1 ||7673 mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS)7674 return -EINVAL;7675 for (i = 0; ; i++) {7676 if (!mqprio_qopt->qopt.count[i])7677 return -EINVAL;7678 if (mqprio_qopt->min_rate[i]) {7679 dev_err(&vsi->back->pdev->dev,7680 "Invalid min tx rate (greater than 0) specified\n");7681 return -EINVAL;7682 }7683 max_rate = mqprio_qopt->max_rate[i];7684 do_div(max_rate, I40E_BW_MBPS_DIVISOR);7685 sum_max_rate += max_rate;7686 7687 if (i >= mqprio_qopt->qopt.num_tc - 1)7688 break;7689 if (mqprio_qopt->qopt.offset[i + 1] !=7690 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))7691 return -EINVAL;7692 }7693 if (vsi->num_queue_pairs <7694 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) {7695 dev_err(&vsi->back->pdev->dev,7696 "Failed to create traffic channel, insufficient number of queues.\n");7697 return -EINVAL;7698 }7699 if (sum_max_rate > i40e_get_link_speed(vsi)) {7700 dev_err(&vsi->back->pdev->dev,7701 "Invalid max tx rate specified\n");7702 return -EINVAL;7703 }7704 return 0;7705}7706 7707/**7708 * i40e_vsi_set_default_tc_config - set default values for tc configuration7709 * @vsi: the VSI being configured7710 **/7711static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi)7712{7713 u16 qcount;7714 int i;7715 7716 /* Only TC0 is enabled */7717 vsi->tc_config.numtc = 1;7718 vsi->tc_config.enabled_tc = 1;7719 qcount = min_t(int, vsi->alloc_queue_pairs,7720 i40e_pf_get_max_q_per_tc(vsi->back));7721 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {7722 /* For the TC that is not enabled set the offset to default7723 * queue and allocate one queue for the given TC.7724 */7725 vsi->tc_config.tc_info[i].qoffset = 0;7726 if (i == 0)7727 vsi->tc_config.tc_info[i].qcount = qcount;7728 else7729 vsi->tc_config.tc_info[i].qcount = 1;7730 vsi->tc_config.tc_info[i].netdev_tc = 0;7731 }7732}7733 7734/**7735 * i40e_del_macvlan_filter7736 * @hw: pointer to the HW structure7737 * @seid: seid of the channel VSI7738 * @macaddr: the mac address to apply as a filter7739 * @aq_err: store the admin Q error7740 *7741 * This function deletes a mac filter on the channel VSI which serves as the7742 * macvlan. Returns 0 on success.7743 **/7744static int i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid,7745 const u8 *macaddr, int *aq_err)7746{7747 struct i40e_aqc_remove_macvlan_element_data element;7748 int status;7749 7750 memset(&element, 0, sizeof(element));7751 ether_addr_copy(element.mac_addr, macaddr);7752 element.vlan_tag = 0;7753 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;7754 status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL);7755 *aq_err = hw->aq.asq_last_status;7756 7757 return status;7758}7759 7760/**7761 * i40e_add_macvlan_filter7762 * @hw: pointer to the HW structure7763 * @seid: seid of the channel VSI7764 * @macaddr: the mac address to apply as a filter7765 * @aq_err: store the admin Q error7766 *7767 * This function adds a mac filter on the channel VSI which serves as the7768 * macvlan. Returns 0 on success.7769 **/7770static int i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid,7771 const u8 *macaddr, int *aq_err)7772{7773 struct i40e_aqc_add_macvlan_element_data element;7774 u16 cmd_flags = 0;7775 int status;7776 7777 ether_addr_copy(element.mac_addr, macaddr);7778 element.vlan_tag = 0;7779 element.queue_number = 0;7780 element.match_method = I40E_AQC_MM_ERR_NO_RES;7781 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;7782 element.flags = cpu_to_le16(cmd_flags);7783 status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL);7784 *aq_err = hw->aq.asq_last_status;7785 7786 return status;7787}7788 7789/**7790 * i40e_reset_ch_rings - Reset the queue contexts in a channel7791 * @vsi: the VSI we want to access7792 * @ch: the channel we want to access7793 */7794static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch)7795{7796 struct i40e_ring *tx_ring, *rx_ring;7797 u16 pf_q;7798 int i;7799 7800 for (i = 0; i < ch->num_queue_pairs; i++) {7801 pf_q = ch->base_queue + i;7802 tx_ring = vsi->tx_rings[pf_q];7803 tx_ring->ch = NULL;7804 rx_ring = vsi->rx_rings[pf_q];7805 rx_ring->ch = NULL;7806 }7807}7808 7809/**7810 * i40e_free_macvlan_channels7811 * @vsi: the VSI we want to access7812 *7813 * This function frees the Qs of the channel VSI from7814 * the stack and also deletes the channel VSIs which7815 * serve as macvlans.7816 */7817static void i40e_free_macvlan_channels(struct i40e_vsi *vsi)7818{7819 struct i40e_channel *ch, *ch_tmp;7820 int ret;7821 7822 if (list_empty(&vsi->macvlan_list))7823 return;7824 7825 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {7826 struct i40e_vsi *parent_vsi;7827 7828 if (i40e_is_channel_macvlan(ch)) {7829 i40e_reset_ch_rings(vsi, ch);7830 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);7831 netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev);7832 netdev_set_sb_channel(ch->fwd->netdev, 0);7833 kfree(ch->fwd);7834 ch->fwd = NULL;7835 }7836 7837 list_del(&ch->list);7838 parent_vsi = ch->parent_vsi;7839 if (!parent_vsi || !ch->initialized) {7840 kfree(ch);7841 continue;7842 }7843 7844 /* remove the VSI */7845 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,7846 NULL);7847 if (ret)7848 dev_err(&vsi->back->pdev->dev,7849 "unable to remove channel (%d) for parent VSI(%d)\n",7850 ch->seid, parent_vsi->seid);7851 kfree(ch);7852 }7853 vsi->macvlan_cnt = 0;7854}7855 7856/**7857 * i40e_fwd_ring_up - bring the macvlan device up7858 * @vsi: the VSI we want to access7859 * @vdev: macvlan netdevice7860 * @fwd: the private fwd structure7861 */7862static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev,7863 struct i40e_fwd_adapter *fwd)7864{7865 struct i40e_channel *ch = NULL, *ch_tmp, *iter;7866 int ret = 0, num_tc = 1, i, aq_err;7867 struct i40e_pf *pf = vsi->back;7868 struct i40e_hw *hw = &pf->hw;7869 7870 /* Go through the list and find an available channel */7871 list_for_each_entry_safe(iter, ch_tmp, &vsi->macvlan_list, list) {7872 if (!i40e_is_channel_macvlan(iter)) {7873 iter->fwd = fwd;7874 /* record configuration for macvlan interface in vdev */7875 for (i = 0; i < num_tc; i++)7876 netdev_bind_sb_channel_queue(vsi->netdev, vdev,7877 i,7878 iter->num_queue_pairs,7879 iter->base_queue);7880 for (i = 0; i < iter->num_queue_pairs; i++) {7881 struct i40e_ring *tx_ring, *rx_ring;7882 u16 pf_q;7883 7884 pf_q = iter->base_queue + i;7885 7886 /* Get to TX ring ptr */7887 tx_ring = vsi->tx_rings[pf_q];7888 tx_ring->ch = iter;7889 7890 /* Get the RX ring ptr */7891 rx_ring = vsi->rx_rings[pf_q];7892 rx_ring->ch = iter;7893 }7894 ch = iter;7895 break;7896 }7897 }7898 7899 if (!ch)7900 return -EINVAL;7901 7902 /* Guarantee all rings are updated before we update the7903 * MAC address filter.7904 */7905 wmb();7906 7907 /* Add a mac filter */7908 ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err);7909 if (ret) {7910 /* if we cannot add the MAC rule then disable the offload */7911 macvlan_release_l2fw_offload(vdev);7912 for (i = 0; i < ch->num_queue_pairs; i++) {7913 struct i40e_ring *rx_ring;7914 u16 pf_q;7915 7916 pf_q = ch->base_queue + i;7917 rx_ring = vsi->rx_rings[pf_q];7918 rx_ring->netdev = NULL;7919 }7920 dev_info(&pf->pdev->dev,7921 "Error adding mac filter on macvlan err %pe, aq_err %s\n",7922 ERR_PTR(ret),7923 i40e_aq_str(hw, aq_err));7924 netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n");7925 }7926 7927 return ret;7928}7929 7930/**7931 * i40e_setup_macvlans - create the channels which will be macvlans7932 * @vsi: the VSI we want to access7933 * @macvlan_cnt: no. of macvlans to be setup7934 * @qcnt: no. of Qs per macvlan7935 * @vdev: macvlan netdevice7936 */7937static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt,7938 struct net_device *vdev)7939{7940 struct i40e_pf *pf = vsi->back;7941 struct i40e_hw *hw = &pf->hw;7942 struct i40e_vsi_context ctxt;7943 u16 sections, qmap, num_qps;7944 struct i40e_channel *ch;7945 int i, pow, ret = 0;7946 u8 offset = 0;7947 7948 if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt)7949 return -EINVAL;7950 7951 num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt);7952 7953 /* find the next higher power-of-2 of num queue pairs */7954 pow = fls(roundup_pow_of_two(num_qps) - 1);7955 7956 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |7957 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);7958 7959 /* Setup context bits for the main VSI */7960 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;7961 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;7962 memset(&ctxt, 0, sizeof(ctxt));7963 ctxt.seid = vsi->seid;7964 ctxt.pf_num = vsi->back->hw.pf_id;7965 ctxt.vf_num = 0;7966 ctxt.uplink_seid = vsi->uplink_seid;7967 ctxt.info = vsi->info;7968 ctxt.info.tc_mapping[0] = cpu_to_le16(qmap);7969 ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);7970 ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);7971 ctxt.info.valid_sections |= cpu_to_le16(sections);7972 7973 /* Reconfigure RSS for main VSI with new max queue count */7974 vsi->rss_size = max_t(u16, num_qps, qcnt);7975 ret = i40e_vsi_config_rss(vsi);7976 if (ret) {7977 dev_info(&pf->pdev->dev,7978 "Failed to reconfig RSS for num_queues (%u)\n",7979 vsi->rss_size);7980 return ret;7981 }7982 vsi->reconfig_rss = true;7983 dev_dbg(&vsi->back->pdev->dev,7984 "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size);7985 vsi->next_base_queue = num_qps;7986 vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps;7987 7988 /* Update the VSI after updating the VSI queue-mapping7989 * information7990 */7991 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);7992 if (ret) {7993 dev_info(&pf->pdev->dev,7994 "Update vsi tc config failed, err %pe aq_err %s\n",7995 ERR_PTR(ret),7996 i40e_aq_str(hw, hw->aq.asq_last_status));7997 return ret;7998 }7999 /* update the local VSI info with updated queue map */8000 i40e_vsi_update_queue_map(vsi, &ctxt);8001 vsi->info.valid_sections = 0;8002 8003 /* Create channels for macvlans */8004 INIT_LIST_HEAD(&vsi->macvlan_list);8005 for (i = 0; i < macvlan_cnt; i++) {8006 ch = kzalloc(sizeof(*ch), GFP_KERNEL);8007 if (!ch) {8008 ret = -ENOMEM;8009 goto err_free;8010 }8011 INIT_LIST_HEAD(&ch->list);8012 ch->num_queue_pairs = qcnt;8013 if (!i40e_setup_channel(pf, vsi, ch)) {8014 ret = -EINVAL;8015 kfree(ch);8016 goto err_free;8017 }8018 ch->parent_vsi = vsi;8019 vsi->cnt_q_avail -= ch->num_queue_pairs;8020 vsi->macvlan_cnt++;8021 list_add_tail(&ch->list, &vsi->macvlan_list);8022 }8023 8024 return ret;8025 8026err_free:8027 dev_info(&pf->pdev->dev, "Failed to setup macvlans\n");8028 i40e_free_macvlan_channels(vsi);8029 8030 return ret;8031}8032 8033/**8034 * i40e_fwd_add - configure macvlans8035 * @netdev: net device to configure8036 * @vdev: macvlan netdevice8037 **/8038static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev)8039{8040 struct i40e_netdev_priv *np = netdev_priv(netdev);8041 u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors;8042 struct i40e_vsi *vsi = np->vsi;8043 struct i40e_pf *pf = vsi->back;8044 struct i40e_fwd_adapter *fwd;8045 int avail_macvlan, ret;8046 8047 if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {8048 netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n");8049 return ERR_PTR(-EINVAL);8050 }8051 if (i40e_is_tc_mqprio_enabled(pf)) {8052 netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n");8053 return ERR_PTR(-EINVAL);8054 }8055 if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) {8056 netdev_info(netdev, "Not enough vectors available to support macvlans\n");8057 return ERR_PTR(-EINVAL);8058 }8059 8060 /* The macvlan device has to be a single Q device so that the8061 * tc_to_txq field can be reused to pick the tx queue.8062 */8063 if (netif_is_multiqueue(vdev))8064 return ERR_PTR(-ERANGE);8065 8066 if (!vsi->macvlan_cnt) {8067 /* reserve bit 0 for the pf device */8068 set_bit(0, vsi->fwd_bitmask);8069 8070 /* Try to reserve as many queues as possible for macvlans. First8071 * reserve 3/4th of max vectors, then half, then quarter and8072 * calculate Qs per macvlan as you go8073 */8074 vectors = pf->num_lan_msix;8075 if (vectors <= I40E_MAX_MACVLANS && vectors > 64) {8076 /* allocate 4 Qs per macvlan and 32 Qs to the PF*/8077 q_per_macvlan = 4;8078 macvlan_cnt = (vectors - 32) / 4;8079 } else if (vectors <= 64 && vectors > 32) {8080 /* allocate 2 Qs per macvlan and 16 Qs to the PF*/8081 q_per_macvlan = 2;8082 macvlan_cnt = (vectors - 16) / 2;8083 } else if (vectors <= 32 && vectors > 16) {8084 /* allocate 1 Q per macvlan and 16 Qs to the PF*/8085 q_per_macvlan = 1;8086 macvlan_cnt = vectors - 16;8087 } else if (vectors <= 16 && vectors > 8) {8088 /* allocate 1 Q per macvlan and 8 Qs to the PF */8089 q_per_macvlan = 1;8090 macvlan_cnt = vectors - 8;8091 } else {8092 /* allocate 1 Q per macvlan and 1 Q to the PF */8093 q_per_macvlan = 1;8094 macvlan_cnt = vectors - 1;8095 }8096 8097 if (macvlan_cnt == 0)8098 return ERR_PTR(-EBUSY);8099 8100 /* Quiesce VSI queues */8101 i40e_quiesce_vsi(vsi);8102 8103 /* sets up the macvlans but does not "enable" them */8104 ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan,8105 vdev);8106 if (ret)8107 return ERR_PTR(ret);8108 8109 /* Unquiesce VSI */8110 i40e_unquiesce_vsi(vsi);8111 }8112 avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask,8113 vsi->macvlan_cnt);8114 if (avail_macvlan >= I40E_MAX_MACVLANS)8115 return ERR_PTR(-EBUSY);8116 8117 /* create the fwd struct */8118 fwd = kzalloc(sizeof(*fwd), GFP_KERNEL);8119 if (!fwd)8120 return ERR_PTR(-ENOMEM);8121 8122 set_bit(avail_macvlan, vsi->fwd_bitmask);8123 fwd->bit_no = avail_macvlan;8124 netdev_set_sb_channel(vdev, avail_macvlan);8125 fwd->netdev = vdev;8126 8127 if (!netif_running(netdev))8128 return fwd;8129 8130 /* Set fwd ring up */8131 ret = i40e_fwd_ring_up(vsi, vdev, fwd);8132 if (ret) {8133 /* unbind the queues and drop the subordinate channel config */8134 netdev_unbind_sb_channel(netdev, vdev);8135 netdev_set_sb_channel(vdev, 0);8136 8137 kfree(fwd);8138 return ERR_PTR(-EINVAL);8139 }8140 8141 return fwd;8142}8143 8144/**8145 * i40e_del_all_macvlans - Delete all the mac filters on the channels8146 * @vsi: the VSI we want to access8147 */8148static void i40e_del_all_macvlans(struct i40e_vsi *vsi)8149{8150 struct i40e_channel *ch, *ch_tmp;8151 struct i40e_pf *pf = vsi->back;8152 struct i40e_hw *hw = &pf->hw;8153 int aq_err, ret = 0;8154 8155 if (list_empty(&vsi->macvlan_list))8156 return;8157 8158 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {8159 if (i40e_is_channel_macvlan(ch)) {8160 ret = i40e_del_macvlan_filter(hw, ch->seid,8161 i40e_channel_mac(ch),8162 &aq_err);8163 if (!ret) {8164 /* Reset queue contexts */8165 i40e_reset_ch_rings(vsi, ch);8166 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);8167 netdev_unbind_sb_channel(vsi->netdev,8168 ch->fwd->netdev);8169 netdev_set_sb_channel(ch->fwd->netdev, 0);8170 kfree(ch->fwd);8171 ch->fwd = NULL;8172 }8173 }8174 }8175}8176 8177/**8178 * i40e_fwd_del - delete macvlan interfaces8179 * @netdev: net device to configure8180 * @vdev: macvlan netdevice8181 */8182static void i40e_fwd_del(struct net_device *netdev, void *vdev)8183{8184 struct i40e_netdev_priv *np = netdev_priv(netdev);8185 struct i40e_fwd_adapter *fwd = vdev;8186 struct i40e_channel *ch, *ch_tmp;8187 struct i40e_vsi *vsi = np->vsi;8188 struct i40e_pf *pf = vsi->back;8189 struct i40e_hw *hw = &pf->hw;8190 int aq_err, ret = 0;8191 8192 /* Find the channel associated with the macvlan and del mac filter */8193 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {8194 if (i40e_is_channel_macvlan(ch) &&8195 ether_addr_equal(i40e_channel_mac(ch),8196 fwd->netdev->dev_addr)) {8197 ret = i40e_del_macvlan_filter(hw, ch->seid,8198 i40e_channel_mac(ch),8199 &aq_err);8200 if (!ret) {8201 /* Reset queue contexts */8202 i40e_reset_ch_rings(vsi, ch);8203 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);8204 netdev_unbind_sb_channel(netdev, fwd->netdev);8205 netdev_set_sb_channel(fwd->netdev, 0);8206 kfree(ch->fwd);8207 ch->fwd = NULL;8208 } else {8209 dev_info(&pf->pdev->dev,8210 "Error deleting mac filter on macvlan err %pe, aq_err %s\n",8211 ERR_PTR(ret),8212 i40e_aq_str(hw, aq_err));8213 }8214 break;8215 }8216 }8217}8218 8219/**8220 * i40e_setup_tc - configure multiple traffic classes8221 * @netdev: net device to configure8222 * @type_data: tc offload data8223 **/8224static int i40e_setup_tc(struct net_device *netdev, void *type_data)8225{8226 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;8227 struct i40e_netdev_priv *np = netdev_priv(netdev);8228 struct i40e_vsi *vsi = np->vsi;8229 struct i40e_pf *pf = vsi->back;8230 u8 enabled_tc = 0, num_tc, hw;8231 bool need_reset = false;8232 int old_queue_pairs;8233 int ret = -EINVAL;8234 u16 mode;8235 int i;8236 8237 old_queue_pairs = vsi->num_queue_pairs;8238 num_tc = mqprio_qopt->qopt.num_tc;8239 hw = mqprio_qopt->qopt.hw;8240 mode = mqprio_qopt->mode;8241 if (!hw) {8242 clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);8243 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));8244 goto config_tc;8245 }8246 8247 /* Check if MFP enabled */8248 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {8249 netdev_info(netdev,8250 "Configuring TC not supported in MFP mode\n");8251 return ret;8252 }8253 switch (mode) {8254 case TC_MQPRIO_MODE_DCB:8255 clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);8256 8257 /* Check if DCB enabled to continue */8258 if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {8259 netdev_info(netdev,8260 "DCB is not enabled for adapter\n");8261 return ret;8262 }8263 8264 /* Check whether tc count is within enabled limit */8265 if (num_tc > i40e_pf_get_num_tc(pf)) {8266 netdev_info(netdev,8267 "TC count greater than enabled on link for adapter\n");8268 return ret;8269 }8270 break;8271 case TC_MQPRIO_MODE_CHANNEL:8272 if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {8273 netdev_info(netdev,8274 "Full offload of TC Mqprio options is not supported when DCB is enabled\n");8275 return ret;8276 }8277 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))8278 return ret;8279 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt);8280 if (ret)8281 return ret;8282 memcpy(&vsi->mqprio_qopt, mqprio_qopt,8283 sizeof(*mqprio_qopt));8284 set_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);8285 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);8286 break;8287 default:8288 return -EINVAL;8289 }8290 8291config_tc:8292 /* Generate TC map for number of tc requested */8293 for (i = 0; i < num_tc; i++)8294 enabled_tc |= BIT(i);8295 8296 /* Requesting same TC configuration as already enabled */8297 if (enabled_tc == vsi->tc_config.enabled_tc &&8298 mode != TC_MQPRIO_MODE_CHANNEL)8299 return 0;8300 8301 /* Quiesce VSI queues */8302 i40e_quiesce_vsi(vsi);8303 8304 if (!hw && !i40e_is_tc_mqprio_enabled(pf))8305 i40e_remove_queue_channels(vsi);8306 8307 /* Configure VSI for enabled TCs */8308 ret = i40e_vsi_config_tc(vsi, enabled_tc);8309 if (ret) {8310 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",8311 vsi->seid);8312 need_reset = true;8313 goto exit;8314 } else if (enabled_tc &&8315 (!is_power_of_2(vsi->tc_config.tc_info[0].qcount))) {8316 netdev_info(netdev,8317 "Failed to create channel. Override queues (%u) not power of 2\n",8318 vsi->tc_config.tc_info[0].qcount);8319 ret = -EINVAL;8320 need_reset = true;8321 goto exit;8322 }8323 8324 dev_info(&vsi->back->pdev->dev,8325 "Setup channel (id:%u) utilizing num_queues %d\n",8326 vsi->seid, vsi->tc_config.tc_info[0].qcount);8327 8328 if (i40e_is_tc_mqprio_enabled(pf)) {8329 if (vsi->mqprio_qopt.max_rate[0]) {8330 u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,8331 vsi->mqprio_qopt.max_rate[0]);8332 8333 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);8334 if (!ret) {8335 u64 credits = max_tx_rate;8336 8337 do_div(credits, I40E_BW_CREDIT_DIVISOR);8338 dev_dbg(&vsi->back->pdev->dev,8339 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",8340 max_tx_rate,8341 credits,8342 vsi->seid);8343 } else {8344 need_reset = true;8345 goto exit;8346 }8347 }8348 ret = i40e_configure_queue_channels(vsi);8349 if (ret) {8350 vsi->num_queue_pairs = old_queue_pairs;8351 netdev_info(netdev,8352 "Failed configuring queue channels\n");8353 need_reset = true;8354 goto exit;8355 }8356 }8357 8358exit:8359 /* Reset the configuration data to defaults, only TC0 is enabled */8360 if (need_reset) {8361 i40e_vsi_set_default_tc_config(vsi);8362 need_reset = false;8363 }8364 8365 /* Unquiesce VSI */8366 i40e_unquiesce_vsi(vsi);8367 return ret;8368}8369 8370/**8371 * i40e_set_cld_element - sets cloud filter element data8372 * @filter: cloud filter rule8373 * @cld: ptr to cloud filter element data8374 *8375 * This is helper function to copy data into cloud filter element8376 **/8377static inline void8378i40e_set_cld_element(struct i40e_cloud_filter *filter,8379 struct i40e_aqc_cloud_filters_element_data *cld)8380{8381 u32 ipa;8382 int i;8383 8384 memset(cld, 0, sizeof(*cld));8385 ether_addr_copy(cld->outer_mac, filter->dst_mac);8386 ether_addr_copy(cld->inner_mac, filter->src_mac);8387 8388 if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6)8389 return;8390 8391 if (filter->n_proto == ETH_P_IPV6) {8392#define IPV6_MAX_INDEX (ARRAY_SIZE(filter->dst_ipv6) - 1)8393 for (i = 0; i < ARRAY_SIZE(filter->dst_ipv6); i++) {8394 ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]);8395 8396 *(__le32 *)&cld->ipaddr.raw_v6.data[i * 2] = cpu_to_le32(ipa);8397 }8398 } else {8399 ipa = be32_to_cpu(filter->dst_ipv4);8400 8401 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa));8402 }8403 8404 cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id));8405 8406 /* tenant_id is not supported by FW now, once the support is enabled8407 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id)8408 */8409 if (filter->tenant_id)8410 return;8411}8412 8413/**8414 * i40e_add_del_cloud_filter - Add/del cloud filter8415 * @vsi: pointer to VSI8416 * @filter: cloud filter rule8417 * @add: if true, add, if false, delete8418 *8419 * Add or delete a cloud filter for a specific flow spec.8420 * Returns 0 if the filter were successfully added.8421 **/8422int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,8423 struct i40e_cloud_filter *filter, bool add)8424{8425 struct i40e_aqc_cloud_filters_element_data cld_filter;8426 struct i40e_pf *pf = vsi->back;8427 int ret;8428 static const u16 flag_table[128] = {8429 [I40E_CLOUD_FILTER_FLAGS_OMAC] =8430 I40E_AQC_ADD_CLOUD_FILTER_OMAC,8431 [I40E_CLOUD_FILTER_FLAGS_IMAC] =8432 I40E_AQC_ADD_CLOUD_FILTER_IMAC,8433 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN] =8434 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN,8435 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] =8436 I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID,8437 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] =8438 I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC,8439 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] =8440 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID,8441 [I40E_CLOUD_FILTER_FLAGS_IIP] =8442 I40E_AQC_ADD_CLOUD_FILTER_IIP,8443 };8444 8445 if (filter->flags >= ARRAY_SIZE(flag_table))8446 return -EIO;8447 8448 memset(&cld_filter, 0, sizeof(cld_filter));8449 8450 /* copy element needed to add cloud filter from filter */8451 i40e_set_cld_element(filter, &cld_filter);8452 8453 if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE)8454 cld_filter.flags = cpu_to_le16(filter->tunnel_type <<8455 I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT);8456 8457 if (filter->n_proto == ETH_P_IPV6)8458 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |8459 I40E_AQC_ADD_CLOUD_FLAGS_IPV6);8460 else8461 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |8462 I40E_AQC_ADD_CLOUD_FLAGS_IPV4);8463 8464 if (add)8465 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid,8466 &cld_filter, 1);8467 else8468 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid,8469 &cld_filter, 1);8470 if (ret)8471 dev_dbg(&pf->pdev->dev,8472 "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n",8473 add ? "add" : "delete", filter->dst_port, ret,8474 pf->hw.aq.asq_last_status);8475 else8476 dev_info(&pf->pdev->dev,8477 "%s cloud filter for VSI: %d\n",8478 add ? "Added" : "Deleted", filter->seid);8479 return ret;8480}8481 8482/**8483 * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf8484 * @vsi: pointer to VSI8485 * @filter: cloud filter rule8486 * @add: if true, add, if false, delete8487 *8488 * Add or delete a cloud filter for a specific flow spec using big buffer.8489 * Returns 0 if the filter were successfully added.8490 **/8491int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,8492 struct i40e_cloud_filter *filter,8493 bool add)8494{8495 struct i40e_aqc_cloud_filters_element_bb cld_filter;8496 struct i40e_pf *pf = vsi->back;8497 int ret;8498 8499 /* Both (src/dst) valid mac_addr are not supported */8500 if ((is_valid_ether_addr(filter->dst_mac) &&8501 is_valid_ether_addr(filter->src_mac)) ||8502 (is_multicast_ether_addr(filter->dst_mac) &&8503 is_multicast_ether_addr(filter->src_mac)))8504 return -EOPNOTSUPP;8505 8506 /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP8507 * ports are not supported via big buffer now.8508 */8509 if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP)8510 return -EOPNOTSUPP;8511 8512 /* adding filter using src_port/src_ip is not supported at this stage */8513 if (filter->src_port ||8514 (filter->src_ipv4 && filter->n_proto != ETH_P_IPV6) ||8515 !ipv6_addr_any(&filter->ip.v6.src_ip6))8516 return -EOPNOTSUPP;8517 8518 memset(&cld_filter, 0, sizeof(cld_filter));8519 8520 /* copy element needed to add cloud filter from filter */8521 i40e_set_cld_element(filter, &cld_filter.element);8522 8523 if (is_valid_ether_addr(filter->dst_mac) ||8524 is_valid_ether_addr(filter->src_mac) ||8525 is_multicast_ether_addr(filter->dst_mac) ||8526 is_multicast_ether_addr(filter->src_mac)) {8527 /* MAC + IP : unsupported mode */8528 if (filter->dst_ipv4)8529 return -EOPNOTSUPP;8530 8531 /* since we validated that L4 port must be valid before8532 * we get here, start with respective "flags" value8533 * and update if vlan is present or not8534 */8535 cld_filter.element.flags =8536 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT);8537 8538 if (filter->vlan_id) {8539 cld_filter.element.flags =8540 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT);8541 }8542 8543 } else if ((filter->dst_ipv4 && filter->n_proto != ETH_P_IPV6) ||8544 !ipv6_addr_any(&filter->ip.v6.dst_ip6)) {8545 cld_filter.element.flags =8546 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT);8547 if (filter->n_proto == ETH_P_IPV6)8548 cld_filter.element.flags |=8549 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6);8550 else8551 cld_filter.element.flags |=8552 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4);8553 } else {8554 dev_err(&pf->pdev->dev,8555 "either mac or ip has to be valid for cloud filter\n");8556 return -EINVAL;8557 }8558 8559 /* Now copy L4 port in Byte 6..7 in general fields */8560 cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] =8561 be16_to_cpu(filter->dst_port);8562 8563 if (add) {8564 /* Validate current device switch mode, change if necessary */8565 ret = i40e_validate_and_set_switch_mode(vsi);8566 if (ret) {8567 dev_err(&pf->pdev->dev,8568 "failed to set switch mode, ret %d\n",8569 ret);8570 return ret;8571 }8572 8573 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid,8574 &cld_filter, 1);8575 } else {8576 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid,8577 &cld_filter, 1);8578 }8579 8580 if (ret)8581 dev_dbg(&pf->pdev->dev,8582 "Failed to %s cloud filter(big buffer) err %d aq_err %d\n",8583 add ? "add" : "delete", ret, pf->hw.aq.asq_last_status);8584 else8585 dev_info(&pf->pdev->dev,8586 "%s cloud filter for VSI: %d, L4 port: %d\n",8587 add ? "add" : "delete", filter->seid,8588 ntohs(filter->dst_port));8589 return ret;8590}8591 8592/**8593 * i40e_parse_cls_flower - Parse tc flower filters provided by kernel8594 * @vsi: Pointer to VSI8595 * @f: Pointer to struct flow_cls_offload8596 * @filter: Pointer to cloud filter structure8597 *8598 **/8599static int i40e_parse_cls_flower(struct i40e_vsi *vsi,8600 struct flow_cls_offload *f,8601 struct i40e_cloud_filter *filter)8602{8603 struct flow_rule *rule = flow_cls_offload_flow_rule(f);8604 struct flow_dissector *dissector = rule->match.dissector;8605 u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0;8606 struct i40e_pf *pf = vsi->back;8607 u8 field_flags = 0;8608 8609 if (dissector->used_keys &8610 ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |8611 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |8612 BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |8613 BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) |8614 BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |8615 BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |8616 BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |8617 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID))) {8618 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%llx\n",8619 dissector->used_keys);8620 return -EOPNOTSUPP;8621 }8622 8623 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {8624 struct flow_match_enc_keyid match;8625 8626 flow_rule_match_enc_keyid(rule, &match);8627 if (match.mask->keyid != 0)8628 field_flags |= I40E_CLOUD_FIELD_TEN_ID;8629 8630 filter->tenant_id = be32_to_cpu(match.key->keyid);8631 }8632 8633 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {8634 struct flow_match_basic match;8635 8636 flow_rule_match_basic(rule, &match);8637 n_proto_key = ntohs(match.key->n_proto);8638 n_proto_mask = ntohs(match.mask->n_proto);8639 8640 if (n_proto_key == ETH_P_ALL) {8641 n_proto_key = 0;8642 n_proto_mask = 0;8643 }8644 filter->n_proto = n_proto_key & n_proto_mask;8645 filter->ip_proto = match.key->ip_proto;8646 }8647 8648 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {8649 struct flow_match_eth_addrs match;8650 8651 flow_rule_match_eth_addrs(rule, &match);8652 8653 /* use is_broadcast and is_zero to check for all 0xf or 0 */8654 if (!is_zero_ether_addr(match.mask->dst)) {8655 if (is_broadcast_ether_addr(match.mask->dst)) {8656 field_flags |= I40E_CLOUD_FIELD_OMAC;8657 } else {8658 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n",8659 match.mask->dst);8660 return -EIO;8661 }8662 }8663 8664 if (!is_zero_ether_addr(match.mask->src)) {8665 if (is_broadcast_ether_addr(match.mask->src)) {8666 field_flags |= I40E_CLOUD_FIELD_IMAC;8667 } else {8668 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n",8669 match.mask->src);8670 return -EIO;8671 }8672 }8673 ether_addr_copy(filter->dst_mac, match.key->dst);8674 ether_addr_copy(filter->src_mac, match.key->src);8675 }8676 8677 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {8678 struct flow_match_vlan match;8679 8680 flow_rule_match_vlan(rule, &match);8681 if (match.mask->vlan_id) {8682 if (match.mask->vlan_id == VLAN_VID_MASK) {8683 field_flags |= I40E_CLOUD_FIELD_IVLAN;8684 8685 } else {8686 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n",8687 match.mask->vlan_id);8688 return -EIO;8689 }8690 }8691 8692 filter->vlan_id = cpu_to_be16(match.key->vlan_id);8693 }8694 8695 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {8696 struct flow_match_control match;8697 8698 flow_rule_match_control(rule, &match);8699 addr_type = match.key->addr_type;8700 8701 if (flow_rule_has_control_flags(match.mask->flags,8702 f->common.extack))8703 return -EOPNOTSUPP;8704 }8705 8706 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {8707 struct flow_match_ipv4_addrs match;8708 8709 flow_rule_match_ipv4_addrs(rule, &match);8710 if (match.mask->dst) {8711 if (match.mask->dst == cpu_to_be32(0xffffffff)) {8712 field_flags |= I40E_CLOUD_FIELD_IIP;8713 } else {8714 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n",8715 &match.mask->dst);8716 return -EIO;8717 }8718 }8719 8720 if (match.mask->src) {8721 if (match.mask->src == cpu_to_be32(0xffffffff)) {8722 field_flags |= I40E_CLOUD_FIELD_IIP;8723 } else {8724 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n",8725 &match.mask->src);8726 return -EIO;8727 }8728 }8729 8730 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) {8731 dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n");8732 return -EIO;8733 }8734 filter->dst_ipv4 = match.key->dst;8735 filter->src_ipv4 = match.key->src;8736 }8737 8738 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {8739 struct flow_match_ipv6_addrs match;8740 8741 flow_rule_match_ipv6_addrs(rule, &match);8742 8743 /* src and dest IPV6 address should not be LOOPBACK8744 * (0:0:0:0:0:0:0:1), which can be represented as ::18745 */8746 if (ipv6_addr_loopback(&match.key->dst) ||8747 ipv6_addr_loopback(&match.key->src)) {8748 dev_err(&pf->pdev->dev,8749 "Bad ipv6, addr is LOOPBACK\n");8750 return -EIO;8751 }8752 if (!ipv6_addr_any(&match.mask->dst) ||8753 !ipv6_addr_any(&match.mask->src))8754 field_flags |= I40E_CLOUD_FIELD_IIP;8755 8756 memcpy(&filter->src_ipv6, &match.key->src.s6_addr32,8757 sizeof(filter->src_ipv6));8758 memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32,8759 sizeof(filter->dst_ipv6));8760 }8761 8762 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {8763 struct flow_match_ports match;8764 8765 flow_rule_match_ports(rule, &match);8766 if (match.mask->src) {8767 if (match.mask->src == cpu_to_be16(0xffff)) {8768 field_flags |= I40E_CLOUD_FIELD_IIP;8769 } else {8770 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n",8771 be16_to_cpu(match.mask->src));8772 return -EIO;8773 }8774 }8775 8776 if (match.mask->dst) {8777 if (match.mask->dst == cpu_to_be16(0xffff)) {8778 field_flags |= I40E_CLOUD_FIELD_IIP;8779 } else {8780 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n",8781 be16_to_cpu(match.mask->dst));8782 return -EIO;8783 }8784 }8785 8786 filter->dst_port = match.key->dst;8787 filter->src_port = match.key->src;8788 8789 switch (filter->ip_proto) {8790 case IPPROTO_TCP:8791 case IPPROTO_UDP:8792 break;8793 default:8794 dev_err(&pf->pdev->dev,8795 "Only UDP and TCP transport are supported\n");8796 return -EINVAL;8797 }8798 }8799 filter->flags = field_flags;8800 return 0;8801}8802 8803/**8804 * i40e_handle_tclass: Forward to a traffic class on the device8805 * @vsi: Pointer to VSI8806 * @tc: traffic class index on the device8807 * @filter: Pointer to cloud filter structure8808 *8809 **/8810static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc,8811 struct i40e_cloud_filter *filter)8812{8813 struct i40e_channel *ch, *ch_tmp;8814 8815 /* direct to a traffic class on the same device */8816 if (tc == 0) {8817 filter->seid = vsi->seid;8818 return 0;8819 } else if (vsi->tc_config.enabled_tc & BIT(tc)) {8820 if (!filter->dst_port) {8821 dev_err(&vsi->back->pdev->dev,8822 "Specify destination port to direct to traffic class that is not default\n");8823 return -EINVAL;8824 }8825 if (list_empty(&vsi->ch_list))8826 return -EINVAL;8827 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list,8828 list) {8829 if (ch->seid == vsi->tc_seid_map[tc])8830 filter->seid = ch->seid;8831 }8832 return 0;8833 }8834 dev_err(&vsi->back->pdev->dev, "TC is not enabled\n");8835 return -EINVAL;8836}8837 8838/**8839 * i40e_configure_clsflower - Configure tc flower filters8840 * @vsi: Pointer to VSI8841 * @cls_flower: Pointer to struct flow_cls_offload8842 *8843 **/8844static int i40e_configure_clsflower(struct i40e_vsi *vsi,8845 struct flow_cls_offload *cls_flower)8846{8847 int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid);8848 struct i40e_cloud_filter *filter = NULL;8849 struct i40e_pf *pf = vsi->back;8850 int err = 0;8851 8852 if (tc < 0) {8853 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n");8854 return -EOPNOTSUPP;8855 }8856 8857 if (!tc) {8858 dev_err(&pf->pdev->dev, "Unable to add filter because of invalid destination");8859 return -EINVAL;8860 }8861 8862 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||8863 test_bit(__I40E_RESET_INTR_RECEIVED, pf->state))8864 return -EBUSY;8865 8866 if (pf->fdir_pf_active_filters ||8867 (!hlist_empty(&pf->fdir_filter_list))) {8868 dev_err(&vsi->back->pdev->dev,8869 "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n");8870 return -EINVAL;8871 }8872 8873 if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags)) {8874 dev_err(&vsi->back->pdev->dev,8875 "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n");8876 clear_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags);8877 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, vsi->back->flags);8878 }8879 8880 filter = kzalloc(sizeof(*filter), GFP_KERNEL);8881 if (!filter)8882 return -ENOMEM;8883 8884 filter->cookie = cls_flower->cookie;8885 8886 err = i40e_parse_cls_flower(vsi, cls_flower, filter);8887 if (err < 0)8888 goto err;8889 8890 err = i40e_handle_tclass(vsi, tc, filter);8891 if (err < 0)8892 goto err;8893 8894 /* Add cloud filter */8895 if (filter->dst_port)8896 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true);8897 else8898 err = i40e_add_del_cloud_filter(vsi, filter, true);8899 8900 if (err) {8901 dev_err(&pf->pdev->dev, "Failed to add cloud filter, err %d\n",8902 err);8903 goto err;8904 }8905 8906 /* add filter to the ordered list */8907 INIT_HLIST_NODE(&filter->cloud_node);8908 8909 hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list);8910 8911 pf->num_cloud_filters++;8912 8913 return err;8914err:8915 kfree(filter);8916 return err;8917}8918 8919/**8920 * i40e_find_cloud_filter - Find the could filter in the list8921 * @vsi: Pointer to VSI8922 * @cookie: filter specific cookie8923 *8924 **/8925static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi,8926 unsigned long *cookie)8927{8928 struct i40e_cloud_filter *filter = NULL;8929 struct hlist_node *node2;8930 8931 hlist_for_each_entry_safe(filter, node2,8932 &vsi->back->cloud_filter_list, cloud_node)8933 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))8934 return filter;8935 return NULL;8936}8937 8938/**8939 * i40e_delete_clsflower - Remove tc flower filters8940 * @vsi: Pointer to VSI8941 * @cls_flower: Pointer to struct flow_cls_offload8942 *8943 **/8944static int i40e_delete_clsflower(struct i40e_vsi *vsi,8945 struct flow_cls_offload *cls_flower)8946{8947 struct i40e_cloud_filter *filter = NULL;8948 struct i40e_pf *pf = vsi->back;8949 int err = 0;8950 8951 filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie);8952 8953 if (!filter)8954 return -EINVAL;8955 8956 hash_del(&filter->cloud_node);8957 8958 if (filter->dst_port)8959 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false);8960 else8961 err = i40e_add_del_cloud_filter(vsi, filter, false);8962 8963 kfree(filter);8964 if (err) {8965 dev_err(&pf->pdev->dev,8966 "Failed to delete cloud filter, err %pe\n",8967 ERR_PTR(err));8968 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status);8969 }8970 8971 pf->num_cloud_filters--;8972 if (!pf->num_cloud_filters)8973 if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) &&8974 !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) {8975 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);8976 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags);8977 clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);8978 }8979 return 0;8980}8981 8982/**8983 * i40e_setup_tc_cls_flower - flower classifier offloads8984 * @np: net device to configure8985 * @cls_flower: offload data8986 **/8987static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np,8988 struct flow_cls_offload *cls_flower)8989{8990 struct i40e_vsi *vsi = np->vsi;8991 8992 switch (cls_flower->command) {8993 case FLOW_CLS_REPLACE:8994 return i40e_configure_clsflower(vsi, cls_flower);8995 case FLOW_CLS_DESTROY:8996 return i40e_delete_clsflower(vsi, cls_flower);8997 case FLOW_CLS_STATS:8998 return -EOPNOTSUPP;8999 default:9000 return -EOPNOTSUPP;9001 }9002}9003 9004static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,9005 void *cb_priv)9006{9007 struct i40e_netdev_priv *np = cb_priv;9008 9009 if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data))9010 return -EOPNOTSUPP;9011 9012 switch (type) {9013 case TC_SETUP_CLSFLOWER:9014 return i40e_setup_tc_cls_flower(np, type_data);9015 9016 default:9017 return -EOPNOTSUPP;9018 }9019}9020 9021static LIST_HEAD(i40e_block_cb_list);9022 9023static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,9024 void *type_data)9025{9026 struct i40e_netdev_priv *np = netdev_priv(netdev);9027 9028 switch (type) {9029 case TC_SETUP_QDISC_MQPRIO:9030 return i40e_setup_tc(netdev, type_data);9031 case TC_SETUP_BLOCK:9032 return flow_block_cb_setup_simple(type_data,9033 &i40e_block_cb_list,9034 i40e_setup_tc_block_cb,9035 np, np, true);9036 default:9037 return -EOPNOTSUPP;9038 }9039}9040 9041/**9042 * i40e_open - Called when a network interface is made active9043 * @netdev: network interface device structure9044 *9045 * The open entry point is called when a network interface is made9046 * active by the system (IFF_UP). At this point all resources needed9047 * for transmit and receive operations are allocated, the interrupt9048 * handler is registered with the OS, the netdev watchdog subtask is9049 * enabled, and the stack is notified that the interface is ready.9050 *9051 * Returns 0 on success, negative value on failure9052 **/9053int i40e_open(struct net_device *netdev)9054{9055 struct i40e_netdev_priv *np = netdev_priv(netdev);9056 struct i40e_vsi *vsi = np->vsi;9057 struct i40e_pf *pf = vsi->back;9058 int err;9059 9060 /* disallow open during test or if eeprom is broken */9061 if (test_bit(__I40E_TESTING, pf->state) ||9062 test_bit(__I40E_BAD_EEPROM, pf->state))9063 return -EBUSY;9064 9065 netif_carrier_off(netdev);9066 9067 if (i40e_force_link_state(pf, true))9068 return -EAGAIN;9069 9070 err = i40e_vsi_open(vsi);9071 if (err)9072 return err;9073 9074 /* configure global TSO hardware offload settings */9075 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |9076 TCP_FLAG_FIN) >> 16);9077 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |9078 TCP_FLAG_FIN |9079 TCP_FLAG_CWR) >> 16);9080 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);9081 udp_tunnel_get_rx_info(netdev);9082 9083 return 0;9084}9085 9086/**9087 * i40e_netif_set_realnum_tx_rx_queues - Update number of tx/rx queues9088 * @vsi: vsi structure9089 *9090 * This updates netdev's number of tx/rx queues9091 *9092 * Returns status of setting tx/rx queues9093 **/9094static int i40e_netif_set_realnum_tx_rx_queues(struct i40e_vsi *vsi)9095{9096 int ret;9097 9098 ret = netif_set_real_num_rx_queues(vsi->netdev,9099 vsi->num_queue_pairs);9100 if (ret)9101 return ret;9102 9103 return netif_set_real_num_tx_queues(vsi->netdev,9104 vsi->num_queue_pairs);9105}9106 9107/**9108 * i40e_vsi_open -9109 * @vsi: the VSI to open9110 *9111 * Finish initialization of the VSI.9112 *9113 * Returns 0 on success, negative value on failure9114 *9115 * Note: expects to be called while under rtnl_lock()9116 **/9117int i40e_vsi_open(struct i40e_vsi *vsi)9118{9119 struct i40e_pf *pf = vsi->back;9120 char int_name[I40E_INT_NAME_STR_LEN];9121 int err;9122 9123 /* allocate descriptors */9124 err = i40e_vsi_setup_tx_resources(vsi);9125 if (err)9126 goto err_setup_tx;9127 err = i40e_vsi_setup_rx_resources(vsi);9128 if (err)9129 goto err_setup_rx;9130 9131 err = i40e_vsi_configure(vsi);9132 if (err)9133 goto err_setup_rx;9134 9135 if (vsi->netdev) {9136 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",9137 dev_driver_string(&pf->pdev->dev), vsi->netdev->name);9138 err = i40e_vsi_request_irq(vsi, int_name);9139 if (err)9140 goto err_setup_rx;9141 9142 /* Notify the stack of the actual queue counts. */9143 err = i40e_netif_set_realnum_tx_rx_queues(vsi);9144 if (err)9145 goto err_set_queues;9146 9147 } else if (vsi->type == I40E_VSI_FDIR) {9148 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",9149 dev_driver_string(&pf->pdev->dev),9150 dev_name(&pf->pdev->dev));9151 err = i40e_vsi_request_irq(vsi, int_name);9152 if (err)9153 goto err_setup_rx;9154 9155 } else {9156 err = -EINVAL;9157 goto err_setup_rx;9158 }9159 9160 err = i40e_up_complete(vsi);9161 if (err)9162 goto err_up_complete;9163 9164 return 0;9165 9166err_up_complete:9167 i40e_down(vsi);9168err_set_queues:9169 i40e_vsi_free_irq(vsi);9170err_setup_rx:9171 i40e_vsi_free_rx_resources(vsi);9172err_setup_tx:9173 i40e_vsi_free_tx_resources(vsi);9174 if (vsi->type == I40E_VSI_MAIN)9175 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);9176 9177 return err;9178}9179 9180/**9181 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting9182 * @pf: Pointer to PF9183 *9184 * This function destroys the hlist where all the Flow Director9185 * filters were saved.9186 **/9187static void i40e_fdir_filter_exit(struct i40e_pf *pf)9188{9189 struct i40e_fdir_filter *filter;9190 struct i40e_flex_pit *pit_entry, *tmp;9191 struct hlist_node *node2;9192 9193 hlist_for_each_entry_safe(filter, node2,9194 &pf->fdir_filter_list, fdir_node) {9195 hlist_del(&filter->fdir_node);9196 kfree(filter);9197 }9198 9199 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {9200 list_del(&pit_entry->list);9201 kfree(pit_entry);9202 }9203 INIT_LIST_HEAD(&pf->l3_flex_pit_list);9204 9205 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {9206 list_del(&pit_entry->list);9207 kfree(pit_entry);9208 }9209 INIT_LIST_HEAD(&pf->l4_flex_pit_list);9210 9211 pf->fdir_pf_active_filters = 0;9212 i40e_reset_fdir_filter_cnt(pf);9213 9214 /* Reprogram the default input set for TCP/IPv4 */9215 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,9216 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |9217 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);9218 9219 /* Reprogram the default input set for TCP/IPv6 */9220 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_TCP,9221 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |9222 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);9223 9224 /* Reprogram the default input set for UDP/IPv4 */9225 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,9226 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |9227 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);9228 9229 /* Reprogram the default input set for UDP/IPv6 */9230 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_UDP,9231 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |9232 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);9233 9234 /* Reprogram the default input set for SCTP/IPv4 */9235 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,9236 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |9237 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);9238 9239 /* Reprogram the default input set for SCTP/IPv6 */9240 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_SCTP,9241 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |9242 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);9243 9244 /* Reprogram the default input set for Other/IPv4 */9245 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,9246 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);9247 9248 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4,9249 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);9250 9251 /* Reprogram the default input set for Other/IPv6 */9252 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_OTHER,9253 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);9254 9255 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV6,9256 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);9257}9258 9259/**9260 * i40e_cloud_filter_exit - Cleans up the cloud filters9261 * @pf: Pointer to PF9262 *9263 * This function destroys the hlist where all the cloud filters9264 * were saved.9265 **/9266static void i40e_cloud_filter_exit(struct i40e_pf *pf)9267{9268 struct i40e_cloud_filter *cfilter;9269 struct hlist_node *node;9270 9271 hlist_for_each_entry_safe(cfilter, node,9272 &pf->cloud_filter_list, cloud_node) {9273 hlist_del(&cfilter->cloud_node);9274 kfree(cfilter);9275 }9276 pf->num_cloud_filters = 0;9277 9278 if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) &&9279 !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) {9280 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);9281 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags);9282 clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);9283 }9284}9285 9286/**9287 * i40e_close - Disables a network interface9288 * @netdev: network interface device structure9289 *9290 * The close entry point is called when an interface is de-activated9291 * by the OS. The hardware is still under the driver's control, but9292 * this netdev interface is disabled.9293 *9294 * Returns 0, this is not allowed to fail9295 **/9296int i40e_close(struct net_device *netdev)9297{9298 struct i40e_netdev_priv *np = netdev_priv(netdev);9299 struct i40e_vsi *vsi = np->vsi;9300 9301 i40e_vsi_close(vsi);9302 9303 return 0;9304}9305 9306/**9307 * i40e_do_reset - Start a PF or Core Reset sequence9308 * @pf: board private structure9309 * @reset_flags: which reset is requested9310 * @lock_acquired: indicates whether or not the lock has been acquired9311 * before this function was called.9312 *9313 * The essential difference in resets is that the PF Reset9314 * doesn't clear the packet buffers, doesn't reset the PE9315 * firmware, and doesn't bother the other PFs on the chip.9316 **/9317void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)9318{9319 struct i40e_vsi *vsi;9320 u32 val;9321 int i;9322 9323 /* do the biggest reset indicated */9324 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {9325 9326 /* Request a Global Reset9327 *9328 * This will start the chip's countdown to the actual full9329 * chip reset event, and a warning interrupt to be sent9330 * to all PFs, including the requestor. Our handler9331 * for the warning interrupt will deal with the shutdown9332 * and recovery of the switch setup.9333 */9334 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");9335 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);9336 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;9337 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);9338 9339 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {9340 9341 /* Request a Core Reset9342 *9343 * Same as Global Reset, except does *not* include the MAC/PHY9344 */9345 dev_dbg(&pf->pdev->dev, "CoreR requested\n");9346 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);9347 val |= I40E_GLGEN_RTRIG_CORER_MASK;9348 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);9349 i40e_flush(&pf->hw);9350 9351 } else if (reset_flags & I40E_PF_RESET_FLAG) {9352 9353 /* Request a PF Reset9354 *9355 * Resets only the PF-specific registers9356 *9357 * This goes directly to the tear-down and rebuild of9358 * the switch, since we need to do all the recovery as9359 * for the Core Reset.9360 */9361 dev_dbg(&pf->pdev->dev, "PFR requested\n");9362 i40e_handle_reset_warning(pf, lock_acquired);9363 9364 } else if (reset_flags & I40E_PF_RESET_AND_REBUILD_FLAG) {9365 /* Request a PF Reset9366 *9367 * Resets PF and reinitializes PFs VSI.9368 */9369 i40e_prep_for_reset(pf);9370 i40e_reset_and_rebuild(pf, true, lock_acquired);9371 dev_info(&pf->pdev->dev,9372 test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ?9373 "FW LLDP is disabled\n" :9374 "FW LLDP is enabled\n");9375 9376 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {9377 /* Find the VSI(s) that requested a re-init */9378 dev_info(&pf->pdev->dev, "VSI reinit requested\n");9379 9380 i40e_pf_for_each_vsi(pf, i, vsi) {9381 if (test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,9382 vsi->state))9383 i40e_vsi_reinit_locked(vsi);9384 }9385 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {9386 /* Find the VSI(s) that needs to be brought down */9387 dev_info(&pf->pdev->dev, "VSI down requested\n");9388 9389 i40e_pf_for_each_vsi(pf, i, vsi) {9390 if (test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,9391 vsi->state)) {9392 set_bit(__I40E_VSI_DOWN, vsi->state);9393 i40e_down(vsi);9394 }9395 }9396 } else {9397 dev_info(&pf->pdev->dev,9398 "bad reset request 0x%08x\n", reset_flags);9399 }9400}9401 9402#ifdef CONFIG_I40E_DCB9403/**9404 * i40e_dcb_need_reconfig - Check if DCB needs reconfig9405 * @pf: board private structure9406 * @old_cfg: current DCB config9407 * @new_cfg: new DCB config9408 **/9409bool i40e_dcb_need_reconfig(struct i40e_pf *pf,9410 struct i40e_dcbx_config *old_cfg,9411 struct i40e_dcbx_config *new_cfg)9412{9413 bool need_reconfig = false;9414 9415 /* Check if ETS configuration has changed */9416 if (memcmp(&new_cfg->etscfg,9417 &old_cfg->etscfg,9418 sizeof(new_cfg->etscfg))) {9419 /* If Priority Table has changed reconfig is needed */9420 if (memcmp(&new_cfg->etscfg.prioritytable,9421 &old_cfg->etscfg.prioritytable,9422 sizeof(new_cfg->etscfg.prioritytable))) {9423 need_reconfig = true;9424 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");9425 }9426 9427 if (memcmp(&new_cfg->etscfg.tcbwtable,9428 &old_cfg->etscfg.tcbwtable,9429 sizeof(new_cfg->etscfg.tcbwtable)))9430 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");9431 9432 if (memcmp(&new_cfg->etscfg.tsatable,9433 &old_cfg->etscfg.tsatable,9434 sizeof(new_cfg->etscfg.tsatable)))9435 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");9436 }9437 9438 /* Check if PFC configuration has changed */9439 if (memcmp(&new_cfg->pfc,9440 &old_cfg->pfc,9441 sizeof(new_cfg->pfc))) {9442 need_reconfig = true;9443 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");9444 }9445 9446 /* Check if APP Table has changed */9447 if (memcmp(&new_cfg->app,9448 &old_cfg->app,9449 sizeof(new_cfg->app))) {9450 need_reconfig = true;9451 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");9452 }9453 9454 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);9455 return need_reconfig;9456}9457 9458/**9459 * i40e_handle_lldp_event - Handle LLDP Change MIB event9460 * @pf: board private structure9461 * @e: event info posted on ARQ9462 **/9463static int i40e_handle_lldp_event(struct i40e_pf *pf,9464 struct i40e_arq_event_info *e)9465{9466 struct i40e_aqc_lldp_get_mib *mib =9467 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;9468 struct i40e_hw *hw = &pf->hw;9469 struct i40e_dcbx_config tmp_dcbx_cfg;9470 bool need_reconfig = false;9471 int ret = 0;9472 u8 type;9473 9474 /* X710-T*L 2.5G and 5G speeds don't support DCB */9475 if (I40E_IS_X710TL_DEVICE(hw->device_id) &&9476 (hw->phy.link_info.link_speed &9477 ~(I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB)) &&9478 !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))9479 /* let firmware decide if the DCB should be disabled */9480 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);9481 9482 /* Not DCB capable or capability disabled */9483 if (!test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))9484 return ret;9485 9486 /* Ignore if event is not for Nearest Bridge */9487 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)9488 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);9489 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);9490 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)9491 return ret;9492 9493 /* Check MIB Type and return if event for Remote MIB update */9494 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;9495 dev_dbg(&pf->pdev->dev,9496 "LLDP event mib type %s\n", type ? "remote" : "local");9497 if (type == I40E_AQ_LLDP_MIB_REMOTE) {9498 /* Update the remote cached instance and return */9499 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,9500 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,9501 &hw->remote_dcbx_config);9502 goto exit;9503 }9504 9505 /* Store the old configuration */9506 tmp_dcbx_cfg = hw->local_dcbx_config;9507 9508 /* Reset the old DCBx configuration data */9509 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));9510 /* Get updated DCBX data from firmware */9511 ret = i40e_get_dcb_config(&pf->hw);9512 if (ret) {9513 /* X710-T*L 2.5G and 5G speeds don't support DCB */9514 if (I40E_IS_X710TL_DEVICE(hw->device_id) &&9515 (hw->phy.link_info.link_speed &9516 (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {9517 dev_warn(&pf->pdev->dev,9518 "DCB is not supported for X710-T*L 2.5/5G speeds\n");9519 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);9520 } else {9521 dev_info(&pf->pdev->dev,9522 "Failed querying DCB configuration data from firmware, err %pe aq_err %s\n",9523 ERR_PTR(ret),9524 i40e_aq_str(&pf->hw,9525 pf->hw.aq.asq_last_status));9526 }9527 goto exit;9528 }9529 9530 /* No change detected in DCBX configs */9531 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,9532 sizeof(tmp_dcbx_cfg))) {9533 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");9534 goto exit;9535 }9536 9537 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,9538 &hw->local_dcbx_config);9539 9540 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);9541 9542 if (!need_reconfig)9543 goto exit;9544 9545 /* Enable DCB tagging only when more than one TC */9546 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)9547 set_bit(I40E_FLAG_DCB_ENA, pf->flags);9548 else9549 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);9550 9551 set_bit(__I40E_PORT_SUSPENDED, pf->state);9552 /* Reconfiguration needed quiesce all VSIs */9553 i40e_pf_quiesce_all_vsi(pf);9554 9555 /* Changes in configuration update VEB/VSI */9556 i40e_dcb_reconfigure(pf);9557 9558 ret = i40e_resume_port_tx(pf);9559 9560 clear_bit(__I40E_PORT_SUSPENDED, pf->state);9561 /* In case of error no point in resuming VSIs */9562 if (ret)9563 goto exit;9564 9565 /* Wait for the PF's queues to be disabled */9566 ret = i40e_pf_wait_queues_disabled(pf);9567 if (ret) {9568 /* Schedule PF reset to recover */9569 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);9570 i40e_service_event_schedule(pf);9571 } else {9572 i40e_pf_unquiesce_all_vsi(pf);9573 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);9574 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);9575 }9576 9577exit:9578 return ret;9579}9580#endif /* CONFIG_I40E_DCB */9581 9582/**9583 * i40e_do_reset_safe - Protected reset path for userland calls.9584 * @pf: board private structure9585 * @reset_flags: which reset is requested9586 *9587 **/9588void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)9589{9590 rtnl_lock();9591 i40e_do_reset(pf, reset_flags, true);9592 rtnl_unlock();9593}9594 9595/**9596 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event9597 * @pf: board private structure9598 * @e: event info posted on ARQ9599 *9600 * Handler for LAN Queue Overflow Event generated by the firmware for PF9601 * and VF queues9602 **/9603static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,9604 struct i40e_arq_event_info *e)9605{9606 struct i40e_aqc_lan_overflow *data =9607 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;9608 u32 queue = le32_to_cpu(data->prtdcb_rupto);9609 u32 qtx_ctl = le32_to_cpu(data->otx_ctl);9610 struct i40e_hw *hw = &pf->hw;9611 struct i40e_vf *vf;9612 u16 vf_id;9613 9614 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",9615 queue, qtx_ctl);9616 9617 if (FIELD_GET(I40E_QTX_CTL_PFVF_Q_MASK, qtx_ctl) !=9618 I40E_QTX_CTL_VF_QUEUE)9619 return;9620 9621 /* Queue belongs to VF, find the VF and issue VF reset */9622 vf_id = FIELD_GET(I40E_QTX_CTL_VFVM_INDX_MASK, qtx_ctl);9623 vf_id -= hw->func_caps.vf_base_id;9624 vf = &pf->vf[vf_id];9625 i40e_vc_notify_vf_reset(vf);9626 /* Allow VF to process pending reset notification */9627 msleep(20);9628 i40e_reset_vf(vf, false);9629}9630 9631/**9632 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters9633 * @pf: board private structure9634 **/9635u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)9636{9637 u32 val, fcnt_prog;9638 9639 val = rd32(&pf->hw, I40E_PFQF_FDSTAT);9640 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);9641 return fcnt_prog;9642}9643 9644/**9645 * i40e_get_current_fd_count - Get total FD filters programmed for this PF9646 * @pf: board private structure9647 **/9648u32 i40e_get_current_fd_count(struct i40e_pf *pf)9649{9650 u32 val, fcnt_prog;9651 9652 val = rd32(&pf->hw, I40E_PFQF_FDSTAT);9653 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +9654 FIELD_GET(I40E_PFQF_FDSTAT_BEST_CNT_MASK, val);9655 return fcnt_prog;9656}9657 9658/**9659 * i40e_get_global_fd_count - Get total FD filters programmed on device9660 * @pf: board private structure9661 **/9662u32 i40e_get_global_fd_count(struct i40e_pf *pf)9663{9664 u32 val, fcnt_prog;9665 9666 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);9667 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +9668 FIELD_GET(I40E_GLQF_FDCNT_0_BESTCNT_MASK, val);9669 return fcnt_prog;9670}9671 9672/**9673 * i40e_reenable_fdir_sb - Restore FDir SB capability9674 * @pf: board private structure9675 **/9676static void i40e_reenable_fdir_sb(struct i40e_pf *pf)9677{9678 if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))9679 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&9680 (I40E_DEBUG_FD & pf->hw.debug_mask))9681 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");9682}9683 9684/**9685 * i40e_reenable_fdir_atr - Restore FDir ATR capability9686 * @pf: board private structure9687 **/9688static void i40e_reenable_fdir_atr(struct i40e_pf *pf)9689{9690 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) {9691 /* ATR uses the same filtering logic as SB rules. It only9692 * functions properly if the input set mask is at the default9693 * settings. It is safe to restore the default input set9694 * because there are no active TCPv4 filter rules.9695 */9696 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,9697 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |9698 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);9699 9700 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&9701 (I40E_DEBUG_FD & pf->hw.debug_mask))9702 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");9703 }9704}9705 9706/**9707 * i40e_delete_invalid_filter - Delete an invalid FDIR filter9708 * @pf: board private structure9709 * @filter: FDir filter to remove9710 */9711static void i40e_delete_invalid_filter(struct i40e_pf *pf,9712 struct i40e_fdir_filter *filter)9713{9714 /* Update counters */9715 pf->fdir_pf_active_filters--;9716 pf->fd_inv = 0;9717 9718 switch (filter->flow_type) {9719 case TCP_V4_FLOW:9720 pf->fd_tcp4_filter_cnt--;9721 break;9722 case UDP_V4_FLOW:9723 pf->fd_udp4_filter_cnt--;9724 break;9725 case SCTP_V4_FLOW:9726 pf->fd_sctp4_filter_cnt--;9727 break;9728 case TCP_V6_FLOW:9729 pf->fd_tcp6_filter_cnt--;9730 break;9731 case UDP_V6_FLOW:9732 pf->fd_udp6_filter_cnt--;9733 break;9734 case SCTP_V6_FLOW:9735 pf->fd_udp6_filter_cnt--;9736 break;9737 case IP_USER_FLOW:9738 switch (filter->ipl4_proto) {9739 case IPPROTO_TCP:9740 pf->fd_tcp4_filter_cnt--;9741 break;9742 case IPPROTO_UDP:9743 pf->fd_udp4_filter_cnt--;9744 break;9745 case IPPROTO_SCTP:9746 pf->fd_sctp4_filter_cnt--;9747 break;9748 case IPPROTO_IP:9749 pf->fd_ip4_filter_cnt--;9750 break;9751 }9752 break;9753 case IPV6_USER_FLOW:9754 switch (filter->ipl4_proto) {9755 case IPPROTO_TCP:9756 pf->fd_tcp6_filter_cnt--;9757 break;9758 case IPPROTO_UDP:9759 pf->fd_udp6_filter_cnt--;9760 break;9761 case IPPROTO_SCTP:9762 pf->fd_sctp6_filter_cnt--;9763 break;9764 case IPPROTO_IP:9765 pf->fd_ip6_filter_cnt--;9766 break;9767 }9768 break;9769 }9770 9771 /* Remove the filter from the list and free memory */9772 hlist_del(&filter->fdir_node);9773 kfree(filter);9774}9775 9776/**9777 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled9778 * @pf: board private structure9779 **/9780void i40e_fdir_check_and_reenable(struct i40e_pf *pf)9781{9782 struct i40e_fdir_filter *filter;9783 u32 fcnt_prog, fcnt_avail;9784 struct hlist_node *node;9785 9786 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))9787 return;9788 9789 /* Check if we have enough room to re-enable FDir SB capability. */9790 fcnt_prog = i40e_get_global_fd_count(pf);9791 fcnt_avail = pf->fdir_pf_filter_count;9792 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||9793 (pf->fd_add_err == 0) ||9794 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt))9795 i40e_reenable_fdir_sb(pf);9796 9797 /* We should wait for even more space before re-enabling ATR.9798 * Additionally, we cannot enable ATR as long as we still have TCP SB9799 * rules active.9800 */9801 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&9802 pf->fd_tcp4_filter_cnt == 0 && pf->fd_tcp6_filter_cnt == 0)9803 i40e_reenable_fdir_atr(pf);9804 9805 /* if hw had a problem adding a filter, delete it */9806 if (pf->fd_inv > 0) {9807 hlist_for_each_entry_safe(filter, node,9808 &pf->fdir_filter_list, fdir_node)9809 if (filter->fd_id == pf->fd_inv)9810 i40e_delete_invalid_filter(pf, filter);9811 }9812}9813 9814#define I40E_MIN_FD_FLUSH_INTERVAL 109815#define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 309816/**9817 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB9818 * @pf: board private structure9819 **/9820static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)9821{9822 unsigned long min_flush_time;9823 int flush_wait_retry = 50;9824 bool disable_atr = false;9825 int fd_room;9826 int reg;9827 9828 if (!time_after(jiffies, pf->fd_flush_timestamp +9829 (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))9830 return;9831 9832 /* If the flush is happening too quick and we have mostly SB rules we9833 * should not re-enable ATR for some time.9834 */9835 min_flush_time = pf->fd_flush_timestamp +9836 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);9837 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;9838 9839 if (!(time_after(jiffies, min_flush_time)) &&9840 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {9841 if (I40E_DEBUG_FD & pf->hw.debug_mask)9842 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");9843 disable_atr = true;9844 }9845 9846 pf->fd_flush_timestamp = jiffies;9847 set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);9848 /* flush all filters */9849 wr32(&pf->hw, I40E_PFQF_CTL_1,9850 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);9851 i40e_flush(&pf->hw);9852 pf->fd_flush_cnt++;9853 pf->fd_add_err = 0;9854 do {9855 /* Check FD flush status every 5-6msec */9856 usleep_range(5000, 6000);9857 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);9858 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))9859 break;9860 } while (flush_wait_retry--);9861 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {9862 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");9863 } else {9864 /* replay sideband filters */9865 i40e_fdir_filter_restore(i40e_pf_get_main_vsi(pf));9866 if (!disable_atr && !pf->fd_tcp4_filter_cnt)9867 clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);9868 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);9869 if (I40E_DEBUG_FD & pf->hw.debug_mask)9870 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");9871 }9872}9873 9874/**9875 * i40e_get_current_atr_cnt - Get the count of total FD ATR filters programmed9876 * @pf: board private structure9877 **/9878u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)9879{9880 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;9881}9882 9883/**9884 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table9885 * @pf: board private structure9886 **/9887static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)9888{9889 9890 /* if interface is down do nothing */9891 if (test_bit(__I40E_DOWN, pf->state))9892 return;9893 9894 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))9895 i40e_fdir_flush_and_replay(pf);9896 9897 i40e_fdir_check_and_reenable(pf);9898 9899}9900 9901/**9902 * i40e_vsi_link_event - notify VSI of a link event9903 * @vsi: vsi to be notified9904 * @link_up: link up or down9905 **/9906static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)9907{9908 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))9909 return;9910 9911 switch (vsi->type) {9912 case I40E_VSI_MAIN:9913 if (!vsi->netdev || !vsi->netdev_registered)9914 break;9915 9916 if (link_up) {9917 netif_carrier_on(vsi->netdev);9918 netif_tx_wake_all_queues(vsi->netdev);9919 } else {9920 netif_carrier_off(vsi->netdev);9921 netif_tx_stop_all_queues(vsi->netdev);9922 }9923 break;9924 9925 case I40E_VSI_SRIOV:9926 case I40E_VSI_VMDQ2:9927 case I40E_VSI_CTRL:9928 case I40E_VSI_IWARP:9929 case I40E_VSI_MIRROR:9930 default:9931 /* there is no notification for other VSIs */9932 break;9933 }9934}9935 9936/**9937 * i40e_veb_link_event - notify elements on the veb of a link event9938 * @veb: veb to be notified9939 * @link_up: link up or down9940 **/9941static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)9942{9943 struct i40e_vsi *vsi;9944 struct i40e_pf *pf;9945 int i;9946 9947 if (!veb || !veb->pf)9948 return;9949 pf = veb->pf;9950 9951 /* Send link event to contained VSIs */9952 i40e_pf_for_each_vsi(pf, i, vsi)9953 if (vsi->uplink_seid == veb->seid)9954 i40e_vsi_link_event(vsi, link_up);9955}9956 9957/**9958 * i40e_link_event - Update netif_carrier status9959 * @pf: board private structure9960 **/9961static void i40e_link_event(struct i40e_pf *pf)9962{9963 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);9964 struct i40e_veb *veb = i40e_pf_get_main_veb(pf);9965 u8 new_link_speed, old_link_speed;9966 bool new_link, old_link;9967 int status;9968#ifdef CONFIG_I40E_DCB9969 int err;9970#endif /* CONFIG_I40E_DCB */9971 9972 /* set this to force the get_link_status call to refresh state */9973 pf->hw.phy.get_link_info = true;9974 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);9975 status = i40e_get_link_status(&pf->hw, &new_link);9976 9977 /* On success, disable temp link polling */9978 if (status == 0) {9979 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state);9980 } else {9981 /* Enable link polling temporarily until i40e_get_link_status9982 * returns 09983 */9984 set_bit(__I40E_TEMP_LINK_POLLING, pf->state);9985 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",9986 status);9987 return;9988 }9989 9990 old_link_speed = pf->hw.phy.link_info_old.link_speed;9991 new_link_speed = pf->hw.phy.link_info.link_speed;9992 9993 if (new_link == old_link &&9994 new_link_speed == old_link_speed &&9995 (test_bit(__I40E_VSI_DOWN, vsi->state) ||9996 new_link == netif_carrier_ok(vsi->netdev)))9997 return;9998 9999 i40e_print_link_message(vsi, new_link);10000 10001 /* Notify the base of the switch tree connected to10002 * the link. Floating VEBs are not notified.10003 */10004 if (veb)10005 i40e_veb_link_event(veb, new_link);10006 else10007 i40e_vsi_link_event(vsi, new_link);10008 10009 if (pf->vf)10010 i40e_vc_notify_link_state(pf);10011 10012 if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))10013 i40e_ptp_set_increment(pf);10014#ifdef CONFIG_I40E_DCB10015 if (new_link == old_link)10016 return;10017 /* Not SW DCB so firmware will take care of default settings */10018 if (pf->dcbx_cap & DCB_CAP_DCBX_LLD_MANAGED)10019 return;10020 10021 /* We cover here only link down, as after link up in case of SW DCB10022 * SW LLDP agent will take care of setting it up10023 */10024 if (!new_link) {10025 dev_dbg(&pf->pdev->dev, "Reconfig DCB to single TC as result of Link Down\n");10026 memset(&pf->tmp_cfg, 0, sizeof(pf->tmp_cfg));10027 err = i40e_dcb_sw_default_config(pf);10028 if (err) {10029 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);10030 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);10031 } else {10032 pf->dcbx_cap = DCB_CAP_DCBX_HOST |10033 DCB_CAP_DCBX_VER_IEEE;10034 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);10035 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);10036 }10037 }10038#endif /* CONFIG_I40E_DCB */10039}10040 10041/**10042 * i40e_watchdog_subtask - periodic checks not using event driven response10043 * @pf: board private structure10044 **/10045static void i40e_watchdog_subtask(struct i40e_pf *pf)10046{10047 struct i40e_vsi *vsi;10048 struct i40e_veb *veb;10049 int i;10050 10051 /* if interface is down do nothing */10052 if (test_bit(__I40E_DOWN, pf->state) ||10053 test_bit(__I40E_CONFIG_BUSY, pf->state))10054 return;10055 10056 /* make sure we don't do these things too often */10057 if (time_before(jiffies, (pf->service_timer_previous +10058 pf->service_timer_period)))10059 return;10060 pf->service_timer_previous = jiffies;10061 10062 if (test_bit(I40E_FLAG_LINK_POLLING_ENA, pf->flags) ||10063 test_bit(__I40E_TEMP_LINK_POLLING, pf->state))10064 i40e_link_event(pf);10065 10066 /* Update the stats for active netdevs so the network stack10067 * can look at updated numbers whenever it cares to10068 */10069 i40e_pf_for_each_vsi(pf, i, vsi)10070 if (vsi->netdev)10071 i40e_update_stats(vsi);10072 10073 if (test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags)) {10074 /* Update the stats for the active switching components */10075 i40e_pf_for_each_veb(pf, i, veb)10076 i40e_update_veb_stats(veb);10077 }10078 10079 i40e_ptp_rx_hang(pf);10080 i40e_ptp_tx_hang(pf);10081}10082 10083/**10084 * i40e_reset_subtask - Set up for resetting the device and driver10085 * @pf: board private structure10086 **/10087static void i40e_reset_subtask(struct i40e_pf *pf)10088{10089 u32 reset_flags = 0;10090 10091 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {10092 reset_flags |= BIT(__I40E_REINIT_REQUESTED);10093 clear_bit(__I40E_REINIT_REQUESTED, pf->state);10094 }10095 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {10096 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);10097 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);10098 }10099 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {10100 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);10101 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);10102 }10103 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {10104 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);10105 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);10106 }10107 if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {10108 reset_flags |= BIT(__I40E_DOWN_REQUESTED);10109 clear_bit(__I40E_DOWN_REQUESTED, pf->state);10110 }10111 10112 /* If there's a recovery already waiting, it takes10113 * precedence before starting a new reset sequence.10114 */10115 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {10116 i40e_prep_for_reset(pf);10117 i40e_reset(pf);10118 i40e_rebuild(pf, false, false);10119 }10120 10121 /* If we're already down or resetting, just bail */10122 if (reset_flags &&10123 !test_bit(__I40E_DOWN, pf->state) &&10124 !test_bit(__I40E_CONFIG_BUSY, pf->state)) {10125 i40e_do_reset(pf, reset_flags, false);10126 }10127}10128 10129/**10130 * i40e_handle_link_event - Handle link event10131 * @pf: board private structure10132 * @e: event info posted on ARQ10133 **/10134static void i40e_handle_link_event(struct i40e_pf *pf,10135 struct i40e_arq_event_info *e)10136{10137 struct i40e_aqc_get_link_status *status =10138 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;10139 10140 /* Do a new status request to re-enable LSE reporting10141 * and load new status information into the hw struct10142 * This completely ignores any state information10143 * in the ARQ event info, instead choosing to always10144 * issue the AQ update link status command.10145 */10146 i40e_link_event(pf);10147 10148 /* Check if module meets thermal requirements */10149 if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) {10150 dev_err(&pf->pdev->dev,10151 "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n");10152 dev_err(&pf->pdev->dev,10153 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");10154 } else {10155 /* check for unqualified module, if link is down, suppress10156 * the message if link was forced to be down.10157 */10158 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&10159 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&10160 (!(status->link_info & I40E_AQ_LINK_UP)) &&10161 (!test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags))) {10162 dev_err(&pf->pdev->dev,10163 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n");10164 dev_err(&pf->pdev->dev,10165 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");10166 }10167 }10168}10169 10170/**10171 * i40e_clean_adminq_subtask - Clean the AdminQ rings10172 * @pf: board private structure10173 **/10174static void i40e_clean_adminq_subtask(struct i40e_pf *pf)10175{10176 struct i40e_arq_event_info event;10177 struct i40e_hw *hw = &pf->hw;10178 u16 pending, i = 0;10179 u16 opcode;10180 u32 oldval;10181 int ret;10182 u32 val;10183 10184 /* Do not run clean AQ when PF reset fails */10185 if (test_bit(__I40E_RESET_FAILED, pf->state))10186 return;10187 10188 /* check for error indications */10189 val = rd32(&pf->hw, I40E_PF_ARQLEN);10190 oldval = val;10191 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {10192 if (hw->debug_mask & I40E_DEBUG_AQ)10193 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");10194 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;10195 }10196 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {10197 if (hw->debug_mask & I40E_DEBUG_AQ)10198 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");10199 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;10200 pf->arq_overflows++;10201 }10202 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {10203 if (hw->debug_mask & I40E_DEBUG_AQ)10204 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");10205 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;10206 }10207 if (oldval != val)10208 wr32(&pf->hw, I40E_PF_ARQLEN, val);10209 10210 val = rd32(&pf->hw, I40E_PF_ATQLEN);10211 oldval = val;10212 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {10213 if (pf->hw.debug_mask & I40E_DEBUG_AQ)10214 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");10215 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;10216 }10217 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {10218 if (pf->hw.debug_mask & I40E_DEBUG_AQ)10219 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");10220 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;10221 }10222 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {10223 if (pf->hw.debug_mask & I40E_DEBUG_AQ)10224 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");10225 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;10226 }10227 if (oldval != val)10228 wr32(&pf->hw, I40E_PF_ATQLEN, val);10229 10230 event.buf_len = I40E_MAX_AQ_BUF_SIZE;10231 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);10232 if (!event.msg_buf)10233 return;10234 10235 do {10236 ret = i40e_clean_arq_element(hw, &event, &pending);10237 if (ret == -EALREADY)10238 break;10239 else if (ret) {10240 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);10241 break;10242 }10243 10244 opcode = le16_to_cpu(event.desc.opcode);10245 switch (opcode) {10246 10247 case i40e_aqc_opc_get_link_status:10248 rtnl_lock();10249 i40e_handle_link_event(pf, &event);10250 rtnl_unlock();10251 break;10252 case i40e_aqc_opc_send_msg_to_pf:10253 ret = i40e_vc_process_vf_msg(pf,10254 le16_to_cpu(event.desc.retval),10255 le32_to_cpu(event.desc.cookie_high),10256 le32_to_cpu(event.desc.cookie_low),10257 event.msg_buf,10258 event.msg_len);10259 break;10260 case i40e_aqc_opc_lldp_update_mib:10261 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");10262#ifdef CONFIG_I40E_DCB10263 rtnl_lock();10264 i40e_handle_lldp_event(pf, &event);10265 rtnl_unlock();10266#endif /* CONFIG_I40E_DCB */10267 break;10268 case i40e_aqc_opc_event_lan_overflow:10269 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");10270 i40e_handle_lan_overflow_event(pf, &event);10271 break;10272 case i40e_aqc_opc_send_msg_to_peer:10273 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");10274 break;10275 case i40e_aqc_opc_nvm_erase:10276 case i40e_aqc_opc_nvm_update:10277 case i40e_aqc_opc_oem_post_update:10278 i40e_debug(&pf->hw, I40E_DEBUG_NVM,10279 "ARQ NVM operation 0x%04x completed\n",10280 opcode);10281 break;10282 default:10283 dev_info(&pf->pdev->dev,10284 "ARQ: Unknown event 0x%04x ignored\n",10285 opcode);10286 break;10287 }10288 } while (i++ < I40E_AQ_WORK_LIMIT);10289 10290 if (i < I40E_AQ_WORK_LIMIT)10291 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);10292 10293 /* re-enable Admin queue interrupt cause */10294 val = rd32(hw, I40E_PFINT_ICR0_ENA);10295 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK;10296 wr32(hw, I40E_PFINT_ICR0_ENA, val);10297 i40e_flush(hw);10298 10299 kfree(event.msg_buf);10300}10301 10302/**10303 * i40e_verify_eeprom - make sure eeprom is good to use10304 * @pf: board private structure10305 **/10306static void i40e_verify_eeprom(struct i40e_pf *pf)10307{10308 int err;10309 10310 err = i40e_diag_eeprom_test(&pf->hw);10311 if (err) {10312 /* retry in case of garbage read */10313 err = i40e_diag_eeprom_test(&pf->hw);10314 if (err) {10315 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",10316 err);10317 set_bit(__I40E_BAD_EEPROM, pf->state);10318 }10319 }10320 10321 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {10322 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");10323 clear_bit(__I40E_BAD_EEPROM, pf->state);10324 }10325}10326 10327/**10328 * i40e_enable_pf_switch_lb10329 * @pf: pointer to the PF structure10330 *10331 * enable switch loop back or die - no point in a return value10332 **/10333static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)10334{10335 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);10336 struct i40e_vsi_context ctxt;10337 int ret;10338 10339 ctxt.seid = pf->main_vsi_seid;10340 ctxt.pf_num = pf->hw.pf_id;10341 ctxt.vf_num = 0;10342 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);10343 if (ret) {10344 dev_info(&pf->pdev->dev,10345 "couldn't get PF vsi config, err %pe aq_err %s\n",10346 ERR_PTR(ret),10347 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));10348 return;10349 }10350 ctxt.flags = I40E_AQ_VSI_TYPE_PF;10351 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);10352 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);10353 10354 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);10355 if (ret) {10356 dev_info(&pf->pdev->dev,10357 "update vsi switch failed, err %pe aq_err %s\n",10358 ERR_PTR(ret),10359 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));10360 }10361}10362 10363/**10364 * i40e_disable_pf_switch_lb10365 * @pf: pointer to the PF structure10366 *10367 * disable switch loop back or die - no point in a return value10368 **/10369static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)10370{10371 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);10372 struct i40e_vsi_context ctxt;10373 int ret;10374 10375 ctxt.seid = pf->main_vsi_seid;10376 ctxt.pf_num = pf->hw.pf_id;10377 ctxt.vf_num = 0;10378 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);10379 if (ret) {10380 dev_info(&pf->pdev->dev,10381 "couldn't get PF vsi config, err %pe aq_err %s\n",10382 ERR_PTR(ret),10383 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));10384 return;10385 }10386 ctxt.flags = I40E_AQ_VSI_TYPE_PF;10387 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);10388 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);10389 10390 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);10391 if (ret) {10392 dev_info(&pf->pdev->dev,10393 "update vsi switch failed, err %pe aq_err %s\n",10394 ERR_PTR(ret),10395 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));10396 }10397}10398 10399/**10400 * i40e_config_bridge_mode - Configure the HW bridge mode10401 * @veb: pointer to the bridge instance10402 *10403 * Configure the loop back mode for the LAN VSI that is downlink to the10404 * specified HW bridge instance. It is expected this function is called10405 * when a new HW bridge is instantiated.10406 **/10407static void i40e_config_bridge_mode(struct i40e_veb *veb)10408{10409 struct i40e_pf *pf = veb->pf;10410 10411 if (pf->hw.debug_mask & I40E_DEBUG_LAN)10412 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",10413 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");10414 if (veb->bridge_mode & BRIDGE_MODE_VEPA)10415 i40e_disable_pf_switch_lb(pf);10416 else10417 i40e_enable_pf_switch_lb(pf);10418}10419 10420/**10421 * i40e_reconstitute_veb - rebuild the VEB and VSIs connected to it10422 * @veb: pointer to the VEB instance10423 *10424 * This is a function that builds the attached VSIs. We track the connections10425 * through our own index numbers because the seid's from the HW could change10426 * across the reset.10427 **/10428static int i40e_reconstitute_veb(struct i40e_veb *veb)10429{10430 struct i40e_vsi *ctl_vsi = NULL;10431 struct i40e_pf *pf = veb->pf;10432 struct i40e_vsi *vsi;10433 int v, ret;10434 10435 /* As we do not maintain PV (port virtualizer) switch element then10436 * there can be only one non-floating VEB that have uplink to MAC SEID10437 * and its control VSI is the main one.10438 */10439 if (WARN_ON(veb->uplink_seid && veb->uplink_seid != pf->mac_seid)) {10440 dev_err(&pf->pdev->dev,10441 "Invalid uplink SEID for VEB %d\n", veb->idx);10442 return -ENOENT;10443 }10444 10445 if (veb->uplink_seid == pf->mac_seid) {10446 /* Check that the LAN VSI has VEB owning flag set */10447 ctl_vsi = i40e_pf_get_main_vsi(pf);10448 10449 if (WARN_ON(ctl_vsi->veb_idx != veb->idx ||10450 !(ctl_vsi->flags & I40E_VSI_FLAG_VEB_OWNER))) {10451 dev_err(&pf->pdev->dev,10452 "Invalid control VSI for VEB %d\n", veb->idx);10453 return -ENOENT;10454 }10455 10456 /* Add the control VSI to switch */10457 ret = i40e_add_vsi(ctl_vsi);10458 if (ret) {10459 dev_err(&pf->pdev->dev,10460 "Rebuild of owner VSI for VEB %d failed: %d\n",10461 veb->idx, ret);10462 return ret;10463 }10464 10465 i40e_vsi_reset_stats(ctl_vsi);10466 }10467 10468 /* create the VEB in the switch and move the VSI onto the VEB */10469 ret = i40e_add_veb(veb, ctl_vsi);10470 if (ret)10471 return ret;10472 10473 if (veb->uplink_seid) {10474 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags))10475 veb->bridge_mode = BRIDGE_MODE_VEB;10476 else10477 veb->bridge_mode = BRIDGE_MODE_VEPA;10478 i40e_config_bridge_mode(veb);10479 }10480 10481 /* create the remaining VSIs attached to this VEB */10482 i40e_pf_for_each_vsi(pf, v, vsi) {10483 if (vsi == ctl_vsi)10484 continue;10485 10486 if (vsi->veb_idx == veb->idx) {10487 vsi->uplink_seid = veb->seid;10488 ret = i40e_add_vsi(vsi);10489 if (ret) {10490 dev_info(&pf->pdev->dev,10491 "rebuild of vsi_idx %d failed: %d\n",10492 v, ret);10493 return ret;10494 }10495 i40e_vsi_reset_stats(vsi);10496 }10497 }10498 10499 return ret;10500}10501 10502/**10503 * i40e_get_capabilities - get info about the HW10504 * @pf: the PF struct10505 * @list_type: AQ capability to be queried10506 **/10507static int i40e_get_capabilities(struct i40e_pf *pf,10508 enum i40e_admin_queue_opc list_type)10509{10510 struct i40e_aqc_list_capabilities_element_resp *cap_buf;10511 u16 data_size;10512 int buf_len;10513 int err;10514 10515 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);10516 do {10517 cap_buf = kzalloc(buf_len, GFP_KERNEL);10518 if (!cap_buf)10519 return -ENOMEM;10520 10521 /* this loads the data into the hw struct for us */10522 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,10523 &data_size, list_type,10524 NULL);10525 /* data loaded, buffer no longer needed */10526 kfree(cap_buf);10527 10528 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {10529 /* retry with a larger buffer */10530 buf_len = data_size;10531 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK || err) {10532 dev_info(&pf->pdev->dev,10533 "capability discovery failed, err %pe aq_err %s\n",10534 ERR_PTR(err),10535 i40e_aq_str(&pf->hw,10536 pf->hw.aq.asq_last_status));10537 return -ENODEV;10538 }10539 } while (err);10540 10541 if (pf->hw.debug_mask & I40E_DEBUG_USER) {10542 if (list_type == i40e_aqc_opc_list_func_capabilities) {10543 dev_info(&pf->pdev->dev,10544 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",10545 pf->hw.pf_id, pf->hw.func_caps.num_vfs,10546 pf->hw.func_caps.num_msix_vectors,10547 pf->hw.func_caps.num_msix_vectors_vf,10548 pf->hw.func_caps.fd_filters_guaranteed,10549 pf->hw.func_caps.fd_filters_best_effort,10550 pf->hw.func_caps.num_tx_qp,10551 pf->hw.func_caps.num_vsis);10552 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) {10553 dev_info(&pf->pdev->dev,10554 "switch_mode=0x%04x, function_valid=0x%08x\n",10555 pf->hw.dev_caps.switch_mode,10556 pf->hw.dev_caps.valid_functions);10557 dev_info(&pf->pdev->dev,10558 "SR-IOV=%d, num_vfs for all function=%u\n",10559 pf->hw.dev_caps.sr_iov_1_1,10560 pf->hw.dev_caps.num_vfs);10561 dev_info(&pf->pdev->dev,10562 "num_vsis=%u, num_rx:%u, num_tx=%u\n",10563 pf->hw.dev_caps.num_vsis,10564 pf->hw.dev_caps.num_rx_qp,10565 pf->hw.dev_caps.num_tx_qp);10566 }10567 }10568 if (list_type == i40e_aqc_opc_list_func_capabilities) {10569#define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \10570 + pf->hw.func_caps.num_vfs)10571 if (pf->hw.revision_id == 0 &&10572 pf->hw.func_caps.num_vsis < DEF_NUM_VSI) {10573 dev_info(&pf->pdev->dev,10574 "got num_vsis %d, setting num_vsis to %d\n",10575 pf->hw.func_caps.num_vsis, DEF_NUM_VSI);10576 pf->hw.func_caps.num_vsis = DEF_NUM_VSI;10577 }10578 }10579 return 0;10580}10581 10582static int i40e_vsi_clear(struct i40e_vsi *vsi);10583 10584/**10585 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband10586 * @pf: board private structure10587 **/10588static void i40e_fdir_sb_setup(struct i40e_pf *pf)10589{10590 struct i40e_vsi *main_vsi, *vsi;10591 10592 /* quick workaround for an NVM issue that leaves a critical register10593 * uninitialized10594 */10595 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {10596 static const u32 hkey[] = {10597 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,10598 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,10599 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,10600 0x95b3a76d};10601 int i;10602 10603 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)10604 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);10605 }10606 10607 if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))10608 return;10609 10610 /* find existing VSI and see if it needs configuring */10611 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);10612 10613 /* create a new VSI if none exists */10614 if (!vsi) {10615 main_vsi = i40e_pf_get_main_vsi(pf);10616 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, main_vsi->seid, 0);10617 if (!vsi) {10618 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");10619 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);10620 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);10621 return;10622 }10623 }10624 10625 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);10626}10627 10628/**10629 * i40e_fdir_teardown - release the Flow Director resources10630 * @pf: board private structure10631 **/10632static void i40e_fdir_teardown(struct i40e_pf *pf)10633{10634 struct i40e_vsi *vsi;10635 10636 i40e_fdir_filter_exit(pf);10637 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);10638 if (vsi)10639 i40e_vsi_release(vsi);10640}10641 10642/**10643 * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs10644 * @vsi: PF main vsi10645 * @seid: seid of main or channel VSIs10646 *10647 * Rebuilds cloud filters associated with main VSI and channel VSIs if they10648 * existed before reset10649 **/10650static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid)10651{10652 struct i40e_cloud_filter *cfilter;10653 struct i40e_pf *pf = vsi->back;10654 struct hlist_node *node;10655 int ret;10656 10657 /* Add cloud filters back if they exist */10658 hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list,10659 cloud_node) {10660 if (cfilter->seid != seid)10661 continue;10662 10663 if (cfilter->dst_port)10664 ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter,10665 true);10666 else10667 ret = i40e_add_del_cloud_filter(vsi, cfilter, true);10668 10669 if (ret) {10670 dev_dbg(&pf->pdev->dev,10671 "Failed to rebuild cloud filter, err %pe aq_err %s\n",10672 ERR_PTR(ret),10673 i40e_aq_str(&pf->hw,10674 pf->hw.aq.asq_last_status));10675 return ret;10676 }10677 }10678 return 0;10679}10680 10681/**10682 * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset10683 * @vsi: PF main vsi10684 *10685 * Rebuilds channel VSIs if they existed before reset10686 **/10687static int i40e_rebuild_channels(struct i40e_vsi *vsi)10688{10689 struct i40e_channel *ch, *ch_tmp;10690 int ret;10691 10692 if (list_empty(&vsi->ch_list))10693 return 0;10694 10695 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {10696 if (!ch->initialized)10697 break;10698 /* Proceed with creation of channel (VMDq2) VSI */10699 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch);10700 if (ret) {10701 dev_info(&vsi->back->pdev->dev,10702 "failed to rebuild channels using uplink_seid %u\n",10703 vsi->uplink_seid);10704 return ret;10705 }10706 /* Reconfigure TX queues using QTX_CTL register */10707 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch);10708 if (ret) {10709 dev_info(&vsi->back->pdev->dev,10710 "failed to configure TX rings for channel %u\n",10711 ch->seid);10712 return ret;10713 }10714 /* update 'next_base_queue' */10715 vsi->next_base_queue = vsi->next_base_queue +10716 ch->num_queue_pairs;10717 if (ch->max_tx_rate) {10718 u64 credits = ch->max_tx_rate;10719 10720 if (i40e_set_bw_limit(vsi, ch->seid,10721 ch->max_tx_rate))10722 return -EINVAL;10723 10724 do_div(credits, I40E_BW_CREDIT_DIVISOR);10725 dev_dbg(&vsi->back->pdev->dev,10726 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",10727 ch->max_tx_rate,10728 credits,10729 ch->seid);10730 }10731 ret = i40e_rebuild_cloud_filters(vsi, ch->seid);10732 if (ret) {10733 dev_dbg(&vsi->back->pdev->dev,10734 "Failed to rebuild cloud filters for channel VSI %u\n",10735 ch->seid);10736 return ret;10737 }10738 }10739 return 0;10740}10741 10742/**10743 * i40e_clean_xps_state - clean xps state for every tx_ring10744 * @vsi: ptr to the VSI10745 **/10746static void i40e_clean_xps_state(struct i40e_vsi *vsi)10747{10748 int i;10749 10750 if (vsi->tx_rings)10751 for (i = 0; i < vsi->num_queue_pairs; i++)10752 if (vsi->tx_rings[i])10753 clear_bit(__I40E_TX_XPS_INIT_DONE,10754 vsi->tx_rings[i]->state);10755}10756 10757/**10758 * i40e_prep_for_reset - prep for the core to reset10759 * @pf: board private structure10760 *10761 * Close up the VFs and other things in prep for PF Reset.10762 **/10763static void i40e_prep_for_reset(struct i40e_pf *pf)10764{10765 struct i40e_hw *hw = &pf->hw;10766 struct i40e_vsi *vsi;10767 int ret = 0;10768 u32 v;10769 10770 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);10771 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))10772 return;10773 if (i40e_check_asq_alive(&pf->hw))10774 i40e_vc_notify_reset(pf);10775 10776 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");10777 10778 /* quiesce the VSIs and their queues that are not already DOWN */10779 i40e_pf_quiesce_all_vsi(pf);10780 10781 i40e_pf_for_each_vsi(pf, v, vsi) {10782 i40e_clean_xps_state(vsi);10783 vsi->seid = 0;10784 }10785 10786 i40e_shutdown_adminq(&pf->hw);10787 10788 /* call shutdown HMC */10789 if (hw->hmc.hmc_obj) {10790 ret = i40e_shutdown_lan_hmc(hw);10791 if (ret)10792 dev_warn(&pf->pdev->dev,10793 "shutdown_lan_hmc failed: %d\n", ret);10794 }10795 10796 /* Save the current PTP time so that we can restore the time after the10797 * reset completes.10798 */10799 i40e_ptp_save_hw_time(pf);10800}10801 10802/**10803 * i40e_send_version - update firmware with driver version10804 * @pf: PF struct10805 */10806static void i40e_send_version(struct i40e_pf *pf)10807{10808 struct i40e_driver_version dv;10809 10810 dv.major_version = 0xff;10811 dv.minor_version = 0xff;10812 dv.build_version = 0xff;10813 dv.subbuild_version = 0;10814 strscpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string));10815 i40e_aq_send_driver_version(&pf->hw, &dv, NULL);10816}10817 10818/**10819 * i40e_get_oem_version - get OEM specific version information10820 * @hw: pointer to the hardware structure10821 **/10822static void i40e_get_oem_version(struct i40e_hw *hw)10823{10824 u16 block_offset = 0xffff;10825 u16 block_length = 0;10826 u16 capabilities = 0;10827 u16 gen_snap = 0;10828 u16 release = 0;10829 10830#define I40E_SR_NVM_OEM_VERSION_PTR 0x1B10831#define I40E_NVM_OEM_LENGTH_OFFSET 0x0010832#define I40E_NVM_OEM_CAPABILITIES_OFFSET 0x0110833#define I40E_NVM_OEM_GEN_OFFSET 0x0210834#define I40E_NVM_OEM_RELEASE_OFFSET 0x0310835#define I40E_NVM_OEM_CAPABILITIES_MASK 0x000F10836#define I40E_NVM_OEM_LENGTH 310837 10838 /* Check if pointer to OEM version block is valid. */10839 i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset);10840 if (block_offset == 0xffff)10841 return;10842 10843 /* Check if OEM version block has correct length. */10844 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET,10845 &block_length);10846 if (block_length < I40E_NVM_OEM_LENGTH)10847 return;10848 10849 /* Check if OEM version format is as expected. */10850 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,10851 &capabilities);10852 if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)10853 return;10854 10855 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET,10856 &gen_snap);10857 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET,10858 &release);10859 hw->nvm.oem_ver =10860 FIELD_PREP(I40E_OEM_GEN_MASK | I40E_OEM_SNAP_MASK, gen_snap) |10861 FIELD_PREP(I40E_OEM_RELEASE_MASK, release);10862 hw->nvm.eetrack = I40E_OEM_EETRACK_ID;10863}10864 10865/**10866 * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen10867 * @pf: board private structure10868 **/10869static int i40e_reset(struct i40e_pf *pf)10870{10871 struct i40e_hw *hw = &pf->hw;10872 int ret;10873 10874 ret = i40e_pf_reset(hw);10875 if (ret) {10876 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);10877 set_bit(__I40E_RESET_FAILED, pf->state);10878 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);10879 } else {10880 pf->pfr_count++;10881 }10882 return ret;10883}10884 10885/**10886 * i40e_rebuild - rebuild using a saved config10887 * @pf: board private structure10888 * @reinit: if the Main VSI needs to re-initialized.10889 * @lock_acquired: indicates whether or not the lock has been acquired10890 * before this function was called.10891 **/10892static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)10893{10894 const bool is_recovery_mode_reported = i40e_check_recovery_mode(pf);10895 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);10896 struct i40e_hw *hw = &pf->hw;10897 struct i40e_veb *veb;10898 int ret;10899 u32 val;10900 int v;10901 10902 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&10903 is_recovery_mode_reported)10904 i40e_set_ethtool_ops(vsi->netdev);10905 10906 if (test_bit(__I40E_DOWN, pf->state) &&10907 !test_bit(__I40E_RECOVERY_MODE, pf->state))10908 goto clear_recovery;10909 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");10910 10911 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */10912 ret = i40e_init_adminq(&pf->hw);10913 if (ret) {10914 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %pe aq_err %s\n",10915 ERR_PTR(ret),10916 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));10917 goto clear_recovery;10918 }10919 i40e_get_oem_version(&pf->hw);10920 10921 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) {10922 /* The following delay is necessary for firmware update. */10923 mdelay(1000);10924 }10925 10926 /* re-verify the eeprom if we just had an EMP reset */10927 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))10928 i40e_verify_eeprom(pf);10929 10930 /* if we are going out of or into recovery mode we have to act10931 * accordingly with regard to resources initialization10932 * and deinitialization10933 */10934 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {10935 if (i40e_get_capabilities(pf,10936 i40e_aqc_opc_list_func_capabilities))10937 goto end_unlock;10938 10939 if (is_recovery_mode_reported) {10940 /* we're staying in recovery mode so we'll reinitialize10941 * misc vector here10942 */10943 if (i40e_setup_misc_vector_for_recovery_mode(pf))10944 goto end_unlock;10945 } else {10946 if (!lock_acquired)10947 rtnl_lock();10948 /* we're going out of recovery mode so we'll free10949 * the IRQ allocated specifically for recovery mode10950 * and restore the interrupt scheme10951 */10952 free_irq(pf->pdev->irq, pf);10953 i40e_clear_interrupt_scheme(pf);10954 if (i40e_restore_interrupt_scheme(pf))10955 goto end_unlock;10956 }10957 10958 /* tell the firmware that we're starting */10959 i40e_send_version(pf);10960 10961 /* bail out in case recovery mode was detected, as there is10962 * no need for further configuration.10963 */10964 goto end_unlock;10965 }10966 10967 i40e_clear_pxe_mode(hw);10968 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);10969 if (ret)10970 goto end_core_reset;10971 10972 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,10973 hw->func_caps.num_rx_qp, 0, 0);10974 if (ret) {10975 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);10976 goto end_core_reset;10977 }10978 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);10979 if (ret) {10980 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);10981 goto end_core_reset;10982 }10983 10984#ifdef CONFIG_I40E_DCB10985 /* Enable FW to write a default DCB config on link-up10986 * unless I40E_FLAG_TC_MQPRIO was enabled or DCB10987 * is not supported with new link speed10988 */10989 if (i40e_is_tc_mqprio_enabled(pf)) {10990 i40e_aq_set_dcb_parameters(hw, false, NULL);10991 } else {10992 if (I40E_IS_X710TL_DEVICE(hw->device_id) &&10993 (hw->phy.link_info.link_speed &10994 (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {10995 i40e_aq_set_dcb_parameters(hw, false, NULL);10996 dev_warn(&pf->pdev->dev,10997 "DCB is not supported for X710-T*L 2.5/5G speeds\n");10998 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);10999 } else {11000 i40e_aq_set_dcb_parameters(hw, true, NULL);11001 ret = i40e_init_pf_dcb(pf);11002 if (ret) {11003 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n",11004 ret);11005 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);11006 /* Continue without DCB enabled */11007 }11008 }11009 }11010 11011#endif /* CONFIG_I40E_DCB */11012 if (!lock_acquired)11013 rtnl_lock();11014 ret = i40e_setup_pf_switch(pf, reinit, true);11015 if (ret)11016 goto end_unlock;11017 11018 /* The driver only wants link up/down and module qualification11019 * reports from firmware. Note the negative logic.11020 */11021 ret = i40e_aq_set_phy_int_mask(&pf->hw,11022 ~(I40E_AQ_EVENT_LINK_UPDOWN |11023 I40E_AQ_EVENT_MEDIA_NA |11024 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);11025 if (ret)11026 dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n",11027 ERR_PTR(ret),11028 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));11029 11030 /* Rebuild the VSIs and VEBs that existed before reset.11031 * They are still in our local switch element arrays, so only11032 * need to rebuild the switch model in the HW.11033 *11034 * If there were VEBs but the reconstitution failed, we'll try11035 * to recover minimal use by getting the basic PF VSI working.11036 */11037 if (vsi->uplink_seid != pf->mac_seid) {11038 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");11039 11040 /* Rebuild VEBs */11041 i40e_pf_for_each_veb(pf, v, veb) {11042 ret = i40e_reconstitute_veb(veb);11043 if (!ret)11044 continue;11045 11046 /* If Main VEB failed, we're in deep doodoo,11047 * so give up rebuilding the switch and set up11048 * for minimal rebuild of PF VSI.11049 * If orphan failed, we'll report the error11050 * but try to keep going.11051 */11052 if (veb->uplink_seid == pf->mac_seid) {11053 dev_info(&pf->pdev->dev,11054 "rebuild of switch failed: %d, will try to set up simple PF connection\n",11055 ret);11056 vsi->uplink_seid = pf->mac_seid;11057 break;11058 } else if (veb->uplink_seid == 0) {11059 dev_info(&pf->pdev->dev,11060 "rebuild of orphan VEB failed: %d\n",11061 ret);11062 }11063 }11064 }11065 11066 if (vsi->uplink_seid == pf->mac_seid) {11067 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");11068 /* no VEB, so rebuild only the Main VSI */11069 ret = i40e_add_vsi(vsi);11070 if (ret) {11071 dev_info(&pf->pdev->dev,11072 "rebuild of Main VSI failed: %d\n", ret);11073 goto end_unlock;11074 }11075 }11076 11077 if (vsi->mqprio_qopt.max_rate[0]) {11078 u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,11079 vsi->mqprio_qopt.max_rate[0]);11080 u64 credits = 0;11081 11082 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);11083 if (ret)11084 goto end_unlock;11085 11086 credits = max_tx_rate;11087 do_div(credits, I40E_BW_CREDIT_DIVISOR);11088 dev_dbg(&vsi->back->pdev->dev,11089 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",11090 max_tx_rate,11091 credits,11092 vsi->seid);11093 }11094 11095 ret = i40e_rebuild_cloud_filters(vsi, vsi->seid);11096 if (ret)11097 goto end_unlock;11098 11099 /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs11100 * for this main VSI if they exist11101 */11102 ret = i40e_rebuild_channels(vsi);11103 if (ret)11104 goto end_unlock;11105 11106 /* Reconfigure hardware for allowing smaller MSS in the case11107 * of TSO, so that we avoid the MDD being fired and causing11108 * a reset in the case of small MSS+TSO.11109 */11110#define I40E_REG_MSS 0x000E64DC11111#define I40E_REG_MSS_MIN_MASK 0x3FF000011112#define I40E_64BYTE_MSS 0x40000011113 val = rd32(hw, I40E_REG_MSS);11114 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {11115 val &= ~I40E_REG_MSS_MIN_MASK;11116 val |= I40E_64BYTE_MSS;11117 wr32(hw, I40E_REG_MSS, val);11118 }11119 11120 if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) {11121 msleep(75);11122 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);11123 if (ret)11124 dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n",11125 ERR_PTR(ret),11126 i40e_aq_str(&pf->hw,11127 pf->hw.aq.asq_last_status));11128 }11129 /* reinit the misc interrupt */11130 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {11131 ret = i40e_setup_misc_vector(pf);11132 if (ret)11133 goto end_unlock;11134 }11135 11136 /* Add a filter to drop all Flow control frames from any VSI from being11137 * transmitted. By doing so we stop a malicious VF from sending out11138 * PAUSE or PFC frames and potentially controlling traffic for other11139 * PF/VF VSIs.11140 * The FW can still send Flow control frames if enabled.11141 */11142 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,11143 pf->main_vsi_seid);11144 11145 /* restart the VSIs that were rebuilt and running before the reset */11146 i40e_pf_unquiesce_all_vsi(pf);11147 11148 /* Release the RTNL lock before we start resetting VFs */11149 if (!lock_acquired)11150 rtnl_unlock();11151 11152 /* Restore promiscuous settings */11153 ret = i40e_set_promiscuous(pf, pf->cur_promisc);11154 if (ret)11155 dev_warn(&pf->pdev->dev,11156 "Failed to restore promiscuous setting: %s, err %pe aq_err %s\n",11157 pf->cur_promisc ? "on" : "off",11158 ERR_PTR(ret),11159 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));11160 11161 i40e_reset_all_vfs(pf, true);11162 11163 /* tell the firmware that we're starting */11164 i40e_send_version(pf);11165 11166 /* We've already released the lock, so don't do it again */11167 goto end_core_reset;11168 11169end_unlock:11170 if (!lock_acquired)11171 rtnl_unlock();11172end_core_reset:11173 clear_bit(__I40E_RESET_FAILED, pf->state);11174clear_recovery:11175 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);11176 clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state);11177}11178 11179/**11180 * i40e_reset_and_rebuild - reset and rebuild using a saved config11181 * @pf: board private structure11182 * @reinit: if the Main VSI needs to re-initialized.11183 * @lock_acquired: indicates whether or not the lock has been acquired11184 * before this function was called.11185 **/11186static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,11187 bool lock_acquired)11188{11189 int ret;11190 11191 if (test_bit(__I40E_IN_REMOVE, pf->state))11192 return;11193 /* Now we wait for GRST to settle out.11194 * We don't have to delete the VEBs or VSIs from the hw switch11195 * because the reset will make them disappear.11196 */11197 ret = i40e_reset(pf);11198 if (!ret)11199 i40e_rebuild(pf, reinit, lock_acquired);11200 else11201 dev_err(&pf->pdev->dev, "%s: i40e_reset() FAILED", __func__);11202}11203 11204/**11205 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild11206 * @pf: board private structure11207 *11208 * Close up the VFs and other things in prep for a Core Reset,11209 * then get ready to rebuild the world.11210 * @lock_acquired: indicates whether or not the lock has been acquired11211 * before this function was called.11212 **/11213static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)11214{11215 i40e_prep_for_reset(pf);11216 i40e_reset_and_rebuild(pf, false, lock_acquired);11217}11218 11219/**11220 * i40e_handle_mdd_event11221 * @pf: pointer to the PF structure11222 *11223 * Called from the MDD irq handler to identify possibly malicious vfs11224 **/11225static void i40e_handle_mdd_event(struct i40e_pf *pf)11226{11227 struct i40e_hw *hw = &pf->hw;11228 bool mdd_detected = false;11229 struct i40e_vf *vf;11230 u32 reg;11231 int i;11232 11233 if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))11234 return;11235 11236 /* find what triggered the MDD event */11237 reg = rd32(hw, I40E_GL_MDET_TX);11238 if (reg & I40E_GL_MDET_TX_VALID_MASK) {11239 u8 pf_num = FIELD_GET(I40E_GL_MDET_TX_PF_NUM_MASK, reg);11240 u16 vf_num = FIELD_GET(I40E_GL_MDET_TX_VF_NUM_MASK, reg);11241 u8 event = FIELD_GET(I40E_GL_MDET_TX_EVENT_MASK, reg);11242 u16 queue = FIELD_GET(I40E_GL_MDET_TX_QUEUE_MASK, reg) -11243 pf->hw.func_caps.base_queue;11244 if (netif_msg_tx_err(pf))11245 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",11246 event, queue, pf_num, vf_num);11247 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);11248 mdd_detected = true;11249 }11250 reg = rd32(hw, I40E_GL_MDET_RX);11251 if (reg & I40E_GL_MDET_RX_VALID_MASK) {11252 u8 func = FIELD_GET(I40E_GL_MDET_RX_FUNCTION_MASK, reg);11253 u8 event = FIELD_GET(I40E_GL_MDET_RX_EVENT_MASK, reg);11254 u16 queue = FIELD_GET(I40E_GL_MDET_RX_QUEUE_MASK, reg) -11255 pf->hw.func_caps.base_queue;11256 if (netif_msg_rx_err(pf))11257 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",11258 event, queue, func);11259 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);11260 mdd_detected = true;11261 }11262 11263 if (mdd_detected) {11264 reg = rd32(hw, I40E_PF_MDET_TX);11265 if (reg & I40E_PF_MDET_TX_VALID_MASK) {11266 wr32(hw, I40E_PF_MDET_TX, 0xFFFF);11267 dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n");11268 }11269 reg = rd32(hw, I40E_PF_MDET_RX);11270 if (reg & I40E_PF_MDET_RX_VALID_MASK) {11271 wr32(hw, I40E_PF_MDET_RX, 0xFFFF);11272 dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n");11273 }11274 }11275 11276 /* see if one of the VFs needs its hand slapped */11277 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {11278 vf = &(pf->vf[i]);11279 reg = rd32(hw, I40E_VP_MDET_TX(i));11280 if (reg & I40E_VP_MDET_TX_VALID_MASK) {11281 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);11282 vf->num_mdd_events++;11283 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",11284 i);11285 dev_info(&pf->pdev->dev,11286 "Use PF Control I/F to re-enable the VF\n");11287 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);11288 }11289 11290 reg = rd32(hw, I40E_VP_MDET_RX(i));11291 if (reg & I40E_VP_MDET_RX_VALID_MASK) {11292 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);11293 vf->num_mdd_events++;11294 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",11295 i);11296 dev_info(&pf->pdev->dev,11297 "Use PF Control I/F to re-enable the VF\n");11298 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);11299 }11300 }11301 11302 /* re-enable mdd interrupt cause */11303 clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);11304 reg = rd32(hw, I40E_PFINT_ICR0_ENA);11305 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;11306 wr32(hw, I40E_PFINT_ICR0_ENA, reg);11307 i40e_flush(hw);11308}11309 11310/**11311 * i40e_service_task - Run the driver's async subtasks11312 * @work: pointer to work_struct containing our data11313 **/11314static void i40e_service_task(struct work_struct *work)11315{11316 struct i40e_pf *pf = container_of(work,11317 struct i40e_pf,11318 service_task);11319 unsigned long start_time = jiffies;11320 11321 /* don't bother with service tasks if a reset is in progress */11322 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||11323 test_bit(__I40E_SUSPENDED, pf->state))11324 return;11325 11326 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))11327 return;11328 11329 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) {11330 i40e_detect_recover_hung(pf);11331 i40e_sync_filters_subtask(pf);11332 i40e_reset_subtask(pf);11333 i40e_handle_mdd_event(pf);11334 i40e_vc_process_vflr_event(pf);11335 i40e_watchdog_subtask(pf);11336 i40e_fdir_reinit_subtask(pf);11337 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) {11338 /* Client subtask will reopen next time through. */11339 i40e_notify_client_of_netdev_close(pf, true);11340 } else {11341 i40e_client_subtask(pf);11342 if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE,11343 pf->state))11344 i40e_notify_client_of_l2_param_changes(pf);11345 }11346 i40e_sync_filters_subtask(pf);11347 } else {11348 i40e_reset_subtask(pf);11349 }11350 11351 i40e_clean_adminq_subtask(pf);11352 11353 /* flush memory to make sure state is correct before next watchdog */11354 smp_mb__before_atomic();11355 clear_bit(__I40E_SERVICE_SCHED, pf->state);11356 11357 /* If the tasks have taken longer than one timer cycle or there11358 * is more work to be done, reschedule the service task now11359 * rather than wait for the timer to tick again.11360 */11361 if (time_after(jiffies, (start_time + pf->service_timer_period)) ||11362 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) ||11363 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) ||11364 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))11365 i40e_service_event_schedule(pf);11366}11367 11368/**11369 * i40e_service_timer - timer callback11370 * @t: timer list pointer11371 **/11372static void i40e_service_timer(struct timer_list *t)11373{11374 struct i40e_pf *pf = from_timer(pf, t, service_timer);11375 11376 mod_timer(&pf->service_timer,11377 round_jiffies(jiffies + pf->service_timer_period));11378 i40e_service_event_schedule(pf);11379}11380 11381/**11382 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI11383 * @vsi: the VSI being configured11384 **/11385static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)11386{11387 struct i40e_pf *pf = vsi->back;11388 11389 switch (vsi->type) {11390 case I40E_VSI_MAIN:11391 vsi->alloc_queue_pairs = pf->num_lan_qps;11392 if (!vsi->num_tx_desc)11393 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,11394 I40E_REQ_DESCRIPTOR_MULTIPLE);11395 if (!vsi->num_rx_desc)11396 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,11397 I40E_REQ_DESCRIPTOR_MULTIPLE);11398 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))11399 vsi->num_q_vectors = pf->num_lan_msix;11400 else11401 vsi->num_q_vectors = 1;11402 11403 break;11404 11405 case I40E_VSI_FDIR:11406 vsi->alloc_queue_pairs = 1;11407 vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT,11408 I40E_REQ_DESCRIPTOR_MULTIPLE);11409 vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT,11410 I40E_REQ_DESCRIPTOR_MULTIPLE);11411 vsi->num_q_vectors = pf->num_fdsb_msix;11412 break;11413 11414 case I40E_VSI_VMDQ2:11415 vsi->alloc_queue_pairs = pf->num_vmdq_qps;11416 if (!vsi->num_tx_desc)11417 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,11418 I40E_REQ_DESCRIPTOR_MULTIPLE);11419 if (!vsi->num_rx_desc)11420 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,11421 I40E_REQ_DESCRIPTOR_MULTIPLE);11422 vsi->num_q_vectors = pf->num_vmdq_msix;11423 break;11424 11425 case I40E_VSI_SRIOV:11426 vsi->alloc_queue_pairs = pf->num_vf_qps;11427 if (!vsi->num_tx_desc)11428 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,11429 I40E_REQ_DESCRIPTOR_MULTIPLE);11430 if (!vsi->num_rx_desc)11431 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,11432 I40E_REQ_DESCRIPTOR_MULTIPLE);11433 break;11434 11435 default:11436 WARN_ON(1);11437 return -ENODATA;11438 }11439 11440 if (is_kdump_kernel()) {11441 vsi->num_tx_desc = I40E_MIN_NUM_DESCRIPTORS;11442 vsi->num_rx_desc = I40E_MIN_NUM_DESCRIPTORS;11443 }11444 11445 return 0;11446}11447 11448/**11449 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi11450 * @vsi: VSI pointer11451 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.11452 *11453 * On error: returns error code (negative)11454 * On success: returns 011455 **/11456static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)11457{11458 struct i40e_ring **next_rings;11459 int size;11460 int ret = 0;11461 11462 /* allocate memory for both Tx, XDP Tx and Rx ring pointers */11463 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *11464 (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);11465 vsi->tx_rings = kzalloc(size, GFP_KERNEL);11466 if (!vsi->tx_rings)11467 return -ENOMEM;11468 next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;11469 if (i40e_enabled_xdp_vsi(vsi)) {11470 vsi->xdp_rings = next_rings;11471 next_rings += vsi->alloc_queue_pairs;11472 }11473 vsi->rx_rings = next_rings;11474 11475 if (alloc_qvectors) {11476 /* allocate memory for q_vector pointers */11477 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;11478 vsi->q_vectors = kzalloc(size, GFP_KERNEL);11479 if (!vsi->q_vectors) {11480 ret = -ENOMEM;11481 goto err_vectors;11482 }11483 }11484 return ret;11485 11486err_vectors:11487 kfree(vsi->tx_rings);11488 return ret;11489}11490 11491/**11492 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF11493 * @pf: board private structure11494 * @type: type of VSI11495 *11496 * On error: returns error code (negative)11497 * On success: returns vsi index in PF (positive)11498 **/11499static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)11500{11501 int ret = -ENODEV;11502 struct i40e_vsi *vsi;11503 int vsi_idx;11504 int i;11505 11506 /* Need to protect the allocation of the VSIs at the PF level */11507 mutex_lock(&pf->switch_mutex);11508 11509 /* VSI list may be fragmented if VSI creation/destruction has11510 * been happening. We can afford to do a quick scan to look11511 * for any free VSIs in the list.11512 *11513 * find next empty vsi slot, looping back around if necessary11514 */11515 i = pf->next_vsi;11516 while (i < pf->num_alloc_vsi && pf->vsi[i])11517 i++;11518 if (i >= pf->num_alloc_vsi) {11519 i = 0;11520 while (i < pf->next_vsi && pf->vsi[i])11521 i++;11522 }11523 11524 if (i < pf->num_alloc_vsi && !pf->vsi[i]) {11525 vsi_idx = i; /* Found one! */11526 } else {11527 ret = -ENODEV;11528 goto unlock_pf; /* out of VSI slots! */11529 }11530 pf->next_vsi = ++i;11531 11532 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);11533 if (!vsi) {11534 ret = -ENOMEM;11535 goto unlock_pf;11536 }11537 vsi->type = type;11538 vsi->back = pf;11539 set_bit(__I40E_VSI_DOWN, vsi->state);11540 vsi->flags = 0;11541 vsi->idx = vsi_idx;11542 vsi->int_rate_limit = 0;11543 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?11544 pf->rss_table_size : 64;11545 vsi->netdev_registered = false;11546 vsi->work_limit = I40E_DEFAULT_IRQ_WORK;11547 hash_init(vsi->mac_filter_hash);11548 vsi->irqs_ready = false;11549 11550 if (type == I40E_VSI_MAIN) {11551 vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL);11552 if (!vsi->af_xdp_zc_qps)11553 goto err_rings;11554 }11555 11556 ret = i40e_set_num_rings_in_vsi(vsi);11557 if (ret)11558 goto err_rings;11559 11560 ret = i40e_vsi_alloc_arrays(vsi, true);11561 if (ret)11562 goto err_rings;11563 11564 /* Setup default MSIX irq handler for VSI */11565 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);11566 11567 /* Initialize VSI lock */11568 spin_lock_init(&vsi->mac_filter_hash_lock);11569 pf->vsi[vsi_idx] = vsi;11570 ret = vsi_idx;11571 goto unlock_pf;11572 11573err_rings:11574 bitmap_free(vsi->af_xdp_zc_qps);11575 pf->next_vsi = i - 1;11576 kfree(vsi);11577unlock_pf:11578 mutex_unlock(&pf->switch_mutex);11579 return ret;11580}11581 11582/**11583 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI11584 * @vsi: VSI pointer11585 * @free_qvectors: a bool to specify if q_vectors need to be freed.11586 *11587 * On error: returns error code (negative)11588 * On success: returns 011589 **/11590static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)11591{11592 /* free the ring and vector containers */11593 if (free_qvectors) {11594 kfree(vsi->q_vectors);11595 vsi->q_vectors = NULL;11596 }11597 kfree(vsi->tx_rings);11598 vsi->tx_rings = NULL;11599 vsi->rx_rings = NULL;11600 vsi->xdp_rings = NULL;11601}11602 11603/**11604 * i40e_clear_rss_config_user - clear the user configured RSS hash keys11605 * and lookup table11606 * @vsi: Pointer to VSI structure11607 */11608static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)11609{11610 if (!vsi)11611 return;11612 11613 kfree(vsi->rss_hkey_user);11614 vsi->rss_hkey_user = NULL;11615 11616 kfree(vsi->rss_lut_user);11617 vsi->rss_lut_user = NULL;11618}11619 11620/**11621 * i40e_vsi_clear - Deallocate the VSI provided11622 * @vsi: the VSI being un-configured11623 **/11624static int i40e_vsi_clear(struct i40e_vsi *vsi)11625{11626 struct i40e_pf *pf;11627 11628 if (!vsi)11629 return 0;11630 11631 if (!vsi->back)11632 goto free_vsi;11633 pf = vsi->back;11634 11635 mutex_lock(&pf->switch_mutex);11636 if (!pf->vsi[vsi->idx]) {11637 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n",11638 vsi->idx, vsi->idx, vsi->type);11639 goto unlock_vsi;11640 }11641 11642 if (pf->vsi[vsi->idx] != vsi) {11643 dev_err(&pf->pdev->dev,11644 "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n",11645 pf->vsi[vsi->idx]->idx,11646 pf->vsi[vsi->idx]->type,11647 vsi->idx, vsi->type);11648 goto unlock_vsi;11649 }11650 11651 /* updates the PF for this cleared vsi */11652 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);11653 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);11654 11655 bitmap_free(vsi->af_xdp_zc_qps);11656 i40e_vsi_free_arrays(vsi, true);11657 i40e_clear_rss_config_user(vsi);11658 11659 pf->vsi[vsi->idx] = NULL;11660 if (vsi->idx < pf->next_vsi)11661 pf->next_vsi = vsi->idx;11662 11663unlock_vsi:11664 mutex_unlock(&pf->switch_mutex);11665free_vsi:11666 kfree(vsi);11667 11668 return 0;11669}11670 11671/**11672 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI11673 * @vsi: the VSI being cleaned11674 **/11675static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)11676{11677 int i;11678 11679 if (vsi->tx_rings && vsi->tx_rings[0]) {11680 for (i = 0; i < vsi->alloc_queue_pairs; i++) {11681 kfree_rcu(vsi->tx_rings[i], rcu);11682 WRITE_ONCE(vsi->tx_rings[i], NULL);11683 WRITE_ONCE(vsi->rx_rings[i], NULL);11684 if (vsi->xdp_rings)11685 WRITE_ONCE(vsi->xdp_rings[i], NULL);11686 }11687 }11688}11689 11690/**11691 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI11692 * @vsi: the VSI being configured11693 **/11694static int i40e_alloc_rings(struct i40e_vsi *vsi)11695{11696 int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;11697 struct i40e_pf *pf = vsi->back;11698 struct i40e_ring *ring;11699 11700 /* Set basic values in the rings to be used later during open() */11701 for (i = 0; i < vsi->alloc_queue_pairs; i++) {11702 /* allocate space for both Tx and Rx in one shot */11703 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);11704 if (!ring)11705 goto err_out;11706 11707 ring->queue_index = i;11708 ring->reg_idx = vsi->base_queue + i;11709 ring->ring_active = false;11710 ring->vsi = vsi;11711 ring->netdev = vsi->netdev;11712 ring->dev = &pf->pdev->dev;11713 ring->count = vsi->num_tx_desc;11714 ring->size = 0;11715 ring->dcb_tc = 0;11716 if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps))11717 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;11718 ring->itr_setting = pf->tx_itr_default;11719 WRITE_ONCE(vsi->tx_rings[i], ring++);11720 11721 if (!i40e_enabled_xdp_vsi(vsi))11722 goto setup_rx;11723 11724 ring->queue_index = vsi->alloc_queue_pairs + i;11725 ring->reg_idx = vsi->base_queue + ring->queue_index;11726 ring->ring_active = false;11727 ring->vsi = vsi;11728 ring->netdev = NULL;11729 ring->dev = &pf->pdev->dev;11730 ring->count = vsi->num_tx_desc;11731 ring->size = 0;11732 ring->dcb_tc = 0;11733 if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps))11734 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;11735 set_ring_xdp(ring);11736 ring->itr_setting = pf->tx_itr_default;11737 WRITE_ONCE(vsi->xdp_rings[i], ring++);11738 11739setup_rx:11740 ring->queue_index = i;11741 ring->reg_idx = vsi->base_queue + i;11742 ring->ring_active = false;11743 ring->vsi = vsi;11744 ring->netdev = vsi->netdev;11745 ring->dev = &pf->pdev->dev;11746 ring->count = vsi->num_rx_desc;11747 ring->size = 0;11748 ring->dcb_tc = 0;11749 ring->itr_setting = pf->rx_itr_default;11750 WRITE_ONCE(vsi->rx_rings[i], ring);11751 }11752 11753 return 0;11754 11755err_out:11756 i40e_vsi_clear_rings(vsi);11757 return -ENOMEM;11758}11759 11760/**11761 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel11762 * @pf: board private structure11763 * @vectors: the number of MSI-X vectors to request11764 *11765 * Returns the number of vectors reserved, or error11766 **/11767static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)11768{11769 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,11770 I40E_MIN_MSIX, vectors);11771 if (vectors < 0) {11772 dev_info(&pf->pdev->dev,11773 "MSI-X vector reservation failed: %d\n", vectors);11774 vectors = 0;11775 }11776 11777 return vectors;11778}11779 11780/**11781 * i40e_init_msix - Setup the MSIX capability11782 * @pf: board private structure11783 *11784 * Work with the OS to set up the MSIX vectors needed.11785 *11786 * Returns the number of vectors reserved or negative on failure11787 **/11788static int i40e_init_msix(struct i40e_pf *pf)11789{11790 struct i40e_hw *hw = &pf->hw;11791 int cpus, extra_vectors;11792 int vectors_left;11793 int v_budget, i;11794 int v_actual;11795 int iwarp_requested = 0;11796 11797 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))11798 return -ENODEV;11799 11800 /* The number of vectors we'll request will be comprised of:11801 * - Add 1 for "other" cause for Admin Queue events, etc.11802 * - The number of LAN queue pairs11803 * - Queues being used for RSS.11804 * We don't need as many as max_rss_size vectors.11805 * use rss_size instead in the calculation since that11806 * is governed by number of cpus in the system.11807 * - assumes symmetric Tx/Rx pairing11808 * - The number of VMDq pairs11809 * - The CPU count within the NUMA node if iWARP is enabled11810 * Once we count this up, try the request.11811 *11812 * If we can't get what we want, we'll simplify to nearly nothing11813 * and try again. If that still fails, we punt.11814 */11815 vectors_left = hw->func_caps.num_msix_vectors;11816 v_budget = 0;11817 11818 /* reserve one vector for miscellaneous handler */11819 if (vectors_left) {11820 v_budget++;11821 vectors_left--;11822 }11823 11824 /* reserve some vectors for the main PF traffic queues. Initially we11825 * only reserve at most 50% of the available vectors, in the case that11826 * the number of online CPUs is large. This ensures that we can enable11827 * extra features as well. Once we've enabled the other features, we11828 * will use any remaining vectors to reach as close as we can to the11829 * number of online CPUs.11830 */11831 cpus = num_online_cpus();11832 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);11833 vectors_left -= pf->num_lan_msix;11834 11835 /* reserve one vector for sideband flow director */11836 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {11837 if (vectors_left) {11838 pf->num_fdsb_msix = 1;11839 v_budget++;11840 vectors_left--;11841 } else {11842 pf->num_fdsb_msix = 0;11843 }11844 }11845 11846 /* can we reserve enough for iWARP? */11847 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {11848 iwarp_requested = pf->num_iwarp_msix;11849 11850 if (!vectors_left)11851 pf->num_iwarp_msix = 0;11852 else if (vectors_left < pf->num_iwarp_msix)11853 pf->num_iwarp_msix = 1;11854 v_budget += pf->num_iwarp_msix;11855 vectors_left -= pf->num_iwarp_msix;11856 }11857 11858 /* any vectors left over go for VMDq support */11859 if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags)) {11860 if (!vectors_left) {11861 pf->num_vmdq_msix = 0;11862 pf->num_vmdq_qps = 0;11863 } else {11864 int vmdq_vecs_wanted =11865 pf->num_vmdq_vsis * pf->num_vmdq_qps;11866 int vmdq_vecs =11867 min_t(int, vectors_left, vmdq_vecs_wanted);11868 11869 /* if we're short on vectors for what's desired, we limit11870 * the queues per vmdq. If this is still more than are11871 * available, the user will need to change the number of11872 * queues/vectors used by the PF later with the ethtool11873 * channels command11874 */11875 if (vectors_left < vmdq_vecs_wanted) {11876 pf->num_vmdq_qps = 1;11877 vmdq_vecs_wanted = pf->num_vmdq_vsis;11878 vmdq_vecs = min_t(int,11879 vectors_left,11880 vmdq_vecs_wanted);11881 }11882 pf->num_vmdq_msix = pf->num_vmdq_qps;11883 11884 v_budget += vmdq_vecs;11885 vectors_left -= vmdq_vecs;11886 }11887 }11888 11889 /* On systems with a large number of SMP cores, we previously limited11890 * the number of vectors for num_lan_msix to be at most 50% of the11891 * available vectors, to allow for other features. Now, we add back11892 * the remaining vectors. However, we ensure that the total11893 * num_lan_msix will not exceed num_online_cpus(). To do this, we11894 * calculate the number of vectors we can add without going over the11895 * cap of CPUs. For systems with a small number of CPUs this will be11896 * zero.11897 */11898 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);11899 pf->num_lan_msix += extra_vectors;11900 vectors_left -= extra_vectors;11901 11902 WARN(vectors_left < 0,11903 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");11904 11905 v_budget += pf->num_lan_msix;11906 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),11907 GFP_KERNEL);11908 if (!pf->msix_entries)11909 return -ENOMEM;11910 11911 for (i = 0; i < v_budget; i++)11912 pf->msix_entries[i].entry = i;11913 v_actual = i40e_reserve_msix_vectors(pf, v_budget);11914 11915 if (v_actual < I40E_MIN_MSIX) {11916 clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);11917 kfree(pf->msix_entries);11918 pf->msix_entries = NULL;11919 pci_disable_msix(pf->pdev);11920 return -ENODEV;11921 11922 } else if (v_actual == I40E_MIN_MSIX) {11923 /* Adjust for minimal MSIX use */11924 pf->num_vmdq_vsis = 0;11925 pf->num_vmdq_qps = 0;11926 pf->num_lan_qps = 1;11927 pf->num_lan_msix = 1;11928 11929 } else if (v_actual != v_budget) {11930 /* If we have limited resources, we will start with no vectors11931 * for the special features and then allocate vectors to some11932 * of these features based on the policy and at the end disable11933 * the features that did not get any vectors.11934 */11935 int vec;11936 11937 dev_info(&pf->pdev->dev,11938 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n",11939 v_actual, v_budget);11940 /* reserve the misc vector */11941 vec = v_actual - 1;11942 11943 /* Scale vector usage down */11944 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */11945 pf->num_vmdq_vsis = 1;11946 pf->num_vmdq_qps = 1;11947 11948 /* partition out the remaining vectors */11949 switch (vec) {11950 case 2:11951 pf->num_lan_msix = 1;11952 break;11953 case 3:11954 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {11955 pf->num_lan_msix = 1;11956 pf->num_iwarp_msix = 1;11957 } else {11958 pf->num_lan_msix = 2;11959 }11960 break;11961 default:11962 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {11963 pf->num_iwarp_msix = min_t(int, (vec / 3),11964 iwarp_requested);11965 pf->num_vmdq_vsis = min_t(int, (vec / 3),11966 I40E_DEFAULT_NUM_VMDQ_VSI);11967 } else {11968 pf->num_vmdq_vsis = min_t(int, (vec / 2),11969 I40E_DEFAULT_NUM_VMDQ_VSI);11970 }11971 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {11972 pf->num_fdsb_msix = 1;11973 vec--;11974 }11975 pf->num_lan_msix = min_t(int,11976 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),11977 pf->num_lan_msix);11978 pf->num_lan_qps = pf->num_lan_msix;11979 break;11980 }11981 }11982 11983 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && pf->num_fdsb_msix == 0) {11984 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");11985 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);11986 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);11987 }11988 if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) && pf->num_vmdq_msix == 0) {11989 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");11990 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);11991 }11992 11993 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) &&11994 pf->num_iwarp_msix == 0) {11995 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");11996 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);11997 }11998 i40e_debug(&pf->hw, I40E_DEBUG_INIT,11999 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",12000 pf->num_lan_msix,12001 pf->num_vmdq_msix * pf->num_vmdq_vsis,12002 pf->num_fdsb_msix,12003 pf->num_iwarp_msix);12004 12005 return v_actual;12006}12007 12008/**12009 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector12010 * @vsi: the VSI being configured12011 * @v_idx: index of the vector in the vsi struct12012 *12013 * We allocate one q_vector. If allocation fails we return -ENOMEM.12014 **/12015static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)12016{12017 struct i40e_q_vector *q_vector;12018 12019 /* allocate q_vector */12020 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);12021 if (!q_vector)12022 return -ENOMEM;12023 12024 q_vector->vsi = vsi;12025 q_vector->v_idx = v_idx;12026 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);12027 12028 if (vsi->netdev)12029 netif_napi_add(vsi->netdev, &q_vector->napi, i40e_napi_poll);12030 12031 /* tie q_vector and vsi together */12032 vsi->q_vectors[v_idx] = q_vector;12033 12034 return 0;12035}12036 12037/**12038 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors12039 * @vsi: the VSI being configured12040 *12041 * We allocate one q_vector per queue interrupt. If allocation fails we12042 * return -ENOMEM.12043 **/12044static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)12045{12046 struct i40e_pf *pf = vsi->back;12047 int err, v_idx, num_q_vectors;12048 12049 /* if not MSIX, give the one vector only to the LAN VSI */12050 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))12051 num_q_vectors = vsi->num_q_vectors;12052 else if (vsi->type == I40E_VSI_MAIN)12053 num_q_vectors = 1;12054 else12055 return -EINVAL;12056 12057 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {12058 err = i40e_vsi_alloc_q_vector(vsi, v_idx);12059 if (err)12060 goto err_out;12061 }12062 12063 return 0;12064 12065err_out:12066 while (v_idx--)12067 i40e_free_q_vector(vsi, v_idx);12068 12069 return err;12070}12071 12072/**12073 * i40e_init_interrupt_scheme - Determine proper interrupt scheme12074 * @pf: board private structure to initialize12075 **/12076static int i40e_init_interrupt_scheme(struct i40e_pf *pf)12077{12078 int vectors = 0;12079 ssize_t size;12080 12081 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {12082 vectors = i40e_init_msix(pf);12083 if (vectors < 0) {12084 clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);12085 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);12086 clear_bit(I40E_FLAG_RSS_ENA, pf->flags);12087 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);12088 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);12089 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);12090 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);12091 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);12092 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);12093 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);12094 12095 /* rework the queue expectations without MSIX */12096 i40e_determine_queue_usage(pf);12097 }12098 }12099 12100 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&12101 test_bit(I40E_FLAG_MSI_ENA, pf->flags)) {12102 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");12103 vectors = pci_enable_msi(pf->pdev);12104 if (vectors < 0) {12105 dev_info(&pf->pdev->dev, "MSI init failed - %d\n",12106 vectors);12107 clear_bit(I40E_FLAG_MSI_ENA, pf->flags);12108 }12109 vectors = 1; /* one MSI or Legacy vector */12110 }12111 12112 if (!test_bit(I40E_FLAG_MSI_ENA, pf->flags) &&12113 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))12114 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");12115 12116 /* set up vector assignment tracking */12117 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);12118 pf->irq_pile = kzalloc(size, GFP_KERNEL);12119 if (!pf->irq_pile)12120 return -ENOMEM;12121 12122 pf->irq_pile->num_entries = vectors;12123 12124 /* track first vector for misc interrupts, ignore return */12125 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);12126 12127 return 0;12128}12129 12130/**12131 * i40e_restore_interrupt_scheme - Restore the interrupt scheme12132 * @pf: private board data structure12133 *12134 * Restore the interrupt scheme that was cleared when we suspended the12135 * device. This should be called during resume to re-allocate the q_vectors12136 * and reacquire IRQs.12137 */12138static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)12139{12140 struct i40e_vsi *vsi;12141 int err, i;12142 12143 /* We cleared the MSI and MSI-X flags when disabling the old interrupt12144 * scheme. We need to re-enabled them here in order to attempt to12145 * re-acquire the MSI or MSI-X vectors12146 */12147 set_bit(I40E_FLAG_MSI_ENA, pf->flags);12148 set_bit(I40E_FLAG_MSIX_ENA, pf->flags);12149 12150 err = i40e_init_interrupt_scheme(pf);12151 if (err)12152 return err;12153 12154 /* Now that we've re-acquired IRQs, we need to remap the vectors and12155 * rings together again.12156 */12157 i40e_pf_for_each_vsi(pf, i, vsi) {12158 err = i40e_vsi_alloc_q_vectors(vsi);12159 if (err)12160 goto err_unwind;12161 12162 i40e_vsi_map_rings_to_vectors(vsi);12163 }12164 12165 err = i40e_setup_misc_vector(pf);12166 if (err)12167 goto err_unwind;12168 12169 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags))12170 i40e_client_update_msix_info(pf);12171 12172 return 0;12173 12174err_unwind:12175 while (i--) {12176 if (pf->vsi[i])12177 i40e_vsi_free_q_vectors(pf->vsi[i]);12178 }12179 12180 return err;12181}12182 12183/**12184 * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle12185 * non queue events in recovery mode12186 * @pf: board private structure12187 *12188 * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage12189 * the non-queue interrupts, e.g. AdminQ and errors in recovery mode.12190 * This is handled differently than in recovery mode since no Tx/Rx resources12191 * are being allocated.12192 **/12193static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf)12194{12195 int err;12196 12197 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {12198 err = i40e_setup_misc_vector(pf);12199 12200 if (err) {12201 dev_info(&pf->pdev->dev,12202 "MSI-X misc vector request failed, error %d\n",12203 err);12204 return err;12205 }12206 } else {12207 u32 flags = test_bit(I40E_FLAG_MSI_ENA, pf->flags) ? 0 : IRQF_SHARED;12208 12209 err = request_irq(pf->pdev->irq, i40e_intr, flags,12210 pf->int_name, pf);12211 12212 if (err) {12213 dev_info(&pf->pdev->dev,12214 "MSI/legacy misc vector request failed, error %d\n",12215 err);12216 return err;12217 }12218 i40e_enable_misc_int_causes(pf);12219 i40e_irq_dynamic_enable_icr0(pf);12220 }12221 12222 return 0;12223}12224 12225/**12226 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events12227 * @pf: board private structure12228 *12229 * This sets up the handler for MSIX 0, which is used to manage the12230 * non-queue interrupts, e.g. AdminQ and errors. This is not used12231 * when in MSI or Legacy interrupt mode.12232 **/12233static int i40e_setup_misc_vector(struct i40e_pf *pf)12234{12235 struct i40e_hw *hw = &pf->hw;12236 int err = 0;12237 12238 /* Only request the IRQ once, the first time through. */12239 if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) {12240 err = request_irq(pf->msix_entries[0].vector,12241 i40e_intr, 0, pf->int_name, pf);12242 if (err) {12243 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);12244 dev_info(&pf->pdev->dev,12245 "request_irq for %s failed: %d\n",12246 pf->int_name, err);12247 return -EFAULT;12248 }12249 }12250 12251 i40e_enable_misc_int_causes(pf);12252 12253 /* associate no queues to the misc vector */12254 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);12255 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1);12256 12257 i40e_flush(hw);12258 12259 i40e_irq_dynamic_enable_icr0(pf);12260 12261 return err;12262}12263 12264/**12265 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands12266 * @vsi: Pointer to vsi structure12267 * @seed: Buffter to store the hash keys12268 * @lut: Buffer to store the lookup table entries12269 * @lut_size: Size of buffer to store the lookup table entries12270 *12271 * Return 0 on success, negative on failure12272 */12273static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,12274 u8 *lut, u16 lut_size)12275{12276 struct i40e_pf *pf = vsi->back;12277 struct i40e_hw *hw = &pf->hw;12278 int ret = 0;12279 12280 if (seed) {12281 ret = i40e_aq_get_rss_key(hw, vsi->id,12282 (struct i40e_aqc_get_set_rss_key_data *)seed);12283 if (ret) {12284 dev_info(&pf->pdev->dev,12285 "Cannot get RSS key, err %pe aq_err %s\n",12286 ERR_PTR(ret),12287 i40e_aq_str(&pf->hw,12288 pf->hw.aq.asq_last_status));12289 return ret;12290 }12291 }12292 12293 if (lut) {12294 bool pf_lut = vsi->type == I40E_VSI_MAIN;12295 12296 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);12297 if (ret) {12298 dev_info(&pf->pdev->dev,12299 "Cannot get RSS lut, err %pe aq_err %s\n",12300 ERR_PTR(ret),12301 i40e_aq_str(&pf->hw,12302 pf->hw.aq.asq_last_status));12303 return ret;12304 }12305 }12306 12307 return ret;12308}12309 12310/**12311 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers12312 * @vsi: Pointer to vsi structure12313 * @seed: RSS hash seed12314 * @lut: Lookup table12315 * @lut_size: Lookup table size12316 *12317 * Returns 0 on success, negative on failure12318 **/12319static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,12320 const u8 *lut, u16 lut_size)12321{12322 struct i40e_pf *pf = vsi->back;12323 struct i40e_hw *hw = &pf->hw;12324 u16 vf_id = vsi->vf_id;12325 u8 i;12326 12327 /* Fill out hash function seed */12328 if (seed) {12329 u32 *seed_dw = (u32 *)seed;12330 12331 if (vsi->type == I40E_VSI_MAIN) {12332 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)12333 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);12334 } else if (vsi->type == I40E_VSI_SRIOV) {12335 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)12336 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);12337 } else {12338 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");12339 }12340 }12341 12342 if (lut) {12343 u32 *lut_dw = (u32 *)lut;12344 12345 if (vsi->type == I40E_VSI_MAIN) {12346 if (lut_size != I40E_HLUT_ARRAY_SIZE)12347 return -EINVAL;12348 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)12349 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);12350 } else if (vsi->type == I40E_VSI_SRIOV) {12351 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)12352 return -EINVAL;12353 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)12354 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);12355 } else {12356 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");12357 }12358 }12359 i40e_flush(hw);12360 12361 return 0;12362}12363 12364/**12365 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers12366 * @vsi: Pointer to VSI structure12367 * @seed: Buffer to store the keys12368 * @lut: Buffer to store the lookup table entries12369 * @lut_size: Size of buffer to store the lookup table entries12370 *12371 * Returns 0 on success, negative on failure12372 */12373static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,12374 u8 *lut, u16 lut_size)12375{12376 struct i40e_pf *pf = vsi->back;12377 struct i40e_hw *hw = &pf->hw;12378 u16 i;12379 12380 if (seed) {12381 u32 *seed_dw = (u32 *)seed;12382 12383 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)12384 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));12385 }12386 if (lut) {12387 u32 *lut_dw = (u32 *)lut;12388 12389 if (lut_size != I40E_HLUT_ARRAY_SIZE)12390 return -EINVAL;12391 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)12392 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));12393 }12394 12395 return 0;12396}12397 12398/**12399 * i40e_config_rss - Configure RSS keys and lut12400 * @vsi: Pointer to VSI structure12401 * @seed: RSS hash seed12402 * @lut: Lookup table12403 * @lut_size: Lookup table size12404 *12405 * Returns 0 on success, negative on failure12406 */12407int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)12408{12409 struct i40e_pf *pf = vsi->back;12410 12411 if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))12412 return i40e_config_rss_aq(vsi, seed, lut, lut_size);12413 else12414 return i40e_config_rss_reg(vsi, seed, lut, lut_size);12415}12416 12417/**12418 * i40e_get_rss - Get RSS keys and lut12419 * @vsi: Pointer to VSI structure12420 * @seed: Buffer to store the keys12421 * @lut: Buffer to store the lookup table entries12422 * @lut_size: Size of buffer to store the lookup table entries12423 *12424 * Returns 0 on success, negative on failure12425 */12426int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)12427{12428 struct i40e_pf *pf = vsi->back;12429 12430 if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))12431 return i40e_get_rss_aq(vsi, seed, lut, lut_size);12432 else12433 return i40e_get_rss_reg(vsi, seed, lut, lut_size);12434}12435 12436/**12437 * i40e_fill_rss_lut - Fill the RSS lookup table with default values12438 * @pf: Pointer to board private structure12439 * @lut: Lookup table12440 * @rss_table_size: Lookup table size12441 * @rss_size: Range of queue number for hashing12442 */12443void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,12444 u16 rss_table_size, u16 rss_size)12445{12446 u16 i;12447 12448 for (i = 0; i < rss_table_size; i++)12449 lut[i] = i % rss_size;12450}12451 12452/**12453 * i40e_pf_config_rss - Prepare for RSS if used12454 * @pf: board private structure12455 **/12456static int i40e_pf_config_rss(struct i40e_pf *pf)12457{12458 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);12459 u8 seed[I40E_HKEY_ARRAY_SIZE];12460 u8 *lut;12461 struct i40e_hw *hw = &pf->hw;12462 u32 reg_val;12463 u64 hena;12464 int ret;12465 12466 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */12467 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |12468 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);12469 hena |= i40e_pf_get_default_rss_hena(pf);12470 12471 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);12472 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));12473 12474 /* Determine the RSS table size based on the hardware capabilities */12475 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);12476 reg_val = (pf->rss_table_size == 512) ?12477 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :12478 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);12479 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);12480 12481 /* Determine the RSS size of the VSI */12482 if (!vsi->rss_size) {12483 u16 qcount;12484 /* If the firmware does something weird during VSI init, we12485 * could end up with zero TCs. Check for that to avoid12486 * divide-by-zero. It probably won't pass traffic, but it also12487 * won't panic.12488 */12489 qcount = vsi->num_queue_pairs /12490 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1);12491 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);12492 }12493 if (!vsi->rss_size)12494 return -EINVAL;12495 12496 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);12497 if (!lut)12498 return -ENOMEM;12499 12500 /* Use user configured lut if there is one, otherwise use default */12501 if (vsi->rss_lut_user)12502 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);12503 else12504 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);12505 12506 /* Use user configured hash key if there is one, otherwise12507 * use default.12508 */12509 if (vsi->rss_hkey_user)12510 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);12511 else12512 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);12513 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);12514 kfree(lut);12515 12516 return ret;12517}12518 12519/**12520 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild12521 * @pf: board private structure12522 * @queue_count: the requested queue count for rss.12523 *12524 * returns 0 if rss is not enabled, if enabled returns the final rss queue12525 * count which may be different from the requested queue count.12526 * Note: expects to be called while under rtnl_lock()12527 **/12528int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)12529{12530 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);12531 int new_rss_size;12532 12533 if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags))12534 return 0;12535 12536 queue_count = min_t(int, queue_count, num_online_cpus());12537 new_rss_size = min_t(int, queue_count, pf->rss_size_max);12538 12539 if (queue_count != vsi->num_queue_pairs) {12540 u16 qcount;12541 12542 vsi->req_queue_pairs = queue_count;12543 i40e_prep_for_reset(pf);12544 if (test_bit(__I40E_IN_REMOVE, pf->state))12545 return pf->alloc_rss_size;12546 12547 pf->alloc_rss_size = new_rss_size;12548 12549 i40e_reset_and_rebuild(pf, true, true);12550 12551 /* Discard the user configured hash keys and lut, if less12552 * queues are enabled.12553 */12554 if (queue_count < vsi->rss_size) {12555 i40e_clear_rss_config_user(vsi);12556 dev_dbg(&pf->pdev->dev,12557 "discard user configured hash keys and lut\n");12558 }12559 12560 /* Reset vsi->rss_size, as number of enabled queues changed */12561 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;12562 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);12563 12564 i40e_pf_config_rss(pf);12565 }12566 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n",12567 vsi->req_queue_pairs, pf->rss_size_max);12568 return pf->alloc_rss_size;12569}12570 12571/**12572 * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition12573 * @pf: board private structure12574 **/12575int i40e_get_partition_bw_setting(struct i40e_pf *pf)12576{12577 bool min_valid, max_valid;12578 u32 max_bw, min_bw;12579 int status;12580 12581 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,12582 &min_valid, &max_valid);12583 12584 if (!status) {12585 if (min_valid)12586 pf->min_bw = min_bw;12587 if (max_valid)12588 pf->max_bw = max_bw;12589 }12590 12591 return status;12592}12593 12594/**12595 * i40e_set_partition_bw_setting - Set BW settings for this PF partition12596 * @pf: board private structure12597 **/12598int i40e_set_partition_bw_setting(struct i40e_pf *pf)12599{12600 struct i40e_aqc_configure_partition_bw_data bw_data;12601 int status;12602 12603 memset(&bw_data, 0, sizeof(bw_data));12604 12605 /* Set the valid bit for this PF */12606 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));12607 bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;12608 bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;12609 12610 /* Set the new bandwidths */12611 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);12612 12613 return status;12614}12615 12616/**12617 * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition12618 * @pf: board private structure12619 **/12620int i40e_commit_partition_bw_setting(struct i40e_pf *pf)12621{12622 /* Commit temporary BW setting to permanent NVM image */12623 enum i40e_admin_queue_err last_aq_status;12624 u16 nvm_word;12625 int ret;12626 12627 if (pf->hw.partition_id != 1) {12628 dev_info(&pf->pdev->dev,12629 "Commit BW only works on partition 1! This is partition %d",12630 pf->hw.partition_id);12631 ret = -EOPNOTSUPP;12632 goto bw_commit_out;12633 }12634 12635 /* Acquire NVM for read access */12636 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);12637 last_aq_status = pf->hw.aq.asq_last_status;12638 if (ret) {12639 dev_info(&pf->pdev->dev,12640 "Cannot acquire NVM for read access, err %pe aq_err %s\n",12641 ERR_PTR(ret),12642 i40e_aq_str(&pf->hw, last_aq_status));12643 goto bw_commit_out;12644 }12645 12646 /* Read word 0x10 of NVM - SW compatibility word 1 */12647 ret = i40e_aq_read_nvm(&pf->hw,12648 I40E_SR_NVM_CONTROL_WORD,12649 0x10, sizeof(nvm_word), &nvm_word,12650 false, NULL);12651 /* Save off last admin queue command status before releasing12652 * the NVM12653 */12654 last_aq_status = pf->hw.aq.asq_last_status;12655 i40e_release_nvm(&pf->hw);12656 if (ret) {12657 dev_info(&pf->pdev->dev, "NVM read error, err %pe aq_err %s\n",12658 ERR_PTR(ret),12659 i40e_aq_str(&pf->hw, last_aq_status));12660 goto bw_commit_out;12661 }12662 12663 /* Wait a bit for NVM release to complete */12664 msleep(50);12665 12666 /* Acquire NVM for write access */12667 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);12668 last_aq_status = pf->hw.aq.asq_last_status;12669 if (ret) {12670 dev_info(&pf->pdev->dev,12671 "Cannot acquire NVM for write access, err %pe aq_err %s\n",12672 ERR_PTR(ret),12673 i40e_aq_str(&pf->hw, last_aq_status));12674 goto bw_commit_out;12675 }12676 /* Write it back out unchanged to initiate update NVM,12677 * which will force a write of the shadow (alt) RAM to12678 * the NVM - thus storing the bandwidth values permanently.12679 */12680 ret = i40e_aq_update_nvm(&pf->hw,12681 I40E_SR_NVM_CONTROL_WORD,12682 0x10, sizeof(nvm_word),12683 &nvm_word, true, 0, NULL);12684 /* Save off last admin queue command status before releasing12685 * the NVM12686 */12687 last_aq_status = pf->hw.aq.asq_last_status;12688 i40e_release_nvm(&pf->hw);12689 if (ret)12690 dev_info(&pf->pdev->dev,12691 "BW settings NOT SAVED, err %pe aq_err %s\n",12692 ERR_PTR(ret),12693 i40e_aq_str(&pf->hw, last_aq_status));12694bw_commit_out:12695 12696 return ret;12697}12698 12699/**12700 * i40e_is_total_port_shutdown_enabled - read NVM and return value12701 * if total port shutdown feature is enabled for this PF12702 * @pf: board private structure12703 **/12704static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf)12705{12706#define I40E_TOTAL_PORT_SHUTDOWN_ENABLED BIT(4)12707#define I40E_FEATURES_ENABLE_PTR 0x2A12708#define I40E_CURRENT_SETTING_PTR 0x2B12709#define I40E_LINK_BEHAVIOR_WORD_OFFSET 0x2D12710#define I40E_LINK_BEHAVIOR_WORD_LENGTH 0x112711#define I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED BIT(0)12712#define I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH 412713 u16 sr_emp_sr_settings_ptr = 0;12714 u16 features_enable = 0;12715 u16 link_behavior = 0;12716 int read_status = 0;12717 bool ret = false;12718 12719 read_status = i40e_read_nvm_word(&pf->hw,12720 I40E_SR_EMP_SR_SETTINGS_PTR,12721 &sr_emp_sr_settings_ptr);12722 if (read_status)12723 goto err_nvm;12724 read_status = i40e_read_nvm_word(&pf->hw,12725 sr_emp_sr_settings_ptr +12726 I40E_FEATURES_ENABLE_PTR,12727 &features_enable);12728 if (read_status)12729 goto err_nvm;12730 if (I40E_TOTAL_PORT_SHUTDOWN_ENABLED & features_enable) {12731 read_status = i40e_read_nvm_module_data(&pf->hw,12732 I40E_SR_EMP_SR_SETTINGS_PTR,12733 I40E_CURRENT_SETTING_PTR,12734 I40E_LINK_BEHAVIOR_WORD_OFFSET,12735 I40E_LINK_BEHAVIOR_WORD_LENGTH,12736 &link_behavior);12737 if (read_status)12738 goto err_nvm;12739 link_behavior >>= (pf->hw.port * I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH);12740 ret = I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED & link_behavior;12741 }12742 return ret;12743 12744err_nvm:12745 dev_warn(&pf->pdev->dev,12746 "total-port-shutdown feature is off due to read nvm error: %pe\n",12747 ERR_PTR(read_status));12748 return ret;12749}12750 12751/**12752 * i40e_sw_init - Initialize general software structures (struct i40e_pf)12753 * @pf: board private structure to initialize12754 *12755 * i40e_sw_init initializes the Adapter private data structure.12756 * Fields are initialized based on PCI device information and12757 * OS network device settings (MTU size).12758 **/12759static int i40e_sw_init(struct i40e_pf *pf)12760{12761 int err = 0;12762 int size;12763 u16 pow;12764 12765 /* Set default capability flags */12766 bitmap_zero(pf->flags, I40E_PF_FLAGS_NBITS);12767 set_bit(I40E_FLAG_MSI_ENA, pf->flags);12768 set_bit(I40E_FLAG_MSIX_ENA, pf->flags);12769 12770 /* Set default ITR */12771 pf->rx_itr_default = I40E_ITR_RX_DEF;12772 pf->tx_itr_default = I40E_ITR_TX_DEF;12773 12774 /* Depending on PF configurations, it is possible that the RSS12775 * maximum might end up larger than the available queues12776 */12777 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);12778 pf->alloc_rss_size = 1;12779 pf->rss_table_size = pf->hw.func_caps.rss_table_size;12780 pf->rss_size_max = min_t(int, pf->rss_size_max,12781 pf->hw.func_caps.num_tx_qp);12782 12783 /* find the next higher power-of-2 of num cpus */12784 pow = roundup_pow_of_two(num_online_cpus());12785 pf->rss_size_max = min_t(int, pf->rss_size_max, pow);12786 12787 if (pf->hw.func_caps.rss) {12788 set_bit(I40E_FLAG_RSS_ENA, pf->flags);12789 pf->alloc_rss_size = min_t(int, pf->rss_size_max,12790 num_online_cpus());12791 }12792 12793 /* MFP mode enabled */12794 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {12795 set_bit(I40E_FLAG_MFP_ENA, pf->flags);12796 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");12797 if (i40e_get_partition_bw_setting(pf)) {12798 dev_warn(&pf->pdev->dev,12799 "Could not get partition bw settings\n");12800 } else {12801 dev_info(&pf->pdev->dev,12802 "Partition BW Min = %8.8x, Max = %8.8x\n",12803 pf->min_bw, pf->max_bw);12804 12805 /* nudge the Tx scheduler */12806 i40e_set_partition_bw_setting(pf);12807 }12808 }12809 12810 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||12811 (pf->hw.func_caps.fd_filters_best_effort > 0)) {12812 set_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);12813 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) &&12814 pf->hw.num_partitions > 1)12815 dev_info(&pf->pdev->dev,12816 "Flow Director Sideband mode Disabled in MFP mode\n");12817 else12818 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);12819 pf->fdir_pf_filter_count =12820 pf->hw.func_caps.fd_filters_guaranteed;12821 pf->hw.fdir_shared_filter_count =12822 pf->hw.func_caps.fd_filters_best_effort;12823 }12824 12825 /* Enable HW ATR eviction if possible */12826 if (test_bit(I40E_HW_CAP_ATR_EVICT, pf->hw.caps))12827 set_bit(I40E_FLAG_HW_ATR_EVICT_ENA, pf->flags);12828 12829 if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) {12830 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;12831 set_bit(I40E_FLAG_VMDQ_ENA, pf->flags);12832 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);12833 }12834 12835 if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) {12836 set_bit(I40E_FLAG_IWARP_ENA, pf->flags);12837 /* IWARP needs one extra vector for CQP just like MISC.*/12838 pf->num_iwarp_msix = (int)num_online_cpus() + 1;12839 }12840 /* Stopping FW LLDP engine is supported on XL710 and X72212841 * starting from FW versions determined in i40e_init_adminq.12842 * Stopping the FW LLDP engine is not supported on XL71012843 * if NPAR is functioning so unset this hw flag in this case.12844 */12845 if (pf->hw.mac.type == I40E_MAC_XL710 &&12846 pf->hw.func_caps.npar_enable)12847 clear_bit(I40E_HW_CAP_FW_LLDP_STOPPABLE, pf->hw.caps);12848 12849#ifdef CONFIG_PCI_IOV12850 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {12851 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;12852 set_bit(I40E_FLAG_SRIOV_ENA, pf->flags);12853 pf->num_req_vfs = min_t(int,12854 pf->hw.func_caps.num_vfs,12855 I40E_MAX_VF_COUNT);12856 }12857#endif /* CONFIG_PCI_IOV */12858 pf->lan_veb = I40E_NO_VEB;12859 pf->lan_vsi = I40E_NO_VSI;12860 12861 /* By default FW has this off for performance reasons */12862 clear_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags);12863 12864 /* set up queue assignment tracking */12865 size = sizeof(struct i40e_lump_tracking)12866 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);12867 pf->qp_pile = kzalloc(size, GFP_KERNEL);12868 if (!pf->qp_pile) {12869 err = -ENOMEM;12870 goto sw_init_done;12871 }12872 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;12873 12874 pf->tx_timeout_recovery_level = 1;12875 12876 if (pf->hw.mac.type != I40E_MAC_X722 &&12877 i40e_is_total_port_shutdown_enabled(pf)) {12878 /* Link down on close must be on when total port shutdown12879 * is enabled for a given port12880 */12881 set_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags);12882 set_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags);12883 dev_info(&pf->pdev->dev,12884 "total-port-shutdown was enabled, link-down-on-close is forced on\n");12885 }12886 mutex_init(&pf->switch_mutex);12887 12888sw_init_done:12889 return err;12890}12891 12892/**12893 * i40e_set_ntuple - set the ntuple feature flag and take action12894 * @pf: board private structure to initialize12895 * @features: the feature set that the stack is suggesting12896 *12897 * returns a bool to indicate if reset needs to happen12898 **/12899bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)12900{12901 bool need_reset = false;12902 12903 /* Check if Flow Director n-tuple support was enabled or disabled. If12904 * the state changed, we need to reset.12905 */12906 if (features & NETIF_F_NTUPLE) {12907 /* Enable filters and mark for reset */12908 if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))12909 need_reset = true;12910 /* enable FD_SB only if there is MSI-X vector and no cloud12911 * filters exist12912 */12913 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) {12914 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);12915 clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);12916 }12917 } else {12918 /* turn off filters, mark for reset and clear SW filter list */12919 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {12920 need_reset = true;12921 i40e_fdir_filter_exit(pf);12922 }12923 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);12924 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state);12925 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);12926 12927 /* reset fd counters */12928 pf->fd_add_err = 0;12929 pf->fd_atr_cnt = 0;12930 /* if ATR was auto disabled it can be re-enabled. */12931 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))12932 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&12933 (I40E_DEBUG_FD & pf->hw.debug_mask))12934 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");12935 }12936 return need_reset;12937}12938 12939/**12940 * i40e_clear_rss_lut - clear the rx hash lookup table12941 * @vsi: the VSI being configured12942 **/12943static void i40e_clear_rss_lut(struct i40e_vsi *vsi)12944{12945 struct i40e_pf *pf = vsi->back;12946 struct i40e_hw *hw = &pf->hw;12947 u16 vf_id = vsi->vf_id;12948 u8 i;12949 12950 if (vsi->type == I40E_VSI_MAIN) {12951 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)12952 wr32(hw, I40E_PFQF_HLUT(i), 0);12953 } else if (vsi->type == I40E_VSI_SRIOV) {12954 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)12955 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);12956 } else {12957 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");12958 }12959}12960 12961/**12962 * i40e_set_loopback - turn on/off loopback mode on underlying PF12963 * @vsi: ptr to VSI12964 * @ena: flag to indicate the on/off setting12965 */12966static int i40e_set_loopback(struct i40e_vsi *vsi, bool ena)12967{12968 bool if_running = netif_running(vsi->netdev) &&12969 !test_and_set_bit(__I40E_VSI_DOWN, vsi->state);12970 int ret;12971 12972 if (if_running)12973 i40e_down(vsi);12974 12975 ret = i40e_aq_set_mac_loopback(&vsi->back->hw, ena, NULL);12976 if (ret)12977 netdev_err(vsi->netdev, "Failed to toggle loopback state\n");12978 if (if_running)12979 i40e_up(vsi);12980 12981 return ret;12982}12983 12984/**12985 * i40e_set_features - set the netdev feature flags12986 * @netdev: ptr to the netdev being adjusted12987 * @features: the feature set that the stack is suggesting12988 * Note: expects to be called while under rtnl_lock()12989 **/12990static int i40e_set_features(struct net_device *netdev,12991 netdev_features_t features)12992{12993 struct i40e_netdev_priv *np = netdev_priv(netdev);12994 struct i40e_vsi *vsi = np->vsi;12995 struct i40e_pf *pf = vsi->back;12996 bool need_reset;12997 12998 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))12999 i40e_pf_config_rss(pf);13000 else if (!(features & NETIF_F_RXHASH) &&13001 netdev->features & NETIF_F_RXHASH)13002 i40e_clear_rss_lut(vsi);13003 13004 if (features & NETIF_F_HW_VLAN_CTAG_RX)13005 i40e_vlan_stripping_enable(vsi);13006 else13007 i40e_vlan_stripping_disable(vsi);13008 13009 if (!(features & NETIF_F_HW_TC) &&13010 (netdev->features & NETIF_F_HW_TC) && pf->num_cloud_filters) {13011 dev_err(&pf->pdev->dev,13012 "Offloaded tc filters active, can't turn hw_tc_offload off");13013 return -EINVAL;13014 }13015 13016 if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt)13017 i40e_del_all_macvlans(vsi);13018 13019 need_reset = i40e_set_ntuple(pf, features);13020 13021 if (need_reset)13022 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);13023 13024 if ((features ^ netdev->features) & NETIF_F_LOOPBACK)13025 return i40e_set_loopback(vsi, !!(features & NETIF_F_LOOPBACK));13026 13027 return 0;13028}13029 13030static int i40e_udp_tunnel_set_port(struct net_device *netdev,13031 unsigned int table, unsigned int idx,13032 struct udp_tunnel_info *ti)13033{13034 struct i40e_netdev_priv *np = netdev_priv(netdev);13035 struct i40e_hw *hw = &np->vsi->back->hw;13036 u8 type, filter_index;13037 int ret;13038 13039 type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? I40E_AQC_TUNNEL_TYPE_VXLAN :13040 I40E_AQC_TUNNEL_TYPE_NGE;13041 13042 ret = i40e_aq_add_udp_tunnel(hw, ntohs(ti->port), type, &filter_index,13043 NULL);13044 if (ret) {13045 netdev_info(netdev, "add UDP port failed, err %pe aq_err %s\n",13046 ERR_PTR(ret),13047 i40e_aq_str(hw, hw->aq.asq_last_status));13048 return -EIO;13049 }13050 13051 udp_tunnel_nic_set_port_priv(netdev, table, idx, filter_index);13052 return 0;13053}13054 13055static int i40e_udp_tunnel_unset_port(struct net_device *netdev,13056 unsigned int table, unsigned int idx,13057 struct udp_tunnel_info *ti)13058{13059 struct i40e_netdev_priv *np = netdev_priv(netdev);13060 struct i40e_hw *hw = &np->vsi->back->hw;13061 int ret;13062 13063 ret = i40e_aq_del_udp_tunnel(hw, ti->hw_priv, NULL);13064 if (ret) {13065 netdev_info(netdev, "delete UDP port failed, err %pe aq_err %s\n",13066 ERR_PTR(ret),13067 i40e_aq_str(hw, hw->aq.asq_last_status));13068 return -EIO;13069 }13070 13071 return 0;13072}13073 13074static int i40e_get_phys_port_id(struct net_device *netdev,13075 struct netdev_phys_item_id *ppid)13076{13077 struct i40e_netdev_priv *np = netdev_priv(netdev);13078 struct i40e_pf *pf = np->vsi->back;13079 struct i40e_hw *hw = &pf->hw;13080 13081 if (!test_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps))13082 return -EOPNOTSUPP;13083 13084 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));13085 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);13086 13087 return 0;13088}13089 13090/**13091 * i40e_ndo_fdb_add - add an entry to the hardware database13092 * @ndm: the input from the stack13093 * @tb: pointer to array of nladdr (unused)13094 * @dev: the net device pointer13095 * @addr: the MAC address entry being added13096 * @vid: VLAN ID13097 * @flags: instructions from stack about fdb operation13098 * @extack: netlink extended ack, unused currently13099 */13100static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],13101 struct net_device *dev,13102 const unsigned char *addr, u16 vid,13103 u16 flags,13104 struct netlink_ext_ack *extack)13105{13106 struct i40e_netdev_priv *np = netdev_priv(dev);13107 struct i40e_pf *pf = np->vsi->back;13108 int err = 0;13109 13110 if (!test_bit(I40E_FLAG_SRIOV_ENA, pf->flags))13111 return -EOPNOTSUPP;13112 13113 if (vid) {13114 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);13115 return -EINVAL;13116 }13117 13118 /* Hardware does not support aging addresses so if a13119 * ndm_state is given only allow permanent addresses13120 */13121 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {13122 netdev_info(dev, "FDB only supports static addresses\n");13123 return -EINVAL;13124 }13125 13126 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))13127 err = dev_uc_add_excl(dev, addr);13128 else if (is_multicast_ether_addr(addr))13129 err = dev_mc_add_excl(dev, addr);13130 else13131 err = -EINVAL;13132 13133 /* Only return duplicate errors if NLM_F_EXCL is set */13134 if (err == -EEXIST && !(flags & NLM_F_EXCL))13135 err = 0;13136 13137 return err;13138}13139 13140/**13141 * i40e_ndo_bridge_setlink - Set the hardware bridge mode13142 * @dev: the netdev being configured13143 * @nlh: RTNL message13144 * @flags: bridge flags13145 * @extack: netlink extended ack13146 *13147 * Inserts a new hardware bridge if not already created and13148 * enables the bridging mode requested (VEB or VEPA). If the13149 * hardware bridge has already been inserted and the request13150 * is to change the mode then that requires a PF reset to13151 * allow rebuild of the components with required hardware13152 * bridge mode enabled.13153 *13154 * Note: expects to be called while under rtnl_lock()13155 **/13156static int i40e_ndo_bridge_setlink(struct net_device *dev,13157 struct nlmsghdr *nlh,13158 u16 flags,13159 struct netlink_ext_ack *extack)13160{13161 struct i40e_netdev_priv *np = netdev_priv(dev);13162 struct i40e_vsi *vsi = np->vsi;13163 struct i40e_pf *pf = vsi->back;13164 struct nlattr *attr, *br_spec;13165 struct i40e_veb *veb;13166 int rem;13167 13168 /* Only for PF VSI for now */13169 if (vsi->type != I40E_VSI_MAIN)13170 return -EOPNOTSUPP;13171 13172 /* Find the HW bridge for PF VSI */13173 veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid);13174 13175 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);13176 if (!br_spec)13177 return -EINVAL;13178 13179 nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) {13180 __u16 mode = nla_get_u16(attr);13181 13182 if ((mode != BRIDGE_MODE_VEPA) &&13183 (mode != BRIDGE_MODE_VEB))13184 return -EINVAL;13185 13186 /* Insert a new HW bridge */13187 if (!veb) {13188 veb = i40e_veb_setup(pf, vsi->uplink_seid, vsi->seid,13189 vsi->tc_config.enabled_tc);13190 if (veb) {13191 veb->bridge_mode = mode;13192 i40e_config_bridge_mode(veb);13193 } else {13194 /* No Bridge HW offload available */13195 return -ENOENT;13196 }13197 break;13198 } else if (mode != veb->bridge_mode) {13199 /* Existing HW bridge but different mode needs reset */13200 veb->bridge_mode = mode;13201 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */13202 if (mode == BRIDGE_MODE_VEB)13203 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);13204 else13205 clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);13206 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);13207 break;13208 }13209 }13210 13211 return 0;13212}13213 13214/**13215 * i40e_ndo_bridge_getlink - Get the hardware bridge mode13216 * @skb: skb buff13217 * @pid: process id13218 * @seq: RTNL message seq #13219 * @dev: the netdev being configured13220 * @filter_mask: unused13221 * @nlflags: netlink flags passed in13222 *13223 * Return the mode in which the hardware bridge is operating in13224 * i.e VEB or VEPA.13225 **/13226static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,13227 struct net_device *dev,13228 u32 __always_unused filter_mask,13229 int nlflags)13230{13231 struct i40e_netdev_priv *np = netdev_priv(dev);13232 struct i40e_vsi *vsi = np->vsi;13233 struct i40e_pf *pf = vsi->back;13234 struct i40e_veb *veb;13235 13236 /* Only for PF VSI for now */13237 if (vsi->type != I40E_VSI_MAIN)13238 return -EOPNOTSUPP;13239 13240 /* Find the HW bridge for the PF VSI */13241 veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid);13242 if (!veb)13243 return 0;13244 13245 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,13246 0, 0, nlflags, filter_mask, NULL);13247}13248 13249/**13250 * i40e_features_check - Validate encapsulated packet conforms to limits13251 * @skb: skb buff13252 * @dev: This physical port's netdev13253 * @features: Offload features that the stack believes apply13254 **/13255static netdev_features_t i40e_features_check(struct sk_buff *skb,13256 struct net_device *dev,13257 netdev_features_t features)13258{13259 size_t len;13260 13261 /* No point in doing any of this if neither checksum nor GSO are13262 * being requested for this frame. We can rule out both by just13263 * checking for CHECKSUM_PARTIAL13264 */13265 if (skb->ip_summed != CHECKSUM_PARTIAL)13266 return features;13267 13268 /* We cannot support GSO if the MSS is going to be less than13269 * 64 bytes. If it is then we need to drop support for GSO.13270 */13271 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))13272 features &= ~NETIF_F_GSO_MASK;13273 13274 /* MACLEN can support at most 63 words */13275 len = skb_network_offset(skb);13276 if (len & ~(63 * 2))13277 goto out_err;13278 13279 /* IPLEN and EIPLEN can support at most 127 dwords */13280 len = skb_network_header_len(skb);13281 if (len & ~(127 * 4))13282 goto out_err;13283 13284 if (skb->encapsulation) {13285 /* L4TUNLEN can support 127 words */13286 len = skb_inner_network_header(skb) - skb_transport_header(skb);13287 if (len & ~(127 * 2))13288 goto out_err;13289 13290 /* IPLEN can support at most 127 dwords */13291 len = skb_inner_transport_header(skb) -13292 skb_inner_network_header(skb);13293 if (len & ~(127 * 4))13294 goto out_err;13295 }13296 13297 /* No need to validate L4LEN as TCP is the only protocol with a13298 * flexible value and we support all possible values supported13299 * by TCP, which is at most 15 dwords13300 */13301 13302 return features;13303out_err:13304 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);13305}13306 13307/**13308 * i40e_xdp_setup - add/remove an XDP program13309 * @vsi: VSI to changed13310 * @prog: XDP program13311 * @extack: netlink extended ack13312 **/13313static int i40e_xdp_setup(struct i40e_vsi *vsi, struct bpf_prog *prog,13314 struct netlink_ext_ack *extack)13315{13316 int frame_size = i40e_max_vsi_frame_size(vsi, prog);13317 struct i40e_pf *pf = vsi->back;13318 struct bpf_prog *old_prog;13319 bool need_reset;13320 int i;13321 13322 /* VSI shall be deleted in a moment, block loading new programs */13323 if (prog && test_bit(__I40E_IN_REMOVE, pf->state))13324 return -EINVAL;13325 13326 /* Don't allow frames that span over multiple buffers */13327 if (vsi->netdev->mtu > frame_size - I40E_PACKET_HDR_PAD) {13328 NL_SET_ERR_MSG_MOD(extack, "MTU too large for linear frames and XDP prog does not support frags");13329 return -EINVAL;13330 }13331 13332 /* When turning XDP on->off/off->on we reset and rebuild the rings. */13333 need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);13334 if (need_reset)13335 i40e_prep_for_reset(pf);13336 13337 old_prog = xchg(&vsi->xdp_prog, prog);13338 13339 if (need_reset) {13340 if (!prog) {13341 xdp_features_clear_redirect_target(vsi->netdev);13342 /* Wait until ndo_xsk_wakeup completes. */13343 synchronize_rcu();13344 }13345 i40e_reset_and_rebuild(pf, true, true);13346 }13347 13348 if (!i40e_enabled_xdp_vsi(vsi) && prog) {13349 if (i40e_realloc_rx_bi_zc(vsi, true))13350 return -ENOMEM;13351 } else if (i40e_enabled_xdp_vsi(vsi) && !prog) {13352 if (i40e_realloc_rx_bi_zc(vsi, false))13353 return -ENOMEM;13354 }13355 13356 for (i = 0; i < vsi->num_queue_pairs; i++)13357 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);13358 13359 if (old_prog)13360 bpf_prog_put(old_prog);13361 13362 /* Kick start the NAPI context if there is an AF_XDP socket open13363 * on that queue id. This so that receiving will start.13364 */13365 if (need_reset && prog) {13366 for (i = 0; i < vsi->num_queue_pairs; i++)13367 if (vsi->xdp_rings[i]->xsk_pool)13368 (void)i40e_xsk_wakeup(vsi->netdev, i,13369 XDP_WAKEUP_RX);13370 xdp_features_set_redirect_target(vsi->netdev, true);13371 }13372 13373 return 0;13374}13375 13376/**13377 * i40e_enter_busy_conf - Enters busy config state13378 * @vsi: vsi13379 *13380 * Returns 0 on success, <0 for failure.13381 **/13382static int i40e_enter_busy_conf(struct i40e_vsi *vsi)13383{13384 struct i40e_pf *pf = vsi->back;13385 int timeout = 50;13386 13387 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) {13388 timeout--;13389 if (!timeout)13390 return -EBUSY;13391 usleep_range(1000, 2000);13392 }13393 13394 return 0;13395}13396 13397/**13398 * i40e_exit_busy_conf - Exits busy config state13399 * @vsi: vsi13400 **/13401static void i40e_exit_busy_conf(struct i40e_vsi *vsi)13402{13403 struct i40e_pf *pf = vsi->back;13404 13405 clear_bit(__I40E_CONFIG_BUSY, pf->state);13406}13407 13408/**13409 * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair13410 * @vsi: vsi13411 * @queue_pair: queue pair13412 **/13413static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair)13414{13415 memset(&vsi->rx_rings[queue_pair]->rx_stats, 0,13416 sizeof(vsi->rx_rings[queue_pair]->rx_stats));13417 memset(&vsi->tx_rings[queue_pair]->stats, 0,13418 sizeof(vsi->tx_rings[queue_pair]->stats));13419 if (i40e_enabled_xdp_vsi(vsi)) {13420 memset(&vsi->xdp_rings[queue_pair]->stats, 0,13421 sizeof(vsi->xdp_rings[queue_pair]->stats));13422 }13423}13424 13425/**13426 * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair13427 * @vsi: vsi13428 * @queue_pair: queue pair13429 **/13430static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair)13431{13432 i40e_clean_tx_ring(vsi->tx_rings[queue_pair]);13433 if (i40e_enabled_xdp_vsi(vsi)) {13434 /* Make sure that in-progress ndo_xdp_xmit calls are13435 * completed.13436 */13437 synchronize_rcu();13438 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]);13439 }13440 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);13441}13442 13443/**13444 * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair13445 * @vsi: vsi13446 * @queue_pair: queue pair13447 * @enable: true for enable, false for disable13448 **/13449static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair,13450 bool enable)13451{13452 struct i40e_ring *rxr = vsi->rx_rings[queue_pair];13453 struct i40e_q_vector *q_vector = rxr->q_vector;13454 13455 if (!vsi->netdev)13456 return;13457 13458 /* All rings in a qp belong to the same qvector. */13459 if (q_vector->rx.ring || q_vector->tx.ring) {13460 if (enable)13461 napi_enable(&q_vector->napi);13462 else13463 napi_disable(&q_vector->napi);13464 }13465}13466 13467/**13468 * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair13469 * @vsi: vsi13470 * @queue_pair: queue pair13471 * @enable: true for enable, false for disable13472 *13473 * Returns 0 on success, <0 on failure.13474 **/13475static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair,13476 bool enable)13477{13478 struct i40e_pf *pf = vsi->back;13479 int pf_q, ret = 0;13480 13481 pf_q = vsi->base_queue + queue_pair;13482 ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q,13483 false /*is xdp*/, enable);13484 if (ret) {13485 dev_info(&pf->pdev->dev,13486 "VSI seid %d Tx ring %d %sable timeout\n",13487 vsi->seid, pf_q, (enable ? "en" : "dis"));13488 return ret;13489 }13490 13491 i40e_control_rx_q(pf, pf_q, enable);13492 ret = i40e_pf_rxq_wait(pf, pf_q, enable);13493 if (ret) {13494 dev_info(&pf->pdev->dev,13495 "VSI seid %d Rx ring %d %sable timeout\n",13496 vsi->seid, pf_q, (enable ? "en" : "dis"));13497 return ret;13498 }13499 13500 /* Due to HW errata, on Rx disable only, the register can13501 * indicate done before it really is. Needs 50ms to be sure13502 */13503 if (!enable)13504 mdelay(50);13505 13506 if (!i40e_enabled_xdp_vsi(vsi))13507 return ret;13508 13509 ret = i40e_control_wait_tx_q(vsi->seid, pf,13510 pf_q + vsi->alloc_queue_pairs,13511 true /*is xdp*/, enable);13512 if (ret) {13513 dev_info(&pf->pdev->dev,13514 "VSI seid %d XDP Tx ring %d %sable timeout\n",13515 vsi->seid, pf_q, (enable ? "en" : "dis"));13516 }13517 13518 return ret;13519}13520 13521/**13522 * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair13523 * @vsi: vsi13524 * @queue_pair: queue_pair13525 **/13526static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair)13527{13528 struct i40e_ring *rxr = vsi->rx_rings[queue_pair];13529 struct i40e_pf *pf = vsi->back;13530 struct i40e_hw *hw = &pf->hw;13531 13532 /* All rings in a qp belong to the same qvector. */13533 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))13534 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx);13535 else13536 i40e_irq_dynamic_enable_icr0(pf);13537 13538 i40e_flush(hw);13539}13540 13541/**13542 * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair13543 * @vsi: vsi13544 * @queue_pair: queue_pair13545 **/13546static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair)13547{13548 struct i40e_ring *rxr = vsi->rx_rings[queue_pair];13549 struct i40e_pf *pf = vsi->back;13550 struct i40e_hw *hw = &pf->hw;13551 13552 /* For simplicity, instead of removing the qp interrupt causes13553 * from the interrupt linked list, we simply disable the interrupt, and13554 * leave the list intact.13555 *13556 * All rings in a qp belong to the same qvector.13557 */13558 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {13559 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx;13560 13561 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0);13562 i40e_flush(hw);13563 synchronize_irq(pf->msix_entries[intpf].vector);13564 } else {13565 /* Legacy and MSI mode - this stops all interrupt handling */13566 wr32(hw, I40E_PFINT_ICR0_ENA, 0);13567 wr32(hw, I40E_PFINT_DYN_CTL0, 0);13568 i40e_flush(hw);13569 synchronize_irq(pf->pdev->irq);13570 }13571}13572 13573/**13574 * i40e_queue_pair_disable - Disables a queue pair13575 * @vsi: vsi13576 * @queue_pair: queue pair13577 *13578 * Returns 0 on success, <0 on failure.13579 **/13580int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair)13581{13582 int err;13583 13584 err = i40e_enter_busy_conf(vsi);13585 if (err)13586 return err;13587 13588 i40e_queue_pair_disable_irq(vsi, queue_pair);13589 i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */);13590 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */);13591 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);13592 i40e_queue_pair_clean_rings(vsi, queue_pair);13593 i40e_queue_pair_reset_stats(vsi, queue_pair);13594 13595 return err;13596}13597 13598/**13599 * i40e_queue_pair_enable - Enables a queue pair13600 * @vsi: vsi13601 * @queue_pair: queue pair13602 *13603 * Returns 0 on success, <0 on failure.13604 **/13605int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair)13606{13607 int err;13608 13609 err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]);13610 if (err)13611 return err;13612 13613 if (i40e_enabled_xdp_vsi(vsi)) {13614 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]);13615 if (err)13616 return err;13617 }13618 13619 err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]);13620 if (err)13621 return err;13622 13623 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */);13624 i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */);13625 i40e_queue_pair_enable_irq(vsi, queue_pair);13626 13627 i40e_exit_busy_conf(vsi);13628 13629 return err;13630}13631 13632/**13633 * i40e_xdp - implements ndo_bpf for i40e13634 * @dev: netdevice13635 * @xdp: XDP command13636 **/13637static int i40e_xdp(struct net_device *dev,13638 struct netdev_bpf *xdp)13639{13640 struct i40e_netdev_priv *np = netdev_priv(dev);13641 struct i40e_vsi *vsi = np->vsi;13642 13643 if (vsi->type != I40E_VSI_MAIN)13644 return -EINVAL;13645 13646 switch (xdp->command) {13647 case XDP_SETUP_PROG:13648 return i40e_xdp_setup(vsi, xdp->prog, xdp->extack);13649 case XDP_SETUP_XSK_POOL:13650 return i40e_xsk_pool_setup(vsi, xdp->xsk.pool,13651 xdp->xsk.queue_id);13652 default:13653 return -EINVAL;13654 }13655}13656 13657static const struct net_device_ops i40e_netdev_ops = {13658 .ndo_open = i40e_open,13659 .ndo_stop = i40e_close,13660 .ndo_start_xmit = i40e_lan_xmit_frame,13661 .ndo_get_stats64 = i40e_get_netdev_stats_struct,13662 .ndo_set_rx_mode = i40e_set_rx_mode,13663 .ndo_validate_addr = eth_validate_addr,13664 .ndo_set_mac_address = i40e_set_mac,13665 .ndo_change_mtu = i40e_change_mtu,13666 .ndo_eth_ioctl = i40e_ioctl,13667 .ndo_tx_timeout = i40e_tx_timeout,13668 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid,13669 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid,13670#ifdef CONFIG_NET_POLL_CONTROLLER13671 .ndo_poll_controller = i40e_netpoll,13672#endif13673 .ndo_setup_tc = __i40e_setup_tc,13674 .ndo_select_queue = i40e_lan_select_queue,13675 .ndo_set_features = i40e_set_features,13676 .ndo_set_vf_mac = i40e_ndo_set_vf_mac,13677 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan,13678 .ndo_get_vf_stats = i40e_get_vf_stats,13679 .ndo_set_vf_rate = i40e_ndo_set_vf_bw,13680 .ndo_get_vf_config = i40e_ndo_get_vf_config,13681 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state,13682 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk,13683 .ndo_set_vf_trust = i40e_ndo_set_vf_trust,13684 .ndo_get_phys_port_id = i40e_get_phys_port_id,13685 .ndo_fdb_add = i40e_ndo_fdb_add,13686 .ndo_features_check = i40e_features_check,13687 .ndo_bridge_getlink = i40e_ndo_bridge_getlink,13688 .ndo_bridge_setlink = i40e_ndo_bridge_setlink,13689 .ndo_bpf = i40e_xdp,13690 .ndo_xdp_xmit = i40e_xdp_xmit,13691 .ndo_xsk_wakeup = i40e_xsk_wakeup,13692 .ndo_dfwd_add_station = i40e_fwd_add,13693 .ndo_dfwd_del_station = i40e_fwd_del,13694};13695 13696/**13697 * i40e_config_netdev - Setup the netdev flags13698 * @vsi: the VSI being configured13699 *13700 * Returns 0 on success, negative value on failure13701 **/13702static int i40e_config_netdev(struct i40e_vsi *vsi)13703{13704 struct i40e_pf *pf = vsi->back;13705 struct i40e_hw *hw = &pf->hw;13706 struct i40e_netdev_priv *np;13707 struct net_device *netdev;13708 u8 broadcast[ETH_ALEN];13709 u8 mac_addr[ETH_ALEN];13710 int etherdev_size;13711 netdev_features_t hw_enc_features;13712 netdev_features_t hw_features;13713 13714 etherdev_size = sizeof(struct i40e_netdev_priv);13715 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);13716 if (!netdev)13717 return -ENOMEM;13718 13719 vsi->netdev = netdev;13720 np = netdev_priv(netdev);13721 np->vsi = vsi;13722 13723 hw_enc_features = NETIF_F_SG |13724 NETIF_F_HW_CSUM |13725 NETIF_F_HIGHDMA |13726 NETIF_F_SOFT_FEATURES |13727 NETIF_F_TSO |13728 NETIF_F_TSO_ECN |13729 NETIF_F_TSO6 |13730 NETIF_F_GSO_GRE |13731 NETIF_F_GSO_GRE_CSUM |13732 NETIF_F_GSO_PARTIAL |13733 NETIF_F_GSO_IPXIP4 |13734 NETIF_F_GSO_IPXIP6 |13735 NETIF_F_GSO_UDP_TUNNEL |13736 NETIF_F_GSO_UDP_TUNNEL_CSUM |13737 NETIF_F_GSO_UDP_L4 |13738 NETIF_F_SCTP_CRC |13739 NETIF_F_RXHASH |13740 NETIF_F_RXCSUM |13741 0;13742 13743 if (!test_bit(I40E_HW_CAP_OUTER_UDP_CSUM, pf->hw.caps))13744 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;13745 13746 netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic;13747 13748 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;13749 13750 netdev->hw_enc_features |= hw_enc_features;13751 13752 /* record features VLANs can make use of */13753 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;13754 13755#define I40E_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \13756 NETIF_F_GSO_GRE_CSUM | \13757 NETIF_F_GSO_IPXIP4 | \13758 NETIF_F_GSO_IPXIP6 | \13759 NETIF_F_GSO_UDP_TUNNEL | \13760 NETIF_F_GSO_UDP_TUNNEL_CSUM)13761 13762 netdev->gso_partial_features = I40E_GSO_PARTIAL_FEATURES;13763 netdev->features |= NETIF_F_GSO_PARTIAL |13764 I40E_GSO_PARTIAL_FEATURES;13765 13766 netdev->mpls_features |= NETIF_F_SG;13767 netdev->mpls_features |= NETIF_F_HW_CSUM;13768 netdev->mpls_features |= NETIF_F_TSO;13769 netdev->mpls_features |= NETIF_F_TSO6;13770 netdev->mpls_features |= I40E_GSO_PARTIAL_FEATURES;13771 13772 /* enable macvlan offloads */13773 netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD;13774 13775 hw_features = hw_enc_features |13776 NETIF_F_HW_VLAN_CTAG_TX |13777 NETIF_F_HW_VLAN_CTAG_RX;13778 13779 if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))13780 hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC;13781 13782 netdev->hw_features |= hw_features | NETIF_F_LOOPBACK;13783 13784 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;13785 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;13786 13787 netdev->features &= ~NETIF_F_HW_TC;13788 13789 if (vsi->type == I40E_VSI_MAIN) {13790 SET_NETDEV_DEV(netdev, &pf->pdev->dev);13791 ether_addr_copy(mac_addr, hw->mac.perm_addr);13792 /* The following steps are necessary for two reasons. First,13793 * some older NVM configurations load a default MAC-VLAN13794 * filter that will accept any tagged packet, and we want to13795 * replace this with a normal filter. Additionally, it is13796 * possible our MAC address was provided by the platform using13797 * Open Firmware or similar.13798 *13799 * Thus, we need to remove the default filter and install one13800 * specific to the MAC address.13801 */13802 i40e_rm_default_mac_filter(vsi, mac_addr);13803 spin_lock_bh(&vsi->mac_filter_hash_lock);13804 i40e_add_mac_filter(vsi, mac_addr);13805 spin_unlock_bh(&vsi->mac_filter_hash_lock);13806 13807 netdev->xdp_features = NETDEV_XDP_ACT_BASIC |13808 NETDEV_XDP_ACT_REDIRECT |13809 NETDEV_XDP_ACT_XSK_ZEROCOPY |13810 NETDEV_XDP_ACT_RX_SG;13811 netdev->xdp_zc_max_segs = I40E_MAX_BUFFER_TXD;13812 } else {13813 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we13814 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to13815 * the end, which is 4 bytes long, so force truncation of the13816 * original name by IFNAMSIZ - 413817 */13818 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf);13819 13820 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d", IFNAMSIZ - 4,13821 main_vsi->netdev->name);13822 eth_random_addr(mac_addr);13823 13824 spin_lock_bh(&vsi->mac_filter_hash_lock);13825 i40e_add_mac_filter(vsi, mac_addr);13826 spin_unlock_bh(&vsi->mac_filter_hash_lock);13827 }13828 13829 /* Add the broadcast filter so that we initially will receive13830 * broadcast packets. Note that when a new VLAN is first added the13831 * driver will convert all filters marked I40E_VLAN_ANY into VLAN13832 * specific filters as part of transitioning into "vlan" operation.13833 * When more VLANs are added, the driver will copy each existing MAC13834 * filter and add it for the new VLAN.13835 *13836 * Broadcast filters are handled specially by13837 * i40e_sync_filters_subtask, as the driver must to set the broadcast13838 * promiscuous bit instead of adding this directly as a MAC/VLAN13839 * filter. The subtask will update the correct broadcast promiscuous13840 * bits as VLANs become active or inactive.13841 */13842 eth_broadcast_addr(broadcast);13843 spin_lock_bh(&vsi->mac_filter_hash_lock);13844 i40e_add_mac_filter(vsi, broadcast);13845 spin_unlock_bh(&vsi->mac_filter_hash_lock);13846 13847 eth_hw_addr_set(netdev, mac_addr);13848 ether_addr_copy(netdev->perm_addr, mac_addr);13849 13850 /* i40iw_net_event() reads 16 bytes from neigh->primary_key */13851 netdev->neigh_priv_len = sizeof(u32) * 4;13852 13853 netdev->priv_flags |= IFF_UNICAST_FLT;13854 netdev->priv_flags |= IFF_SUPP_NOFCS;13855 /* Setup netdev TC information */13856 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);13857 13858 netdev->netdev_ops = &i40e_netdev_ops;13859 netdev->watchdog_timeo = 5 * HZ;13860 i40e_set_ethtool_ops(netdev);13861 13862 /* MTU range: 68 - 9706 */13863 netdev->min_mtu = ETH_MIN_MTU;13864 netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;13865 13866 return 0;13867}13868 13869/**13870 * i40e_vsi_delete - Delete a VSI from the switch13871 * @vsi: the VSI being removed13872 *13873 * Returns 0 on success, negative value on failure13874 **/13875static void i40e_vsi_delete(struct i40e_vsi *vsi)13876{13877 /* remove default VSI is not allowed */13878 if (vsi == vsi->back->vsi[vsi->back->lan_vsi])13879 return;13880 13881 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);13882}13883 13884/**13885 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB13886 * @vsi: the VSI being queried13887 *13888 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode13889 **/13890int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)13891{13892 struct i40e_veb *veb;13893 struct i40e_pf *pf = vsi->back;13894 13895 /* Uplink is not a bridge so default to VEB */13896 if (vsi->veb_idx >= I40E_MAX_VEB)13897 return 1;13898 13899 veb = pf->veb[vsi->veb_idx];13900 if (!veb) {13901 dev_info(&pf->pdev->dev,13902 "There is no veb associated with the bridge\n");13903 return -ENOENT;13904 }13905 13906 /* Uplink is a bridge in VEPA mode */13907 if (veb->bridge_mode & BRIDGE_MODE_VEPA) {13908 return 0;13909 } else {13910 /* Uplink is a bridge in VEB mode */13911 return 1;13912 }13913 13914 /* VEPA is now default bridge, so return 0 */13915 return 0;13916}13917 13918/**13919 * i40e_add_vsi - Add a VSI to the switch13920 * @vsi: the VSI being configured13921 *13922 * This initializes a VSI context depending on the VSI type to be added and13923 * passes it down to the add_vsi aq command.13924 **/13925static int i40e_add_vsi(struct i40e_vsi *vsi)13926{13927 int ret = -ENODEV;13928 struct i40e_pf *pf = vsi->back;13929 struct i40e_hw *hw = &pf->hw;13930 struct i40e_vsi_context ctxt;13931 struct i40e_mac_filter *f;13932 struct hlist_node *h;13933 int bkt;13934 13935 u8 enabled_tc = 0x1; /* TC0 enabled */13936 int f_count = 0;13937 13938 memset(&ctxt, 0, sizeof(ctxt));13939 switch (vsi->type) {13940 case I40E_VSI_MAIN:13941 /* The PF's main VSI is already setup as part of the13942 * device initialization, so we'll not bother with13943 * the add_vsi call, but we will retrieve the current13944 * VSI context.13945 */13946 ctxt.seid = pf->main_vsi_seid;13947 ctxt.pf_num = pf->hw.pf_id;13948 ctxt.vf_num = 0;13949 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);13950 ctxt.flags = I40E_AQ_VSI_TYPE_PF;13951 if (ret) {13952 dev_info(&pf->pdev->dev,13953 "couldn't get PF vsi config, err %pe aq_err %s\n",13954 ERR_PTR(ret),13955 i40e_aq_str(&pf->hw,13956 pf->hw.aq.asq_last_status));13957 return -ENOENT;13958 }13959 vsi->info = ctxt.info;13960 vsi->info.valid_sections = 0;13961 13962 vsi->seid = ctxt.seid;13963 vsi->id = ctxt.vsi_number;13964 13965 enabled_tc = i40e_pf_get_tc_map(pf);13966 13967 /* Source pruning is enabled by default, so the flag is13968 * negative logic - if it's set, we need to fiddle with13969 * the VSI to disable source pruning.13970 */13971 if (test_bit(I40E_FLAG_SOURCE_PRUNING_DIS, pf->flags)) {13972 memset(&ctxt, 0, sizeof(ctxt));13973 ctxt.seid = pf->main_vsi_seid;13974 ctxt.pf_num = pf->hw.pf_id;13975 ctxt.vf_num = 0;13976 ctxt.info.valid_sections |=13977 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);13978 ctxt.info.switch_id =13979 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);13980 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);13981 if (ret) {13982 dev_info(&pf->pdev->dev,13983 "update vsi failed, err %d aq_err %s\n",13984 ret,13985 i40e_aq_str(&pf->hw,13986 pf->hw.aq.asq_last_status));13987 ret = -ENOENT;13988 goto err;13989 }13990 }13991 13992 /* MFP mode setup queue map and update VSI */13993 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) &&13994 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */13995 memset(&ctxt, 0, sizeof(ctxt));13996 ctxt.seid = pf->main_vsi_seid;13997 ctxt.pf_num = pf->hw.pf_id;13998 ctxt.vf_num = 0;13999 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);14000 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);14001 if (ret) {14002 dev_info(&pf->pdev->dev,14003 "update vsi failed, err %pe aq_err %s\n",14004 ERR_PTR(ret),14005 i40e_aq_str(&pf->hw,14006 pf->hw.aq.asq_last_status));14007 ret = -ENOENT;14008 goto err;14009 }14010 /* update the local VSI info queue map */14011 i40e_vsi_update_queue_map(vsi, &ctxt);14012 vsi->info.valid_sections = 0;14013 } else {14014 /* Default/Main VSI is only enabled for TC014015 * reconfigure it to enable all TCs that are14016 * available on the port in SFP mode.14017 * For MFP case the iSCSI PF would use this14018 * flow to enable LAN+iSCSI TC.14019 */14020 ret = i40e_vsi_config_tc(vsi, enabled_tc);14021 if (ret) {14022 /* Single TC condition is not fatal,14023 * message and continue14024 */14025 dev_info(&pf->pdev->dev,14026 "failed to configure TCs for main VSI tc_map 0x%08x, err %pe aq_err %s\n",14027 enabled_tc,14028 ERR_PTR(ret),14029 i40e_aq_str(&pf->hw,14030 pf->hw.aq.asq_last_status));14031 }14032 }14033 break;14034 14035 case I40E_VSI_FDIR:14036 ctxt.pf_num = hw->pf_id;14037 ctxt.vf_num = 0;14038 ctxt.uplink_seid = vsi->uplink_seid;14039 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;14040 ctxt.flags = I40E_AQ_VSI_TYPE_PF;14041 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags) &&14042 (i40e_is_vsi_uplink_mode_veb(vsi))) {14043 ctxt.info.valid_sections |=14044 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);14045 ctxt.info.switch_id =14046 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);14047 }14048 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);14049 break;14050 14051 case I40E_VSI_VMDQ2:14052 ctxt.pf_num = hw->pf_id;14053 ctxt.vf_num = 0;14054 ctxt.uplink_seid = vsi->uplink_seid;14055 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;14056 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;14057 14058 /* This VSI is connected to VEB so the switch_id14059 * should be set to zero by default.14060 */14061 if (i40e_is_vsi_uplink_mode_veb(vsi)) {14062 ctxt.info.valid_sections |=14063 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);14064 ctxt.info.switch_id =14065 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);14066 }14067 14068 /* Setup the VSI tx/rx queue map for TC0 only for now */14069 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);14070 break;14071 14072 case I40E_VSI_SRIOV:14073 ctxt.pf_num = hw->pf_id;14074 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;14075 ctxt.uplink_seid = vsi->uplink_seid;14076 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;14077 ctxt.flags = I40E_AQ_VSI_TYPE_VF;14078 14079 /* This VSI is connected to VEB so the switch_id14080 * should be set to zero by default.14081 */14082 if (i40e_is_vsi_uplink_mode_veb(vsi)) {14083 ctxt.info.valid_sections |=14084 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);14085 ctxt.info.switch_id =14086 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);14087 }14088 14089 if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) {14090 ctxt.info.valid_sections |=14091 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);14092 ctxt.info.queueing_opt_flags |=14093 (I40E_AQ_VSI_QUE_OPT_TCP_ENA |14094 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);14095 }14096 14097 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);14098 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;14099 if (pf->vf[vsi->vf_id].spoofchk) {14100 ctxt.info.valid_sections |=14101 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);14102 ctxt.info.sec_flags |=14103 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |14104 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);14105 }14106 /* Setup the VSI tx/rx queue map for TC0 only for now */14107 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);14108 break;14109 14110 case I40E_VSI_IWARP:14111 /* send down message to iWARP */14112 break;14113 14114 default:14115 return -ENODEV;14116 }14117 14118 if (vsi->type != I40E_VSI_MAIN) {14119 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);14120 if (ret) {14121 dev_info(&vsi->back->pdev->dev,14122 "add vsi failed, err %pe aq_err %s\n",14123 ERR_PTR(ret),14124 i40e_aq_str(&pf->hw,14125 pf->hw.aq.asq_last_status));14126 ret = -ENOENT;14127 goto err;14128 }14129 vsi->info = ctxt.info;14130 vsi->info.valid_sections = 0;14131 vsi->seid = ctxt.seid;14132 vsi->id = ctxt.vsi_number;14133 }14134 14135 spin_lock_bh(&vsi->mac_filter_hash_lock);14136 vsi->active_filters = 0;14137 /* If macvlan filters already exist, force them to get loaded */14138 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {14139 f->state = I40E_FILTER_NEW;14140 f_count++;14141 }14142 spin_unlock_bh(&vsi->mac_filter_hash_lock);14143 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);14144 14145 if (f_count) {14146 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;14147 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);14148 }14149 14150 /* Update VSI BW information */14151 ret = i40e_vsi_get_bw_info(vsi);14152 if (ret) {14153 dev_info(&pf->pdev->dev,14154 "couldn't get vsi bw info, err %pe aq_err %s\n",14155 ERR_PTR(ret),14156 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));14157 /* VSI is already added so not tearing that up */14158 ret = 0;14159 }14160 14161err:14162 return ret;14163}14164 14165/**14166 * i40e_vsi_release - Delete a VSI and free its resources14167 * @vsi: the VSI being removed14168 *14169 * Returns 0 on success or < 0 on error14170 **/14171int i40e_vsi_release(struct i40e_vsi *vsi)14172{14173 struct i40e_mac_filter *f;14174 struct hlist_node *h;14175 struct i40e_veb *veb;14176 struct i40e_pf *pf;14177 u16 uplink_seid;14178 int i, n, bkt;14179 14180 pf = vsi->back;14181 14182 /* release of a VEB-owner or last VSI is not allowed */14183 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {14184 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",14185 vsi->seid, vsi->uplink_seid);14186 return -ENODEV;14187 }14188 if (vsi->type == I40E_VSI_MAIN && !test_bit(__I40E_DOWN, pf->state)) {14189 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");14190 return -ENODEV;14191 }14192 set_bit(__I40E_VSI_RELEASING, vsi->state);14193 uplink_seid = vsi->uplink_seid;14194 14195 if (vsi->type != I40E_VSI_SRIOV) {14196 if (vsi->netdev_registered) {14197 vsi->netdev_registered = false;14198 if (vsi->netdev) {14199 /* results in a call to i40e_close() */14200 unregister_netdev(vsi->netdev);14201 }14202 } else {14203 i40e_vsi_close(vsi);14204 }14205 i40e_vsi_disable_irq(vsi);14206 }14207 14208 if (vsi->type == I40E_VSI_MAIN)14209 i40e_devlink_destroy_port(pf);14210 14211 spin_lock_bh(&vsi->mac_filter_hash_lock);14212 14213 /* clear the sync flag on all filters */14214 if (vsi->netdev) {14215 __dev_uc_unsync(vsi->netdev, NULL);14216 __dev_mc_unsync(vsi->netdev, NULL);14217 }14218 14219 /* make sure any remaining filters are marked for deletion */14220 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)14221 __i40e_del_filter(vsi, f);14222 14223 spin_unlock_bh(&vsi->mac_filter_hash_lock);14224 14225 i40e_sync_vsi_filters(vsi);14226 14227 i40e_vsi_delete(vsi);14228 i40e_vsi_free_q_vectors(vsi);14229 if (vsi->netdev) {14230 free_netdev(vsi->netdev);14231 vsi->netdev = NULL;14232 }14233 i40e_vsi_clear_rings(vsi);14234 i40e_vsi_clear(vsi);14235 14236 /* If this was the last thing on the VEB, except for the14237 * controlling VSI, remove the VEB, which puts the controlling14238 * VSI onto the uplink port.14239 *14240 * Well, okay, there's one more exception here: don't remove14241 * the floating VEBs yet. We'll wait for an explicit remove request14242 * from up the network stack.14243 */14244 veb = i40e_pf_get_veb_by_seid(pf, uplink_seid);14245 if (veb && veb->uplink_seid) {14246 n = 0;14247 14248 /* Count non-controlling VSIs present on the VEB */14249 i40e_pf_for_each_vsi(pf, i, vsi)14250 if (vsi->uplink_seid == uplink_seid &&14251 (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)14252 n++;14253 14254 /* If there is no VSI except the control one then release14255 * the VEB and put the control VSI onto VEB uplink.14256 */14257 if (!n)14258 i40e_veb_release(veb);14259 }14260 14261 return 0;14262}14263 14264/**14265 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI14266 * @vsi: ptr to the VSI14267 *14268 * This should only be called after i40e_vsi_mem_alloc() which allocates the14269 * corresponding SW VSI structure and initializes num_queue_pairs for the14270 * newly allocated VSI.14271 *14272 * Returns 0 on success or negative on failure14273 **/14274static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)14275{14276 int ret = -ENOENT;14277 struct i40e_pf *pf = vsi->back;14278 14279 if (vsi->q_vectors[0]) {14280 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",14281 vsi->seid);14282 return -EEXIST;14283 }14284 14285 if (vsi->base_vector) {14286 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",14287 vsi->seid, vsi->base_vector);14288 return -EEXIST;14289 }14290 14291 ret = i40e_vsi_alloc_q_vectors(vsi);14292 if (ret) {14293 dev_info(&pf->pdev->dev,14294 "failed to allocate %d q_vector for VSI %d, ret=%d\n",14295 vsi->num_q_vectors, vsi->seid, ret);14296 vsi->num_q_vectors = 0;14297 goto vector_setup_out;14298 }14299 14300 /* In Legacy mode, we do not have to get any other vector since we14301 * piggyback on the misc/ICR0 for queue interrupts.14302 */14303 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))14304 return ret;14305 if (vsi->num_q_vectors)14306 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,14307 vsi->num_q_vectors, vsi->idx);14308 if (vsi->base_vector < 0) {14309 dev_info(&pf->pdev->dev,14310 "failed to get tracking for %d vectors for VSI %d, err=%d\n",14311 vsi->num_q_vectors, vsi->seid, vsi->base_vector);14312 i40e_vsi_free_q_vectors(vsi);14313 ret = -ENOENT;14314 goto vector_setup_out;14315 }14316 14317vector_setup_out:14318 return ret;14319}14320 14321/**14322 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI14323 * @vsi: pointer to the vsi.14324 *14325 * This re-allocates a vsi's queue resources.14326 *14327 * Returns pointer to the successfully allocated and configured VSI sw struct14328 * on success, otherwise returns NULL on failure.14329 **/14330static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)14331{14332 struct i40e_vsi *main_vsi;14333 u16 alloc_queue_pairs;14334 struct i40e_pf *pf;14335 int ret;14336 14337 if (!vsi)14338 return NULL;14339 14340 pf = vsi->back;14341 14342 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);14343 i40e_vsi_clear_rings(vsi);14344 14345 i40e_vsi_free_arrays(vsi, false);14346 i40e_set_num_rings_in_vsi(vsi);14347 ret = i40e_vsi_alloc_arrays(vsi, false);14348 if (ret)14349 goto err_vsi;14350 14351 alloc_queue_pairs = vsi->alloc_queue_pairs *14352 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);14353 14354 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);14355 if (ret < 0) {14356 dev_info(&pf->pdev->dev,14357 "failed to get tracking for %d queues for VSI %d err %d\n",14358 alloc_queue_pairs, vsi->seid, ret);14359 goto err_vsi;14360 }14361 vsi->base_queue = ret;14362 14363 /* Update the FW view of the VSI. Force a reset of TC and queue14364 * layout configurations.14365 */14366 main_vsi = i40e_pf_get_main_vsi(pf);14367 main_vsi->seid = pf->main_vsi_seid;14368 i40e_vsi_reconfig_tc(main_vsi);14369 14370 if (vsi->type == I40E_VSI_MAIN)14371 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);14372 14373 /* assign it some queues */14374 ret = i40e_alloc_rings(vsi);14375 if (ret)14376 goto err_rings;14377 14378 /* map all of the rings to the q_vectors */14379 i40e_vsi_map_rings_to_vectors(vsi);14380 return vsi;14381 14382err_rings:14383 i40e_vsi_free_q_vectors(vsi);14384 if (vsi->netdev_registered) {14385 vsi->netdev_registered = false;14386 unregister_netdev(vsi->netdev);14387 free_netdev(vsi->netdev);14388 vsi->netdev = NULL;14389 }14390 if (vsi->type == I40E_VSI_MAIN)14391 i40e_devlink_destroy_port(pf);14392 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);14393err_vsi:14394 i40e_vsi_clear(vsi);14395 return NULL;14396}14397 14398/**14399 * i40e_vsi_setup - Set up a VSI by a given type14400 * @pf: board private structure14401 * @type: VSI type14402 * @uplink_seid: the switch element to link to14403 * @param1: usage depends upon VSI type. For VF types, indicates VF id14404 *14405 * This allocates the sw VSI structure and its queue resources, then add a VSI14406 * to the identified VEB.14407 *14408 * Returns pointer to the successfully allocated and configure VSI sw struct on14409 * success, otherwise returns NULL on failure.14410 **/14411struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,14412 u16 uplink_seid, u32 param1)14413{14414 struct i40e_vsi *vsi = NULL;14415 struct i40e_veb *veb = NULL;14416 u16 alloc_queue_pairs;14417 int v_idx;14418 int ret;14419 14420 /* The requested uplink_seid must be either14421 * - the PF's port seid14422 * no VEB is needed because this is the PF14423 * or this is a Flow Director special case VSI14424 * - seid of an existing VEB14425 * - seid of a VSI that owns an existing VEB14426 * - seid of a VSI that doesn't own a VEB14427 * a new VEB is created and the VSI becomes the owner14428 * - seid of the PF VSI, which is what creates the first VEB14429 * this is a special case of the previous14430 *14431 * Find which uplink_seid we were given and create a new VEB if needed14432 */14433 veb = i40e_pf_get_veb_by_seid(pf, uplink_seid);14434 if (!veb && uplink_seid != pf->mac_seid) {14435 vsi = i40e_pf_get_vsi_by_seid(pf, uplink_seid);14436 if (!vsi) {14437 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",14438 uplink_seid);14439 return NULL;14440 }14441 14442 if (vsi->uplink_seid == pf->mac_seid)14443 veb = i40e_veb_setup(pf, pf->mac_seid, vsi->seid,14444 vsi->tc_config.enabled_tc);14445 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)14446 veb = i40e_veb_setup(pf, vsi->uplink_seid, vsi->seid,14447 vsi->tc_config.enabled_tc);14448 if (veb) {14449 if (vsi->type != I40E_VSI_MAIN) {14450 dev_info(&vsi->back->pdev->dev,14451 "New VSI creation error, uplink seid of LAN VSI expected.\n");14452 return NULL;14453 }14454 /* We come up by default in VEPA mode if SRIOV is not14455 * already enabled, in which case we can't force VEPA14456 * mode.14457 */14458 if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {14459 veb->bridge_mode = BRIDGE_MODE_VEPA;14460 clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);14461 }14462 i40e_config_bridge_mode(veb);14463 }14464 veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid);14465 if (!veb) {14466 dev_info(&pf->pdev->dev, "couldn't add VEB\n");14467 return NULL;14468 }14469 14470 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;14471 uplink_seid = veb->seid;14472 }14473 14474 /* get vsi sw struct */14475 v_idx = i40e_vsi_mem_alloc(pf, type);14476 if (v_idx < 0)14477 goto err_alloc;14478 vsi = pf->vsi[v_idx];14479 if (!vsi)14480 goto err_alloc;14481 vsi->type = type;14482 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);14483 14484 if (type == I40E_VSI_MAIN)14485 pf->lan_vsi = v_idx;14486 else if (type == I40E_VSI_SRIOV)14487 vsi->vf_id = param1;14488 /* assign it some queues */14489 alloc_queue_pairs = vsi->alloc_queue_pairs *14490 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);14491 14492 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);14493 if (ret < 0) {14494 dev_info(&pf->pdev->dev,14495 "failed to get tracking for %d queues for VSI %d err=%d\n",14496 alloc_queue_pairs, vsi->seid, ret);14497 goto err_vsi;14498 }14499 vsi->base_queue = ret;14500 14501 /* get a VSI from the hardware */14502 vsi->uplink_seid = uplink_seid;14503 ret = i40e_add_vsi(vsi);14504 if (ret)14505 goto err_vsi;14506 14507 switch (vsi->type) {14508 /* setup the netdev if needed */14509 case I40E_VSI_MAIN:14510 case I40E_VSI_VMDQ2:14511 ret = i40e_config_netdev(vsi);14512 if (ret)14513 goto err_netdev;14514 ret = i40e_netif_set_realnum_tx_rx_queues(vsi);14515 if (ret)14516 goto err_netdev;14517 if (vsi->type == I40E_VSI_MAIN) {14518 ret = i40e_devlink_create_port(pf);14519 if (ret)14520 goto err_netdev;14521 SET_NETDEV_DEVLINK_PORT(vsi->netdev, &pf->devlink_port);14522 }14523 ret = register_netdev(vsi->netdev);14524 if (ret)14525 goto err_dl_port;14526 vsi->netdev_registered = true;14527 netif_carrier_off(vsi->netdev);14528#ifdef CONFIG_I40E_DCB14529 /* Setup DCB netlink interface */14530 i40e_dcbnl_setup(vsi);14531#endif /* CONFIG_I40E_DCB */14532 fallthrough;14533 case I40E_VSI_FDIR:14534 /* set up vectors and rings if needed */14535 ret = i40e_vsi_setup_vectors(vsi);14536 if (ret)14537 goto err_msix;14538 14539 ret = i40e_alloc_rings(vsi);14540 if (ret)14541 goto err_rings;14542 14543 /* map all of the rings to the q_vectors */14544 i40e_vsi_map_rings_to_vectors(vsi);14545 14546 i40e_vsi_reset_stats(vsi);14547 break;14548 default:14549 /* no netdev or rings for the other VSI types */14550 break;14551 }14552 14553 if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps) &&14554 vsi->type == I40E_VSI_VMDQ2) {14555 ret = i40e_vsi_config_rss(vsi);14556 if (ret)14557 goto err_config;14558 }14559 return vsi;14560 14561err_config:14562 i40e_vsi_clear_rings(vsi);14563err_rings:14564 i40e_vsi_free_q_vectors(vsi);14565err_msix:14566 if (vsi->netdev_registered) {14567 vsi->netdev_registered = false;14568 unregister_netdev(vsi->netdev);14569 free_netdev(vsi->netdev);14570 vsi->netdev = NULL;14571 }14572err_dl_port:14573 if (vsi->type == I40E_VSI_MAIN)14574 i40e_devlink_destroy_port(pf);14575err_netdev:14576 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);14577err_vsi:14578 i40e_vsi_clear(vsi);14579err_alloc:14580 return NULL;14581}14582 14583/**14584 * i40e_veb_get_bw_info - Query VEB BW information14585 * @veb: the veb to query14586 *14587 * Query the Tx scheduler BW configuration data for given VEB14588 **/14589static int i40e_veb_get_bw_info(struct i40e_veb *veb)14590{14591 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;14592 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;14593 struct i40e_pf *pf = veb->pf;14594 struct i40e_hw *hw = &pf->hw;14595 u32 tc_bw_max;14596 int ret = 0;14597 int i;14598 14599 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,14600 &bw_data, NULL);14601 if (ret) {14602 dev_info(&pf->pdev->dev,14603 "query veb bw config failed, err %pe aq_err %s\n",14604 ERR_PTR(ret),14605 i40e_aq_str(&pf->hw, hw->aq.asq_last_status));14606 goto out;14607 }14608 14609 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,14610 &ets_data, NULL);14611 if (ret) {14612 dev_info(&pf->pdev->dev,14613 "query veb bw ets config failed, err %pe aq_err %s\n",14614 ERR_PTR(ret),14615 i40e_aq_str(&pf->hw, hw->aq.asq_last_status));14616 goto out;14617 }14618 14619 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);14620 veb->bw_max_quanta = ets_data.tc_bw_max;14621 veb->is_abs_credits = bw_data.absolute_credits_enable;14622 veb->enabled_tc = ets_data.tc_valid_bits;14623 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |14624 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);14625 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {14626 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];14627 veb->bw_tc_limit_credits[i] =14628 le16_to_cpu(bw_data.tc_bw_limits[i]);14629 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);14630 }14631 14632out:14633 return ret;14634}14635 14636/**14637 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF14638 * @pf: board private structure14639 *14640 * On error: returns error code (negative)14641 * On success: returns vsi index in PF (positive)14642 **/14643static int i40e_veb_mem_alloc(struct i40e_pf *pf)14644{14645 int ret = -ENOENT;14646 struct i40e_veb *veb;14647 int i;14648 14649 /* Need to protect the allocation of switch elements at the PF level */14650 mutex_lock(&pf->switch_mutex);14651 14652 /* VEB list may be fragmented if VEB creation/destruction has14653 * been happening. We can afford to do a quick scan to look14654 * for any free slots in the list.14655 *14656 * find next empty veb slot, looping back around if necessary14657 */14658 i = 0;14659 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))14660 i++;14661 if (i >= I40E_MAX_VEB) {14662 ret = -ENOMEM;14663 goto err_alloc_veb; /* out of VEB slots! */14664 }14665 14666 veb = kzalloc(sizeof(*veb), GFP_KERNEL);14667 if (!veb) {14668 ret = -ENOMEM;14669 goto err_alloc_veb;14670 }14671 veb->pf = pf;14672 veb->idx = i;14673 veb->enabled_tc = 1;14674 14675 pf->veb[i] = veb;14676 ret = i;14677err_alloc_veb:14678 mutex_unlock(&pf->switch_mutex);14679 return ret;14680}14681 14682/**14683 * i40e_switch_branch_release - Delete a branch of the switch tree14684 * @branch: where to start deleting14685 *14686 * This uses recursion to find the tips of the branch to be14687 * removed, deleting until we get back to and can delete this VEB.14688 **/14689static void i40e_switch_branch_release(struct i40e_veb *branch)14690{14691 struct i40e_pf *pf = branch->pf;14692 u16 branch_seid = branch->seid;14693 u16 veb_idx = branch->idx;14694 struct i40e_vsi *vsi;14695 struct i40e_veb *veb;14696 int i;14697 14698 /* release any VEBs on this VEB - RECURSION */14699 i40e_pf_for_each_veb(pf, i, veb)14700 if (veb->uplink_seid == branch->seid)14701 i40e_switch_branch_release(veb);14702 14703 /* Release the VSIs on this VEB, but not the owner VSI.14704 *14705 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing14706 * the VEB itself, so don't use (*branch) after this loop.14707 */14708 i40e_pf_for_each_vsi(pf, i, vsi)14709 if (vsi->uplink_seid == branch_seid &&14710 (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)14711 i40e_vsi_release(vsi);14712 14713 /* There's one corner case where the VEB might not have been14714 * removed, so double check it here and remove it if needed.14715 * This case happens if the veb was created from the debugfs14716 * commands and no VSIs were added to it.14717 */14718 if (pf->veb[veb_idx])14719 i40e_veb_release(pf->veb[veb_idx]);14720}14721 14722/**14723 * i40e_veb_clear - remove veb struct14724 * @veb: the veb to remove14725 **/14726static void i40e_veb_clear(struct i40e_veb *veb)14727{14728 if (!veb)14729 return;14730 14731 if (veb->pf) {14732 struct i40e_pf *pf = veb->pf;14733 14734 mutex_lock(&pf->switch_mutex);14735 if (pf->veb[veb->idx] == veb)14736 pf->veb[veb->idx] = NULL;14737 mutex_unlock(&pf->switch_mutex);14738 }14739 14740 kfree(veb);14741}14742 14743/**14744 * i40e_veb_release - Delete a VEB and free its resources14745 * @veb: the VEB being removed14746 **/14747void i40e_veb_release(struct i40e_veb *veb)14748{14749 struct i40e_vsi *vsi, *vsi_it;14750 struct i40e_pf *pf;14751 int i, n = 0;14752 14753 pf = veb->pf;14754 14755 /* find the remaining VSI and check for extras */14756 i40e_pf_for_each_vsi(pf, i, vsi_it)14757 if (vsi_it->uplink_seid == veb->seid) {14758 if (vsi_it->flags & I40E_VSI_FLAG_VEB_OWNER)14759 vsi = vsi_it;14760 n++;14761 }14762 14763 /* Floating VEB has to be empty and regular one must have14764 * single owner VSI.14765 */14766 if ((veb->uplink_seid && n != 1) || (!veb->uplink_seid && n != 0)) {14767 dev_info(&pf->pdev->dev,14768 "can't remove VEB %d with %d VSIs left\n",14769 veb->seid, n);14770 return;14771 }14772 14773 /* For regular VEB move the owner VSI to uplink port */14774 if (veb->uplink_seid) {14775 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;14776 vsi->uplink_seid = veb->uplink_seid;14777 vsi->veb_idx = I40E_NO_VEB;14778 }14779 14780 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);14781 i40e_veb_clear(veb);14782}14783 14784/**14785 * i40e_add_veb - create the VEB in the switch14786 * @veb: the VEB to be instantiated14787 * @vsi: the controlling VSI14788 **/14789static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)14790{14791 struct i40e_pf *pf = veb->pf;14792 bool enable_stats = !!test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags);14793 int ret;14794 14795 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi ? vsi->seid : 0,14796 veb->enabled_tc, vsi ? false : true,14797 &veb->seid, enable_stats, NULL);14798 14799 /* get a VEB from the hardware */14800 if (ret) {14801 dev_info(&pf->pdev->dev,14802 "couldn't add VEB, err %pe aq_err %s\n",14803 ERR_PTR(ret),14804 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));14805 return -EPERM;14806 }14807 14808 /* get statistics counter */14809 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,14810 &veb->stats_idx, NULL, NULL, NULL);14811 if (ret) {14812 dev_info(&pf->pdev->dev,14813 "couldn't get VEB statistics idx, err %pe aq_err %s\n",14814 ERR_PTR(ret),14815 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));14816 return -EPERM;14817 }14818 ret = i40e_veb_get_bw_info(veb);14819 if (ret) {14820 dev_info(&pf->pdev->dev,14821 "couldn't get VEB bw info, err %pe aq_err %s\n",14822 ERR_PTR(ret),14823 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));14824 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);14825 return -ENOENT;14826 }14827 14828 if (vsi) {14829 vsi->uplink_seid = veb->seid;14830 vsi->veb_idx = veb->idx;14831 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;14832 }14833 14834 return 0;14835}14836 14837/**14838 * i40e_veb_setup - Set up a VEB14839 * @pf: board private structure14840 * @uplink_seid: the switch element to link to14841 * @vsi_seid: the initial VSI seid14842 * @enabled_tc: Enabled TC bit-map14843 *14844 * This allocates the sw VEB structure and links it into the switch14845 * It is possible and legal for this to be a duplicate of an already14846 * existing VEB. It is also possible for both uplink and vsi seids14847 * to be zero, in order to create a floating VEB.14848 *14849 * Returns pointer to the successfully allocated VEB sw struct on14850 * success, otherwise returns NULL on failure.14851 **/14852struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 uplink_seid,14853 u16 vsi_seid, u8 enabled_tc)14854{14855 struct i40e_vsi *vsi = NULL;14856 struct i40e_veb *veb;14857 int veb_idx;14858 int ret;14859 14860 /* if one seid is 0, the other must be 0 to create a floating relay */14861 if ((uplink_seid == 0 || vsi_seid == 0) &&14862 (uplink_seid + vsi_seid != 0)) {14863 dev_info(&pf->pdev->dev,14864 "one, not both seid's are 0: uplink=%d vsi=%d\n",14865 uplink_seid, vsi_seid);14866 return NULL;14867 }14868 14869 /* make sure there is such a vsi and uplink */14870 if (vsi_seid) {14871 vsi = i40e_pf_get_vsi_by_seid(pf, vsi_seid);14872 if (!vsi) {14873 dev_err(&pf->pdev->dev, "vsi seid %d not found\n",14874 vsi_seid);14875 return NULL;14876 }14877 }14878 14879 /* get veb sw struct */14880 veb_idx = i40e_veb_mem_alloc(pf);14881 if (veb_idx < 0)14882 goto err_alloc;14883 veb = pf->veb[veb_idx];14884 veb->uplink_seid = uplink_seid;14885 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);14886 14887 /* create the VEB in the switch */14888 ret = i40e_add_veb(veb, vsi);14889 if (ret)14890 goto err_veb;14891 14892 if (vsi && vsi->idx == pf->lan_vsi)14893 pf->lan_veb = veb->idx;14894 14895 return veb;14896 14897err_veb:14898 i40e_veb_clear(veb);14899err_alloc:14900 return NULL;14901}14902 14903/**14904 * i40e_setup_pf_switch_element - set PF vars based on switch type14905 * @pf: board private structure14906 * @ele: element we are building info from14907 * @num_reported: total number of elements14908 * @printconfig: should we print the contents14909 *14910 * helper function to assist in extracting a few useful SEID values.14911 **/14912static void i40e_setup_pf_switch_element(struct i40e_pf *pf,14913 struct i40e_aqc_switch_config_element_resp *ele,14914 u16 num_reported, bool printconfig)14915{14916 u16 downlink_seid = le16_to_cpu(ele->downlink_seid);14917 u16 uplink_seid = le16_to_cpu(ele->uplink_seid);14918 u8 element_type = ele->element_type;14919 u16 seid = le16_to_cpu(ele->seid);14920 struct i40e_veb *veb;14921 14922 if (printconfig)14923 dev_info(&pf->pdev->dev,14924 "type=%d seid=%d uplink=%d downlink=%d\n",14925 element_type, seid, uplink_seid, downlink_seid);14926 14927 switch (element_type) {14928 case I40E_SWITCH_ELEMENT_TYPE_MAC:14929 pf->mac_seid = seid;14930 break;14931 case I40E_SWITCH_ELEMENT_TYPE_VEB:14932 /* Main VEB? */14933 if (uplink_seid != pf->mac_seid)14934 break;14935 veb = i40e_pf_get_main_veb(pf);14936 if (!veb) {14937 int v;14938 14939 /* find existing or else empty VEB */14940 veb = i40e_pf_get_veb_by_seid(pf, seid);14941 if (veb) {14942 pf->lan_veb = veb->idx;14943 } else {14944 v = i40e_veb_mem_alloc(pf);14945 if (v < 0)14946 break;14947 pf->lan_veb = v;14948 }14949 }14950 14951 /* Try to get again main VEB as pf->lan_veb may have changed */14952 veb = i40e_pf_get_main_veb(pf);14953 if (!veb)14954 break;14955 14956 veb->seid = seid;14957 veb->uplink_seid = pf->mac_seid;14958 veb->pf = pf;14959 break;14960 case I40E_SWITCH_ELEMENT_TYPE_VSI:14961 if (num_reported != 1)14962 break;14963 /* This is immediately after a reset so we can assume this is14964 * the PF's VSI14965 */14966 pf->mac_seid = uplink_seid;14967 pf->main_vsi_seid = seid;14968 if (printconfig)14969 dev_info(&pf->pdev->dev,14970 "pf_seid=%d main_vsi_seid=%d\n",14971 downlink_seid, pf->main_vsi_seid);14972 break;14973 case I40E_SWITCH_ELEMENT_TYPE_PF:14974 case I40E_SWITCH_ELEMENT_TYPE_VF:14975 case I40E_SWITCH_ELEMENT_TYPE_EMP:14976 case I40E_SWITCH_ELEMENT_TYPE_BMC:14977 case I40E_SWITCH_ELEMENT_TYPE_PE:14978 case I40E_SWITCH_ELEMENT_TYPE_PA:14979 /* ignore these for now */14980 break;14981 default:14982 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",14983 element_type, seid);14984 break;14985 }14986}14987 14988/**14989 * i40e_fetch_switch_configuration - Get switch config from firmware14990 * @pf: board private structure14991 * @printconfig: should we print the contents14992 *14993 * Get the current switch configuration from the device and14994 * extract a few useful SEID values.14995 **/14996int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)14997{14998 struct i40e_aqc_get_switch_config_resp *sw_config;14999 u16 next_seid = 0;15000 int ret = 0;15001 u8 *aq_buf;15002 int i;15003 15004 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);15005 if (!aq_buf)15006 return -ENOMEM;15007 15008 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;15009 do {15010 u16 num_reported, num_total;15011 15012 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,15013 I40E_AQ_LARGE_BUF,15014 &next_seid, NULL);15015 if (ret) {15016 dev_info(&pf->pdev->dev,15017 "get switch config failed err %d aq_err %s\n",15018 ret,15019 i40e_aq_str(&pf->hw,15020 pf->hw.aq.asq_last_status));15021 kfree(aq_buf);15022 return -ENOENT;15023 }15024 15025 num_reported = le16_to_cpu(sw_config->header.num_reported);15026 num_total = le16_to_cpu(sw_config->header.num_total);15027 15028 if (printconfig)15029 dev_info(&pf->pdev->dev,15030 "header: %d reported %d total\n",15031 num_reported, num_total);15032 15033 for (i = 0; i < num_reported; i++) {15034 struct i40e_aqc_switch_config_element_resp *ele =15035 &sw_config->element[i];15036 15037 i40e_setup_pf_switch_element(pf, ele, num_reported,15038 printconfig);15039 }15040 } while (next_seid != 0);15041 15042 kfree(aq_buf);15043 return ret;15044}15045 15046/**15047 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset15048 * @pf: board private structure15049 * @reinit: if the Main VSI needs to re-initialized.15050 * @lock_acquired: indicates whether or not the lock has been acquired15051 *15052 * Returns 0 on success, negative value on failure15053 **/15054static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired)15055{15056 struct i40e_vsi *main_vsi;15057 u16 flags = 0;15058 int ret;15059 15060 /* find out what's out there already */15061 ret = i40e_fetch_switch_configuration(pf, false);15062 if (ret) {15063 dev_info(&pf->pdev->dev,15064 "couldn't fetch switch config, err %pe aq_err %s\n",15065 ERR_PTR(ret),15066 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));15067 return ret;15068 }15069 i40e_pf_reset_stats(pf);15070 15071 /* set the switch config bit for the whole device to15072 * support limited promisc or true promisc15073 * when user requests promisc. The default is limited15074 * promisc.15075 */15076 15077 if ((pf->hw.pf_id == 0) &&15078 !test_bit(I40E_FLAG_TRUE_PROMISC_ENA, pf->flags)) {15079 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;15080 pf->last_sw_conf_flags = flags;15081 }15082 15083 if (pf->hw.pf_id == 0) {15084 u16 valid_flags;15085 15086 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;15087 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0,15088 NULL);15089 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {15090 dev_info(&pf->pdev->dev,15091 "couldn't set switch config bits, err %pe aq_err %s\n",15092 ERR_PTR(ret),15093 i40e_aq_str(&pf->hw,15094 pf->hw.aq.asq_last_status));15095 /* not a fatal problem, just keep going */15096 }15097 pf->last_sw_conf_valid_flags = valid_flags;15098 }15099 15100 /* first time setup */15101 main_vsi = i40e_pf_get_main_vsi(pf);15102 if (!main_vsi || reinit) {15103 struct i40e_veb *veb;15104 u16 uplink_seid;15105 15106 /* Set up the PF VSI associated with the PF's main VSI15107 * that is already in the HW switch15108 */15109 veb = i40e_pf_get_main_veb(pf);15110 if (veb)15111 uplink_seid = veb->seid;15112 else15113 uplink_seid = pf->mac_seid;15114 if (!main_vsi)15115 main_vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN,15116 uplink_seid, 0);15117 else if (reinit)15118 main_vsi = i40e_vsi_reinit_setup(main_vsi);15119 if (!main_vsi) {15120 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");15121 i40e_cloud_filter_exit(pf);15122 i40e_fdir_teardown(pf);15123 return -EAGAIN;15124 }15125 } else {15126 /* force a reset of TC and queue layout configurations */15127 main_vsi->seid = pf->main_vsi_seid;15128 i40e_vsi_reconfig_tc(main_vsi);15129 }15130 i40e_vlan_stripping_disable(main_vsi);15131 15132 i40e_fdir_sb_setup(pf);15133 15134 /* Setup static PF queue filter control settings */15135 ret = i40e_setup_pf_filter_control(pf);15136 if (ret) {15137 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",15138 ret);15139 /* Failure here should not stop continuing other steps */15140 }15141 15142 /* enable RSS in the HW, even for only one queue, as the stack can use15143 * the hash15144 */15145 if (test_bit(I40E_FLAG_RSS_ENA, pf->flags))15146 i40e_pf_config_rss(pf);15147 15148 /* fill in link information and enable LSE reporting */15149 i40e_link_event(pf);15150 15151 i40e_ptp_init(pf);15152 15153 if (!lock_acquired)15154 rtnl_lock();15155 15156 /* repopulate tunnel port filters */15157 udp_tunnel_nic_reset_ntf(main_vsi->netdev);15158 15159 if (!lock_acquired)15160 rtnl_unlock();15161 15162 return ret;15163}15164 15165/**15166 * i40e_determine_queue_usage - Work out queue distribution15167 * @pf: board private structure15168 **/15169static void i40e_determine_queue_usage(struct i40e_pf *pf)15170{15171 int queues_left;15172 int q_max;15173 15174 pf->num_lan_qps = 0;15175 15176 /* Find the max queues to be put into basic use. We'll always be15177 * using TC0, whether or not DCB is running, and TC0 will get the15178 * big RSS set.15179 */15180 queues_left = pf->hw.func_caps.num_tx_qp;15181 15182 if ((queues_left == 1) ||15183 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {15184 /* one qp for PF, no queues for anything else */15185 queues_left = 0;15186 pf->alloc_rss_size = pf->num_lan_qps = 1;15187 15188 /* make sure all the fancies are disabled */15189 clear_bit(I40E_FLAG_RSS_ENA, pf->flags);15190 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);15191 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);15192 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);15193 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);15194 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);15195 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);15196 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);15197 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);15198 } else if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags) &&15199 !test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&15200 !test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&15201 !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) {15202 /* one qp for PF */15203 pf->alloc_rss_size = pf->num_lan_qps = 1;15204 queues_left -= pf->num_lan_qps;15205 15206 clear_bit(I40E_FLAG_RSS_ENA, pf->flags);15207 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);15208 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);15209 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);15210 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);15211 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);15212 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);15213 } else {15214 /* Not enough queues for all TCs */15215 if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags) &&15216 queues_left < I40E_MAX_TRAFFIC_CLASS) {15217 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);15218 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);15219 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");15220 }15221 15222 /* limit lan qps to the smaller of qps, cpus or msix */15223 q_max = max_t(int, pf->rss_size_max, num_online_cpus());15224 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp);15225 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors);15226 pf->num_lan_qps = q_max;15227 15228 queues_left -= pf->num_lan_qps;15229 }15230 15231 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {15232 if (queues_left > 1) {15233 queues_left -= 1; /* save 1 queue for FD */15234 } else {15235 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);15236 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);15237 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");15238 }15239 }15240 15241 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&15242 pf->num_vf_qps && pf->num_req_vfs && queues_left) {15243 pf->num_req_vfs = min_t(int, pf->num_req_vfs,15244 (queues_left / pf->num_vf_qps));15245 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);15246 }15247 15248 if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) &&15249 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {15250 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,15251 (queues_left / pf->num_vmdq_qps));15252 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);15253 }15254 15255 pf->queues_left = queues_left;15256 dev_dbg(&pf->pdev->dev,15257 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",15258 pf->hw.func_caps.num_tx_qp,15259 !!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags),15260 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,15261 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,15262 queues_left);15263}15264 15265/**15266 * i40e_setup_pf_filter_control - Setup PF static filter control15267 * @pf: PF to be setup15268 *15269 * i40e_setup_pf_filter_control sets up a PF's initial filter control15270 * settings. If PE/FCoE are enabled then it will also set the per PF15271 * based filter sizes required for them. It also enables Flow director,15272 * ethertype and macvlan type filter settings for the pf.15273 *15274 * Returns 0 on success, negative on failure15275 **/15276static int i40e_setup_pf_filter_control(struct i40e_pf *pf)15277{15278 struct i40e_filter_control_settings *settings = &pf->filter_settings;15279 15280 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;15281 15282 /* Flow Director is enabled */15283 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) ||15284 test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags))15285 settings->enable_fdir = true;15286 15287 /* Ethtype and MACVLAN filters enabled for PF */15288 settings->enable_ethtype = true;15289 settings->enable_macvlan = true;15290 15291 if (i40e_set_filter_control(&pf->hw, settings))15292 return -ENOENT;15293 15294 return 0;15295}15296 15297#define INFO_STRING_LEN 25515298#define REMAIN(__x) (INFO_STRING_LEN - (__x))15299static void i40e_print_features(struct i40e_pf *pf)15300{15301 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf);15302 struct i40e_hw *hw = &pf->hw;15303 char *buf;15304 int i;15305 15306 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);15307 if (!buf)15308 return;15309 15310 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);15311#ifdef CONFIG_PCI_IOV15312 i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);15313#endif15314 i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",15315 pf->hw.func_caps.num_vsis, main_vsi->num_queue_pairs);15316 if (test_bit(I40E_FLAG_RSS_ENA, pf->flags))15317 i += scnprintf(&buf[i], REMAIN(i), " RSS");15318 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags))15319 i += scnprintf(&buf[i], REMAIN(i), " FD_ATR");15320 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {15321 i += scnprintf(&buf[i], REMAIN(i), " FD_SB");15322 i += scnprintf(&buf[i], REMAIN(i), " NTUPLE");15323 }15324 if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))15325 i += scnprintf(&buf[i], REMAIN(i), " DCB");15326 i += scnprintf(&buf[i], REMAIN(i), " VxLAN");15327 i += scnprintf(&buf[i], REMAIN(i), " Geneve");15328 if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))15329 i += scnprintf(&buf[i], REMAIN(i), " PTP");15330 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags))15331 i += scnprintf(&buf[i], REMAIN(i), " VEB");15332 else15333 i += scnprintf(&buf[i], REMAIN(i), " VEPA");15334 15335 dev_info(&pf->pdev->dev, "%s\n", buf);15336 kfree(buf);15337 WARN_ON(i > INFO_STRING_LEN);15338}15339 15340/**15341 * i40e_get_platform_mac_addr - get platform-specific MAC address15342 * @pdev: PCI device information struct15343 * @pf: board private structure15344 *15345 * Look up the MAC address for the device. First we'll try15346 * eth_platform_get_mac_address, which will check Open Firmware, or arch15347 * specific fallback. Otherwise, we'll default to the stored value in15348 * firmware.15349 **/15350static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)15351{15352 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))15353 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);15354}15355 15356/**15357 * i40e_set_fec_in_flags - helper function for setting FEC options in flags15358 * @fec_cfg: FEC option to set in flags15359 * @flags: ptr to flags in which we set FEC option15360 **/15361void i40e_set_fec_in_flags(u8 fec_cfg, unsigned long *flags)15362{15363 if (fec_cfg & I40E_AQ_SET_FEC_AUTO) {15364 set_bit(I40E_FLAG_RS_FEC, flags);15365 set_bit(I40E_FLAG_BASE_R_FEC, flags);15366 }15367 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) ||15368 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) {15369 set_bit(I40E_FLAG_RS_FEC, flags);15370 clear_bit(I40E_FLAG_BASE_R_FEC, flags);15371 }15372 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) ||15373 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) {15374 set_bit(I40E_FLAG_BASE_R_FEC, flags);15375 clear_bit(I40E_FLAG_RS_FEC, flags);15376 }15377 if (fec_cfg == 0) {15378 clear_bit(I40E_FLAG_RS_FEC, flags);15379 clear_bit(I40E_FLAG_BASE_R_FEC, flags);15380 }15381}15382 15383/**15384 * i40e_check_recovery_mode - check if we are running transition firmware15385 * @pf: board private structure15386 *15387 * Check registers indicating the firmware runs in recovery mode. Sets the15388 * appropriate driver state.15389 *15390 * Returns true if the recovery mode was detected, false otherwise15391 **/15392static bool i40e_check_recovery_mode(struct i40e_pf *pf)15393{15394 u32 val = rd32(&pf->hw, I40E_GL_FWSTS);15395 15396 if (val & I40E_GL_FWSTS_FWS1B_MASK) {15397 dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n");15398 dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");15399 set_bit(__I40E_RECOVERY_MODE, pf->state);15400 15401 return true;15402 }15403 if (test_bit(__I40E_RECOVERY_MODE, pf->state))15404 dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full functionality.\n");15405 15406 return false;15407}15408 15409/**15410 * i40e_pf_loop_reset - perform reset in a loop.15411 * @pf: board private structure15412 *15413 * This function is useful when a NIC is about to enter recovery mode.15414 * When a NIC's internal data structures are corrupted the NIC's15415 * firmware is going to enter recovery mode.15416 * Right after a POR it takes about 7 minutes for firmware to enter15417 * recovery mode. Until that time a NIC is in some kind of intermediate15418 * state. After that time period the NIC almost surely enters15419 * recovery mode. The only way for a driver to detect intermediate15420 * state is to issue a series of pf-resets and check a return value.15421 * If a PF reset returns success then the firmware could be in recovery15422 * mode so the caller of this code needs to check for recovery mode15423 * if this function returns success. There is a little chance that15424 * firmware will hang in intermediate state forever.15425 * Since waiting 7 minutes is quite a lot of time this function waits15426 * 10 seconds and then gives up by returning an error.15427 *15428 * Return 0 on success, negative on failure.15429 **/15430static int i40e_pf_loop_reset(struct i40e_pf *pf)15431{15432 /* wait max 10 seconds for PF reset to succeed */15433 const unsigned long time_end = jiffies + 10 * HZ;15434 struct i40e_hw *hw = &pf->hw;15435 int ret;15436 15437 ret = i40e_pf_reset(hw);15438 while (ret != 0 && time_before(jiffies, time_end)) {15439 usleep_range(10000, 20000);15440 ret = i40e_pf_reset(hw);15441 }15442 15443 if (ret == 0)15444 pf->pfr_count++;15445 else15446 dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret);15447 15448 return ret;15449}15450 15451/**15452 * i40e_check_fw_empr - check if FW issued unexpected EMP Reset15453 * @pf: board private structure15454 *15455 * Check FW registers to determine if FW issued unexpected EMP Reset.15456 * Every time when unexpected EMP Reset occurs the FW increments15457 * a counter of unexpected EMP Resets. When the counter reaches 1015458 * the FW should enter the Recovery mode15459 *15460 * Returns true if FW issued unexpected EMP Reset15461 **/15462static bool i40e_check_fw_empr(struct i40e_pf *pf)15463{15464 const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) &15465 I40E_GL_FWSTS_FWS1B_MASK;15466 return (fw_sts > I40E_GL_FWSTS_FWS1B_EMPR_0) &&15467 (fw_sts <= I40E_GL_FWSTS_FWS1B_EMPR_10);15468}15469 15470/**15471 * i40e_handle_resets - handle EMP resets and PF resets15472 * @pf: board private structure15473 *15474 * Handle both EMP resets and PF resets and conclude whether there are15475 * any issues regarding these resets. If there are any issues then15476 * generate log entry.15477 *15478 * Return 0 if NIC is healthy or negative value when there are issues15479 * with resets15480 **/15481static int i40e_handle_resets(struct i40e_pf *pf)15482{15483 const int pfr = i40e_pf_loop_reset(pf);15484 const bool is_empr = i40e_check_fw_empr(pf);15485 15486 if (is_empr || pfr != 0)15487 dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n");15488 15489 return is_empr ? -EIO : pfr;15490}15491 15492/**15493 * i40e_init_recovery_mode - initialize subsystems needed in recovery mode15494 * @pf: board private structure15495 * @hw: ptr to the hardware info15496 *15497 * This function does a minimal setup of all subsystems needed for running15498 * recovery mode.15499 *15500 * Returns 0 on success, negative on failure15501 **/15502static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw)15503{15504 struct i40e_vsi *vsi;15505 int err;15506 int v_idx;15507 15508 pci_set_drvdata(pf->pdev, pf);15509 pci_save_state(pf->pdev);15510 15511 /* set up periodic task facility */15512 timer_setup(&pf->service_timer, i40e_service_timer, 0);15513 pf->service_timer_period = HZ;15514 15515 INIT_WORK(&pf->service_task, i40e_service_task);15516 clear_bit(__I40E_SERVICE_SCHED, pf->state);15517 15518 err = i40e_init_interrupt_scheme(pf);15519 if (err)15520 goto err_switch_setup;15521 15522 /* The number of VSIs reported by the FW is the minimum guaranteed15523 * to us; HW supports far more and we share the remaining pool with15524 * the other PFs. We allocate space for more than the guarantee with15525 * the understanding that we might not get them all later.15526 */15527 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)15528 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;15529 else15530 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;15531 15532 /* Set up the vsi struct and our local tracking of the MAIN PF vsi. */15533 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),15534 GFP_KERNEL);15535 if (!pf->vsi) {15536 err = -ENOMEM;15537 goto err_switch_setup;15538 }15539 15540 /* We allocate one VSI which is needed as absolute minimum15541 * in order to register the netdev15542 */15543 v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN);15544 if (v_idx < 0) {15545 err = v_idx;15546 goto err_switch_setup;15547 }15548 pf->lan_vsi = v_idx;15549 vsi = pf->vsi[v_idx];15550 if (!vsi) {15551 err = -EFAULT;15552 goto err_switch_setup;15553 }15554 vsi->alloc_queue_pairs = 1;15555 err = i40e_config_netdev(vsi);15556 if (err)15557 goto err_switch_setup;15558 err = register_netdev(vsi->netdev);15559 if (err)15560 goto err_switch_setup;15561 vsi->netdev_registered = true;15562 i40e_dbg_pf_init(pf);15563 15564 err = i40e_setup_misc_vector_for_recovery_mode(pf);15565 if (err)15566 goto err_switch_setup;15567 15568 /* tell the firmware that we're starting */15569 i40e_send_version(pf);15570 15571 /* since everything's happy, start the service_task timer */15572 mod_timer(&pf->service_timer,15573 round_jiffies(jiffies + pf->service_timer_period));15574 15575 return 0;15576 15577err_switch_setup:15578 i40e_reset_interrupt_capability(pf);15579 timer_shutdown_sync(&pf->service_timer);15580 i40e_shutdown_adminq(hw);15581 iounmap(hw->hw_addr);15582 pci_release_mem_regions(pf->pdev);15583 pci_disable_device(pf->pdev);15584 i40e_free_pf(pf);15585 15586 return err;15587}15588 15589/**15590 * i40e_set_subsystem_device_id - set subsystem device id15591 * @hw: pointer to the hardware info15592 *15593 * Set PCI subsystem device id either from a pci_dev structure or15594 * a specific FW register.15595 **/15596static inline void i40e_set_subsystem_device_id(struct i40e_hw *hw)15597{15598 struct i40e_pf *pf = i40e_hw_to_pf(hw);15599 15600 hw->subsystem_device_id = pf->pdev->subsystem_device ?15601 pf->pdev->subsystem_device :15602 (ushort)(rd32(hw, I40E_PFPCI_SUBSYSID) & USHRT_MAX);15603}15604 15605/**15606 * i40e_probe - Device initialization routine15607 * @pdev: PCI device information struct15608 * @ent: entry in i40e_pci_tbl15609 *15610 * i40e_probe initializes a PF identified by a pci_dev structure.15611 * The OS initialization, configuring of the PF private structure,15612 * and a hardware reset occur.15613 *15614 * Returns 0 on success, negative on failure15615 **/15616static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)15617{15618 struct i40e_aq_get_phy_abilities_resp abilities;15619#ifdef CONFIG_I40E_DCB15620 enum i40e_get_fw_lldp_status_resp lldp_status;15621#endif /* CONFIG_I40E_DCB */15622 struct i40e_vsi *vsi;15623 struct i40e_pf *pf;15624 struct i40e_hw *hw;15625 u16 wol_nvm_bits;15626 char nvm_ver[32];15627 u16 link_status;15628#ifdef CONFIG_I40E_DCB15629 int status;15630#endif /* CONFIG_I40E_DCB */15631 int err;15632 u32 val;15633 15634 err = pci_enable_device_mem(pdev);15635 if (err)15636 return err;15637 15638 /* set up for high or low dma */15639 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));15640 if (err) {15641 dev_err(&pdev->dev,15642 "DMA configuration failed: 0x%x\n", err);15643 goto err_dma;15644 }15645 15646 /* set up pci connections */15647 err = pci_request_mem_regions(pdev, i40e_driver_name);15648 if (err) {15649 dev_info(&pdev->dev,15650 "pci_request_selected_regions failed %d\n", err);15651 goto err_pci_reg;15652 }15653 15654 pci_set_master(pdev);15655 15656 /* Now that we have a PCI connection, we need to do the15657 * low level device setup. This is primarily setting up15658 * the Admin Queue structures and then querying for the15659 * device's current profile information.15660 */15661 pf = i40e_alloc_pf(&pdev->dev);15662 if (!pf) {15663 err = -ENOMEM;15664 goto err_pf_alloc;15665 }15666 pf->next_vsi = 0;15667 pf->pdev = pdev;15668 set_bit(__I40E_DOWN, pf->state);15669 15670 hw = &pf->hw;15671 15672 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),15673 I40E_MAX_CSR_SPACE);15674 /* We believe that the highest register to read is15675 * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size15676 * is not less than that before mapping to prevent a15677 * kernel panic.15678 */15679 if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) {15680 dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n",15681 pf->ioremap_len);15682 err = -ENOMEM;15683 goto err_ioremap;15684 }15685 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);15686 if (!hw->hw_addr) {15687 err = -EIO;15688 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",15689 (unsigned int)pci_resource_start(pdev, 0),15690 pf->ioremap_len, err);15691 goto err_ioremap;15692 }15693 hw->vendor_id = pdev->vendor;15694 hw->device_id = pdev->device;15695 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);15696 hw->subsystem_vendor_id = pdev->subsystem_vendor;15697 i40e_set_subsystem_device_id(hw);15698 hw->bus.device = PCI_SLOT(pdev->devfn);15699 hw->bus.func = PCI_FUNC(pdev->devfn);15700 hw->bus.bus_id = pdev->bus->number;15701 15702 /* Select something other than the 802.1ad ethertype for the15703 * switch to use internally and drop on ingress.15704 */15705 hw->switch_tag = 0xffff;15706 hw->first_tag = ETH_P_8021AD;15707 hw->second_tag = ETH_P_8021Q;15708 15709 INIT_LIST_HEAD(&pf->l3_flex_pit_list);15710 INIT_LIST_HEAD(&pf->l4_flex_pit_list);15711 INIT_LIST_HEAD(&pf->ddp_old_prof);15712 15713 /* set up the locks for the AQ, do this only once in probe15714 * and destroy them only once in remove15715 */15716 mutex_init(&hw->aq.asq_mutex);15717 mutex_init(&hw->aq.arq_mutex);15718 15719 pf->msg_enable = netif_msg_init(debug,15720 NETIF_MSG_DRV |15721 NETIF_MSG_PROBE |15722 NETIF_MSG_LINK);15723 if (debug < -1)15724 pf->hw.debug_mask = debug;15725 15726 /* do a special CORER for clearing PXE mode once at init */15727 if (hw->revision_id == 0 &&15728 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {15729 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);15730 i40e_flush(hw);15731 msleep(200);15732 pf->corer_count++;15733 15734 i40e_clear_pxe_mode(hw);15735 }15736 15737 /* Reset here to make sure all is clean and to define PF 'n' */15738 i40e_clear_hw(hw);15739 15740 err = i40e_set_mac_type(hw);15741 if (err) {15742 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",15743 err);15744 goto err_pf_reset;15745 }15746 15747 err = i40e_handle_resets(pf);15748 if (err)15749 goto err_pf_reset;15750 15751 i40e_check_recovery_mode(pf);15752 15753 if (is_kdump_kernel()) {15754 hw->aq.num_arq_entries = I40E_MIN_ARQ_LEN;15755 hw->aq.num_asq_entries = I40E_MIN_ASQ_LEN;15756 } else {15757 hw->aq.num_arq_entries = I40E_AQ_LEN;15758 hw->aq.num_asq_entries = I40E_AQ_LEN;15759 }15760 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;15761 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;15762 15763 snprintf(pf->int_name, sizeof(pf->int_name) - 1,15764 "%s-%s:misc",15765 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));15766 15767 err = i40e_init_shared_code(hw);15768 if (err) {15769 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",15770 err);15771 goto err_pf_reset;15772 }15773 15774 /* set up a default setting for link flow control */15775 pf->hw.fc.requested_mode = I40E_FC_NONE;15776 15777 err = i40e_init_adminq(hw);15778 if (err) {15779 if (err == -EIO)15780 dev_info(&pdev->dev,15781 "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n",15782 hw->aq.api_maj_ver,15783 hw->aq.api_min_ver,15784 I40E_FW_API_VERSION_MAJOR,15785 I40E_FW_MINOR_VERSION(hw));15786 else15787 dev_info(&pdev->dev,15788 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");15789 15790 goto err_pf_reset;15791 }15792 i40e_get_oem_version(hw);15793 i40e_get_pba_string(hw);15794 15795 /* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */15796 i40e_nvm_version_str(hw, nvm_ver, sizeof(nvm_ver));15797 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n",15798 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,15799 hw->aq.api_maj_ver, hw->aq.api_min_ver, nvm_ver,15800 hw->vendor_id, hw->device_id, hw->subsystem_vendor_id,15801 hw->subsystem_device_id);15802 15803 if (i40e_is_aq_api_ver_ge(hw, I40E_FW_API_VERSION_MAJOR,15804 I40E_FW_MINOR_VERSION(hw) + 1))15805 dev_dbg(&pdev->dev,15806 "The driver for the device detected a newer version of the NVM image v%u.%u than v%u.%u.\n",15807 hw->aq.api_maj_ver,15808 hw->aq.api_min_ver,15809 I40E_FW_API_VERSION_MAJOR,15810 I40E_FW_MINOR_VERSION(hw));15811 else if (i40e_is_aq_api_ver_lt(hw, 1, 4))15812 dev_info(&pdev->dev,15813 "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n",15814 hw->aq.api_maj_ver,15815 hw->aq.api_min_ver,15816 I40E_FW_API_VERSION_MAJOR,15817 I40E_FW_MINOR_VERSION(hw));15818 15819 i40e_verify_eeprom(pf);15820 15821 /* Rev 0 hardware was never productized */15822 if (hw->revision_id < 1)15823 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");15824 15825 i40e_clear_pxe_mode(hw);15826 15827 err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);15828 if (err)15829 goto err_adminq_setup;15830 15831 err = i40e_sw_init(pf);15832 if (err) {15833 dev_info(&pdev->dev, "sw_init failed: %d\n", err);15834 goto err_sw_init;15835 }15836 15837 if (test_bit(__I40E_RECOVERY_MODE, pf->state))15838 return i40e_init_recovery_mode(pf, hw);15839 15840 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,15841 hw->func_caps.num_rx_qp, 0, 0);15842 if (err) {15843 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);15844 goto err_init_lan_hmc;15845 }15846 15847 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);15848 if (err) {15849 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);15850 err = -ENOENT;15851 goto err_configure_lan_hmc;15852 }15853 15854 /* Disable LLDP for NICs that have firmware versions lower than v4.3.15855 * Ignore error return codes because if it was already disabled via15856 * hardware settings this will fail15857 */15858 if (test_bit(I40E_HW_CAP_STOP_FW_LLDP, pf->hw.caps)) {15859 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");15860 i40e_aq_stop_lldp(hw, true, false, NULL);15861 }15862 15863 /* allow a platform config to override the HW addr */15864 i40e_get_platform_mac_addr(pdev, pf);15865 15866 if (!is_valid_ether_addr(hw->mac.addr)) {15867 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);15868 err = -EIO;15869 goto err_mac_addr;15870 }15871 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);15872 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);15873 i40e_get_port_mac_addr(hw, hw->mac.port_addr);15874 if (is_valid_ether_addr(hw->mac.port_addr))15875 set_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps);15876 15877 i40e_ptp_alloc_pins(pf);15878 pci_set_drvdata(pdev, pf);15879 pci_save_state(pdev);15880 15881#ifdef CONFIG_I40E_DCB15882 status = i40e_get_fw_lldp_status(&pf->hw, &lldp_status);15883 (!status &&15884 lldp_status == I40E_GET_FW_LLDP_STATUS_ENABLED) ?15885 (clear_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) :15886 (set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags));15887 dev_info(&pdev->dev,15888 test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ?15889 "FW LLDP is disabled\n" :15890 "FW LLDP is enabled\n");15891 15892 /* Enable FW to write default DCB config on link-up */15893 i40e_aq_set_dcb_parameters(hw, true, NULL);15894 15895 err = i40e_init_pf_dcb(pf);15896 if (err) {15897 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);15898 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);15899 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);15900 /* Continue without DCB enabled */15901 }15902#endif /* CONFIG_I40E_DCB */15903 15904 /* set up periodic task facility */15905 timer_setup(&pf->service_timer, i40e_service_timer, 0);15906 pf->service_timer_period = HZ;15907 15908 INIT_WORK(&pf->service_task, i40e_service_task);15909 clear_bit(__I40E_SERVICE_SCHED, pf->state);15910 15911 /* NVM bit on means WoL disabled for the port */15912 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);15913 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)15914 pf->wol_en = false;15915 else15916 pf->wol_en = true;15917 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);15918 15919 /* set up the main switch operations */15920 i40e_determine_queue_usage(pf);15921 err = i40e_init_interrupt_scheme(pf);15922 if (err)15923 goto err_switch_setup;15924 15925 /* Reduce Tx and Rx pairs for kdump15926 * When MSI-X is enabled, it's not allowed to use more TC queue15927 * pairs than MSI-X vectors (pf->num_lan_msix) exist. Thus15928 * vsi->num_queue_pairs will be equal to pf->num_lan_msix, i.e., 1.15929 */15930 if (is_kdump_kernel())15931 pf->num_lan_msix = 1;15932 15933 pf->udp_tunnel_nic.set_port = i40e_udp_tunnel_set_port;15934 pf->udp_tunnel_nic.unset_port = i40e_udp_tunnel_unset_port;15935 pf->udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP;15936 pf->udp_tunnel_nic.shared = &pf->udp_tunnel_shared;15937 pf->udp_tunnel_nic.tables[0].n_entries = I40E_MAX_PF_UDP_OFFLOAD_PORTS;15938 pf->udp_tunnel_nic.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN |15939 UDP_TUNNEL_TYPE_GENEVE;15940 15941 /* The number of VSIs reported by the FW is the minimum guaranteed15942 * to us; HW supports far more and we share the remaining pool with15943 * the other PFs. We allocate space for more than the guarantee with15944 * the understanding that we might not get them all later.15945 */15946 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)15947 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;15948 else15949 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;15950 if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) {15951 dev_warn(&pf->pdev->dev,15952 "limiting the VSI count due to UDP tunnel limitation %d > %d\n",15953 pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES);15954 pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES;15955 }15956 15957 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */15958 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),15959 GFP_KERNEL);15960 if (!pf->vsi) {15961 err = -ENOMEM;15962 goto err_switch_setup;15963 }15964 15965#ifdef CONFIG_PCI_IOV15966 /* prep for VF support */15967 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&15968 test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&15969 !test_bit(__I40E_BAD_EEPROM, pf->state)) {15970 if (pci_num_vf(pdev))15971 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);15972 }15973#endif15974 err = i40e_setup_pf_switch(pf, false, false);15975 if (err) {15976 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);15977 goto err_vsis;15978 }15979 15980 vsi = i40e_pf_get_main_vsi(pf);15981 INIT_LIST_HEAD(&vsi->ch_list);15982 15983 /* if FDIR VSI was set up, start it now */15984 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);15985 if (vsi)15986 i40e_vsi_open(vsi);15987 15988 /* The driver only wants link up/down and module qualification15989 * reports from firmware. Note the negative logic.15990 */15991 err = i40e_aq_set_phy_int_mask(&pf->hw,15992 ~(I40E_AQ_EVENT_LINK_UPDOWN |15993 I40E_AQ_EVENT_MEDIA_NA |15994 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);15995 if (err)15996 dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n",15997 ERR_PTR(err),15998 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));15999 16000 /* Reconfigure hardware for allowing smaller MSS in the case16001 * of TSO, so that we avoid the MDD being fired and causing16002 * a reset in the case of small MSS+TSO.16003 */16004 val = rd32(hw, I40E_REG_MSS);16005 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {16006 val &= ~I40E_REG_MSS_MIN_MASK;16007 val |= I40E_64BYTE_MSS;16008 wr32(hw, I40E_REG_MSS, val);16009 }16010 16011 if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) {16012 msleep(75);16013 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);16014 if (err)16015 dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n",16016 ERR_PTR(err),16017 i40e_aq_str(&pf->hw,16018 pf->hw.aq.asq_last_status));16019 }16020 /* The main driver is (mostly) up and happy. We need to set this state16021 * before setting up the misc vector or we get a race and the vector16022 * ends up disabled forever.16023 */16024 clear_bit(__I40E_DOWN, pf->state);16025 16026 /* In case of MSIX we are going to setup the misc vector right here16027 * to handle admin queue events etc. In case of legacy and MSI16028 * the misc functionality and queue processing is combined in16029 * the same vector and that gets setup at open.16030 */16031 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {16032 err = i40e_setup_misc_vector(pf);16033 if (err) {16034 dev_info(&pdev->dev,16035 "setup of misc vector failed: %d\n", err);16036 i40e_cloud_filter_exit(pf);16037 i40e_fdir_teardown(pf);16038 goto err_vsis;16039 }16040 }16041 16042#ifdef CONFIG_PCI_IOV16043 /* prep for VF support */16044 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&16045 test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&16046 !test_bit(__I40E_BAD_EEPROM, pf->state)) {16047 /* disable link interrupts for VFs */16048 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);16049 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;16050 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);16051 i40e_flush(hw);16052 16053 if (pci_num_vf(pdev)) {16054 dev_info(&pdev->dev,16055 "Active VFs found, allocating resources.\n");16056 err = i40e_alloc_vfs(pf, pci_num_vf(pdev));16057 if (err)16058 dev_info(&pdev->dev,16059 "Error %d allocating resources for existing VFs\n",16060 err);16061 }16062 }16063#endif /* CONFIG_PCI_IOV */16064 16065 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {16066 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,16067 pf->num_iwarp_msix,16068 I40E_IWARP_IRQ_PILE_ID);16069 if (pf->iwarp_base_vector < 0) {16070 dev_info(&pdev->dev,16071 "failed to get tracking for %d vectors for IWARP err=%d\n",16072 pf->num_iwarp_msix, pf->iwarp_base_vector);16073 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);16074 }16075 }16076 16077 i40e_dbg_pf_init(pf);16078 16079 /* tell the firmware that we're starting */16080 i40e_send_version(pf);16081 16082 /* since everything's happy, start the service_task timer */16083 mod_timer(&pf->service_timer,16084 round_jiffies(jiffies + pf->service_timer_period));16085 16086 /* add this PF to client device list and launch a client service task */16087 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {16088 err = i40e_lan_add_device(pf);16089 if (err)16090 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",16091 err);16092 }16093 16094#define PCI_SPEED_SIZE 816095#define PCI_WIDTH_SIZE 816096 /* Devices on the IOSF bus do not have this information16097 * and will report PCI Gen 1 x 1 by default so don't bother16098 * checking them.16099 */16100 if (!test_bit(I40E_HW_CAP_NO_PCI_LINK_CHECK, pf->hw.caps)) {16101 char speed[PCI_SPEED_SIZE] = "Unknown";16102 char width[PCI_WIDTH_SIZE] = "Unknown";16103 16104 /* Get the negotiated link width and speed from PCI config16105 * space16106 */16107 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,16108 &link_status);16109 16110 i40e_set_pci_config_data(hw, link_status);16111 16112 switch (hw->bus.speed) {16113 case i40e_bus_speed_8000:16114 strscpy(speed, "8.0", PCI_SPEED_SIZE); break;16115 case i40e_bus_speed_5000:16116 strscpy(speed, "5.0", PCI_SPEED_SIZE); break;16117 case i40e_bus_speed_2500:16118 strscpy(speed, "2.5", PCI_SPEED_SIZE); break;16119 default:16120 break;16121 }16122 switch (hw->bus.width) {16123 case i40e_bus_width_pcie_x8:16124 strscpy(width, "8", PCI_WIDTH_SIZE); break;16125 case i40e_bus_width_pcie_x4:16126 strscpy(width, "4", PCI_WIDTH_SIZE); break;16127 case i40e_bus_width_pcie_x2:16128 strscpy(width, "2", PCI_WIDTH_SIZE); break;16129 case i40e_bus_width_pcie_x1:16130 strscpy(width, "1", PCI_WIDTH_SIZE); break;16131 default:16132 break;16133 }16134 16135 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",16136 speed, width);16137 16138 if (hw->bus.width < i40e_bus_width_pcie_x8 ||16139 hw->bus.speed < i40e_bus_speed_8000) {16140 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");16141 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");16142 }16143 }16144 16145 /* get the requested speeds from the fw */16146 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);16147 if (err)16148 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %pe last_status = %s\n",16149 ERR_PTR(err),16150 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));16151 pf->hw.phy.link_info.requested_speeds = abilities.link_speed;16152 16153 /* set the FEC config due to the board capabilities */16154 i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, pf->flags);16155 16156 /* get the supported phy types from the fw */16157 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);16158 if (err)16159 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %pe last_status = %s\n",16160 ERR_PTR(err),16161 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));16162 16163 /* make sure the MFS hasn't been set lower than the default */16164#define MAX_FRAME_SIZE_DEFAULT 0x260016165 val = FIELD_GET(I40E_PRTGL_SAH_MFS_MASK,16166 rd32(&pf->hw, I40E_PRTGL_SAH));16167 if (val < MAX_FRAME_SIZE_DEFAULT)16168 dev_warn(&pdev->dev, "MFS for port %x (%d) has been set below the default (%d)\n",16169 pf->hw.port, val, MAX_FRAME_SIZE_DEFAULT);16170 16171 /* Add a filter to drop all Flow control frames from any VSI from being16172 * transmitted. By doing so we stop a malicious VF from sending out16173 * PAUSE or PFC frames and potentially controlling traffic for other16174 * PF/VF VSIs.16175 * The FW can still send Flow control frames if enabled.16176 */16177 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,16178 pf->main_vsi_seid);16179 16180 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||16181 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))16182 set_bit(I40E_HW_CAP_PHY_CONTROLS_LEDS, pf->hw.caps);16183 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)16184 set_bit(I40E_HW_CAP_CRT_RETIMER, pf->hw.caps);16185 /* print a string summarizing features */16186 i40e_print_features(pf);16187 16188 i40e_devlink_register(pf);16189 16190 return 0;16191 16192 /* Unwind what we've done if something failed in the setup */16193err_vsis:16194 set_bit(__I40E_DOWN, pf->state);16195 i40e_clear_interrupt_scheme(pf);16196 kfree(pf->vsi);16197err_switch_setup:16198 i40e_reset_interrupt_capability(pf);16199 timer_shutdown_sync(&pf->service_timer);16200err_mac_addr:16201err_configure_lan_hmc:16202 (void)i40e_shutdown_lan_hmc(hw);16203err_init_lan_hmc:16204 kfree(pf->qp_pile);16205err_sw_init:16206err_adminq_setup:16207err_pf_reset:16208 iounmap(hw->hw_addr);16209err_ioremap:16210 i40e_free_pf(pf);16211err_pf_alloc:16212 pci_release_mem_regions(pdev);16213err_pci_reg:16214err_dma:16215 pci_disable_device(pdev);16216 return err;16217}16218 16219/**16220 * i40e_remove - Device removal routine16221 * @pdev: PCI device information struct16222 *16223 * i40e_remove is called by the PCI subsystem to alert the driver16224 * that is should release a PCI device. This could be caused by a16225 * Hot-Plug event, or because the driver is going to be removed from16226 * memory.16227 **/16228static void i40e_remove(struct pci_dev *pdev)16229{16230 struct i40e_pf *pf = pci_get_drvdata(pdev);16231 struct i40e_hw *hw = &pf->hw;16232 struct i40e_vsi *vsi;16233 struct i40e_veb *veb;16234 int ret_code;16235 int i;16236 16237 i40e_devlink_unregister(pf);16238 16239 i40e_dbg_pf_exit(pf);16240 16241 i40e_ptp_stop(pf);16242 16243 /* Disable RSS in hw */16244 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);16245 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);16246 16247 /* Grab __I40E_RESET_RECOVERY_PENDING and set __I40E_IN_REMOVE16248 * flags, once they are set, i40e_rebuild should not be called as16249 * i40e_prep_for_reset always returns early.16250 */16251 while (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))16252 usleep_range(1000, 2000);16253 set_bit(__I40E_IN_REMOVE, pf->state);16254 16255 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags)) {16256 set_bit(__I40E_VF_RESETS_DISABLED, pf->state);16257 i40e_free_vfs(pf);16258 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);16259 }16260 /* no more scheduling of any task */16261 set_bit(__I40E_SUSPENDED, pf->state);16262 set_bit(__I40E_DOWN, pf->state);16263 if (pf->service_timer.function)16264 timer_shutdown_sync(&pf->service_timer);16265 if (pf->service_task.func)16266 cancel_work_sync(&pf->service_task);16267 16268 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {16269 struct i40e_vsi *vsi = pf->vsi[0];16270 16271 /* We know that we have allocated only one vsi for this PF,16272 * it was just for registering netdevice, so the interface16273 * could be visible in the 'ifconfig' output16274 */16275 unregister_netdev(vsi->netdev);16276 free_netdev(vsi->netdev);16277 16278 goto unmap;16279 }16280 16281 /* Client close must be called explicitly here because the timer16282 * has been stopped.16283 */16284 i40e_notify_client_of_netdev_close(pf, false);16285 16286 i40e_fdir_teardown(pf);16287 16288 /* If there is a switch structure or any orphans, remove them.16289 * This will leave only the PF's VSI remaining.16290 */16291 i40e_pf_for_each_veb(pf, i, veb)16292 if (veb->uplink_seid == pf->mac_seid ||16293 veb->uplink_seid == 0)16294 i40e_switch_branch_release(veb);16295 16296 /* Now we can shutdown the PF's VSIs, just before we kill16297 * adminq and hmc.16298 */16299 i40e_pf_for_each_vsi(pf, i, vsi) {16300 i40e_vsi_close(vsi);16301 i40e_vsi_release(vsi);16302 pf->vsi[i] = NULL;16303 }16304 16305 i40e_cloud_filter_exit(pf);16306 16307 /* remove attached clients */16308 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {16309 ret_code = i40e_lan_del_device(pf);16310 if (ret_code)16311 dev_warn(&pdev->dev, "Failed to delete client device: %d\n",16312 ret_code);16313 }16314 16315 /* shutdown and destroy the HMC */16316 if (hw->hmc.hmc_obj) {16317 ret_code = i40e_shutdown_lan_hmc(hw);16318 if (ret_code)16319 dev_warn(&pdev->dev,16320 "Failed to destroy the HMC resources: %d\n",16321 ret_code);16322 }16323 16324unmap:16325 /* Free MSI/legacy interrupt 0 when in recovery mode. */16326 if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&16327 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))16328 free_irq(pf->pdev->irq, pf);16329 16330 /* shutdown the adminq */16331 i40e_shutdown_adminq(hw);16332 16333 /* destroy the locks only once, here */16334 mutex_destroy(&hw->aq.arq_mutex);16335 mutex_destroy(&hw->aq.asq_mutex);16336 16337 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */16338 rtnl_lock();16339 i40e_clear_interrupt_scheme(pf);16340 i40e_pf_for_each_vsi(pf, i, vsi) {16341 if (!test_bit(__I40E_RECOVERY_MODE, pf->state))16342 i40e_vsi_clear_rings(vsi);16343 16344 i40e_vsi_clear(vsi);16345 pf->vsi[i] = NULL;16346 }16347 rtnl_unlock();16348 16349 i40e_pf_for_each_veb(pf, i, veb) {16350 kfree(veb);16351 pf->veb[i] = NULL;16352 }16353 16354 kfree(pf->qp_pile);16355 kfree(pf->vsi);16356 16357 iounmap(hw->hw_addr);16358 i40e_free_pf(pf);16359 pci_release_mem_regions(pdev);16360 16361 pci_disable_device(pdev);16362}16363 16364/**16365 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up16366 * using the mac_address_write admin q function16367 * @pf: pointer to i40e_pf struct16368 **/16369static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)16370{16371 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf);16372 struct i40e_hw *hw = &pf->hw;16373 u8 mac_addr[6];16374 u16 flags = 0;16375 int ret;16376 16377 /* Get current MAC address in case it's an LAA */16378 if (main_vsi && main_vsi->netdev) {16379 ether_addr_copy(mac_addr, main_vsi->netdev->dev_addr);16380 } else {16381 dev_err(&pf->pdev->dev,16382 "Failed to retrieve MAC address; using default\n");16383 ether_addr_copy(mac_addr, hw->mac.addr);16384 }16385 16386 /* The FW expects the mac address write cmd to first be called with16387 * one of these flags before calling it again with the multicast16388 * enable flags.16389 */16390 flags = I40E_AQC_WRITE_TYPE_LAA_WOL;16391 16392 if (hw->func_caps.flex10_enable && hw->partition_id != 1)16393 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;16394 16395 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);16396 if (ret) {16397 dev_err(&pf->pdev->dev,16398 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");16399 return;16400 }16401 16402 flags = I40E_AQC_MC_MAG_EN16403 | I40E_AQC_WOL_PRESERVE_ON_PFR16404 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;16405 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);16406 if (ret)16407 dev_err(&pf->pdev->dev,16408 "Failed to enable Multicast Magic Packet wake up\n");16409}16410 16411/**16412 * i40e_io_suspend - suspend all IO operations16413 * @pf: pointer to i40e_pf struct16414 *16415 **/16416static int i40e_io_suspend(struct i40e_pf *pf)16417{16418 struct i40e_hw *hw = &pf->hw;16419 16420 set_bit(__I40E_DOWN, pf->state);16421 16422 /* Ensure service task will not be running */16423 del_timer_sync(&pf->service_timer);16424 cancel_work_sync(&pf->service_task);16425 16426 /* Client close must be called explicitly here because the timer16427 * has been stopped.16428 */16429 i40e_notify_client_of_netdev_close(pf, false);16430 16431 if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) &&16432 pf->wol_en)16433 i40e_enable_mc_magic_wake(pf);16434 16435 /* Since we're going to destroy queues during the16436 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this16437 * whole section16438 */16439 rtnl_lock();16440 16441 i40e_prep_for_reset(pf);16442 16443 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));16444 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));16445 16446 /* Clear the interrupt scheme and release our IRQs so that the system16447 * can safely hibernate even when there are a large number of CPUs.16448 * Otherwise hibernation might fail when mapping all the vectors back16449 * to CPU0.16450 */16451 i40e_clear_interrupt_scheme(pf);16452 16453 rtnl_unlock();16454 16455 return 0;16456}16457 16458/**16459 * i40e_io_resume - resume IO operations16460 * @pf: pointer to i40e_pf struct16461 *16462 **/16463static int i40e_io_resume(struct i40e_pf *pf)16464{16465 struct device *dev = &pf->pdev->dev;16466 int err;16467 16468 /* We need to hold the RTNL lock prior to restoring interrupt schemes,16469 * since we're going to be restoring queues16470 */16471 rtnl_lock();16472 16473 /* We cleared the interrupt scheme when we suspended, so we need to16474 * restore it now to resume device functionality.16475 */16476 err = i40e_restore_interrupt_scheme(pf);16477 if (err) {16478 dev_err(dev, "Cannot restore interrupt scheme: %d\n",16479 err);16480 }16481 16482 clear_bit(__I40E_DOWN, pf->state);16483 i40e_reset_and_rebuild(pf, false, true);16484 16485 rtnl_unlock();16486 16487 /* Clear suspended state last after everything is recovered */16488 clear_bit(__I40E_SUSPENDED, pf->state);16489 16490 /* Restart the service task */16491 mod_timer(&pf->service_timer,16492 round_jiffies(jiffies + pf->service_timer_period));16493 16494 return 0;16495}16496 16497/**16498 * i40e_pci_error_detected - warning that something funky happened in PCI land16499 * @pdev: PCI device information struct16500 * @error: the type of PCI error16501 *16502 * Called to warn that something happened and the error handling steps16503 * are in progress. Allows the driver to quiesce things, be ready for16504 * remediation.16505 **/16506static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,16507 pci_channel_state_t error)16508{16509 struct i40e_pf *pf = pci_get_drvdata(pdev);16510 16511 dev_info(&pdev->dev, "%s: error %d\n", __func__, error);16512 16513 if (!pf) {16514 dev_info(&pdev->dev,16515 "Cannot recover - error happened during device probe\n");16516 return PCI_ERS_RESULT_DISCONNECT;16517 }16518 16519 /* shutdown all operations */16520 if (!test_bit(__I40E_SUSPENDED, pf->state))16521 i40e_io_suspend(pf);16522 16523 /* Request a slot reset */16524 return PCI_ERS_RESULT_NEED_RESET;16525}16526 16527/**16528 * i40e_pci_error_slot_reset - a PCI slot reset just happened16529 * @pdev: PCI device information struct16530 *16531 * Called to find if the driver can work with the device now that16532 * the pci slot has been reset. If a basic connection seems good16533 * (registers are readable and have sane content) then return a16534 * happy little PCI_ERS_RESULT_xxx.16535 **/16536static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)16537{16538 struct i40e_pf *pf = pci_get_drvdata(pdev);16539 pci_ers_result_t result;16540 u32 reg;16541 16542 dev_dbg(&pdev->dev, "%s\n", __func__);16543 /* enable I/O and memory of the device */16544 if (pci_enable_device(pdev)) {16545 dev_info(&pdev->dev,16546 "Cannot re-enable PCI device after reset.\n");16547 result = PCI_ERS_RESULT_DISCONNECT;16548 } else {16549 pci_set_master(pdev);16550 pci_restore_state(pdev);16551 pci_save_state(pdev);16552 pci_wake_from_d3(pdev, false);16553 16554 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);16555 if (reg == 0)16556 result = PCI_ERS_RESULT_RECOVERED;16557 else16558 result = PCI_ERS_RESULT_DISCONNECT;16559 }16560 16561 return result;16562}16563 16564/**16565 * i40e_pci_error_reset_prepare - prepare device driver for pci reset16566 * @pdev: PCI device information struct16567 */16568static void i40e_pci_error_reset_prepare(struct pci_dev *pdev)16569{16570 struct i40e_pf *pf = pci_get_drvdata(pdev);16571 16572 i40e_prep_for_reset(pf);16573}16574 16575/**16576 * i40e_pci_error_reset_done - pci reset done, device driver reset can begin16577 * @pdev: PCI device information struct16578 */16579static void i40e_pci_error_reset_done(struct pci_dev *pdev)16580{16581 struct i40e_pf *pf = pci_get_drvdata(pdev);16582 16583 if (test_bit(__I40E_IN_REMOVE, pf->state))16584 return;16585 16586 i40e_reset_and_rebuild(pf, false, false);16587#ifdef CONFIG_PCI_IOV16588 i40e_restore_all_vfs_msi_state(pdev);16589#endif /* CONFIG_PCI_IOV */16590}16591 16592/**16593 * i40e_pci_error_resume - restart operations after PCI error recovery16594 * @pdev: PCI device information struct16595 *16596 * Called to allow the driver to bring things back up after PCI error16597 * and/or reset recovery has finished.16598 **/16599static void i40e_pci_error_resume(struct pci_dev *pdev)16600{16601 struct i40e_pf *pf = pci_get_drvdata(pdev);16602 16603 dev_dbg(&pdev->dev, "%s\n", __func__);16604 if (test_bit(__I40E_SUSPENDED, pf->state))16605 return;16606 16607 i40e_io_resume(pf);16608}16609 16610/**16611 * i40e_shutdown - PCI callback for shutting down16612 * @pdev: PCI device information struct16613 **/16614static void i40e_shutdown(struct pci_dev *pdev)16615{16616 struct i40e_pf *pf = pci_get_drvdata(pdev);16617 struct i40e_hw *hw = &pf->hw;16618 16619 set_bit(__I40E_SUSPENDED, pf->state);16620 set_bit(__I40E_DOWN, pf->state);16621 16622 del_timer_sync(&pf->service_timer);16623 cancel_work_sync(&pf->service_task);16624 i40e_cloud_filter_exit(pf);16625 i40e_fdir_teardown(pf);16626 16627 /* Client close must be called explicitly here because the timer16628 * has been stopped.16629 */16630 i40e_notify_client_of_netdev_close(pf, false);16631 16632 if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) &&16633 pf->wol_en)16634 i40e_enable_mc_magic_wake(pf);16635 16636 i40e_prep_for_reset(pf);16637 16638 wr32(hw, I40E_PFPM_APM,16639 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));16640 wr32(hw, I40E_PFPM_WUFC,16641 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));16642 16643 /* Free MSI/legacy interrupt 0 when in recovery mode. */16644 if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&16645 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))16646 free_irq(pf->pdev->irq, pf);16647 16648 /* Since we're going to destroy queues during the16649 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this16650 * whole section16651 */16652 rtnl_lock();16653 i40e_clear_interrupt_scheme(pf);16654 rtnl_unlock();16655 16656 if (system_state == SYSTEM_POWER_OFF) {16657 pci_wake_from_d3(pdev, pf->wol_en);16658 pci_set_power_state(pdev, PCI_D3hot);16659 }16660}16661 16662/**16663 * i40e_suspend - PM callback for moving to D316664 * @dev: generic device information structure16665 **/16666static int i40e_suspend(struct device *dev)16667{16668 struct i40e_pf *pf = dev_get_drvdata(dev);16669 16670 /* If we're already suspended, then there is nothing to do */16671 if (test_and_set_bit(__I40E_SUSPENDED, pf->state))16672 return 0;16673 return i40e_io_suspend(pf);16674}16675 16676/**16677 * i40e_resume - PM callback for waking up from D316678 * @dev: generic device information structure16679 **/16680static int i40e_resume(struct device *dev)16681{16682 struct i40e_pf *pf = dev_get_drvdata(dev);16683 16684 /* If we're not suspended, then there is nothing to do */16685 if (!test_bit(__I40E_SUSPENDED, pf->state))16686 return 0;16687 return i40e_io_resume(pf);16688}16689 16690static const struct pci_error_handlers i40e_err_handler = {16691 .error_detected = i40e_pci_error_detected,16692 .slot_reset = i40e_pci_error_slot_reset,16693 .reset_prepare = i40e_pci_error_reset_prepare,16694 .reset_done = i40e_pci_error_reset_done,16695 .resume = i40e_pci_error_resume,16696};16697 16698static DEFINE_SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume);16699 16700static struct pci_driver i40e_driver = {16701 .name = i40e_driver_name,16702 .id_table = i40e_pci_tbl,16703 .probe = i40e_probe,16704 .remove = i40e_remove,16705 .driver.pm = pm_sleep_ptr(&i40e_pm_ops),16706 .shutdown = i40e_shutdown,16707 .err_handler = &i40e_err_handler,16708 .sriov_configure = i40e_pci_sriov_configure,16709};16710 16711/**16712 * i40e_init_module - Driver registration routine16713 *16714 * i40e_init_module is the first routine called when the driver is16715 * loaded. All it does is register with the PCI subsystem.16716 **/16717static int __init i40e_init_module(void)16718{16719 int err;16720 16721 pr_info("%s: %s\n", i40e_driver_name, i40e_driver_string);16722 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);16723 16724 /* There is no need to throttle the number of active tasks because16725 * each device limits its own task using a state bit for scheduling16726 * the service task, and the device tasks do not interfere with each16727 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM16728 * since we need to be able to guarantee forward progress even under16729 * memory pressure.16730 */16731 i40e_wq = alloc_workqueue("%s", 0, 0, i40e_driver_name);16732 if (!i40e_wq) {16733 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);16734 return -ENOMEM;16735 }16736 16737 i40e_dbg_init();16738 err = pci_register_driver(&i40e_driver);16739 if (err) {16740 destroy_workqueue(i40e_wq);16741 i40e_dbg_exit();16742 return err;16743 }16744 16745 return 0;16746}16747module_init(i40e_init_module);16748 16749/**16750 * i40e_exit_module - Driver exit cleanup routine16751 *16752 * i40e_exit_module is called just before the driver is removed16753 * from memory.16754 **/16755static void __exit i40e_exit_module(void)16756{16757 pci_unregister_driver(&i40e_driver);16758 destroy_workqueue(i40e_wq);16759 ida_destroy(&i40e_client_ida);16760 i40e_dbg_exit();16761}16762module_exit(i40e_exit_module);16763