11668 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright(c) 1999 - 2018 Intel Corporation. */3 4#include <linux/types.h>5#include <linux/module.h>6#include <linux/pci.h>7#include <linux/netdevice.h>8#include <linux/vmalloc.h>9#include <linux/string.h>10#include <linux/in.h>11#include <linux/interrupt.h>12#include <linux/ip.h>13#include <linux/tcp.h>14#include <linux/sctp.h>15#include <linux/pkt_sched.h>16#include <linux/ipv6.h>17#include <linux/slab.h>18#include <net/checksum.h>19#include <net/ip6_checksum.h>20#include <linux/etherdevice.h>21#include <linux/ethtool.h>22#include <linux/if.h>23#include <linux/if_vlan.h>24#include <linux/if_macvlan.h>25#include <linux/if_bridge.h>26#include <linux/prefetch.h>27#include <linux/bpf.h>28#include <linux/bpf_trace.h>29#include <linux/atomic.h>30#include <linux/numa.h>31#include <generated/utsrelease.h>32#include <scsi/fc/fc_fcoe.h>33#include <net/udp_tunnel.h>34#include <net/pkt_cls.h>35#include <net/tc_act/tc_gact.h>36#include <net/tc_act/tc_mirred.h>37#include <net/vxlan.h>38#include <net/mpls.h>39#include <net/netdev_queues.h>40#include <net/xdp_sock_drv.h>41#include <net/xfrm.h>42 43#include "ixgbe.h"44#include "ixgbe_common.h"45#include "ixgbe_dcb_82599.h"46#include "ixgbe_phy.h"47#include "ixgbe_sriov.h"48#include "ixgbe_model.h"49#include "ixgbe_txrx_common.h"50 51char ixgbe_driver_name[] = "ixgbe";52static const char ixgbe_driver_string[] =53 "Intel(R) 10 Gigabit PCI Express Network Driver";54#ifdef IXGBE_FCOE55char ixgbe_default_device_descr[] =56 "Intel(R) 10 Gigabit Network Connection";57#else58static char ixgbe_default_device_descr[] =59 "Intel(R) 10 Gigabit Network Connection";60#endif61static const char ixgbe_copyright[] =62 "Copyright (c) 1999-2016 Intel Corporation.";63 64static const char ixgbe_overheat_msg[] = "Network adapter has been stopped because it has over heated. Restart the computer. If the problem persists, power off the system and replace the adapter";65 66static const struct ixgbe_info *ixgbe_info_tbl[] = {67 [board_82598] = &ixgbe_82598_info,68 [board_82599] = &ixgbe_82599_info,69 [board_X540] = &ixgbe_X540_info,70 [board_X550] = &ixgbe_X550_info,71 [board_X550EM_x] = &ixgbe_X550EM_x_info,72 [board_x550em_x_fw] = &ixgbe_x550em_x_fw_info,73 [board_x550em_a] = &ixgbe_x550em_a_info,74 [board_x550em_a_fw] = &ixgbe_x550em_a_fw_info,75};76 77/* ixgbe_pci_tbl - PCI Device ID Table78 *79 * Wildcard entries (PCI_ANY_ID) should come last80 * Last entry must be all 0s81 *82 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,83 * Class, Class Mask, private data (not used) }84 */85static const struct pci_device_id ixgbe_pci_tbl[] = {86 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 },87 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 },88 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 },89 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 },90 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 },91 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 },92 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 },93 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 },94 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 },95 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 },96 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 },97 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 },98 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 },99 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 },100 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 },101 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 },102 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 },103 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 },104 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 },105 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 },106 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 },107 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 },108 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 },109 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 },110 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 },111 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 },112 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_QSFP_SF_QP), board_82599 },113 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), board_82599 },114 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 },115 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T1), board_X540 },116 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T), board_X550},117 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T1), board_X550},118 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KX4), board_X550EM_x},119 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_XFI), board_X550EM_x},120 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KR), board_X550EM_x},121 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_10G_T), board_X550EM_x},122 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_SFP), board_X550EM_x},123 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_1G_T), board_x550em_x_fw},124 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR), board_x550em_a },125 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR_L), board_x550em_a },126 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP_N), board_x550em_a },127 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII), board_x550em_a },128 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII_L), board_x550em_a },129 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_10G_T), board_x550em_a},130 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP), board_x550em_a },131 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T), board_x550em_a_fw },132 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T_L), board_x550em_a_fw },133 /* required last entry */134 {0, }135};136MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);137 138#ifdef CONFIG_IXGBE_DCA139static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,140 void *p);141static struct notifier_block dca_notifier = {142 .notifier_call = ixgbe_notify_dca,143 .next = NULL,144 .priority = 0145};146#endif147 148#ifdef CONFIG_PCI_IOV149static unsigned int max_vfs;150module_param(max_vfs, uint, 0);151MODULE_PARM_DESC(max_vfs,152 "Maximum number of virtual functions to allocate per physical function - default is zero and maximum value is 63. (Deprecated)");153#endif /* CONFIG_PCI_IOV */154 155static bool allow_unsupported_sfp;156module_param(allow_unsupported_sfp, bool, 0444);157MODULE_PARM_DESC(allow_unsupported_sfp,158 "Allow unsupported and untested SFP+ modules on 82599-based adapters");159 160#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)161static int debug = -1;162module_param(debug, int, 0);163MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");164 165MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");166MODULE_LICENSE("GPL v2");167 168DEFINE_STATIC_KEY_FALSE(ixgbe_xdp_locking_key);169EXPORT_SYMBOL(ixgbe_xdp_locking_key);170 171static struct workqueue_struct *ixgbe_wq;172 173static bool ixgbe_check_cfg_remove(struct ixgbe_hw *hw, struct pci_dev *pdev);174static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *);175 176static const struct net_device_ops ixgbe_netdev_ops;177 178static bool netif_is_ixgbe(struct net_device *dev)179{180 return dev && (dev->netdev_ops == &ixgbe_netdev_ops);181}182 183static int ixgbe_read_pci_cfg_word_parent(struct ixgbe_adapter *adapter,184 u32 reg, u16 *value)185{186 struct pci_dev *parent_dev;187 struct pci_bus *parent_bus;188 189 parent_bus = adapter->pdev->bus->parent;190 if (!parent_bus)191 return -1;192 193 parent_dev = parent_bus->self;194 if (!parent_dev)195 return -1;196 197 if (!pci_is_pcie(parent_dev))198 return -1;199 200 pcie_capability_read_word(parent_dev, reg, value);201 if (*value == IXGBE_FAILED_READ_CFG_WORD &&202 ixgbe_check_cfg_remove(&adapter->hw, parent_dev))203 return -1;204 return 0;205}206 207static int ixgbe_get_parent_bus_info(struct ixgbe_adapter *adapter)208{209 struct ixgbe_hw *hw = &adapter->hw;210 u16 link_status = 0;211 int err;212 213 hw->bus.type = ixgbe_bus_type_pci_express;214 215 /* Get the negotiated link width and speed from PCI config space of the216 * parent, as this device is behind a switch217 */218 err = ixgbe_read_pci_cfg_word_parent(adapter, 18, &link_status);219 220 /* assume caller will handle error case */221 if (err)222 return err;223 224 hw->bus.width = ixgbe_convert_bus_width(link_status);225 hw->bus.speed = ixgbe_convert_bus_speed(link_status);226 227 return 0;228}229 230/**231 * ixgbe_pcie_from_parent - Determine whether PCIe info should come from parent232 * @hw: hw specific details233 *234 * This function is used by probe to determine whether a device's PCI-Express235 * bandwidth details should be gathered from the parent bus instead of from the236 * device. Used to ensure that various locations all have the correct device ID237 * checks.238 */239static inline bool ixgbe_pcie_from_parent(struct ixgbe_hw *hw)240{241 switch (hw->device_id) {242 case IXGBE_DEV_ID_82599_SFP_SF_QP:243 case IXGBE_DEV_ID_82599_QSFP_SF_QP:244 return true;245 default:246 return false;247 }248}249 250static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter,251 int expected_gts)252{253 struct ixgbe_hw *hw = &adapter->hw;254 struct pci_dev *pdev;255 256 /* Some devices are not connected over PCIe and thus do not negotiate257 * speed. These devices do not have valid bus info, and thus any report258 * we generate may not be correct.259 */260 if (hw->bus.type == ixgbe_bus_type_internal)261 return;262 263 /* determine whether to use the parent device */264 if (ixgbe_pcie_from_parent(&adapter->hw))265 pdev = adapter->pdev->bus->parent->self;266 else267 pdev = adapter->pdev;268 269 pcie_print_link_status(pdev);270}271 272static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)273{274 if (!test_bit(__IXGBE_DOWN, &adapter->state) &&275 !test_bit(__IXGBE_REMOVING, &adapter->state) &&276 !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state))277 queue_work(ixgbe_wq, &adapter->service_task);278}279 280static void ixgbe_remove_adapter(struct ixgbe_hw *hw)281{282 struct ixgbe_adapter *adapter = hw->back;283 284 if (!hw->hw_addr)285 return;286 hw->hw_addr = NULL;287 e_dev_err("Adapter removed\n");288 if (test_bit(__IXGBE_SERVICE_INITED, &adapter->state))289 ixgbe_service_event_schedule(adapter);290}291 292static u32 ixgbe_check_remove(struct ixgbe_hw *hw, u32 reg)293{294 u8 __iomem *reg_addr;295 u32 value;296 int i;297 298 reg_addr = READ_ONCE(hw->hw_addr);299 if (ixgbe_removed(reg_addr))300 return IXGBE_FAILED_READ_REG;301 302 /* Register read of 0xFFFFFFF can indicate the adapter has been removed,303 * so perform several status register reads to determine if the adapter304 * has been removed.305 */306 for (i = 0; i < IXGBE_FAILED_READ_RETRIES; i++) {307 value = readl(reg_addr + IXGBE_STATUS);308 if (value != IXGBE_FAILED_READ_REG)309 break;310 mdelay(3);311 }312 313 if (value == IXGBE_FAILED_READ_REG)314 ixgbe_remove_adapter(hw);315 else316 value = readl(reg_addr + reg);317 return value;318}319 320/**321 * ixgbe_read_reg - Read from device register322 * @hw: hw specific details323 * @reg: offset of register to read324 *325 * Returns : value read or IXGBE_FAILED_READ_REG if removed326 *327 * This function is used to read device registers. It checks for device328 * removal by confirming any read that returns all ones by checking the329 * status register value for all ones. This function avoids reading from330 * the hardware if a removal was previously detected in which case it331 * returns IXGBE_FAILED_READ_REG (all ones).332 */333u32 ixgbe_read_reg(struct ixgbe_hw *hw, u32 reg)334{335 u8 __iomem *reg_addr = READ_ONCE(hw->hw_addr);336 u32 value;337 338 if (ixgbe_removed(reg_addr))339 return IXGBE_FAILED_READ_REG;340 if (unlikely(hw->phy.nw_mng_if_sel &341 IXGBE_NW_MNG_IF_SEL_SGMII_ENABLE)) {342 struct ixgbe_adapter *adapter;343 int i;344 345 for (i = 0; i < 200; ++i) {346 value = readl(reg_addr + IXGBE_MAC_SGMII_BUSY);347 if (likely(!value))348 goto writes_completed;349 if (value == IXGBE_FAILED_READ_REG) {350 ixgbe_remove_adapter(hw);351 return IXGBE_FAILED_READ_REG;352 }353 udelay(5);354 }355 356 adapter = hw->back;357 e_warn(hw, "register writes incomplete %08x\n", value);358 }359 360writes_completed:361 value = readl(reg_addr + reg);362 if (unlikely(value == IXGBE_FAILED_READ_REG))363 value = ixgbe_check_remove(hw, reg);364 return value;365}366 367static bool ixgbe_check_cfg_remove(struct ixgbe_hw *hw, struct pci_dev *pdev)368{369 u16 value;370 371 pci_read_config_word(pdev, PCI_VENDOR_ID, &value);372 if (value == IXGBE_FAILED_READ_CFG_WORD) {373 ixgbe_remove_adapter(hw);374 return true;375 }376 return false;377}378 379u16 ixgbe_read_pci_cfg_word(struct ixgbe_hw *hw, u32 reg)380{381 struct ixgbe_adapter *adapter = hw->back;382 u16 value;383 384 if (ixgbe_removed(hw->hw_addr))385 return IXGBE_FAILED_READ_CFG_WORD;386 pci_read_config_word(adapter->pdev, reg, &value);387 if (value == IXGBE_FAILED_READ_CFG_WORD &&388 ixgbe_check_cfg_remove(hw, adapter->pdev))389 return IXGBE_FAILED_READ_CFG_WORD;390 return value;391}392 393#ifdef CONFIG_PCI_IOV394static u32 ixgbe_read_pci_cfg_dword(struct ixgbe_hw *hw, u32 reg)395{396 struct ixgbe_adapter *adapter = hw->back;397 u32 value;398 399 if (ixgbe_removed(hw->hw_addr))400 return IXGBE_FAILED_READ_CFG_DWORD;401 pci_read_config_dword(adapter->pdev, reg, &value);402 if (value == IXGBE_FAILED_READ_CFG_DWORD &&403 ixgbe_check_cfg_remove(hw, adapter->pdev))404 return IXGBE_FAILED_READ_CFG_DWORD;405 return value;406}407#endif /* CONFIG_PCI_IOV */408 409void ixgbe_write_pci_cfg_word(struct ixgbe_hw *hw, u32 reg, u16 value)410{411 struct ixgbe_adapter *adapter = hw->back;412 413 if (ixgbe_removed(hw->hw_addr))414 return;415 pci_write_config_word(adapter->pdev, reg, value);416}417 418static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter)419{420 BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state));421 422 /* flush memory to make sure state is correct before next watchdog */423 smp_mb__before_atomic();424 clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);425}426 427struct ixgbe_reg_info {428 u32 ofs;429 char *name;430};431 432static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {433 434 /* General Registers */435 {IXGBE_CTRL, "CTRL"},436 {IXGBE_STATUS, "STATUS"},437 {IXGBE_CTRL_EXT, "CTRL_EXT"},438 439 /* Interrupt Registers */440 {IXGBE_EICR, "EICR"},441 442 /* RX Registers */443 {IXGBE_SRRCTL(0), "SRRCTL"},444 {IXGBE_DCA_RXCTRL(0), "DRXCTL"},445 {IXGBE_RDLEN(0), "RDLEN"},446 {IXGBE_RDH(0), "RDH"},447 {IXGBE_RDT(0), "RDT"},448 {IXGBE_RXDCTL(0), "RXDCTL"},449 {IXGBE_RDBAL(0), "RDBAL"},450 {IXGBE_RDBAH(0), "RDBAH"},451 452 /* TX Registers */453 {IXGBE_TDBAL(0), "TDBAL"},454 {IXGBE_TDBAH(0), "TDBAH"},455 {IXGBE_TDLEN(0), "TDLEN"},456 {IXGBE_TDH(0), "TDH"},457 {IXGBE_TDT(0), "TDT"},458 {IXGBE_TXDCTL(0), "TXDCTL"},459 460 /* List Terminator */461 { .name = NULL }462};463 464 465/*466 * ixgbe_regdump - register printout routine467 */468static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)469{470 int i;471 char rname[16];472 u32 regs[64];473 474 switch (reginfo->ofs) {475 case IXGBE_SRRCTL(0):476 for (i = 0; i < 64; i++)477 regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));478 break;479 case IXGBE_DCA_RXCTRL(0):480 for (i = 0; i < 64; i++)481 regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));482 break;483 case IXGBE_RDLEN(0):484 for (i = 0; i < 64; i++)485 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));486 break;487 case IXGBE_RDH(0):488 for (i = 0; i < 64; i++)489 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));490 break;491 case IXGBE_RDT(0):492 for (i = 0; i < 64; i++)493 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));494 break;495 case IXGBE_RXDCTL(0):496 for (i = 0; i < 64; i++)497 regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));498 break;499 case IXGBE_RDBAL(0):500 for (i = 0; i < 64; i++)501 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));502 break;503 case IXGBE_RDBAH(0):504 for (i = 0; i < 64; i++)505 regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));506 break;507 case IXGBE_TDBAL(0):508 for (i = 0; i < 64; i++)509 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));510 break;511 case IXGBE_TDBAH(0):512 for (i = 0; i < 64; i++)513 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));514 break;515 case IXGBE_TDLEN(0):516 for (i = 0; i < 64; i++)517 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));518 break;519 case IXGBE_TDH(0):520 for (i = 0; i < 64; i++)521 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));522 break;523 case IXGBE_TDT(0):524 for (i = 0; i < 64; i++)525 regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));526 break;527 case IXGBE_TXDCTL(0):528 for (i = 0; i < 64; i++)529 regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));530 break;531 default:532 pr_info("%-15s %08x\n",533 reginfo->name, IXGBE_READ_REG(hw, reginfo->ofs));534 return;535 }536 537 i = 0;538 while (i < 64) {539 int j;540 char buf[9 * 8 + 1];541 char *p = buf;542 543 snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i, i + 7);544 for (j = 0; j < 8; j++)545 p += sprintf(p, " %08x", regs[i++]);546 pr_err("%-15s%s\n", rname, buf);547 }548 549}550 551static void ixgbe_print_buffer(struct ixgbe_ring *ring, int n)552{553 struct ixgbe_tx_buffer *tx_buffer;554 555 tx_buffer = &ring->tx_buffer_info[ring->next_to_clean];556 pr_info(" %5d %5X %5X %016llX %08X %p %016llX\n",557 n, ring->next_to_use, ring->next_to_clean,558 (u64)dma_unmap_addr(tx_buffer, dma),559 dma_unmap_len(tx_buffer, len),560 tx_buffer->next_to_watch,561 (u64)tx_buffer->time_stamp);562}563 564/*565 * ixgbe_dump - Print registers, tx-rings and rx-rings566 */567static void ixgbe_dump(struct ixgbe_adapter *adapter)568{569 struct net_device *netdev = adapter->netdev;570 struct ixgbe_hw *hw = &adapter->hw;571 struct ixgbe_reg_info *reginfo;572 int n = 0;573 struct ixgbe_ring *ring;574 struct ixgbe_tx_buffer *tx_buffer;575 union ixgbe_adv_tx_desc *tx_desc;576 struct my_u0 { u64 a; u64 b; } *u0;577 struct ixgbe_ring *rx_ring;578 union ixgbe_adv_rx_desc *rx_desc;579 struct ixgbe_rx_buffer *rx_buffer_info;580 int i = 0;581 582 if (!netif_msg_hw(adapter))583 return;584 585 /* Print netdevice Info */586 if (netdev) {587 dev_info(&adapter->pdev->dev, "Net device Info\n");588 pr_info("Device Name state "589 "trans_start\n");590 pr_info("%-15s %016lX %016lX\n",591 netdev->name,592 netdev->state,593 dev_trans_start(netdev));594 }595 596 /* Print Registers */597 dev_info(&adapter->pdev->dev, "Register Dump\n");598 pr_info(" Register Name Value\n");599 for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;600 reginfo->name; reginfo++) {601 ixgbe_regdump(hw, reginfo);602 }603 604 /* Print TX Ring Summary */605 if (!netdev || !netif_running(netdev))606 return;607 608 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");609 pr_info(" %s %s %s %s\n",610 "Queue [NTU] [NTC] [bi(ntc)->dma ]",611 "leng", "ntw", "timestamp");612 for (n = 0; n < adapter->num_tx_queues; n++) {613 ring = adapter->tx_ring[n];614 ixgbe_print_buffer(ring, n);615 }616 617 for (n = 0; n < adapter->num_xdp_queues; n++) {618 ring = adapter->xdp_ring[n];619 ixgbe_print_buffer(ring, n);620 }621 622 /* Print TX Rings */623 if (!netif_msg_tx_done(adapter))624 goto rx_ring_summary;625 626 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");627 628 /* Transmit Descriptor Formats629 *630 * 82598 Advanced Transmit Descriptor631 * +--------------------------------------------------------------+632 * 0 | Buffer Address [63:0] |633 * +--------------------------------------------------------------+634 * 8 | PAYLEN | POPTS | IDX | STA | DCMD |DTYP | RSV | DTALEN |635 * +--------------------------------------------------------------+636 * 63 46 45 40 39 36 35 32 31 24 23 20 19 0637 *638 * 82598 Advanced Transmit Descriptor (Write-Back Format)639 * +--------------------------------------------------------------+640 * 0 | RSV [63:0] |641 * +--------------------------------------------------------------+642 * 8 | RSV | STA | NXTSEQ |643 * +--------------------------------------------------------------+644 * 63 36 35 32 31 0645 *646 * 82599+ Advanced Transmit Descriptor647 * +--------------------------------------------------------------+648 * 0 | Buffer Address [63:0] |649 * +--------------------------------------------------------------+650 * 8 |PAYLEN |POPTS|CC|IDX |STA |DCMD |DTYP |MAC |RSV |DTALEN |651 * +--------------------------------------------------------------+652 * 63 46 45 40 39 38 36 35 32 31 24 23 20 19 18 17 16 15 0653 *654 * 82599+ Advanced Transmit Descriptor (Write-Back Format)655 * +--------------------------------------------------------------+656 * 0 | RSV [63:0] |657 * +--------------------------------------------------------------+658 * 8 | RSV | STA | RSV |659 * +--------------------------------------------------------------+660 * 63 36 35 32 31 0661 */662 663 for (n = 0; n < adapter->num_tx_queues; n++) {664 ring = adapter->tx_ring[n];665 pr_info("------------------------------------\n");666 pr_info("TX QUEUE INDEX = %d\n", ring->queue_index);667 pr_info("------------------------------------\n");668 pr_info("%s%s %s %s %s %s\n",669 "T [desc] [address 63:0 ] ",670 "[PlPOIdStDDt Ln] [bi->dma ] ",671 "leng", "ntw", "timestamp", "bi->skb");672 673 for (i = 0; ring->desc && (i < ring->count); i++) {674 tx_desc = IXGBE_TX_DESC(ring, i);675 tx_buffer = &ring->tx_buffer_info[i];676 u0 = (struct my_u0 *)tx_desc;677 if (dma_unmap_len(tx_buffer, len) > 0) {678 const char *ring_desc;679 680 if (i == ring->next_to_use &&681 i == ring->next_to_clean)682 ring_desc = " NTC/U";683 else if (i == ring->next_to_use)684 ring_desc = " NTU";685 else if (i == ring->next_to_clean)686 ring_desc = " NTC";687 else688 ring_desc = "";689 pr_info("T [0x%03X] %016llX %016llX %016llX %08X %p %016llX %p%s",690 i,691 le64_to_cpu((__force __le64)u0->a),692 le64_to_cpu((__force __le64)u0->b),693 (u64)dma_unmap_addr(tx_buffer, dma),694 dma_unmap_len(tx_buffer, len),695 tx_buffer->next_to_watch,696 (u64)tx_buffer->time_stamp,697 tx_buffer->skb,698 ring_desc);699 700 if (netif_msg_pktdata(adapter) &&701 tx_buffer->skb)702 print_hex_dump(KERN_INFO, "",703 DUMP_PREFIX_ADDRESS, 16, 1,704 tx_buffer->skb->data,705 dma_unmap_len(tx_buffer, len),706 true);707 }708 }709 }710 711 /* Print RX Rings Summary */712rx_ring_summary:713 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");714 pr_info("Queue [NTU] [NTC]\n");715 for (n = 0; n < adapter->num_rx_queues; n++) {716 rx_ring = adapter->rx_ring[n];717 pr_info("%5d %5X %5X\n",718 n, rx_ring->next_to_use, rx_ring->next_to_clean);719 }720 721 /* Print RX Rings */722 if (!netif_msg_rx_status(adapter))723 return;724 725 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");726 727 /* Receive Descriptor Formats728 *729 * 82598 Advanced Receive Descriptor (Read) Format730 * 63 1 0731 * +-----------------------------------------------------+732 * 0 | Packet Buffer Address [63:1] |A0/NSE|733 * +----------------------------------------------+------+734 * 8 | Header Buffer Address [63:1] | DD |735 * +-----------------------------------------------------+736 *737 *738 * 82598 Advanced Receive Descriptor (Write-Back) Format739 *740 * 63 48 47 32 31 30 21 20 16 15 4 3 0741 * +------------------------------------------------------+742 * 0 | RSS Hash / |SPH| HDR_LEN | RSV |Packet| RSS |743 * | Packet | IP | | | | Type | Type |744 * | Checksum | Ident | | | | | |745 * +------------------------------------------------------+746 * 8 | VLAN Tag | Length | Extended Error | Extended Status |747 * +------------------------------------------------------+748 * 63 48 47 32 31 20 19 0749 *750 * 82599+ Advanced Receive Descriptor (Read) Format751 * 63 1 0752 * +-----------------------------------------------------+753 * 0 | Packet Buffer Address [63:1] |A0/NSE|754 * +----------------------------------------------+------+755 * 8 | Header Buffer Address [63:1] | DD |756 * +-----------------------------------------------------+757 *758 *759 * 82599+ Advanced Receive Descriptor (Write-Back) Format760 *761 * 63 48 47 32 31 30 21 20 17 16 4 3 0762 * +------------------------------------------------------+763 * 0 |RSS / Frag Checksum|SPH| HDR_LEN |RSC- |Packet| RSS |764 * |/ RTT / PCoE_PARAM | | | CNT | Type | Type |765 * |/ Flow Dir Flt ID | | | | | |766 * +------------------------------------------------------+767 * 8 | VLAN Tag | Length |Extended Error| Xtnd Status/NEXTP |768 * +------------------------------------------------------+769 * 63 48 47 32 31 20 19 0770 */771 772 for (n = 0; n < adapter->num_rx_queues; n++) {773 rx_ring = adapter->rx_ring[n];774 pr_info("------------------------------------\n");775 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);776 pr_info("------------------------------------\n");777 pr_info("%s%s%s\n",778 "R [desc] [ PktBuf A0] ",779 "[ HeadBuf DD] [bi->dma ] [bi->skb ] ",780 "<-- Adv Rx Read format");781 pr_info("%s%s%s\n",782 "RWB[desc] [PcsmIpSHl PtRs] ",783 "[vl er S cks ln] ---------------- [bi->skb ] ",784 "<-- Adv Rx Write-Back format");785 786 for (i = 0; i < rx_ring->count; i++) {787 const char *ring_desc;788 789 if (i == rx_ring->next_to_use)790 ring_desc = " NTU";791 else if (i == rx_ring->next_to_clean)792 ring_desc = " NTC";793 else794 ring_desc = "";795 796 rx_buffer_info = &rx_ring->rx_buffer_info[i];797 rx_desc = IXGBE_RX_DESC(rx_ring, i);798 u0 = (struct my_u0 *)rx_desc;799 if (rx_desc->wb.upper.length) {800 /* Descriptor Done */801 pr_info("RWB[0x%03X] %016llX %016llX ---------------- %p%s\n",802 i,803 le64_to_cpu((__force __le64)u0->a),804 le64_to_cpu((__force __le64)u0->b),805 rx_buffer_info->skb,806 ring_desc);807 } else {808 pr_info("R [0x%03X] %016llX %016llX %016llX %p%s\n",809 i,810 le64_to_cpu((__force __le64)u0->a),811 le64_to_cpu((__force __le64)u0->b),812 (u64)rx_buffer_info->dma,813 rx_buffer_info->skb,814 ring_desc);815 816 if (netif_msg_pktdata(adapter) &&817 rx_buffer_info->dma) {818 print_hex_dump(KERN_INFO, "",819 DUMP_PREFIX_ADDRESS, 16, 1,820 page_address(rx_buffer_info->page) +821 rx_buffer_info->page_offset,822 ixgbe_rx_bufsz(rx_ring), true);823 }824 }825 }826 }827}828 829static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)830{831 u32 ctrl_ext;832 833 /* Let firmware take over control of h/w */834 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);835 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,836 ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);837}838 839static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)840{841 u32 ctrl_ext;842 843 /* Let firmware know the driver has taken over */844 ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);845 IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,846 ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);847}848 849/**850 * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors851 * @adapter: pointer to adapter struct852 * @direction: 0 for Rx, 1 for Tx, -1 for other causes853 * @queue: queue to map the corresponding interrupt to854 * @msix_vector: the vector to map to the corresponding queue855 *856 */857static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,858 u8 queue, u8 msix_vector)859{860 u32 ivar, index;861 struct ixgbe_hw *hw = &adapter->hw;862 switch (hw->mac.type) {863 case ixgbe_mac_82598EB:864 msix_vector |= IXGBE_IVAR_ALLOC_VAL;865 if (direction == -1)866 direction = 0;867 index = (((direction * 64) + queue) >> 2) & 0x1F;868 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));869 ivar &= ~(0xFF << (8 * (queue & 0x3)));870 ivar |= (msix_vector << (8 * (queue & 0x3)));871 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);872 break;873 case ixgbe_mac_82599EB:874 case ixgbe_mac_X540:875 case ixgbe_mac_X550:876 case ixgbe_mac_X550EM_x:877 case ixgbe_mac_x550em_a:878 if (direction == -1) {879 /* other causes */880 msix_vector |= IXGBE_IVAR_ALLOC_VAL;881 index = ((queue & 1) * 8);882 ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);883 ivar &= ~(0xFF << index);884 ivar |= (msix_vector << index);885 IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);886 break;887 } else {888 /* tx or rx causes */889 msix_vector |= IXGBE_IVAR_ALLOC_VAL;890 index = ((16 * (queue & 1)) + (8 * direction));891 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));892 ivar &= ~(0xFF << index);893 ivar |= (msix_vector << index);894 IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);895 break;896 }897 default:898 break;899 }900}901 902void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,903 u64 qmask)904{905 u32 mask;906 907 switch (adapter->hw.mac.type) {908 case ixgbe_mac_82598EB:909 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);910 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);911 break;912 case ixgbe_mac_82599EB:913 case ixgbe_mac_X540:914 case ixgbe_mac_X550:915 case ixgbe_mac_X550EM_x:916 case ixgbe_mac_x550em_a:917 mask = (qmask & 0xFFFFFFFF);918 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);919 mask = (qmask >> 32);920 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);921 break;922 default:923 break;924 }925}926 927static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter)928{929 struct ixgbe_hw *hw = &adapter->hw;930 struct ixgbe_hw_stats *hwstats = &adapter->stats;931 int i;932 u32 data;933 934 if ((hw->fc.current_mode != ixgbe_fc_full) &&935 (hw->fc.current_mode != ixgbe_fc_rx_pause))936 return;937 938 switch (hw->mac.type) {939 case ixgbe_mac_82598EB:940 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);941 break;942 default:943 data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);944 }945 hwstats->lxoffrxc += data;946 947 /* refill credits (no tx hang) if we received xoff */948 if (!data)949 return;950 951 for (i = 0; i < adapter->num_tx_queues; i++)952 clear_bit(__IXGBE_HANG_CHECK_ARMED,953 &adapter->tx_ring[i]->state);954 955 for (i = 0; i < adapter->num_xdp_queues; i++)956 clear_bit(__IXGBE_HANG_CHECK_ARMED,957 &adapter->xdp_ring[i]->state);958}959 960static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)961{962 struct ixgbe_hw *hw = &adapter->hw;963 struct ixgbe_hw_stats *hwstats = &adapter->stats;964 u32 xoff[8] = {0};965 u8 tc;966 int i;967 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;968 969 if (adapter->ixgbe_ieee_pfc)970 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);971 972 if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED) || !pfc_en) {973 ixgbe_update_xoff_rx_lfc(adapter);974 return;975 }976 977 /* update stats for each tc, only valid with PFC enabled */978 for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {979 u32 pxoffrxc;980 981 switch (hw->mac.type) {982 case ixgbe_mac_82598EB:983 pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));984 break;985 default:986 pxoffrxc = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));987 }988 hwstats->pxoffrxc[i] += pxoffrxc;989 /* Get the TC for given UP */990 tc = netdev_get_prio_tc_map(adapter->netdev, i);991 xoff[tc] += pxoffrxc;992 }993 994 /* disarm tx queues that have received xoff frames */995 for (i = 0; i < adapter->num_tx_queues; i++) {996 struct ixgbe_ring *tx_ring = adapter->tx_ring[i];997 998 tc = tx_ring->dcb_tc;999 if (xoff[tc])1000 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);1001 }1002 1003 for (i = 0; i < adapter->num_xdp_queues; i++) {1004 struct ixgbe_ring *xdp_ring = adapter->xdp_ring[i];1005 1006 tc = xdp_ring->dcb_tc;1007 if (xoff[tc])1008 clear_bit(__IXGBE_HANG_CHECK_ARMED, &xdp_ring->state);1009 }1010}1011 1012static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)1013{1014 return ring->stats.packets;1015}1016 1017static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)1018{1019 unsigned int head, tail;1020 1021 head = ring->next_to_clean;1022 tail = ring->next_to_use;1023 1024 return ((head <= tail) ? tail : tail + ring->count) - head;1025}1026 1027static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)1028{1029 u32 tx_done = ixgbe_get_tx_completed(tx_ring);1030 u32 tx_done_old = tx_ring->tx_stats.tx_done_old;1031 u32 tx_pending = ixgbe_get_tx_pending(tx_ring);1032 1033 clear_check_for_tx_hang(tx_ring);1034 1035 /*1036 * Check for a hung queue, but be thorough. This verifies1037 * that a transmit has been completed since the previous1038 * check AND there is at least one packet pending. The1039 * ARMED bit is set to indicate a potential hang. The1040 * bit is cleared if a pause frame is received to remove1041 * false hang detection due to PFC or 802.3x frames. By1042 * requiring this to fail twice we avoid races with1043 * pfc clearing the ARMED bit and conditions where we1044 * run the check_tx_hang logic with a transmit completion1045 * pending but without time to complete it yet.1046 */1047 if (tx_done_old == tx_done && tx_pending)1048 /* make sure it is true for two checks in a row */1049 return test_and_set_bit(__IXGBE_HANG_CHECK_ARMED,1050 &tx_ring->state);1051 /* update completed stats and continue */1052 tx_ring->tx_stats.tx_done_old = tx_done;1053 /* reset the countdown */1054 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);1055 1056 return false;1057}1058 1059/**1060 * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout1061 * @adapter: driver private struct1062 **/1063static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)1064{1065 1066 /* Do the reset outside of interrupt context */1067 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {1068 set_bit(__IXGBE_RESET_REQUESTED, &adapter->state);1069 e_warn(drv, "initiating reset due to tx timeout\n");1070 ixgbe_service_event_schedule(adapter);1071 }1072}1073 1074/**1075 * ixgbe_tx_maxrate - callback to set the maximum per-queue bitrate1076 * @netdev: network interface device structure1077 * @queue_index: Tx queue to set1078 * @maxrate: desired maximum transmit bitrate1079 **/1080static int ixgbe_tx_maxrate(struct net_device *netdev,1081 int queue_index, u32 maxrate)1082{1083 struct ixgbe_adapter *adapter = netdev_priv(netdev);1084 struct ixgbe_hw *hw = &adapter->hw;1085 u32 bcnrc_val = ixgbe_link_mbps(adapter);1086 1087 if (!maxrate)1088 return 0;1089 1090 /* Calculate the rate factor values to set */1091 bcnrc_val <<= IXGBE_RTTBCNRC_RF_INT_SHIFT;1092 bcnrc_val /= maxrate;1093 1094 /* clear everything but the rate factor */1095 bcnrc_val &= IXGBE_RTTBCNRC_RF_INT_MASK |1096 IXGBE_RTTBCNRC_RF_DEC_MASK;1097 1098 /* enable the rate scheduler */1099 bcnrc_val |= IXGBE_RTTBCNRC_RS_ENA;1100 1101 IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_index);1102 IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val);1103 1104 return 0;1105}1106 1107/**1108 * ixgbe_update_tx_ring_stats - Update Tx ring specific counters1109 * @tx_ring: ring to update1110 * @q_vector: queue vector ring belongs to1111 * @pkts: number of processed packets1112 * @bytes: number of processed bytes1113 */1114void ixgbe_update_tx_ring_stats(struct ixgbe_ring *tx_ring,1115 struct ixgbe_q_vector *q_vector, u64 pkts,1116 u64 bytes)1117{1118 u64_stats_update_begin(&tx_ring->syncp);1119 tx_ring->stats.bytes += bytes;1120 tx_ring->stats.packets += pkts;1121 u64_stats_update_end(&tx_ring->syncp);1122 q_vector->tx.total_bytes += bytes;1123 q_vector->tx.total_packets += pkts;1124}1125 1126/**1127 * ixgbe_update_rx_ring_stats - Update Rx ring specific counters1128 * @rx_ring: ring to update1129 * @q_vector: queue vector ring belongs to1130 * @pkts: number of processed packets1131 * @bytes: number of processed bytes1132 */1133void ixgbe_update_rx_ring_stats(struct ixgbe_ring *rx_ring,1134 struct ixgbe_q_vector *q_vector, u64 pkts,1135 u64 bytes)1136{1137 u64_stats_update_begin(&rx_ring->syncp);1138 rx_ring->stats.bytes += bytes;1139 rx_ring->stats.packets += pkts;1140 u64_stats_update_end(&rx_ring->syncp);1141 q_vector->rx.total_bytes += bytes;1142 q_vector->rx.total_packets += pkts;1143}1144 1145/**1146 * ixgbe_clean_tx_irq - Reclaim resources after transmit completes1147 * @q_vector: structure containing interrupt and ring information1148 * @tx_ring: tx ring to clean1149 * @napi_budget: Used to determine if we are in netpoll1150 **/1151static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,1152 struct ixgbe_ring *tx_ring, int napi_budget)1153{1154 struct ixgbe_adapter *adapter = q_vector->adapter;1155 struct ixgbe_tx_buffer *tx_buffer;1156 union ixgbe_adv_tx_desc *tx_desc;1157 unsigned int total_bytes = 0, total_packets = 0, total_ipsec = 0;1158 unsigned int budget = q_vector->tx.work_limit;1159 unsigned int i = tx_ring->next_to_clean;1160 struct netdev_queue *txq;1161 1162 if (test_bit(__IXGBE_DOWN, &adapter->state))1163 return true;1164 1165 tx_buffer = &tx_ring->tx_buffer_info[i];1166 tx_desc = IXGBE_TX_DESC(tx_ring, i);1167 i -= tx_ring->count;1168 1169 do {1170 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;1171 1172 /* if next_to_watch is not set then there is no work pending */1173 if (!eop_desc)1174 break;1175 1176 /* prevent any other reads prior to eop_desc */1177 smp_rmb();1178 1179 /* if DD is not set pending work has not been completed */1180 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))1181 break;1182 1183 /* clear next_to_watch to prevent false hangs */1184 tx_buffer->next_to_watch = NULL;1185 1186 /* update the statistics for this packet */1187 total_bytes += tx_buffer->bytecount;1188 total_packets += tx_buffer->gso_segs;1189 if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_IPSEC)1190 total_ipsec++;1191 1192 /* free the skb */1193 if (ring_is_xdp(tx_ring))1194 xdp_return_frame(tx_buffer->xdpf);1195 else1196 napi_consume_skb(tx_buffer->skb, napi_budget);1197 1198 /* unmap skb header data */1199 dma_unmap_single(tx_ring->dev,1200 dma_unmap_addr(tx_buffer, dma),1201 dma_unmap_len(tx_buffer, len),1202 DMA_TO_DEVICE);1203 1204 /* clear tx_buffer data */1205 dma_unmap_len_set(tx_buffer, len, 0);1206 1207 /* unmap remaining buffers */1208 while (tx_desc != eop_desc) {1209 tx_buffer++;1210 tx_desc++;1211 i++;1212 if (unlikely(!i)) {1213 i -= tx_ring->count;1214 tx_buffer = tx_ring->tx_buffer_info;1215 tx_desc = IXGBE_TX_DESC(tx_ring, 0);1216 }1217 1218 /* unmap any remaining paged data */1219 if (dma_unmap_len(tx_buffer, len)) {1220 dma_unmap_page(tx_ring->dev,1221 dma_unmap_addr(tx_buffer, dma),1222 dma_unmap_len(tx_buffer, len),1223 DMA_TO_DEVICE);1224 dma_unmap_len_set(tx_buffer, len, 0);1225 }1226 }1227 1228 /* move us one more past the eop_desc for start of next pkt */1229 tx_buffer++;1230 tx_desc++;1231 i++;1232 if (unlikely(!i)) {1233 i -= tx_ring->count;1234 tx_buffer = tx_ring->tx_buffer_info;1235 tx_desc = IXGBE_TX_DESC(tx_ring, 0);1236 }1237 1238 /* issue prefetch for next Tx descriptor */1239 prefetch(tx_desc);1240 1241 /* update budget accounting */1242 budget--;1243 } while (likely(budget));1244 1245 i += tx_ring->count;1246 tx_ring->next_to_clean = i;1247 ixgbe_update_tx_ring_stats(tx_ring, q_vector, total_packets,1248 total_bytes);1249 adapter->tx_ipsec += total_ipsec;1250 1251 if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {1252 /* schedule immediate reset if we believe we hung */1253 struct ixgbe_hw *hw = &adapter->hw;1254 e_err(drv, "Detected Tx Unit Hang %s\n"1255 " Tx Queue <%d>\n"1256 " TDH, TDT <%x>, <%x>\n"1257 " next_to_use <%x>\n"1258 " next_to_clean <%x>\n"1259 "tx_buffer_info[next_to_clean]\n"1260 " time_stamp <%lx>\n"1261 " jiffies <%lx>\n",1262 ring_is_xdp(tx_ring) ? "(XDP)" : "",1263 tx_ring->queue_index,1264 IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)),1265 IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)),1266 tx_ring->next_to_use, i,1267 tx_ring->tx_buffer_info[i].time_stamp, jiffies);1268 1269 if (!ring_is_xdp(tx_ring))1270 netif_stop_subqueue(tx_ring->netdev,1271 tx_ring->queue_index);1272 1273 e_info(probe,1274 "tx hang %d detected on queue %d, resetting adapter\n",1275 adapter->tx_timeout_count + 1, tx_ring->queue_index);1276 1277 /* schedule immediate reset if we believe we hung */1278 ixgbe_tx_timeout_reset(adapter);1279 1280 /* the adapter is about to reset, no point in enabling stuff */1281 return true;1282 }1283 1284 if (ring_is_xdp(tx_ring))1285 return !!budget;1286 1287#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)1288 txq = netdev_get_tx_queue(tx_ring->netdev, tx_ring->queue_index);1289 if (!__netif_txq_completed_wake(txq, total_packets, total_bytes,1290 ixgbe_desc_unused(tx_ring),1291 TX_WAKE_THRESHOLD,1292 !netif_carrier_ok(tx_ring->netdev) ||1293 test_bit(__IXGBE_DOWN, &adapter->state)))1294 ++tx_ring->tx_stats.restart_queue;1295 1296 return !!budget;1297}1298 1299#ifdef CONFIG_IXGBE_DCA1300static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter,1301 struct ixgbe_ring *tx_ring,1302 int cpu)1303{1304 struct ixgbe_hw *hw = &adapter->hw;1305 u32 txctrl = 0;1306 u16 reg_offset;1307 1308 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)1309 txctrl = dca3_get_tag(tx_ring->dev, cpu);1310 1311 switch (hw->mac.type) {1312 case ixgbe_mac_82598EB:1313 reg_offset = IXGBE_DCA_TXCTRL(tx_ring->reg_idx);1314 break;1315 case ixgbe_mac_82599EB:1316 case ixgbe_mac_X540:1317 reg_offset = IXGBE_DCA_TXCTRL_82599(tx_ring->reg_idx);1318 txctrl <<= IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599;1319 break;1320 default:1321 /* for unknown hardware do not write register */1322 return;1323 }1324 1325 /*1326 * We can enable relaxed ordering for reads, but not writes when1327 * DCA is enabled. This is due to a known issue in some chipsets1328 * which will cause the DCA tag to be cleared.1329 */1330 txctrl |= IXGBE_DCA_TXCTRL_DESC_RRO_EN |1331 IXGBE_DCA_TXCTRL_DATA_RRO_EN |1332 IXGBE_DCA_TXCTRL_DESC_DCA_EN;1333 1334 IXGBE_WRITE_REG(hw, reg_offset, txctrl);1335}1336 1337static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter,1338 struct ixgbe_ring *rx_ring,1339 int cpu)1340{1341 struct ixgbe_hw *hw = &adapter->hw;1342 u32 rxctrl = 0;1343 u8 reg_idx = rx_ring->reg_idx;1344 1345 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)1346 rxctrl = dca3_get_tag(rx_ring->dev, cpu);1347 1348 switch (hw->mac.type) {1349 case ixgbe_mac_82599EB:1350 case ixgbe_mac_X540:1351 rxctrl <<= IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599;1352 break;1353 default:1354 break;1355 }1356 1357 /*1358 * We can enable relaxed ordering for reads, but not writes when1359 * DCA is enabled. This is due to a known issue in some chipsets1360 * which will cause the DCA tag to be cleared.1361 */1362 rxctrl |= IXGBE_DCA_RXCTRL_DESC_RRO_EN |1363 IXGBE_DCA_RXCTRL_DATA_DCA_EN |1364 IXGBE_DCA_RXCTRL_DESC_DCA_EN;1365 1366 IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl);1367}1368 1369static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector)1370{1371 struct ixgbe_adapter *adapter = q_vector->adapter;1372 struct ixgbe_ring *ring;1373 int cpu = get_cpu();1374 1375 if (q_vector->cpu == cpu)1376 goto out_no_update;1377 1378 ixgbe_for_each_ring(ring, q_vector->tx)1379 ixgbe_update_tx_dca(adapter, ring, cpu);1380 1381 ixgbe_for_each_ring(ring, q_vector->rx)1382 ixgbe_update_rx_dca(adapter, ring, cpu);1383 1384 q_vector->cpu = cpu;1385out_no_update:1386 put_cpu();1387}1388 1389static void ixgbe_setup_dca(struct ixgbe_adapter *adapter)1390{1391 int i;1392 1393 /* always use CB2 mode, difference is masked in the CB driver */1394 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)1395 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,1396 IXGBE_DCA_CTRL_DCA_MODE_CB2);1397 else1398 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,1399 IXGBE_DCA_CTRL_DCA_DISABLE);1400 1401 for (i = 0; i < adapter->num_q_vectors; i++) {1402 adapter->q_vector[i]->cpu = -1;1403 ixgbe_update_dca(adapter->q_vector[i]);1404 }1405}1406 1407static int __ixgbe_notify_dca(struct device *dev, void *data)1408{1409 struct ixgbe_adapter *adapter = dev_get_drvdata(dev);1410 unsigned long event = *(unsigned long *)data;1411 1412 if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE))1413 return 0;1414 1415 switch (event) {1416 case DCA_PROVIDER_ADD:1417 /* if we're already enabled, don't do it again */1418 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)1419 break;1420 if (dca_add_requester(dev) == 0) {1421 adapter->flags |= IXGBE_FLAG_DCA_ENABLED;1422 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,1423 IXGBE_DCA_CTRL_DCA_MODE_CB2);1424 break;1425 }1426 fallthrough; /* DCA is disabled. */1427 case DCA_PROVIDER_REMOVE:1428 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {1429 dca_remove_requester(dev);1430 adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;1431 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,1432 IXGBE_DCA_CTRL_DCA_DISABLE);1433 }1434 break;1435 }1436 1437 return 0;1438}1439 1440#endif /* CONFIG_IXGBE_DCA */1441 1442#define IXGBE_RSS_L4_TYPES_MASK \1443 ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \1444 (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \1445 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \1446 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP))1447 1448static inline void ixgbe_rx_hash(struct ixgbe_ring *ring,1449 union ixgbe_adv_rx_desc *rx_desc,1450 struct sk_buff *skb)1451{1452 u16 rss_type;1453 1454 if (!(ring->netdev->features & NETIF_F_RXHASH))1455 return;1456 1457 rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &1458 IXGBE_RXDADV_RSSTYPE_MASK;1459 1460 if (!rss_type)1461 return;1462 1463 skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),1464 (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ?1465 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);1466}1467 1468#ifdef IXGBE_FCOE1469/**1470 * ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type1471 * @ring: structure containing ring specific data1472 * @rx_desc: advanced rx descriptor1473 *1474 * Returns : true if it is FCoE pkt1475 */1476static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring,1477 union ixgbe_adv_rx_desc *rx_desc)1478{1479 __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;1480 1481 return test_bit(__IXGBE_RX_FCOE, &ring->state) &&1482 ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) ==1483 (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE <<1484 IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT)));1485}1486 1487#endif /* IXGBE_FCOE */1488/**1489 * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum1490 * @ring: structure containing ring specific data1491 * @rx_desc: current Rx descriptor being processed1492 * @skb: skb currently being received and modified1493 **/1494static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring,1495 union ixgbe_adv_rx_desc *rx_desc,1496 struct sk_buff *skb)1497{1498 __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;1499 bool encap_pkt = false;1500 1501 skb_checksum_none_assert(skb);1502 1503 /* Rx csum disabled */1504 if (!(ring->netdev->features & NETIF_F_RXCSUM))1505 return;1506 1507 /* check for VXLAN and Geneve packets */1508 if (pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_VXLAN)) {1509 encap_pkt = true;1510 skb->encapsulation = 1;1511 }1512 1513 /* if IP and error */1514 if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) &&1515 ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) {1516 ring->rx_stats.csum_err++;1517 return;1518 }1519 1520 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS))1521 return;1522 1523 if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) {1524 /*1525 * 82599 errata, UDP frames with a 0 checksum can be marked as1526 * checksum errors.1527 */1528 if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) &&1529 test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state))1530 return;1531 1532 ring->rx_stats.csum_err++;1533 return;1534 }1535 1536 /* It must be a TCP or UDP packet with a valid checksum */1537 skb->ip_summed = CHECKSUM_UNNECESSARY;1538 if (encap_pkt) {1539 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_OUTERIPCS))1540 return;1541 1542 if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_OUTERIPER)) {1543 skb->ip_summed = CHECKSUM_NONE;1544 return;1545 }1546 /* If we checked the outer header let the stack know */1547 skb->csum_level = 1;1548 }1549}1550 1551static unsigned int ixgbe_rx_offset(struct ixgbe_ring *rx_ring)1552{1553 return ring_uses_build_skb(rx_ring) ? IXGBE_SKB_PAD : 0;1554}1555 1556static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring,1557 struct ixgbe_rx_buffer *bi)1558{1559 struct page *page = bi->page;1560 dma_addr_t dma;1561 1562 /* since we are recycling buffers we should seldom need to alloc */1563 if (likely(page))1564 return true;1565 1566 /* alloc new page for storage */1567 page = dev_alloc_pages(ixgbe_rx_pg_order(rx_ring));1568 if (unlikely(!page)) {1569 rx_ring->rx_stats.alloc_rx_page_failed++;1570 return false;1571 }1572 1573 /* map page for use */1574 dma = dma_map_page_attrs(rx_ring->dev, page, 0,1575 ixgbe_rx_pg_size(rx_ring),1576 DMA_FROM_DEVICE,1577 IXGBE_RX_DMA_ATTR);1578 1579 /*1580 * if mapping failed free memory back to system since1581 * there isn't much point in holding memory we can't use1582 */1583 if (dma_mapping_error(rx_ring->dev, dma)) {1584 __free_pages(page, ixgbe_rx_pg_order(rx_ring));1585 1586 rx_ring->rx_stats.alloc_rx_page_failed++;1587 return false;1588 }1589 1590 bi->dma = dma;1591 bi->page = page;1592 bi->page_offset = rx_ring->rx_offset;1593 page_ref_add(page, USHRT_MAX - 1);1594 bi->pagecnt_bias = USHRT_MAX;1595 rx_ring->rx_stats.alloc_rx_page++;1596 1597 return true;1598}1599 1600/**1601 * ixgbe_alloc_rx_buffers - Replace used receive buffers1602 * @rx_ring: ring to place buffers on1603 * @cleaned_count: number of buffers to replace1604 **/1605void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count)1606{1607 union ixgbe_adv_rx_desc *rx_desc;1608 struct ixgbe_rx_buffer *bi;1609 u16 i = rx_ring->next_to_use;1610 u16 bufsz;1611 1612 /* nothing to do */1613 if (!cleaned_count)1614 return;1615 1616 rx_desc = IXGBE_RX_DESC(rx_ring, i);1617 bi = &rx_ring->rx_buffer_info[i];1618 i -= rx_ring->count;1619 1620 bufsz = ixgbe_rx_bufsz(rx_ring);1621 1622 do {1623 if (!ixgbe_alloc_mapped_page(rx_ring, bi))1624 break;1625 1626 /* sync the buffer for use by the device */1627 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,1628 bi->page_offset, bufsz,1629 DMA_FROM_DEVICE);1630 1631 /*1632 * Refresh the desc even if buffer_addrs didn't change1633 * because each write-back erases this info.1634 */1635 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);1636 1637 rx_desc++;1638 bi++;1639 i++;1640 if (unlikely(!i)) {1641 rx_desc = IXGBE_RX_DESC(rx_ring, 0);1642 bi = rx_ring->rx_buffer_info;1643 i -= rx_ring->count;1644 }1645 1646 /* clear the length for the next_to_use descriptor */1647 rx_desc->wb.upper.length = 0;1648 1649 cleaned_count--;1650 } while (cleaned_count);1651 1652 i += rx_ring->count;1653 1654 if (rx_ring->next_to_use != i) {1655 rx_ring->next_to_use = i;1656 1657 /* update next to alloc since we have filled the ring */1658 rx_ring->next_to_alloc = i;1659 1660 /* Force memory writes to complete before letting h/w1661 * know there are new descriptors to fetch. (Only1662 * applicable for weak-ordered memory model archs,1663 * such as IA-64).1664 */1665 wmb();1666 writel(i, rx_ring->tail);1667 }1668}1669 1670static void ixgbe_set_rsc_gso_size(struct ixgbe_ring *ring,1671 struct sk_buff *skb)1672{1673 u16 hdr_len = skb_headlen(skb);1674 1675 /* set gso_size to avoid messing up TCP MSS */1676 skb_shinfo(skb)->gso_size = DIV_ROUND_UP((skb->len - hdr_len),1677 IXGBE_CB(skb)->append_cnt);1678 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;1679}1680 1681static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring,1682 struct sk_buff *skb)1683{1684 /* if append_cnt is 0 then frame is not RSC */1685 if (!IXGBE_CB(skb)->append_cnt)1686 return;1687 1688 rx_ring->rx_stats.rsc_count += IXGBE_CB(skb)->append_cnt;1689 rx_ring->rx_stats.rsc_flush++;1690 1691 ixgbe_set_rsc_gso_size(rx_ring, skb);1692 1693 /* gso_size is computed using append_cnt so always clear it last */1694 IXGBE_CB(skb)->append_cnt = 0;1695}1696 1697/**1698 * ixgbe_process_skb_fields - Populate skb header fields from Rx descriptor1699 * @rx_ring: rx descriptor ring packet is being transacted on1700 * @rx_desc: pointer to the EOP Rx descriptor1701 * @skb: pointer to current skb being populated1702 *1703 * This function checks the ring, descriptor, and packet information in1704 * order to populate the hash, checksum, VLAN, timestamp, protocol, and1705 * other fields within the skb.1706 **/1707void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring,1708 union ixgbe_adv_rx_desc *rx_desc,1709 struct sk_buff *skb)1710{1711 struct net_device *dev = rx_ring->netdev;1712 u32 flags = rx_ring->q_vector->adapter->flags;1713 1714 ixgbe_update_rsc_stats(rx_ring, skb);1715 1716 ixgbe_rx_hash(rx_ring, rx_desc, skb);1717 1718 ixgbe_rx_checksum(rx_ring, rx_desc, skb);1719 1720 if (unlikely(flags & IXGBE_FLAG_RX_HWTSTAMP_ENABLED))1721 ixgbe_ptp_rx_hwtstamp(rx_ring, rx_desc, skb);1722 1723 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&1724 ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) {1725 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);1726 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);1727 }1728 1729 if (ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_STAT_SECP))1730 ixgbe_ipsec_rx(rx_ring, rx_desc, skb);1731 1732 /* record Rx queue, or update MACVLAN statistics */1733 if (netif_is_ixgbe(dev))1734 skb_record_rx_queue(skb, rx_ring->queue_index);1735 else1736 macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, true,1737 false);1738 1739 skb->protocol = eth_type_trans(skb, dev);1740}1741 1742void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector,1743 struct sk_buff *skb)1744{1745 napi_gro_receive(&q_vector->napi, skb);1746}1747 1748/**1749 * ixgbe_is_non_eop - process handling of non-EOP buffers1750 * @rx_ring: Rx ring being processed1751 * @rx_desc: Rx descriptor for current buffer1752 * @skb: Current socket buffer containing buffer in progress1753 *1754 * This function updates next to clean. If the buffer is an EOP buffer1755 * this function exits returning false, otherwise it will place the1756 * sk_buff in the next buffer to be chained and return true indicating1757 * that this is in fact a non-EOP buffer.1758 **/1759static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring,1760 union ixgbe_adv_rx_desc *rx_desc,1761 struct sk_buff *skb)1762{1763 u32 ntc = rx_ring->next_to_clean + 1;1764 1765 /* fetch, update, and store next to clean */1766 ntc = (ntc < rx_ring->count) ? ntc : 0;1767 rx_ring->next_to_clean = ntc;1768 1769 prefetch(IXGBE_RX_DESC(rx_ring, ntc));1770 1771 /* update RSC append count if present */1772 if (ring_is_rsc_enabled(rx_ring)) {1773 __le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data &1774 cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK);1775 1776 if (unlikely(rsc_enabled)) {1777 u32 rsc_cnt = le32_to_cpu(rsc_enabled);1778 1779 rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT;1780 IXGBE_CB(skb)->append_cnt += rsc_cnt - 1;1781 1782 /* update ntc based on RSC value */1783 ntc = le32_to_cpu(rx_desc->wb.upper.status_error);1784 ntc &= IXGBE_RXDADV_NEXTP_MASK;1785 ntc >>= IXGBE_RXDADV_NEXTP_SHIFT;1786 }1787 }1788 1789 /* if we are the last buffer then there is nothing else to do */1790 if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))1791 return false;1792 1793 /* place skb in next buffer to be received */1794 rx_ring->rx_buffer_info[ntc].skb = skb;1795 rx_ring->rx_stats.non_eop_descs++;1796 1797 return true;1798}1799 1800/**1801 * ixgbe_pull_tail - ixgbe specific version of skb_pull_tail1802 * @rx_ring: rx descriptor ring packet is being transacted on1803 * @skb: pointer to current skb being adjusted1804 *1805 * This function is an ixgbe specific version of __pskb_pull_tail. The1806 * main difference between this version and the original function is that1807 * this function can make several assumptions about the state of things1808 * that allow for significant optimizations versus the standard function.1809 * As a result we can do things like drop a frag and maintain an accurate1810 * truesize for the skb.1811 */1812static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring,1813 struct sk_buff *skb)1814{1815 skb_frag_t *frag = &skb_shinfo(skb)->frags[0];1816 unsigned char *va;1817 unsigned int pull_len;1818 1819 /*1820 * it is valid to use page_address instead of kmap since we are1821 * working with pages allocated out of the lomem pool per1822 * alloc_page(GFP_ATOMIC)1823 */1824 va = skb_frag_address(frag);1825 1826 /*1827 * we need the header to contain the greater of either ETH_HLEN or1828 * 60 bytes if the skb->len is less than 60 for skb_pad.1829 */1830 pull_len = eth_get_headlen(skb->dev, va, IXGBE_RX_HDR_SIZE);1831 1832 /* align pull length to size of long to optimize memcpy performance */1833 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));1834 1835 /* update all of the pointers */1836 skb_frag_size_sub(frag, pull_len);1837 skb_frag_off_add(frag, pull_len);1838 skb->data_len -= pull_len;1839 skb->tail += pull_len;1840}1841 1842/**1843 * ixgbe_dma_sync_frag - perform DMA sync for first frag of SKB1844 * @rx_ring: rx descriptor ring packet is being transacted on1845 * @skb: pointer to current skb being updated1846 *1847 * This function provides a basic DMA sync up for the first fragment of an1848 * skb. The reason for doing this is that the first fragment cannot be1849 * unmapped until we have reached the end of packet descriptor for a buffer1850 * chain.1851 */1852static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring,1853 struct sk_buff *skb)1854{1855 if (ring_uses_build_skb(rx_ring)) {1856 unsigned long mask = (unsigned long)ixgbe_rx_pg_size(rx_ring) - 1;1857 unsigned long offset = (unsigned long)(skb->data) & mask;1858 1859 dma_sync_single_range_for_cpu(rx_ring->dev,1860 IXGBE_CB(skb)->dma,1861 offset,1862 skb_headlen(skb),1863 DMA_FROM_DEVICE);1864 } else {1865 skb_frag_t *frag = &skb_shinfo(skb)->frags[0];1866 1867 dma_sync_single_range_for_cpu(rx_ring->dev,1868 IXGBE_CB(skb)->dma,1869 skb_frag_off(frag),1870 skb_frag_size(frag),1871 DMA_FROM_DEVICE);1872 }1873 1874 /* If the page was released, just unmap it. */1875 if (unlikely(IXGBE_CB(skb)->page_released)) {1876 dma_unmap_page_attrs(rx_ring->dev, IXGBE_CB(skb)->dma,1877 ixgbe_rx_pg_size(rx_ring),1878 DMA_FROM_DEVICE,1879 IXGBE_RX_DMA_ATTR);1880 }1881}1882 1883/**1884 * ixgbe_cleanup_headers - Correct corrupted or empty headers1885 * @rx_ring: rx descriptor ring packet is being transacted on1886 * @rx_desc: pointer to the EOP Rx descriptor1887 * @skb: pointer to current skb being fixed1888 *1889 * Check if the skb is valid in the XDP case it will be an error pointer.1890 * Return true in this case to abort processing and advance to next1891 * descriptor.1892 *1893 * Check for corrupted packet headers caused by senders on the local L21894 * embedded NIC switch not setting up their Tx Descriptors right. These1895 * should be very rare.1896 *1897 * Also address the case where we are pulling data in on pages only1898 * and as such no data is present in the skb header.1899 *1900 * In addition if skb is not at least 60 bytes we need to pad it so that1901 * it is large enough to qualify as a valid Ethernet frame.1902 *1903 * Returns true if an error was encountered and skb was freed.1904 **/1905bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring,1906 union ixgbe_adv_rx_desc *rx_desc,1907 struct sk_buff *skb)1908{1909 struct net_device *netdev = rx_ring->netdev;1910 1911 /* XDP packets use error pointer so abort at this point */1912 if (IS_ERR(skb))1913 return true;1914 1915 /* Verify netdev is present, and that packet does not have any1916 * errors that would be unacceptable to the netdev.1917 */1918 if (!netdev ||1919 (unlikely(ixgbe_test_staterr(rx_desc,1920 IXGBE_RXDADV_ERR_FRAME_ERR_MASK) &&1921 !(netdev->features & NETIF_F_RXALL)))) {1922 dev_kfree_skb_any(skb);1923 return true;1924 }1925 1926 /* place header in linear portion of buffer */1927 if (!skb_headlen(skb))1928 ixgbe_pull_tail(rx_ring, skb);1929 1930#ifdef IXGBE_FCOE1931 /* do not attempt to pad FCoE Frames as this will disrupt DDP */1932 if (ixgbe_rx_is_fcoe(rx_ring, rx_desc))1933 return false;1934 1935#endif1936 /* if eth_skb_pad returns an error the skb was freed */1937 if (eth_skb_pad(skb))1938 return true;1939 1940 return false;1941}1942 1943/**1944 * ixgbe_reuse_rx_page - page flip buffer and store it back on the ring1945 * @rx_ring: rx descriptor ring to store buffers on1946 * @old_buff: donor buffer to have page reused1947 *1948 * Synchronizes page for reuse by the adapter1949 **/1950static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring,1951 struct ixgbe_rx_buffer *old_buff)1952{1953 struct ixgbe_rx_buffer *new_buff;1954 u16 nta = rx_ring->next_to_alloc;1955 1956 new_buff = &rx_ring->rx_buffer_info[nta];1957 1958 /* update, and store next to alloc */1959 nta++;1960 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;1961 1962 /* Transfer page from old buffer to new buffer.1963 * Move each member individually to avoid possible store1964 * forwarding stalls and unnecessary copy of skb.1965 */1966 new_buff->dma = old_buff->dma;1967 new_buff->page = old_buff->page;1968 new_buff->page_offset = old_buff->page_offset;1969 new_buff->pagecnt_bias = old_buff->pagecnt_bias;1970}1971 1972static bool ixgbe_can_reuse_rx_page(struct ixgbe_rx_buffer *rx_buffer,1973 int rx_buffer_pgcnt)1974{1975 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;1976 struct page *page = rx_buffer->page;1977 1978 /* avoid re-using remote and pfmemalloc pages */1979 if (!dev_page_is_reusable(page))1980 return false;1981 1982#if (PAGE_SIZE < 8192)1983 /* if we are only owner of page we can reuse it */1984 if (unlikely((rx_buffer_pgcnt - pagecnt_bias) > 1))1985 return false;1986#else1987 /* The last offset is a bit aggressive in that we assume the1988 * worst case of FCoE being enabled and using a 3K buffer.1989 * However this should have minimal impact as the 1K extra is1990 * still less than one buffer in size.1991 */1992#define IXGBE_LAST_OFFSET \1993 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBE_RXBUFFER_3K)1994 if (rx_buffer->page_offset > IXGBE_LAST_OFFSET)1995 return false;1996#endif1997 1998 /* If we have drained the page fragment pool we need to update1999 * the pagecnt_bias and page count so that we fully restock the2000 * number of references the driver holds.2001 */2002 if (unlikely(pagecnt_bias == 1)) {2003 page_ref_add(page, USHRT_MAX - 1);2004 rx_buffer->pagecnt_bias = USHRT_MAX;2005 }2006 2007 return true;2008}2009 2010/**2011 * ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff2012 * @rx_ring: rx descriptor ring to transact packets on2013 * @rx_buffer: buffer containing page to add2014 * @skb: sk_buff to place the data into2015 * @size: size of data in rx_buffer2016 *2017 * This function will add the data contained in rx_buffer->page to the skb.2018 * This is done either through a direct copy if the data in the buffer is2019 * less than the skb header size, otherwise it will just attach the page as2020 * a frag to the skb.2021 *2022 * The function will then update the page offset if necessary and return2023 * true if the buffer can be reused by the adapter.2024 **/2025static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring,2026 struct ixgbe_rx_buffer *rx_buffer,2027 struct sk_buff *skb,2028 unsigned int size)2029{2030#if (PAGE_SIZE < 8192)2031 unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;2032#else2033 unsigned int truesize = rx_ring->rx_offset ?2034 SKB_DATA_ALIGN(rx_ring->rx_offset + size) :2035 SKB_DATA_ALIGN(size);2036#endif2037 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,2038 rx_buffer->page_offset, size, truesize);2039#if (PAGE_SIZE < 8192)2040 rx_buffer->page_offset ^= truesize;2041#else2042 rx_buffer->page_offset += truesize;2043#endif2044}2045 2046static struct ixgbe_rx_buffer *ixgbe_get_rx_buffer(struct ixgbe_ring *rx_ring,2047 union ixgbe_adv_rx_desc *rx_desc,2048 struct sk_buff **skb,2049 const unsigned int size,2050 int *rx_buffer_pgcnt)2051{2052 struct ixgbe_rx_buffer *rx_buffer;2053 2054 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];2055 *rx_buffer_pgcnt =2056#if (PAGE_SIZE < 8192)2057 page_count(rx_buffer->page);2058#else2059 0;2060#endif2061 prefetchw(rx_buffer->page);2062 *skb = rx_buffer->skb;2063 2064 /* Delay unmapping of the first packet. It carries the header2065 * information, HW may still access the header after the writeback.2066 * Only unmap it when EOP is reached2067 */2068 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) {2069 if (!*skb)2070 goto skip_sync;2071 } else {2072 if (*skb)2073 ixgbe_dma_sync_frag(rx_ring, *skb);2074 }2075 2076 /* we are reusing so sync this buffer for CPU use */2077 dma_sync_single_range_for_cpu(rx_ring->dev,2078 rx_buffer->dma,2079 rx_buffer->page_offset,2080 size,2081 DMA_FROM_DEVICE);2082skip_sync:2083 rx_buffer->pagecnt_bias--;2084 2085 return rx_buffer;2086}2087 2088static void ixgbe_put_rx_buffer(struct ixgbe_ring *rx_ring,2089 struct ixgbe_rx_buffer *rx_buffer,2090 struct sk_buff *skb,2091 int rx_buffer_pgcnt)2092{2093 if (ixgbe_can_reuse_rx_page(rx_buffer, rx_buffer_pgcnt)) {2094 /* hand second half of page back to the ring */2095 ixgbe_reuse_rx_page(rx_ring, rx_buffer);2096 } else {2097 if (!IS_ERR(skb) && IXGBE_CB(skb)->dma == rx_buffer->dma) {2098 /* the page has been released from the ring */2099 IXGBE_CB(skb)->page_released = true;2100 } else {2101 /* we are not reusing the buffer so unmap it */2102 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,2103 ixgbe_rx_pg_size(rx_ring),2104 DMA_FROM_DEVICE,2105 IXGBE_RX_DMA_ATTR);2106 }2107 __page_frag_cache_drain(rx_buffer->page,2108 rx_buffer->pagecnt_bias);2109 }2110 2111 /* clear contents of rx_buffer */2112 rx_buffer->page = NULL;2113 rx_buffer->skb = NULL;2114}2115 2116static struct sk_buff *ixgbe_construct_skb(struct ixgbe_ring *rx_ring,2117 struct ixgbe_rx_buffer *rx_buffer,2118 struct xdp_buff *xdp,2119 union ixgbe_adv_rx_desc *rx_desc)2120{2121 unsigned int size = xdp->data_end - xdp->data;2122#if (PAGE_SIZE < 8192)2123 unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;2124#else2125 unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end -2126 xdp->data_hard_start);2127#endif2128 struct sk_buff *skb;2129 2130 /* prefetch first cache line of first page */2131 net_prefetch(xdp->data);2132 2133 /* Note, we get here by enabling legacy-rx via:2134 *2135 * ethtool --set-priv-flags <dev> legacy-rx on2136 *2137 * In this mode, we currently get 0 extra XDP headroom as2138 * opposed to having legacy-rx off, where we process XDP2139 * packets going to stack via ixgbe_build_skb(). The latter2140 * provides us currently with 192 bytes of headroom.2141 *2142 * For ixgbe_construct_skb() mode it means that the2143 * xdp->data_meta will always point to xdp->data, since2144 * the helper cannot expand the head. Should this ever2145 * change in future for legacy-rx mode on, then lets also2146 * add xdp->data_meta handling here.2147 */2148 2149 /* allocate a skb to store the frags */2150 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IXGBE_RX_HDR_SIZE);2151 if (unlikely(!skb))2152 return NULL;2153 2154 if (size > IXGBE_RX_HDR_SIZE) {2155 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))2156 IXGBE_CB(skb)->dma = rx_buffer->dma;2157 2158 skb_add_rx_frag(skb, 0, rx_buffer->page,2159 xdp->data - page_address(rx_buffer->page),2160 size, truesize);2161#if (PAGE_SIZE < 8192)2162 rx_buffer->page_offset ^= truesize;2163#else2164 rx_buffer->page_offset += truesize;2165#endif2166 } else {2167 memcpy(__skb_put(skb, size),2168 xdp->data, ALIGN(size, sizeof(long)));2169 rx_buffer->pagecnt_bias++;2170 }2171 2172 return skb;2173}2174 2175static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring,2176 struct ixgbe_rx_buffer *rx_buffer,2177 struct xdp_buff *xdp,2178 union ixgbe_adv_rx_desc *rx_desc)2179{2180 unsigned int metasize = xdp->data - xdp->data_meta;2181#if (PAGE_SIZE < 8192)2182 unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2;2183#else2184 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +2185 SKB_DATA_ALIGN(xdp->data_end -2186 xdp->data_hard_start);2187#endif2188 struct sk_buff *skb;2189 2190 /* Prefetch first cache line of first page. If xdp->data_meta2191 * is unused, this points extactly as xdp->data, otherwise we2192 * likely have a consumer accessing first few bytes of meta2193 * data, and then actual data.2194 */2195 net_prefetch(xdp->data_meta);2196 2197 /* build an skb to around the page buffer */2198 skb = napi_build_skb(xdp->data_hard_start, truesize);2199 if (unlikely(!skb))2200 return NULL;2201 2202 /* update pointers within the skb to store the data */2203 skb_reserve(skb, xdp->data - xdp->data_hard_start);2204 __skb_put(skb, xdp->data_end - xdp->data);2205 if (metasize)2206 skb_metadata_set(skb, metasize);2207 2208 /* record DMA address if this is the start of a chain of buffers */2209 if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))2210 IXGBE_CB(skb)->dma = rx_buffer->dma;2211 2212 /* update buffer offset */2213#if (PAGE_SIZE < 8192)2214 rx_buffer->page_offset ^= truesize;2215#else2216 rx_buffer->page_offset += truesize;2217#endif2218 2219 return skb;2220}2221 2222static struct sk_buff *ixgbe_run_xdp(struct ixgbe_adapter *adapter,2223 struct ixgbe_ring *rx_ring,2224 struct xdp_buff *xdp)2225{2226 int err, result = IXGBE_XDP_PASS;2227 struct bpf_prog *xdp_prog;2228 struct ixgbe_ring *ring;2229 struct xdp_frame *xdpf;2230 u32 act;2231 2232 xdp_prog = READ_ONCE(rx_ring->xdp_prog);2233 2234 if (!xdp_prog)2235 goto xdp_out;2236 2237 prefetchw(xdp->data_hard_start); /* xdp_frame write */2238 2239 act = bpf_prog_run_xdp(xdp_prog, xdp);2240 switch (act) {2241 case XDP_PASS:2242 break;2243 case XDP_TX:2244 xdpf = xdp_convert_buff_to_frame(xdp);2245 if (unlikely(!xdpf))2246 goto out_failure;2247 ring = ixgbe_determine_xdp_ring(adapter);2248 if (static_branch_unlikely(&ixgbe_xdp_locking_key))2249 spin_lock(&ring->tx_lock);2250 result = ixgbe_xmit_xdp_ring(ring, xdpf);2251 if (static_branch_unlikely(&ixgbe_xdp_locking_key))2252 spin_unlock(&ring->tx_lock);2253 if (result == IXGBE_XDP_CONSUMED)2254 goto out_failure;2255 break;2256 case XDP_REDIRECT:2257 err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog);2258 if (err)2259 goto out_failure;2260 result = IXGBE_XDP_REDIR;2261 break;2262 default:2263 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act);2264 fallthrough;2265 case XDP_ABORTED:2266out_failure:2267 trace_xdp_exception(rx_ring->netdev, xdp_prog, act);2268 fallthrough; /* handle aborts by dropping packet */2269 case XDP_DROP:2270 result = IXGBE_XDP_CONSUMED;2271 break;2272 }2273xdp_out:2274 return ERR_PTR(-result);2275}2276 2277static unsigned int ixgbe_rx_frame_truesize(struct ixgbe_ring *rx_ring,2278 unsigned int size)2279{2280 unsigned int truesize;2281 2282#if (PAGE_SIZE < 8192)2283 truesize = ixgbe_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */2284#else2285 truesize = rx_ring->rx_offset ?2286 SKB_DATA_ALIGN(rx_ring->rx_offset + size) +2287 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) :2288 SKB_DATA_ALIGN(size);2289#endif2290 return truesize;2291}2292 2293static void ixgbe_rx_buffer_flip(struct ixgbe_ring *rx_ring,2294 struct ixgbe_rx_buffer *rx_buffer,2295 unsigned int size)2296{2297 unsigned int truesize = ixgbe_rx_frame_truesize(rx_ring, size);2298#if (PAGE_SIZE < 8192)2299 rx_buffer->page_offset ^= truesize;2300#else2301 rx_buffer->page_offset += truesize;2302#endif2303}2304 2305/**2306 * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf2307 * @q_vector: structure containing interrupt and ring information2308 * @rx_ring: rx descriptor ring to transact packets on2309 * @budget: Total limit on number of packets to process2310 *2311 * This function provides a "bounce buffer" approach to Rx interrupt2312 * processing. The advantage to this is that on systems that have2313 * expensive overhead for IOMMU access this provides a means of avoiding2314 * it by maintaining the mapping of the page to the syste.2315 *2316 * Returns amount of work completed2317 **/2318static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,2319 struct ixgbe_ring *rx_ring,2320 const int budget)2321{2322 unsigned int total_rx_bytes = 0, total_rx_packets = 0, frame_sz = 0;2323 struct ixgbe_adapter *adapter = q_vector->adapter;2324#ifdef IXGBE_FCOE2325 int ddp_bytes;2326 unsigned int mss = 0;2327#endif /* IXGBE_FCOE */2328 u16 cleaned_count = ixgbe_desc_unused(rx_ring);2329 unsigned int offset = rx_ring->rx_offset;2330 unsigned int xdp_xmit = 0;2331 struct xdp_buff xdp;2332 2333 /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */2334#if (PAGE_SIZE < 8192)2335 frame_sz = ixgbe_rx_frame_truesize(rx_ring, 0);2336#endif2337 xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq);2338 2339 while (likely(total_rx_packets < budget)) {2340 union ixgbe_adv_rx_desc *rx_desc;2341 struct ixgbe_rx_buffer *rx_buffer;2342 struct sk_buff *skb;2343 int rx_buffer_pgcnt;2344 unsigned int size;2345 2346 /* return some buffers to hardware, one at a time is too slow */2347 if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {2348 ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);2349 cleaned_count = 0;2350 }2351 2352 rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean);2353 size = le16_to_cpu(rx_desc->wb.upper.length);2354 if (!size)2355 break;2356 2357 /* This memory barrier is needed to keep us from reading2358 * any other fields out of the rx_desc until we know the2359 * descriptor has been written back2360 */2361 dma_rmb();2362 2363 rx_buffer = ixgbe_get_rx_buffer(rx_ring, rx_desc, &skb, size, &rx_buffer_pgcnt);2364 2365 /* retrieve a buffer from the ring */2366 if (!skb) {2367 unsigned char *hard_start;2368 2369 hard_start = page_address(rx_buffer->page) +2370 rx_buffer->page_offset - offset;2371 xdp_prepare_buff(&xdp, hard_start, offset, size, true);2372 xdp_buff_clear_frags_flag(&xdp);2373#if (PAGE_SIZE > 4096)2374 /* At larger PAGE_SIZE, frame_sz depend on len size */2375 xdp.frame_sz = ixgbe_rx_frame_truesize(rx_ring, size);2376#endif2377 skb = ixgbe_run_xdp(adapter, rx_ring, &xdp);2378 }2379 2380 if (IS_ERR(skb)) {2381 unsigned int xdp_res = -PTR_ERR(skb);2382 2383 if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) {2384 xdp_xmit |= xdp_res;2385 ixgbe_rx_buffer_flip(rx_ring, rx_buffer, size);2386 } else {2387 rx_buffer->pagecnt_bias++;2388 }2389 total_rx_packets++;2390 total_rx_bytes += size;2391 } else if (skb) {2392 ixgbe_add_rx_frag(rx_ring, rx_buffer, skb, size);2393 } else if (ring_uses_build_skb(rx_ring)) {2394 skb = ixgbe_build_skb(rx_ring, rx_buffer,2395 &xdp, rx_desc);2396 } else {2397 skb = ixgbe_construct_skb(rx_ring, rx_buffer,2398 &xdp, rx_desc);2399 }2400 2401 /* exit if we failed to retrieve a buffer */2402 if (!skb) {2403 rx_ring->rx_stats.alloc_rx_buff_failed++;2404 rx_buffer->pagecnt_bias++;2405 break;2406 }2407 2408 ixgbe_put_rx_buffer(rx_ring, rx_buffer, skb, rx_buffer_pgcnt);2409 cleaned_count++;2410 2411 /* place incomplete frames back on ring for completion */2412 if (ixgbe_is_non_eop(rx_ring, rx_desc, skb))2413 continue;2414 2415 /* verify the packet layout is correct */2416 if (ixgbe_cleanup_headers(rx_ring, rx_desc, skb))2417 continue;2418 2419 /* probably a little skewed due to removing CRC */2420 total_rx_bytes += skb->len;2421 2422 /* populate checksum, timestamp, VLAN, and protocol */2423 ixgbe_process_skb_fields(rx_ring, rx_desc, skb);2424 2425#ifdef IXGBE_FCOE2426 /* if ddp, not passing to ULD unless for FCP_RSP or error */2427 if (ixgbe_rx_is_fcoe(rx_ring, rx_desc)) {2428 ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb);2429 /* include DDPed FCoE data */2430 if (ddp_bytes > 0) {2431 if (!mss) {2432 mss = rx_ring->netdev->mtu -2433 sizeof(struct fcoe_hdr) -2434 sizeof(struct fc_frame_header) -2435 sizeof(struct fcoe_crc_eof);2436 if (mss > 512)2437 mss &= ~511;2438 }2439 total_rx_bytes += ddp_bytes;2440 total_rx_packets += DIV_ROUND_UP(ddp_bytes,2441 mss);2442 }2443 if (!ddp_bytes) {2444 dev_kfree_skb_any(skb);2445 continue;2446 }2447 }2448 2449#endif /* IXGBE_FCOE */2450 ixgbe_rx_skb(q_vector, skb);2451 2452 /* update budget accounting */2453 total_rx_packets++;2454 }2455 2456 if (xdp_xmit & IXGBE_XDP_REDIR)2457 xdp_do_flush();2458 2459 if (xdp_xmit & IXGBE_XDP_TX) {2460 struct ixgbe_ring *ring = ixgbe_determine_xdp_ring(adapter);2461 2462 ixgbe_xdp_ring_update_tail_locked(ring);2463 }2464 2465 ixgbe_update_rx_ring_stats(rx_ring, q_vector, total_rx_packets,2466 total_rx_bytes);2467 2468 return total_rx_packets;2469}2470 2471/**2472 * ixgbe_configure_msix - Configure MSI-X hardware2473 * @adapter: board private structure2474 *2475 * ixgbe_configure_msix sets up the hardware to properly generate MSI-X2476 * interrupts.2477 **/2478static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)2479{2480 struct ixgbe_q_vector *q_vector;2481 int v_idx;2482 u32 mask;2483 2484 /* Populate MSIX to EITR Select */2485 if (adapter->num_vfs > 32) {2486 u32 eitrsel = BIT(adapter->num_vfs - 32) - 1;2487 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel);2488 }2489 2490 /*2491 * Populate the IVAR table and set the ITR values to the2492 * corresponding register.2493 */2494 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {2495 struct ixgbe_ring *ring;2496 q_vector = adapter->q_vector[v_idx];2497 2498 ixgbe_for_each_ring(ring, q_vector->rx)2499 ixgbe_set_ivar(adapter, 0, ring->reg_idx, v_idx);2500 2501 ixgbe_for_each_ring(ring, q_vector->tx)2502 ixgbe_set_ivar(adapter, 1, ring->reg_idx, v_idx);2503 2504 ixgbe_write_eitr(q_vector);2505 }2506 2507 switch (adapter->hw.mac.type) {2508 case ixgbe_mac_82598EB:2509 ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,2510 v_idx);2511 break;2512 case ixgbe_mac_82599EB:2513 case ixgbe_mac_X540:2514 case ixgbe_mac_X550:2515 case ixgbe_mac_X550EM_x:2516 case ixgbe_mac_x550em_a:2517 ixgbe_set_ivar(adapter, -1, 1, v_idx);2518 break;2519 default:2520 break;2521 }2522 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950);2523 2524 /* set up to autoclear timer, and the vectors */2525 mask = IXGBE_EIMS_ENABLE_MASK;2526 mask &= ~(IXGBE_EIMS_OTHER |2527 IXGBE_EIMS_MAILBOX |2528 IXGBE_EIMS_LSC);2529 2530 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);2531}2532 2533/**2534 * ixgbe_update_itr - update the dynamic ITR value based on statistics2535 * @q_vector: structure containing interrupt and ring information2536 * @ring_container: structure containing ring performance data2537 *2538 * Stores a new ITR value based on packets and byte2539 * counts during the last interrupt. The advantage of per interrupt2540 * computation is faster updates and more accurate ITR for the current2541 * traffic pattern. Constants in this function were computed2542 * based on theoretical maximum wire speed and thresholds were set based2543 * on testing data as well as attempting to minimize response time2544 * while increasing bulk throughput.2545 **/2546static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector,2547 struct ixgbe_ring_container *ring_container)2548{2549 unsigned int itr = IXGBE_ITR_ADAPTIVE_MIN_USECS |2550 IXGBE_ITR_ADAPTIVE_LATENCY;2551 unsigned int avg_wire_size, packets, bytes;2552 unsigned long next_update = jiffies;2553 2554 /* If we don't have any rings just leave ourselves set for maximum2555 * possible latency so we take ourselves out of the equation.2556 */2557 if (!ring_container->ring)2558 return;2559 2560 /* If we didn't update within up to 1 - 2 jiffies we can assume2561 * that either packets are coming in so slow there hasn't been2562 * any work, or that there is so much work that NAPI is dealing2563 * with interrupt moderation and we don't need to do anything.2564 */2565 if (time_after(next_update, ring_container->next_update))2566 goto clear_counts;2567 2568 packets = ring_container->total_packets;2569 2570 /* We have no packets to actually measure against. This means2571 * either one of the other queues on this vector is active or2572 * we are a Tx queue doing TSO with too high of an interrupt rate.2573 *2574 * When this occurs just tick up our delay by the minimum value2575 * and hope that this extra delay will prevent us from being called2576 * without any work on our queue.2577 */2578 if (!packets) {2579 itr = (q_vector->itr >> 2) + IXGBE_ITR_ADAPTIVE_MIN_INC;2580 if (itr > IXGBE_ITR_ADAPTIVE_MAX_USECS)2581 itr = IXGBE_ITR_ADAPTIVE_MAX_USECS;2582 itr += ring_container->itr & IXGBE_ITR_ADAPTIVE_LATENCY;2583 goto clear_counts;2584 }2585 2586 bytes = ring_container->total_bytes;2587 2588 /* If packets are less than 4 or bytes are less than 9000 assume2589 * insufficient data to use bulk rate limiting approach. We are2590 * likely latency driven.2591 */2592 if (packets < 4 && bytes < 9000) {2593 itr = IXGBE_ITR_ADAPTIVE_LATENCY;2594 goto adjust_by_size;2595 }2596 2597 /* Between 4 and 48 we can assume that our current interrupt delay2598 * is only slightly too low. As such we should increase it by a small2599 * fixed amount.2600 */2601 if (packets < 48) {2602 itr = (q_vector->itr >> 2) + IXGBE_ITR_ADAPTIVE_MIN_INC;2603 if (itr > IXGBE_ITR_ADAPTIVE_MAX_USECS)2604 itr = IXGBE_ITR_ADAPTIVE_MAX_USECS;2605 goto clear_counts;2606 }2607 2608 /* Between 48 and 96 is our "goldilocks" zone where we are working2609 * out "just right". Just report that our current ITR is good for us.2610 */2611 if (packets < 96) {2612 itr = q_vector->itr >> 2;2613 goto clear_counts;2614 }2615 2616 /* If packet count is 96 or greater we are likely looking at a slight2617 * overrun of the delay we want. Try halving our delay to see if that2618 * will cut the number of packets in half per interrupt.2619 */2620 if (packets < 256) {2621 itr = q_vector->itr >> 3;2622 if (itr < IXGBE_ITR_ADAPTIVE_MIN_USECS)2623 itr = IXGBE_ITR_ADAPTIVE_MIN_USECS;2624 goto clear_counts;2625 }2626 2627 /* The paths below assume we are dealing with a bulk ITR since number2628 * of packets is 256 or greater. We are just going to have to compute2629 * a value and try to bring the count under control, though for smaller2630 * packet sizes there isn't much we can do as NAPI polling will likely2631 * be kicking in sooner rather than later.2632 */2633 itr = IXGBE_ITR_ADAPTIVE_BULK;2634 2635adjust_by_size:2636 /* If packet counts are 256 or greater we can assume we have a gross2637 * overestimation of what the rate should be. Instead of trying to fine2638 * tune it just use the formula below to try and dial in an exact value2639 * give the current packet size of the frame.2640 */2641 avg_wire_size = bytes / packets;2642 2643 /* The following is a crude approximation of:2644 * wmem_default / (size + overhead) = desired_pkts_per_int2645 * rate / bits_per_byte / (size + ethernet overhead) = pkt_rate2646 * (desired_pkt_rate / pkt_rate) * usecs_per_sec = ITR value2647 *2648 * Assuming wmem_default is 212992 and overhead is 640 bytes per2649 * packet, (256 skb, 64 headroom, 320 shared info), we can reduce the2650 * formula down to2651 *2652 * (170 * (size + 24)) / (size + 640) = ITR2653 *2654 * We first do some math on the packet size and then finally bitshift2655 * by 8 after rounding up. We also have to account for PCIe link speed2656 * difference as ITR scales based on this.2657 */2658 if (avg_wire_size <= 60) {2659 /* Start at 50k ints/sec */2660 avg_wire_size = 5120;2661 } else if (avg_wire_size <= 316) {2662 /* 50K ints/sec to 16K ints/sec */2663 avg_wire_size *= 40;2664 avg_wire_size += 2720;2665 } else if (avg_wire_size <= 1084) {2666 /* 16K ints/sec to 9.2K ints/sec */2667 avg_wire_size *= 15;2668 avg_wire_size += 11452;2669 } else if (avg_wire_size < 1968) {2670 /* 9.2K ints/sec to 8K ints/sec */2671 avg_wire_size *= 5;2672 avg_wire_size += 22420;2673 } else {2674 /* plateau at a limit of 8K ints/sec */2675 avg_wire_size = 32256;2676 }2677 2678 /* If we are in low latency mode half our delay which doubles the rate2679 * to somewhere between 100K to 16K ints/sec2680 */2681 if (itr & IXGBE_ITR_ADAPTIVE_LATENCY)2682 avg_wire_size >>= 1;2683 2684 /* Resultant value is 256 times larger than it needs to be. This2685 * gives us room to adjust the value as needed to either increase2686 * or decrease the value based on link speeds of 10G, 2.5G, 1G, etc.2687 *2688 * Use addition as we have already recorded the new latency flag2689 * for the ITR value.2690 */2691 switch (q_vector->adapter->link_speed) {2692 case IXGBE_LINK_SPEED_10GB_FULL:2693 case IXGBE_LINK_SPEED_100_FULL:2694 default:2695 itr += DIV_ROUND_UP(avg_wire_size,2696 IXGBE_ITR_ADAPTIVE_MIN_INC * 256) *2697 IXGBE_ITR_ADAPTIVE_MIN_INC;2698 break;2699 case IXGBE_LINK_SPEED_2_5GB_FULL:2700 case IXGBE_LINK_SPEED_1GB_FULL:2701 case IXGBE_LINK_SPEED_10_FULL:2702 if (avg_wire_size > 8064)2703 avg_wire_size = 8064;2704 itr += DIV_ROUND_UP(avg_wire_size,2705 IXGBE_ITR_ADAPTIVE_MIN_INC * 64) *2706 IXGBE_ITR_ADAPTIVE_MIN_INC;2707 break;2708 }2709 2710clear_counts:2711 /* write back value */2712 ring_container->itr = itr;2713 2714 /* next update should occur within next jiffy */2715 ring_container->next_update = next_update + 1;2716 2717 ring_container->total_bytes = 0;2718 ring_container->total_packets = 0;2719}2720 2721/**2722 * ixgbe_write_eitr - write EITR register in hardware specific way2723 * @q_vector: structure containing interrupt and ring information2724 *2725 * This function is made to be called by ethtool and by the driver2726 * when it needs to update EITR registers at runtime. Hardware2727 * specific quirks/differences are taken care of here.2728 */2729void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector)2730{2731 struct ixgbe_adapter *adapter = q_vector->adapter;2732 struct ixgbe_hw *hw = &adapter->hw;2733 int v_idx = q_vector->v_idx;2734 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;2735 2736 switch (adapter->hw.mac.type) {2737 case ixgbe_mac_82598EB:2738 /* must write high and low 16 bits to reset counter */2739 itr_reg |= (itr_reg << 16);2740 break;2741 case ixgbe_mac_82599EB:2742 case ixgbe_mac_X540:2743 case ixgbe_mac_X550:2744 case ixgbe_mac_X550EM_x:2745 case ixgbe_mac_x550em_a:2746 /*2747 * set the WDIS bit to not clear the timer bits and cause an2748 * immediate assertion of the interrupt2749 */2750 itr_reg |= IXGBE_EITR_CNT_WDIS;2751 break;2752 default:2753 break;2754 }2755 IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg);2756}2757 2758static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector)2759{2760 u32 new_itr;2761 2762 ixgbe_update_itr(q_vector, &q_vector->tx);2763 ixgbe_update_itr(q_vector, &q_vector->rx);2764 2765 /* use the smallest value of new ITR delay calculations */2766 new_itr = min(q_vector->rx.itr, q_vector->tx.itr);2767 2768 /* Clear latency flag if set, shift into correct position */2769 new_itr &= ~IXGBE_ITR_ADAPTIVE_LATENCY;2770 new_itr <<= 2;2771 2772 if (new_itr != q_vector->itr) {2773 /* save the algorithm value here */2774 q_vector->itr = new_itr;2775 2776 ixgbe_write_eitr(q_vector);2777 }2778}2779 2780/**2781 * ixgbe_check_overtemp_subtask - check for over temperature2782 * @adapter: pointer to adapter2783 **/2784static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter)2785{2786 struct ixgbe_hw *hw = &adapter->hw;2787 u32 eicr = adapter->interrupt_event;2788 2789 if (test_bit(__IXGBE_DOWN, &adapter->state))2790 return;2791 2792 if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT))2793 return;2794 2795 adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT;2796 2797 switch (hw->device_id) {2798 case IXGBE_DEV_ID_82599_T3_LOM:2799 /*2800 * Since the warning interrupt is for both ports2801 * we don't have to check if:2802 * - This interrupt wasn't for our port.2803 * - We may have missed the interrupt so always have to2804 * check if we got a LSC2805 */2806 if (!(eicr & IXGBE_EICR_GPI_SDP0_8259X) &&2807 !(eicr & IXGBE_EICR_LSC))2808 return;2809 2810 if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) {2811 u32 speed;2812 bool link_up = false;2813 2814 hw->mac.ops.check_link(hw, &speed, &link_up, false);2815 2816 if (link_up)2817 return;2818 }2819 2820 /* Check if this is not due to overtemp */2821 if (!hw->phy.ops.check_overtemp(hw))2822 return;2823 2824 break;2825 case IXGBE_DEV_ID_X550EM_A_1G_T:2826 case IXGBE_DEV_ID_X550EM_A_1G_T_L:2827 if (!hw->phy.ops.check_overtemp(hw))2828 return;2829 break;2830 default:2831 if (adapter->hw.mac.type >= ixgbe_mac_X540)2832 return;2833 if (!(eicr & IXGBE_EICR_GPI_SDP0(hw)))2834 return;2835 break;2836 }2837 e_crit(drv, "%s\n", ixgbe_overheat_msg);2838 2839 adapter->interrupt_event = 0;2840}2841 2842static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr)2843{2844 struct ixgbe_hw *hw = &adapter->hw;2845 2846 if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&2847 (eicr & IXGBE_EICR_GPI_SDP1(hw))) {2848 e_crit(probe, "Fan has stopped, replace the adapter\n");2849 /* write to clear the interrupt */2850 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw));2851 }2852}2853 2854static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr)2855{2856 struct ixgbe_hw *hw = &adapter->hw;2857 2858 if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE))2859 return;2860 2861 switch (adapter->hw.mac.type) {2862 case ixgbe_mac_82599EB:2863 /*2864 * Need to check link state so complete overtemp check2865 * on service task2866 */2867 if (((eicr & IXGBE_EICR_GPI_SDP0(hw)) ||2868 (eicr & IXGBE_EICR_LSC)) &&2869 (!test_bit(__IXGBE_DOWN, &adapter->state))) {2870 adapter->interrupt_event = eicr;2871 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;2872 ixgbe_service_event_schedule(adapter);2873 return;2874 }2875 return;2876 case ixgbe_mac_x550em_a:2877 if (eicr & IXGBE_EICR_GPI_SDP0_X550EM_a) {2878 adapter->interrupt_event = eicr;2879 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;2880 ixgbe_service_event_schedule(adapter);2881 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC,2882 IXGBE_EICR_GPI_SDP0_X550EM_a);2883 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICR,2884 IXGBE_EICR_GPI_SDP0_X550EM_a);2885 }2886 return;2887 case ixgbe_mac_X550:2888 case ixgbe_mac_X540:2889 if (!(eicr & IXGBE_EICR_TS))2890 return;2891 break;2892 default:2893 return;2894 }2895 2896 e_crit(drv, "%s\n", ixgbe_overheat_msg);2897}2898 2899static inline bool ixgbe_is_sfp(struct ixgbe_hw *hw)2900{2901 switch (hw->mac.type) {2902 case ixgbe_mac_82598EB:2903 if (hw->phy.type == ixgbe_phy_nl)2904 return true;2905 return false;2906 case ixgbe_mac_82599EB:2907 case ixgbe_mac_X550EM_x:2908 case ixgbe_mac_x550em_a:2909 switch (hw->mac.ops.get_media_type(hw)) {2910 case ixgbe_media_type_fiber:2911 case ixgbe_media_type_fiber_qsfp:2912 return true;2913 default:2914 return false;2915 }2916 default:2917 return false;2918 }2919}2920 2921static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr)2922{2923 struct ixgbe_hw *hw = &adapter->hw;2924 u32 eicr_mask = IXGBE_EICR_GPI_SDP2(hw);2925 2926 if (!ixgbe_is_sfp(hw))2927 return;2928 2929 /* Later MAC's use different SDP */2930 if (hw->mac.type >= ixgbe_mac_X540)2931 eicr_mask = IXGBE_EICR_GPI_SDP0_X540;2932 2933 if (eicr & eicr_mask) {2934 /* Clear the interrupt */2935 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr_mask);2936 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {2937 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;2938 adapter->sfp_poll_time = 0;2939 ixgbe_service_event_schedule(adapter);2940 }2941 }2942 2943 if (adapter->hw.mac.type == ixgbe_mac_82599EB &&2944 (eicr & IXGBE_EICR_GPI_SDP1(hw))) {2945 /* Clear the interrupt */2946 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw));2947 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {2948 adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;2949 ixgbe_service_event_schedule(adapter);2950 }2951 }2952}2953 2954static void ixgbe_check_lsc(struct ixgbe_adapter *adapter)2955{2956 struct ixgbe_hw *hw = &adapter->hw;2957 2958 adapter->lsc_int++;2959 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;2960 adapter->link_check_timeout = jiffies;2961 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {2962 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);2963 IXGBE_WRITE_FLUSH(hw);2964 ixgbe_service_event_schedule(adapter);2965 }2966}2967 2968static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter,2969 u64 qmask)2970{2971 struct ixgbe_hw *hw = &adapter->hw;2972 u32 mask;2973 2974 switch (hw->mac.type) {2975 case ixgbe_mac_82598EB:2976 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);2977 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);2978 break;2979 case ixgbe_mac_82599EB:2980 case ixgbe_mac_X540:2981 case ixgbe_mac_X550:2982 case ixgbe_mac_X550EM_x:2983 case ixgbe_mac_x550em_a:2984 mask = (qmask & 0xFFFFFFFF);2985 if (mask)2986 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);2987 mask = (qmask >> 32);2988 if (mask)2989 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);2990 break;2991 default:2992 break;2993 }2994 /* skip the flush */2995}2996 2997/**2998 * ixgbe_irq_enable - Enable default interrupt generation settings2999 * @adapter: board private structure3000 * @queues: enable irqs for queues3001 * @flush: flush register write3002 **/3003static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,3004 bool flush)3005{3006 struct ixgbe_hw *hw = &adapter->hw;3007 u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);3008 3009 /* don't reenable LSC while waiting for link */3010 if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)3011 mask &= ~IXGBE_EIMS_LSC;3012 3013 if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)3014 switch (adapter->hw.mac.type) {3015 case ixgbe_mac_82599EB:3016 mask |= IXGBE_EIMS_GPI_SDP0(hw);3017 break;3018 case ixgbe_mac_X540:3019 case ixgbe_mac_X550:3020 case ixgbe_mac_X550EM_x:3021 case ixgbe_mac_x550em_a:3022 mask |= IXGBE_EIMS_TS;3023 break;3024 default:3025 break;3026 }3027 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)3028 mask |= IXGBE_EIMS_GPI_SDP1(hw);3029 switch (adapter->hw.mac.type) {3030 case ixgbe_mac_82599EB:3031 mask |= IXGBE_EIMS_GPI_SDP1(hw);3032 mask |= IXGBE_EIMS_GPI_SDP2(hw);3033 fallthrough;3034 case ixgbe_mac_X540:3035 case ixgbe_mac_X550:3036 case ixgbe_mac_X550EM_x:3037 case ixgbe_mac_x550em_a:3038 if (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_X_SFP ||3039 adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP ||3040 adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP_N)3041 mask |= IXGBE_EIMS_GPI_SDP0(&adapter->hw);3042 if (adapter->hw.phy.type == ixgbe_phy_x550em_ext_t)3043 mask |= IXGBE_EICR_GPI_SDP0_X540;3044 mask |= IXGBE_EIMS_ECC;3045 mask |= IXGBE_EIMS_MAILBOX;3046 break;3047 default:3048 break;3049 }3050 3051 if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&3052 !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))3053 mask |= IXGBE_EIMS_FLOW_DIR;3054 3055 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask);3056 if (queues)3057 ixgbe_irq_enable_queues(adapter, ~0);3058 if (flush)3059 IXGBE_WRITE_FLUSH(&adapter->hw);3060}3061 3062static irqreturn_t ixgbe_msix_other(int irq, void *data)3063{3064 struct ixgbe_adapter *adapter = data;3065 struct ixgbe_hw *hw = &adapter->hw;3066 u32 eicr;3067 3068 /*3069 * Workaround for Silicon errata. Use clear-by-write instead3070 * of clear-by-read. Reading with EICS will return the3071 * interrupt causes without clearing, which later be done3072 * with the write to EICR.3073 */3074 eicr = IXGBE_READ_REG(hw, IXGBE_EICS);3075 3076 /* The lower 16bits of the EICR register are for the queue interrupts3077 * which should be masked here in order to not accidentally clear them if3078 * the bits are high when ixgbe_msix_other is called. There is a race3079 * condition otherwise which results in possible performance loss3080 * especially if the ixgbe_msix_other interrupt is triggering3081 * consistently (as it would when PPS is turned on for the X540 device)3082 */3083 eicr &= 0xFFFF0000;3084 3085 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);3086 3087 if (eicr & IXGBE_EICR_LSC)3088 ixgbe_check_lsc(adapter);3089 3090 if (eicr & IXGBE_EICR_MAILBOX)3091 ixgbe_msg_task(adapter);3092 3093 switch (hw->mac.type) {3094 case ixgbe_mac_82599EB:3095 case ixgbe_mac_X540:3096 case ixgbe_mac_X550:3097 case ixgbe_mac_X550EM_x:3098 case ixgbe_mac_x550em_a:3099 if (hw->phy.type == ixgbe_phy_x550em_ext_t &&3100 (eicr & IXGBE_EICR_GPI_SDP0_X540)) {3101 adapter->flags2 |= IXGBE_FLAG2_PHY_INTERRUPT;3102 ixgbe_service_event_schedule(adapter);3103 IXGBE_WRITE_REG(hw, IXGBE_EICR,3104 IXGBE_EICR_GPI_SDP0_X540);3105 }3106 if (eicr & IXGBE_EICR_ECC) {3107 e_info(link, "Received ECC Err, initiating reset\n");3108 set_bit(__IXGBE_RESET_REQUESTED, &adapter->state);3109 ixgbe_service_event_schedule(adapter);3110 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);3111 }3112 /* Handle Flow Director Full threshold interrupt */3113 if (eicr & IXGBE_EICR_FLOW_DIR) {3114 int reinit_count = 0;3115 int i;3116 for (i = 0; i < adapter->num_tx_queues; i++) {3117 struct ixgbe_ring *ring = adapter->tx_ring[i];3118 if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE,3119 &ring->state))3120 reinit_count++;3121 }3122 if (reinit_count) {3123 /* no more flow director interrupts until after init */3124 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR);3125 adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT;3126 ixgbe_service_event_schedule(adapter);3127 }3128 }3129 ixgbe_check_sfp_event(adapter, eicr);3130 ixgbe_check_overtemp_event(adapter, eicr);3131 break;3132 default:3133 break;3134 }3135 3136 ixgbe_check_fan_failure(adapter, eicr);3137 3138 if (unlikely(eicr & IXGBE_EICR_TIMESYNC))3139 ixgbe_ptp_check_pps_event(adapter);3140 3141 /* re-enable the original interrupt state, no lsc, no queues */3142 if (!test_bit(__IXGBE_DOWN, &adapter->state))3143 ixgbe_irq_enable(adapter, false, false);3144 3145 return IRQ_HANDLED;3146}3147 3148static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data)3149{3150 struct ixgbe_q_vector *q_vector = data;3151 3152 /* EIAM disabled interrupts (on this vector) for us */3153 3154 if (q_vector->rx.ring || q_vector->tx.ring)3155 napi_schedule_irqoff(&q_vector->napi);3156 3157 return IRQ_HANDLED;3158}3159 3160/**3161 * ixgbe_poll - NAPI Rx polling callback3162 * @napi: structure for representing this polling device3163 * @budget: how many packets driver is allowed to clean3164 *3165 * This function is used for legacy and MSI, NAPI mode3166 **/3167int ixgbe_poll(struct napi_struct *napi, int budget)3168{3169 struct ixgbe_q_vector *q_vector =3170 container_of(napi, struct ixgbe_q_vector, napi);3171 struct ixgbe_adapter *adapter = q_vector->adapter;3172 struct ixgbe_ring *ring;3173 int per_ring_budget, work_done = 0;3174 bool clean_complete = true;3175 3176#ifdef CONFIG_IXGBE_DCA3177 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)3178 ixgbe_update_dca(q_vector);3179#endif3180 3181 ixgbe_for_each_ring(ring, q_vector->tx) {3182 bool wd = ring->xsk_pool ?3183 ixgbe_clean_xdp_tx_irq(q_vector, ring, budget) :3184 ixgbe_clean_tx_irq(q_vector, ring, budget);3185 3186 if (!wd)3187 clean_complete = false;3188 }3189 3190 /* Exit if we are called by netpoll */3191 if (budget <= 0)3192 return budget;3193 3194 /* attempt to distribute budget to each queue fairly, but don't allow3195 * the budget to go below 1 because we'll exit polling */3196 if (q_vector->rx.count > 1)3197 per_ring_budget = max(budget/q_vector->rx.count, 1);3198 else3199 per_ring_budget = budget;3200 3201 ixgbe_for_each_ring(ring, q_vector->rx) {3202 int cleaned = ring->xsk_pool ?3203 ixgbe_clean_rx_irq_zc(q_vector, ring,3204 per_ring_budget) :3205 ixgbe_clean_rx_irq(q_vector, ring,3206 per_ring_budget);3207 3208 work_done += cleaned;3209 if (cleaned >= per_ring_budget)3210 clean_complete = false;3211 }3212 3213 /* If all work not completed, return budget and keep polling */3214 if (!clean_complete)3215 return budget;3216 3217 /* all work done, exit the polling mode */3218 if (likely(napi_complete_done(napi, work_done))) {3219 if (adapter->rx_itr_setting & 1)3220 ixgbe_set_itr(q_vector);3221 if (!test_bit(__IXGBE_DOWN, &adapter->state))3222 ixgbe_irq_enable_queues(adapter,3223 BIT_ULL(q_vector->v_idx));3224 }3225 3226 return min(work_done, budget - 1);3227}3228 3229/**3230 * ixgbe_request_msix_irqs - Initialize MSI-X interrupts3231 * @adapter: board private structure3232 *3233 * ixgbe_request_msix_irqs allocates MSI-X vectors and requests3234 * interrupts from the kernel.3235 **/3236static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter)3237{3238 struct net_device *netdev = adapter->netdev;3239 unsigned int ri = 0, ti = 0;3240 int vector, err;3241 3242 for (vector = 0; vector < adapter->num_q_vectors; vector++) {3243 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];3244 struct msix_entry *entry = &adapter->msix_entries[vector];3245 3246 if (q_vector->tx.ring && q_vector->rx.ring) {3247 snprintf(q_vector->name, sizeof(q_vector->name),3248 "%s-TxRx-%u", netdev->name, ri++);3249 ti++;3250 } else if (q_vector->rx.ring) {3251 snprintf(q_vector->name, sizeof(q_vector->name),3252 "%s-rx-%u", netdev->name, ri++);3253 } else if (q_vector->tx.ring) {3254 snprintf(q_vector->name, sizeof(q_vector->name),3255 "%s-tx-%u", netdev->name, ti++);3256 } else {3257 /* skip this unused q_vector */3258 continue;3259 }3260 err = request_irq(entry->vector, &ixgbe_msix_clean_rings, 0,3261 q_vector->name, q_vector);3262 if (err) {3263 e_err(probe, "request_irq failed for MSIX interrupt "3264 "Error: %d\n", err);3265 goto free_queue_irqs;3266 }3267 /* If Flow Director is enabled, set interrupt affinity */3268 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {3269 /* assign the mask for this irq */3270 irq_update_affinity_hint(entry->vector,3271 &q_vector->affinity_mask);3272 }3273 }3274 3275 err = request_irq(adapter->msix_entries[vector].vector,3276 ixgbe_msix_other, 0, netdev->name, adapter);3277 if (err) {3278 e_err(probe, "request_irq for msix_other failed: %d\n", err);3279 goto free_queue_irqs;3280 }3281 3282 return 0;3283 3284free_queue_irqs:3285 while (vector) {3286 vector--;3287 irq_update_affinity_hint(adapter->msix_entries[vector].vector,3288 NULL);3289 free_irq(adapter->msix_entries[vector].vector,3290 adapter->q_vector[vector]);3291 }3292 adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;3293 pci_disable_msix(adapter->pdev);3294 kfree(adapter->msix_entries);3295 adapter->msix_entries = NULL;3296 return err;3297}3298 3299/**3300 * ixgbe_intr - legacy mode Interrupt Handler3301 * @irq: interrupt number3302 * @data: pointer to a network interface device structure3303 **/3304static irqreturn_t ixgbe_intr(int irq, void *data)3305{3306 struct ixgbe_adapter *adapter = data;3307 struct ixgbe_hw *hw = &adapter->hw;3308 struct ixgbe_q_vector *q_vector = adapter->q_vector[0];3309 u32 eicr;3310 3311 /*3312 * Workaround for silicon errata #26 on 82598. Mask the interrupt3313 * before the read of EICR.3314 */3315 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);3316 3317 /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read3318 * therefore no explicit interrupt disable is necessary */3319 eicr = IXGBE_READ_REG(hw, IXGBE_EICR);3320 if (!eicr) {3321 /*3322 * shared interrupt alert!3323 * make sure interrupts are enabled because the read will3324 * have disabled interrupts due to EIAM3325 * finish the workaround of silicon errata on 82598. Unmask3326 * the interrupt that we masked before the EICR read.3327 */3328 if (!test_bit(__IXGBE_DOWN, &adapter->state))3329 ixgbe_irq_enable(adapter, true, true);3330 return IRQ_NONE; /* Not our interrupt */3331 }3332 3333 if (eicr & IXGBE_EICR_LSC)3334 ixgbe_check_lsc(adapter);3335 3336 switch (hw->mac.type) {3337 case ixgbe_mac_82599EB:3338 ixgbe_check_sfp_event(adapter, eicr);3339 fallthrough;3340 case ixgbe_mac_X540:3341 case ixgbe_mac_X550:3342 case ixgbe_mac_X550EM_x:3343 case ixgbe_mac_x550em_a:3344 if (eicr & IXGBE_EICR_ECC) {3345 e_info(link, "Received ECC Err, initiating reset\n");3346 set_bit(__IXGBE_RESET_REQUESTED, &adapter->state);3347 ixgbe_service_event_schedule(adapter);3348 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_ECC);3349 }3350 ixgbe_check_overtemp_event(adapter, eicr);3351 break;3352 default:3353 break;3354 }3355 3356 ixgbe_check_fan_failure(adapter, eicr);3357 if (unlikely(eicr & IXGBE_EICR_TIMESYNC))3358 ixgbe_ptp_check_pps_event(adapter);3359 3360 /* would disable interrupts here but EIAM disabled it */3361 napi_schedule_irqoff(&q_vector->napi);3362 3363 /*3364 * re-enable link(maybe) and non-queue interrupts, no flush.3365 * ixgbe_poll will re-enable the queue interrupts3366 */3367 if (!test_bit(__IXGBE_DOWN, &adapter->state))3368 ixgbe_irq_enable(adapter, false, false);3369 3370 return IRQ_HANDLED;3371}3372 3373/**3374 * ixgbe_request_irq - initialize interrupts3375 * @adapter: board private structure3376 *3377 * Attempts to configure interrupts using the best available3378 * capabilities of the hardware and kernel.3379 **/3380static int ixgbe_request_irq(struct ixgbe_adapter *adapter)3381{3382 struct net_device *netdev = adapter->netdev;3383 int err;3384 3385 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)3386 err = ixgbe_request_msix_irqs(adapter);3387 else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED)3388 err = request_irq(adapter->pdev->irq, ixgbe_intr, 0,3389 netdev->name, adapter);3390 else3391 err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED,3392 netdev->name, adapter);3393 3394 if (err)3395 e_err(probe, "request_irq failed, Error %d\n", err);3396 3397 return err;3398}3399 3400static void ixgbe_free_irq(struct ixgbe_adapter *adapter)3401{3402 int vector;3403 3404 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {3405 free_irq(adapter->pdev->irq, adapter);3406 return;3407 }3408 3409 if (!adapter->msix_entries)3410 return;3411 3412 for (vector = 0; vector < adapter->num_q_vectors; vector++) {3413 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];3414 struct msix_entry *entry = &adapter->msix_entries[vector];3415 3416 /* free only the irqs that were actually requested */3417 if (!q_vector->rx.ring && !q_vector->tx.ring)3418 continue;3419 3420 /* clear the affinity_mask in the IRQ descriptor */3421 irq_update_affinity_hint(entry->vector, NULL);3422 3423 free_irq(entry->vector, q_vector);3424 }3425 3426 free_irq(adapter->msix_entries[vector].vector, adapter);3427}3428 3429/**3430 * ixgbe_irq_disable - Mask off interrupt generation on the NIC3431 * @adapter: board private structure3432 **/3433static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)3434{3435 switch (adapter->hw.mac.type) {3436 case ixgbe_mac_82598EB:3437 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);3438 break;3439 case ixgbe_mac_82599EB:3440 case ixgbe_mac_X540:3441 case ixgbe_mac_X550:3442 case ixgbe_mac_X550EM_x:3443 case ixgbe_mac_x550em_a:3444 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);3445 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);3446 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);3447 break;3448 default:3449 break;3450 }3451 IXGBE_WRITE_FLUSH(&adapter->hw);3452 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {3453 int vector;3454 3455 for (vector = 0; vector < adapter->num_q_vectors; vector++)3456 synchronize_irq(adapter->msix_entries[vector].vector);3457 3458 synchronize_irq(adapter->msix_entries[vector++].vector);3459 } else {3460 synchronize_irq(adapter->pdev->irq);3461 }3462}3463 3464/**3465 * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts3466 * @adapter: board private structure3467 *3468 **/3469static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)3470{3471 struct ixgbe_q_vector *q_vector = adapter->q_vector[0];3472 3473 ixgbe_write_eitr(q_vector);3474 3475 ixgbe_set_ivar(adapter, 0, 0, 0);3476 ixgbe_set_ivar(adapter, 1, 0, 0);3477 3478 e_info(hw, "Legacy interrupt IVAR setup done\n");3479}3480 3481/**3482 * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset3483 * @adapter: board private structure3484 * @ring: structure containing ring specific data3485 *3486 * Configure the Tx descriptor ring after a reset.3487 **/3488void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,3489 struct ixgbe_ring *ring)3490{3491 struct ixgbe_hw *hw = &adapter->hw;3492 u64 tdba = ring->dma;3493 int wait_loop = 10;3494 u32 txdctl = IXGBE_TXDCTL_ENABLE;3495 u8 reg_idx = ring->reg_idx;3496 3497 ring->xsk_pool = NULL;3498 if (ring_is_xdp(ring))3499 ring->xsk_pool = ixgbe_xsk_pool(adapter, ring);3500 3501 /* disable queue to avoid issues while updating state */3502 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0);3503 IXGBE_WRITE_FLUSH(hw);3504 3505 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx),3506 (tdba & DMA_BIT_MASK(32)));3507 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32));3508 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx),3509 ring->count * sizeof(union ixgbe_adv_tx_desc));3510 IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0);3511 IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0);3512 ring->tail = adapter->io_addr + IXGBE_TDT(reg_idx);3513 3514 /*3515 * set WTHRESH to encourage burst writeback, it should not be set3516 * higher than 1 when:3517 * - ITR is 0 as it could cause false TX hangs3518 * - ITR is set to > 100k int/sec and BQL is enabled3519 *3520 * In order to avoid issues WTHRESH + PTHRESH should always be equal3521 * to or less than the number of on chip descriptors, which is3522 * currently 40.3523 */3524 if (!ring->q_vector || (ring->q_vector->itr < IXGBE_100K_ITR))3525 txdctl |= 1u << 16; /* WTHRESH = 1 */3526 else3527 txdctl |= 8u << 16; /* WTHRESH = 8 */3528 3529 /*3530 * Setting PTHRESH to 32 both improves performance3531 * and avoids a TX hang with DFP enabled3532 */3533 txdctl |= (1u << 8) | /* HTHRESH = 1 */3534 32; /* PTHRESH = 32 */3535 3536 /* reinitialize flowdirector state */3537 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {3538 ring->atr_sample_rate = adapter->atr_sample_rate;3539 ring->atr_count = 0;3540 set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state);3541 } else {3542 ring->atr_sample_rate = 0;3543 }3544 3545 /* initialize XPS */3546 if (!test_and_set_bit(__IXGBE_TX_XPS_INIT_DONE, &ring->state)) {3547 struct ixgbe_q_vector *q_vector = ring->q_vector;3548 3549 if (q_vector)3550 netif_set_xps_queue(ring->netdev,3551 &q_vector->affinity_mask,3552 ring->queue_index);3553 }3554 3555 clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state);3556 3557 /* reinitialize tx_buffer_info */3558 memset(ring->tx_buffer_info, 0,3559 sizeof(struct ixgbe_tx_buffer) * ring->count);3560 3561 /* enable queue */3562 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl);3563 3564 /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */3565 if (hw->mac.type == ixgbe_mac_82598EB &&3566 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))3567 return;3568 3569 /* poll to verify queue is enabled */3570 do {3571 usleep_range(1000, 2000);3572 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));3573 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));3574 if (!wait_loop)3575 hw_dbg(hw, "Could not enable Tx Queue %d\n", reg_idx);3576}3577 3578static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)3579{3580 struct ixgbe_hw *hw = &adapter->hw;3581 u32 rttdcs, mtqc;3582 u8 tcs = adapter->hw_tcs;3583 3584 if (hw->mac.type == ixgbe_mac_82598EB)3585 return;3586 3587 /* disable the arbiter while setting MTQC */3588 rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);3589 rttdcs |= IXGBE_RTTDCS_ARBDIS;3590 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);3591 3592 /* set transmit pool layout */3593 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {3594 mtqc = IXGBE_MTQC_VT_ENA;3595 if (tcs > 4)3596 mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;3597 else if (tcs > 1)3598 mtqc |= IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;3599 else if (adapter->ring_feature[RING_F_VMDQ].mask ==3600 IXGBE_82599_VMDQ_4Q_MASK)3601 mtqc |= IXGBE_MTQC_32VF;3602 else3603 mtqc |= IXGBE_MTQC_64VF;3604 } else {3605 if (tcs > 4) {3606 mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;3607 } else if (tcs > 1) {3608 mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;3609 } else {3610 u8 max_txq = adapter->num_tx_queues +3611 adapter->num_xdp_queues;3612 if (max_txq > 63)3613 mtqc = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;3614 else3615 mtqc = IXGBE_MTQC_64Q_1PB;3616 }3617 }3618 3619 IXGBE_WRITE_REG(hw, IXGBE_MTQC, mtqc);3620 3621 /* Enable Security TX Buffer IFG for multiple pb */3622 if (tcs) {3623 u32 sectx = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);3624 sectx |= IXGBE_SECTX_DCB;3625 IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, sectx);3626 }3627 3628 /* re-enable the arbiter */3629 rttdcs &= ~IXGBE_RTTDCS_ARBDIS;3630 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);3631}3632 3633/**3634 * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset3635 * @adapter: board private structure3636 *3637 * Configure the Tx unit of the MAC after a reset.3638 **/3639static void ixgbe_configure_tx(struct ixgbe_adapter *adapter)3640{3641 struct ixgbe_hw *hw = &adapter->hw;3642 u32 dmatxctl;3643 u32 i;3644 3645 ixgbe_setup_mtqc(adapter);3646 3647 if (hw->mac.type != ixgbe_mac_82598EB) {3648 /* DMATXCTL.EN must be before Tx queues are enabled */3649 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);3650 dmatxctl |= IXGBE_DMATXCTL_TE;3651 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);3652 }3653 3654 /* Setup the HW Tx Head and Tail descriptor pointers */3655 for (i = 0; i < adapter->num_tx_queues; i++)3656 ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]);3657 for (i = 0; i < adapter->num_xdp_queues; i++)3658 ixgbe_configure_tx_ring(adapter, adapter->xdp_ring[i]);3659}3660 3661static void ixgbe_enable_rx_drop(struct ixgbe_adapter *adapter,3662 struct ixgbe_ring *ring)3663{3664 struct ixgbe_hw *hw = &adapter->hw;3665 u8 reg_idx = ring->reg_idx;3666 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));3667 3668 srrctl |= IXGBE_SRRCTL_DROP_EN;3669 3670 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);3671}3672 3673static void ixgbe_disable_rx_drop(struct ixgbe_adapter *adapter,3674 struct ixgbe_ring *ring)3675{3676 struct ixgbe_hw *hw = &adapter->hw;3677 u8 reg_idx = ring->reg_idx;3678 u32 srrctl = IXGBE_READ_REG(hw, IXGBE_SRRCTL(reg_idx));3679 3680 srrctl &= ~IXGBE_SRRCTL_DROP_EN;3681 3682 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);3683}3684 3685#ifdef CONFIG_IXGBE_DCB3686void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)3687#else3688static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter)3689#endif3690{3691 int i;3692 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;3693 3694 if (adapter->ixgbe_ieee_pfc)3695 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);3696 3697 /*3698 * We should set the drop enable bit if:3699 * SR-IOV is enabled3700 * or3701 * Number of Rx queues > 1 and flow control is disabled3702 *3703 * This allows us to avoid head of line blocking for security3704 * and performance reasons.3705 */3706 if (adapter->num_vfs || (adapter->num_rx_queues > 1 &&3707 !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) {3708 for (i = 0; i < adapter->num_rx_queues; i++)3709 ixgbe_enable_rx_drop(adapter, adapter->rx_ring[i]);3710 } else {3711 for (i = 0; i < adapter->num_rx_queues; i++)3712 ixgbe_disable_rx_drop(adapter, adapter->rx_ring[i]);3713 }3714}3715 3716#define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 23717 3718static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,3719 struct ixgbe_ring *rx_ring)3720{3721 struct ixgbe_hw *hw = &adapter->hw;3722 u32 srrctl;3723 u8 reg_idx = rx_ring->reg_idx;3724 3725 if (hw->mac.type == ixgbe_mac_82598EB) {3726 u16 mask = adapter->ring_feature[RING_F_RSS].mask;3727 3728 /*3729 * if VMDq is not active we must program one srrctl register3730 * per RSS queue since we have enabled RDRXCTL.MVMEN3731 */3732 reg_idx &= mask;3733 }3734 3735 /* configure header buffer length, needed for RSC */3736 srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT;3737 3738 /* configure the packet buffer length */3739 if (rx_ring->xsk_pool) {3740 u32 xsk_buf_len = xsk_pool_get_rx_frame_size(rx_ring->xsk_pool);3741 3742 /* If the MAC support setting RXDCTL.RLPML, the3743 * SRRCTL[n].BSIZEPKT is set to PAGE_SIZE and3744 * RXDCTL.RLPML is set to the actual UMEM buffer3745 * size. If not, then we are stuck with a 1k buffer3746 * size resolution. In this case frames larger than3747 * the UMEM buffer size viewed in a 1k resolution will3748 * be dropped.3749 */3750 if (hw->mac.type != ixgbe_mac_82599EB)3751 srrctl |= PAGE_SIZE >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;3752 else3753 srrctl |= xsk_buf_len >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;3754 } else if (test_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state)) {3755 srrctl |= IXGBE_RXBUFFER_3K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;3756 } else {3757 srrctl |= IXGBE_RXBUFFER_2K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;3758 }3759 3760 /* configure descriptor type */3761 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;3762 3763 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(reg_idx), srrctl);3764}3765 3766/**3767 * ixgbe_rss_indir_tbl_entries - Return RSS indirection table entries3768 * @adapter: device handle3769 *3770 * - 82598/82599/X540: 1283771 * - X550(non-SRIOV mode): 5123772 * - X550(SRIOV mode): 643773 */3774u32 ixgbe_rss_indir_tbl_entries(struct ixgbe_adapter *adapter)3775{3776 if (adapter->hw.mac.type < ixgbe_mac_X550)3777 return 128;3778 else if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)3779 return 64;3780 else3781 return 512;3782}3783 3784/**3785 * ixgbe_store_key - Write the RSS key to HW3786 * @adapter: device handle3787 *3788 * Write the RSS key stored in adapter.rss_key to HW.3789 */3790void ixgbe_store_key(struct ixgbe_adapter *adapter)3791{3792 struct ixgbe_hw *hw = &adapter->hw;3793 int i;3794 3795 for (i = 0; i < 10; i++)3796 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), adapter->rss_key[i]);3797}3798 3799/**3800 * ixgbe_init_rss_key - Initialize adapter RSS key3801 * @adapter: device handle3802 *3803 * Allocates and initializes the RSS key if it is not allocated.3804 **/3805static inline int ixgbe_init_rss_key(struct ixgbe_adapter *adapter)3806{3807 u32 *rss_key;3808 3809 if (!adapter->rss_key) {3810 rss_key = kzalloc(IXGBE_RSS_KEY_SIZE, GFP_KERNEL);3811 if (unlikely(!rss_key))3812 return -ENOMEM;3813 3814 netdev_rss_key_fill(rss_key, IXGBE_RSS_KEY_SIZE);3815 adapter->rss_key = rss_key;3816 }3817 3818 return 0;3819}3820 3821/**3822 * ixgbe_store_reta - Write the RETA table to HW3823 * @adapter: device handle3824 *3825 * Write the RSS redirection table stored in adapter.rss_indir_tbl[] to HW.3826 */3827void ixgbe_store_reta(struct ixgbe_adapter *adapter)3828{3829 u32 i, reta_entries = ixgbe_rss_indir_tbl_entries(adapter);3830 struct ixgbe_hw *hw = &adapter->hw;3831 u32 reta = 0;3832 u32 indices_multi;3833 u8 *indir_tbl = adapter->rss_indir_tbl;3834 3835 /* Fill out the redirection table as follows:3836 * - 82598: 8 bit wide entries containing pair of 4 bit RSS3837 * indices.3838 * - 82599/X540: 8 bit wide entries containing 4 bit RSS index3839 * - X550: 8 bit wide entries containing 6 bit RSS index3840 */3841 if (adapter->hw.mac.type == ixgbe_mac_82598EB)3842 indices_multi = 0x11;3843 else3844 indices_multi = 0x1;3845 3846 /* Write redirection table to HW */3847 for (i = 0; i < reta_entries; i++) {3848 reta |= indices_multi * indir_tbl[i] << (i & 0x3) * 8;3849 if ((i & 3) == 3) {3850 if (i < 128)3851 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);3852 else3853 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32),3854 reta);3855 reta = 0;3856 }3857 }3858}3859 3860/**3861 * ixgbe_store_vfreta - Write the RETA table to HW (x550 devices in SRIOV mode)3862 * @adapter: device handle3863 *3864 * Write the RSS redirection table stored in adapter.rss_indir_tbl[] to HW.3865 */3866static void ixgbe_store_vfreta(struct ixgbe_adapter *adapter)3867{3868 u32 i, reta_entries = ixgbe_rss_indir_tbl_entries(adapter);3869 struct ixgbe_hw *hw = &adapter->hw;3870 u32 vfreta = 0;3871 3872 /* Write redirection table to HW */3873 for (i = 0; i < reta_entries; i++) {3874 u16 pool = adapter->num_rx_pools;3875 3876 vfreta |= (u32)adapter->rss_indir_tbl[i] << (i & 0x3) * 8;3877 if ((i & 3) != 3)3878 continue;3879 3880 while (pool--)3881 IXGBE_WRITE_REG(hw,3882 IXGBE_PFVFRETA(i >> 2, VMDQ_P(pool)),3883 vfreta);3884 vfreta = 0;3885 }3886}3887 3888static void ixgbe_setup_reta(struct ixgbe_adapter *adapter)3889{3890 u32 i, j;3891 u32 reta_entries = ixgbe_rss_indir_tbl_entries(adapter);3892 u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;3893 3894 /* Program table for at least 4 queues w/ SR-IOV so that VFs can3895 * make full use of any rings they may have. We will use the3896 * PSRTYPE register to control how many rings we use within the PF.3897 */3898 if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && (rss_i < 4))3899 rss_i = 4;3900 3901 /* Fill out hash function seeds */3902 ixgbe_store_key(adapter);3903 3904 /* Fill out redirection table */3905 memset(adapter->rss_indir_tbl, 0, sizeof(adapter->rss_indir_tbl));3906 3907 for (i = 0, j = 0; i < reta_entries; i++, j++) {3908 if (j == rss_i)3909 j = 0;3910 3911 adapter->rss_indir_tbl[i] = j;3912 }3913 3914 ixgbe_store_reta(adapter);3915}3916 3917static void ixgbe_setup_vfreta(struct ixgbe_adapter *adapter)3918{3919 struct ixgbe_hw *hw = &adapter->hw;3920 u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;3921 int i, j;3922 3923 /* Fill out hash function seeds */3924 for (i = 0; i < 10; i++) {3925 u16 pool = adapter->num_rx_pools;3926 3927 while (pool--)3928 IXGBE_WRITE_REG(hw,3929 IXGBE_PFVFRSSRK(i, VMDQ_P(pool)),3930 *(adapter->rss_key + i));3931 }3932 3933 /* Fill out the redirection table */3934 for (i = 0, j = 0; i < 64; i++, j++) {3935 if (j == rss_i)3936 j = 0;3937 3938 adapter->rss_indir_tbl[i] = j;3939 }3940 3941 ixgbe_store_vfreta(adapter);3942}3943 3944static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)3945{3946 struct ixgbe_hw *hw = &adapter->hw;3947 u32 mrqc = 0, rss_field = 0, vfmrqc = 0;3948 u32 rxcsum;3949 3950 /* Disable indicating checksum in descriptor, enables RSS hash */3951 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);3952 rxcsum |= IXGBE_RXCSUM_PCSD;3953 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);3954 3955 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {3956 if (adapter->ring_feature[RING_F_RSS].mask)3957 mrqc = IXGBE_MRQC_RSSEN;3958 } else {3959 u8 tcs = adapter->hw_tcs;3960 3961 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {3962 if (tcs > 4)3963 mrqc = IXGBE_MRQC_VMDQRT8TCEN; /* 8 TCs */3964 else if (tcs > 1)3965 mrqc = IXGBE_MRQC_VMDQRT4TCEN; /* 4 TCs */3966 else if (adapter->ring_feature[RING_F_VMDQ].mask ==3967 IXGBE_82599_VMDQ_4Q_MASK)3968 mrqc = IXGBE_MRQC_VMDQRSS32EN;3969 else3970 mrqc = IXGBE_MRQC_VMDQRSS64EN;3971 3972 /* Enable L3/L4 for Tx Switched packets only for X550,3973 * older devices do not support this feature3974 */3975 if (hw->mac.type >= ixgbe_mac_X550)3976 mrqc |= IXGBE_MRQC_L3L4TXSWEN;3977 } else {3978 if (tcs > 4)3979 mrqc = IXGBE_MRQC_RTRSS8TCEN;3980 else if (tcs > 1)3981 mrqc = IXGBE_MRQC_RTRSS4TCEN;3982 else3983 mrqc = IXGBE_MRQC_RSSEN;3984 }3985 }3986 3987 /* Perform hash on these packet types */3988 rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4 |3989 IXGBE_MRQC_RSS_FIELD_IPV4_TCP |3990 IXGBE_MRQC_RSS_FIELD_IPV6 |3991 IXGBE_MRQC_RSS_FIELD_IPV6_TCP;3992 3993 if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV4_UDP)3994 rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP;3995 if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV6_UDP)3996 rss_field |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP;3997 3998 if ((hw->mac.type >= ixgbe_mac_X550) &&3999 (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) {4000 u16 pool = adapter->num_rx_pools;4001 4002 /* Enable VF RSS mode */4003 mrqc |= IXGBE_MRQC_MULTIPLE_RSS;4004 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);4005 4006 /* Setup RSS through the VF registers */4007 ixgbe_setup_vfreta(adapter);4008 vfmrqc = IXGBE_MRQC_RSSEN;4009 vfmrqc |= rss_field;4010 4011 while (pool--)4012 IXGBE_WRITE_REG(hw,4013 IXGBE_PFVFMRQC(VMDQ_P(pool)),4014 vfmrqc);4015 } else {4016 ixgbe_setup_reta(adapter);4017 mrqc |= rss_field;4018 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);4019 }4020}4021 4022/**4023 * ixgbe_configure_rscctl - enable RSC for the indicated ring4024 * @adapter: address of board private structure4025 * @ring: structure containing ring specific data4026 **/4027static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter,4028 struct ixgbe_ring *ring)4029{4030 struct ixgbe_hw *hw = &adapter->hw;4031 u32 rscctrl;4032 u8 reg_idx = ring->reg_idx;4033 4034 if (!ring_is_rsc_enabled(ring))4035 return;4036 4037 rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx));4038 rscctrl |= IXGBE_RSCCTL_RSCEN;4039 /*4040 * we must limit the number of descriptors so that the4041 * total size of max desc * buf_len is not greater4042 * than 655364043 */4044 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;4045 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl);4046}4047 4048#define IXGBE_MAX_RX_DESC_POLL 104049static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter,4050 struct ixgbe_ring *ring)4051{4052 struct ixgbe_hw *hw = &adapter->hw;4053 int wait_loop = IXGBE_MAX_RX_DESC_POLL;4054 u32 rxdctl;4055 u8 reg_idx = ring->reg_idx;4056 4057 if (ixgbe_removed(hw->hw_addr))4058 return;4059 /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */4060 if (hw->mac.type == ixgbe_mac_82598EB &&4061 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))4062 return;4063 4064 do {4065 usleep_range(1000, 2000);4066 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));4067 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));4068 4069 if (!wait_loop) {4070 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within "4071 "the polling period\n", reg_idx);4072 }4073}4074 4075void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,4076 struct ixgbe_ring *ring)4077{4078 struct ixgbe_hw *hw = &adapter->hw;4079 union ixgbe_adv_rx_desc *rx_desc;4080 u64 rdba = ring->dma;4081 u32 rxdctl;4082 u8 reg_idx = ring->reg_idx;4083 4084 xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);4085 ring->xsk_pool = ixgbe_xsk_pool(adapter, ring);4086 if (ring->xsk_pool) {4087 WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,4088 MEM_TYPE_XSK_BUFF_POOL,4089 NULL));4090 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq);4091 } else {4092 WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,4093 MEM_TYPE_PAGE_SHARED, NULL));4094 }4095 4096 /* disable queue to avoid use of these values while updating state */4097 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));4098 rxdctl &= ~IXGBE_RXDCTL_ENABLE;4099 4100 /* write value back with RXDCTL.ENABLE bit cleared */4101 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);4102 IXGBE_WRITE_FLUSH(hw);4103 4104 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32)));4105 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32));4106 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx),4107 ring->count * sizeof(union ixgbe_adv_rx_desc));4108 /* Force flushing of IXGBE_RDLEN to prevent MDD */4109 IXGBE_WRITE_FLUSH(hw);4110 4111 IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0);4112 IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0);4113 ring->tail = adapter->io_addr + IXGBE_RDT(reg_idx);4114 4115 ixgbe_configure_srrctl(adapter, ring);4116 ixgbe_configure_rscctl(adapter, ring);4117 4118 if (hw->mac.type == ixgbe_mac_82598EB) {4119 /*4120 * enable cache line friendly hardware writes:4121 * PTHRESH=32 descriptors (half the internal cache),4122 * this also removes ugly rx_no_buffer_count increment4123 * HTHRESH=4 descriptors (to minimize latency on fetch)4124 * WTHRESH=8 burst writeback up to two cache lines4125 */4126 rxdctl &= ~0x3FFFFF;4127 rxdctl |= 0x080420;4128#if (PAGE_SIZE < 8192)4129 /* RXDCTL.RLPML does not work on 82599 */4130 } else if (hw->mac.type != ixgbe_mac_82599EB) {4131 rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK |4132 IXGBE_RXDCTL_RLPML_EN);4133 4134 /* Limit the maximum frame size so we don't overrun the skb.4135 * This can happen in SRIOV mode when the MTU of the VF is4136 * higher than the MTU of the PF.4137 */4138 if (ring_uses_build_skb(ring) &&4139 !test_bit(__IXGBE_RX_3K_BUFFER, &ring->state))4140 rxdctl |= IXGBE_MAX_2K_FRAME_BUILD_SKB |4141 IXGBE_RXDCTL_RLPML_EN;4142#endif4143 }4144 4145 ring->rx_offset = ixgbe_rx_offset(ring);4146 4147 if (ring->xsk_pool && hw->mac.type != ixgbe_mac_82599EB) {4148 u32 xsk_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool);4149 4150 rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK |4151 IXGBE_RXDCTL_RLPML_EN);4152 rxdctl |= xsk_buf_len | IXGBE_RXDCTL_RLPML_EN;4153 4154 ring->rx_buf_len = xsk_buf_len;4155 }4156 4157 /* initialize rx_buffer_info */4158 memset(ring->rx_buffer_info, 0,4159 sizeof(struct ixgbe_rx_buffer) * ring->count);4160 4161 /* initialize Rx descriptor 0 */4162 rx_desc = IXGBE_RX_DESC(ring, 0);4163 rx_desc->wb.upper.length = 0;4164 4165 /* enable receive descriptor ring */4166 rxdctl |= IXGBE_RXDCTL_ENABLE;4167 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);4168 4169 ixgbe_rx_desc_queue_enable(adapter, ring);4170 if (ring->xsk_pool)4171 ixgbe_alloc_rx_buffers_zc(ring, ixgbe_desc_unused(ring));4172 else4173 ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring));4174}4175 4176static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter)4177{4178 struct ixgbe_hw *hw = &adapter->hw;4179 int rss_i = adapter->ring_feature[RING_F_RSS].indices;4180 u16 pool = adapter->num_rx_pools;4181 4182 /* PSRTYPE must be initialized in non 82598 adapters */4183 u32 psrtype = IXGBE_PSRTYPE_TCPHDR |4184 IXGBE_PSRTYPE_UDPHDR |4185 IXGBE_PSRTYPE_IPV4HDR |4186 IXGBE_PSRTYPE_L2HDR |4187 IXGBE_PSRTYPE_IPV6HDR;4188 4189 if (hw->mac.type == ixgbe_mac_82598EB)4190 return;4191 4192 if (rss_i > 3)4193 psrtype |= 2u << 29;4194 else if (rss_i > 1)4195 psrtype |= 1u << 29;4196 4197 while (pool--)4198 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(VMDQ_P(pool)), psrtype);4199}4200 4201static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter)4202{4203 struct ixgbe_hw *hw = &adapter->hw;4204 u16 pool = adapter->num_rx_pools;4205 u32 reg_offset, vf_shift, vmolr;4206 u32 gcr_ext, vmdctl;4207 int i;4208 4209 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))4210 return;4211 4212 vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);4213 vmdctl |= IXGBE_VMD_CTL_VMDQ_EN;4214 vmdctl &= ~IXGBE_VT_CTL_POOL_MASK;4215 vmdctl |= VMDQ_P(0) << IXGBE_VT_CTL_POOL_SHIFT;4216 vmdctl |= IXGBE_VT_CTL_REPLEN;4217 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl);4218 4219 /* accept untagged packets until a vlan tag is4220 * specifically set for the VMDQ queue/pool4221 */4222 vmolr = IXGBE_VMOLR_AUPE;4223 while (pool--)4224 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(VMDQ_P(pool)), vmolr);4225 4226 vf_shift = VMDQ_P(0) % 32;4227 reg_offset = (VMDQ_P(0) >= 32) ? 1 : 0;4228 4229 /* Enable only the PF's pool for Tx/Rx */4230 IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), GENMASK(31, vf_shift));4231 IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), reg_offset - 1);4232 IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), GENMASK(31, vf_shift));4233 IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), reg_offset - 1);4234 if (adapter->bridge_mode == BRIDGE_MODE_VEB)4235 IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);4236 4237 /* Map PF MAC address in RAR Entry 0 to first pool following VFs */4238 hw->mac.ops.set_vmdq(hw, 0, VMDQ_P(0));4239 4240 /* clear VLAN promisc flag so VFTA will be updated if necessary */4241 adapter->flags2 &= ~IXGBE_FLAG2_VLAN_PROMISC;4242 4243 /*4244 * Set up VF register offsets for selected VT Mode,4245 * i.e. 32 or 64 VFs for SR-IOV4246 */4247 switch (adapter->ring_feature[RING_F_VMDQ].mask) {4248 case IXGBE_82599_VMDQ_8Q_MASK:4249 gcr_ext = IXGBE_GCR_EXT_VT_MODE_16;4250 break;4251 case IXGBE_82599_VMDQ_4Q_MASK:4252 gcr_ext = IXGBE_GCR_EXT_VT_MODE_32;4253 break;4254 default:4255 gcr_ext = IXGBE_GCR_EXT_VT_MODE_64;4256 break;4257 }4258 4259 IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext);4260 4261 for (i = 0; i < adapter->num_vfs; i++) {4262 /* configure spoof checking */4263 ixgbe_ndo_set_vf_spoofchk(adapter->netdev, i,4264 adapter->vfinfo[i].spoofchk_enabled);4265 4266 /* Enable/Disable RSS query feature */4267 ixgbe_ndo_set_vf_rss_query_en(adapter->netdev, i,4268 adapter->vfinfo[i].rss_query_enabled);4269 }4270}4271 4272static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter)4273{4274 struct ixgbe_hw *hw = &adapter->hw;4275 struct net_device *netdev = adapter->netdev;4276 int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;4277 struct ixgbe_ring *rx_ring;4278 int i;4279 u32 mhadd, hlreg0;4280 4281#ifdef IXGBE_FCOE4282 /* adjust max frame to be able to do baby jumbo for FCoE */4283 if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&4284 (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE))4285 max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE;4286 4287#endif /* IXGBE_FCOE */4288 4289 /* adjust max frame to be at least the size of a standard frame */4290 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))4291 max_frame = (ETH_FRAME_LEN + ETH_FCS_LEN);4292 4293 mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);4294 if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) {4295 mhadd &= ~IXGBE_MHADD_MFS_MASK;4296 mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT;4297 4298 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);4299 }4300 4301 hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);4302 /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */4303 hlreg0 |= IXGBE_HLREG0_JUMBOEN;4304 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);4305 4306 /*4307 * Setup the HW Rx Head and Tail Descriptor Pointers and4308 * the Base and Length of the Rx Descriptor Ring4309 */4310 for (i = 0; i < adapter->num_rx_queues; i++) {4311 rx_ring = adapter->rx_ring[i];4312 4313 clear_ring_rsc_enabled(rx_ring);4314 clear_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state);4315 clear_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &rx_ring->state);4316 4317 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)4318 set_ring_rsc_enabled(rx_ring);4319 4320 if (test_bit(__IXGBE_RX_FCOE, &rx_ring->state))4321 set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state);4322 4323 if (adapter->flags2 & IXGBE_FLAG2_RX_LEGACY)4324 continue;4325 4326 set_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &rx_ring->state);4327 4328#if (PAGE_SIZE < 8192)4329 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)4330 set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state);4331 4332 if (IXGBE_2K_TOO_SMALL_WITH_PADDING ||4333 (max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN)))4334 set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state);4335#endif4336 }4337}4338 4339static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter)4340{4341 struct ixgbe_hw *hw = &adapter->hw;4342 u32 rdrxctl = IXGBE_READ_REG(hw, IXGBE_RDRXCTL);4343 4344 switch (hw->mac.type) {4345 case ixgbe_mac_82598EB:4346 /*4347 * For VMDq support of different descriptor types or4348 * buffer sizes through the use of multiple SRRCTL4349 * registers, RDRXCTL.MVMEN must be set to 14350 *4351 * also, the manual doesn't mention it clearly but DCA hints4352 * will only use queue 0's tags unless this bit is set. Side4353 * effects of setting this bit are only that SRRCTL must be4354 * fully programmed [0..15]4355 */4356 rdrxctl |= IXGBE_RDRXCTL_MVMEN;4357 break;4358 case ixgbe_mac_X550:4359 case ixgbe_mac_X550EM_x:4360 case ixgbe_mac_x550em_a:4361 if (adapter->num_vfs)4362 rdrxctl |= IXGBE_RDRXCTL_PSP;4363 fallthrough;4364 case ixgbe_mac_82599EB:4365 case ixgbe_mac_X540:4366 /* Disable RSC for ACK packets */4367 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU,4368 (IXGBE_RSCDBU_RSCACKDIS | IXGBE_READ_REG(hw, IXGBE_RSCDBU)));4369 rdrxctl &= ~IXGBE_RDRXCTL_RSCFRSTSIZE;4370 /* hardware requires some bits to be set by default */4371 rdrxctl |= (IXGBE_RDRXCTL_RSCACKC | IXGBE_RDRXCTL_FCOE_WRFIX);4372 rdrxctl |= IXGBE_RDRXCTL_CRCSTRIP;4373 break;4374 default:4375 /* We should do nothing since we don't know this hardware */4376 return;4377 }4378 4379 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, rdrxctl);4380}4381 4382/**4383 * ixgbe_configure_rx - Configure 8259x Receive Unit after Reset4384 * @adapter: board private structure4385 *4386 * Configure the Rx unit of the MAC after a reset.4387 **/4388static void ixgbe_configure_rx(struct ixgbe_adapter *adapter)4389{4390 struct ixgbe_hw *hw = &adapter->hw;4391 int i;4392 u32 rxctrl, rfctl;4393 4394 /* disable receives while setting up the descriptors */4395 hw->mac.ops.disable_rx(hw);4396 4397 ixgbe_setup_psrtype(adapter);4398 ixgbe_setup_rdrxctl(adapter);4399 4400 /* RSC Setup */4401 rfctl = IXGBE_READ_REG(hw, IXGBE_RFCTL);4402 rfctl &= ~IXGBE_RFCTL_RSC_DIS;4403 if (!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED))4404 rfctl |= IXGBE_RFCTL_RSC_DIS;4405 4406 /* disable NFS filtering */4407 rfctl |= (IXGBE_RFCTL_NFSW_DIS | IXGBE_RFCTL_NFSR_DIS);4408 IXGBE_WRITE_REG(hw, IXGBE_RFCTL, rfctl);4409 4410 /* Program registers for the distribution of queues */4411 ixgbe_setup_mrqc(adapter);4412 4413 /* set_rx_buffer_len must be called before ring initialization */4414 ixgbe_set_rx_buffer_len(adapter);4415 4416 /*4417 * Setup the HW Rx Head and Tail Descriptor Pointers and4418 * the Base and Length of the Rx Descriptor Ring4419 */4420 for (i = 0; i < adapter->num_rx_queues; i++)4421 ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]);4422 4423 rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL);4424 /* disable drop enable for 82598 parts */4425 if (hw->mac.type == ixgbe_mac_82598EB)4426 rxctrl |= IXGBE_RXCTRL_DMBYPS;4427 4428 /* enable all receives */4429 rxctrl |= IXGBE_RXCTRL_RXEN;4430 hw->mac.ops.enable_rx_dma(hw, rxctrl);4431}4432 4433static int ixgbe_vlan_rx_add_vid(struct net_device *netdev,4434 __be16 proto, u16 vid)4435{4436 struct ixgbe_adapter *adapter = netdev_priv(netdev);4437 struct ixgbe_hw *hw = &adapter->hw;4438 4439 /* add VID to filter table */4440 if (!vid || !(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))4441 hw->mac.ops.set_vfta(&adapter->hw, vid, VMDQ_P(0), true, !!vid);4442 4443 set_bit(vid, adapter->active_vlans);4444 4445 return 0;4446}4447 4448static int ixgbe_find_vlvf_entry(struct ixgbe_hw *hw, u32 vlan)4449{4450 u32 vlvf;4451 int idx;4452 4453 /* short cut the special case */4454 if (vlan == 0)4455 return 0;4456 4457 /* Search for the vlan id in the VLVF entries */4458 for (idx = IXGBE_VLVF_ENTRIES; --idx;) {4459 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(idx));4460 if ((vlvf & VLAN_VID_MASK) == vlan)4461 break;4462 }4463 4464 return idx;4465}4466 4467void ixgbe_update_pf_promisc_vlvf(struct ixgbe_adapter *adapter, u32 vid)4468{4469 struct ixgbe_hw *hw = &adapter->hw;4470 u32 bits, word;4471 int idx;4472 4473 idx = ixgbe_find_vlvf_entry(hw, vid);4474 if (!idx)4475 return;4476 4477 /* See if any other pools are set for this VLAN filter4478 * entry other than the PF.4479 */4480 word = idx * 2 + (VMDQ_P(0) / 32);4481 bits = ~BIT(VMDQ_P(0) % 32);4482 bits &= IXGBE_READ_REG(hw, IXGBE_VLVFB(word));4483 4484 /* Disable the filter so this falls into the default pool. */4485 if (!bits && !IXGBE_READ_REG(hw, IXGBE_VLVFB(word ^ 1))) {4486 if (!(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))4487 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(word), 0);4488 IXGBE_WRITE_REG(hw, IXGBE_VLVF(idx), 0);4489 }4490}4491 4492static int ixgbe_vlan_rx_kill_vid(struct net_device *netdev,4493 __be16 proto, u16 vid)4494{4495 struct ixgbe_adapter *adapter = netdev_priv(netdev);4496 struct ixgbe_hw *hw = &adapter->hw;4497 4498 /* remove VID from filter table */4499 if (vid && !(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))4500 hw->mac.ops.set_vfta(hw, vid, VMDQ_P(0), false, true);4501 4502 clear_bit(vid, adapter->active_vlans);4503 4504 return 0;4505}4506 4507/**4508 * ixgbe_vlan_strip_disable - helper to disable hw vlan stripping4509 * @adapter: driver data4510 */4511static void ixgbe_vlan_strip_disable(struct ixgbe_adapter *adapter)4512{4513 struct ixgbe_hw *hw = &adapter->hw;4514 u32 vlnctrl;4515 int i, j;4516 4517 switch (hw->mac.type) {4518 case ixgbe_mac_82598EB:4519 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);4520 vlnctrl &= ~IXGBE_VLNCTRL_VME;4521 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);4522 break;4523 case ixgbe_mac_82599EB:4524 case ixgbe_mac_X540:4525 case ixgbe_mac_X550:4526 case ixgbe_mac_X550EM_x:4527 case ixgbe_mac_x550em_a:4528 for (i = 0; i < adapter->num_rx_queues; i++) {4529 struct ixgbe_ring *ring = adapter->rx_ring[i];4530 4531 if (!netif_is_ixgbe(ring->netdev))4532 continue;4533 4534 j = ring->reg_idx;4535 vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));4536 vlnctrl &= ~IXGBE_RXDCTL_VME;4537 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);4538 }4539 break;4540 default:4541 break;4542 }4543}4544 4545/**4546 * ixgbe_vlan_strip_enable - helper to enable hw vlan stripping4547 * @adapter: driver data4548 */4549static void ixgbe_vlan_strip_enable(struct ixgbe_adapter *adapter)4550{4551 struct ixgbe_hw *hw = &adapter->hw;4552 u32 vlnctrl;4553 int i, j;4554 4555 switch (hw->mac.type) {4556 case ixgbe_mac_82598EB:4557 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);4558 vlnctrl |= IXGBE_VLNCTRL_VME;4559 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);4560 break;4561 case ixgbe_mac_82599EB:4562 case ixgbe_mac_X540:4563 case ixgbe_mac_X550:4564 case ixgbe_mac_X550EM_x:4565 case ixgbe_mac_x550em_a:4566 for (i = 0; i < adapter->num_rx_queues; i++) {4567 struct ixgbe_ring *ring = adapter->rx_ring[i];4568 4569 if (!netif_is_ixgbe(ring->netdev))4570 continue;4571 4572 j = ring->reg_idx;4573 vlnctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(j));4574 vlnctrl |= IXGBE_RXDCTL_VME;4575 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(j), vlnctrl);4576 }4577 break;4578 default:4579 break;4580 }4581}4582 4583static void ixgbe_vlan_promisc_enable(struct ixgbe_adapter *adapter)4584{4585 struct ixgbe_hw *hw = &adapter->hw;4586 u32 vlnctrl, i;4587 4588 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);4589 4590 if (adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) {4591 /* For VMDq and SR-IOV we must leave VLAN filtering enabled */4592 vlnctrl |= IXGBE_VLNCTRL_VFE;4593 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);4594 } else {4595 vlnctrl &= ~IXGBE_VLNCTRL_VFE;4596 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);4597 return;4598 }4599 4600 /* Nothing to do for 82598 */4601 if (hw->mac.type == ixgbe_mac_82598EB)4602 return;4603 4604 /* We are already in VLAN promisc, nothing to do */4605 if (adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC)4606 return;4607 4608 /* Set flag so we don't redo unnecessary work */4609 adapter->flags2 |= IXGBE_FLAG2_VLAN_PROMISC;4610 4611 /* Add PF to all active pools */4612 for (i = IXGBE_VLVF_ENTRIES; --i;) {4613 u32 reg_offset = IXGBE_VLVFB(i * 2 + VMDQ_P(0) / 32);4614 u32 vlvfb = IXGBE_READ_REG(hw, reg_offset);4615 4616 vlvfb |= BIT(VMDQ_P(0) % 32);4617 IXGBE_WRITE_REG(hw, reg_offset, vlvfb);4618 }4619 4620 /* Set all bits in the VLAN filter table array */4621 for (i = hw->mac.vft_size; i--;)4622 IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), ~0U);4623}4624 4625#define VFTA_BLOCK_SIZE 84626static void ixgbe_scrub_vfta(struct ixgbe_adapter *adapter, u32 vfta_offset)4627{4628 struct ixgbe_hw *hw = &adapter->hw;4629 u32 vfta[VFTA_BLOCK_SIZE] = { 0 };4630 u32 vid_start = vfta_offset * 32;4631 u32 vid_end = vid_start + (VFTA_BLOCK_SIZE * 32);4632 u32 i, vid, word, bits;4633 4634 for (i = IXGBE_VLVF_ENTRIES; --i;) {4635 u32 vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(i));4636 4637 /* pull VLAN ID from VLVF */4638 vid = vlvf & VLAN_VID_MASK;4639 4640 /* only concern outselves with a certain range */4641 if (vid < vid_start || vid >= vid_end)4642 continue;4643 4644 if (vlvf) {4645 /* record VLAN ID in VFTA */4646 vfta[(vid - vid_start) / 32] |= BIT(vid % 32);4647 4648 /* if PF is part of this then continue */4649 if (test_bit(vid, adapter->active_vlans))4650 continue;4651 }4652 4653 /* remove PF from the pool */4654 word = i * 2 + VMDQ_P(0) / 32;4655 bits = ~BIT(VMDQ_P(0) % 32);4656 bits &= IXGBE_READ_REG(hw, IXGBE_VLVFB(word));4657 IXGBE_WRITE_REG(hw, IXGBE_VLVFB(word), bits);4658 }4659 4660 /* extract values from active_vlans and write back to VFTA */4661 for (i = VFTA_BLOCK_SIZE; i--;) {4662 vid = (vfta_offset + i) * 32;4663 word = vid / BITS_PER_LONG;4664 bits = vid % BITS_PER_LONG;4665 4666 vfta[i] |= adapter->active_vlans[word] >> bits;4667 4668 IXGBE_WRITE_REG(hw, IXGBE_VFTA(vfta_offset + i), vfta[i]);4669 }4670}4671 4672static void ixgbe_vlan_promisc_disable(struct ixgbe_adapter *adapter)4673{4674 struct ixgbe_hw *hw = &adapter->hw;4675 u32 vlnctrl, i;4676 4677 /* Set VLAN filtering to enabled */4678 vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);4679 vlnctrl |= IXGBE_VLNCTRL_VFE;4680 IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);4681 4682 if (!(adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) ||4683 hw->mac.type == ixgbe_mac_82598EB)4684 return;4685 4686 /* We are not in VLAN promisc, nothing to do */4687 if (!(adapter->flags2 & IXGBE_FLAG2_VLAN_PROMISC))4688 return;4689 4690 /* Set flag so we don't redo unnecessary work */4691 adapter->flags2 &= ~IXGBE_FLAG2_VLAN_PROMISC;4692 4693 for (i = 0; i < hw->mac.vft_size; i += VFTA_BLOCK_SIZE)4694 ixgbe_scrub_vfta(adapter, i);4695}4696 4697static void ixgbe_restore_vlan(struct ixgbe_adapter *adapter)4698{4699 u16 vid = 1;4700 4701 ixgbe_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0);4702 4703 for_each_set_bit_from(vid, adapter->active_vlans, VLAN_N_VID)4704 ixgbe_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);4705}4706 4707/**4708 * ixgbe_write_mc_addr_list - write multicast addresses to MTA4709 * @netdev: network interface device structure4710 *4711 * Writes multicast address list to the MTA hash table.4712 * Returns: -ENOMEM on failure4713 * 0 on no addresses written4714 * X on writing X addresses to MTA4715 **/4716static int ixgbe_write_mc_addr_list(struct net_device *netdev)4717{4718 struct ixgbe_adapter *adapter = netdev_priv(netdev);4719 struct ixgbe_hw *hw = &adapter->hw;4720 4721 if (!netif_running(netdev))4722 return 0;4723 4724 if (hw->mac.ops.update_mc_addr_list)4725 hw->mac.ops.update_mc_addr_list(hw, netdev);4726 else4727 return -ENOMEM;4728 4729#ifdef CONFIG_PCI_IOV4730 ixgbe_restore_vf_multicasts(adapter);4731#endif4732 4733 return netdev_mc_count(netdev);4734}4735 4736#ifdef CONFIG_PCI_IOV4737void ixgbe_full_sync_mac_table(struct ixgbe_adapter *adapter)4738{4739 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];4740 struct ixgbe_hw *hw = &adapter->hw;4741 int i;4742 4743 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {4744 mac_table->state &= ~IXGBE_MAC_STATE_MODIFIED;4745 4746 if (mac_table->state & IXGBE_MAC_STATE_IN_USE)4747 hw->mac.ops.set_rar(hw, i,4748 mac_table->addr,4749 mac_table->pool,4750 IXGBE_RAH_AV);4751 else4752 hw->mac.ops.clear_rar(hw, i);4753 }4754}4755 4756#endif4757static void ixgbe_sync_mac_table(struct ixgbe_adapter *adapter)4758{4759 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];4760 struct ixgbe_hw *hw = &adapter->hw;4761 int i;4762 4763 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {4764 if (!(mac_table->state & IXGBE_MAC_STATE_MODIFIED))4765 continue;4766 4767 mac_table->state &= ~IXGBE_MAC_STATE_MODIFIED;4768 4769 if (mac_table->state & IXGBE_MAC_STATE_IN_USE)4770 hw->mac.ops.set_rar(hw, i,4771 mac_table->addr,4772 mac_table->pool,4773 IXGBE_RAH_AV);4774 else4775 hw->mac.ops.clear_rar(hw, i);4776 }4777}4778 4779static void ixgbe_flush_sw_mac_table(struct ixgbe_adapter *adapter)4780{4781 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];4782 struct ixgbe_hw *hw = &adapter->hw;4783 int i;4784 4785 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {4786 mac_table->state |= IXGBE_MAC_STATE_MODIFIED;4787 mac_table->state &= ~IXGBE_MAC_STATE_IN_USE;4788 }4789 4790 ixgbe_sync_mac_table(adapter);4791}4792 4793static int ixgbe_available_rars(struct ixgbe_adapter *adapter, u16 pool)4794{4795 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];4796 struct ixgbe_hw *hw = &adapter->hw;4797 int i, count = 0;4798 4799 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {4800 /* do not count default RAR as available */4801 if (mac_table->state & IXGBE_MAC_STATE_DEFAULT)4802 continue;4803 4804 /* only count unused and addresses that belong to us */4805 if (mac_table->state & IXGBE_MAC_STATE_IN_USE) {4806 if (mac_table->pool != pool)4807 continue;4808 }4809 4810 count++;4811 }4812 4813 return count;4814}4815 4816/* this function destroys the first RAR entry */4817static void ixgbe_mac_set_default_filter(struct ixgbe_adapter *adapter)4818{4819 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];4820 struct ixgbe_hw *hw = &adapter->hw;4821 4822 memcpy(&mac_table->addr, hw->mac.addr, ETH_ALEN);4823 mac_table->pool = VMDQ_P(0);4824 4825 mac_table->state = IXGBE_MAC_STATE_DEFAULT | IXGBE_MAC_STATE_IN_USE;4826 4827 hw->mac.ops.set_rar(hw, 0, mac_table->addr, mac_table->pool,4828 IXGBE_RAH_AV);4829}4830 4831int ixgbe_add_mac_filter(struct ixgbe_adapter *adapter,4832 const u8 *addr, u16 pool)4833{4834 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];4835 struct ixgbe_hw *hw = &adapter->hw;4836 int i;4837 4838 if (is_zero_ether_addr(addr))4839 return -EINVAL;4840 4841 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {4842 if (mac_table->state & IXGBE_MAC_STATE_IN_USE)4843 continue;4844 4845 ether_addr_copy(mac_table->addr, addr);4846 mac_table->pool = pool;4847 4848 mac_table->state |= IXGBE_MAC_STATE_MODIFIED |4849 IXGBE_MAC_STATE_IN_USE;4850 4851 ixgbe_sync_mac_table(adapter);4852 4853 return i;4854 }4855 4856 return -ENOMEM;4857}4858 4859int ixgbe_del_mac_filter(struct ixgbe_adapter *adapter,4860 const u8 *addr, u16 pool)4861{4862 struct ixgbe_mac_addr *mac_table = &adapter->mac_table[0];4863 struct ixgbe_hw *hw = &adapter->hw;4864 int i;4865 4866 if (is_zero_ether_addr(addr))4867 return -EINVAL;4868 4869 /* search table for addr, if found clear IN_USE flag and sync */4870 for (i = 0; i < hw->mac.num_rar_entries; i++, mac_table++) {4871 /* we can only delete an entry if it is in use */4872 if (!(mac_table->state & IXGBE_MAC_STATE_IN_USE))4873 continue;4874 /* we only care about entries that belong to the given pool */4875 if (mac_table->pool != pool)4876 continue;4877 /* we only care about a specific MAC address */4878 if (!ether_addr_equal(addr, mac_table->addr))4879 continue;4880 4881 mac_table->state |= IXGBE_MAC_STATE_MODIFIED;4882 mac_table->state &= ~IXGBE_MAC_STATE_IN_USE;4883 4884 ixgbe_sync_mac_table(adapter);4885 4886 return 0;4887 }4888 4889 return -ENOMEM;4890}4891 4892static int ixgbe_uc_sync(struct net_device *netdev, const unsigned char *addr)4893{4894 struct ixgbe_adapter *adapter = netdev_priv(netdev);4895 int ret;4896 4897 ret = ixgbe_add_mac_filter(adapter, addr, VMDQ_P(0));4898 4899 return min_t(int, ret, 0);4900}4901 4902static int ixgbe_uc_unsync(struct net_device *netdev, const unsigned char *addr)4903{4904 struct ixgbe_adapter *adapter = netdev_priv(netdev);4905 4906 ixgbe_del_mac_filter(adapter, addr, VMDQ_P(0));4907 4908 return 0;4909}4910 4911/**4912 * ixgbe_set_rx_mode - Unicast, Multicast and Promiscuous mode set4913 * @netdev: network interface device structure4914 *4915 * The set_rx_method entry point is called whenever the unicast/multicast4916 * address list or the network interface flags are updated. This routine is4917 * responsible for configuring the hardware for proper unicast, multicast and4918 * promiscuous mode.4919 **/4920void ixgbe_set_rx_mode(struct net_device *netdev)4921{4922 struct ixgbe_adapter *adapter = netdev_priv(netdev);4923 struct ixgbe_hw *hw = &adapter->hw;4924 u32 fctrl, vmolr = IXGBE_VMOLR_BAM | IXGBE_VMOLR_AUPE;4925 netdev_features_t features = netdev->features;4926 int count;4927 4928 /* Check for Promiscuous and All Multicast modes */4929 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);4930 4931 /* set all bits that we expect to always be set */4932 fctrl &= ~IXGBE_FCTRL_SBP; /* disable store-bad-packets */4933 fctrl |= IXGBE_FCTRL_BAM;4934 fctrl |= IXGBE_FCTRL_DPF; /* discard pause frames when FC enabled */4935 fctrl |= IXGBE_FCTRL_PMCF;4936 4937 /* clear the bits we are changing the status of */4938 fctrl &= ~(IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);4939 if (netdev->flags & IFF_PROMISC) {4940 hw->addr_ctrl.user_set_promisc = true;4941 fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);4942 vmolr |= IXGBE_VMOLR_MPE;4943 features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;4944 } else {4945 if (netdev->flags & IFF_ALLMULTI) {4946 fctrl |= IXGBE_FCTRL_MPE;4947 vmolr |= IXGBE_VMOLR_MPE;4948 }4949 hw->addr_ctrl.user_set_promisc = false;4950 }4951 4952 /*4953 * Write addresses to available RAR registers, if there is not4954 * sufficient space to store all the addresses then enable4955 * unicast promiscuous mode4956 */4957 if (__dev_uc_sync(netdev, ixgbe_uc_sync, ixgbe_uc_unsync)) {4958 fctrl |= IXGBE_FCTRL_UPE;4959 vmolr |= IXGBE_VMOLR_ROPE;4960 }4961 4962 /* Write addresses to the MTA, if the attempt fails4963 * then we should just turn on promiscuous mode so4964 * that we can at least receive multicast traffic4965 */4966 count = ixgbe_write_mc_addr_list(netdev);4967 if (count < 0) {4968 fctrl |= IXGBE_FCTRL_MPE;4969 vmolr |= IXGBE_VMOLR_MPE;4970 } else if (count) {4971 vmolr |= IXGBE_VMOLR_ROMPE;4972 }4973 4974 if (hw->mac.type != ixgbe_mac_82598EB) {4975 vmolr |= IXGBE_READ_REG(hw, IXGBE_VMOLR(VMDQ_P(0))) &4976 ~(IXGBE_VMOLR_MPE | IXGBE_VMOLR_ROMPE |4977 IXGBE_VMOLR_ROPE);4978 IXGBE_WRITE_REG(hw, IXGBE_VMOLR(VMDQ_P(0)), vmolr);4979 }4980 4981 /* This is useful for sniffing bad packets. */4982 if (features & NETIF_F_RXALL) {4983 /* UPE and MPE will be handled by normal PROMISC logic4984 * in e1000e_set_rx_mode */4985 fctrl |= (IXGBE_FCTRL_SBP | /* Receive bad packets */4986 IXGBE_FCTRL_BAM | /* RX All Bcast Pkts */4987 IXGBE_FCTRL_PMCF); /* RX All MAC Ctrl Pkts */4988 4989 fctrl &= ~(IXGBE_FCTRL_DPF);4990 /* NOTE: VLAN filtering is disabled by setting PROMISC */4991 }4992 4993 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);4994 4995 if (features & NETIF_F_HW_VLAN_CTAG_RX)4996 ixgbe_vlan_strip_enable(adapter);4997 else4998 ixgbe_vlan_strip_disable(adapter);4999 5000 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)5001 ixgbe_vlan_promisc_disable(adapter);5002 else5003 ixgbe_vlan_promisc_enable(adapter);5004}5005 5006static void ixgbe_napi_enable_all(struct ixgbe_adapter *adapter)5007{5008 int q_idx;5009 5010 for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++)5011 napi_enable(&adapter->q_vector[q_idx]->napi);5012}5013 5014static void ixgbe_napi_disable_all(struct ixgbe_adapter *adapter)5015{5016 int q_idx;5017 5018 for (q_idx = 0; q_idx < adapter->num_q_vectors; q_idx++)5019 napi_disable(&adapter->q_vector[q_idx]->napi);5020}5021 5022static int ixgbe_udp_tunnel_sync(struct net_device *dev, unsigned int table)5023{5024 struct ixgbe_adapter *adapter = netdev_priv(dev);5025 struct ixgbe_hw *hw = &adapter->hw;5026 struct udp_tunnel_info ti;5027 5028 udp_tunnel_nic_get_port(dev, table, 0, &ti);5029 if (ti.type == UDP_TUNNEL_TYPE_VXLAN)5030 adapter->vxlan_port = ti.port;5031 else5032 adapter->geneve_port = ti.port;5033 5034 IXGBE_WRITE_REG(hw, IXGBE_VXLANCTRL,5035 ntohs(adapter->vxlan_port) |5036 ntohs(adapter->geneve_port) <<5037 IXGBE_VXLANCTRL_GENEVE_UDPPORT_SHIFT);5038 return 0;5039}5040 5041static const struct udp_tunnel_nic_info ixgbe_udp_tunnels_x550 = {5042 .sync_table = ixgbe_udp_tunnel_sync,5043 .flags = UDP_TUNNEL_NIC_INFO_IPV4_ONLY,5044 .tables = {5045 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },5046 },5047};5048 5049static const struct udp_tunnel_nic_info ixgbe_udp_tunnels_x550em_a = {5050 .sync_table = ixgbe_udp_tunnel_sync,5051 .flags = UDP_TUNNEL_NIC_INFO_IPV4_ONLY,5052 .tables = {5053 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, },5054 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_GENEVE, },5055 },5056};5057 5058#ifdef CONFIG_IXGBE_DCB5059/**5060 * ixgbe_configure_dcb - Configure DCB hardware5061 * @adapter: ixgbe adapter struct5062 *5063 * This is called by the driver on open to configure the DCB hardware.5064 * This is also called by the gennetlink interface when reconfiguring5065 * the DCB state.5066 */5067static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter)5068{5069 struct ixgbe_hw *hw = &adapter->hw;5070 int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN;5071 5072 if (!(adapter->flags & IXGBE_FLAG_DCB_ENABLED)) {5073 if (hw->mac.type == ixgbe_mac_82598EB)5074 netif_set_tso_max_size(adapter->netdev, 65536);5075 return;5076 }5077 5078 if (hw->mac.type == ixgbe_mac_82598EB)5079 netif_set_tso_max_size(adapter->netdev, 32768);5080 5081#ifdef IXGBE_FCOE5082 if (adapter->netdev->fcoe_mtu)5083 max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE);5084#endif5085 5086 /* reconfigure the hardware */5087 if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE) {5088 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,5089 DCB_TX_CONFIG);5090 ixgbe_dcb_calculate_tc_credits(hw, &adapter->dcb_cfg, max_frame,5091 DCB_RX_CONFIG);5092 ixgbe_dcb_hw_config(hw, &adapter->dcb_cfg);5093 } else if (adapter->ixgbe_ieee_ets && adapter->ixgbe_ieee_pfc) {5094 ixgbe_dcb_hw_ets(&adapter->hw,5095 adapter->ixgbe_ieee_ets,5096 max_frame);5097 ixgbe_dcb_hw_pfc_config(&adapter->hw,5098 adapter->ixgbe_ieee_pfc->pfc_en,5099 adapter->ixgbe_ieee_ets->prio_tc);5100 }5101 5102 /* Enable RSS Hash per TC */5103 if (hw->mac.type != ixgbe_mac_82598EB) {5104 u32 msb = 0;5105 u16 rss_i = adapter->ring_feature[RING_F_RSS].indices - 1;5106 5107 while (rss_i) {5108 msb++;5109 rss_i >>= 1;5110 }5111 5112 /* write msb to all 8 TCs in one write */5113 IXGBE_WRITE_REG(hw, IXGBE_RQTC, msb * 0x11111111);5114 }5115}5116#endif5117 5118/* Additional bittime to account for IXGBE framing */5119#define IXGBE_ETH_FRAMING 205120 5121/**5122 * ixgbe_hpbthresh - calculate high water mark for flow control5123 *5124 * @adapter: board private structure to calculate for5125 * @pb: packet buffer to calculate5126 */5127static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb)5128{5129 struct ixgbe_hw *hw = &adapter->hw;5130 struct net_device *dev = adapter->netdev;5131 int link, tc, kb, marker;5132 u32 dv_id, rx_pba;5133 5134 /* Calculate max LAN frame size */5135 tc = link = dev->mtu + ETH_HLEN + ETH_FCS_LEN + IXGBE_ETH_FRAMING;5136 5137#ifdef IXGBE_FCOE5138 /* FCoE traffic class uses FCOE jumbo frames */5139 if (dev->fcoe_mtu && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE &&5140 (pb == ixgbe_fcoe_get_tc(adapter)))5141 tc = IXGBE_FCOE_JUMBO_FRAME_SIZE;5142#endif5143 5144 /* Calculate delay value for device */5145 switch (hw->mac.type) {5146 case ixgbe_mac_X540:5147 case ixgbe_mac_X550:5148 case ixgbe_mac_X550EM_x:5149 case ixgbe_mac_x550em_a:5150 dv_id = IXGBE_DV_X540(link, tc);5151 break;5152 default:5153 dv_id = IXGBE_DV(link, tc);5154 break;5155 }5156 5157 /* Loopback switch introduces additional latency */5158 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)5159 dv_id += IXGBE_B2BT(tc);5160 5161 /* Delay value is calculated in bit times convert to KB */5162 kb = IXGBE_BT2KB(dv_id);5163 rx_pba = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(pb)) >> 10;5164 5165 marker = rx_pba - kb;5166 5167 /* It is possible that the packet buffer is not large enough5168 * to provide required headroom. In this case throw an error5169 * to user and a do the best we can.5170 */5171 if (marker < 0) {5172 e_warn(drv, "Packet Buffer(%i) can not provide enough"5173 "headroom to support flow control."5174 "Decrease MTU or number of traffic classes\n", pb);5175 marker = tc + 1;5176 }5177 5178 return marker;5179}5180 5181/**5182 * ixgbe_lpbthresh - calculate low water mark for flow control5183 *5184 * @adapter: board private structure to calculate for5185 * @pb: packet buffer to calculate5186 */5187static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter, int pb)5188{5189 struct ixgbe_hw *hw = &adapter->hw;5190 struct net_device *dev = adapter->netdev;5191 int tc;5192 u32 dv_id;5193 5194 /* Calculate max LAN frame size */5195 tc = dev->mtu + ETH_HLEN + ETH_FCS_LEN;5196 5197#ifdef IXGBE_FCOE5198 /* FCoE traffic class uses FCOE jumbo frames */5199 if (dev->fcoe_mtu && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE &&5200 (pb == netdev_get_prio_tc_map(dev, adapter->fcoe.up)))5201 tc = IXGBE_FCOE_JUMBO_FRAME_SIZE;5202#endif5203 5204 /* Calculate delay value for device */5205 switch (hw->mac.type) {5206 case ixgbe_mac_X540:5207 case ixgbe_mac_X550:5208 case ixgbe_mac_X550EM_x:5209 case ixgbe_mac_x550em_a:5210 dv_id = IXGBE_LOW_DV_X540(tc);5211 break;5212 default:5213 dv_id = IXGBE_LOW_DV(tc);5214 break;5215 }5216 5217 /* Delay value is calculated in bit times convert to KB */5218 return IXGBE_BT2KB(dv_id);5219}5220 5221/*5222 * ixgbe_pbthresh_setup - calculate and setup high low water marks5223 */5224static void ixgbe_pbthresh_setup(struct ixgbe_adapter *adapter)5225{5226 struct ixgbe_hw *hw = &adapter->hw;5227 int num_tc = adapter->hw_tcs;5228 int i;5229 5230 if (!num_tc)5231 num_tc = 1;5232 5233 for (i = 0; i < num_tc; i++) {5234 hw->fc.high_water[i] = ixgbe_hpbthresh(adapter, i);5235 hw->fc.low_water[i] = ixgbe_lpbthresh(adapter, i);5236 5237 /* Low water marks must not be larger than high water marks */5238 if (hw->fc.low_water[i] > hw->fc.high_water[i])5239 hw->fc.low_water[i] = 0;5240 }5241 5242 for (; i < MAX_TRAFFIC_CLASS; i++)5243 hw->fc.high_water[i] = 0;5244}5245 5246static void ixgbe_configure_pb(struct ixgbe_adapter *adapter)5247{5248 struct ixgbe_hw *hw = &adapter->hw;5249 int hdrm;5250 u8 tc = adapter->hw_tcs;5251 5252 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE ||5253 adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)5254 hdrm = 32 << adapter->fdir_pballoc;5255 else5256 hdrm = 0;5257 5258 hw->mac.ops.set_rxpba(hw, tc, hdrm, PBA_STRATEGY_EQUAL);5259 ixgbe_pbthresh_setup(adapter);5260}5261 5262static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter)5263{5264 struct ixgbe_hw *hw = &adapter->hw;5265 struct hlist_node *node2;5266 struct ixgbe_fdir_filter *filter;5267 u8 queue;5268 5269 spin_lock(&adapter->fdir_perfect_lock);5270 5271 if (!hlist_empty(&adapter->fdir_filter_list))5272 ixgbe_fdir_set_input_mask_82599(hw, &adapter->fdir_mask);5273 5274 hlist_for_each_entry_safe(filter, node2,5275 &adapter->fdir_filter_list, fdir_node) {5276 if (filter->action == IXGBE_FDIR_DROP_QUEUE) {5277 queue = IXGBE_FDIR_DROP_QUEUE;5278 } else {5279 u32 ring = ethtool_get_flow_spec_ring(filter->action);5280 u8 vf = ethtool_get_flow_spec_ring_vf(filter->action);5281 5282 if (!vf && (ring >= adapter->num_rx_queues)) {5283 e_err(drv, "FDIR restore failed without VF, ring: %u\n",5284 ring);5285 continue;5286 } else if (vf &&5287 ((vf > adapter->num_vfs) ||5288 ring >= adapter->num_rx_queues_per_pool)) {5289 e_err(drv, "FDIR restore failed with VF, vf: %hhu, ring: %u\n",5290 vf, ring);5291 continue;5292 }5293 5294 /* Map the ring onto the absolute queue index */5295 if (!vf)5296 queue = adapter->rx_ring[ring]->reg_idx;5297 else5298 queue = ((vf - 1) *5299 adapter->num_rx_queues_per_pool) + ring;5300 }5301 5302 ixgbe_fdir_write_perfect_filter_82599(hw,5303 &filter->filter, filter->sw_idx, queue);5304 }5305 5306 spin_unlock(&adapter->fdir_perfect_lock);5307}5308 5309/**5310 * ixgbe_clean_rx_ring - Free Rx Buffers per Queue5311 * @rx_ring: ring to free buffers from5312 **/5313static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring)5314{5315 u16 i = rx_ring->next_to_clean;5316 struct ixgbe_rx_buffer *rx_buffer = &rx_ring->rx_buffer_info[i];5317 5318 if (rx_ring->xsk_pool) {5319 ixgbe_xsk_clean_rx_ring(rx_ring);5320 goto skip_free;5321 }5322 5323 /* Free all the Rx ring sk_buffs */5324 while (i != rx_ring->next_to_alloc) {5325 if (rx_buffer->skb) {5326 struct sk_buff *skb = rx_buffer->skb;5327 if (IXGBE_CB(skb)->page_released)5328 dma_unmap_page_attrs(rx_ring->dev,5329 IXGBE_CB(skb)->dma,5330 ixgbe_rx_pg_size(rx_ring),5331 DMA_FROM_DEVICE,5332 IXGBE_RX_DMA_ATTR);5333 dev_kfree_skb(skb);5334 }5335 5336 /* Invalidate cache lines that may have been written to by5337 * device so that we avoid corrupting memory.5338 */5339 dma_sync_single_range_for_cpu(rx_ring->dev,5340 rx_buffer->dma,5341 rx_buffer->page_offset,5342 ixgbe_rx_bufsz(rx_ring),5343 DMA_FROM_DEVICE);5344 5345 /* free resources associated with mapping */5346 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,5347 ixgbe_rx_pg_size(rx_ring),5348 DMA_FROM_DEVICE,5349 IXGBE_RX_DMA_ATTR);5350 __page_frag_cache_drain(rx_buffer->page,5351 rx_buffer->pagecnt_bias);5352 5353 i++;5354 rx_buffer++;5355 if (i == rx_ring->count) {5356 i = 0;5357 rx_buffer = rx_ring->rx_buffer_info;5358 }5359 }5360 5361skip_free:5362 rx_ring->next_to_alloc = 0;5363 rx_ring->next_to_clean = 0;5364 rx_ring->next_to_use = 0;5365}5366 5367static int ixgbe_fwd_ring_up(struct ixgbe_adapter *adapter,5368 struct ixgbe_fwd_adapter *accel)5369{5370 u16 rss_i = adapter->ring_feature[RING_F_RSS].indices;5371 int num_tc = netdev_get_num_tc(adapter->netdev);5372 struct net_device *vdev = accel->netdev;5373 int i, baseq, err;5374 5375 baseq = accel->pool * adapter->num_rx_queues_per_pool;5376 netdev_dbg(vdev, "pool %i:%i queues %i:%i\n",5377 accel->pool, adapter->num_rx_pools,5378 baseq, baseq + adapter->num_rx_queues_per_pool);5379 5380 accel->rx_base_queue = baseq;5381 accel->tx_base_queue = baseq;5382 5383 /* record configuration for macvlan interface in vdev */5384 for (i = 0; i < num_tc; i++)5385 netdev_bind_sb_channel_queue(adapter->netdev, vdev,5386 i, rss_i, baseq + (rss_i * i));5387 5388 for (i = 0; i < adapter->num_rx_queues_per_pool; i++)5389 adapter->rx_ring[baseq + i]->netdev = vdev;5390 5391 /* Guarantee all rings are updated before we update the5392 * MAC address filter.5393 */5394 wmb();5395 5396 /* ixgbe_add_mac_filter will return an index if it succeeds, so we5397 * need to only treat it as an error value if it is negative.5398 */5399 err = ixgbe_add_mac_filter(adapter, vdev->dev_addr,5400 VMDQ_P(accel->pool));5401 if (err >= 0)5402 return 0;5403 5404 /* if we cannot add the MAC rule then disable the offload */5405 macvlan_release_l2fw_offload(vdev);5406 5407 for (i = 0; i < adapter->num_rx_queues_per_pool; i++)5408 adapter->rx_ring[baseq + i]->netdev = NULL;5409 5410 netdev_err(vdev, "L2FW offload disabled due to L2 filter error\n");5411 5412 /* unbind the queues and drop the subordinate channel config */5413 netdev_unbind_sb_channel(adapter->netdev, vdev);5414 netdev_set_sb_channel(vdev, 0);5415 5416 clear_bit(accel->pool, adapter->fwd_bitmask);5417 kfree(accel);5418 5419 return err;5420}5421 5422static int ixgbe_macvlan_up(struct net_device *vdev,5423 struct netdev_nested_priv *priv)5424{5425 struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)priv->data;5426 struct ixgbe_fwd_adapter *accel;5427 5428 if (!netif_is_macvlan(vdev))5429 return 0;5430 5431 accel = macvlan_accel_priv(vdev);5432 if (!accel)5433 return 0;5434 5435 ixgbe_fwd_ring_up(adapter, accel);5436 5437 return 0;5438}5439 5440static void ixgbe_configure_dfwd(struct ixgbe_adapter *adapter)5441{5442 struct netdev_nested_priv priv = {5443 .data = (void *)adapter,5444 };5445 5446 netdev_walk_all_upper_dev_rcu(adapter->netdev,5447 ixgbe_macvlan_up, &priv);5448}5449 5450static void ixgbe_configure(struct ixgbe_adapter *adapter)5451{5452 struct ixgbe_hw *hw = &adapter->hw;5453 5454 ixgbe_configure_pb(adapter);5455#ifdef CONFIG_IXGBE_DCB5456 ixgbe_configure_dcb(adapter);5457#endif5458 /*5459 * We must restore virtualization before VLANs or else5460 * the VLVF registers will not be populated5461 */5462 ixgbe_configure_virtualization(adapter);5463 5464 ixgbe_set_rx_mode(adapter->netdev);5465 ixgbe_restore_vlan(adapter);5466 ixgbe_ipsec_restore(adapter);5467 5468 switch (hw->mac.type) {5469 case ixgbe_mac_82599EB:5470 case ixgbe_mac_X540:5471 hw->mac.ops.disable_rx_buff(hw);5472 break;5473 default:5474 break;5475 }5476 5477 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {5478 ixgbe_init_fdir_signature_82599(&adapter->hw,5479 adapter->fdir_pballoc);5480 } else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) {5481 ixgbe_init_fdir_perfect_82599(&adapter->hw,5482 adapter->fdir_pballoc);5483 ixgbe_fdir_filter_restore(adapter);5484 }5485 5486 switch (hw->mac.type) {5487 case ixgbe_mac_82599EB:5488 case ixgbe_mac_X540:5489 hw->mac.ops.enable_rx_buff(hw);5490 break;5491 default:5492 break;5493 }5494 5495#ifdef CONFIG_IXGBE_DCA5496 /* configure DCA */5497 if (adapter->flags & IXGBE_FLAG_DCA_CAPABLE)5498 ixgbe_setup_dca(adapter);5499#endif /* CONFIG_IXGBE_DCA */5500 5501#ifdef IXGBE_FCOE5502 /* configure FCoE L2 filters, redirection table, and Rx control */5503 ixgbe_configure_fcoe(adapter);5504 5505#endif /* IXGBE_FCOE */5506 ixgbe_configure_tx(adapter);5507 ixgbe_configure_rx(adapter);5508 ixgbe_configure_dfwd(adapter);5509}5510 5511/**5512 * ixgbe_sfp_link_config - set up SFP+ link5513 * @adapter: pointer to private adapter struct5514 **/5515static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter)5516{5517 /*5518 * We are assuming the worst case scenario here, and that5519 * is that an SFP was inserted/removed after the reset5520 * but before SFP detection was enabled. As such the best5521 * solution is to just start searching as soon as we start5522 */5523 if (adapter->hw.mac.type == ixgbe_mac_82598EB)5524 adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;5525 5526 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;5527 adapter->sfp_poll_time = 0;5528}5529 5530/**5531 * ixgbe_non_sfp_link_config - set up non-SFP+ link5532 * @hw: pointer to private hardware struct5533 *5534 * Returns 0 on success, negative on failure5535 **/5536static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw)5537{5538 u32 speed;5539 bool autoneg, link_up = false;5540 int ret = -EIO;5541 5542 if (hw->mac.ops.check_link)5543 ret = hw->mac.ops.check_link(hw, &speed, &link_up, false);5544 5545 if (ret)5546 return ret;5547 5548 speed = hw->phy.autoneg_advertised;5549 if (!speed && hw->mac.ops.get_link_capabilities) {5550 ret = hw->mac.ops.get_link_capabilities(hw, &speed,5551 &autoneg);5552 /* remove NBASE-T speeds from default autonegotiation5553 * to accommodate broken network switches in the field5554 * which cannot cope with advertised NBASE-T speeds5555 */5556 speed &= ~(IXGBE_LINK_SPEED_5GB_FULL |5557 IXGBE_LINK_SPEED_2_5GB_FULL);5558 }5559 5560 if (ret)5561 return ret;5562 5563 if (hw->mac.ops.setup_link)5564 ret = hw->mac.ops.setup_link(hw, speed, link_up);5565 5566 return ret;5567}5568 5569/**5570 * ixgbe_clear_vf_stats_counters - Clear out VF stats after reset5571 * @adapter: board private structure5572 *5573 * On a reset we need to clear out the VF stats or accounting gets5574 * messed up because they're not clear on read.5575 **/5576static void ixgbe_clear_vf_stats_counters(struct ixgbe_adapter *adapter)5577{5578 struct ixgbe_hw *hw = &adapter->hw;5579 int i;5580 5581 for (i = 0; i < adapter->num_vfs; i++) {5582 adapter->vfinfo[i].last_vfstats.gprc =5583 IXGBE_READ_REG(hw, IXGBE_PVFGPRC(i));5584 adapter->vfinfo[i].saved_rst_vfstats.gprc +=5585 adapter->vfinfo[i].vfstats.gprc;5586 adapter->vfinfo[i].vfstats.gprc = 0;5587 adapter->vfinfo[i].last_vfstats.gptc =5588 IXGBE_READ_REG(hw, IXGBE_PVFGPTC(i));5589 adapter->vfinfo[i].saved_rst_vfstats.gptc +=5590 adapter->vfinfo[i].vfstats.gptc;5591 adapter->vfinfo[i].vfstats.gptc = 0;5592 adapter->vfinfo[i].last_vfstats.gorc =5593 IXGBE_READ_REG(hw, IXGBE_PVFGORC_LSB(i));5594 adapter->vfinfo[i].saved_rst_vfstats.gorc +=5595 adapter->vfinfo[i].vfstats.gorc;5596 adapter->vfinfo[i].vfstats.gorc = 0;5597 adapter->vfinfo[i].last_vfstats.gotc =5598 IXGBE_READ_REG(hw, IXGBE_PVFGOTC_LSB(i));5599 adapter->vfinfo[i].saved_rst_vfstats.gotc +=5600 adapter->vfinfo[i].vfstats.gotc;5601 adapter->vfinfo[i].vfstats.gotc = 0;5602 adapter->vfinfo[i].last_vfstats.mprc =5603 IXGBE_READ_REG(hw, IXGBE_PVFMPRC(i));5604 adapter->vfinfo[i].saved_rst_vfstats.mprc +=5605 adapter->vfinfo[i].vfstats.mprc;5606 adapter->vfinfo[i].vfstats.mprc = 0;5607 }5608}5609 5610static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter)5611{5612 struct ixgbe_hw *hw = &adapter->hw;5613 u32 gpie = 0;5614 5615 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {5616 gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT |5617 IXGBE_GPIE_OCD;5618 gpie |= IXGBE_GPIE_EIAME;5619 /*5620 * use EIAM to auto-mask when MSI-X interrupt is asserted5621 * this saves a register write for every interrupt5622 */5623 switch (hw->mac.type) {5624 case ixgbe_mac_82598EB:5625 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);5626 break;5627 case ixgbe_mac_82599EB:5628 case ixgbe_mac_X540:5629 case ixgbe_mac_X550:5630 case ixgbe_mac_X550EM_x:5631 case ixgbe_mac_x550em_a:5632 default:5633 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);5634 IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);5635 break;5636 }5637 } else {5638 /* legacy interrupts, use EIAM to auto-mask when reading EICR,5639 * specifically only auto mask tx and rx interrupts */5640 IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);5641 }5642 5643 /* XXX: to interrupt immediately for EICS writes, enable this */5644 /* gpie |= IXGBE_GPIE_EIMEN; */5645 5646 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {5647 gpie &= ~IXGBE_GPIE_VTMODE_MASK;5648 5649 switch (adapter->ring_feature[RING_F_VMDQ].mask) {5650 case IXGBE_82599_VMDQ_8Q_MASK:5651 gpie |= IXGBE_GPIE_VTMODE_16;5652 break;5653 case IXGBE_82599_VMDQ_4Q_MASK:5654 gpie |= IXGBE_GPIE_VTMODE_32;5655 break;5656 default:5657 gpie |= IXGBE_GPIE_VTMODE_64;5658 break;5659 }5660 }5661 5662 /* Enable Thermal over heat sensor interrupt */5663 if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) {5664 switch (adapter->hw.mac.type) {5665 case ixgbe_mac_82599EB:5666 gpie |= IXGBE_SDP0_GPIEN_8259X;5667 break;5668 default:5669 break;5670 }5671 }5672 5673 /* Enable fan failure interrupt */5674 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)5675 gpie |= IXGBE_SDP1_GPIEN(hw);5676 5677 switch (hw->mac.type) {5678 case ixgbe_mac_82599EB:5679 gpie |= IXGBE_SDP1_GPIEN_8259X | IXGBE_SDP2_GPIEN_8259X;5680 break;5681 case ixgbe_mac_X550EM_x:5682 case ixgbe_mac_x550em_a:5683 gpie |= IXGBE_SDP0_GPIEN_X540;5684 break;5685 default:5686 break;5687 }5688 5689 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);5690}5691 5692static void ixgbe_up_complete(struct ixgbe_adapter *adapter)5693{5694 struct ixgbe_hw *hw = &adapter->hw;5695 int err;5696 u32 ctrl_ext;5697 5698 ixgbe_get_hw_control(adapter);5699 ixgbe_setup_gpie(adapter);5700 5701 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)5702 ixgbe_configure_msix(adapter);5703 else5704 ixgbe_configure_msi_and_legacy(adapter);5705 5706 /* enable the optics for 82599 SFP+ fiber */5707 if (hw->mac.ops.enable_tx_laser)5708 hw->mac.ops.enable_tx_laser(hw);5709 5710 if (hw->phy.ops.set_phy_power)5711 hw->phy.ops.set_phy_power(hw, true);5712 5713 smp_mb__before_atomic();5714 clear_bit(__IXGBE_DOWN, &adapter->state);5715 ixgbe_napi_enable_all(adapter);5716 5717 if (ixgbe_is_sfp(hw)) {5718 ixgbe_sfp_link_config(adapter);5719 } else {5720 err = ixgbe_non_sfp_link_config(hw);5721 if (err)5722 e_err(probe, "link_config FAILED %d\n", err);5723 }5724 5725 /* clear any pending interrupts, may auto mask */5726 IXGBE_READ_REG(hw, IXGBE_EICR);5727 ixgbe_irq_enable(adapter, true, true);5728 5729 /*5730 * If this adapter has a fan, check to see if we had a failure5731 * before we enabled the interrupt.5732 */5733 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {5734 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);5735 if (esdp & IXGBE_ESDP_SDP1)5736 e_crit(drv, "Fan has stopped, replace the adapter\n");5737 }5738 5739 /* bring the link up in the watchdog, this could race with our first5740 * link up interrupt but shouldn't be a problem */5741 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;5742 adapter->link_check_timeout = jiffies;5743 mod_timer(&adapter->service_timer, jiffies);5744 5745 ixgbe_clear_vf_stats_counters(adapter);5746 /* Set PF Reset Done bit so PF/VF Mail Ops can work */5747 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);5748 ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;5749 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);5750 5751 /* update setting rx tx for all active vfs */5752 ixgbe_set_all_vfs(adapter);5753}5754 5755void ixgbe_reinit_locked(struct ixgbe_adapter *adapter)5756{5757 /* put off any impending NetWatchDogTimeout */5758 netif_trans_update(adapter->netdev);5759 5760 while (test_and_set_bit(__IXGBE_RESETTING, &adapter->state))5761 usleep_range(1000, 2000);5762 if (adapter->hw.phy.type == ixgbe_phy_fw)5763 ixgbe_watchdog_link_is_down(adapter);5764 ixgbe_down(adapter);5765 /*5766 * If SR-IOV enabled then wait a bit before bringing the adapter5767 * back up to give the VFs time to respond to the reset. The5768 * two second wait is based upon the watchdog timer cycle in5769 * the VF driver.5770 */5771 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)5772 msleep(2000);5773 ixgbe_up(adapter);5774 clear_bit(__IXGBE_RESETTING, &adapter->state);5775}5776 5777void ixgbe_up(struct ixgbe_adapter *adapter)5778{5779 /* hardware has been reset, we need to reload some things */5780 ixgbe_configure(adapter);5781 5782 ixgbe_up_complete(adapter);5783}5784 5785static unsigned long ixgbe_get_completion_timeout(struct ixgbe_adapter *adapter)5786{5787 u16 devctl2;5788 5789 pcie_capability_read_word(adapter->pdev, PCI_EXP_DEVCTL2, &devctl2);5790 5791 switch (devctl2 & IXGBE_PCIDEVCTRL2_TIMEO_MASK) {5792 case IXGBE_PCIDEVCTRL2_17_34s:5793 case IXGBE_PCIDEVCTRL2_4_8s:5794 /* For now we cap the upper limit on delay to 2 seconds5795 * as we end up going up to 34 seconds of delay in worst5796 * case timeout value.5797 */5798 case IXGBE_PCIDEVCTRL2_1_2s:5799 return 2000000ul; /* 2.0 s */5800 case IXGBE_PCIDEVCTRL2_260_520ms:5801 return 520000ul; /* 520 ms */5802 case IXGBE_PCIDEVCTRL2_65_130ms:5803 return 130000ul; /* 130 ms */5804 case IXGBE_PCIDEVCTRL2_16_32ms:5805 return 32000ul; /* 32 ms */5806 case IXGBE_PCIDEVCTRL2_1_2ms:5807 return 2000ul; /* 2 ms */5808 case IXGBE_PCIDEVCTRL2_50_100us:5809 return 100ul; /* 100 us */5810 case IXGBE_PCIDEVCTRL2_16_32ms_def:5811 return 32000ul; /* 32 ms */5812 default:5813 break;5814 }5815 5816 /* We shouldn't need to hit this path, but just in case default as5817 * though completion timeout is not supported and support 32ms.5818 */5819 return 32000ul;5820}5821 5822void ixgbe_disable_rx(struct ixgbe_adapter *adapter)5823{5824 unsigned long wait_delay, delay_interval;5825 struct ixgbe_hw *hw = &adapter->hw;5826 int i, wait_loop;5827 u32 rxdctl;5828 5829 /* disable receives */5830 hw->mac.ops.disable_rx(hw);5831 5832 if (ixgbe_removed(hw->hw_addr))5833 return;5834 5835 /* disable all enabled Rx queues */5836 for (i = 0; i < adapter->num_rx_queues; i++) {5837 struct ixgbe_ring *ring = adapter->rx_ring[i];5838 u8 reg_idx = ring->reg_idx;5839 5840 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));5841 rxdctl &= ~IXGBE_RXDCTL_ENABLE;5842 rxdctl |= IXGBE_RXDCTL_SWFLSH;5843 5844 /* write value back with RXDCTL.ENABLE bit cleared */5845 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);5846 }5847 5848 /* RXDCTL.EN may not change on 82598 if link is down, so skip it */5849 if (hw->mac.type == ixgbe_mac_82598EB &&5850 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))5851 return;5852 5853 /* Determine our minimum delay interval. We will increase this value5854 * with each subsequent test. This way if the device returns quickly5855 * we should spend as little time as possible waiting, however as5856 * the time increases we will wait for larger periods of time.5857 *5858 * The trick here is that we increase the interval using the5859 * following pattern: 1x 3x 5x 7x 9x 11x 13x 15x 17x 19x. The result5860 * of that wait is that it totals up to 100x whatever interval we5861 * choose. Since our minimum wait is 100us we can just divide the5862 * total timeout by 100 to get our minimum delay interval.5863 */5864 delay_interval = ixgbe_get_completion_timeout(adapter) / 100;5865 5866 wait_loop = IXGBE_MAX_RX_DESC_POLL;5867 wait_delay = delay_interval;5868 5869 while (wait_loop--) {5870 usleep_range(wait_delay, wait_delay + 10);5871 wait_delay += delay_interval * 2;5872 rxdctl = 0;5873 5874 /* OR together the reading of all the active RXDCTL registers,5875 * and then test the result. We need the disable to complete5876 * before we start freeing the memory and invalidating the5877 * DMA mappings.5878 */5879 for (i = 0; i < adapter->num_rx_queues; i++) {5880 struct ixgbe_ring *ring = adapter->rx_ring[i];5881 u8 reg_idx = ring->reg_idx;5882 5883 rxdctl |= IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));5884 }5885 5886 if (!(rxdctl & IXGBE_RXDCTL_ENABLE))5887 return;5888 }5889 5890 e_err(drv,5891 "RXDCTL.ENABLE for one or more queues not cleared within the polling period\n");5892}5893 5894void ixgbe_disable_tx(struct ixgbe_adapter *adapter)5895{5896 unsigned long wait_delay, delay_interval;5897 struct ixgbe_hw *hw = &adapter->hw;5898 int i, wait_loop;5899 u32 txdctl;5900 5901 if (ixgbe_removed(hw->hw_addr))5902 return;5903 5904 /* disable all enabled Tx queues */5905 for (i = 0; i < adapter->num_tx_queues; i++) {5906 struct ixgbe_ring *ring = adapter->tx_ring[i];5907 u8 reg_idx = ring->reg_idx;5908 5909 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);5910 }5911 5912 /* disable all enabled XDP Tx queues */5913 for (i = 0; i < adapter->num_xdp_queues; i++) {5914 struct ixgbe_ring *ring = adapter->xdp_ring[i];5915 u8 reg_idx = ring->reg_idx;5916 5917 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);5918 }5919 5920 /* If the link is not up there shouldn't be much in the way of5921 * pending transactions. Those that are left will be flushed out5922 * when the reset logic goes through the flush sequence to clean out5923 * the pending Tx transactions.5924 */5925 if (!(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))5926 goto dma_engine_disable;5927 5928 /* Determine our minimum delay interval. We will increase this value5929 * with each subsequent test. This way if the device returns quickly5930 * we should spend as little time as possible waiting, however as5931 * the time increases we will wait for larger periods of time.5932 *5933 * The trick here is that we increase the interval using the5934 * following pattern: 1x 3x 5x 7x 9x 11x 13x 15x 17x 19x. The result5935 * of that wait is that it totals up to 100x whatever interval we5936 * choose. Since our minimum wait is 100us we can just divide the5937 * total timeout by 100 to get our minimum delay interval.5938 */5939 delay_interval = ixgbe_get_completion_timeout(adapter) / 100;5940 5941 wait_loop = IXGBE_MAX_RX_DESC_POLL;5942 wait_delay = delay_interval;5943 5944 while (wait_loop--) {5945 usleep_range(wait_delay, wait_delay + 10);5946 wait_delay += delay_interval * 2;5947 txdctl = 0;5948 5949 /* OR together the reading of all the active TXDCTL registers,5950 * and then test the result. We need the disable to complete5951 * before we start freeing the memory and invalidating the5952 * DMA mappings.5953 */5954 for (i = 0; i < adapter->num_tx_queues; i++) {5955 struct ixgbe_ring *ring = adapter->tx_ring[i];5956 u8 reg_idx = ring->reg_idx;5957 5958 txdctl |= IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));5959 }5960 for (i = 0; i < adapter->num_xdp_queues; i++) {5961 struct ixgbe_ring *ring = adapter->xdp_ring[i];5962 u8 reg_idx = ring->reg_idx;5963 5964 txdctl |= IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));5965 }5966 5967 if (!(txdctl & IXGBE_TXDCTL_ENABLE))5968 goto dma_engine_disable;5969 }5970 5971 e_err(drv,5972 "TXDCTL.ENABLE for one or more queues not cleared within the polling period\n");5973 5974dma_engine_disable:5975 /* Disable the Tx DMA engine on 82599 and later MAC */5976 switch (hw->mac.type) {5977 case ixgbe_mac_82599EB:5978 case ixgbe_mac_X540:5979 case ixgbe_mac_X550:5980 case ixgbe_mac_X550EM_x:5981 case ixgbe_mac_x550em_a:5982 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL,5983 (IXGBE_READ_REG(hw, IXGBE_DMATXCTL) &5984 ~IXGBE_DMATXCTL_TE));5985 fallthrough;5986 default:5987 break;5988 }5989}5990 5991void ixgbe_reset(struct ixgbe_adapter *adapter)5992{5993 struct ixgbe_hw *hw = &adapter->hw;5994 struct net_device *netdev = adapter->netdev;5995 int err;5996 5997 if (ixgbe_removed(hw->hw_addr))5998 return;5999 /* lock SFP init bit to prevent race conditions with the watchdog */6000 while (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))6001 usleep_range(1000, 2000);6002 6003 /* clear all SFP and link config related flags while holding SFP_INIT */6004 adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP |6005 IXGBE_FLAG2_SFP_NEEDS_RESET);6006 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;6007 6008 err = hw->mac.ops.init_hw(hw);6009 switch (err) {6010 case 0:6011 case -ENOENT:6012 case -EOPNOTSUPP:6013 break;6014 case -EALREADY:6015 e_dev_err("primary disable timed out\n");6016 break;6017 case -EACCES:6018 /* We are running on a pre-production device, log a warning */6019 e_dev_warn("This device is a pre-production adapter/LOM. "6020 "Please be aware there may be issues associated with "6021 "your hardware. If you are experiencing problems "6022 "please contact your Intel or hardware "6023 "representative who provided you with this "6024 "hardware.\n");6025 break;6026 default:6027 e_dev_err("Hardware Error: %d\n", err);6028 }6029 6030 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);6031 6032 /* flush entries out of MAC table */6033 ixgbe_flush_sw_mac_table(adapter);6034 __dev_uc_unsync(netdev, NULL);6035 6036 /* do not flush user set addresses */6037 ixgbe_mac_set_default_filter(adapter);6038 6039 /* update SAN MAC vmdq pool selection */6040 if (hw->mac.san_mac_rar_index)6041 hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0));6042 6043 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state))6044 ixgbe_ptp_reset(adapter);6045 6046 if (hw->phy.ops.set_phy_power) {6047 if (!netif_running(adapter->netdev) && !adapter->wol)6048 hw->phy.ops.set_phy_power(hw, false);6049 else6050 hw->phy.ops.set_phy_power(hw, true);6051 }6052}6053 6054/**6055 * ixgbe_clean_tx_ring - Free Tx Buffers6056 * @tx_ring: ring to be cleaned6057 **/6058static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring)6059{6060 u16 i = tx_ring->next_to_clean;6061 struct ixgbe_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];6062 6063 if (tx_ring->xsk_pool) {6064 ixgbe_xsk_clean_tx_ring(tx_ring);6065 goto out;6066 }6067 6068 while (i != tx_ring->next_to_use) {6069 union ixgbe_adv_tx_desc *eop_desc, *tx_desc;6070 6071 /* Free all the Tx ring sk_buffs */6072 if (ring_is_xdp(tx_ring))6073 xdp_return_frame(tx_buffer->xdpf);6074 else6075 dev_kfree_skb_any(tx_buffer->skb);6076 6077 /* unmap skb header data */6078 dma_unmap_single(tx_ring->dev,6079 dma_unmap_addr(tx_buffer, dma),6080 dma_unmap_len(tx_buffer, len),6081 DMA_TO_DEVICE);6082 6083 /* check for eop_desc to determine the end of the packet */6084 eop_desc = tx_buffer->next_to_watch;6085 tx_desc = IXGBE_TX_DESC(tx_ring, i);6086 6087 /* unmap remaining buffers */6088 while (tx_desc != eop_desc) {6089 tx_buffer++;6090 tx_desc++;6091 i++;6092 if (unlikely(i == tx_ring->count)) {6093 i = 0;6094 tx_buffer = tx_ring->tx_buffer_info;6095 tx_desc = IXGBE_TX_DESC(tx_ring, 0);6096 }6097 6098 /* unmap any remaining paged data */6099 if (dma_unmap_len(tx_buffer, len))6100 dma_unmap_page(tx_ring->dev,6101 dma_unmap_addr(tx_buffer, dma),6102 dma_unmap_len(tx_buffer, len),6103 DMA_TO_DEVICE);6104 }6105 6106 /* move us one more past the eop_desc for start of next pkt */6107 tx_buffer++;6108 i++;6109 if (unlikely(i == tx_ring->count)) {6110 i = 0;6111 tx_buffer = tx_ring->tx_buffer_info;6112 }6113 }6114 6115 /* reset BQL for queue */6116 if (!ring_is_xdp(tx_ring))6117 netdev_tx_reset_queue(txring_txq(tx_ring));6118 6119out:6120 /* reset next_to_use and next_to_clean */6121 tx_ring->next_to_use = 0;6122 tx_ring->next_to_clean = 0;6123}6124 6125/**6126 * ixgbe_clean_all_rx_rings - Free Rx Buffers for all queues6127 * @adapter: board private structure6128 **/6129static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter)6130{6131 int i;6132 6133 for (i = 0; i < adapter->num_rx_queues; i++)6134 ixgbe_clean_rx_ring(adapter->rx_ring[i]);6135}6136 6137/**6138 * ixgbe_clean_all_tx_rings - Free Tx Buffers for all queues6139 * @adapter: board private structure6140 **/6141static void ixgbe_clean_all_tx_rings(struct ixgbe_adapter *adapter)6142{6143 int i;6144 6145 for (i = 0; i < adapter->num_tx_queues; i++)6146 ixgbe_clean_tx_ring(adapter->tx_ring[i]);6147 for (i = 0; i < adapter->num_xdp_queues; i++)6148 ixgbe_clean_tx_ring(adapter->xdp_ring[i]);6149}6150 6151static void ixgbe_fdir_filter_exit(struct ixgbe_adapter *adapter)6152{6153 struct hlist_node *node2;6154 struct ixgbe_fdir_filter *filter;6155 6156 spin_lock(&adapter->fdir_perfect_lock);6157 6158 hlist_for_each_entry_safe(filter, node2,6159 &adapter->fdir_filter_list, fdir_node) {6160 hlist_del(&filter->fdir_node);6161 kfree(filter);6162 }6163 adapter->fdir_filter_count = 0;6164 6165 spin_unlock(&adapter->fdir_perfect_lock);6166}6167 6168void ixgbe_down(struct ixgbe_adapter *adapter)6169{6170 struct net_device *netdev = adapter->netdev;6171 struct ixgbe_hw *hw = &adapter->hw;6172 int i;6173 6174 /* signal that we are down to the interrupt handler */6175 if (test_and_set_bit(__IXGBE_DOWN, &adapter->state))6176 return; /* do nothing if already down */6177 6178 /* Shut off incoming Tx traffic */6179 netif_tx_stop_all_queues(netdev);6180 6181 /* call carrier off first to avoid false dev_watchdog timeouts */6182 netif_carrier_off(netdev);6183 netif_tx_disable(netdev);6184 6185 /* Disable Rx */6186 ixgbe_disable_rx(adapter);6187 6188 /* synchronize_rcu() needed for pending XDP buffers to drain */6189 if (adapter->xdp_ring[0])6190 synchronize_rcu();6191 6192 ixgbe_irq_disable(adapter);6193 6194 ixgbe_napi_disable_all(adapter);6195 6196 clear_bit(__IXGBE_RESET_REQUESTED, &adapter->state);6197 adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT;6198 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;6199 6200 del_timer_sync(&adapter->service_timer);6201 6202 if (adapter->num_vfs) {6203 /* Clear EITR Select mapping */6204 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, 0);6205 6206 /* Mark all the VFs as inactive */6207 for (i = 0 ; i < adapter->num_vfs; i++)6208 adapter->vfinfo[i].clear_to_send = false;6209 6210 /* update setting rx tx for all active vfs */6211 ixgbe_set_all_vfs(adapter);6212 }6213 6214 /* disable transmits in the hardware now that interrupts are off */6215 ixgbe_disable_tx(adapter);6216 6217 if (!pci_channel_offline(adapter->pdev))6218 ixgbe_reset(adapter);6219 6220 /* power down the optics for 82599 SFP+ fiber */6221 if (hw->mac.ops.disable_tx_laser)6222 hw->mac.ops.disable_tx_laser(hw);6223 6224 ixgbe_clean_all_tx_rings(adapter);6225 ixgbe_clean_all_rx_rings(adapter);6226}6227 6228/**6229 * ixgbe_set_eee_capable - helper function to determine EEE support on X5506230 * @adapter: board private structure6231 */6232static void ixgbe_set_eee_capable(struct ixgbe_adapter *adapter)6233{6234 struct ixgbe_hw *hw = &adapter->hw;6235 6236 switch (hw->device_id) {6237 case IXGBE_DEV_ID_X550EM_A_1G_T:6238 case IXGBE_DEV_ID_X550EM_A_1G_T_L:6239 if (!hw->phy.eee_speeds_supported)6240 break;6241 adapter->flags2 |= IXGBE_FLAG2_EEE_CAPABLE;6242 if (!hw->phy.eee_speeds_advertised)6243 break;6244 adapter->flags2 |= IXGBE_FLAG2_EEE_ENABLED;6245 break;6246 default:6247 adapter->flags2 &= ~IXGBE_FLAG2_EEE_CAPABLE;6248 adapter->flags2 &= ~IXGBE_FLAG2_EEE_ENABLED;6249 break;6250 }6251}6252 6253/**6254 * ixgbe_tx_timeout - Respond to a Tx Hang6255 * @netdev: network interface device structure6256 * @txqueue: queue number that timed out6257 **/6258static void ixgbe_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)6259{6260 struct ixgbe_adapter *adapter = netdev_priv(netdev);6261 6262 /* Do the reset outside of interrupt context */6263 ixgbe_tx_timeout_reset(adapter);6264}6265 6266#ifdef CONFIG_IXGBE_DCB6267static void ixgbe_init_dcb(struct ixgbe_adapter *adapter)6268{6269 struct ixgbe_hw *hw = &adapter->hw;6270 struct tc_configuration *tc;6271 int j;6272 6273 switch (hw->mac.type) {6274 case ixgbe_mac_82598EB:6275 case ixgbe_mac_82599EB:6276 adapter->dcb_cfg.num_tcs.pg_tcs = MAX_TRAFFIC_CLASS;6277 adapter->dcb_cfg.num_tcs.pfc_tcs = MAX_TRAFFIC_CLASS;6278 break;6279 case ixgbe_mac_X540:6280 case ixgbe_mac_X550:6281 adapter->dcb_cfg.num_tcs.pg_tcs = X540_TRAFFIC_CLASS;6282 adapter->dcb_cfg.num_tcs.pfc_tcs = X540_TRAFFIC_CLASS;6283 break;6284 case ixgbe_mac_X550EM_x:6285 case ixgbe_mac_x550em_a:6286 default:6287 adapter->dcb_cfg.num_tcs.pg_tcs = DEF_TRAFFIC_CLASS;6288 adapter->dcb_cfg.num_tcs.pfc_tcs = DEF_TRAFFIC_CLASS;6289 break;6290 }6291 6292 /* Configure DCB traffic classes */6293 for (j = 0; j < MAX_TRAFFIC_CLASS; j++) {6294 tc = &adapter->dcb_cfg.tc_config[j];6295 tc->path[DCB_TX_CONFIG].bwg_id = 0;6296 tc->path[DCB_TX_CONFIG].bwg_percent = 12 + (j & 1);6297 tc->path[DCB_RX_CONFIG].bwg_id = 0;6298 tc->path[DCB_RX_CONFIG].bwg_percent = 12 + (j & 1);6299 tc->dcb_pfc = pfc_disabled;6300 }6301 6302 /* Initialize default user to priority mapping, UPx->TC0 */6303 tc = &adapter->dcb_cfg.tc_config[0];6304 tc->path[DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;6305 tc->path[DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;6306 6307 adapter->dcb_cfg.bw_percentage[DCB_TX_CONFIG][0] = 100;6308 adapter->dcb_cfg.bw_percentage[DCB_RX_CONFIG][0] = 100;6309 adapter->dcb_cfg.pfc_mode_enable = false;6310 adapter->dcb_set_bitmap = 0x00;6311 if (adapter->flags & IXGBE_FLAG_DCB_CAPABLE)6312 adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE;6313 memcpy(&adapter->temp_dcb_cfg, &adapter->dcb_cfg,6314 sizeof(adapter->temp_dcb_cfg));6315}6316#endif6317 6318/**6319 * ixgbe_sw_init - Initialize general software structures (struct ixgbe_adapter)6320 * @adapter: board private structure to initialize6321 * @ii: pointer to ixgbe_info for device6322 *6323 * ixgbe_sw_init initializes the Adapter private data structure.6324 * Fields are initialized based on PCI device information and6325 * OS network device settings (MTU size).6326 **/6327static int ixgbe_sw_init(struct ixgbe_adapter *adapter,6328 const struct ixgbe_info *ii)6329{6330 struct ixgbe_hw *hw = &adapter->hw;6331 struct pci_dev *pdev = adapter->pdev;6332 unsigned int rss, fdir;6333 u32 fwsm;6334 int i;6335 6336 /* PCI config space info */6337 6338 hw->vendor_id = pdev->vendor;6339 hw->device_id = pdev->device;6340 hw->revision_id = pdev->revision;6341 hw->subsystem_vendor_id = pdev->subsystem_vendor;6342 hw->subsystem_device_id = pdev->subsystem_device;6343 6344 /* get_invariants needs the device IDs */6345 ii->get_invariants(hw);6346 6347 /* Set common capability flags and settings */6348 rss = min_t(int, ixgbe_max_rss_indices(adapter), num_online_cpus());6349 adapter->ring_feature[RING_F_RSS].limit = rss;6350 adapter->flags2 |= IXGBE_FLAG2_RSC_CAPABLE;6351 adapter->max_q_vectors = MAX_Q_VECTORS_82599;6352 adapter->atr_sample_rate = 20;6353 fdir = min_t(int, IXGBE_MAX_FDIR_INDICES, num_online_cpus());6354 adapter->ring_feature[RING_F_FDIR].limit = fdir;6355 adapter->fdir_pballoc = IXGBE_FDIR_PBALLOC_64K;6356 adapter->ring_feature[RING_F_VMDQ].limit = 1;6357#ifdef CONFIG_IXGBE_DCA6358 adapter->flags |= IXGBE_FLAG_DCA_CAPABLE;6359#endif6360#ifdef CONFIG_IXGBE_DCB6361 adapter->flags |= IXGBE_FLAG_DCB_CAPABLE;6362 adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED;6363#endif6364#ifdef IXGBE_FCOE6365 adapter->flags |= IXGBE_FLAG_FCOE_CAPABLE;6366 adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED;6367#ifdef CONFIG_IXGBE_DCB6368 /* Default traffic class to use for FCoE */6369 adapter->fcoe.up = IXGBE_FCOE_DEFTC;6370#endif /* CONFIG_IXGBE_DCB */6371#endif /* IXGBE_FCOE */6372 6373 /* initialize static ixgbe jump table entries */6374 adapter->jump_tables[0] = kzalloc(sizeof(*adapter->jump_tables[0]),6375 GFP_KERNEL);6376 if (!adapter->jump_tables[0])6377 return -ENOMEM;6378 adapter->jump_tables[0]->mat = ixgbe_ipv4_fields;6379 6380 for (i = 1; i < IXGBE_MAX_LINK_HANDLE; i++)6381 adapter->jump_tables[i] = NULL;6382 6383 adapter->mac_table = kcalloc(hw->mac.num_rar_entries,6384 sizeof(struct ixgbe_mac_addr),6385 GFP_KERNEL);6386 if (!adapter->mac_table)6387 return -ENOMEM;6388 6389 if (ixgbe_init_rss_key(adapter))6390 return -ENOMEM;6391 6392 adapter->af_xdp_zc_qps = bitmap_zalloc(IXGBE_MAX_XDP_QS, GFP_KERNEL);6393 if (!adapter->af_xdp_zc_qps)6394 return -ENOMEM;6395 6396 /* Set MAC specific capability flags and exceptions */6397 switch (hw->mac.type) {6398 case ixgbe_mac_82598EB:6399 adapter->flags2 &= ~IXGBE_FLAG2_RSC_CAPABLE;6400 6401 if (hw->device_id == IXGBE_DEV_ID_82598AT)6402 adapter->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE;6403 6404 adapter->max_q_vectors = MAX_Q_VECTORS_82598;6405 adapter->ring_feature[RING_F_FDIR].limit = 0;6406 adapter->atr_sample_rate = 0;6407 adapter->fdir_pballoc = 0;6408#ifdef IXGBE_FCOE6409 adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;6410 adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED;6411#ifdef CONFIG_IXGBE_DCB6412 adapter->fcoe.up = 0;6413#endif /* IXGBE_DCB */6414#endif /* IXGBE_FCOE */6415 break;6416 case ixgbe_mac_82599EB:6417 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM)6418 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;6419 break;6420 case ixgbe_mac_X540:6421 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM(hw));6422 if (fwsm & IXGBE_FWSM_TS_ENABLED)6423 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;6424 break;6425 case ixgbe_mac_x550em_a:6426 switch (hw->device_id) {6427 case IXGBE_DEV_ID_X550EM_A_1G_T:6428 case IXGBE_DEV_ID_X550EM_A_1G_T_L:6429 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;6430 break;6431 default:6432 break;6433 }6434 fallthrough;6435 case ixgbe_mac_X550EM_x:6436#ifdef CONFIG_IXGBE_DCB6437 adapter->flags &= ~IXGBE_FLAG_DCB_CAPABLE;6438#endif6439#ifdef IXGBE_FCOE6440 adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;6441#ifdef CONFIG_IXGBE_DCB6442 adapter->fcoe.up = 0;6443#endif /* IXGBE_DCB */6444#endif /* IXGBE_FCOE */6445 fallthrough;6446 case ixgbe_mac_X550:6447 if (hw->mac.type == ixgbe_mac_X550)6448 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE;6449#ifdef CONFIG_IXGBE_DCA6450 adapter->flags &= ~IXGBE_FLAG_DCA_CAPABLE;6451#endif6452 break;6453 default:6454 break;6455 }6456 6457#ifdef IXGBE_FCOE6458 /* FCoE support exists, always init the FCoE lock */6459 spin_lock_init(&adapter->fcoe.lock);6460 6461#endif6462 /* n-tuple support exists, always init our spinlock */6463 spin_lock_init(&adapter->fdir_perfect_lock);6464 6465 /* init spinlock to avoid concurrency of VF resources */6466 spin_lock_init(&adapter->vfs_lock);6467 6468#ifdef CONFIG_IXGBE_DCB6469 ixgbe_init_dcb(adapter);6470#endif6471 ixgbe_init_ipsec_offload(adapter);6472 6473 /* default flow control settings */6474 hw->fc.requested_mode = ixgbe_fc_full;6475 hw->fc.current_mode = ixgbe_fc_full; /* init for ethtool output */6476 ixgbe_pbthresh_setup(adapter);6477 hw->fc.pause_time = IXGBE_DEFAULT_FCPAUSE;6478 hw->fc.send_xon = true;6479 hw->fc.disable_fc_autoneg = ixgbe_device_supports_autoneg_fc(hw);6480 6481#ifdef CONFIG_PCI_IOV6482 if (max_vfs > 0)6483 e_dev_warn("Enabling SR-IOV VFs using the max_vfs module parameter is deprecated - please use the pci sysfs interface instead.\n");6484 6485 /* assign number of SR-IOV VFs */6486 if (hw->mac.type != ixgbe_mac_82598EB) {6487 if (max_vfs > IXGBE_MAX_VFS_DRV_LIMIT) {6488 max_vfs = 0;6489 e_dev_warn("max_vfs parameter out of range. Not assigning any SR-IOV VFs\n");6490 }6491 }6492#endif /* CONFIG_PCI_IOV */6493 6494 /* enable itr by default in dynamic mode */6495 adapter->rx_itr_setting = 1;6496 adapter->tx_itr_setting = 1;6497 6498 /* set default ring sizes */6499 adapter->tx_ring_count = IXGBE_DEFAULT_TXD;6500 adapter->rx_ring_count = IXGBE_DEFAULT_RXD;6501 6502 /* set default work limits */6503 adapter->tx_work_limit = IXGBE_DEFAULT_TX_WORK;6504 6505 /* initialize eeprom parameters */6506 if (ixgbe_init_eeprom_params_generic(hw)) {6507 e_dev_err("EEPROM initialization failed\n");6508 return -EIO;6509 }6510 6511 /* PF holds first pool slot */6512 set_bit(0, adapter->fwd_bitmask);6513 set_bit(__IXGBE_DOWN, &adapter->state);6514 6515 /* enable locking for XDP_TX if we have more CPUs than queues */6516 if (nr_cpu_ids > IXGBE_MAX_XDP_QS)6517 static_branch_enable(&ixgbe_xdp_locking_key);6518 6519 return 0;6520}6521 6522/**6523 * ixgbe_setup_tx_resources - allocate Tx resources (Descriptors)6524 * @tx_ring: tx descriptor ring (for a specific queue) to setup6525 *6526 * Return 0 on success, negative on failure6527 **/6528int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring)6529{6530 struct device *dev = tx_ring->dev;6531 int orig_node = dev_to_node(dev);6532 int ring_node = NUMA_NO_NODE;6533 int size;6534 6535 size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count;6536 6537 if (tx_ring->q_vector)6538 ring_node = tx_ring->q_vector->numa_node;6539 6540 tx_ring->tx_buffer_info = vmalloc_node(size, ring_node);6541 if (!tx_ring->tx_buffer_info)6542 tx_ring->tx_buffer_info = vmalloc(size);6543 if (!tx_ring->tx_buffer_info)6544 goto err;6545 6546 /* round up to nearest 4K */6547 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);6548 tx_ring->size = ALIGN(tx_ring->size, 4096);6549 6550 set_dev_node(dev, ring_node);6551 tx_ring->desc = dma_alloc_coherent(dev,6552 tx_ring->size,6553 &tx_ring->dma,6554 GFP_KERNEL);6555 set_dev_node(dev, orig_node);6556 if (!tx_ring->desc)6557 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,6558 &tx_ring->dma, GFP_KERNEL);6559 if (!tx_ring->desc)6560 goto err;6561 6562 tx_ring->next_to_use = 0;6563 tx_ring->next_to_clean = 0;6564 return 0;6565 6566err:6567 vfree(tx_ring->tx_buffer_info);6568 tx_ring->tx_buffer_info = NULL;6569 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");6570 return -ENOMEM;6571}6572 6573/**6574 * ixgbe_setup_all_tx_resources - allocate all queues Tx resources6575 * @adapter: board private structure6576 *6577 * If this function returns with an error, then it's possible one or6578 * more of the rings is populated (while the rest are not). It is the6579 * callers duty to clean those orphaned rings.6580 *6581 * Return 0 on success, negative on failure6582 **/6583static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter)6584{6585 int i, j = 0, err = 0;6586 6587 for (i = 0; i < adapter->num_tx_queues; i++) {6588 err = ixgbe_setup_tx_resources(adapter->tx_ring[i]);6589 if (!err)6590 continue;6591 6592 e_err(probe, "Allocation for Tx Queue %u failed\n", i);6593 goto err_setup_tx;6594 }6595 for (j = 0; j < adapter->num_xdp_queues; j++) {6596 err = ixgbe_setup_tx_resources(adapter->xdp_ring[j]);6597 if (!err)6598 continue;6599 6600 e_err(probe, "Allocation for Tx Queue %u failed\n", j);6601 goto err_setup_tx;6602 }6603 6604 return 0;6605err_setup_tx:6606 /* rewind the index freeing the rings as we go */6607 while (j--)6608 ixgbe_free_tx_resources(adapter->xdp_ring[j]);6609 while (i--)6610 ixgbe_free_tx_resources(adapter->tx_ring[i]);6611 return err;6612}6613 6614static int ixgbe_rx_napi_id(struct ixgbe_ring *rx_ring)6615{6616 struct ixgbe_q_vector *q_vector = rx_ring->q_vector;6617 6618 return q_vector ? q_vector->napi.napi_id : 0;6619}6620 6621/**6622 * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors)6623 * @adapter: pointer to ixgbe_adapter6624 * @rx_ring: rx descriptor ring (for a specific queue) to setup6625 *6626 * Returns 0 on success, negative on failure6627 **/6628int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter,6629 struct ixgbe_ring *rx_ring)6630{6631 struct device *dev = rx_ring->dev;6632 int orig_node = dev_to_node(dev);6633 int ring_node = NUMA_NO_NODE;6634 int size;6635 6636 size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count;6637 6638 if (rx_ring->q_vector)6639 ring_node = rx_ring->q_vector->numa_node;6640 6641 rx_ring->rx_buffer_info = vmalloc_node(size, ring_node);6642 if (!rx_ring->rx_buffer_info)6643 rx_ring->rx_buffer_info = vmalloc(size);6644 if (!rx_ring->rx_buffer_info)6645 goto err;6646 6647 /* Round up to nearest 4K */6648 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);6649 rx_ring->size = ALIGN(rx_ring->size, 4096);6650 6651 set_dev_node(dev, ring_node);6652 rx_ring->desc = dma_alloc_coherent(dev,6653 rx_ring->size,6654 &rx_ring->dma,6655 GFP_KERNEL);6656 set_dev_node(dev, orig_node);6657 if (!rx_ring->desc)6658 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,6659 &rx_ring->dma, GFP_KERNEL);6660 if (!rx_ring->desc)6661 goto err;6662 6663 rx_ring->next_to_clean = 0;6664 rx_ring->next_to_use = 0;6665 6666 /* XDP RX-queue info */6667 if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev,6668 rx_ring->queue_index, ixgbe_rx_napi_id(rx_ring)) < 0)6669 goto err;6670 6671 WRITE_ONCE(rx_ring->xdp_prog, adapter->xdp_prog);6672 6673 return 0;6674err:6675 vfree(rx_ring->rx_buffer_info);6676 rx_ring->rx_buffer_info = NULL;6677 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");6678 return -ENOMEM;6679}6680 6681/**6682 * ixgbe_setup_all_rx_resources - allocate all queues Rx resources6683 * @adapter: board private structure6684 *6685 * If this function returns with an error, then it's possible one or6686 * more of the rings is populated (while the rest are not). It is the6687 * callers duty to clean those orphaned rings.6688 *6689 * Return 0 on success, negative on failure6690 **/6691static int ixgbe_setup_all_rx_resources(struct ixgbe_adapter *adapter)6692{6693 int i, err = 0;6694 6695 for (i = 0; i < adapter->num_rx_queues; i++) {6696 err = ixgbe_setup_rx_resources(adapter, adapter->rx_ring[i]);6697 if (!err)6698 continue;6699 6700 e_err(probe, "Allocation for Rx Queue %u failed\n", i);6701 goto err_setup_rx;6702 }6703 6704#ifdef IXGBE_FCOE6705 err = ixgbe_setup_fcoe_ddp_resources(adapter);6706 if (!err)6707#endif6708 return 0;6709err_setup_rx:6710 /* rewind the index freeing the rings as we go */6711 while (i--)6712 ixgbe_free_rx_resources(adapter->rx_ring[i]);6713 return err;6714}6715 6716/**6717 * ixgbe_free_tx_resources - Free Tx Resources per Queue6718 * @tx_ring: Tx descriptor ring for a specific queue6719 *6720 * Free all transmit software resources6721 **/6722void ixgbe_free_tx_resources(struct ixgbe_ring *tx_ring)6723{6724 ixgbe_clean_tx_ring(tx_ring);6725 6726 vfree(tx_ring->tx_buffer_info);6727 tx_ring->tx_buffer_info = NULL;6728 6729 /* if not set, then don't free */6730 if (!tx_ring->desc)6731 return;6732 6733 dma_free_coherent(tx_ring->dev, tx_ring->size,6734 tx_ring->desc, tx_ring->dma);6735 6736 tx_ring->desc = NULL;6737}6738 6739/**6740 * ixgbe_free_all_tx_resources - Free Tx Resources for All Queues6741 * @adapter: board private structure6742 *6743 * Free all transmit software resources6744 **/6745static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter)6746{6747 int i;6748 6749 for (i = 0; i < adapter->num_tx_queues; i++)6750 if (adapter->tx_ring[i]->desc)6751 ixgbe_free_tx_resources(adapter->tx_ring[i]);6752 for (i = 0; i < adapter->num_xdp_queues; i++)6753 if (adapter->xdp_ring[i]->desc)6754 ixgbe_free_tx_resources(adapter->xdp_ring[i]);6755}6756 6757/**6758 * ixgbe_free_rx_resources - Free Rx Resources6759 * @rx_ring: ring to clean the resources from6760 *6761 * Free all receive software resources6762 **/6763void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring)6764{6765 ixgbe_clean_rx_ring(rx_ring);6766 6767 rx_ring->xdp_prog = NULL;6768 xdp_rxq_info_unreg(&rx_ring->xdp_rxq);6769 vfree(rx_ring->rx_buffer_info);6770 rx_ring->rx_buffer_info = NULL;6771 6772 /* if not set, then don't free */6773 if (!rx_ring->desc)6774 return;6775 6776 dma_free_coherent(rx_ring->dev, rx_ring->size,6777 rx_ring->desc, rx_ring->dma);6778 6779 rx_ring->desc = NULL;6780}6781 6782/**6783 * ixgbe_free_all_rx_resources - Free Rx Resources for All Queues6784 * @adapter: board private structure6785 *6786 * Free all receive software resources6787 **/6788static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter)6789{6790 int i;6791 6792#ifdef IXGBE_FCOE6793 ixgbe_free_fcoe_ddp_resources(adapter);6794 6795#endif6796 for (i = 0; i < adapter->num_rx_queues; i++)6797 if (adapter->rx_ring[i]->desc)6798 ixgbe_free_rx_resources(adapter->rx_ring[i]);6799}6800 6801/**6802 * ixgbe_max_xdp_frame_size - returns the maximum allowed frame size for XDP6803 * @adapter: device handle, pointer to adapter6804 */6805static int ixgbe_max_xdp_frame_size(struct ixgbe_adapter *adapter)6806{6807 if (PAGE_SIZE >= 8192 || adapter->flags2 & IXGBE_FLAG2_RX_LEGACY)6808 return IXGBE_RXBUFFER_2K;6809 else6810 return IXGBE_RXBUFFER_3K;6811}6812 6813/**6814 * ixgbe_change_mtu - Change the Maximum Transfer Unit6815 * @netdev: network interface device structure6816 * @new_mtu: new value for maximum frame size6817 *6818 * Returns 0 on success, negative on failure6819 **/6820static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu)6821{6822 struct ixgbe_adapter *adapter = netdev_priv(netdev);6823 6824 if (ixgbe_enabled_xdp_adapter(adapter)) {6825 int new_frame_size = new_mtu + IXGBE_PKT_HDR_PAD;6826 6827 if (new_frame_size > ixgbe_max_xdp_frame_size(adapter)) {6828 e_warn(probe, "Requested MTU size is not supported with XDP\n");6829 return -EINVAL;6830 }6831 }6832 6833 /*6834 * For 82599EB we cannot allow legacy VFs to enable their receive6835 * paths when MTU greater than 1500 is configured. So display a6836 * warning that legacy VFs will be disabled.6837 */6838 if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) &&6839 (adapter->hw.mac.type == ixgbe_mac_82599EB) &&6840 (new_mtu > ETH_DATA_LEN))6841 e_warn(probe, "Setting MTU > 1500 will disable legacy VFs\n");6842 6843 netdev_dbg(netdev, "changing MTU from %d to %d\n",6844 netdev->mtu, new_mtu);6845 6846 /* must set new MTU before calling down or up */6847 WRITE_ONCE(netdev->mtu, new_mtu);6848 6849 if (netif_running(netdev))6850 ixgbe_reinit_locked(adapter);6851 6852 return 0;6853}6854 6855/**6856 * ixgbe_open - Called when a network interface is made active6857 * @netdev: network interface device structure6858 *6859 * Returns 0 on success, negative value on failure6860 *6861 * The open entry point is called when a network interface is made6862 * active by the system (IFF_UP). At this point all resources needed6863 * for transmit and receive operations are allocated, the interrupt6864 * handler is registered with the OS, the watchdog timer is started,6865 * and the stack is notified that the interface is ready.6866 **/6867int ixgbe_open(struct net_device *netdev)6868{6869 struct ixgbe_adapter *adapter = netdev_priv(netdev);6870 struct ixgbe_hw *hw = &adapter->hw;6871 int err, queues;6872 6873 /* disallow open during test */6874 if (test_bit(__IXGBE_TESTING, &adapter->state))6875 return -EBUSY;6876 6877 netif_carrier_off(netdev);6878 6879 /* allocate transmit descriptors */6880 err = ixgbe_setup_all_tx_resources(adapter);6881 if (err)6882 goto err_setup_tx;6883 6884 /* allocate receive descriptors */6885 err = ixgbe_setup_all_rx_resources(adapter);6886 if (err)6887 goto err_setup_rx;6888 6889 ixgbe_configure(adapter);6890 6891 err = ixgbe_request_irq(adapter);6892 if (err)6893 goto err_req_irq;6894 6895 /* Notify the stack of the actual queue counts. */6896 queues = adapter->num_tx_queues;6897 err = netif_set_real_num_tx_queues(netdev, queues);6898 if (err)6899 goto err_set_queues;6900 6901 queues = adapter->num_rx_queues;6902 err = netif_set_real_num_rx_queues(netdev, queues);6903 if (err)6904 goto err_set_queues;6905 6906 ixgbe_ptp_init(adapter);6907 6908 ixgbe_up_complete(adapter);6909 6910 udp_tunnel_nic_reset_ntf(netdev);6911 6912 return 0;6913 6914err_set_queues:6915 ixgbe_free_irq(adapter);6916err_req_irq:6917 ixgbe_free_all_rx_resources(adapter);6918 if (hw->phy.ops.set_phy_power && !adapter->wol)6919 hw->phy.ops.set_phy_power(&adapter->hw, false);6920err_setup_rx:6921 ixgbe_free_all_tx_resources(adapter);6922err_setup_tx:6923 ixgbe_reset(adapter);6924 6925 return err;6926}6927 6928static void ixgbe_close_suspend(struct ixgbe_adapter *adapter)6929{6930 ixgbe_ptp_suspend(adapter);6931 6932 if (adapter->hw.phy.ops.enter_lplu) {6933 adapter->hw.phy.reset_disable = true;6934 ixgbe_down(adapter);6935 adapter->hw.phy.ops.enter_lplu(&adapter->hw);6936 adapter->hw.phy.reset_disable = false;6937 } else {6938 ixgbe_down(adapter);6939 }6940 6941 ixgbe_free_irq(adapter);6942 6943 ixgbe_free_all_tx_resources(adapter);6944 ixgbe_free_all_rx_resources(adapter);6945}6946 6947/**6948 * ixgbe_close - Disables a network interface6949 * @netdev: network interface device structure6950 *6951 * Returns 0, this is not allowed to fail6952 *6953 * The close entry point is called when an interface is de-activated6954 * by the OS. The hardware is still under the drivers control, but6955 * needs to be disabled. A global MAC reset is issued to stop the6956 * hardware, and all transmit and receive resources are freed.6957 **/6958int ixgbe_close(struct net_device *netdev)6959{6960 struct ixgbe_adapter *adapter = netdev_priv(netdev);6961 6962 ixgbe_ptp_stop(adapter);6963 6964 if (netif_device_present(netdev))6965 ixgbe_close_suspend(adapter);6966 6967 ixgbe_fdir_filter_exit(adapter);6968 6969 ixgbe_release_hw_control(adapter);6970 6971 return 0;6972}6973 6974static int ixgbe_resume(struct device *dev_d)6975{6976 struct pci_dev *pdev = to_pci_dev(dev_d);6977 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);6978 struct net_device *netdev = adapter->netdev;6979 u32 err;6980 6981 adapter->hw.hw_addr = adapter->io_addr;6982 6983 err = pci_enable_device_mem(pdev);6984 if (err) {6985 e_dev_err("Cannot enable PCI device from suspend\n");6986 return err;6987 }6988 smp_mb__before_atomic();6989 clear_bit(__IXGBE_DISABLED, &adapter->state);6990 pci_set_master(pdev);6991 6992 device_wakeup_disable(dev_d);6993 6994 ixgbe_reset(adapter);6995 6996 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);6997 6998 rtnl_lock();6999 err = ixgbe_init_interrupt_scheme(adapter);7000 if (!err && netif_running(netdev))7001 err = ixgbe_open(netdev);7002 7003 7004 if (!err)7005 netif_device_attach(netdev);7006 rtnl_unlock();7007 7008 return err;7009}7010 7011static int __ixgbe_shutdown(struct pci_dev *pdev, bool *enable_wake)7012{7013 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);7014 struct net_device *netdev = adapter->netdev;7015 struct ixgbe_hw *hw = &adapter->hw;7016 u32 ctrl;7017 u32 wufc = adapter->wol;7018 7019 rtnl_lock();7020 netif_device_detach(netdev);7021 7022 if (netif_running(netdev))7023 ixgbe_close_suspend(adapter);7024 7025 ixgbe_clear_interrupt_scheme(adapter);7026 rtnl_unlock();7027 7028 if (hw->mac.ops.stop_link_on_d3)7029 hw->mac.ops.stop_link_on_d3(hw);7030 7031 if (wufc) {7032 u32 fctrl;7033 7034 ixgbe_set_rx_mode(netdev);7035 7036 /* enable the optics for 82599 SFP+ fiber as we can WoL */7037 if (hw->mac.ops.enable_tx_laser)7038 hw->mac.ops.enable_tx_laser(hw);7039 7040 /* enable the reception of multicast packets */7041 fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);7042 fctrl |= IXGBE_FCTRL_MPE;7043 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);7044 7045 ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL);7046 ctrl |= IXGBE_CTRL_GIO_DIS;7047 IXGBE_WRITE_REG(hw, IXGBE_CTRL, ctrl);7048 7049 IXGBE_WRITE_REG(hw, IXGBE_WUFC, wufc);7050 } else {7051 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0);7052 IXGBE_WRITE_REG(hw, IXGBE_WUFC, 0);7053 }7054 7055 switch (hw->mac.type) {7056 case ixgbe_mac_82598EB:7057 pci_wake_from_d3(pdev, false);7058 break;7059 case ixgbe_mac_82599EB:7060 case ixgbe_mac_X540:7061 case ixgbe_mac_X550:7062 case ixgbe_mac_X550EM_x:7063 case ixgbe_mac_x550em_a:7064 pci_wake_from_d3(pdev, !!wufc);7065 break;7066 default:7067 break;7068 }7069 7070 *enable_wake = !!wufc;7071 if (hw->phy.ops.set_phy_power && !*enable_wake)7072 hw->phy.ops.set_phy_power(hw, false);7073 7074 ixgbe_release_hw_control(adapter);7075 7076 if (!test_and_set_bit(__IXGBE_DISABLED, &adapter->state))7077 pci_disable_device(pdev);7078 7079 return 0;7080}7081 7082static int ixgbe_suspend(struct device *dev_d)7083{7084 struct pci_dev *pdev = to_pci_dev(dev_d);7085 int retval;7086 bool wake;7087 7088 retval = __ixgbe_shutdown(pdev, &wake);7089 7090 device_set_wakeup_enable(dev_d, wake);7091 7092 return retval;7093}7094 7095static void ixgbe_shutdown(struct pci_dev *pdev)7096{7097 bool wake;7098 7099 __ixgbe_shutdown(pdev, &wake);7100 7101 if (system_state == SYSTEM_POWER_OFF) {7102 pci_wake_from_d3(pdev, wake);7103 pci_set_power_state(pdev, PCI_D3hot);7104 }7105}7106 7107/**7108 * ixgbe_update_stats - Update the board statistics counters.7109 * @adapter: board private structure7110 **/7111void ixgbe_update_stats(struct ixgbe_adapter *adapter)7112{7113 struct net_device *netdev = adapter->netdev;7114 struct ixgbe_hw *hw = &adapter->hw;7115 struct ixgbe_hw_stats *hwstats = &adapter->stats;7116 u64 total_mpc = 0;7117 u32 i, missed_rx = 0, mpc, bprc, lxon, lxoff, xon_off_tot;7118 u64 non_eop_descs = 0, restart_queue = 0, tx_busy = 0;7119 u64 alloc_rx_page_failed = 0, alloc_rx_buff_failed = 0;7120 u64 alloc_rx_page = 0;7121 u64 bytes = 0, packets = 0, hw_csum_rx_error = 0;7122 7123 if (test_bit(__IXGBE_DOWN, &adapter->state) ||7124 test_bit(__IXGBE_RESETTING, &adapter->state))7125 return;7126 7127 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) {7128 u64 rsc_count = 0;7129 u64 rsc_flush = 0;7130 for (i = 0; i < adapter->num_rx_queues; i++) {7131 rsc_count += adapter->rx_ring[i]->rx_stats.rsc_count;7132 rsc_flush += adapter->rx_ring[i]->rx_stats.rsc_flush;7133 }7134 adapter->rsc_total_count = rsc_count;7135 adapter->rsc_total_flush = rsc_flush;7136 }7137 7138 for (i = 0; i < adapter->num_rx_queues; i++) {7139 struct ixgbe_ring *rx_ring = READ_ONCE(adapter->rx_ring[i]);7140 7141 if (!rx_ring)7142 continue;7143 non_eop_descs += rx_ring->rx_stats.non_eop_descs;7144 alloc_rx_page += rx_ring->rx_stats.alloc_rx_page;7145 alloc_rx_page_failed += rx_ring->rx_stats.alloc_rx_page_failed;7146 alloc_rx_buff_failed += rx_ring->rx_stats.alloc_rx_buff_failed;7147 hw_csum_rx_error += rx_ring->rx_stats.csum_err;7148 bytes += rx_ring->stats.bytes;7149 packets += rx_ring->stats.packets;7150 }7151 adapter->non_eop_descs = non_eop_descs;7152 adapter->alloc_rx_page = alloc_rx_page;7153 adapter->alloc_rx_page_failed = alloc_rx_page_failed;7154 adapter->alloc_rx_buff_failed = alloc_rx_buff_failed;7155 adapter->hw_csum_rx_error = hw_csum_rx_error;7156 netdev->stats.rx_bytes = bytes;7157 netdev->stats.rx_packets = packets;7158 7159 bytes = 0;7160 packets = 0;7161 /* gather some stats to the adapter struct that are per queue */7162 for (i = 0; i < adapter->num_tx_queues; i++) {7163 struct ixgbe_ring *tx_ring = READ_ONCE(adapter->tx_ring[i]);7164 7165 if (!tx_ring)7166 continue;7167 restart_queue += tx_ring->tx_stats.restart_queue;7168 tx_busy += tx_ring->tx_stats.tx_busy;7169 bytes += tx_ring->stats.bytes;7170 packets += tx_ring->stats.packets;7171 }7172 for (i = 0; i < adapter->num_xdp_queues; i++) {7173 struct ixgbe_ring *xdp_ring = READ_ONCE(adapter->xdp_ring[i]);7174 7175 if (!xdp_ring)7176 continue;7177 restart_queue += xdp_ring->tx_stats.restart_queue;7178 tx_busy += xdp_ring->tx_stats.tx_busy;7179 bytes += xdp_ring->stats.bytes;7180 packets += xdp_ring->stats.packets;7181 }7182 adapter->restart_queue = restart_queue;7183 adapter->tx_busy = tx_busy;7184 netdev->stats.tx_bytes = bytes;7185 netdev->stats.tx_packets = packets;7186 7187 hwstats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);7188 7189 /* 8 register reads */7190 for (i = 0; i < 8; i++) {7191 /* for packet buffers not used, the register should read 0 */7192 mpc = IXGBE_READ_REG(hw, IXGBE_MPC(i));7193 missed_rx += mpc;7194 hwstats->mpc[i] += mpc;7195 total_mpc += hwstats->mpc[i];7196 hwstats->pxontxc[i] += IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));7197 hwstats->pxofftxc[i] += IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));7198 switch (hw->mac.type) {7199 case ixgbe_mac_82598EB:7200 hwstats->rnbc[i] += IXGBE_READ_REG(hw, IXGBE_RNBC(i));7201 hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC(i));7202 hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC(i));7203 hwstats->pxonrxc[i] +=7204 IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));7205 break;7206 case ixgbe_mac_82599EB:7207 case ixgbe_mac_X540:7208 case ixgbe_mac_X550:7209 case ixgbe_mac_X550EM_x:7210 case ixgbe_mac_x550em_a:7211 hwstats->pxonrxc[i] +=7212 IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));7213 break;7214 default:7215 break;7216 }7217 }7218 7219 /*16 register reads */7220 for (i = 0; i < 16; i++) {7221 hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i));7222 hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i));7223 if ((hw->mac.type == ixgbe_mac_82599EB) ||7224 (hw->mac.type == ixgbe_mac_X540) ||7225 (hw->mac.type == ixgbe_mac_X550) ||7226 (hw->mac.type == ixgbe_mac_X550EM_x) ||7227 (hw->mac.type == ixgbe_mac_x550em_a)) {7228 hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));7229 IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)); /* to clear */7230 hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));7231 IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)); /* to clear */7232 }7233 }7234 7235 hwstats->gprc += IXGBE_READ_REG(hw, IXGBE_GPRC);7236 /* work around hardware counting issue */7237 hwstats->gprc -= missed_rx;7238 7239 ixgbe_update_xoff_received(adapter);7240 7241 /* 82598 hardware only has a 32 bit counter in the high register */7242 switch (hw->mac.type) {7243 case ixgbe_mac_82598EB:7244 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);7245 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);7246 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);7247 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);7248 break;7249 case ixgbe_mac_X540:7250 case ixgbe_mac_X550:7251 case ixgbe_mac_X550EM_x:7252 case ixgbe_mac_x550em_a:7253 /* OS2BMC stats are X540 and later */7254 hwstats->o2bgptc += IXGBE_READ_REG(hw, IXGBE_O2BGPTC);7255 hwstats->o2bspc += IXGBE_READ_REG(hw, IXGBE_O2BSPC);7256 hwstats->b2ospc += IXGBE_READ_REG(hw, IXGBE_B2OSPC);7257 hwstats->b2ogprc += IXGBE_READ_REG(hw, IXGBE_B2OGPRC);7258 fallthrough;7259 case ixgbe_mac_82599EB:7260 for (i = 0; i < 16; i++)7261 adapter->hw_rx_no_dma_resources +=7262 IXGBE_READ_REG(hw, IXGBE_QPRDC(i));7263 hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);7264 IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */7265 hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);7266 IXGBE_READ_REG(hw, IXGBE_GOTCH); /* to clear */7267 hwstats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);7268 IXGBE_READ_REG(hw, IXGBE_TORH); /* to clear */7269 hwstats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);7270 hwstats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);7271 hwstats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);7272#ifdef IXGBE_FCOE7273 hwstats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);7274 hwstats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);7275 hwstats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);7276 hwstats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);7277 hwstats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);7278 hwstats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);7279 /* Add up per cpu counters for total ddp aloc fail */7280 if (adapter->fcoe.ddp_pool) {7281 struct ixgbe_fcoe *fcoe = &adapter->fcoe;7282 struct ixgbe_fcoe_ddp_pool *ddp_pool;7283 unsigned int cpu;7284 u64 noddp = 0, noddp_ext_buff = 0;7285 for_each_possible_cpu(cpu) {7286 ddp_pool = per_cpu_ptr(fcoe->ddp_pool, cpu);7287 noddp += ddp_pool->noddp;7288 noddp_ext_buff += ddp_pool->noddp_ext_buff;7289 }7290 hwstats->fcoe_noddp = noddp;7291 hwstats->fcoe_noddp_ext_buff = noddp_ext_buff;7292 }7293#endif /* IXGBE_FCOE */7294 break;7295 default:7296 break;7297 }7298 bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);7299 hwstats->bprc += bprc;7300 hwstats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);7301 if (hw->mac.type == ixgbe_mac_82598EB)7302 hwstats->mprc -= bprc;7303 hwstats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);7304 hwstats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);7305 hwstats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);7306 hwstats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);7307 hwstats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);7308 hwstats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);7309 hwstats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);7310 hwstats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);7311 lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);7312 hwstats->lxontxc += lxon;7313 lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);7314 hwstats->lxofftxc += lxoff;7315 hwstats->gptc += IXGBE_READ_REG(hw, IXGBE_GPTC);7316 hwstats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);7317 /*7318 * 82598 errata - tx of flow control packets is included in tx counters7319 */7320 xon_off_tot = lxon + lxoff;7321 hwstats->gptc -= xon_off_tot;7322 hwstats->mptc -= xon_off_tot;7323 hwstats->gotc -= (xon_off_tot * (ETH_ZLEN + ETH_FCS_LEN));7324 hwstats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);7325 hwstats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);7326 hwstats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);7327 hwstats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);7328 hwstats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);7329 hwstats->ptc64 -= xon_off_tot;7330 hwstats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);7331 hwstats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);7332 hwstats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);7333 hwstats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);7334 hwstats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);7335 hwstats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);7336 7337 /* Fill out the OS statistics structure */7338 netdev->stats.multicast = hwstats->mprc;7339 7340 /* Rx Errors */7341 netdev->stats.rx_errors = hwstats->crcerrs + hwstats->rlec;7342 netdev->stats.rx_dropped = 0;7343 netdev->stats.rx_length_errors = hwstats->rlec;7344 netdev->stats.rx_crc_errors = hwstats->crcerrs;7345 netdev->stats.rx_missed_errors = total_mpc;7346 7347 /* VF Stats Collection - skip while resetting because these7348 * are not clear on read and otherwise you'll sometimes get7349 * crazy values.7350 */7351 if (!test_bit(__IXGBE_RESETTING, &adapter->state)) {7352 for (i = 0; i < adapter->num_vfs; i++) {7353 UPDATE_VF_COUNTER_32bit(IXGBE_PVFGPRC(i),7354 adapter->vfinfo[i].last_vfstats.gprc,7355 adapter->vfinfo[i].vfstats.gprc);7356 UPDATE_VF_COUNTER_32bit(IXGBE_PVFGPTC(i),7357 adapter->vfinfo[i].last_vfstats.gptc,7358 adapter->vfinfo[i].vfstats.gptc);7359 UPDATE_VF_COUNTER_36bit(IXGBE_PVFGORC_LSB(i),7360 IXGBE_PVFGORC_MSB(i),7361 adapter->vfinfo[i].last_vfstats.gorc,7362 adapter->vfinfo[i].vfstats.gorc);7363 UPDATE_VF_COUNTER_36bit(IXGBE_PVFGOTC_LSB(i),7364 IXGBE_PVFGOTC_MSB(i),7365 adapter->vfinfo[i].last_vfstats.gotc,7366 adapter->vfinfo[i].vfstats.gotc);7367 UPDATE_VF_COUNTER_32bit(IXGBE_PVFMPRC(i),7368 adapter->vfinfo[i].last_vfstats.mprc,7369 adapter->vfinfo[i].vfstats.mprc);7370 }7371 }7372}7373 7374/**7375 * ixgbe_fdir_reinit_subtask - worker thread to reinit FDIR filter table7376 * @adapter: pointer to the device adapter structure7377 **/7378static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter)7379{7380 struct ixgbe_hw *hw = &adapter->hw;7381 int i;7382 7383 if (!(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))7384 return;7385 7386 adapter->flags2 &= ~IXGBE_FLAG2_FDIR_REQUIRES_REINIT;7387 7388 /* if interface is down do nothing */7389 if (test_bit(__IXGBE_DOWN, &adapter->state))7390 return;7391 7392 /* do nothing if we are not using signature filters */7393 if (!(adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE))7394 return;7395 7396 adapter->fdir_overflow++;7397 7398 if (ixgbe_reinit_fdir_tables_82599(hw) == 0) {7399 for (i = 0; i < adapter->num_tx_queues; i++)7400 set_bit(__IXGBE_TX_FDIR_INIT_DONE,7401 &(adapter->tx_ring[i]->state));7402 for (i = 0; i < adapter->num_xdp_queues; i++)7403 set_bit(__IXGBE_TX_FDIR_INIT_DONE,7404 &adapter->xdp_ring[i]->state);7405 /* re-enable flow director interrupts */7406 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR);7407 } else {7408 e_err(probe, "failed to finish FDIR re-initialization, "7409 "ignored adding FDIR ATR filters\n");7410 }7411}7412 7413/**7414 * ixgbe_check_hang_subtask - check for hung queues and dropped interrupts7415 * @adapter: pointer to the device adapter structure7416 *7417 * This function serves two purposes. First it strobes the interrupt lines7418 * in order to make certain interrupts are occurring. Secondly it sets the7419 * bits needed to check for TX hangs. As a result we should immediately7420 * determine if a hang has occurred.7421 */7422static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter)7423{7424 struct ixgbe_hw *hw = &adapter->hw;7425 u64 eics = 0;7426 int i;7427 7428 /* If we're down, removing or resetting, just bail */7429 if (test_bit(__IXGBE_DOWN, &adapter->state) ||7430 test_bit(__IXGBE_REMOVING, &adapter->state) ||7431 test_bit(__IXGBE_RESETTING, &adapter->state))7432 return;7433 7434 /* Force detection of hung controller */7435 if (netif_carrier_ok(adapter->netdev)) {7436 for (i = 0; i < adapter->num_tx_queues; i++)7437 set_check_for_tx_hang(adapter->tx_ring[i]);7438 for (i = 0; i < adapter->num_xdp_queues; i++)7439 set_check_for_tx_hang(adapter->xdp_ring[i]);7440 }7441 7442 if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) {7443 /*7444 * for legacy and MSI interrupts don't set any bits7445 * that are enabled for EIAM, because this operation7446 * would set *both* EIMS and EICS for any bit in EIAM7447 */7448 IXGBE_WRITE_REG(hw, IXGBE_EICS,7449 (IXGBE_EICS_TCP_TIMER | IXGBE_EICS_OTHER));7450 } else {7451 /* get one bit for every active tx/rx interrupt vector */7452 for (i = 0; i < adapter->num_q_vectors; i++) {7453 struct ixgbe_q_vector *qv = adapter->q_vector[i];7454 if (qv->rx.ring || qv->tx.ring)7455 eics |= BIT_ULL(i);7456 }7457 }7458 7459 /* Cause software interrupt to ensure rings are cleaned */7460 ixgbe_irq_rearm_queues(adapter, eics);7461}7462 7463/**7464 * ixgbe_watchdog_update_link - update the link status7465 * @adapter: pointer to the device adapter structure7466 **/7467static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter)7468{7469 struct ixgbe_hw *hw = &adapter->hw;7470 u32 link_speed = adapter->link_speed;7471 bool link_up = adapter->link_up;7472 bool pfc_en = adapter->dcb_cfg.pfc_mode_enable;7473 7474 if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE))7475 return;7476 7477 if (hw->mac.ops.check_link) {7478 hw->mac.ops.check_link(hw, &link_speed, &link_up, false);7479 } else {7480 /* always assume link is up, if no check link function */7481 link_speed = IXGBE_LINK_SPEED_10GB_FULL;7482 link_up = true;7483 }7484 7485 if (adapter->ixgbe_ieee_pfc)7486 pfc_en |= !!(adapter->ixgbe_ieee_pfc->pfc_en);7487 7488 if (link_up && !((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && pfc_en)) {7489 hw->mac.ops.fc_enable(hw);7490 ixgbe_set_rx_drop_en(adapter);7491 }7492 7493 if (link_up ||7494 time_after(jiffies, (adapter->link_check_timeout +7495 IXGBE_TRY_LINK_TIMEOUT))) {7496 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;7497 IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC);7498 IXGBE_WRITE_FLUSH(hw);7499 }7500 7501 adapter->link_up = link_up;7502 adapter->link_speed = link_speed;7503}7504 7505static void ixgbe_update_default_up(struct ixgbe_adapter *adapter)7506{7507#ifdef CONFIG_IXGBE_DCB7508 struct net_device *netdev = adapter->netdev;7509 struct dcb_app app = {7510 .selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE,7511 .protocol = 0,7512 };7513 u8 up = 0;7514 7515 if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_IEEE)7516 up = dcb_ieee_getapp_mask(netdev, &app);7517 7518 adapter->default_up = (up > 1) ? (ffs(up) - 1) : 0;7519#endif7520}7521 7522/**7523 * ixgbe_watchdog_link_is_up - update netif_carrier status and7524 * print link up message7525 * @adapter: pointer to the device adapter structure7526 **/7527static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter)7528{7529 struct net_device *netdev = adapter->netdev;7530 struct ixgbe_hw *hw = &adapter->hw;7531 u32 link_speed = adapter->link_speed;7532 const char *speed_str;7533 bool flow_rx, flow_tx;7534 7535 /* only continue if link was previously down */7536 if (netif_carrier_ok(netdev))7537 return;7538 7539 adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;7540 7541 switch (hw->mac.type) {7542 case ixgbe_mac_82598EB: {7543 u32 frctl = IXGBE_READ_REG(hw, IXGBE_FCTRL);7544 u32 rmcs = IXGBE_READ_REG(hw, IXGBE_RMCS);7545 flow_rx = !!(frctl & IXGBE_FCTRL_RFCE);7546 flow_tx = !!(rmcs & IXGBE_RMCS_TFCE_802_3X);7547 }7548 break;7549 case ixgbe_mac_X540:7550 case ixgbe_mac_X550:7551 case ixgbe_mac_X550EM_x:7552 case ixgbe_mac_x550em_a:7553 case ixgbe_mac_82599EB: {7554 u32 mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);7555 u32 fccfg = IXGBE_READ_REG(hw, IXGBE_FCCFG);7556 flow_rx = !!(mflcn & IXGBE_MFLCN_RFCE);7557 flow_tx = !!(fccfg & IXGBE_FCCFG_TFCE_802_3X);7558 }7559 break;7560 default:7561 flow_tx = false;7562 flow_rx = false;7563 break;7564 }7565 7566 adapter->last_rx_ptp_check = jiffies;7567 7568 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state))7569 ixgbe_ptp_start_cyclecounter(adapter);7570 7571 switch (link_speed) {7572 case IXGBE_LINK_SPEED_10GB_FULL:7573 speed_str = "10 Gbps";7574 break;7575 case IXGBE_LINK_SPEED_5GB_FULL:7576 speed_str = "5 Gbps";7577 break;7578 case IXGBE_LINK_SPEED_2_5GB_FULL:7579 speed_str = "2.5 Gbps";7580 break;7581 case IXGBE_LINK_SPEED_1GB_FULL:7582 speed_str = "1 Gbps";7583 break;7584 case IXGBE_LINK_SPEED_100_FULL:7585 speed_str = "100 Mbps";7586 break;7587 case IXGBE_LINK_SPEED_10_FULL:7588 speed_str = "10 Mbps";7589 break;7590 default:7591 speed_str = "unknown speed";7592 break;7593 }7594 e_info(drv, "NIC Link is Up %s, Flow Control: %s\n", speed_str,7595 ((flow_rx && flow_tx) ? "RX/TX" :7596 (flow_rx ? "RX" :7597 (flow_tx ? "TX" : "None"))));7598 7599 netif_carrier_on(netdev);7600 ixgbe_check_vf_rate_limit(adapter);7601 7602 /* enable transmits */7603 netif_tx_wake_all_queues(adapter->netdev);7604 7605 /* update the default user priority for VFs */7606 ixgbe_update_default_up(adapter);7607 7608 /* ping all the active vfs to let them know link has changed */7609 ixgbe_ping_all_vfs(adapter);7610}7611 7612/**7613 * ixgbe_watchdog_link_is_down - update netif_carrier status and7614 * print link down message7615 * @adapter: pointer to the adapter structure7616 **/7617static void ixgbe_watchdog_link_is_down(struct ixgbe_adapter *adapter)7618{7619 struct net_device *netdev = adapter->netdev;7620 struct ixgbe_hw *hw = &adapter->hw;7621 7622 adapter->link_up = false;7623 adapter->link_speed = 0;7624 7625 /* only continue if link was up previously */7626 if (!netif_carrier_ok(netdev))7627 return;7628 7629 /* poll for SFP+ cable when link is down */7630 if (ixgbe_is_sfp(hw) && hw->mac.type == ixgbe_mac_82598EB)7631 adapter->flags2 |= IXGBE_FLAG2_SEARCH_FOR_SFP;7632 7633 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state))7634 ixgbe_ptp_start_cyclecounter(adapter);7635 7636 e_info(drv, "NIC Link is Down\n");7637 netif_carrier_off(netdev);7638 7639 /* ping all the active vfs to let them know link has changed */7640 ixgbe_ping_all_vfs(adapter);7641}7642 7643static bool ixgbe_ring_tx_pending(struct ixgbe_adapter *adapter)7644{7645 int i;7646 7647 for (i = 0; i < adapter->num_tx_queues; i++) {7648 struct ixgbe_ring *tx_ring = adapter->tx_ring[i];7649 7650 if (tx_ring->next_to_use != tx_ring->next_to_clean)7651 return true;7652 }7653 7654 for (i = 0; i < adapter->num_xdp_queues; i++) {7655 struct ixgbe_ring *ring = adapter->xdp_ring[i];7656 7657 if (ring->next_to_use != ring->next_to_clean)7658 return true;7659 }7660 7661 return false;7662}7663 7664static bool ixgbe_vf_tx_pending(struct ixgbe_adapter *adapter)7665{7666 struct ixgbe_hw *hw = &adapter->hw;7667 struct ixgbe_ring_feature *vmdq = &adapter->ring_feature[RING_F_VMDQ];7668 u32 q_per_pool = __ALIGN_MASK(1, ~vmdq->mask);7669 7670 int i, j;7671 7672 if (!adapter->num_vfs)7673 return false;7674 7675 /* resetting the PF is only needed for MAC before X550 */7676 if (hw->mac.type >= ixgbe_mac_X550)7677 return false;7678 7679 for (i = 0; i < adapter->num_vfs; i++) {7680 for (j = 0; j < q_per_pool; j++) {7681 u32 h, t;7682 7683 h = IXGBE_READ_REG(hw, IXGBE_PVFTDHN(q_per_pool, i, j));7684 t = IXGBE_READ_REG(hw, IXGBE_PVFTDTN(q_per_pool, i, j));7685 7686 if (h != t)7687 return true;7688 }7689 }7690 7691 return false;7692}7693 7694/**7695 * ixgbe_watchdog_flush_tx - flush queues on link down7696 * @adapter: pointer to the device adapter structure7697 **/7698static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter)7699{7700 if (!netif_carrier_ok(adapter->netdev)) {7701 if (ixgbe_ring_tx_pending(adapter) ||7702 ixgbe_vf_tx_pending(adapter)) {7703 /* We've lost link, so the controller stops DMA,7704 * but we've got queued Tx work that's never going7705 * to get done, so reset controller to flush Tx.7706 * (Do the reset outside of interrupt context).7707 */7708 e_warn(drv, "initiating reset to clear Tx work after link loss\n");7709 set_bit(__IXGBE_RESET_REQUESTED, &adapter->state);7710 }7711 }7712}7713 7714#ifdef CONFIG_PCI_IOV7715static void ixgbe_bad_vf_abort(struct ixgbe_adapter *adapter, u32 vf)7716{7717 struct ixgbe_hw *hw = &adapter->hw;7718 7719 if (adapter->hw.mac.type == ixgbe_mac_82599EB &&7720 adapter->flags2 & IXGBE_FLAG2_AUTO_DISABLE_VF) {7721 adapter->vfinfo[vf].primary_abort_count++;7722 if (adapter->vfinfo[vf].primary_abort_count ==7723 IXGBE_PRIMARY_ABORT_LIMIT) {7724 ixgbe_set_vf_link_state(adapter, vf,7725 IFLA_VF_LINK_STATE_DISABLE);7726 adapter->vfinfo[vf].primary_abort_count = 0;7727 7728 e_info(drv,7729 "Malicious Driver Detection event detected on PF %d VF %d MAC: %pM mdd-disable-vf=on",7730 hw->bus.func, vf,7731 adapter->vfinfo[vf].vf_mac_addresses);7732 }7733 }7734}7735 7736static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter)7737{7738 struct ixgbe_hw *hw = &adapter->hw;7739 struct pci_dev *pdev = adapter->pdev;7740 unsigned int vf;7741 u32 gpc;7742 7743 if (!(netif_carrier_ok(adapter->netdev)))7744 return;7745 7746 gpc = IXGBE_READ_REG(hw, IXGBE_TXDGPC);7747 if (gpc) /* If incrementing then no need for the check below */7748 return;7749 /* Check to see if a bad DMA write target from an errant or7750 * malicious VF has caused a PCIe error. If so then we can7751 * issue a VFLR to the offending VF(s) and then resume without7752 * requesting a full slot reset.7753 */7754 7755 if (!pdev)7756 return;7757 7758 /* check status reg for all VFs owned by this PF */7759 for (vf = 0; vf < adapter->num_vfs; ++vf) {7760 struct pci_dev *vfdev = adapter->vfinfo[vf].vfdev;7761 u16 status_reg;7762 7763 if (!vfdev)7764 continue;7765 pci_read_config_word(vfdev, PCI_STATUS, &status_reg);7766 if (status_reg != IXGBE_FAILED_READ_CFG_WORD &&7767 status_reg & PCI_STATUS_REC_MASTER_ABORT) {7768 ixgbe_bad_vf_abort(adapter, vf);7769 pcie_flr(vfdev);7770 }7771 }7772}7773 7774static void ixgbe_spoof_check(struct ixgbe_adapter *adapter)7775{7776 u32 ssvpc;7777 7778 /* Do not perform spoof check for 82598 or if not in IOV mode */7779 if (adapter->hw.mac.type == ixgbe_mac_82598EB ||7780 adapter->num_vfs == 0)7781 return;7782 7783 ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC);7784 7785 /*7786 * ssvpc register is cleared on read, if zero then no7787 * spoofed packets in the last interval.7788 */7789 if (!ssvpc)7790 return;7791 7792 e_warn(drv, "%u Spoofed packets detected\n", ssvpc);7793}7794#else7795static void ixgbe_spoof_check(struct ixgbe_adapter __always_unused *adapter)7796{7797}7798 7799static void7800ixgbe_check_for_bad_vf(struct ixgbe_adapter __always_unused *adapter)7801{7802}7803#endif /* CONFIG_PCI_IOV */7804 7805 7806/**7807 * ixgbe_watchdog_subtask - check and bring link up7808 * @adapter: pointer to the device adapter structure7809 **/7810static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter)7811{7812 /* if interface is down, removing or resetting, do nothing */7813 if (test_bit(__IXGBE_DOWN, &adapter->state) ||7814 test_bit(__IXGBE_REMOVING, &adapter->state) ||7815 test_bit(__IXGBE_RESETTING, &adapter->state))7816 return;7817 7818 ixgbe_watchdog_update_link(adapter);7819 7820 if (adapter->link_up)7821 ixgbe_watchdog_link_is_up(adapter);7822 else7823 ixgbe_watchdog_link_is_down(adapter);7824 7825 ixgbe_check_for_bad_vf(adapter);7826 ixgbe_spoof_check(adapter);7827 ixgbe_update_stats(adapter);7828 7829 ixgbe_watchdog_flush_tx(adapter);7830}7831 7832/**7833 * ixgbe_sfp_detection_subtask - poll for SFP+ cable7834 * @adapter: the ixgbe adapter structure7835 **/7836static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter)7837{7838 struct ixgbe_hw *hw = &adapter->hw;7839 int err;7840 7841 /* not searching for SFP so there is nothing to do here */7842 if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) &&7843 !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))7844 return;7845 7846 if (adapter->sfp_poll_time &&7847 time_after(adapter->sfp_poll_time, jiffies))7848 return; /* If not yet time to poll for SFP */7849 7850 /* someone else is in init, wait until next service event */7851 if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))7852 return;7853 7854 adapter->sfp_poll_time = jiffies + IXGBE_SFP_POLL_JIFFIES - 1;7855 7856 err = hw->phy.ops.identify_sfp(hw);7857 if (err == -EOPNOTSUPP)7858 goto sfp_out;7859 7860 if (err == -ENOENT) {7861 /* If no cable is present, then we need to reset7862 * the next time we find a good cable. */7863 adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;7864 }7865 7866 /* exit on error */7867 if (err)7868 goto sfp_out;7869 7870 /* exit if reset not needed */7871 if (!(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET))7872 goto sfp_out;7873 7874 adapter->flags2 &= ~IXGBE_FLAG2_SFP_NEEDS_RESET;7875 7876 /*7877 * A module may be identified correctly, but the EEPROM may not have7878 * support for that module. setup_sfp() will fail in that case, so7879 * we should not allow that module to load.7880 */7881 if (hw->mac.type == ixgbe_mac_82598EB)7882 err = hw->phy.ops.reset(hw);7883 else7884 err = hw->mac.ops.setup_sfp(hw);7885 7886 if (err == -EOPNOTSUPP)7887 goto sfp_out;7888 7889 adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;7890 e_info(probe, "detected SFP+: %d\n", hw->phy.sfp_type);7891 7892sfp_out:7893 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);7894 7895 if (err == -EOPNOTSUPP &&7896 adapter->netdev->reg_state == NETREG_REGISTERED) {7897 e_dev_err("failed to initialize because an unsupported "7898 "SFP+ module type was detected.\n");7899 e_dev_err("Reload the driver after installing a "7900 "supported module.\n");7901 unregister_netdev(adapter->netdev);7902 }7903}7904 7905/**7906 * ixgbe_sfp_link_config_subtask - set up link SFP after module install7907 * @adapter: the ixgbe adapter structure7908 **/7909static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter)7910{7911 struct ixgbe_hw *hw = &adapter->hw;7912 u32 cap_speed;7913 u32 speed;7914 bool autoneg = false;7915 7916 if (!(adapter->flags & IXGBE_FLAG_NEED_LINK_CONFIG))7917 return;7918 7919 /* someone else is in init, wait until next service event */7920 if (test_and_set_bit(__IXGBE_IN_SFP_INIT, &adapter->state))7921 return;7922 7923 adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG;7924 7925 hw->mac.ops.get_link_capabilities(hw, &cap_speed, &autoneg);7926 7927 /* advertise highest capable link speed */7928 if (!autoneg && (cap_speed & IXGBE_LINK_SPEED_10GB_FULL))7929 speed = IXGBE_LINK_SPEED_10GB_FULL;7930 else7931 speed = cap_speed & (IXGBE_LINK_SPEED_10GB_FULL |7932 IXGBE_LINK_SPEED_1GB_FULL);7933 7934 if (hw->mac.ops.setup_link)7935 hw->mac.ops.setup_link(hw, speed, true);7936 7937 adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;7938 adapter->link_check_timeout = jiffies;7939 clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state);7940}7941 7942/**7943 * ixgbe_service_timer - Timer Call-back7944 * @t: pointer to timer_list structure7945 **/7946static void ixgbe_service_timer(struct timer_list *t)7947{7948 struct ixgbe_adapter *adapter = from_timer(adapter, t, service_timer);7949 unsigned long next_event_offset;7950 7951 /* poll faster when waiting for link */7952 if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)7953 next_event_offset = HZ / 10;7954 else7955 next_event_offset = HZ * 2;7956 7957 /* Reset the timer */7958 mod_timer(&adapter->service_timer, next_event_offset + jiffies);7959 7960 ixgbe_service_event_schedule(adapter);7961}7962 7963static void ixgbe_phy_interrupt_subtask(struct ixgbe_adapter *adapter)7964{7965 struct ixgbe_hw *hw = &adapter->hw;7966 bool overtemp;7967 7968 if (!(adapter->flags2 & IXGBE_FLAG2_PHY_INTERRUPT))7969 return;7970 7971 adapter->flags2 &= ~IXGBE_FLAG2_PHY_INTERRUPT;7972 7973 if (!hw->phy.ops.handle_lasi)7974 return;7975 7976 hw->phy.ops.handle_lasi(&adapter->hw, &overtemp);7977 if (overtemp)7978 e_crit(drv, "%s\n", ixgbe_overheat_msg);7979}7980 7981static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter)7982{7983 if (!test_and_clear_bit(__IXGBE_RESET_REQUESTED, &adapter->state))7984 return;7985 7986 rtnl_lock();7987 /* If we're already down, removing or resetting, just bail */7988 if (test_bit(__IXGBE_DOWN, &adapter->state) ||7989 test_bit(__IXGBE_REMOVING, &adapter->state) ||7990 test_bit(__IXGBE_RESETTING, &adapter->state)) {7991 rtnl_unlock();7992 return;7993 }7994 7995 ixgbe_dump(adapter);7996 netdev_err(adapter->netdev, "Reset adapter\n");7997 adapter->tx_timeout_count++;7998 7999 ixgbe_reinit_locked(adapter);8000 rtnl_unlock();8001}8002 8003/**8004 * ixgbe_check_fw_error - Check firmware for errors8005 * @adapter: the adapter private structure8006 *8007 * Check firmware errors in register FWSM8008 */8009static bool ixgbe_check_fw_error(struct ixgbe_adapter *adapter)8010{8011 struct ixgbe_hw *hw = &adapter->hw;8012 u32 fwsm;8013 8014 /* read fwsm.ext_err_ind register and log errors */8015 fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM(hw));8016 8017 if (fwsm & IXGBE_FWSM_EXT_ERR_IND_MASK ||8018 !(fwsm & IXGBE_FWSM_FW_VAL_BIT))8019 e_dev_warn("Warning firmware error detected FWSM: 0x%08X\n",8020 fwsm);8021 8022 if (hw->mac.ops.fw_recovery_mode && hw->mac.ops.fw_recovery_mode(hw)) {8023 e_dev_err("Firmware recovery mode detected. Limiting functionality. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");8024 return true;8025 }8026 8027 return false;8028}8029 8030/**8031 * ixgbe_service_task - manages and runs subtasks8032 * @work: pointer to work_struct containing our data8033 **/8034static void ixgbe_service_task(struct work_struct *work)8035{8036 struct ixgbe_adapter *adapter = container_of(work,8037 struct ixgbe_adapter,8038 service_task);8039 if (ixgbe_removed(adapter->hw.hw_addr)) {8040 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {8041 rtnl_lock();8042 ixgbe_down(adapter);8043 rtnl_unlock();8044 }8045 ixgbe_service_event_complete(adapter);8046 return;8047 }8048 if (ixgbe_check_fw_error(adapter)) {8049 if (!test_bit(__IXGBE_DOWN, &adapter->state))8050 unregister_netdev(adapter->netdev);8051 ixgbe_service_event_complete(adapter);8052 return;8053 }8054 ixgbe_reset_subtask(adapter);8055 ixgbe_phy_interrupt_subtask(adapter);8056 ixgbe_sfp_detection_subtask(adapter);8057 ixgbe_sfp_link_config_subtask(adapter);8058 ixgbe_check_overtemp_subtask(adapter);8059 ixgbe_watchdog_subtask(adapter);8060 ixgbe_fdir_reinit_subtask(adapter);8061 ixgbe_check_hang_subtask(adapter);8062 8063 if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state)) {8064 ixgbe_ptp_overflow_check(adapter);8065 if (adapter->flags & IXGBE_FLAG_RX_HWTSTAMP_IN_REGISTER)8066 ixgbe_ptp_rx_hang(adapter);8067 ixgbe_ptp_tx_hang(adapter);8068 }8069 8070 ixgbe_service_event_complete(adapter);8071}8072 8073static int ixgbe_tso(struct ixgbe_ring *tx_ring,8074 struct ixgbe_tx_buffer *first,8075 u8 *hdr_len,8076 struct ixgbe_ipsec_tx_data *itd)8077{8078 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;8079 struct sk_buff *skb = first->skb;8080 union {8081 struct iphdr *v4;8082 struct ipv6hdr *v6;8083 unsigned char *hdr;8084 } ip;8085 union {8086 struct tcphdr *tcp;8087 struct udphdr *udp;8088 unsigned char *hdr;8089 } l4;8090 u32 paylen, l4_offset;8091 u32 fceof_saidx = 0;8092 int err;8093 8094 if (skb->ip_summed != CHECKSUM_PARTIAL)8095 return 0;8096 8097 if (!skb_is_gso(skb))8098 return 0;8099 8100 err = skb_cow_head(skb, 0);8101 if (err < 0)8102 return err;8103 8104 if (eth_p_mpls(first->protocol))8105 ip.hdr = skb_inner_network_header(skb);8106 else8107 ip.hdr = skb_network_header(skb);8108 l4.hdr = skb_checksum_start(skb);8109 8110 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */8111 type_tucmd = (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ?8112 IXGBE_ADVTXD_TUCMD_L4T_UDP : IXGBE_ADVTXD_TUCMD_L4T_TCP;8113 8114 /* initialize outer IP header fields */8115 if (ip.v4->version == 4) {8116 unsigned char *csum_start = skb_checksum_start(skb);8117 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);8118 int len = csum_start - trans_start;8119 8120 /* IP header will have to cancel out any data that8121 * is not a part of the outer IP header, so set to8122 * a reverse csum if needed, else init check to 0.8123 */8124 ip.v4->check = (skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) ?8125 csum_fold(csum_partial(trans_start,8126 len, 0)) : 0;8127 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;8128 8129 ip.v4->tot_len = 0;8130 first->tx_flags |= IXGBE_TX_FLAGS_TSO |8131 IXGBE_TX_FLAGS_CSUM |8132 IXGBE_TX_FLAGS_IPV4;8133 } else {8134 ip.v6->payload_len = 0;8135 first->tx_flags |= IXGBE_TX_FLAGS_TSO |8136 IXGBE_TX_FLAGS_CSUM;8137 }8138 8139 /* determine offset of inner transport header */8140 l4_offset = l4.hdr - skb->data;8141 8142 /* remove payload length from inner checksum */8143 paylen = skb->len - l4_offset;8144 8145 if (type_tucmd & IXGBE_ADVTXD_TUCMD_L4T_TCP) {8146 /* compute length of segmentation header */8147 *hdr_len = (l4.tcp->doff * 4) + l4_offset;8148 csum_replace_by_diff(&l4.tcp->check,8149 (__force __wsum)htonl(paylen));8150 } else {8151 /* compute length of segmentation header */8152 *hdr_len = sizeof(*l4.udp) + l4_offset;8153 csum_replace_by_diff(&l4.udp->check,8154 (__force __wsum)htonl(paylen));8155 }8156 8157 /* update gso size and bytecount with header size */8158 first->gso_segs = skb_shinfo(skb)->gso_segs;8159 first->bytecount += (first->gso_segs - 1) * *hdr_len;8160 8161 /* mss_l4len_id: use 0 as index for TSO */8162 mss_l4len_idx = (*hdr_len - l4_offset) << IXGBE_ADVTXD_L4LEN_SHIFT;8163 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;8164 8165 fceof_saidx |= itd->sa_idx;8166 type_tucmd |= itd->flags | itd->trailer_len;8167 8168 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */8169 vlan_macip_lens = l4.hdr - ip.hdr;8170 vlan_macip_lens |= (ip.hdr - skb->data) << IXGBE_ADVTXD_MACLEN_SHIFT;8171 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;8172 8173 ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, fceof_saidx, type_tucmd,8174 mss_l4len_idx);8175 8176 return 1;8177}8178 8179static void ixgbe_tx_csum(struct ixgbe_ring *tx_ring,8180 struct ixgbe_tx_buffer *first,8181 struct ixgbe_ipsec_tx_data *itd)8182{8183 struct sk_buff *skb = first->skb;8184 u32 vlan_macip_lens = 0;8185 u32 fceof_saidx = 0;8186 u32 type_tucmd = 0;8187 8188 if (skb->ip_summed != CHECKSUM_PARTIAL) {8189csum_failed:8190 if (!(first->tx_flags & (IXGBE_TX_FLAGS_HW_VLAN |8191 IXGBE_TX_FLAGS_CC)))8192 return;8193 goto no_csum;8194 }8195 8196 switch (skb->csum_offset) {8197 case offsetof(struct tcphdr, check):8198 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;8199 fallthrough;8200 case offsetof(struct udphdr, check):8201 break;8202 case offsetof(struct sctphdr, checksum):8203 /* validate that this is actually an SCTP request */8204 if (skb_csum_is_sctp(skb)) {8205 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_SCTP;8206 break;8207 }8208 fallthrough;8209 default:8210 skb_checksum_help(skb);8211 goto csum_failed;8212 }8213 8214 /* update TX checksum flag */8215 first->tx_flags |= IXGBE_TX_FLAGS_CSUM;8216 vlan_macip_lens = skb_checksum_start_offset(skb) -8217 skb_network_offset(skb);8218no_csum:8219 /* vlan_macip_lens: MACLEN, VLAN tag */8220 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;8221 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;8222 8223 fceof_saidx |= itd->sa_idx;8224 type_tucmd |= itd->flags | itd->trailer_len;8225 8226 ixgbe_tx_ctxtdesc(tx_ring, vlan_macip_lens, fceof_saidx, type_tucmd, 0);8227}8228 8229#define IXGBE_SET_FLAG(_input, _flag, _result) \8230 ((_flag <= _result) ? \8231 ((u32)(_input & _flag) * (_result / _flag)) : \8232 ((u32)(_input & _flag) / (_flag / _result)))8233 8234static u32 ixgbe_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)8235{8236 /* set type for advanced descriptor with frame checksum insertion */8237 u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA |8238 IXGBE_ADVTXD_DCMD_DEXT |8239 IXGBE_ADVTXD_DCMD_IFCS;8240 8241 /* set HW vlan bit if vlan is present */8242 cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_HW_VLAN,8243 IXGBE_ADVTXD_DCMD_VLE);8244 8245 /* set segmentation enable bits for TSO/FSO */8246 cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_TSO,8247 IXGBE_ADVTXD_DCMD_TSE);8248 8249 /* set timestamp bit if present */8250 cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_TSTAMP,8251 IXGBE_ADVTXD_MAC_TSTAMP);8252 8253 /* insert frame checksum */8254 cmd_type ^= IXGBE_SET_FLAG(skb->no_fcs, 1, IXGBE_ADVTXD_DCMD_IFCS);8255 8256 return cmd_type;8257}8258 8259static void ixgbe_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc,8260 u32 tx_flags, unsigned int paylen)8261{8262 u32 olinfo_status = paylen << IXGBE_ADVTXD_PAYLEN_SHIFT;8263 8264 /* enable L4 checksum for TSO and TX checksum offload */8265 olinfo_status |= IXGBE_SET_FLAG(tx_flags,8266 IXGBE_TX_FLAGS_CSUM,8267 IXGBE_ADVTXD_POPTS_TXSM);8268 8269 /* enable IPv4 checksum for TSO */8270 olinfo_status |= IXGBE_SET_FLAG(tx_flags,8271 IXGBE_TX_FLAGS_IPV4,8272 IXGBE_ADVTXD_POPTS_IXSM);8273 8274 /* enable IPsec */8275 olinfo_status |= IXGBE_SET_FLAG(tx_flags,8276 IXGBE_TX_FLAGS_IPSEC,8277 IXGBE_ADVTXD_POPTS_IPSEC);8278 8279 /*8280 * Check Context must be set if Tx switch is enabled, which it8281 * always is for case where virtual functions are running8282 */8283 olinfo_status |= IXGBE_SET_FLAG(tx_flags,8284 IXGBE_TX_FLAGS_CC,8285 IXGBE_ADVTXD_CC);8286 8287 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);8288}8289 8290static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)8291{8292 if (!netif_subqueue_try_stop(tx_ring->netdev, tx_ring->queue_index,8293 ixgbe_desc_unused(tx_ring), size))8294 return -EBUSY;8295 8296 ++tx_ring->tx_stats.restart_queue;8297 return 0;8298}8299 8300static inline int ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size)8301{8302 if (likely(ixgbe_desc_unused(tx_ring) >= size))8303 return 0;8304 8305 return __ixgbe_maybe_stop_tx(tx_ring, size);8306}8307 8308static int ixgbe_tx_map(struct ixgbe_ring *tx_ring,8309 struct ixgbe_tx_buffer *first,8310 const u8 hdr_len)8311{8312 struct sk_buff *skb = first->skb;8313 struct ixgbe_tx_buffer *tx_buffer;8314 union ixgbe_adv_tx_desc *tx_desc;8315 skb_frag_t *frag;8316 dma_addr_t dma;8317 unsigned int data_len, size;8318 u32 tx_flags = first->tx_flags;8319 u32 cmd_type = ixgbe_tx_cmd_type(skb, tx_flags);8320 u16 i = tx_ring->next_to_use;8321 8322 tx_desc = IXGBE_TX_DESC(tx_ring, i);8323 8324 ixgbe_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len);8325 8326 size = skb_headlen(skb);8327 data_len = skb->data_len;8328 8329#ifdef IXGBE_FCOE8330 if (tx_flags & IXGBE_TX_FLAGS_FCOE) {8331 if (data_len < sizeof(struct fcoe_crc_eof)) {8332 size -= sizeof(struct fcoe_crc_eof) - data_len;8333 data_len = 0;8334 } else {8335 data_len -= sizeof(struct fcoe_crc_eof);8336 }8337 }8338 8339#endif8340 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);8341 8342 tx_buffer = first;8343 8344 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {8345 if (dma_mapping_error(tx_ring->dev, dma))8346 goto dma_error;8347 8348 /* record length, and DMA address */8349 dma_unmap_len_set(tx_buffer, len, size);8350 dma_unmap_addr_set(tx_buffer, dma, dma);8351 8352 tx_desc->read.buffer_addr = cpu_to_le64(dma);8353 8354 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) {8355 tx_desc->read.cmd_type_len =8356 cpu_to_le32(cmd_type ^ IXGBE_MAX_DATA_PER_TXD);8357 8358 i++;8359 tx_desc++;8360 if (i == tx_ring->count) {8361 tx_desc = IXGBE_TX_DESC(tx_ring, 0);8362 i = 0;8363 }8364 tx_desc->read.olinfo_status = 0;8365 8366 dma += IXGBE_MAX_DATA_PER_TXD;8367 size -= IXGBE_MAX_DATA_PER_TXD;8368 8369 tx_desc->read.buffer_addr = cpu_to_le64(dma);8370 }8371 8372 if (likely(!data_len))8373 break;8374 8375 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);8376 8377 i++;8378 tx_desc++;8379 if (i == tx_ring->count) {8380 tx_desc = IXGBE_TX_DESC(tx_ring, 0);8381 i = 0;8382 }8383 tx_desc->read.olinfo_status = 0;8384 8385#ifdef IXGBE_FCOE8386 size = min_t(unsigned int, data_len, skb_frag_size(frag));8387#else8388 size = skb_frag_size(frag);8389#endif8390 data_len -= size;8391 8392 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,8393 DMA_TO_DEVICE);8394 8395 tx_buffer = &tx_ring->tx_buffer_info[i];8396 }8397 8398 /* write last descriptor with RS and EOP bits */8399 cmd_type |= size | IXGBE_TXD_CMD;8400 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);8401 8402 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);8403 8404 /* set the timestamp */8405 first->time_stamp = jiffies;8406 8407 skb_tx_timestamp(skb);8408 8409 /*8410 * Force memory writes to complete before letting h/w know there8411 * are new descriptors to fetch. (Only applicable for weak-ordered8412 * memory model archs, such as IA-64).8413 *8414 * We also need this memory barrier to make certain all of the8415 * status bits have been updated before next_to_watch is written.8416 */8417 wmb();8418 8419 /* set next_to_watch value indicating a packet is present */8420 first->next_to_watch = tx_desc;8421 8422 i++;8423 if (i == tx_ring->count)8424 i = 0;8425 8426 tx_ring->next_to_use = i;8427 8428 ixgbe_maybe_stop_tx(tx_ring, DESC_NEEDED);8429 8430 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {8431 writel(i, tx_ring->tail);8432 }8433 8434 return 0;8435dma_error:8436 dev_err(tx_ring->dev, "TX DMA map failed\n");8437 8438 /* clear dma mappings for failed tx_buffer_info map */8439 for (;;) {8440 tx_buffer = &tx_ring->tx_buffer_info[i];8441 if (dma_unmap_len(tx_buffer, len))8442 dma_unmap_page(tx_ring->dev,8443 dma_unmap_addr(tx_buffer, dma),8444 dma_unmap_len(tx_buffer, len),8445 DMA_TO_DEVICE);8446 dma_unmap_len_set(tx_buffer, len, 0);8447 if (tx_buffer == first)8448 break;8449 if (i == 0)8450 i += tx_ring->count;8451 i--;8452 }8453 8454 dev_kfree_skb_any(first->skb);8455 first->skb = NULL;8456 8457 tx_ring->next_to_use = i;8458 8459 return -1;8460}8461 8462static void ixgbe_atr(struct ixgbe_ring *ring,8463 struct ixgbe_tx_buffer *first)8464{8465 struct ixgbe_q_vector *q_vector = ring->q_vector;8466 union ixgbe_atr_hash_dword input = { .dword = 0 };8467 union ixgbe_atr_hash_dword common = { .dword = 0 };8468 union {8469 unsigned char *network;8470 struct iphdr *ipv4;8471 struct ipv6hdr *ipv6;8472 } hdr;8473 struct tcphdr *th;8474 unsigned int hlen;8475 struct sk_buff *skb;8476 __be16 vlan_id;8477 int l4_proto;8478 8479 /* if ring doesn't have a interrupt vector, cannot perform ATR */8480 if (!q_vector)8481 return;8482 8483 /* do nothing if sampling is disabled */8484 if (!ring->atr_sample_rate)8485 return;8486 8487 ring->atr_count++;8488 8489 /* currently only IPv4/IPv6 with TCP is supported */8490 if ((first->protocol != htons(ETH_P_IP)) &&8491 (first->protocol != htons(ETH_P_IPV6)))8492 return;8493 8494 /* snag network header to get L4 type and address */8495 skb = first->skb;8496 hdr.network = skb_network_header(skb);8497 if (unlikely(hdr.network <= skb->data))8498 return;8499 if (skb->encapsulation &&8500 first->protocol == htons(ETH_P_IP) &&8501 hdr.ipv4->protocol == IPPROTO_UDP) {8502 struct ixgbe_adapter *adapter = q_vector->adapter;8503 8504 if (unlikely(skb_tail_pointer(skb) < hdr.network +8505 vxlan_headroom(0)))8506 return;8507 8508 /* verify the port is recognized as VXLAN */8509 if (adapter->vxlan_port &&8510 udp_hdr(skb)->dest == adapter->vxlan_port)8511 hdr.network = skb_inner_network_header(skb);8512 8513 if (adapter->geneve_port &&8514 udp_hdr(skb)->dest == adapter->geneve_port)8515 hdr.network = skb_inner_network_header(skb);8516 }8517 8518 /* Make sure we have at least [minimum IPv4 header + TCP]8519 * or [IPv6 header] bytes8520 */8521 if (unlikely(skb_tail_pointer(skb) < hdr.network + 40))8522 return;8523 8524 /* Currently only IPv4/IPv6 with TCP is supported */8525 switch (hdr.ipv4->version) {8526 case IPVERSION:8527 /* access ihl as u8 to avoid unaligned access on ia64 */8528 hlen = (hdr.network[0] & 0x0F) << 2;8529 l4_proto = hdr.ipv4->protocol;8530 break;8531 case 6:8532 hlen = hdr.network - skb->data;8533 l4_proto = ipv6_find_hdr(skb, &hlen, IPPROTO_TCP, NULL, NULL);8534 hlen -= hdr.network - skb->data;8535 break;8536 default:8537 return;8538 }8539 8540 if (l4_proto != IPPROTO_TCP)8541 return;8542 8543 if (unlikely(skb_tail_pointer(skb) < hdr.network +8544 hlen + sizeof(struct tcphdr)))8545 return;8546 8547 th = (struct tcphdr *)(hdr.network + hlen);8548 8549 /* skip this packet since the socket is closing */8550 if (th->fin)8551 return;8552 8553 /* sample on all syn packets or once every atr sample count */8554 if (!th->syn && (ring->atr_count < ring->atr_sample_rate))8555 return;8556 8557 /* reset sample count */8558 ring->atr_count = 0;8559 8560 vlan_id = htons(first->tx_flags >> IXGBE_TX_FLAGS_VLAN_SHIFT);8561 8562 /*8563 * src and dst are inverted, think how the receiver sees them8564 *8565 * The input is broken into two sections, a non-compressed section8566 * containing vm_pool, vlan_id, and flow_type. The rest of the data8567 * is XORed together and stored in the compressed dword.8568 */8569 input.formatted.vlan_id = vlan_id;8570 8571 /*8572 * since src port and flex bytes occupy the same word XOR them together8573 * and write the value to source port portion of compressed dword8574 */8575 if (first->tx_flags & (IXGBE_TX_FLAGS_SW_VLAN | IXGBE_TX_FLAGS_HW_VLAN))8576 common.port.src ^= th->dest ^ htons(ETH_P_8021Q);8577 else8578 common.port.src ^= th->dest ^ first->protocol;8579 common.port.dst ^= th->source;8580 8581 switch (hdr.ipv4->version) {8582 case IPVERSION:8583 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV4;8584 common.ip ^= hdr.ipv4->saddr ^ hdr.ipv4->daddr;8585 break;8586 case 6:8587 input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6;8588 common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^8589 hdr.ipv6->saddr.s6_addr32[1] ^8590 hdr.ipv6->saddr.s6_addr32[2] ^8591 hdr.ipv6->saddr.s6_addr32[3] ^8592 hdr.ipv6->daddr.s6_addr32[0] ^8593 hdr.ipv6->daddr.s6_addr32[1] ^8594 hdr.ipv6->daddr.s6_addr32[2] ^8595 hdr.ipv6->daddr.s6_addr32[3];8596 break;8597 default:8598 break;8599 }8600 8601 if (hdr.network != skb_network_header(skb))8602 input.formatted.flow_type |= IXGBE_ATR_L4TYPE_TUNNEL_MASK;8603 8604 /* This assumes the Rx queue and Tx queue are bound to the same CPU */8605 ixgbe_fdir_add_signature_filter_82599(&q_vector->adapter->hw,8606 input, common, ring->queue_index);8607}8608 8609#ifdef IXGBE_FCOE8610static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb,8611 struct net_device *sb_dev)8612{8613 struct ixgbe_adapter *adapter;8614 struct ixgbe_ring_feature *f;8615 int txq;8616 8617 if (sb_dev) {8618 u8 tc = netdev_get_prio_tc_map(dev, skb->priority);8619 struct net_device *vdev = sb_dev;8620 8621 txq = vdev->tc_to_txq[tc].offset;8622 txq += reciprocal_scale(skb_get_hash(skb),8623 vdev->tc_to_txq[tc].count);8624 8625 return txq;8626 }8627 8628 /*8629 * only execute the code below if protocol is FCoE8630 * or FIP and we have FCoE enabled on the adapter8631 */8632 switch (vlan_get_protocol(skb)) {8633 case htons(ETH_P_FCOE):8634 case htons(ETH_P_FIP):8635 adapter = netdev_priv(dev);8636 8637 if (!sb_dev && (adapter->flags & IXGBE_FLAG_FCOE_ENABLED))8638 break;8639 fallthrough;8640 default:8641 return netdev_pick_tx(dev, skb, sb_dev);8642 }8643 8644 f = &adapter->ring_feature[RING_F_FCOE];8645 8646 txq = skb_rx_queue_recorded(skb) ? skb_get_rx_queue(skb) :8647 smp_processor_id();8648 8649 while (txq >= f->indices)8650 txq -= f->indices;8651 8652 return txq + f->offset;8653}8654 8655#endif8656int ixgbe_xmit_xdp_ring(struct ixgbe_ring *ring,8657 struct xdp_frame *xdpf)8658{8659 struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);8660 u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;8661 u16 i = 0, index = ring->next_to_use;8662 struct ixgbe_tx_buffer *tx_head = &ring->tx_buffer_info[index];8663 struct ixgbe_tx_buffer *tx_buff = tx_head;8664 union ixgbe_adv_tx_desc *tx_desc = IXGBE_TX_DESC(ring, index);8665 u32 cmd_type, len = xdpf->len;8666 void *data = xdpf->data;8667 8668 if (unlikely(ixgbe_desc_unused(ring) < 1 + nr_frags))8669 return IXGBE_XDP_CONSUMED;8670 8671 tx_head->bytecount = xdp_get_frame_len(xdpf);8672 tx_head->gso_segs = 1;8673 tx_head->xdpf = xdpf;8674 8675 tx_desc->read.olinfo_status =8676 cpu_to_le32(tx_head->bytecount << IXGBE_ADVTXD_PAYLEN_SHIFT);8677 8678 for (;;) {8679 dma_addr_t dma;8680 8681 dma = dma_map_single(ring->dev, data, len, DMA_TO_DEVICE);8682 if (dma_mapping_error(ring->dev, dma))8683 goto unmap;8684 8685 dma_unmap_len_set(tx_buff, len, len);8686 dma_unmap_addr_set(tx_buff, dma, dma);8687 8688 cmd_type = IXGBE_ADVTXD_DTYP_DATA | IXGBE_ADVTXD_DCMD_DEXT |8689 IXGBE_ADVTXD_DCMD_IFCS | len;8690 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);8691 tx_desc->read.buffer_addr = cpu_to_le64(dma);8692 tx_buff->protocol = 0;8693 8694 if (++index == ring->count)8695 index = 0;8696 8697 if (i == nr_frags)8698 break;8699 8700 tx_buff = &ring->tx_buffer_info[index];8701 tx_desc = IXGBE_TX_DESC(ring, index);8702 tx_desc->read.olinfo_status = 0;8703 8704 data = skb_frag_address(&sinfo->frags[i]);8705 len = skb_frag_size(&sinfo->frags[i]);8706 i++;8707 }8708 /* put descriptor type bits */8709 tx_desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD);8710 8711 /* Avoid any potential race with xdp_xmit and cleanup */8712 smp_wmb();8713 8714 tx_head->next_to_watch = tx_desc;8715 ring->next_to_use = index;8716 8717 return IXGBE_XDP_TX;8718 8719unmap:8720 for (;;) {8721 tx_buff = &ring->tx_buffer_info[index];8722 if (dma_unmap_len(tx_buff, len))8723 dma_unmap_page(ring->dev, dma_unmap_addr(tx_buff, dma),8724 dma_unmap_len(tx_buff, len),8725 DMA_TO_DEVICE);8726 dma_unmap_len_set(tx_buff, len, 0);8727 if (tx_buff == tx_head)8728 break;8729 8730 if (!index)8731 index += ring->count;8732 index--;8733 }8734 8735 return IXGBE_XDP_CONSUMED;8736}8737 8738netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb,8739 struct ixgbe_adapter *adapter,8740 struct ixgbe_ring *tx_ring)8741{8742 struct ixgbe_tx_buffer *first;8743 int tso;8744 u32 tx_flags = 0;8745 unsigned short f;8746 u16 count = TXD_USE_COUNT(skb_headlen(skb));8747 struct ixgbe_ipsec_tx_data ipsec_tx = { 0 };8748 __be16 protocol = skb->protocol;8749 u8 hdr_len = 0;8750 8751 /*8752 * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,8753 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,8754 * + 2 desc gap to keep tail from touching head,8755 * + 1 desc for context descriptor,8756 * otherwise try next time8757 */8758 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)8759 count += TXD_USE_COUNT(skb_frag_size(8760 &skb_shinfo(skb)->frags[f]));8761 8762 if (ixgbe_maybe_stop_tx(tx_ring, count + 3)) {8763 tx_ring->tx_stats.tx_busy++;8764 return NETDEV_TX_BUSY;8765 }8766 8767 /* record the location of the first descriptor for this packet */8768 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];8769 first->skb = skb;8770 first->bytecount = skb->len;8771 first->gso_segs = 1;8772 8773 /* if we have a HW VLAN tag being added default to the HW one */8774 if (skb_vlan_tag_present(skb)) {8775 tx_flags |= skb_vlan_tag_get(skb) << IXGBE_TX_FLAGS_VLAN_SHIFT;8776 tx_flags |= IXGBE_TX_FLAGS_HW_VLAN;8777 /* else if it is a SW VLAN check the next protocol and store the tag */8778 } else if (protocol == htons(ETH_P_8021Q)) {8779 struct vlan_hdr *vhdr, _vhdr;8780 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);8781 if (!vhdr)8782 goto out_drop;8783 8784 tx_flags |= ntohs(vhdr->h_vlan_TCI) <<8785 IXGBE_TX_FLAGS_VLAN_SHIFT;8786 tx_flags |= IXGBE_TX_FLAGS_SW_VLAN;8787 }8788 protocol = vlan_get_protocol(skb);8789 8790 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&8791 adapter->ptp_clock) {8792 if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON &&8793 !test_and_set_bit_lock(__IXGBE_PTP_TX_IN_PROGRESS,8794 &adapter->state)) {8795 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;8796 tx_flags |= IXGBE_TX_FLAGS_TSTAMP;8797 8798 /* schedule check for Tx timestamp */8799 adapter->ptp_tx_skb = skb_get(skb);8800 adapter->ptp_tx_start = jiffies;8801 schedule_work(&adapter->ptp_tx_work);8802 } else {8803 adapter->tx_hwtstamp_skipped++;8804 }8805 }8806 8807#ifdef CONFIG_PCI_IOV8808 /*8809 * Use the l2switch_enable flag - would be false if the DMA8810 * Tx switch had been disabled.8811 */8812 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)8813 tx_flags |= IXGBE_TX_FLAGS_CC;8814 8815#endif8816 /* DCB maps skb priorities 0-7 onto 3 bit PCP of VLAN tag. */8817 if ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) &&8818 ((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) ||8819 (skb->priority != TC_PRIO_CONTROL))) {8820 tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK;8821 tx_flags |= (skb->priority & 0x7) <<8822 IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT;8823 if (tx_flags & IXGBE_TX_FLAGS_SW_VLAN) {8824 struct vlan_ethhdr *vhdr;8825 8826 if (skb_cow_head(skb, 0))8827 goto out_drop;8828 vhdr = skb_vlan_eth_hdr(skb);8829 vhdr->h_vlan_TCI = htons(tx_flags >>8830 IXGBE_TX_FLAGS_VLAN_SHIFT);8831 } else {8832 tx_flags |= IXGBE_TX_FLAGS_HW_VLAN;8833 }8834 }8835 8836 /* record initial flags and protocol */8837 first->tx_flags = tx_flags;8838 first->protocol = protocol;8839 8840#ifdef IXGBE_FCOE8841 /* setup tx offload for FCoE */8842 if ((protocol == htons(ETH_P_FCOE)) &&8843 (tx_ring->netdev->features & (NETIF_F_FSO | NETIF_F_FCOE_CRC))) {8844 tso = ixgbe_fso(tx_ring, first, &hdr_len);8845 if (tso < 0)8846 goto out_drop;8847 8848 goto xmit_fcoe;8849 }8850 8851#endif /* IXGBE_FCOE */8852 8853#ifdef CONFIG_IXGBE_IPSEC8854 if (xfrm_offload(skb) &&8855 !ixgbe_ipsec_tx(tx_ring, first, &ipsec_tx))8856 goto out_drop;8857#endif8858 tso = ixgbe_tso(tx_ring, first, &hdr_len, &ipsec_tx);8859 if (tso < 0)8860 goto out_drop;8861 else if (!tso)8862 ixgbe_tx_csum(tx_ring, first, &ipsec_tx);8863 8864 /* add the ATR filter if ATR is on */8865 if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, &tx_ring->state))8866 ixgbe_atr(tx_ring, first);8867 8868#ifdef IXGBE_FCOE8869xmit_fcoe:8870#endif /* IXGBE_FCOE */8871 if (ixgbe_tx_map(tx_ring, first, hdr_len))8872 goto cleanup_tx_timestamp;8873 8874 return NETDEV_TX_OK;8875 8876out_drop:8877 dev_kfree_skb_any(first->skb);8878 first->skb = NULL;8879cleanup_tx_timestamp:8880 if (unlikely(tx_flags & IXGBE_TX_FLAGS_TSTAMP)) {8881 dev_kfree_skb_any(adapter->ptp_tx_skb);8882 adapter->ptp_tx_skb = NULL;8883 cancel_work_sync(&adapter->ptp_tx_work);8884 clear_bit_unlock(__IXGBE_PTP_TX_IN_PROGRESS, &adapter->state);8885 }8886 8887 return NETDEV_TX_OK;8888}8889 8890static netdev_tx_t __ixgbe_xmit_frame(struct sk_buff *skb,8891 struct net_device *netdev,8892 struct ixgbe_ring *ring)8893{8894 struct ixgbe_adapter *adapter = netdev_priv(netdev);8895 struct ixgbe_ring *tx_ring;8896 8897 /*8898 * The minimum packet size for olinfo paylen is 17 so pad the skb8899 * in order to meet this minimum size requirement.8900 */8901 if (skb_put_padto(skb, 17))8902 return NETDEV_TX_OK;8903 8904 tx_ring = ring ? ring : adapter->tx_ring[skb_get_queue_mapping(skb)];8905 if (unlikely(test_bit(__IXGBE_TX_DISABLED, &tx_ring->state)))8906 return NETDEV_TX_BUSY;8907 8908 return ixgbe_xmit_frame_ring(skb, adapter, tx_ring);8909}8910 8911static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb,8912 struct net_device *netdev)8913{8914 return __ixgbe_xmit_frame(skb, netdev, NULL);8915}8916 8917/**8918 * ixgbe_set_mac - Change the Ethernet Address of the NIC8919 * @netdev: network interface device structure8920 * @p: pointer to an address structure8921 *8922 * Returns 0 on success, negative on failure8923 **/8924static int ixgbe_set_mac(struct net_device *netdev, void *p)8925{8926 struct ixgbe_adapter *adapter = netdev_priv(netdev);8927 struct ixgbe_hw *hw = &adapter->hw;8928 struct sockaddr *addr = p;8929 8930 if (!is_valid_ether_addr(addr->sa_data))8931 return -EADDRNOTAVAIL;8932 8933 eth_hw_addr_set(netdev, addr->sa_data);8934 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);8935 8936 ixgbe_mac_set_default_filter(adapter);8937 8938 return 0;8939}8940 8941static int8942ixgbe_mdio_read(struct net_device *netdev, int prtad, int devad, u16 addr)8943{8944 struct ixgbe_adapter *adapter = netdev_priv(netdev);8945 struct ixgbe_hw *hw = &adapter->hw;8946 u16 value;8947 int rc;8948 8949 if (adapter->mii_bus) {8950 int regnum = addr;8951 8952 if (devad != MDIO_DEVAD_NONE)8953 return mdiobus_c45_read(adapter->mii_bus, prtad,8954 devad, regnum);8955 8956 return mdiobus_read(adapter->mii_bus, prtad, regnum);8957 }8958 8959 if (prtad != hw->phy.mdio.prtad)8960 return -EINVAL;8961 rc = hw->phy.ops.read_reg(hw, addr, devad, &value);8962 if (!rc)8963 rc = value;8964 return rc;8965}8966 8967static int ixgbe_mdio_write(struct net_device *netdev, int prtad, int devad,8968 u16 addr, u16 value)8969{8970 struct ixgbe_adapter *adapter = netdev_priv(netdev);8971 struct ixgbe_hw *hw = &adapter->hw;8972 8973 if (adapter->mii_bus) {8974 int regnum = addr;8975 8976 if (devad != MDIO_DEVAD_NONE)8977 return mdiobus_c45_write(adapter->mii_bus, prtad, devad,8978 regnum, value);8979 8980 return mdiobus_write(adapter->mii_bus, prtad, regnum, value);8981 }8982 8983 if (prtad != hw->phy.mdio.prtad)8984 return -EINVAL;8985 return hw->phy.ops.write_reg(hw, addr, devad, value);8986}8987 8988static int ixgbe_ioctl(struct net_device *netdev, struct ifreq *req, int cmd)8989{8990 struct ixgbe_adapter *adapter = netdev_priv(netdev);8991 8992 switch (cmd) {8993 case SIOCSHWTSTAMP:8994 return ixgbe_ptp_set_ts_config(adapter, req);8995 case SIOCGHWTSTAMP:8996 return ixgbe_ptp_get_ts_config(adapter, req);8997 case SIOCGMIIPHY:8998 if (!adapter->hw.phy.ops.read_reg)8999 return -EOPNOTSUPP;9000 fallthrough;9001 default:9002 return mdio_mii_ioctl(&adapter->hw.phy.mdio, if_mii(req), cmd);9003 }9004}9005 9006/**9007 * ixgbe_add_sanmac_netdev - Add the SAN MAC address to the corresponding9008 * netdev->dev_addrs9009 * @dev: network interface device structure9010 *9011 * Returns non-zero on failure9012 **/9013static int ixgbe_add_sanmac_netdev(struct net_device *dev)9014{9015 int err = 0;9016 struct ixgbe_adapter *adapter = netdev_priv(dev);9017 struct ixgbe_hw *hw = &adapter->hw;9018 9019 if (is_valid_ether_addr(hw->mac.san_addr)) {9020 rtnl_lock();9021 err = dev_addr_add(dev, hw->mac.san_addr, NETDEV_HW_ADDR_T_SAN);9022 rtnl_unlock();9023 9024 /* update SAN MAC vmdq pool selection */9025 hw->mac.ops.set_vmdq_san_mac(hw, VMDQ_P(0));9026 }9027 return err;9028}9029 9030/**9031 * ixgbe_del_sanmac_netdev - Removes the SAN MAC address to the corresponding9032 * netdev->dev_addrs9033 * @dev: network interface device structure9034 *9035 * Returns non-zero on failure9036 **/9037static int ixgbe_del_sanmac_netdev(struct net_device *dev)9038{9039 int err = 0;9040 struct ixgbe_adapter *adapter = netdev_priv(dev);9041 struct ixgbe_mac_info *mac = &adapter->hw.mac;9042 9043 if (is_valid_ether_addr(mac->san_addr)) {9044 rtnl_lock();9045 err = dev_addr_del(dev, mac->san_addr, NETDEV_HW_ADDR_T_SAN);9046 rtnl_unlock();9047 }9048 return err;9049}9050 9051static void ixgbe_get_ring_stats64(struct rtnl_link_stats64 *stats,9052 struct ixgbe_ring *ring)9053{9054 u64 bytes, packets;9055 unsigned int start;9056 9057 if (ring) {9058 do {9059 start = u64_stats_fetch_begin(&ring->syncp);9060 packets = ring->stats.packets;9061 bytes = ring->stats.bytes;9062 } while (u64_stats_fetch_retry(&ring->syncp, start));9063 stats->tx_packets += packets;9064 stats->tx_bytes += bytes;9065 }9066}9067 9068static void ixgbe_get_stats64(struct net_device *netdev,9069 struct rtnl_link_stats64 *stats)9070{9071 struct ixgbe_adapter *adapter = netdev_priv(netdev);9072 int i;9073 9074 rcu_read_lock();9075 for (i = 0; i < adapter->num_rx_queues; i++) {9076 struct ixgbe_ring *ring = READ_ONCE(adapter->rx_ring[i]);9077 u64 bytes, packets;9078 unsigned int start;9079 9080 if (ring) {9081 do {9082 start = u64_stats_fetch_begin(&ring->syncp);9083 packets = ring->stats.packets;9084 bytes = ring->stats.bytes;9085 } while (u64_stats_fetch_retry(&ring->syncp, start));9086 stats->rx_packets += packets;9087 stats->rx_bytes += bytes;9088 }9089 }9090 9091 for (i = 0; i < adapter->num_tx_queues; i++) {9092 struct ixgbe_ring *ring = READ_ONCE(adapter->tx_ring[i]);9093 9094 ixgbe_get_ring_stats64(stats, ring);9095 }9096 for (i = 0; i < adapter->num_xdp_queues; i++) {9097 struct ixgbe_ring *ring = READ_ONCE(adapter->xdp_ring[i]);9098 9099 ixgbe_get_ring_stats64(stats, ring);9100 }9101 rcu_read_unlock();9102 9103 /* following stats updated by ixgbe_watchdog_task() */9104 stats->multicast = netdev->stats.multicast;9105 stats->rx_errors = netdev->stats.rx_errors;9106 stats->rx_length_errors = netdev->stats.rx_length_errors;9107 stats->rx_crc_errors = netdev->stats.rx_crc_errors;9108 stats->rx_missed_errors = netdev->stats.rx_missed_errors;9109}9110 9111static int ixgbe_ndo_get_vf_stats(struct net_device *netdev, int vf,9112 struct ifla_vf_stats *vf_stats)9113{9114 struct ixgbe_adapter *adapter = netdev_priv(netdev);9115 9116 if (vf < 0 || vf >= adapter->num_vfs)9117 return -EINVAL;9118 9119 vf_stats->rx_packets = adapter->vfinfo[vf].vfstats.gprc;9120 vf_stats->rx_bytes = adapter->vfinfo[vf].vfstats.gorc;9121 vf_stats->tx_packets = adapter->vfinfo[vf].vfstats.gptc;9122 vf_stats->tx_bytes = adapter->vfinfo[vf].vfstats.gotc;9123 vf_stats->multicast = adapter->vfinfo[vf].vfstats.mprc;9124 9125 return 0;9126}9127 9128#ifdef CONFIG_IXGBE_DCB9129/**9130 * ixgbe_validate_rtr - verify 802.1Qp to Rx packet buffer mapping is valid.9131 * @adapter: pointer to ixgbe_adapter9132 * @tc: number of traffic classes currently enabled9133 *9134 * Configure a valid 802.1Qp to Rx packet buffer mapping ie confirm9135 * 802.1Q priority maps to a packet buffer that exists.9136 */9137static void ixgbe_validate_rtr(struct ixgbe_adapter *adapter, u8 tc)9138{9139 struct ixgbe_hw *hw = &adapter->hw;9140 u32 reg, rsave;9141 int i;9142 9143 /* 82598 have a static priority to TC mapping that can not9144 * be changed so no validation is needed.9145 */9146 if (hw->mac.type == ixgbe_mac_82598EB)9147 return;9148 9149 reg = IXGBE_READ_REG(hw, IXGBE_RTRUP2TC);9150 rsave = reg;9151 9152 for (i = 0; i < MAX_TRAFFIC_CLASS; i++) {9153 u8 up2tc = reg >> (i * IXGBE_RTRUP2TC_UP_SHIFT);9154 9155 /* If up2tc is out of bounds default to zero */9156 if (up2tc > tc)9157 reg &= ~(0x7 << IXGBE_RTRUP2TC_UP_SHIFT);9158 }9159 9160 if (reg != rsave)9161 IXGBE_WRITE_REG(hw, IXGBE_RTRUP2TC, reg);9162 9163 return;9164}9165 9166/**9167 * ixgbe_set_prio_tc_map - Configure netdev prio tc map9168 * @adapter: Pointer to adapter struct9169 *9170 * Populate the netdev user priority to tc map9171 */9172static void ixgbe_set_prio_tc_map(struct ixgbe_adapter *adapter)9173{9174 struct net_device *dev = adapter->netdev;9175 struct ixgbe_dcb_config *dcb_cfg = &adapter->dcb_cfg;9176 struct ieee_ets *ets = adapter->ixgbe_ieee_ets;9177 u8 prio;9178 9179 for (prio = 0; prio < MAX_USER_PRIORITY; prio++) {9180 u8 tc = 0;9181 9182 if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_CEE)9183 tc = ixgbe_dcb_get_tc_from_up(dcb_cfg, 0, prio);9184 else if (ets)9185 tc = ets->prio_tc[prio];9186 9187 netdev_set_prio_tc_map(dev, prio, tc);9188 }9189}9190 9191#endif /* CONFIG_IXGBE_DCB */9192static int ixgbe_reassign_macvlan_pool(struct net_device *vdev,9193 struct netdev_nested_priv *priv)9194{9195 struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)priv->data;9196 struct ixgbe_fwd_adapter *accel;9197 int pool;9198 9199 /* we only care about macvlans... */9200 if (!netif_is_macvlan(vdev))9201 return 0;9202 9203 /* that have hardware offload enabled... */9204 accel = macvlan_accel_priv(vdev);9205 if (!accel)9206 return 0;9207 9208 /* If we can relocate to a different bit do so */9209 pool = find_first_zero_bit(adapter->fwd_bitmask, adapter->num_rx_pools);9210 if (pool < adapter->num_rx_pools) {9211 set_bit(pool, adapter->fwd_bitmask);9212 accel->pool = pool;9213 return 0;9214 }9215 9216 /* if we cannot find a free pool then disable the offload */9217 netdev_err(vdev, "L2FW offload disabled due to lack of queue resources\n");9218 macvlan_release_l2fw_offload(vdev);9219 9220 /* unbind the queues and drop the subordinate channel config */9221 netdev_unbind_sb_channel(adapter->netdev, vdev);9222 netdev_set_sb_channel(vdev, 0);9223 9224 kfree(accel);9225 9226 return 0;9227}9228 9229static void ixgbe_defrag_macvlan_pools(struct net_device *dev)9230{9231 struct ixgbe_adapter *adapter = netdev_priv(dev);9232 struct netdev_nested_priv priv = {9233 .data = (void *)adapter,9234 };9235 9236 /* flush any stale bits out of the fwd bitmask */9237 bitmap_clear(adapter->fwd_bitmask, 1, 63);9238 9239 /* walk through upper devices reassigning pools */9240 netdev_walk_all_upper_dev_rcu(dev, ixgbe_reassign_macvlan_pool,9241 &priv);9242}9243 9244/**9245 * ixgbe_setup_tc - configure net_device for multiple traffic classes9246 *9247 * @dev: net device to configure9248 * @tc: number of traffic classes to enable9249 */9250int ixgbe_setup_tc(struct net_device *dev, u8 tc)9251{9252 struct ixgbe_adapter *adapter = netdev_priv(dev);9253 struct ixgbe_hw *hw = &adapter->hw;9254 9255 /* Hardware supports up to 8 traffic classes */9256 if (tc > adapter->dcb_cfg.num_tcs.pg_tcs)9257 return -EINVAL;9258 9259 if (hw->mac.type == ixgbe_mac_82598EB && tc && tc < MAX_TRAFFIC_CLASS)9260 return -EINVAL;9261 9262 /* Hardware has to reinitialize queues and interrupts to9263 * match packet buffer alignment. Unfortunately, the9264 * hardware is not flexible enough to do this dynamically.9265 */9266 if (netif_running(dev))9267 ixgbe_close(dev);9268 else9269 ixgbe_reset(adapter);9270 9271 ixgbe_clear_interrupt_scheme(adapter);9272 9273#ifdef CONFIG_IXGBE_DCB9274 if (tc) {9275 if (adapter->xdp_prog) {9276 e_warn(probe, "DCB is not supported with XDP\n");9277 9278 ixgbe_init_interrupt_scheme(adapter);9279 if (netif_running(dev))9280 ixgbe_open(dev);9281 return -EINVAL;9282 }9283 9284 netdev_set_num_tc(dev, tc);9285 ixgbe_set_prio_tc_map(adapter);9286 9287 adapter->hw_tcs = tc;9288 adapter->flags |= IXGBE_FLAG_DCB_ENABLED;9289 9290 if (adapter->hw.mac.type == ixgbe_mac_82598EB) {9291 adapter->last_lfc_mode = adapter->hw.fc.requested_mode;9292 adapter->hw.fc.requested_mode = ixgbe_fc_none;9293 }9294 } else {9295 netdev_reset_tc(dev);9296 9297 if (adapter->hw.mac.type == ixgbe_mac_82598EB)9298 adapter->hw.fc.requested_mode = adapter->last_lfc_mode;9299 9300 adapter->flags &= ~IXGBE_FLAG_DCB_ENABLED;9301 adapter->hw_tcs = tc;9302 9303 adapter->temp_dcb_cfg.pfc_mode_enable = false;9304 adapter->dcb_cfg.pfc_mode_enable = false;9305 }9306 9307 ixgbe_validate_rtr(adapter, tc);9308 9309#endif /* CONFIG_IXGBE_DCB */9310 ixgbe_init_interrupt_scheme(adapter);9311 9312 ixgbe_defrag_macvlan_pools(dev);9313 9314 if (netif_running(dev))9315 return ixgbe_open(dev);9316 9317 return 0;9318}9319 9320static int ixgbe_delete_clsu32(struct ixgbe_adapter *adapter,9321 struct tc_cls_u32_offload *cls)9322{9323 u32 hdl = cls->knode.handle;9324 u32 uhtid = TC_U32_USERHTID(cls->knode.handle);9325 u32 loc = cls->knode.handle & 0xfffff;9326 int err = 0, i, j;9327 struct ixgbe_jump_table *jump = NULL;9328 9329 if (loc > IXGBE_MAX_HW_ENTRIES)9330 return -EINVAL;9331 9332 if ((uhtid != 0x800) && (uhtid >= IXGBE_MAX_LINK_HANDLE))9333 return -EINVAL;9334 9335 /* Clear this filter in the link data it is associated with */9336 if (uhtid != 0x800) {9337 jump = adapter->jump_tables[uhtid];9338 if (!jump)9339 return -EINVAL;9340 if (!test_bit(loc - 1, jump->child_loc_map))9341 return -EINVAL;9342 clear_bit(loc - 1, jump->child_loc_map);9343 }9344 9345 /* Check if the filter being deleted is a link */9346 for (i = 1; i < IXGBE_MAX_LINK_HANDLE; i++) {9347 jump = adapter->jump_tables[i];9348 if (jump && jump->link_hdl == hdl) {9349 /* Delete filters in the hardware in the child hash9350 * table associated with this link9351 */9352 for (j = 0; j < IXGBE_MAX_HW_ENTRIES; j++) {9353 if (!test_bit(j, jump->child_loc_map))9354 continue;9355 spin_lock(&adapter->fdir_perfect_lock);9356 err = ixgbe_update_ethtool_fdir_entry(adapter,9357 NULL,9358 j + 1);9359 spin_unlock(&adapter->fdir_perfect_lock);9360 clear_bit(j, jump->child_loc_map);9361 }9362 /* Remove resources for this link */9363 kfree(jump->input);9364 kfree(jump->mask);9365 kfree(jump);9366 adapter->jump_tables[i] = NULL;9367 return err;9368 }9369 }9370 9371 spin_lock(&adapter->fdir_perfect_lock);9372 err = ixgbe_update_ethtool_fdir_entry(adapter, NULL, loc);9373 spin_unlock(&adapter->fdir_perfect_lock);9374 return err;9375}9376 9377static int ixgbe_configure_clsu32_add_hnode(struct ixgbe_adapter *adapter,9378 struct tc_cls_u32_offload *cls)9379{9380 u32 uhtid = TC_U32_USERHTID(cls->hnode.handle);9381 9382 if (uhtid >= IXGBE_MAX_LINK_HANDLE)9383 return -EINVAL;9384 9385 /* This ixgbe devices do not support hash tables at the moment9386 * so abort when given hash tables.9387 */9388 if (cls->hnode.divisor > 0)9389 return -EINVAL;9390 9391 set_bit(uhtid - 1, &adapter->tables);9392 return 0;9393}9394 9395static int ixgbe_configure_clsu32_del_hnode(struct ixgbe_adapter *adapter,9396 struct tc_cls_u32_offload *cls)9397{9398 u32 uhtid = TC_U32_USERHTID(cls->hnode.handle);9399 9400 if (uhtid >= IXGBE_MAX_LINK_HANDLE)9401 return -EINVAL;9402 9403 clear_bit(uhtid - 1, &adapter->tables);9404 return 0;9405}9406 9407#ifdef CONFIG_NET_CLS_ACT9408struct upper_walk_data {9409 struct ixgbe_adapter *adapter;9410 u64 action;9411 int ifindex;9412 u8 queue;9413};9414 9415static int get_macvlan_queue(struct net_device *upper,9416 struct netdev_nested_priv *priv)9417{9418 if (netif_is_macvlan(upper)) {9419 struct ixgbe_fwd_adapter *vadapter = macvlan_accel_priv(upper);9420 struct ixgbe_adapter *adapter;9421 struct upper_walk_data *data;9422 int ifindex;9423 9424 data = (struct upper_walk_data *)priv->data;9425 ifindex = data->ifindex;9426 adapter = data->adapter;9427 if (vadapter && upper->ifindex == ifindex) {9428 data->queue = adapter->rx_ring[vadapter->rx_base_queue]->reg_idx;9429 data->action = data->queue;9430 return 1;9431 }9432 }9433 9434 return 0;9435}9436 9437static int handle_redirect_action(struct ixgbe_adapter *adapter, int ifindex,9438 u8 *queue, u64 *action)9439{9440 struct ixgbe_ring_feature *vmdq = &adapter->ring_feature[RING_F_VMDQ];9441 unsigned int num_vfs = adapter->num_vfs, vf;9442 struct netdev_nested_priv priv;9443 struct upper_walk_data data;9444 struct net_device *upper;9445 9446 /* redirect to a SRIOV VF */9447 for (vf = 0; vf < num_vfs; ++vf) {9448 upper = pci_get_drvdata(adapter->vfinfo[vf].vfdev);9449 if (upper->ifindex == ifindex) {9450 *queue = vf * __ALIGN_MASK(1, ~vmdq->mask);9451 *action = vf + 1;9452 *action <<= ETHTOOL_RX_FLOW_SPEC_RING_VF_OFF;9453 return 0;9454 }9455 }9456 9457 /* redirect to a offloaded macvlan netdev */9458 data.adapter = adapter;9459 data.ifindex = ifindex;9460 data.action = 0;9461 data.queue = 0;9462 priv.data = (void *)&data;9463 if (netdev_walk_all_upper_dev_rcu(adapter->netdev,9464 get_macvlan_queue, &priv)) {9465 *action = data.action;9466 *queue = data.queue;9467 9468 return 0;9469 }9470 9471 return -EINVAL;9472}9473 9474static int parse_tc_actions(struct ixgbe_adapter *adapter,9475 struct tcf_exts *exts, u64 *action, u8 *queue)9476{9477 const struct tc_action *a;9478 int i;9479 9480 if (!tcf_exts_has_actions(exts))9481 return -EINVAL;9482 9483 tcf_exts_for_each_action(i, a, exts) {9484 /* Drop action */9485 if (is_tcf_gact_shot(a)) {9486 *action = IXGBE_FDIR_DROP_QUEUE;9487 *queue = IXGBE_FDIR_DROP_QUEUE;9488 return 0;9489 }9490 9491 /* Redirect to a VF or a offloaded macvlan */9492 if (is_tcf_mirred_egress_redirect(a)) {9493 struct net_device *dev = tcf_mirred_dev(a);9494 9495 if (!dev)9496 return -EINVAL;9497 return handle_redirect_action(adapter, dev->ifindex,9498 queue, action);9499 }9500 9501 return -EINVAL;9502 }9503 9504 return -EINVAL;9505}9506#else9507static int parse_tc_actions(struct ixgbe_adapter *adapter,9508 struct tcf_exts *exts, u64 *action, u8 *queue)9509{9510 return -EINVAL;9511}9512#endif /* CONFIG_NET_CLS_ACT */9513 9514static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input,9515 union ixgbe_atr_input *mask,9516 struct tc_cls_u32_offload *cls,9517 struct ixgbe_mat_field *field_ptr,9518 struct ixgbe_nexthdr *nexthdr)9519{9520 int i, j, off;9521 __be32 val, m;9522 bool found_entry = false, found_jump_field = false;9523 9524 for (i = 0; i < cls->knode.sel->nkeys; i++) {9525 off = cls->knode.sel->keys[i].off;9526 val = cls->knode.sel->keys[i].val;9527 m = cls->knode.sel->keys[i].mask;9528 9529 for (j = 0; field_ptr[j].val; j++) {9530 if (field_ptr[j].off == off) {9531 field_ptr[j].val(input, mask, (__force u32)val,9532 (__force u32)m);9533 input->filter.formatted.flow_type |=9534 field_ptr[j].type;9535 found_entry = true;9536 break;9537 }9538 }9539 if (nexthdr) {9540 if (nexthdr->off == cls->knode.sel->keys[i].off &&9541 nexthdr->val ==9542 (__force u32)cls->knode.sel->keys[i].val &&9543 nexthdr->mask ==9544 (__force u32)cls->knode.sel->keys[i].mask)9545 found_jump_field = true;9546 else9547 continue;9548 }9549 }9550 9551 if (nexthdr && !found_jump_field)9552 return -EINVAL;9553 9554 if (!found_entry)9555 return 0;9556 9557 mask->formatted.flow_type = IXGBE_ATR_L4TYPE_IPV6_MASK |9558 IXGBE_ATR_L4TYPE_MASK;9559 9560 if (input->filter.formatted.flow_type == IXGBE_ATR_FLOW_TYPE_IPV4)9561 mask->formatted.flow_type &= IXGBE_ATR_L4TYPE_IPV6_MASK;9562 9563 return 0;9564}9565 9566static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter,9567 struct tc_cls_u32_offload *cls)9568{9569 __be16 protocol = cls->common.protocol;9570 u32 loc = cls->knode.handle & 0xfffff;9571 struct ixgbe_hw *hw = &adapter->hw;9572 struct ixgbe_mat_field *field_ptr;9573 struct ixgbe_fdir_filter *input = NULL;9574 union ixgbe_atr_input *mask = NULL;9575 struct ixgbe_jump_table *jump = NULL;9576 int i, err = -EINVAL;9577 u8 queue;9578 u32 uhtid, link_uhtid;9579 9580 uhtid = TC_U32_USERHTID(cls->knode.handle);9581 link_uhtid = TC_U32_USERHTID(cls->knode.link_handle);9582 9583 /* At the moment cls_u32 jumps to network layer and skips past9584 * L2 headers. The canonical method to match L2 frames is to use9585 * negative values. However this is error prone at best but really9586 * just broken because there is no way to "know" what sort of hdr9587 * is in front of the network layer. Fix cls_u32 to support L29588 * headers when needed.9589 */9590 if (protocol != htons(ETH_P_IP))9591 return err;9592 9593 if (loc >= ((1024 << adapter->fdir_pballoc) - 2)) {9594 e_err(drv, "Location out of range\n");9595 return err;9596 }9597 9598 /* cls u32 is a graph starting at root node 0x800. The driver tracks9599 * links and also the fields used to advance the parser across each9600 * link (e.g. nexthdr/eat parameters from 'tc'). This way we can map9601 * the u32 graph onto the hardware parse graph denoted in ixgbe_model.h9602 * To add support for new nodes update ixgbe_model.h parse structures9603 * this function _should_ be generic try not to hardcode values here.9604 */9605 if (uhtid == 0x800) {9606 field_ptr = (adapter->jump_tables[0])->mat;9607 } else {9608 if (uhtid >= IXGBE_MAX_LINK_HANDLE)9609 return err;9610 if (!adapter->jump_tables[uhtid])9611 return err;9612 field_ptr = (adapter->jump_tables[uhtid])->mat;9613 }9614 9615 if (!field_ptr)9616 return err;9617 9618 /* At this point we know the field_ptr is valid and need to either9619 * build cls_u32 link or attach filter. Because adding a link to9620 * a handle that does not exist is invalid and the same for adding9621 * rules to handles that don't exist.9622 */9623 9624 if (link_uhtid) {9625 struct ixgbe_nexthdr *nexthdr = ixgbe_ipv4_jumps;9626 9627 if (link_uhtid >= IXGBE_MAX_LINK_HANDLE)9628 return err;9629 9630 if (!test_bit(link_uhtid - 1, &adapter->tables))9631 return err;9632 9633 /* Multiple filters as links to the same hash table are not9634 * supported. To add a new filter with the same next header9635 * but different match/jump conditions, create a new hash table9636 * and link to it.9637 */9638 if (adapter->jump_tables[link_uhtid] &&9639 (adapter->jump_tables[link_uhtid])->link_hdl) {9640 e_err(drv, "Link filter exists for link: %x\n",9641 link_uhtid);9642 return err;9643 }9644 9645 for (i = 0; nexthdr[i].jump; i++) {9646 if (nexthdr[i].o != cls->knode.sel->offoff ||9647 nexthdr[i].s != cls->knode.sel->offshift ||9648 nexthdr[i].m !=9649 (__force u32)cls->knode.sel->offmask)9650 return err;9651 9652 jump = kzalloc(sizeof(*jump), GFP_KERNEL);9653 if (!jump)9654 return -ENOMEM;9655 input = kzalloc(sizeof(*input), GFP_KERNEL);9656 if (!input) {9657 err = -ENOMEM;9658 goto free_jump;9659 }9660 mask = kzalloc(sizeof(*mask), GFP_KERNEL);9661 if (!mask) {9662 err = -ENOMEM;9663 goto free_input;9664 }9665 jump->input = input;9666 jump->mask = mask;9667 jump->link_hdl = cls->knode.handle;9668 9669 err = ixgbe_clsu32_build_input(input, mask, cls,9670 field_ptr, &nexthdr[i]);9671 if (!err) {9672 jump->mat = nexthdr[i].jump;9673 adapter->jump_tables[link_uhtid] = jump;9674 break;9675 } else {9676 kfree(mask);9677 kfree(input);9678 kfree(jump);9679 }9680 }9681 return 0;9682 }9683 9684 input = kzalloc(sizeof(*input), GFP_KERNEL);9685 if (!input)9686 return -ENOMEM;9687 mask = kzalloc(sizeof(*mask), GFP_KERNEL);9688 if (!mask) {9689 err = -ENOMEM;9690 goto free_input;9691 }9692 9693 if ((uhtid != 0x800) && (adapter->jump_tables[uhtid])) {9694 if ((adapter->jump_tables[uhtid])->input)9695 memcpy(input, (adapter->jump_tables[uhtid])->input,9696 sizeof(*input));9697 if ((adapter->jump_tables[uhtid])->mask)9698 memcpy(mask, (adapter->jump_tables[uhtid])->mask,9699 sizeof(*mask));9700 9701 /* Lookup in all child hash tables if this location is already9702 * filled with a filter9703 */9704 for (i = 1; i < IXGBE_MAX_LINK_HANDLE; i++) {9705 struct ixgbe_jump_table *link = adapter->jump_tables[i];9706 9707 if (link && (test_bit(loc - 1, link->child_loc_map))) {9708 e_err(drv, "Filter exists in location: %x\n",9709 loc);9710 err = -EINVAL;9711 goto err_out;9712 }9713 }9714 }9715 err = ixgbe_clsu32_build_input(input, mask, cls, field_ptr, NULL);9716 if (err)9717 goto err_out;9718 9719 err = parse_tc_actions(adapter, cls->knode.exts, &input->action,9720 &queue);9721 if (err < 0)9722 goto err_out;9723 9724 input->sw_idx = loc;9725 9726 spin_lock(&adapter->fdir_perfect_lock);9727 9728 if (hlist_empty(&adapter->fdir_filter_list)) {9729 memcpy(&adapter->fdir_mask, mask, sizeof(*mask));9730 err = ixgbe_fdir_set_input_mask_82599(hw, mask);9731 if (err)9732 goto err_out_w_lock;9733 } else if (memcmp(&adapter->fdir_mask, mask, sizeof(*mask))) {9734 err = -EINVAL;9735 goto err_out_w_lock;9736 }9737 9738 ixgbe_atr_compute_perfect_hash_82599(&input->filter, mask);9739 err = ixgbe_fdir_write_perfect_filter_82599(hw, &input->filter,9740 input->sw_idx, queue);9741 if (err)9742 goto err_out_w_lock;9743 9744 ixgbe_update_ethtool_fdir_entry(adapter, input, input->sw_idx);9745 spin_unlock(&adapter->fdir_perfect_lock);9746 9747 if ((uhtid != 0x800) && (adapter->jump_tables[uhtid]))9748 set_bit(loc - 1, (adapter->jump_tables[uhtid])->child_loc_map);9749 9750 kfree(mask);9751 return err;9752err_out_w_lock:9753 spin_unlock(&adapter->fdir_perfect_lock);9754err_out:9755 kfree(mask);9756free_input:9757 kfree(input);9758free_jump:9759 kfree(jump);9760 return err;9761}9762 9763static int ixgbe_setup_tc_cls_u32(struct ixgbe_adapter *adapter,9764 struct tc_cls_u32_offload *cls_u32)9765{9766 switch (cls_u32->command) {9767 case TC_CLSU32_NEW_KNODE:9768 case TC_CLSU32_REPLACE_KNODE:9769 return ixgbe_configure_clsu32(adapter, cls_u32);9770 case TC_CLSU32_DELETE_KNODE:9771 return ixgbe_delete_clsu32(adapter, cls_u32);9772 case TC_CLSU32_NEW_HNODE:9773 case TC_CLSU32_REPLACE_HNODE:9774 return ixgbe_configure_clsu32_add_hnode(adapter, cls_u32);9775 case TC_CLSU32_DELETE_HNODE:9776 return ixgbe_configure_clsu32_del_hnode(adapter, cls_u32);9777 default:9778 return -EOPNOTSUPP;9779 }9780}9781 9782static int ixgbe_setup_tc_block_cb(enum tc_setup_type type, void *type_data,9783 void *cb_priv)9784{9785 struct ixgbe_adapter *adapter = cb_priv;9786 9787 if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data))9788 return -EOPNOTSUPP;9789 9790 switch (type) {9791 case TC_SETUP_CLSU32:9792 return ixgbe_setup_tc_cls_u32(adapter, type_data);9793 default:9794 return -EOPNOTSUPP;9795 }9796}9797 9798static int ixgbe_setup_tc_mqprio(struct net_device *dev,9799 struct tc_mqprio_qopt *mqprio)9800{9801 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;9802 return ixgbe_setup_tc(dev, mqprio->num_tc);9803}9804 9805static LIST_HEAD(ixgbe_block_cb_list);9806 9807static int __ixgbe_setup_tc(struct net_device *dev, enum tc_setup_type type,9808 void *type_data)9809{9810 struct ixgbe_adapter *adapter = netdev_priv(dev);9811 9812 switch (type) {9813 case TC_SETUP_BLOCK:9814 return flow_block_cb_setup_simple(type_data,9815 &ixgbe_block_cb_list,9816 ixgbe_setup_tc_block_cb,9817 adapter, adapter, true);9818 case TC_SETUP_QDISC_MQPRIO:9819 return ixgbe_setup_tc_mqprio(dev, type_data);9820 default:9821 return -EOPNOTSUPP;9822 }9823}9824 9825#ifdef CONFIG_PCI_IOV9826void ixgbe_sriov_reinit(struct ixgbe_adapter *adapter)9827{9828 struct net_device *netdev = adapter->netdev;9829 9830 rtnl_lock();9831 ixgbe_setup_tc(netdev, adapter->hw_tcs);9832 rtnl_unlock();9833}9834 9835#endif9836void ixgbe_do_reset(struct net_device *netdev)9837{9838 struct ixgbe_adapter *adapter = netdev_priv(netdev);9839 9840 if (netif_running(netdev))9841 ixgbe_reinit_locked(adapter);9842 else9843 ixgbe_reset(adapter);9844}9845 9846static netdev_features_t ixgbe_fix_features(struct net_device *netdev,9847 netdev_features_t features)9848{9849 struct ixgbe_adapter *adapter = netdev_priv(netdev);9850 9851 /* If Rx checksum is disabled, then RSC/LRO should also be disabled */9852 if (!(features & NETIF_F_RXCSUM))9853 features &= ~NETIF_F_LRO;9854 9855 /* Turn off LRO if not RSC capable */9856 if (!(adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE))9857 features &= ~NETIF_F_LRO;9858 9859 if (adapter->xdp_prog && (features & NETIF_F_LRO)) {9860 e_dev_err("LRO is not supported with XDP\n");9861 features &= ~NETIF_F_LRO;9862 }9863 9864 return features;9865}9866 9867static void ixgbe_reset_l2fw_offload(struct ixgbe_adapter *adapter)9868{9869 int rss = min_t(int, ixgbe_max_rss_indices(adapter),9870 num_online_cpus());9871 9872 /* go back to full RSS if we're not running SR-IOV */9873 if (!adapter->ring_feature[RING_F_VMDQ].offset)9874 adapter->flags &= ~(IXGBE_FLAG_VMDQ_ENABLED |9875 IXGBE_FLAG_SRIOV_ENABLED);9876 9877 adapter->ring_feature[RING_F_RSS].limit = rss;9878 adapter->ring_feature[RING_F_VMDQ].limit = 1;9879 9880 ixgbe_setup_tc(adapter->netdev, adapter->hw_tcs);9881}9882 9883static int ixgbe_set_features(struct net_device *netdev,9884 netdev_features_t features)9885{9886 struct ixgbe_adapter *adapter = netdev_priv(netdev);9887 netdev_features_t changed = netdev->features ^ features;9888 bool need_reset = false;9889 9890 /* Make sure RSC matches LRO, reset if change */9891 if (!(features & NETIF_F_LRO)) {9892 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)9893 need_reset = true;9894 adapter->flags2 &= ~IXGBE_FLAG2_RSC_ENABLED;9895 } else if ((adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) &&9896 !(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) {9897 if (adapter->rx_itr_setting == 1 ||9898 adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR) {9899 adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED;9900 need_reset = true;9901 } else if ((changed ^ features) & NETIF_F_LRO) {9902 e_info(probe, "rx-usecs set too low, "9903 "disabling RSC\n");9904 }9905 }9906 9907 /*9908 * Check if Flow Director n-tuple support or hw_tc support was9909 * enabled or disabled. If the state changed, we need to reset.9910 */9911 if ((features & NETIF_F_NTUPLE) || (features & NETIF_F_HW_TC)) {9912 /* turn off ATR, enable perfect filters and reset */9913 if (!(adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE))9914 need_reset = true;9915 9916 adapter->flags &= ~IXGBE_FLAG_FDIR_HASH_CAPABLE;9917 adapter->flags |= IXGBE_FLAG_FDIR_PERFECT_CAPABLE;9918 } else {9919 /* turn off perfect filters, enable ATR and reset */9920 if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE)9921 need_reset = true;9922 9923 adapter->flags &= ~IXGBE_FLAG_FDIR_PERFECT_CAPABLE;9924 9925 /* We cannot enable ATR if SR-IOV is enabled */9926 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED ||9927 /* We cannot enable ATR if we have 2 or more tcs */9928 (adapter->hw_tcs > 1) ||9929 /* We cannot enable ATR if RSS is disabled */9930 (adapter->ring_feature[RING_F_RSS].limit <= 1) ||9931 /* A sample rate of 0 indicates ATR disabled */9932 (!adapter->atr_sample_rate))9933 ; /* do nothing not supported */9934 else /* otherwise supported and set the flag */9935 adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE;9936 }9937 9938 if (changed & NETIF_F_RXALL)9939 need_reset = true;9940 9941 netdev->features = features;9942 9943 if ((changed & NETIF_F_HW_L2FW_DOFFLOAD) && adapter->num_rx_pools > 1)9944 ixgbe_reset_l2fw_offload(adapter);9945 else if (need_reset)9946 ixgbe_do_reset(netdev);9947 else if (changed & (NETIF_F_HW_VLAN_CTAG_RX |9948 NETIF_F_HW_VLAN_CTAG_FILTER))9949 ixgbe_set_rx_mode(netdev);9950 9951 return 1;9952}9953 9954static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],9955 struct net_device *dev,9956 const unsigned char *addr, u16 vid,9957 u16 flags,9958 struct netlink_ext_ack *extack)9959{9960 /* guarantee we can provide a unique filter for the unicast address */9961 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) {9962 struct ixgbe_adapter *adapter = netdev_priv(dev);9963 u16 pool = VMDQ_P(0);9964 9965 if (netdev_uc_count(dev) >= ixgbe_available_rars(adapter, pool))9966 return -ENOMEM;9967 }9968 9969 return ndo_dflt_fdb_add(ndm, tb, dev, addr, vid, flags);9970}9971 9972/**9973 * ixgbe_configure_bridge_mode - set various bridge modes9974 * @adapter: the private structure9975 * @mode: requested bridge mode9976 *9977 * Configure some settings require for various bridge modes.9978 **/9979static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter,9980 __u16 mode)9981{9982 struct ixgbe_hw *hw = &adapter->hw;9983 unsigned int p, num_pools;9984 u32 vmdctl;9985 9986 switch (mode) {9987 case BRIDGE_MODE_VEPA:9988 /* disable Tx loopback, rely on switch hairpin mode */9989 IXGBE_WRITE_REG(&adapter->hw, IXGBE_PFDTXGSWC, 0);9990 9991 /* must enable Rx switching replication to allow multicast9992 * packet reception on all VFs, and to enable source address9993 * pruning.9994 */9995 vmdctl = IXGBE_READ_REG(hw, IXGBE_VMD_CTL);9996 vmdctl |= IXGBE_VT_CTL_REPLEN;9997 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);9998 9999 /* enable Rx source address pruning. Note, this requires10000 * replication to be enabled or else it does nothing.10001 */10002 num_pools = adapter->num_vfs + adapter->num_rx_pools;10003 for (p = 0; p < num_pools; p++) {10004 if (hw->mac.ops.set_source_address_pruning)10005 hw->mac.ops.set_source_address_pruning(hw,10006 true,10007 p);10008 }10009 break;10010 case BRIDGE_MODE_VEB:10011 /* enable Tx loopback for internal VF/PF communication */10012 IXGBE_WRITE_REG(&adapter->hw, IXGBE_PFDTXGSWC,10013 IXGBE_PFDTXGSWC_VT_LBEN);10014 10015 /* disable Rx switching replication unless we have SR-IOV10016 * virtual functions10017 */10018 vmdctl = IXGBE_READ_REG(hw, IXGBE_VMD_CTL);10019 if (!adapter->num_vfs)10020 vmdctl &= ~IXGBE_VT_CTL_REPLEN;10021 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl);10022 10023 /* disable Rx source address pruning, since we don't expect to10024 * be receiving external loopback of our transmitted frames.10025 */10026 num_pools = adapter->num_vfs + adapter->num_rx_pools;10027 for (p = 0; p < num_pools; p++) {10028 if (hw->mac.ops.set_source_address_pruning)10029 hw->mac.ops.set_source_address_pruning(hw,10030 false,10031 p);10032 }10033 break;10034 default:10035 return -EINVAL;10036 }10037 10038 adapter->bridge_mode = mode;10039 10040 e_info(drv, "enabling bridge mode: %s\n",10041 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");10042 10043 return 0;10044}10045 10046static int ixgbe_ndo_bridge_setlink(struct net_device *dev,10047 struct nlmsghdr *nlh, u16 flags,10048 struct netlink_ext_ack *extack)10049{10050 struct ixgbe_adapter *adapter = netdev_priv(dev);10051 struct nlattr *attr, *br_spec;10052 int rem;10053 10054 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))10055 return -EOPNOTSUPP;10056 10057 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);10058 if (!br_spec)10059 return -EINVAL;10060 10061 nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) {10062 __u16 mode = nla_get_u16(attr);10063 int status = ixgbe_configure_bridge_mode(adapter, mode);10064 10065 if (status)10066 return status;10067 10068 break;10069 }10070 10071 return 0;10072}10073 10074static int ixgbe_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,10075 struct net_device *dev,10076 u32 filter_mask, int nlflags)10077{10078 struct ixgbe_adapter *adapter = netdev_priv(dev);10079 10080 if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))10081 return 0;10082 10083 return ndo_dflt_bridge_getlink(skb, pid, seq, dev,10084 adapter->bridge_mode, 0, 0, nlflags,10085 filter_mask, NULL);10086}10087 10088static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev)10089{10090 struct ixgbe_adapter *adapter = netdev_priv(pdev);10091 struct ixgbe_fwd_adapter *accel;10092 int tcs = adapter->hw_tcs ? : 1;10093 int pool, err;10094 10095 if (adapter->xdp_prog) {10096 e_warn(probe, "L2FW offload is not supported with XDP\n");10097 return ERR_PTR(-EINVAL);10098 }10099 10100 /* The hardware supported by ixgbe only filters on the destination MAC10101 * address. In order to avoid issues we only support offloading modes10102 * where the hardware can actually provide the functionality.10103 */10104 if (!macvlan_supports_dest_filter(vdev))10105 return ERR_PTR(-EMEDIUMTYPE);10106 10107 /* We need to lock down the macvlan to be a single queue device so that10108 * we can reuse the tc_to_txq field in the macvlan netdev to represent10109 * the queue mapping to our netdev.10110 */10111 if (netif_is_multiqueue(vdev))10112 return ERR_PTR(-ERANGE);10113 10114 pool = find_first_zero_bit(adapter->fwd_bitmask, adapter->num_rx_pools);10115 if (pool == adapter->num_rx_pools) {10116 u16 used_pools = adapter->num_vfs + adapter->num_rx_pools;10117 u16 reserved_pools;10118 10119 if (((adapter->flags & IXGBE_FLAG_DCB_ENABLED) &&10120 adapter->num_rx_pools >= (MAX_TX_QUEUES / tcs)) ||10121 adapter->num_rx_pools > IXGBE_MAX_MACVLANS)10122 return ERR_PTR(-EBUSY);10123 10124 /* Hardware has a limited number of available pools. Each VF,10125 * and the PF require a pool. Check to ensure we don't10126 * attempt to use more then the available number of pools.10127 */10128 if (used_pools >= IXGBE_MAX_VF_FUNCTIONS)10129 return ERR_PTR(-EBUSY);10130 10131 /* Enable VMDq flag so device will be set in VM mode */10132 adapter->flags |= IXGBE_FLAG_VMDQ_ENABLED |10133 IXGBE_FLAG_SRIOV_ENABLED;10134 10135 /* Try to reserve as many queues per pool as possible,10136 * we start with the configurations that support 4 queues10137 * per pools, followed by 2, and then by just 1 per pool.10138 */10139 if (used_pools < 32 && adapter->num_rx_pools < 16)10140 reserved_pools = min_t(u16,10141 32 - used_pools,10142 16 - adapter->num_rx_pools);10143 else if (adapter->num_rx_pools < 32)10144 reserved_pools = min_t(u16,10145 64 - used_pools,10146 32 - adapter->num_rx_pools);10147 else10148 reserved_pools = 64 - used_pools;10149 10150 10151 if (!reserved_pools)10152 return ERR_PTR(-EBUSY);10153 10154 adapter->ring_feature[RING_F_VMDQ].limit += reserved_pools;10155 10156 /* Force reinit of ring allocation with VMDQ enabled */10157 err = ixgbe_setup_tc(pdev, adapter->hw_tcs);10158 if (err)10159 return ERR_PTR(err);10160 10161 if (pool >= adapter->num_rx_pools)10162 return ERR_PTR(-ENOMEM);10163 }10164 10165 accel = kzalloc(sizeof(*accel), GFP_KERNEL);10166 if (!accel)10167 return ERR_PTR(-ENOMEM);10168 10169 set_bit(pool, adapter->fwd_bitmask);10170 netdev_set_sb_channel(vdev, pool);10171 accel->pool = pool;10172 accel->netdev = vdev;10173 10174 if (!netif_running(pdev))10175 return accel;10176 10177 err = ixgbe_fwd_ring_up(adapter, accel);10178 if (err)10179 return ERR_PTR(err);10180 10181 return accel;10182}10183 10184static void ixgbe_fwd_del(struct net_device *pdev, void *priv)10185{10186 struct ixgbe_fwd_adapter *accel = priv;10187 struct ixgbe_adapter *adapter = netdev_priv(pdev);10188 unsigned int rxbase = accel->rx_base_queue;10189 unsigned int i;10190 10191 /* delete unicast filter associated with offloaded interface */10192 ixgbe_del_mac_filter(adapter, accel->netdev->dev_addr,10193 VMDQ_P(accel->pool));10194 10195 /* Allow remaining Rx packets to get flushed out of the10196 * Rx FIFO before we drop the netdev for the ring.10197 */10198 usleep_range(10000, 20000);10199 10200 for (i = 0; i < adapter->num_rx_queues_per_pool; i++) {10201 struct ixgbe_ring *ring = adapter->rx_ring[rxbase + i];10202 struct ixgbe_q_vector *qv = ring->q_vector;10203 10204 /* Make sure we aren't processing any packets and clear10205 * netdev to shut down the ring.10206 */10207 if (netif_running(adapter->netdev))10208 napi_synchronize(&qv->napi);10209 ring->netdev = NULL;10210 }10211 10212 /* unbind the queues and drop the subordinate channel config */10213 netdev_unbind_sb_channel(pdev, accel->netdev);10214 netdev_set_sb_channel(accel->netdev, 0);10215 10216 clear_bit(accel->pool, adapter->fwd_bitmask);10217 kfree(accel);10218}10219 10220#define IXGBE_MAX_MAC_HDR_LEN 12710221#define IXGBE_MAX_NETWORK_HDR_LEN 51110222 10223static netdev_features_t10224ixgbe_features_check(struct sk_buff *skb, struct net_device *dev,10225 netdev_features_t features)10226{10227 unsigned int network_hdr_len, mac_hdr_len;10228 10229 /* Make certain the headers can be described by a context descriptor */10230 mac_hdr_len = skb_network_offset(skb);10231 if (unlikely(mac_hdr_len > IXGBE_MAX_MAC_HDR_LEN))10232 return features & ~(NETIF_F_HW_CSUM |10233 NETIF_F_SCTP_CRC |10234 NETIF_F_GSO_UDP_L4 |10235 NETIF_F_HW_VLAN_CTAG_TX |10236 NETIF_F_TSO |10237 NETIF_F_TSO6);10238 10239 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);10240 if (unlikely(network_hdr_len > IXGBE_MAX_NETWORK_HDR_LEN))10241 return features & ~(NETIF_F_HW_CSUM |10242 NETIF_F_SCTP_CRC |10243 NETIF_F_GSO_UDP_L4 |10244 NETIF_F_TSO |10245 NETIF_F_TSO6);10246 10247 /* We can only support IPV4 TSO in tunnels if we can mangle the10248 * inner IP ID field, so strip TSO if MANGLEID is not supported.10249 * IPsec offoad sets skb->encapsulation but still can handle10250 * the TSO, so it's the exception.10251 */10252 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) {10253#ifdef CONFIG_IXGBE_IPSEC10254 if (!secpath_exists(skb))10255#endif10256 features &= ~NETIF_F_TSO;10257 }10258 10259 return features;10260}10261 10262static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog)10263{10264 int i, frame_size = dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;10265 struct ixgbe_adapter *adapter = netdev_priv(dev);10266 struct bpf_prog *old_prog;10267 bool need_reset;10268 int num_queues;10269 10270 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)10271 return -EINVAL;10272 10273 if (adapter->flags & IXGBE_FLAG_DCB_ENABLED)10274 return -EINVAL;10275 10276 /* verify ixgbe ring attributes are sufficient for XDP */10277 for (i = 0; i < adapter->num_rx_queues; i++) {10278 struct ixgbe_ring *ring = adapter->rx_ring[i];10279 10280 if (ring_is_rsc_enabled(ring))10281 return -EINVAL;10282 10283 if (frame_size > ixgbe_rx_bufsz(ring))10284 return -EINVAL;10285 }10286 10287 /* if the number of cpus is much larger than the maximum of queues,10288 * we should stop it and then return with ENOMEM like before.10289 */10290 if (nr_cpu_ids > IXGBE_MAX_XDP_QS * 2)10291 return -ENOMEM;10292 10293 old_prog = xchg(&adapter->xdp_prog, prog);10294 need_reset = (!!prog != !!old_prog);10295 10296 /* If transitioning XDP modes reconfigure rings */10297 if (need_reset) {10298 int err;10299 10300 if (!prog)10301 /* Wait until ndo_xsk_wakeup completes. */10302 synchronize_rcu();10303 err = ixgbe_setup_tc(dev, adapter->hw_tcs);10304 10305 if (err)10306 return -EINVAL;10307 if (!prog)10308 xdp_features_clear_redirect_target(dev);10309 } else {10310 for (i = 0; i < adapter->num_rx_queues; i++) {10311 WRITE_ONCE(adapter->rx_ring[i]->xdp_prog,10312 adapter->xdp_prog);10313 }10314 }10315 10316 if (old_prog)10317 bpf_prog_put(old_prog);10318 10319 /* Kick start the NAPI context if there is an AF_XDP socket open10320 * on that queue id. This so that receiving will start.10321 */10322 if (need_reset && prog) {10323 num_queues = min_t(int, adapter->num_rx_queues,10324 adapter->num_xdp_queues);10325 for (i = 0; i < num_queues; i++)10326 if (adapter->xdp_ring[i]->xsk_pool)10327 (void)ixgbe_xsk_wakeup(adapter->netdev, i,10328 XDP_WAKEUP_RX);10329 xdp_features_set_redirect_target(dev, true);10330 }10331 10332 return 0;10333}10334 10335static int ixgbe_xdp(struct net_device *dev, struct netdev_bpf *xdp)10336{10337 struct ixgbe_adapter *adapter = netdev_priv(dev);10338 10339 switch (xdp->command) {10340 case XDP_SETUP_PROG:10341 return ixgbe_xdp_setup(dev, xdp->prog);10342 case XDP_SETUP_XSK_POOL:10343 return ixgbe_xsk_pool_setup(adapter, xdp->xsk.pool,10344 xdp->xsk.queue_id);10345 10346 default:10347 return -EINVAL;10348 }10349}10350 10351void ixgbe_xdp_ring_update_tail(struct ixgbe_ring *ring)10352{10353 /* Force memory writes to complete before letting h/w know there10354 * are new descriptors to fetch.10355 */10356 wmb();10357 writel(ring->next_to_use, ring->tail);10358}10359 10360void ixgbe_xdp_ring_update_tail_locked(struct ixgbe_ring *ring)10361{10362 if (static_branch_unlikely(&ixgbe_xdp_locking_key))10363 spin_lock(&ring->tx_lock);10364 ixgbe_xdp_ring_update_tail(ring);10365 if (static_branch_unlikely(&ixgbe_xdp_locking_key))10366 spin_unlock(&ring->tx_lock);10367}10368 10369static int ixgbe_xdp_xmit(struct net_device *dev, int n,10370 struct xdp_frame **frames, u32 flags)10371{10372 struct ixgbe_adapter *adapter = netdev_priv(dev);10373 struct ixgbe_ring *ring;10374 int nxmit = 0;10375 int i;10376 10377 if (unlikely(test_bit(__IXGBE_DOWN, &adapter->state)))10378 return -ENETDOWN;10379 10380 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))10381 return -EINVAL;10382 10383 /* During program transitions its possible adapter->xdp_prog is assigned10384 * but ring has not been configured yet. In this case simply abort xmit.10385 */10386 ring = adapter->xdp_prog ? ixgbe_determine_xdp_ring(adapter) : NULL;10387 if (unlikely(!ring))10388 return -ENXIO;10389 10390 if (unlikely(test_bit(__IXGBE_TX_DISABLED, &ring->state)))10391 return -ENXIO;10392 10393 if (static_branch_unlikely(&ixgbe_xdp_locking_key))10394 spin_lock(&ring->tx_lock);10395 10396 for (i = 0; i < n; i++) {10397 struct xdp_frame *xdpf = frames[i];10398 int err;10399 10400 err = ixgbe_xmit_xdp_ring(ring, xdpf);10401 if (err != IXGBE_XDP_TX)10402 break;10403 nxmit++;10404 }10405 10406 if (unlikely(flags & XDP_XMIT_FLUSH))10407 ixgbe_xdp_ring_update_tail(ring);10408 10409 if (static_branch_unlikely(&ixgbe_xdp_locking_key))10410 spin_unlock(&ring->tx_lock);10411 10412 return nxmit;10413}10414 10415static const struct net_device_ops ixgbe_netdev_ops = {10416 .ndo_open = ixgbe_open,10417 .ndo_stop = ixgbe_close,10418 .ndo_start_xmit = ixgbe_xmit_frame,10419 .ndo_set_rx_mode = ixgbe_set_rx_mode,10420 .ndo_validate_addr = eth_validate_addr,10421 .ndo_set_mac_address = ixgbe_set_mac,10422 .ndo_change_mtu = ixgbe_change_mtu,10423 .ndo_tx_timeout = ixgbe_tx_timeout,10424 .ndo_set_tx_maxrate = ixgbe_tx_maxrate,10425 .ndo_vlan_rx_add_vid = ixgbe_vlan_rx_add_vid,10426 .ndo_vlan_rx_kill_vid = ixgbe_vlan_rx_kill_vid,10427 .ndo_eth_ioctl = ixgbe_ioctl,10428 .ndo_set_vf_mac = ixgbe_ndo_set_vf_mac,10429 .ndo_set_vf_vlan = ixgbe_ndo_set_vf_vlan,10430 .ndo_set_vf_rate = ixgbe_ndo_set_vf_bw,10431 .ndo_set_vf_spoofchk = ixgbe_ndo_set_vf_spoofchk,10432 .ndo_set_vf_link_state = ixgbe_ndo_set_vf_link_state,10433 .ndo_set_vf_rss_query_en = ixgbe_ndo_set_vf_rss_query_en,10434 .ndo_set_vf_trust = ixgbe_ndo_set_vf_trust,10435 .ndo_get_vf_config = ixgbe_ndo_get_vf_config,10436 .ndo_get_vf_stats = ixgbe_ndo_get_vf_stats,10437 .ndo_get_stats64 = ixgbe_get_stats64,10438 .ndo_setup_tc = __ixgbe_setup_tc,10439#ifdef IXGBE_FCOE10440 .ndo_select_queue = ixgbe_select_queue,10441 .ndo_fcoe_ddp_setup = ixgbe_fcoe_ddp_get,10442 .ndo_fcoe_ddp_target = ixgbe_fcoe_ddp_target,10443 .ndo_fcoe_ddp_done = ixgbe_fcoe_ddp_put,10444 .ndo_fcoe_enable = ixgbe_fcoe_enable,10445 .ndo_fcoe_disable = ixgbe_fcoe_disable,10446 .ndo_fcoe_get_wwn = ixgbe_fcoe_get_wwn,10447 .ndo_fcoe_get_hbainfo = ixgbe_fcoe_get_hbainfo,10448#endif /* IXGBE_FCOE */10449 .ndo_set_features = ixgbe_set_features,10450 .ndo_fix_features = ixgbe_fix_features,10451 .ndo_fdb_add = ixgbe_ndo_fdb_add,10452 .ndo_bridge_setlink = ixgbe_ndo_bridge_setlink,10453 .ndo_bridge_getlink = ixgbe_ndo_bridge_getlink,10454 .ndo_dfwd_add_station = ixgbe_fwd_add,10455 .ndo_dfwd_del_station = ixgbe_fwd_del,10456 .ndo_features_check = ixgbe_features_check,10457 .ndo_bpf = ixgbe_xdp,10458 .ndo_xdp_xmit = ixgbe_xdp_xmit,10459 .ndo_xsk_wakeup = ixgbe_xsk_wakeup,10460};10461 10462static void ixgbe_disable_txr_hw(struct ixgbe_adapter *adapter,10463 struct ixgbe_ring *tx_ring)10464{10465 unsigned long wait_delay, delay_interval;10466 struct ixgbe_hw *hw = &adapter->hw;10467 u8 reg_idx = tx_ring->reg_idx;10468 int wait_loop;10469 u32 txdctl;10470 10471 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);10472 10473 /* delay mechanism from ixgbe_disable_tx */10474 delay_interval = ixgbe_get_completion_timeout(adapter) / 100;10475 10476 wait_loop = IXGBE_MAX_RX_DESC_POLL;10477 wait_delay = delay_interval;10478 10479 while (wait_loop--) {10480 usleep_range(wait_delay, wait_delay + 10);10481 wait_delay += delay_interval * 2;10482 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));10483 10484 if (!(txdctl & IXGBE_TXDCTL_ENABLE))10485 return;10486 }10487 10488 e_err(drv, "TXDCTL.ENABLE not cleared within the polling period\n");10489}10490 10491static void ixgbe_disable_txr(struct ixgbe_adapter *adapter,10492 struct ixgbe_ring *tx_ring)10493{10494 set_bit(__IXGBE_TX_DISABLED, &tx_ring->state);10495 ixgbe_disable_txr_hw(adapter, tx_ring);10496}10497 10498static void ixgbe_disable_rxr_hw(struct ixgbe_adapter *adapter,10499 struct ixgbe_ring *rx_ring)10500{10501 unsigned long wait_delay, delay_interval;10502 struct ixgbe_hw *hw = &adapter->hw;10503 u8 reg_idx = rx_ring->reg_idx;10504 int wait_loop;10505 u32 rxdctl;10506 10507 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));10508 rxdctl &= ~IXGBE_RXDCTL_ENABLE;10509 rxdctl |= IXGBE_RXDCTL_SWFLSH;10510 10511 /* write value back with RXDCTL.ENABLE bit cleared */10512 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);10513 10514 /* RXDCTL.EN may not change on 82598 if link is down, so skip it */10515 if (hw->mac.type == ixgbe_mac_82598EB &&10516 !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))10517 return;10518 10519 /* delay mechanism from ixgbe_disable_rx */10520 delay_interval = ixgbe_get_completion_timeout(adapter) / 100;10521 10522 wait_loop = IXGBE_MAX_RX_DESC_POLL;10523 wait_delay = delay_interval;10524 10525 while (wait_loop--) {10526 usleep_range(wait_delay, wait_delay + 10);10527 wait_delay += delay_interval * 2;10528 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));10529 10530 if (!(rxdctl & IXGBE_RXDCTL_ENABLE))10531 return;10532 }10533 10534 e_err(drv, "RXDCTL.ENABLE not cleared within the polling period\n");10535}10536 10537static void ixgbe_reset_txr_stats(struct ixgbe_ring *tx_ring)10538{10539 memset(&tx_ring->stats, 0, sizeof(tx_ring->stats));10540 memset(&tx_ring->tx_stats, 0, sizeof(tx_ring->tx_stats));10541}10542 10543static void ixgbe_reset_rxr_stats(struct ixgbe_ring *rx_ring)10544{10545 memset(&rx_ring->stats, 0, sizeof(rx_ring->stats));10546 memset(&rx_ring->rx_stats, 0, sizeof(rx_ring->rx_stats));10547}10548 10549/**10550 * ixgbe_irq_disable_single - Disable single IRQ vector10551 * @adapter: adapter structure10552 * @ring: ring index10553 **/10554static void ixgbe_irq_disable_single(struct ixgbe_adapter *adapter, u32 ring)10555{10556 struct ixgbe_hw *hw = &adapter->hw;10557 u64 qmask = BIT_ULL(ring);10558 u32 mask;10559 10560 switch (adapter->hw.mac.type) {10561 case ixgbe_mac_82598EB:10562 mask = qmask & IXGBE_EIMC_RTX_QUEUE;10563 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, mask);10564 break;10565 case ixgbe_mac_82599EB:10566 case ixgbe_mac_X540:10567 case ixgbe_mac_X550:10568 case ixgbe_mac_X550EM_x:10569 case ixgbe_mac_x550em_a:10570 mask = (qmask & 0xFFFFFFFF);10571 if (mask)10572 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);10573 mask = (qmask >> 32);10574 if (mask)10575 IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);10576 break;10577 default:10578 break;10579 }10580 IXGBE_WRITE_FLUSH(&adapter->hw);10581 if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)10582 synchronize_irq(adapter->msix_entries[ring].vector);10583 else10584 synchronize_irq(adapter->pdev->irq);10585}10586 10587/**10588 * ixgbe_txrx_ring_disable - Disable Rx/Tx/XDP Tx rings10589 * @adapter: adapter structure10590 * @ring: ring index10591 *10592 * This function disables a certain Rx/Tx/XDP Tx ring. The function10593 * assumes that the netdev is running.10594 **/10595void ixgbe_txrx_ring_disable(struct ixgbe_adapter *adapter, int ring)10596{10597 struct ixgbe_ring *rx_ring, *tx_ring, *xdp_ring;10598 10599 rx_ring = adapter->rx_ring[ring];10600 tx_ring = adapter->tx_ring[ring];10601 xdp_ring = adapter->xdp_ring[ring];10602 10603 ixgbe_irq_disable_single(adapter, ring);10604 10605 /* Rx/Tx/XDP Tx share the same napi context. */10606 napi_disable(&rx_ring->q_vector->napi);10607 10608 ixgbe_disable_txr(adapter, tx_ring);10609 if (xdp_ring)10610 ixgbe_disable_txr(adapter, xdp_ring);10611 ixgbe_disable_rxr_hw(adapter, rx_ring);10612 10613 if (xdp_ring)10614 synchronize_rcu();10615 10616 ixgbe_clean_tx_ring(tx_ring);10617 if (xdp_ring)10618 ixgbe_clean_tx_ring(xdp_ring);10619 ixgbe_clean_rx_ring(rx_ring);10620 10621 ixgbe_reset_txr_stats(tx_ring);10622 if (xdp_ring)10623 ixgbe_reset_txr_stats(xdp_ring);10624 ixgbe_reset_rxr_stats(rx_ring);10625}10626 10627/**10628 * ixgbe_txrx_ring_enable - Enable Rx/Tx/XDP Tx rings10629 * @adapter: adapter structure10630 * @ring: ring index10631 *10632 * This function enables a certain Rx/Tx/XDP Tx ring. The function10633 * assumes that the netdev is running.10634 **/10635void ixgbe_txrx_ring_enable(struct ixgbe_adapter *adapter, int ring)10636{10637 struct ixgbe_ring *rx_ring, *tx_ring, *xdp_ring;10638 10639 rx_ring = adapter->rx_ring[ring];10640 tx_ring = adapter->tx_ring[ring];10641 xdp_ring = adapter->xdp_ring[ring];10642 10643 ixgbe_configure_tx_ring(adapter, tx_ring);10644 if (xdp_ring)10645 ixgbe_configure_tx_ring(adapter, xdp_ring);10646 ixgbe_configure_rx_ring(adapter, rx_ring);10647 10648 clear_bit(__IXGBE_TX_DISABLED, &tx_ring->state);10649 if (xdp_ring)10650 clear_bit(__IXGBE_TX_DISABLED, &xdp_ring->state);10651 10652 /* Rx/Tx/XDP Tx share the same napi context. */10653 napi_enable(&rx_ring->q_vector->napi);10654 ixgbe_irq_enable_queues(adapter, BIT_ULL(ring));10655 IXGBE_WRITE_FLUSH(&adapter->hw);10656}10657 10658/**10659 * ixgbe_enumerate_functions - Get the number of ports this device has10660 * @adapter: adapter structure10661 *10662 * This function enumerates the phsyical functions co-located on a single slot,10663 * in order to determine how many ports a device has. This is most useful in10664 * determining the required GT/s of PCIe bandwidth necessary for optimal10665 * performance.10666 **/10667static inline int ixgbe_enumerate_functions(struct ixgbe_adapter *adapter)10668{10669 struct pci_dev *entry, *pdev = adapter->pdev;10670 int physfns = 0;10671 10672 /* Some cards can not use the generic count PCIe functions method,10673 * because they are behind a parent switch, so we hardcode these with10674 * the correct number of functions.10675 */10676 if (ixgbe_pcie_from_parent(&adapter->hw))10677 physfns = 4;10678 10679 list_for_each_entry(entry, &adapter->pdev->bus->devices, bus_list) {10680 /* don't count virtual functions */10681 if (entry->is_virtfn)10682 continue;10683 10684 /* When the devices on the bus don't all match our device ID,10685 * we can't reliably determine the correct number of10686 * functions. This can occur if a function has been direct10687 * attached to a virtual machine using VT-d, for example. In10688 * this case, simply return -1 to indicate this.10689 */10690 if ((entry->vendor != pdev->vendor) ||10691 (entry->device != pdev->device))10692 return -1;10693 10694 physfns++;10695 }10696 10697 return physfns;10698}10699 10700/**10701 * ixgbe_wol_supported - Check whether device supports WoL10702 * @adapter: the adapter private structure10703 * @device_id: the device ID10704 * @subdevice_id: the subsystem device ID10705 *10706 * This function is used by probe and ethtool to determine10707 * which devices have WoL support10708 *10709 **/10710bool ixgbe_wol_supported(struct ixgbe_adapter *adapter, u16 device_id,10711 u16 subdevice_id)10712{10713 struct ixgbe_hw *hw = &adapter->hw;10714 u16 wol_cap = adapter->eeprom_cap & IXGBE_DEVICE_CAPS_WOL_MASK;10715 10716 /* WOL not supported on 82598 */10717 if (hw->mac.type == ixgbe_mac_82598EB)10718 return false;10719 10720 /* check eeprom to see if WOL is enabled for X540 and newer */10721 if (hw->mac.type >= ixgbe_mac_X540) {10722 if ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0_1) ||10723 ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) &&10724 (hw->bus.func == 0)))10725 return true;10726 }10727 10728 /* WOL is determined based on device IDs for 82599 MACs */10729 switch (device_id) {10730 case IXGBE_DEV_ID_82599_SFP:10731 /* Only these subdevices could supports WOL */10732 switch (subdevice_id) {10733 case IXGBE_SUBDEV_ID_82599_560FLR:10734 case IXGBE_SUBDEV_ID_82599_LOM_SNAP6:10735 case IXGBE_SUBDEV_ID_82599_SFP_WOL0:10736 case IXGBE_SUBDEV_ID_82599_SFP_2OCP:10737 /* only support first port */10738 if (hw->bus.func != 0)10739 break;10740 fallthrough;10741 case IXGBE_SUBDEV_ID_82599_SP_560FLR:10742 case IXGBE_SUBDEV_ID_82599_SFP:10743 case IXGBE_SUBDEV_ID_82599_RNDC:10744 case IXGBE_SUBDEV_ID_82599_ECNA_DP:10745 case IXGBE_SUBDEV_ID_82599_SFP_1OCP:10746 case IXGBE_SUBDEV_ID_82599_SFP_LOM_OEM1:10747 case IXGBE_SUBDEV_ID_82599_SFP_LOM_OEM2:10748 return true;10749 }10750 break;10751 case IXGBE_DEV_ID_82599EN_SFP:10752 /* Only these subdevices support WOL */10753 switch (subdevice_id) {10754 case IXGBE_SUBDEV_ID_82599EN_SFP_OCP1:10755 return true;10756 }10757 break;10758 case IXGBE_DEV_ID_82599_COMBO_BACKPLANE:10759 /* All except this subdevice support WOL */10760 if (subdevice_id != IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ)10761 return true;10762 break;10763 case IXGBE_DEV_ID_82599_KX4:10764 return true;10765 default:10766 break;10767 }10768 10769 return false;10770}10771 10772/**10773 * ixgbe_set_fw_version - Set FW version10774 * @adapter: the adapter private structure10775 *10776 * This function is used by probe and ethtool to determine the FW version to10777 * format to display. The FW version is taken from the EEPROM/NVM.10778 */10779static void ixgbe_set_fw_version(struct ixgbe_adapter *adapter)10780{10781 struct ixgbe_hw *hw = &adapter->hw;10782 struct ixgbe_nvm_version nvm_ver;10783 10784 ixgbe_get_oem_prod_version(hw, &nvm_ver);10785 if (nvm_ver.oem_valid) {10786 snprintf(adapter->eeprom_id, sizeof(adapter->eeprom_id),10787 "%x.%x.%x", nvm_ver.oem_major, nvm_ver.oem_minor,10788 nvm_ver.oem_release);10789 return;10790 }10791 10792 ixgbe_get_etk_id(hw, &nvm_ver);10793 ixgbe_get_orom_version(hw, &nvm_ver);10794 10795 if (nvm_ver.or_valid) {10796 snprintf(adapter->eeprom_id, sizeof(adapter->eeprom_id),10797 "0x%08x, %d.%d.%d", nvm_ver.etk_id, nvm_ver.or_major,10798 nvm_ver.or_build, nvm_ver.or_patch);10799 return;10800 }10801 10802 /* Set ETrack ID format */10803 snprintf(adapter->eeprom_id, sizeof(adapter->eeprom_id),10804 "0x%08x", nvm_ver.etk_id);10805}10806 10807/**10808 * ixgbe_probe - Device Initialization Routine10809 * @pdev: PCI device information struct10810 * @ent: entry in ixgbe_pci_tbl10811 *10812 * Returns 0 on success, negative on failure10813 *10814 * ixgbe_probe initializes an adapter identified by a pci_dev structure.10815 * The OS initialization, configuring of the adapter private structure,10816 * and a hardware reset occur.10817 **/10818static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent)10819{10820 struct net_device *netdev;10821 struct ixgbe_adapter *adapter = NULL;10822 struct ixgbe_hw *hw;10823 const struct ixgbe_info *ii = ixgbe_info_tbl[ent->driver_data];10824 unsigned int indices = MAX_TX_QUEUES;10825 u8 part_str[IXGBE_PBANUM_LENGTH];10826 int i, err, expected_gts;10827 bool disable_dev = false;10828#ifdef IXGBE_FCOE10829 u16 device_caps;10830#endif10831 u32 eec;10832 10833 /* Catch broken hardware that put the wrong VF device ID in10834 * the PCIe SR-IOV capability.10835 */10836 if (pdev->is_virtfn) {10837 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",10838 pci_name(pdev), pdev->vendor, pdev->device);10839 return -EINVAL;10840 }10841 10842 err = pci_enable_device_mem(pdev);10843 if (err)10844 return err;10845 10846 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));10847 if (err) {10848 dev_err(&pdev->dev,10849 "No usable DMA configuration, aborting\n");10850 goto err_dma;10851 }10852 10853 err = pci_request_mem_regions(pdev, ixgbe_driver_name);10854 if (err) {10855 dev_err(&pdev->dev,10856 "pci_request_selected_regions failed 0x%x\n", err);10857 goto err_pci_reg;10858 }10859 10860 pci_set_master(pdev);10861 pci_save_state(pdev);10862 10863 if (ii->mac == ixgbe_mac_82598EB) {10864#ifdef CONFIG_IXGBE_DCB10865 /* 8 TC w/ 4 queues per TC */10866 indices = 4 * MAX_TRAFFIC_CLASS;10867#else10868 indices = IXGBE_MAX_RSS_INDICES;10869#endif10870 }10871 10872 netdev = alloc_etherdev_mq(sizeof(struct ixgbe_adapter), indices);10873 if (!netdev) {10874 err = -ENOMEM;10875 goto err_alloc_etherdev;10876 }10877 10878 SET_NETDEV_DEV(netdev, &pdev->dev);10879 10880 adapter = netdev_priv(netdev);10881 10882 adapter->netdev = netdev;10883 adapter->pdev = pdev;10884 hw = &adapter->hw;10885 hw->back = adapter;10886 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);10887 10888 hw->hw_addr = ioremap(pci_resource_start(pdev, 0),10889 pci_resource_len(pdev, 0));10890 adapter->io_addr = hw->hw_addr;10891 if (!hw->hw_addr) {10892 err = -EIO;10893 goto err_ioremap;10894 }10895 10896 netdev->netdev_ops = &ixgbe_netdev_ops;10897 ixgbe_set_ethtool_ops(netdev);10898 netdev->watchdog_timeo = 5 * HZ;10899 strscpy(netdev->name, pci_name(pdev), sizeof(netdev->name));10900 10901 /* Setup hw api */10902 hw->mac.ops = *ii->mac_ops;10903 hw->mac.type = ii->mac;10904 hw->mvals = ii->mvals;10905 if (ii->link_ops)10906 hw->link.ops = *ii->link_ops;10907 10908 /* EEPROM */10909 hw->eeprom.ops = *ii->eeprom_ops;10910 eec = IXGBE_READ_REG(hw, IXGBE_EEC(hw));10911 if (ixgbe_removed(hw->hw_addr)) {10912 err = -EIO;10913 goto err_ioremap;10914 }10915 /* If EEPROM is valid (bit 8 = 1), use default otherwise use bit bang */10916 if (!(eec & BIT(8)))10917 hw->eeprom.ops.read = &ixgbe_read_eeprom_bit_bang_generic;10918 10919 /* PHY */10920 hw->phy.ops = *ii->phy_ops;10921 hw->phy.sfp_type = ixgbe_sfp_type_unknown;10922 /* ixgbe_identify_phy_generic will set prtad and mmds properly */10923 hw->phy.mdio.prtad = MDIO_PRTAD_NONE;10924 hw->phy.mdio.mmds = 0;10925 hw->phy.mdio.mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22;10926 hw->phy.mdio.dev = netdev;10927 hw->phy.mdio.mdio_read = ixgbe_mdio_read;10928 hw->phy.mdio.mdio_write = ixgbe_mdio_write;10929 10930 /* setup the private structure */10931 err = ixgbe_sw_init(adapter, ii);10932 if (err)10933 goto err_sw_init;10934 10935 if (adapter->hw.mac.type == ixgbe_mac_82599EB)10936 adapter->flags2 |= IXGBE_FLAG2_AUTO_DISABLE_VF;10937 10938 switch (adapter->hw.mac.type) {10939 case ixgbe_mac_X550:10940 case ixgbe_mac_X550EM_x:10941 netdev->udp_tunnel_nic_info = &ixgbe_udp_tunnels_x550;10942 break;10943 case ixgbe_mac_x550em_a:10944 netdev->udp_tunnel_nic_info = &ixgbe_udp_tunnels_x550em_a;10945 break;10946 default:10947 break;10948 }10949 10950 /* Make sure the SWFW semaphore is in a valid state */10951 if (hw->mac.ops.init_swfw_sync)10952 hw->mac.ops.init_swfw_sync(hw);10953 10954 /* Make it possible the adapter to be woken up via WOL */10955 switch (adapter->hw.mac.type) {10956 case ixgbe_mac_82599EB:10957 case ixgbe_mac_X540:10958 case ixgbe_mac_X550:10959 case ixgbe_mac_X550EM_x:10960 case ixgbe_mac_x550em_a:10961 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);10962 break;10963 default:10964 break;10965 }10966 10967 /*10968 * If there is a fan on this device and it has failed log the10969 * failure.10970 */10971 if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) {10972 u32 esdp = IXGBE_READ_REG(hw, IXGBE_ESDP);10973 if (esdp & IXGBE_ESDP_SDP1)10974 e_crit(probe, "Fan has stopped, replace the adapter\n");10975 }10976 10977 if (allow_unsupported_sfp)10978 hw->allow_unsupported_sfp = allow_unsupported_sfp;10979 10980 /* reset_hw fills in the perm_addr as well */10981 hw->phy.reset_if_overtemp = true;10982 err = hw->mac.ops.reset_hw(hw);10983 hw->phy.reset_if_overtemp = false;10984 ixgbe_set_eee_capable(adapter);10985 if (err == -ENOENT) {10986 err = 0;10987 } else if (err == -EOPNOTSUPP) {10988 e_dev_err("failed to load because an unsupported SFP+ or QSFP module type was detected.\n");10989 e_dev_err("Reload the driver after installing a supported module.\n");10990 goto err_sw_init;10991 } else if (err) {10992 e_dev_err("HW Init failed: %d\n", err);10993 goto err_sw_init;10994 }10995 10996#ifdef CONFIG_PCI_IOV10997 /* SR-IOV not supported on the 82598 */10998 if (adapter->hw.mac.type == ixgbe_mac_82598EB)10999 goto skip_sriov;11000 /* Mailbox */11001 ixgbe_init_mbx_params_pf(hw);11002 hw->mbx.ops = ii->mbx_ops;11003 pci_sriov_set_totalvfs(pdev, IXGBE_MAX_VFS_DRV_LIMIT);11004 ixgbe_enable_sriov(adapter, max_vfs);11005skip_sriov:11006 11007#endif11008 netdev->features = NETIF_F_SG |11009 NETIF_F_TSO |11010 NETIF_F_TSO6 |11011 NETIF_F_RXHASH |11012 NETIF_F_RXCSUM |11013 NETIF_F_HW_CSUM;11014 11015#define IXGBE_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \11016 NETIF_F_GSO_GRE_CSUM | \11017 NETIF_F_GSO_IPXIP4 | \11018 NETIF_F_GSO_IPXIP6 | \11019 NETIF_F_GSO_UDP_TUNNEL | \11020 NETIF_F_GSO_UDP_TUNNEL_CSUM)11021 11022 netdev->gso_partial_features = IXGBE_GSO_PARTIAL_FEATURES;11023 netdev->features |= NETIF_F_GSO_PARTIAL |11024 IXGBE_GSO_PARTIAL_FEATURES;11025 11026 if (hw->mac.type >= ixgbe_mac_82599EB)11027 netdev->features |= NETIF_F_SCTP_CRC | NETIF_F_GSO_UDP_L4;11028 11029#ifdef CONFIG_IXGBE_IPSEC11030#define IXGBE_ESP_FEATURES (NETIF_F_HW_ESP | \11031 NETIF_F_HW_ESP_TX_CSUM | \11032 NETIF_F_GSO_ESP)11033 11034 if (adapter->ipsec)11035 netdev->features |= IXGBE_ESP_FEATURES;11036#endif11037 /* copy netdev features into list of user selectable features */11038 netdev->hw_features |= netdev->features |11039 NETIF_F_HW_VLAN_CTAG_FILTER |11040 NETIF_F_HW_VLAN_CTAG_RX |11041 NETIF_F_HW_VLAN_CTAG_TX |11042 NETIF_F_RXALL |11043 NETIF_F_HW_L2FW_DOFFLOAD;11044 11045 if (hw->mac.type >= ixgbe_mac_82599EB)11046 netdev->hw_features |= NETIF_F_NTUPLE |11047 NETIF_F_HW_TC;11048 11049 netdev->features |= NETIF_F_HIGHDMA;11050 11051 netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;11052 netdev->hw_enc_features |= netdev->vlan_features;11053 netdev->mpls_features |= NETIF_F_SG |11054 NETIF_F_TSO |11055 NETIF_F_TSO6 |11056 NETIF_F_HW_CSUM;11057 netdev->mpls_features |= IXGBE_GSO_PARTIAL_FEATURES;11058 11059 /* set this bit last since it cannot be part of vlan_features */11060 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |11061 NETIF_F_HW_VLAN_CTAG_RX |11062 NETIF_F_HW_VLAN_CTAG_TX;11063 11064 netdev->priv_flags |= IFF_UNICAST_FLT;11065 netdev->priv_flags |= IFF_SUPP_NOFCS;11066 11067 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |11068 NETDEV_XDP_ACT_XSK_ZEROCOPY;11069 11070 /* MTU range: 68 - 9710 */11071 netdev->min_mtu = ETH_MIN_MTU;11072 netdev->max_mtu = IXGBE_MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN);11073 11074#ifdef CONFIG_IXGBE_DCB11075 if (adapter->flags & IXGBE_FLAG_DCB_CAPABLE)11076 netdev->dcbnl_ops = &ixgbe_dcbnl_ops;11077#endif11078 11079#ifdef IXGBE_FCOE11080 if (adapter->flags & IXGBE_FLAG_FCOE_CAPABLE) {11081 unsigned int fcoe_l;11082 11083 if (hw->mac.ops.get_device_caps) {11084 hw->mac.ops.get_device_caps(hw, &device_caps);11085 if (device_caps & IXGBE_DEVICE_CAPS_FCOE_OFFLOADS)11086 adapter->flags &= ~IXGBE_FLAG_FCOE_CAPABLE;11087 }11088 11089 11090 fcoe_l = min_t(int, IXGBE_FCRETA_SIZE, num_online_cpus());11091 adapter->ring_feature[RING_F_FCOE].limit = fcoe_l;11092 11093 netdev->features |= NETIF_F_FSO |11094 NETIF_F_FCOE_CRC;11095 11096 netdev->vlan_features |= NETIF_F_FSO |11097 NETIF_F_FCOE_CRC;11098 }11099#endif /* IXGBE_FCOE */11100 if (adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE)11101 netdev->hw_features |= NETIF_F_LRO;11102 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)11103 netdev->features |= NETIF_F_LRO;11104 11105 if (ixgbe_check_fw_error(adapter)) {11106 err = -EIO;11107 goto err_sw_init;11108 }11109 11110 /* make sure the EEPROM is good */11111 if (hw->eeprom.ops.validate_checksum(hw, NULL) < 0) {11112 e_dev_err("The EEPROM Checksum Is Not Valid\n");11113 err = -EIO;11114 goto err_sw_init;11115 }11116 11117 eth_platform_get_mac_address(&adapter->pdev->dev,11118 adapter->hw.mac.perm_addr);11119 11120 eth_hw_addr_set(netdev, hw->mac.perm_addr);11121 11122 if (!is_valid_ether_addr(netdev->dev_addr)) {11123 e_dev_err("invalid MAC address\n");11124 err = -EIO;11125 goto err_sw_init;11126 }11127 11128 /* Set hw->mac.addr to permanent MAC address */11129 ether_addr_copy(hw->mac.addr, hw->mac.perm_addr);11130 ixgbe_mac_set_default_filter(adapter);11131 11132 timer_setup(&adapter->service_timer, ixgbe_service_timer, 0);11133 11134 if (ixgbe_removed(hw->hw_addr)) {11135 err = -EIO;11136 goto err_sw_init;11137 }11138 INIT_WORK(&adapter->service_task, ixgbe_service_task);11139 set_bit(__IXGBE_SERVICE_INITED, &adapter->state);11140 clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);11141 11142 err = ixgbe_init_interrupt_scheme(adapter);11143 if (err)11144 goto err_sw_init;11145 11146 for (i = 0; i < adapter->num_rx_queues; i++)11147 u64_stats_init(&adapter->rx_ring[i]->syncp);11148 for (i = 0; i < adapter->num_tx_queues; i++)11149 u64_stats_init(&adapter->tx_ring[i]->syncp);11150 for (i = 0; i < adapter->num_xdp_queues; i++)11151 u64_stats_init(&adapter->xdp_ring[i]->syncp);11152 11153 /* WOL not supported for all devices */11154 adapter->wol = 0;11155 hw->eeprom.ops.read(hw, 0x2c, &adapter->eeprom_cap);11156 hw->wol_enabled = ixgbe_wol_supported(adapter, pdev->device,11157 pdev->subsystem_device);11158 if (hw->wol_enabled)11159 adapter->wol = IXGBE_WUFC_MAG;11160 11161 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);11162 11163 /* save off EEPROM version number */11164 ixgbe_set_fw_version(adapter);11165 11166 /* pick up the PCI bus settings for reporting later */11167 if (ixgbe_pcie_from_parent(hw))11168 ixgbe_get_parent_bus_info(adapter);11169 else11170 hw->mac.ops.get_bus_info(hw);11171 11172 /* calculate the expected PCIe bandwidth required for optimal11173 * performance. Note that some older parts will never have enough11174 * bandwidth due to being older generation PCIe parts. We clamp these11175 * parts to ensure no warning is displayed if it can't be fixed.11176 */11177 switch (hw->mac.type) {11178 case ixgbe_mac_82598EB:11179 expected_gts = min(ixgbe_enumerate_functions(adapter) * 10, 16);11180 break;11181 default:11182 expected_gts = ixgbe_enumerate_functions(adapter) * 10;11183 break;11184 }11185 11186 /* don't check link if we failed to enumerate functions */11187 if (expected_gts > 0)11188 ixgbe_check_minimum_link(adapter, expected_gts);11189 11190 err = ixgbe_read_pba_string_generic(hw, part_str, sizeof(part_str));11191 if (err)11192 strscpy(part_str, "Unknown", sizeof(part_str));11193 if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)11194 e_dev_info("MAC: %d, PHY: %d, SFP+: %d, PBA No: %s\n",11195 hw->mac.type, hw->phy.type, hw->phy.sfp_type,11196 part_str);11197 else11198 e_dev_info("MAC: %d, PHY: %d, PBA No: %s\n",11199 hw->mac.type, hw->phy.type, part_str);11200 11201 e_dev_info("%pM\n", netdev->dev_addr);11202 11203 /* reset the hardware with the new settings */11204 err = hw->mac.ops.start_hw(hw);11205 if (err == -EACCES) {11206 /* We are running on a pre-production device, log a warning */11207 e_dev_warn("This device is a pre-production adapter/LOM. "11208 "Please be aware there may be issues associated "11209 "with your hardware. If you are experiencing "11210 "problems please contact your Intel or hardware "11211 "representative who provided you with this "11212 "hardware.\n");11213 }11214 strcpy(netdev->name, "eth%d");11215 pci_set_drvdata(pdev, adapter);11216 err = register_netdev(netdev);11217 if (err)11218 goto err_register;11219 11220 11221 /* power down the optics for 82599 SFP+ fiber */11222 if (hw->mac.ops.disable_tx_laser)11223 hw->mac.ops.disable_tx_laser(hw);11224 11225 /* carrier off reporting is important to ethtool even BEFORE open */11226 netif_carrier_off(netdev);11227 11228#ifdef CONFIG_IXGBE_DCA11229 if (dca_add_requester(&pdev->dev) == 0) {11230 adapter->flags |= IXGBE_FLAG_DCA_ENABLED;11231 ixgbe_setup_dca(adapter);11232 }11233#endif11234 if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {11235 e_info(probe, "IOV is enabled with %d VFs\n", adapter->num_vfs);11236 for (i = 0; i < adapter->num_vfs; i++)11237 ixgbe_vf_configuration(pdev, (i | 0x10000000));11238 }11239 11240 /* firmware requires driver version to be 0xFFFFFFFF11241 * since os does not support feature11242 */11243 if (hw->mac.ops.set_fw_drv_ver)11244 hw->mac.ops.set_fw_drv_ver(hw, 0xFF, 0xFF, 0xFF, 0xFF,11245 sizeof(UTS_RELEASE) - 1,11246 UTS_RELEASE);11247 11248 /* add san mac addr to netdev */11249 ixgbe_add_sanmac_netdev(netdev);11250 11251 e_dev_info("%s\n", ixgbe_default_device_descr);11252 11253#ifdef CONFIG_IXGBE_HWMON11254 if (ixgbe_sysfs_init(adapter))11255 e_err(probe, "failed to allocate sysfs resources\n");11256#endif /* CONFIG_IXGBE_HWMON */11257 11258 ixgbe_dbg_adapter_init(adapter);11259 11260 /* setup link for SFP devices with MNG FW, else wait for IXGBE_UP */11261 if (ixgbe_mng_enabled(hw) && ixgbe_is_sfp(hw) && hw->mac.ops.setup_link)11262 hw->mac.ops.setup_link(hw,11263 IXGBE_LINK_SPEED_10GB_FULL | IXGBE_LINK_SPEED_1GB_FULL,11264 true);11265 11266 err = ixgbe_mii_bus_init(hw);11267 if (err)11268 goto err_netdev;11269 11270 return 0;11271 11272err_netdev:11273 unregister_netdev(netdev);11274err_register:11275 ixgbe_release_hw_control(adapter);11276 ixgbe_clear_interrupt_scheme(adapter);11277err_sw_init:11278 ixgbe_disable_sriov(adapter);11279 adapter->flags2 &= ~IXGBE_FLAG2_SEARCH_FOR_SFP;11280 iounmap(adapter->io_addr);11281 kfree(adapter->jump_tables[0]);11282 kfree(adapter->mac_table);11283 kfree(adapter->rss_key);11284 bitmap_free(adapter->af_xdp_zc_qps);11285err_ioremap:11286 disable_dev = !test_and_set_bit(__IXGBE_DISABLED, &adapter->state);11287 free_netdev(netdev);11288err_alloc_etherdev:11289 pci_release_mem_regions(pdev);11290err_pci_reg:11291err_dma:11292 if (!adapter || disable_dev)11293 pci_disable_device(pdev);11294 return err;11295}11296 11297/**11298 * ixgbe_remove - Device Removal Routine11299 * @pdev: PCI device information struct11300 *11301 * ixgbe_remove is called by the PCI subsystem to alert the driver11302 * that it should release a PCI device. The could be caused by a11303 * Hot-Plug event, or because the driver is going to be removed from11304 * memory.11305 **/11306static void ixgbe_remove(struct pci_dev *pdev)11307{11308 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);11309 struct net_device *netdev;11310 bool disable_dev;11311 int i;11312 11313 /* if !adapter then we already cleaned up in probe */11314 if (!adapter)11315 return;11316 11317 netdev = adapter->netdev;11318 ixgbe_dbg_adapter_exit(adapter);11319 11320 set_bit(__IXGBE_REMOVING, &adapter->state);11321 cancel_work_sync(&adapter->service_task);11322 11323 if (adapter->mii_bus)11324 mdiobus_unregister(adapter->mii_bus);11325 11326#ifdef CONFIG_IXGBE_DCA11327 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {11328 adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;11329 dca_remove_requester(&pdev->dev);11330 IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL,11331 IXGBE_DCA_CTRL_DCA_DISABLE);11332 }11333 11334#endif11335#ifdef CONFIG_IXGBE_HWMON11336 ixgbe_sysfs_exit(adapter);11337#endif /* CONFIG_IXGBE_HWMON */11338 11339 /* remove the added san mac */11340 ixgbe_del_sanmac_netdev(netdev);11341 11342#ifdef CONFIG_PCI_IOV11343 ixgbe_disable_sriov(adapter);11344#endif11345 if (netdev->reg_state == NETREG_REGISTERED)11346 unregister_netdev(netdev);11347 11348 ixgbe_stop_ipsec_offload(adapter);11349 ixgbe_clear_interrupt_scheme(adapter);11350 11351 ixgbe_release_hw_control(adapter);11352 11353#ifdef CONFIG_DCB11354 kfree(adapter->ixgbe_ieee_pfc);11355 kfree(adapter->ixgbe_ieee_ets);11356 11357#endif11358 iounmap(adapter->io_addr);11359 pci_release_mem_regions(pdev);11360 11361 e_dev_info("complete\n");11362 11363 for (i = 0; i < IXGBE_MAX_LINK_HANDLE; i++) {11364 if (adapter->jump_tables[i]) {11365 kfree(adapter->jump_tables[i]->input);11366 kfree(adapter->jump_tables[i]->mask);11367 }11368 kfree(adapter->jump_tables[i]);11369 }11370 11371 kfree(adapter->mac_table);11372 kfree(adapter->rss_key);11373 bitmap_free(adapter->af_xdp_zc_qps);11374 disable_dev = !test_and_set_bit(__IXGBE_DISABLED, &adapter->state);11375 free_netdev(netdev);11376 11377 if (disable_dev)11378 pci_disable_device(pdev);11379}11380 11381/**11382 * ixgbe_io_error_detected - called when PCI error is detected11383 * @pdev: Pointer to PCI device11384 * @state: The current pci connection state11385 *11386 * This function is called after a PCI bus error affecting11387 * this device has been detected.11388 */11389static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev,11390 pci_channel_state_t state)11391{11392 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);11393 struct net_device *netdev = adapter->netdev;11394 11395#ifdef CONFIG_PCI_IOV11396 struct ixgbe_hw *hw = &adapter->hw;11397 struct pci_dev *bdev, *vfdev;11398 u32 dw0, dw1, dw2, dw3;11399 int vf, pos;11400 u16 req_id, pf_func;11401 11402 if (adapter->hw.mac.type == ixgbe_mac_82598EB ||11403 adapter->num_vfs == 0)11404 goto skip_bad_vf_detection;11405 11406 bdev = pdev->bus->self;11407 while (bdev && (pci_pcie_type(bdev) != PCI_EXP_TYPE_ROOT_PORT))11408 bdev = bdev->bus->self;11409 11410 if (!bdev)11411 goto skip_bad_vf_detection;11412 11413 pos = pci_find_ext_capability(bdev, PCI_EXT_CAP_ID_ERR);11414 if (!pos)11415 goto skip_bad_vf_detection;11416 11417 dw0 = ixgbe_read_pci_cfg_dword(hw, pos + PCI_ERR_HEADER_LOG);11418 dw1 = ixgbe_read_pci_cfg_dword(hw, pos + PCI_ERR_HEADER_LOG + 4);11419 dw2 = ixgbe_read_pci_cfg_dword(hw, pos + PCI_ERR_HEADER_LOG + 8);11420 dw3 = ixgbe_read_pci_cfg_dword(hw, pos + PCI_ERR_HEADER_LOG + 12);11421 if (ixgbe_removed(hw->hw_addr))11422 goto skip_bad_vf_detection;11423 11424 req_id = dw1 >> 16;11425 /* On the 82599 if bit 7 of the requestor ID is set then it's a VF */11426 if (!(req_id & 0x0080))11427 goto skip_bad_vf_detection;11428 11429 pf_func = req_id & 0x01;11430 if ((pf_func & 1) == (pdev->devfn & 1)) {11431 unsigned int device_id;11432 11433 vf = FIELD_GET(0x7F, req_id);11434 e_dev_err("VF %d has caused a PCIe error\n", vf);11435 e_dev_err("TLP: dw0: %8.8x\tdw1: %8.8x\tdw2: "11436 "%8.8x\tdw3: %8.8x\n",11437 dw0, dw1, dw2, dw3);11438 switch (adapter->hw.mac.type) {11439 case ixgbe_mac_82599EB:11440 device_id = IXGBE_82599_VF_DEVICE_ID;11441 break;11442 case ixgbe_mac_X540:11443 device_id = IXGBE_X540_VF_DEVICE_ID;11444 break;11445 case ixgbe_mac_X550:11446 device_id = IXGBE_DEV_ID_X550_VF;11447 break;11448 case ixgbe_mac_X550EM_x:11449 device_id = IXGBE_DEV_ID_X550EM_X_VF;11450 break;11451 case ixgbe_mac_x550em_a:11452 device_id = IXGBE_DEV_ID_X550EM_A_VF;11453 break;11454 default:11455 device_id = 0;11456 break;11457 }11458 11459 /* Find the pci device of the offending VF */11460 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, device_id, NULL);11461 while (vfdev) {11462 if (vfdev->devfn == (req_id & 0xFF))11463 break;11464 vfdev = pci_get_device(PCI_VENDOR_ID_INTEL,11465 device_id, vfdev);11466 }11467 /*11468 * There's a slim chance the VF could have been hot plugged,11469 * so if it is no longer present we don't need to issue the11470 * VFLR. Just clean up the AER in that case.11471 */11472 if (vfdev) {11473 pcie_flr(vfdev);11474 /* Free device reference count */11475 pci_dev_put(vfdev);11476 }11477 }11478 11479 /*11480 * Even though the error may have occurred on the other port11481 * we still need to increment the vf error reference count for11482 * both ports because the I/O resume function will be called11483 * for both of them.11484 */11485 adapter->vferr_refcount++;11486 11487 return PCI_ERS_RESULT_RECOVERED;11488 11489skip_bad_vf_detection:11490#endif /* CONFIG_PCI_IOV */11491 if (!test_bit(__IXGBE_SERVICE_INITED, &adapter->state))11492 return PCI_ERS_RESULT_DISCONNECT;11493 11494 if (!netif_device_present(netdev))11495 return PCI_ERS_RESULT_DISCONNECT;11496 11497 rtnl_lock();11498 netif_device_detach(netdev);11499 11500 if (netif_running(netdev))11501 ixgbe_close_suspend(adapter);11502 11503 if (state == pci_channel_io_perm_failure) {11504 rtnl_unlock();11505 return PCI_ERS_RESULT_DISCONNECT;11506 }11507 11508 if (!test_and_set_bit(__IXGBE_DISABLED, &adapter->state))11509 pci_disable_device(pdev);11510 rtnl_unlock();11511 11512 /* Request a slot reset. */11513 return PCI_ERS_RESULT_NEED_RESET;11514}11515 11516/**11517 * ixgbe_io_slot_reset - called after the pci bus has been reset.11518 * @pdev: Pointer to PCI device11519 *11520 * Restart the card from scratch, as if from a cold-boot.11521 */11522static pci_ers_result_t ixgbe_io_slot_reset(struct pci_dev *pdev)11523{11524 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);11525 pci_ers_result_t result;11526 11527 if (pci_enable_device_mem(pdev)) {11528 e_err(probe, "Cannot re-enable PCI device after reset.\n");11529 result = PCI_ERS_RESULT_DISCONNECT;11530 } else {11531 smp_mb__before_atomic();11532 clear_bit(__IXGBE_DISABLED, &adapter->state);11533 adapter->hw.hw_addr = adapter->io_addr;11534 pci_set_master(pdev);11535 pci_restore_state(pdev);11536 pci_save_state(pdev);11537 11538 pci_wake_from_d3(pdev, false);11539 11540 ixgbe_reset(adapter);11541 IXGBE_WRITE_REG(&adapter->hw, IXGBE_WUS, ~0);11542 result = PCI_ERS_RESULT_RECOVERED;11543 }11544 11545 return result;11546}11547 11548/**11549 * ixgbe_io_resume - called when traffic can start flowing again.11550 * @pdev: Pointer to PCI device11551 *11552 * This callback is called when the error recovery driver tells us that11553 * its OK to resume normal operation.11554 */11555static void ixgbe_io_resume(struct pci_dev *pdev)11556{11557 struct ixgbe_adapter *adapter = pci_get_drvdata(pdev);11558 struct net_device *netdev = adapter->netdev;11559 11560#ifdef CONFIG_PCI_IOV11561 if (adapter->vferr_refcount) {11562 e_info(drv, "Resuming after VF err\n");11563 adapter->vferr_refcount--;11564 return;11565 }11566 11567#endif11568 rtnl_lock();11569 if (netif_running(netdev))11570 ixgbe_open(netdev);11571 11572 netif_device_attach(netdev);11573 rtnl_unlock();11574}11575 11576static const struct pci_error_handlers ixgbe_err_handler = {11577 .error_detected = ixgbe_io_error_detected,11578 .slot_reset = ixgbe_io_slot_reset,11579 .resume = ixgbe_io_resume,11580};11581 11582static DEFINE_SIMPLE_DEV_PM_OPS(ixgbe_pm_ops, ixgbe_suspend, ixgbe_resume);11583 11584static struct pci_driver ixgbe_driver = {11585 .name = ixgbe_driver_name,11586 .id_table = ixgbe_pci_tbl,11587 .probe = ixgbe_probe,11588 .remove = ixgbe_remove,11589 .driver.pm = pm_sleep_ptr(&ixgbe_pm_ops),11590 .shutdown = ixgbe_shutdown,11591 .sriov_configure = ixgbe_pci_sriov_configure,11592 .err_handler = &ixgbe_err_handler11593};11594 11595/**11596 * ixgbe_init_module - Driver Registration Routine11597 *11598 * ixgbe_init_module is the first routine called when the driver is11599 * loaded. All it does is register with the PCI subsystem.11600 **/11601static int __init ixgbe_init_module(void)11602{11603 int ret;11604 pr_info("%s\n", ixgbe_driver_string);11605 pr_info("%s\n", ixgbe_copyright);11606 11607 ixgbe_wq = create_singlethread_workqueue(ixgbe_driver_name);11608 if (!ixgbe_wq) {11609 pr_err("%s: Failed to create workqueue\n", ixgbe_driver_name);11610 return -ENOMEM;11611 }11612 11613 ixgbe_dbg_init();11614 11615 ret = pci_register_driver(&ixgbe_driver);11616 if (ret) {11617 destroy_workqueue(ixgbe_wq);11618 ixgbe_dbg_exit();11619 return ret;11620 }11621 11622#ifdef CONFIG_IXGBE_DCA11623 dca_register_notify(&dca_notifier);11624#endif11625 11626 return 0;11627}11628 11629module_init(ixgbe_init_module);11630 11631/**11632 * ixgbe_exit_module - Driver Exit Cleanup Routine11633 *11634 * ixgbe_exit_module is called just before the driver is removed11635 * from memory.11636 **/11637static void __exit ixgbe_exit_module(void)11638{11639#ifdef CONFIG_IXGBE_DCA11640 dca_unregister_notify(&dca_notifier);11641#endif11642 pci_unregister_driver(&ixgbe_driver);11643 11644 ixgbe_dbg_exit();11645 if (ixgbe_wq) {11646 destroy_workqueue(ixgbe_wq);11647 ixgbe_wq = NULL;11648 }11649}11650 11651#ifdef CONFIG_IXGBE_DCA11652static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event,11653 void *p)11654{11655 int ret_val;11656 11657 ret_val = driver_for_each_device(&ixgbe_driver.driver, NULL, &event,11658 __ixgbe_notify_dca);11659 11660 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;11661}11662 11663#endif /* CONFIG_IXGBE_DCA */11664 11665module_exit(ixgbe_exit_module);11666 11667/* ixgbe_main.c */11668