7979 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright(c) 1999 - 2018 Intel Corporation. */3 4#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt5 6#include <linux/module.h>7#include <linux/types.h>8#include <linux/init.h>9#include <linux/pci.h>10#include <linux/vmalloc.h>11#include <linux/pagemap.h>12#include <linux/delay.h>13#include <linux/netdevice.h>14#include <linux/interrupt.h>15#include <linux/tcp.h>16#include <linux/ipv6.h>17#include <linux/slab.h>18#include <net/checksum.h>19#include <net/ip6_checksum.h>20#include <linux/ethtool.h>21#include <linux/if_vlan.h>22#include <linux/cpu.h>23#include <linux/smp.h>24#include <linux/pm_qos.h>25#include <linux/pm_runtime.h>26#include <linux/prefetch.h>27#include <linux/suspend.h>28 29#include "e1000.h"30#define CREATE_TRACE_POINTS31#include "e1000e_trace.h"32 33char e1000e_driver_name[] = "e1000e";34 35#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)36static int debug = -1;37module_param(debug, int, 0);38MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");39 40static const struct e1000_info *e1000_info_tbl[] = {41 [board_82571] = &e1000_82571_info,42 [board_82572] = &e1000_82572_info,43 [board_82573] = &e1000_82573_info,44 [board_82574] = &e1000_82574_info,45 [board_82583] = &e1000_82583_info,46 [board_80003es2lan] = &e1000_es2_info,47 [board_ich8lan] = &e1000_ich8_info,48 [board_ich9lan] = &e1000_ich9_info,49 [board_ich10lan] = &e1000_ich10_info,50 [board_pchlan] = &e1000_pch_info,51 [board_pch2lan] = &e1000_pch2_info,52 [board_pch_lpt] = &e1000_pch_lpt_info,53 [board_pch_spt] = &e1000_pch_spt_info,54 [board_pch_cnp] = &e1000_pch_cnp_info,55 [board_pch_tgp] = &e1000_pch_tgp_info,56 [board_pch_adp] = &e1000_pch_adp_info,57 [board_pch_mtp] = &e1000_pch_mtp_info,58};59 60struct e1000_reg_info {61 u32 ofs;62 char *name;63};64 65static const struct e1000_reg_info e1000_reg_info_tbl[] = {66 /* General Registers */67 {E1000_CTRL, "CTRL"},68 {E1000_STATUS, "STATUS"},69 {E1000_CTRL_EXT, "CTRL_EXT"},70 71 /* Interrupt Registers */72 {E1000_ICR, "ICR"},73 74 /* Rx Registers */75 {E1000_RCTL, "RCTL"},76 {E1000_RDLEN(0), "RDLEN"},77 {E1000_RDH(0), "RDH"},78 {E1000_RDT(0), "RDT"},79 {E1000_RDTR, "RDTR"},80 {E1000_RXDCTL(0), "RXDCTL"},81 {E1000_ERT, "ERT"},82 {E1000_RDBAL(0), "RDBAL"},83 {E1000_RDBAH(0), "RDBAH"},84 {E1000_RDFH, "RDFH"},85 {E1000_RDFT, "RDFT"},86 {E1000_RDFHS, "RDFHS"},87 {E1000_RDFTS, "RDFTS"},88 {E1000_RDFPC, "RDFPC"},89 90 /* Tx Registers */91 {E1000_TCTL, "TCTL"},92 {E1000_TDBAL(0), "TDBAL"},93 {E1000_TDBAH(0), "TDBAH"},94 {E1000_TDLEN(0), "TDLEN"},95 {E1000_TDH(0), "TDH"},96 {E1000_TDT(0), "TDT"},97 {E1000_TIDV, "TIDV"},98 {E1000_TXDCTL(0), "TXDCTL"},99 {E1000_TADV, "TADV"},100 {E1000_TARC(0), "TARC"},101 {E1000_TDFH, "TDFH"},102 {E1000_TDFT, "TDFT"},103 {E1000_TDFHS, "TDFHS"},104 {E1000_TDFTS, "TDFTS"},105 {E1000_TDFPC, "TDFPC"},106 107 /* List Terminator */108 {0, NULL}109};110 111/**112 * __ew32_prepare - prepare to write to MAC CSR register on certain parts113 * @hw: pointer to the HW structure114 *115 * When updating the MAC CSR registers, the Manageability Engine (ME) could116 * be accessing the registers at the same time. Normally, this is handled in117 * h/w by an arbiter but on some parts there is a bug that acknowledges Host118 * accesses later than it should which could result in the register to have119 * an incorrect value. Workaround this by checking the FWSM register which120 * has bit 24 set while ME is accessing MAC CSR registers, wait if it is set121 * and try again a number of times.122 **/123static void __ew32_prepare(struct e1000_hw *hw)124{125 s32 i = E1000_ICH_FWSM_PCIM2PCI_COUNT;126 127 while ((er32(FWSM) & E1000_ICH_FWSM_PCIM2PCI) && --i)128 udelay(50);129}130 131void __ew32(struct e1000_hw *hw, unsigned long reg, u32 val)132{133 if (hw->adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)134 __ew32_prepare(hw);135 136 writel(val, hw->hw_addr + reg);137}138 139/**140 * e1000_regdump - register printout routine141 * @hw: pointer to the HW structure142 * @reginfo: pointer to the register info table143 **/144static void e1000_regdump(struct e1000_hw *hw, struct e1000_reg_info *reginfo)145{146 int n = 0;147 char rname[16];148 u32 regs[8];149 150 switch (reginfo->ofs) {151 case E1000_RXDCTL(0):152 for (n = 0; n < 2; n++)153 regs[n] = __er32(hw, E1000_RXDCTL(n));154 break;155 case E1000_TXDCTL(0):156 for (n = 0; n < 2; n++)157 regs[n] = __er32(hw, E1000_TXDCTL(n));158 break;159 case E1000_TARC(0):160 for (n = 0; n < 2; n++)161 regs[n] = __er32(hw, E1000_TARC(n));162 break;163 default:164 pr_info("%-15s %08x\n",165 reginfo->name, __er32(hw, reginfo->ofs));166 return;167 }168 169 snprintf(rname, 16, "%s%s", reginfo->name, "[0-1]");170 pr_info("%-15s %08x %08x\n", rname, regs[0], regs[1]);171}172 173static void e1000e_dump_ps_pages(struct e1000_adapter *adapter,174 struct e1000_buffer *bi)175{176 int i;177 struct e1000_ps_page *ps_page;178 179 for (i = 0; i < adapter->rx_ps_pages; i++) {180 ps_page = &bi->ps_pages[i];181 182 if (ps_page->page) {183 pr_info("packet dump for ps_page %d:\n", i);184 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,185 16, 1, page_address(ps_page->page),186 PAGE_SIZE, true);187 }188 }189}190 191/**192 * e1000e_dump - Print registers, Tx-ring and Rx-ring193 * @adapter: board private structure194 **/195static void e1000e_dump(struct e1000_adapter *adapter)196{197 struct net_device *netdev = adapter->netdev;198 struct e1000_hw *hw = &adapter->hw;199 struct e1000_reg_info *reginfo;200 struct e1000_ring *tx_ring = adapter->tx_ring;201 struct e1000_tx_desc *tx_desc;202 struct my_u0 {203 __le64 a;204 __le64 b;205 } *u0;206 struct e1000_buffer *buffer_info;207 struct e1000_ring *rx_ring = adapter->rx_ring;208 union e1000_rx_desc_packet_split *rx_desc_ps;209 union e1000_rx_desc_extended *rx_desc;210 struct my_u1 {211 __le64 a;212 __le64 b;213 __le64 c;214 __le64 d;215 } *u1;216 u32 staterr;217 int i = 0;218 219 if (!netif_msg_hw(adapter))220 return;221 222 /* Print netdevice Info */223 if (netdev) {224 dev_info(&adapter->pdev->dev, "Net device Info\n");225 pr_info("Device Name state trans_start\n");226 pr_info("%-15s %016lX %016lX\n", netdev->name,227 netdev->state, dev_trans_start(netdev));228 }229 230 /* Print Registers */231 dev_info(&adapter->pdev->dev, "Register Dump\n");232 pr_info(" Register Name Value\n");233 for (reginfo = (struct e1000_reg_info *)e1000_reg_info_tbl;234 reginfo->name; reginfo++) {235 e1000_regdump(hw, reginfo);236 }237 238 /* Print Tx Ring Summary */239 if (!netdev || !netif_running(netdev))240 return;241 242 dev_info(&adapter->pdev->dev, "Tx Ring Summary\n");243 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");244 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];245 pr_info(" %5d %5X %5X %016llX %04X %3X %016llX\n",246 0, tx_ring->next_to_use, tx_ring->next_to_clean,247 (unsigned long long)buffer_info->dma,248 buffer_info->length,249 buffer_info->next_to_watch,250 (unsigned long long)buffer_info->time_stamp);251 252 /* Print Tx Ring */253 if (!netif_msg_tx_done(adapter))254 goto rx_ring_summary;255 256 dev_info(&adapter->pdev->dev, "Tx Ring Dump\n");257 258 /* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)259 *260 * Legacy Transmit Descriptor261 * +--------------------------------------------------------------+262 * 0 | Buffer Address [63:0] (Reserved on Write Back) |263 * +--------------------------------------------------------------+264 * 8 | Special | CSS | Status | CMD | CSO | Length |265 * +--------------------------------------------------------------+266 * 63 48 47 36 35 32 31 24 23 16 15 0267 *268 * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload269 * 63 48 47 40 39 32 31 16 15 8 7 0270 * +----------------------------------------------------------------+271 * 0 | TUCSE | TUCS0 | TUCSS | IPCSE | IPCS0 | IPCSS |272 * +----------------------------------------------------------------+273 * 8 | MSS | HDRLEN | RSV | STA | TUCMD | DTYP | PAYLEN |274 * +----------------------------------------------------------------+275 * 63 48 47 40 39 36 35 32 31 24 23 20 19 0276 *277 * Extended Data Descriptor (DTYP=0x1)278 * +----------------------------------------------------------------+279 * 0 | Buffer Address [63:0] |280 * +----------------------------------------------------------------+281 * 8 | VLAN tag | POPTS | Rsvd | Status | Command | DTYP | DTALEN |282 * +----------------------------------------------------------------+283 * 63 48 47 40 39 36 35 32 31 24 23 20 19 0284 */285 pr_info("Tl[desc] [address 63:0 ] [SpeCssSCmCsLen] [bi->dma ] leng ntw timestamp bi->skb <-- Legacy format\n");286 pr_info("Tc[desc] [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma ] leng ntw timestamp bi->skb <-- Ext Context format\n");287 pr_info("Td[desc] [address 63:0 ] [VlaPoRSCm1Dlen] [bi->dma ] leng ntw timestamp bi->skb <-- Ext Data format\n");288 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {289 const char *next_desc;290 tx_desc = E1000_TX_DESC(*tx_ring, i);291 buffer_info = &tx_ring->buffer_info[i];292 u0 = (struct my_u0 *)tx_desc;293 if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)294 next_desc = " NTC/U";295 else if (i == tx_ring->next_to_use)296 next_desc = " NTU";297 else if (i == tx_ring->next_to_clean)298 next_desc = " NTC";299 else300 next_desc = "";301 pr_info("T%c[0x%03X] %016llX %016llX %016llX %04X %3X %016llX %p%s\n",302 (!(le64_to_cpu(u0->b) & BIT(29)) ? 'l' :303 ((le64_to_cpu(u0->b) & BIT(20)) ? 'd' : 'c')),304 i,305 (unsigned long long)le64_to_cpu(u0->a),306 (unsigned long long)le64_to_cpu(u0->b),307 (unsigned long long)buffer_info->dma,308 buffer_info->length, buffer_info->next_to_watch,309 (unsigned long long)buffer_info->time_stamp,310 buffer_info->skb, next_desc);311 312 if (netif_msg_pktdata(adapter) && buffer_info->skb)313 print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,314 16, 1, buffer_info->skb->data,315 buffer_info->skb->len, true);316 }317 318 /* Print Rx Ring Summary */319rx_ring_summary:320 dev_info(&adapter->pdev->dev, "Rx Ring Summary\n");321 pr_info("Queue [NTU] [NTC]\n");322 pr_info(" %5d %5X %5X\n",323 0, rx_ring->next_to_use, rx_ring->next_to_clean);324 325 /* Print Rx Ring */326 if (!netif_msg_rx_status(adapter))327 return;328 329 dev_info(&adapter->pdev->dev, "Rx Ring Dump\n");330 switch (adapter->rx_ps_pages) {331 case 1:332 case 2:333 case 3:334 /* [Extended] Packet Split Receive Descriptor Format335 *336 * +-----------------------------------------------------+337 * 0 | Buffer Address 0 [63:0] |338 * +-----------------------------------------------------+339 * 8 | Buffer Address 1 [63:0] |340 * +-----------------------------------------------------+341 * 16 | Buffer Address 2 [63:0] |342 * +-----------------------------------------------------+343 * 24 | Buffer Address 3 [63:0] |344 * +-----------------------------------------------------+345 */346 pr_info("R [desc] [buffer 0 63:0 ] [buffer 1 63:0 ] [buffer 2 63:0 ] [buffer 3 63:0 ] [bi->dma ] [bi->skb] <-- Ext Pkt Split format\n");347 /* [Extended] Receive Descriptor (Write-Back) Format348 *349 * 63 48 47 32 31 13 12 8 7 4 3 0350 * +------------------------------------------------------+351 * 0 | Packet | IP | Rsvd | MRQ | Rsvd | MRQ RSS |352 * | Checksum | Ident | | Queue | | Type |353 * +------------------------------------------------------+354 * 8 | VLAN Tag | Length | Extended Error | Extended Status |355 * +------------------------------------------------------+356 * 63 48 47 32 31 20 19 0357 */358 pr_info("RWB[desc] [ck ipid mrqhsh] [vl l0 ee es] [ l3 l2 l1 hs] [reserved ] ---------------- [bi->skb] <-- Ext Rx Write-Back format\n");359 for (i = 0; i < rx_ring->count; i++) {360 const char *next_desc;361 buffer_info = &rx_ring->buffer_info[i];362 rx_desc_ps = E1000_RX_DESC_PS(*rx_ring, i);363 u1 = (struct my_u1 *)rx_desc_ps;364 staterr =365 le32_to_cpu(rx_desc_ps->wb.middle.status_error);366 367 if (i == rx_ring->next_to_use)368 next_desc = " NTU";369 else if (i == rx_ring->next_to_clean)370 next_desc = " NTC";371 else372 next_desc = "";373 374 if (staterr & E1000_RXD_STAT_DD) {375 /* Descriptor Done */376 pr_info("%s[0x%03X] %016llX %016llX %016llX %016llX ---------------- %p%s\n",377 "RWB", i,378 (unsigned long long)le64_to_cpu(u1->a),379 (unsigned long long)le64_to_cpu(u1->b),380 (unsigned long long)le64_to_cpu(u1->c),381 (unsigned long long)le64_to_cpu(u1->d),382 buffer_info->skb, next_desc);383 } else {384 pr_info("%s[0x%03X] %016llX %016llX %016llX %016llX %016llX %p%s\n",385 "R ", i,386 (unsigned long long)le64_to_cpu(u1->a),387 (unsigned long long)le64_to_cpu(u1->b),388 (unsigned long long)le64_to_cpu(u1->c),389 (unsigned long long)le64_to_cpu(u1->d),390 (unsigned long long)buffer_info->dma,391 buffer_info->skb, next_desc);392 393 if (netif_msg_pktdata(adapter))394 e1000e_dump_ps_pages(adapter,395 buffer_info);396 }397 }398 break;399 default:400 case 0:401 /* Extended Receive Descriptor (Read) Format402 *403 * +-----------------------------------------------------+404 * 0 | Buffer Address [63:0] |405 * +-----------------------------------------------------+406 * 8 | Reserved |407 * +-----------------------------------------------------+408 */409 pr_info("R [desc] [buf addr 63:0 ] [reserved 63:0 ] [bi->dma ] [bi->skb] <-- Ext (Read) format\n");410 /* Extended Receive Descriptor (Write-Back) Format411 *412 * 63 48 47 32 31 24 23 4 3 0413 * +------------------------------------------------------+414 * | RSS Hash | | | |415 * 0 +-------------------+ Rsvd | Reserved | MRQ RSS |416 * | Packet | IP | | | Type |417 * | Checksum | Ident | | | |418 * +------------------------------------------------------+419 * 8 | VLAN Tag | Length | Extended Error | Extended Status |420 * +------------------------------------------------------+421 * 63 48 47 32 31 20 19 0422 */423 pr_info("RWB[desc] [cs ipid mrq] [vt ln xe xs] [bi->skb] <-- Ext (Write-Back) format\n");424 425 for (i = 0; i < rx_ring->count; i++) {426 const char *next_desc;427 428 buffer_info = &rx_ring->buffer_info[i];429 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);430 u1 = (struct my_u1 *)rx_desc;431 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);432 433 if (i == rx_ring->next_to_use)434 next_desc = " NTU";435 else if (i == rx_ring->next_to_clean)436 next_desc = " NTC";437 else438 next_desc = "";439 440 if (staterr & E1000_RXD_STAT_DD) {441 /* Descriptor Done */442 pr_info("%s[0x%03X] %016llX %016llX ---------------- %p%s\n",443 "RWB", i,444 (unsigned long long)le64_to_cpu(u1->a),445 (unsigned long long)le64_to_cpu(u1->b),446 buffer_info->skb, next_desc);447 } else {448 pr_info("%s[0x%03X] %016llX %016llX %016llX %p%s\n",449 "R ", i,450 (unsigned long long)le64_to_cpu(u1->a),451 (unsigned long long)le64_to_cpu(u1->b),452 (unsigned long long)buffer_info->dma,453 buffer_info->skb, next_desc);454 455 if (netif_msg_pktdata(adapter) &&456 buffer_info->skb)457 print_hex_dump(KERN_INFO, "",458 DUMP_PREFIX_ADDRESS, 16,459 1,460 buffer_info->skb->data,461 adapter->rx_buffer_len,462 true);463 }464 }465 }466}467 468/**469 * e1000_desc_unused - calculate if we have unused descriptors470 * @ring: pointer to ring struct to perform calculation on471 **/472static int e1000_desc_unused(struct e1000_ring *ring)473{474 if (ring->next_to_clean > ring->next_to_use)475 return ring->next_to_clean - ring->next_to_use - 1;476 477 return ring->count + ring->next_to_clean - ring->next_to_use - 1;478}479 480/**481 * e1000e_systim_to_hwtstamp - convert system time value to hw time stamp482 * @adapter: board private structure483 * @hwtstamps: time stamp structure to update484 * @systim: unsigned 64bit system time value.485 *486 * Convert the system time value stored in the RX/TXSTMP registers into a487 * hwtstamp which can be used by the upper level time stamping functions.488 *489 * The 'systim_lock' spinlock is used to protect the consistency of the490 * system time value. This is needed because reading the 64 bit time491 * value involves reading two 32 bit registers. The first read latches the492 * value.493 **/494static void e1000e_systim_to_hwtstamp(struct e1000_adapter *adapter,495 struct skb_shared_hwtstamps *hwtstamps,496 u64 systim)497{498 u64 ns;499 unsigned long flags;500 501 spin_lock_irqsave(&adapter->systim_lock, flags);502 ns = timecounter_cyc2time(&adapter->tc, systim);503 spin_unlock_irqrestore(&adapter->systim_lock, flags);504 505 memset(hwtstamps, 0, sizeof(*hwtstamps));506 hwtstamps->hwtstamp = ns_to_ktime(ns);507}508 509/**510 * e1000e_rx_hwtstamp - utility function which checks for Rx time stamp511 * @adapter: board private structure512 * @status: descriptor extended error and status field513 * @skb: particular skb to include time stamp514 *515 * If the time stamp is valid, convert it into the timecounter ns value516 * and store that result into the shhwtstamps structure which is passed517 * up the network stack.518 **/519static void e1000e_rx_hwtstamp(struct e1000_adapter *adapter, u32 status,520 struct sk_buff *skb)521{522 struct e1000_hw *hw = &adapter->hw;523 u64 rxstmp;524 525 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP) ||526 !(status & E1000_RXDEXT_STATERR_TST) ||527 !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))528 return;529 530 /* The Rx time stamp registers contain the time stamp. No other531 * received packet will be time stamped until the Rx time stamp532 * registers are read. Because only one packet can be time stamped533 * at a time, the register values must belong to this packet and534 * therefore none of the other additional attributes need to be535 * compared.536 */537 rxstmp = (u64)er32(RXSTMPL);538 rxstmp |= (u64)er32(RXSTMPH) << 32;539 e1000e_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), rxstmp);540 541 adapter->flags2 &= ~FLAG2_CHECK_RX_HWTSTAMP;542}543 544/**545 * e1000_receive_skb - helper function to handle Rx indications546 * @adapter: board private structure547 * @netdev: pointer to netdev struct548 * @staterr: descriptor extended error and status field as written by hardware549 * @vlan: descriptor vlan field as written by hardware (no le/be conversion)550 * @skb: pointer to sk_buff to be indicated to stack551 **/552static void e1000_receive_skb(struct e1000_adapter *adapter,553 struct net_device *netdev, struct sk_buff *skb,554 u32 staterr, __le16 vlan)555{556 u16 tag = le16_to_cpu(vlan);557 558 e1000e_rx_hwtstamp(adapter, staterr, skb);559 560 skb->protocol = eth_type_trans(skb, netdev);561 562 if (staterr & E1000_RXD_STAT_VP)563 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tag);564 565 napi_gro_receive(&adapter->napi, skb);566}567 568/**569 * e1000_rx_checksum - Receive Checksum Offload570 * @adapter: board private structure571 * @status_err: receive descriptor status and error fields572 * @skb: socket buffer with received data573 **/574static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,575 struct sk_buff *skb)576{577 u16 status = (u16)status_err;578 u8 errors = (u8)(status_err >> 24);579 580 skb_checksum_none_assert(skb);581 582 /* Rx checksum disabled */583 if (!(adapter->netdev->features & NETIF_F_RXCSUM))584 return;585 586 /* Ignore Checksum bit is set */587 if (status & E1000_RXD_STAT_IXSM)588 return;589 590 /* TCP/UDP checksum error bit or IP checksum error bit is set */591 if (errors & (E1000_RXD_ERR_TCPE | E1000_RXD_ERR_IPE)) {592 /* let the stack verify checksum errors */593 adapter->hw_csum_err++;594 return;595 }596 597 /* TCP/UDP Checksum has not been calculated */598 if (!(status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)))599 return;600 601 /* It must be a TCP or UDP packet with a valid checksum */602 skb->ip_summed = CHECKSUM_UNNECESSARY;603 adapter->hw_csum_good++;604}605 606static void e1000e_update_rdt_wa(struct e1000_ring *rx_ring, unsigned int i)607{608 struct e1000_adapter *adapter = rx_ring->adapter;609 struct e1000_hw *hw = &adapter->hw;610 611 __ew32_prepare(hw);612 writel(i, rx_ring->tail);613 614 if (unlikely(i != readl(rx_ring->tail))) {615 u32 rctl = er32(RCTL);616 617 ew32(RCTL, rctl & ~E1000_RCTL_EN);618 e_err("ME firmware caused invalid RDT - resetting\n");619 schedule_work(&adapter->reset_task);620 }621}622 623static void e1000e_update_tdt_wa(struct e1000_ring *tx_ring, unsigned int i)624{625 struct e1000_adapter *adapter = tx_ring->adapter;626 struct e1000_hw *hw = &adapter->hw;627 628 __ew32_prepare(hw);629 writel(i, tx_ring->tail);630 631 if (unlikely(i != readl(tx_ring->tail))) {632 u32 tctl = er32(TCTL);633 634 ew32(TCTL, tctl & ~E1000_TCTL_EN);635 e_err("ME firmware caused invalid TDT - resetting\n");636 schedule_work(&adapter->reset_task);637 }638}639 640/**641 * e1000_alloc_rx_buffers - Replace used receive buffers642 * @rx_ring: Rx descriptor ring643 * @cleaned_count: number to reallocate644 * @gfp: flags for allocation645 **/646static void e1000_alloc_rx_buffers(struct e1000_ring *rx_ring,647 int cleaned_count, gfp_t gfp)648{649 struct e1000_adapter *adapter = rx_ring->adapter;650 struct net_device *netdev = adapter->netdev;651 struct pci_dev *pdev = adapter->pdev;652 union e1000_rx_desc_extended *rx_desc;653 struct e1000_buffer *buffer_info;654 struct sk_buff *skb;655 unsigned int i;656 unsigned int bufsz = adapter->rx_buffer_len;657 658 i = rx_ring->next_to_use;659 buffer_info = &rx_ring->buffer_info[i];660 661 while (cleaned_count--) {662 skb = buffer_info->skb;663 if (skb) {664 skb_trim(skb, 0);665 goto map_skb;666 }667 668 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);669 if (!skb) {670 /* Better luck next round */671 adapter->alloc_rx_buff_failed++;672 break;673 }674 675 buffer_info->skb = skb;676map_skb:677 buffer_info->dma = dma_map_single(&pdev->dev, skb->data,678 adapter->rx_buffer_len,679 DMA_FROM_DEVICE);680 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {681 dev_err(&pdev->dev, "Rx DMA map failed\n");682 adapter->rx_dma_failed++;683 break;684 }685 686 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);687 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);688 689 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {690 /* Force memory writes to complete before letting h/w691 * know there are new descriptors to fetch. (Only692 * applicable for weak-ordered memory model archs,693 * such as IA-64).694 */695 wmb();696 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)697 e1000e_update_rdt_wa(rx_ring, i);698 else699 writel(i, rx_ring->tail);700 }701 i++;702 if (i == rx_ring->count)703 i = 0;704 buffer_info = &rx_ring->buffer_info[i];705 }706 707 rx_ring->next_to_use = i;708}709 710/**711 * e1000_alloc_rx_buffers_ps - Replace used receive buffers; packet split712 * @rx_ring: Rx descriptor ring713 * @cleaned_count: number to reallocate714 * @gfp: flags for allocation715 **/716static void e1000_alloc_rx_buffers_ps(struct e1000_ring *rx_ring,717 int cleaned_count, gfp_t gfp)718{719 struct e1000_adapter *adapter = rx_ring->adapter;720 struct net_device *netdev = adapter->netdev;721 struct pci_dev *pdev = adapter->pdev;722 union e1000_rx_desc_packet_split *rx_desc;723 struct e1000_buffer *buffer_info;724 struct e1000_ps_page *ps_page;725 struct sk_buff *skb;726 unsigned int i, j;727 728 i = rx_ring->next_to_use;729 buffer_info = &rx_ring->buffer_info[i];730 731 while (cleaned_count--) {732 rx_desc = E1000_RX_DESC_PS(*rx_ring, i);733 734 for (j = 0; j < PS_PAGE_BUFFERS; j++) {735 ps_page = &buffer_info->ps_pages[j];736 if (j >= adapter->rx_ps_pages) {737 /* all unused desc entries get hw null ptr */738 rx_desc->read.buffer_addr[j + 1] =739 ~cpu_to_le64(0);740 continue;741 }742 if (!ps_page->page) {743 ps_page->page = alloc_page(gfp);744 if (!ps_page->page) {745 adapter->alloc_rx_buff_failed++;746 goto no_buffers;747 }748 ps_page->dma = dma_map_page(&pdev->dev,749 ps_page->page,750 0, PAGE_SIZE,751 DMA_FROM_DEVICE);752 if (dma_mapping_error(&pdev->dev,753 ps_page->dma)) {754 dev_err(&adapter->pdev->dev,755 "Rx DMA page map failed\n");756 adapter->rx_dma_failed++;757 goto no_buffers;758 }759 }760 /* Refresh the desc even if buffer_addrs761 * didn't change because each write-back762 * erases this info.763 */764 rx_desc->read.buffer_addr[j + 1] =765 cpu_to_le64(ps_page->dma);766 }767 768 skb = __netdev_alloc_skb_ip_align(netdev, adapter->rx_ps_bsize0,769 gfp);770 771 if (!skb) {772 adapter->alloc_rx_buff_failed++;773 break;774 }775 776 buffer_info->skb = skb;777 buffer_info->dma = dma_map_single(&pdev->dev, skb->data,778 adapter->rx_ps_bsize0,779 DMA_FROM_DEVICE);780 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {781 dev_err(&pdev->dev, "Rx DMA map failed\n");782 adapter->rx_dma_failed++;783 /* cleanup skb */784 dev_kfree_skb_any(skb);785 buffer_info->skb = NULL;786 break;787 }788 789 rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma);790 791 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {792 /* Force memory writes to complete before letting h/w793 * know there are new descriptors to fetch. (Only794 * applicable for weak-ordered memory model archs,795 * such as IA-64).796 */797 wmb();798 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)799 e1000e_update_rdt_wa(rx_ring, i << 1);800 else801 writel(i << 1, rx_ring->tail);802 }803 804 i++;805 if (i == rx_ring->count)806 i = 0;807 buffer_info = &rx_ring->buffer_info[i];808 }809 810no_buffers:811 rx_ring->next_to_use = i;812}813 814/**815 * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers816 * @rx_ring: Rx descriptor ring817 * @cleaned_count: number of buffers to allocate this pass818 * @gfp: flags for allocation819 **/820 821static void e1000_alloc_jumbo_rx_buffers(struct e1000_ring *rx_ring,822 int cleaned_count, gfp_t gfp)823{824 struct e1000_adapter *adapter = rx_ring->adapter;825 struct net_device *netdev = adapter->netdev;826 struct pci_dev *pdev = adapter->pdev;827 union e1000_rx_desc_extended *rx_desc;828 struct e1000_buffer *buffer_info;829 struct sk_buff *skb;830 unsigned int i;831 unsigned int bufsz = 256 - 16; /* for skb_reserve */832 833 i = rx_ring->next_to_use;834 buffer_info = &rx_ring->buffer_info[i];835 836 while (cleaned_count--) {837 skb = buffer_info->skb;838 if (skb) {839 skb_trim(skb, 0);840 goto check_page;841 }842 843 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);844 if (unlikely(!skb)) {845 /* Better luck next round */846 adapter->alloc_rx_buff_failed++;847 break;848 }849 850 buffer_info->skb = skb;851check_page:852 /* allocate a new page if necessary */853 if (!buffer_info->page) {854 buffer_info->page = alloc_page(gfp);855 if (unlikely(!buffer_info->page)) {856 adapter->alloc_rx_buff_failed++;857 break;858 }859 }860 861 if (!buffer_info->dma) {862 buffer_info->dma = dma_map_page(&pdev->dev,863 buffer_info->page, 0,864 PAGE_SIZE,865 DMA_FROM_DEVICE);866 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {867 adapter->alloc_rx_buff_failed++;868 break;869 }870 }871 872 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);873 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);874 875 if (unlikely(++i == rx_ring->count))876 i = 0;877 buffer_info = &rx_ring->buffer_info[i];878 }879 880 if (likely(rx_ring->next_to_use != i)) {881 rx_ring->next_to_use = i;882 if (unlikely(i-- == 0))883 i = (rx_ring->count - 1);884 885 /* Force memory writes to complete before letting h/w886 * know there are new descriptors to fetch. (Only887 * applicable for weak-ordered memory model archs,888 * such as IA-64).889 */890 wmb();891 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)892 e1000e_update_rdt_wa(rx_ring, i);893 else894 writel(i, rx_ring->tail);895 }896}897 898static inline void e1000_rx_hash(struct net_device *netdev, __le32 rss,899 struct sk_buff *skb)900{901 if (netdev->features & NETIF_F_RXHASH)902 skb_set_hash(skb, le32_to_cpu(rss), PKT_HASH_TYPE_L3);903}904 905/**906 * e1000_clean_rx_irq - Send received data up the network stack907 * @rx_ring: Rx descriptor ring908 * @work_done: output parameter for indicating completed work909 * @work_to_do: how many packets we can clean910 *911 * the return value indicates whether actual cleaning was done, there912 * is no guarantee that everything was cleaned913 **/914static bool e1000_clean_rx_irq(struct e1000_ring *rx_ring, int *work_done,915 int work_to_do)916{917 struct e1000_adapter *adapter = rx_ring->adapter;918 struct net_device *netdev = adapter->netdev;919 struct pci_dev *pdev = adapter->pdev;920 struct e1000_hw *hw = &adapter->hw;921 union e1000_rx_desc_extended *rx_desc, *next_rxd;922 struct e1000_buffer *buffer_info, *next_buffer;923 u32 length, staterr;924 unsigned int i;925 int cleaned_count = 0;926 bool cleaned = false;927 unsigned int total_rx_bytes = 0, total_rx_packets = 0;928 929 i = rx_ring->next_to_clean;930 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);931 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);932 buffer_info = &rx_ring->buffer_info[i];933 934 while (staterr & E1000_RXD_STAT_DD) {935 struct sk_buff *skb;936 937 if (*work_done >= work_to_do)938 break;939 (*work_done)++;940 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */941 942 skb = buffer_info->skb;943 buffer_info->skb = NULL;944 945 prefetch(skb->data - NET_IP_ALIGN);946 947 i++;948 if (i == rx_ring->count)949 i = 0;950 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i);951 prefetch(next_rxd);952 953 next_buffer = &rx_ring->buffer_info[i];954 955 cleaned = true;956 cleaned_count++;957 dma_unmap_single(&pdev->dev, buffer_info->dma,958 adapter->rx_buffer_len, DMA_FROM_DEVICE);959 buffer_info->dma = 0;960 961 length = le16_to_cpu(rx_desc->wb.upper.length);962 963 /* !EOP means multiple descriptors were used to store a single964 * packet, if that's the case we need to toss it. In fact, we965 * need to toss every packet with the EOP bit clear and the966 * next frame that _does_ have the EOP bit set, as it is by967 * definition only a frame fragment968 */969 if (unlikely(!(staterr & E1000_RXD_STAT_EOP)))970 adapter->flags2 |= FLAG2_IS_DISCARDING;971 972 if (adapter->flags2 & FLAG2_IS_DISCARDING) {973 /* All receives must fit into a single buffer */974 e_dbg("Receive packet consumed multiple buffers\n");975 /* recycle */976 buffer_info->skb = skb;977 if (staterr & E1000_RXD_STAT_EOP)978 adapter->flags2 &= ~FLAG2_IS_DISCARDING;979 goto next_desc;980 }981 982 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&983 !(netdev->features & NETIF_F_RXALL))) {984 /* recycle */985 buffer_info->skb = skb;986 goto next_desc;987 }988 989 /* adjust length to remove Ethernet CRC */990 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {991 /* If configured to store CRC, don't subtract FCS,992 * but keep the FCS bytes out of the total_rx_bytes993 * counter994 */995 if (netdev->features & NETIF_F_RXFCS)996 total_rx_bytes -= 4;997 else998 length -= 4;999 }1000 1001 total_rx_bytes += length;1002 total_rx_packets++;1003 1004 /* code added for copybreak, this should improve1005 * performance for small packets with large amounts1006 * of reassembly being done in the stack1007 */1008 if (length < copybreak) {1009 struct sk_buff *new_skb =1010 napi_alloc_skb(&adapter->napi, length);1011 if (new_skb) {1012 skb_copy_to_linear_data_offset(new_skb,1013 -NET_IP_ALIGN,1014 (skb->data -1015 NET_IP_ALIGN),1016 (length +1017 NET_IP_ALIGN));1018 /* save the skb in buffer_info as good */1019 buffer_info->skb = skb;1020 skb = new_skb;1021 }1022 /* else just continue with the old one */1023 }1024 /* end copybreak code */1025 skb_put(skb, length);1026 1027 /* Receive Checksum Offload */1028 e1000_rx_checksum(adapter, staterr, skb);1029 1030 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);1031 1032 e1000_receive_skb(adapter, netdev, skb, staterr,1033 rx_desc->wb.upper.vlan);1034 1035next_desc:1036 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF);1037 1038 /* return some buffers to hardware, one at a time is too slow */1039 if (cleaned_count >= E1000_RX_BUFFER_WRITE) {1040 adapter->alloc_rx_buf(rx_ring, cleaned_count,1041 GFP_ATOMIC);1042 cleaned_count = 0;1043 }1044 1045 /* use prefetched values */1046 rx_desc = next_rxd;1047 buffer_info = next_buffer;1048 1049 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);1050 }1051 rx_ring->next_to_clean = i;1052 1053 cleaned_count = e1000_desc_unused(rx_ring);1054 if (cleaned_count)1055 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);1056 1057 adapter->total_rx_bytes += total_rx_bytes;1058 adapter->total_rx_packets += total_rx_packets;1059 return cleaned;1060}1061 1062static void e1000_put_txbuf(struct e1000_ring *tx_ring,1063 struct e1000_buffer *buffer_info,1064 bool drop)1065{1066 struct e1000_adapter *adapter = tx_ring->adapter;1067 1068 if (buffer_info->dma) {1069 if (buffer_info->mapped_as_page)1070 dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,1071 buffer_info->length, DMA_TO_DEVICE);1072 else1073 dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,1074 buffer_info->length, DMA_TO_DEVICE);1075 buffer_info->dma = 0;1076 }1077 if (buffer_info->skb) {1078 if (drop)1079 dev_kfree_skb_any(buffer_info->skb);1080 else1081 dev_consume_skb_any(buffer_info->skb);1082 buffer_info->skb = NULL;1083 }1084 buffer_info->time_stamp = 0;1085}1086 1087static void e1000_print_hw_hang(struct work_struct *work)1088{1089 struct e1000_adapter *adapter = container_of(work,1090 struct e1000_adapter,1091 print_hang_task);1092 struct net_device *netdev = adapter->netdev;1093 struct e1000_ring *tx_ring = adapter->tx_ring;1094 unsigned int i = tx_ring->next_to_clean;1095 unsigned int eop = tx_ring->buffer_info[i].next_to_watch;1096 struct e1000_tx_desc *eop_desc = E1000_TX_DESC(*tx_ring, eop);1097 struct e1000_hw *hw = &adapter->hw;1098 u16 phy_status, phy_1000t_status, phy_ext_status;1099 u16 pci_status;1100 1101 if (test_bit(__E1000_DOWN, &adapter->state))1102 return;1103 1104 if (!adapter->tx_hang_recheck && (adapter->flags2 & FLAG2_DMA_BURST)) {1105 /* May be block on write-back, flush and detect again1106 * flush pending descriptor writebacks to memory1107 */1108 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);1109 /* execute the writes immediately */1110 e1e_flush();1111 /* Due to rare timing issues, write to TIDV again to ensure1112 * the write is successful1113 */1114 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);1115 /* execute the writes immediately */1116 e1e_flush();1117 adapter->tx_hang_recheck = true;1118 return;1119 }1120 adapter->tx_hang_recheck = false;1121 1122 if (er32(TDH(0)) == er32(TDT(0))) {1123 e_dbg("false hang detected, ignoring\n");1124 return;1125 }1126 1127 /* Real hang detected */1128 netif_stop_queue(netdev);1129 1130 e1e_rphy(hw, MII_BMSR, &phy_status);1131 e1e_rphy(hw, MII_STAT1000, &phy_1000t_status);1132 e1e_rphy(hw, MII_ESTATUS, &phy_ext_status);1133 1134 pci_read_config_word(adapter->pdev, PCI_STATUS, &pci_status);1135 1136 /* detected Hardware unit hang */1137 e_err("Detected Hardware Unit Hang:\n"1138 " TDH <%x>\n"1139 " TDT <%x>\n"1140 " next_to_use <%x>\n"1141 " next_to_clean <%x>\n"1142 "buffer_info[next_to_clean]:\n"1143 " time_stamp <%lx>\n"1144 " next_to_watch <%x>\n"1145 " jiffies <%lx>\n"1146 " next_to_watch.status <%x>\n"1147 "MAC Status <%x>\n"1148 "PHY Status <%x>\n"1149 "PHY 1000BASE-T Status <%x>\n"1150 "PHY Extended Status <%x>\n"1151 "PCI Status <%x>\n",1152 readl(tx_ring->head), readl(tx_ring->tail), tx_ring->next_to_use,1153 tx_ring->next_to_clean, tx_ring->buffer_info[eop].time_stamp,1154 eop, jiffies, eop_desc->upper.fields.status, er32(STATUS),1155 phy_status, phy_1000t_status, phy_ext_status, pci_status);1156 1157 e1000e_dump(adapter);1158 1159 /* Suggest workaround for known h/w issue */1160 if ((hw->mac.type == e1000_pchlan) && (er32(CTRL) & E1000_CTRL_TFCE))1161 e_err("Try turning off Tx pause (flow control) via ethtool\n");1162}1163 1164/**1165 * e1000e_tx_hwtstamp_work - check for Tx time stamp1166 * @work: pointer to work struct1167 *1168 * This work function polls the TSYNCTXCTL valid bit to determine when a1169 * timestamp has been taken for the current stored skb. The timestamp must1170 * be for this skb because only one such packet is allowed in the queue.1171 */1172static void e1000e_tx_hwtstamp_work(struct work_struct *work)1173{1174 struct e1000_adapter *adapter = container_of(work, struct e1000_adapter,1175 tx_hwtstamp_work);1176 struct e1000_hw *hw = &adapter->hw;1177 1178 if (er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID) {1179 struct sk_buff *skb = adapter->tx_hwtstamp_skb;1180 struct skb_shared_hwtstamps shhwtstamps;1181 u64 txstmp;1182 1183 txstmp = er32(TXSTMPL);1184 txstmp |= (u64)er32(TXSTMPH) << 32;1185 1186 e1000e_systim_to_hwtstamp(adapter, &shhwtstamps, txstmp);1187 1188 /* Clear the global tx_hwtstamp_skb pointer and force writes1189 * prior to notifying the stack of a Tx timestamp.1190 */1191 adapter->tx_hwtstamp_skb = NULL;1192 wmb(); /* force write prior to skb_tstamp_tx */1193 1194 skb_tstamp_tx(skb, &shhwtstamps);1195 dev_consume_skb_any(skb);1196 } else if (time_after(jiffies, adapter->tx_hwtstamp_start1197 + adapter->tx_timeout_factor * HZ)) {1198 dev_kfree_skb_any(adapter->tx_hwtstamp_skb);1199 adapter->tx_hwtstamp_skb = NULL;1200 adapter->tx_hwtstamp_timeouts++;1201 e_warn("clearing Tx timestamp hang\n");1202 } else {1203 /* reschedule to check later */1204 schedule_work(&adapter->tx_hwtstamp_work);1205 }1206}1207 1208/**1209 * e1000_clean_tx_irq - Reclaim resources after transmit completes1210 * @tx_ring: Tx descriptor ring1211 *1212 * the return value indicates whether actual cleaning was done, there1213 * is no guarantee that everything was cleaned1214 **/1215static bool e1000_clean_tx_irq(struct e1000_ring *tx_ring)1216{1217 struct e1000_adapter *adapter = tx_ring->adapter;1218 struct net_device *netdev = adapter->netdev;1219 struct e1000_hw *hw = &adapter->hw;1220 struct e1000_tx_desc *tx_desc, *eop_desc;1221 struct e1000_buffer *buffer_info;1222 unsigned int i, eop;1223 unsigned int count = 0;1224 unsigned int total_tx_bytes = 0, total_tx_packets = 0;1225 unsigned int bytes_compl = 0, pkts_compl = 0;1226 1227 i = tx_ring->next_to_clean;1228 eop = tx_ring->buffer_info[i].next_to_watch;1229 eop_desc = E1000_TX_DESC(*tx_ring, eop);1230 1231 while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&1232 (count < tx_ring->count)) {1233 bool cleaned = false;1234 1235 dma_rmb(); /* read buffer_info after eop_desc */1236 for (; !cleaned; count++) {1237 tx_desc = E1000_TX_DESC(*tx_ring, i);1238 buffer_info = &tx_ring->buffer_info[i];1239 cleaned = (i == eop);1240 1241 if (cleaned) {1242 total_tx_packets += buffer_info->segs;1243 total_tx_bytes += buffer_info->bytecount;1244 if (buffer_info->skb) {1245 bytes_compl += buffer_info->skb->len;1246 pkts_compl++;1247 }1248 }1249 1250 e1000_put_txbuf(tx_ring, buffer_info, false);1251 tx_desc->upper.data = 0;1252 1253 i++;1254 if (i == tx_ring->count)1255 i = 0;1256 }1257 1258 if (i == tx_ring->next_to_use)1259 break;1260 eop = tx_ring->buffer_info[i].next_to_watch;1261 eop_desc = E1000_TX_DESC(*tx_ring, eop);1262 }1263 1264 tx_ring->next_to_clean = i;1265 1266 netdev_completed_queue(netdev, pkts_compl, bytes_compl);1267 1268#define TX_WAKE_THRESHOLD 321269 if (count && netif_carrier_ok(netdev) &&1270 e1000_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD) {1271 /* Make sure that anybody stopping the queue after this1272 * sees the new next_to_clean.1273 */1274 smp_mb();1275 1276 if (netif_queue_stopped(netdev) &&1277 !(test_bit(__E1000_DOWN, &adapter->state))) {1278 netif_wake_queue(netdev);1279 ++adapter->restart_queue;1280 }1281 }1282 1283 if (adapter->detect_tx_hung) {1284 /* Detect a transmit hang in hardware, this serializes the1285 * check with the clearing of time_stamp and movement of i1286 */1287 adapter->detect_tx_hung = false;1288 if (tx_ring->buffer_info[i].time_stamp &&1289 time_after(jiffies, tx_ring->buffer_info[i].time_stamp1290 + (adapter->tx_timeout_factor * HZ)) &&1291 !(er32(STATUS) & E1000_STATUS_TXOFF))1292 schedule_work(&adapter->print_hang_task);1293 else1294 adapter->tx_hang_recheck = false;1295 }1296 adapter->total_tx_bytes += total_tx_bytes;1297 adapter->total_tx_packets += total_tx_packets;1298 return count < tx_ring->count;1299}1300 1301/**1302 * e1000_clean_rx_irq_ps - Send received data up the network stack; packet split1303 * @rx_ring: Rx descriptor ring1304 * @work_done: output parameter for indicating completed work1305 * @work_to_do: how many packets we can clean1306 *1307 * the return value indicates whether actual cleaning was done, there1308 * is no guarantee that everything was cleaned1309 **/1310static bool e1000_clean_rx_irq_ps(struct e1000_ring *rx_ring, int *work_done,1311 int work_to_do)1312{1313 struct e1000_adapter *adapter = rx_ring->adapter;1314 struct e1000_hw *hw = &adapter->hw;1315 union e1000_rx_desc_packet_split *rx_desc, *next_rxd;1316 struct net_device *netdev = adapter->netdev;1317 struct pci_dev *pdev = adapter->pdev;1318 struct e1000_buffer *buffer_info, *next_buffer;1319 struct e1000_ps_page *ps_page;1320 struct sk_buff *skb;1321 unsigned int i, j;1322 u32 length, staterr;1323 int cleaned_count = 0;1324 bool cleaned = false;1325 unsigned int total_rx_bytes = 0, total_rx_packets = 0;1326 1327 i = rx_ring->next_to_clean;1328 rx_desc = E1000_RX_DESC_PS(*rx_ring, i);1329 staterr = le32_to_cpu(rx_desc->wb.middle.status_error);1330 buffer_info = &rx_ring->buffer_info[i];1331 1332 while (staterr & E1000_RXD_STAT_DD) {1333 if (*work_done >= work_to_do)1334 break;1335 (*work_done)++;1336 skb = buffer_info->skb;1337 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */1338 1339 /* in the packet split case this is header only */1340 prefetch(skb->data - NET_IP_ALIGN);1341 1342 i++;1343 if (i == rx_ring->count)1344 i = 0;1345 next_rxd = E1000_RX_DESC_PS(*rx_ring, i);1346 prefetch(next_rxd);1347 1348 next_buffer = &rx_ring->buffer_info[i];1349 1350 cleaned = true;1351 cleaned_count++;1352 dma_unmap_single(&pdev->dev, buffer_info->dma,1353 adapter->rx_ps_bsize0, DMA_FROM_DEVICE);1354 buffer_info->dma = 0;1355 1356 /* see !EOP comment in other Rx routine */1357 if (!(staterr & E1000_RXD_STAT_EOP))1358 adapter->flags2 |= FLAG2_IS_DISCARDING;1359 1360 if (adapter->flags2 & FLAG2_IS_DISCARDING) {1361 e_dbg("Packet Split buffers didn't pick up the full packet\n");1362 dev_kfree_skb_irq(skb);1363 if (staterr & E1000_RXD_STAT_EOP)1364 adapter->flags2 &= ~FLAG2_IS_DISCARDING;1365 goto next_desc;1366 }1367 1368 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&1369 !(netdev->features & NETIF_F_RXALL))) {1370 dev_kfree_skb_irq(skb);1371 goto next_desc;1372 }1373 1374 length = le16_to_cpu(rx_desc->wb.middle.length0);1375 1376 if (!length) {1377 e_dbg("Last part of the packet spanning multiple descriptors\n");1378 dev_kfree_skb_irq(skb);1379 goto next_desc;1380 }1381 1382 /* Good Receive */1383 skb_put(skb, length);1384 1385 {1386 /* this looks ugly, but it seems compiler issues make1387 * it more efficient than reusing j1388 */1389 int l1 = le16_to_cpu(rx_desc->wb.upper.length[0]);1390 1391 /* page alloc/put takes too long and effects small1392 * packet throughput, so unsplit small packets and1393 * save the alloc/put1394 */1395 if (l1 && (l1 <= copybreak) &&1396 ((length + l1) <= adapter->rx_ps_bsize0)) {1397 ps_page = &buffer_info->ps_pages[0];1398 1399 dma_sync_single_for_cpu(&pdev->dev,1400 ps_page->dma,1401 PAGE_SIZE,1402 DMA_FROM_DEVICE);1403 memcpy(skb_tail_pointer(skb),1404 page_address(ps_page->page), l1);1405 dma_sync_single_for_device(&pdev->dev,1406 ps_page->dma,1407 PAGE_SIZE,1408 DMA_FROM_DEVICE);1409 1410 /* remove the CRC */1411 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {1412 if (!(netdev->features & NETIF_F_RXFCS))1413 l1 -= 4;1414 }1415 1416 skb_put(skb, l1);1417 goto copydone;1418 } /* if */1419 }1420 1421 for (j = 0; j < PS_PAGE_BUFFERS; j++) {1422 length = le16_to_cpu(rx_desc->wb.upper.length[j]);1423 if (!length)1424 break;1425 1426 ps_page = &buffer_info->ps_pages[j];1427 dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE,1428 DMA_FROM_DEVICE);1429 ps_page->dma = 0;1430 skb_fill_page_desc(skb, j, ps_page->page, 0, length);1431 ps_page->page = NULL;1432 skb->len += length;1433 skb->data_len += length;1434 skb->truesize += PAGE_SIZE;1435 }1436 1437 /* strip the ethernet crc, problem is we're using pages now so1438 * this whole operation can get a little cpu intensive1439 */1440 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {1441 if (!(netdev->features & NETIF_F_RXFCS))1442 pskb_trim(skb, skb->len - 4);1443 }1444 1445copydone:1446 total_rx_bytes += skb->len;1447 total_rx_packets++;1448 1449 e1000_rx_checksum(adapter, staterr, skb);1450 1451 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);1452 1453 if (rx_desc->wb.upper.header_status &1454 cpu_to_le16(E1000_RXDPS_HDRSTAT_HDRSP))1455 adapter->rx_hdr_split++;1456 1457 e1000_receive_skb(adapter, netdev, skb, staterr,1458 rx_desc->wb.middle.vlan);1459 1460next_desc:1461 rx_desc->wb.middle.status_error &= cpu_to_le32(~0xFF);1462 buffer_info->skb = NULL;1463 1464 /* return some buffers to hardware, one at a time is too slow */1465 if (cleaned_count >= E1000_RX_BUFFER_WRITE) {1466 adapter->alloc_rx_buf(rx_ring, cleaned_count,1467 GFP_ATOMIC);1468 cleaned_count = 0;1469 }1470 1471 /* use prefetched values */1472 rx_desc = next_rxd;1473 buffer_info = next_buffer;1474 1475 staterr = le32_to_cpu(rx_desc->wb.middle.status_error);1476 }1477 rx_ring->next_to_clean = i;1478 1479 cleaned_count = e1000_desc_unused(rx_ring);1480 if (cleaned_count)1481 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);1482 1483 adapter->total_rx_bytes += total_rx_bytes;1484 adapter->total_rx_packets += total_rx_packets;1485 return cleaned;1486}1487 1488static void e1000_consume_page(struct e1000_buffer *bi, struct sk_buff *skb,1489 u16 length)1490{1491 bi->page = NULL;1492 skb->len += length;1493 skb->data_len += length;1494 skb->truesize += PAGE_SIZE;1495}1496 1497/**1498 * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy1499 * @rx_ring: Rx descriptor ring1500 * @work_done: output parameter for indicating completed work1501 * @work_to_do: how many packets we can clean1502 *1503 * the return value indicates whether actual cleaning was done, there1504 * is no guarantee that everything was cleaned1505 **/1506static bool e1000_clean_jumbo_rx_irq(struct e1000_ring *rx_ring, int *work_done,1507 int work_to_do)1508{1509 struct e1000_adapter *adapter = rx_ring->adapter;1510 struct net_device *netdev = adapter->netdev;1511 struct pci_dev *pdev = adapter->pdev;1512 union e1000_rx_desc_extended *rx_desc, *next_rxd;1513 struct e1000_buffer *buffer_info, *next_buffer;1514 u32 length, staterr;1515 unsigned int i;1516 int cleaned_count = 0;1517 bool cleaned = false;1518 unsigned int total_rx_bytes = 0, total_rx_packets = 0;1519 struct skb_shared_info *shinfo;1520 1521 i = rx_ring->next_to_clean;1522 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);1523 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);1524 buffer_info = &rx_ring->buffer_info[i];1525 1526 while (staterr & E1000_RXD_STAT_DD) {1527 struct sk_buff *skb;1528 1529 if (*work_done >= work_to_do)1530 break;1531 (*work_done)++;1532 dma_rmb(); /* read descriptor and rx_buffer_info after status DD */1533 1534 skb = buffer_info->skb;1535 buffer_info->skb = NULL;1536 1537 ++i;1538 if (i == rx_ring->count)1539 i = 0;1540 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i);1541 prefetch(next_rxd);1542 1543 next_buffer = &rx_ring->buffer_info[i];1544 1545 cleaned = true;1546 cleaned_count++;1547 dma_unmap_page(&pdev->dev, buffer_info->dma, PAGE_SIZE,1548 DMA_FROM_DEVICE);1549 buffer_info->dma = 0;1550 1551 length = le16_to_cpu(rx_desc->wb.upper.length);1552 1553 /* errors is only valid for DD + EOP descriptors */1554 if (unlikely((staterr & E1000_RXD_STAT_EOP) &&1555 ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&1556 !(netdev->features & NETIF_F_RXALL)))) {1557 /* recycle both page and skb */1558 buffer_info->skb = skb;1559 /* an error means any chain goes out the window too */1560 if (rx_ring->rx_skb_top)1561 dev_kfree_skb_irq(rx_ring->rx_skb_top);1562 rx_ring->rx_skb_top = NULL;1563 goto next_desc;1564 }1565#define rxtop (rx_ring->rx_skb_top)1566 if (!(staterr & E1000_RXD_STAT_EOP)) {1567 /* this descriptor is only the beginning (or middle) */1568 if (!rxtop) {1569 /* this is the beginning of a chain */1570 rxtop = skb;1571 skb_fill_page_desc(rxtop, 0, buffer_info->page,1572 0, length);1573 } else {1574 /* this is the middle of a chain */1575 shinfo = skb_shinfo(rxtop);1576 skb_fill_page_desc(rxtop, shinfo->nr_frags,1577 buffer_info->page, 0,1578 length);1579 /* re-use the skb, only consumed the page */1580 buffer_info->skb = skb;1581 }1582 e1000_consume_page(buffer_info, rxtop, length);1583 goto next_desc;1584 } else {1585 if (rxtop) {1586 /* end of the chain */1587 shinfo = skb_shinfo(rxtop);1588 skb_fill_page_desc(rxtop, shinfo->nr_frags,1589 buffer_info->page, 0,1590 length);1591 /* re-use the current skb, we only consumed the1592 * page1593 */1594 buffer_info->skb = skb;1595 skb = rxtop;1596 rxtop = NULL;1597 e1000_consume_page(buffer_info, skb, length);1598 } else {1599 /* no chain, got EOP, this buf is the packet1600 * copybreak to save the put_page/alloc_page1601 */1602 if (length <= copybreak &&1603 skb_tailroom(skb) >= length) {1604 memcpy(skb_tail_pointer(skb),1605 page_address(buffer_info->page),1606 length);1607 /* re-use the page, so don't erase1608 * buffer_info->page1609 */1610 skb_put(skb, length);1611 } else {1612 skb_fill_page_desc(skb, 0,1613 buffer_info->page, 0,1614 length);1615 e1000_consume_page(buffer_info, skb,1616 length);1617 }1618 }1619 }1620 1621 /* Receive Checksum Offload */1622 e1000_rx_checksum(adapter, staterr, skb);1623 1624 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);1625 1626 /* probably a little skewed due to removing CRC */1627 total_rx_bytes += skb->len;1628 total_rx_packets++;1629 1630 /* eth type trans needs skb->data to point to something */1631 if (!pskb_may_pull(skb, ETH_HLEN)) {1632 e_err("pskb_may_pull failed.\n");1633 dev_kfree_skb_irq(skb);1634 goto next_desc;1635 }1636 1637 e1000_receive_skb(adapter, netdev, skb, staterr,1638 rx_desc->wb.upper.vlan);1639 1640next_desc:1641 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF);1642 1643 /* return some buffers to hardware, one at a time is too slow */1644 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {1645 adapter->alloc_rx_buf(rx_ring, cleaned_count,1646 GFP_ATOMIC);1647 cleaned_count = 0;1648 }1649 1650 /* use prefetched values */1651 rx_desc = next_rxd;1652 buffer_info = next_buffer;1653 1654 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);1655 }1656 rx_ring->next_to_clean = i;1657 1658 cleaned_count = e1000_desc_unused(rx_ring);1659 if (cleaned_count)1660 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);1661 1662 adapter->total_rx_bytes += total_rx_bytes;1663 adapter->total_rx_packets += total_rx_packets;1664 return cleaned;1665}1666 1667/**1668 * e1000_clean_rx_ring - Free Rx Buffers per Queue1669 * @rx_ring: Rx descriptor ring1670 **/1671static void e1000_clean_rx_ring(struct e1000_ring *rx_ring)1672{1673 struct e1000_adapter *adapter = rx_ring->adapter;1674 struct e1000_buffer *buffer_info;1675 struct e1000_ps_page *ps_page;1676 struct pci_dev *pdev = adapter->pdev;1677 unsigned int i, j;1678 1679 /* Free all the Rx ring sk_buffs */1680 for (i = 0; i < rx_ring->count; i++) {1681 buffer_info = &rx_ring->buffer_info[i];1682 if (buffer_info->dma) {1683 if (adapter->clean_rx == e1000_clean_rx_irq)1684 dma_unmap_single(&pdev->dev, buffer_info->dma,1685 adapter->rx_buffer_len,1686 DMA_FROM_DEVICE);1687 else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq)1688 dma_unmap_page(&pdev->dev, buffer_info->dma,1689 PAGE_SIZE, DMA_FROM_DEVICE);1690 else if (adapter->clean_rx == e1000_clean_rx_irq_ps)1691 dma_unmap_single(&pdev->dev, buffer_info->dma,1692 adapter->rx_ps_bsize0,1693 DMA_FROM_DEVICE);1694 buffer_info->dma = 0;1695 }1696 1697 if (buffer_info->page) {1698 put_page(buffer_info->page);1699 buffer_info->page = NULL;1700 }1701 1702 if (buffer_info->skb) {1703 dev_kfree_skb(buffer_info->skb);1704 buffer_info->skb = NULL;1705 }1706 1707 for (j = 0; j < PS_PAGE_BUFFERS; j++) {1708 ps_page = &buffer_info->ps_pages[j];1709 if (!ps_page->page)1710 break;1711 dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE,1712 DMA_FROM_DEVICE);1713 ps_page->dma = 0;1714 put_page(ps_page->page);1715 ps_page->page = NULL;1716 }1717 }1718 1719 /* there also may be some cached data from a chained receive */1720 if (rx_ring->rx_skb_top) {1721 dev_kfree_skb(rx_ring->rx_skb_top);1722 rx_ring->rx_skb_top = NULL;1723 }1724 1725 /* Zero out the descriptor ring */1726 memset(rx_ring->desc, 0, rx_ring->size);1727 1728 rx_ring->next_to_clean = 0;1729 rx_ring->next_to_use = 0;1730 adapter->flags2 &= ~FLAG2_IS_DISCARDING;1731}1732 1733static void e1000e_downshift_workaround(struct work_struct *work)1734{1735 struct e1000_adapter *adapter = container_of(work,1736 struct e1000_adapter,1737 downshift_task);1738 1739 if (test_bit(__E1000_DOWN, &adapter->state))1740 return;1741 1742 e1000e_gig_downshift_workaround_ich8lan(&adapter->hw);1743}1744 1745/**1746 * e1000_intr_msi - Interrupt Handler1747 * @irq: interrupt number1748 * @data: pointer to a network interface device structure1749 **/1750static irqreturn_t e1000_intr_msi(int __always_unused irq, void *data)1751{1752 struct net_device *netdev = data;1753 struct e1000_adapter *adapter = netdev_priv(netdev);1754 struct e1000_hw *hw = &adapter->hw;1755 u32 icr = er32(ICR);1756 1757 /* read ICR disables interrupts using IAM */1758 if (icr & E1000_ICR_LSC) {1759 hw->mac.get_link_status = true;1760 /* ICH8 workaround-- Call gig speed drop workaround on cable1761 * disconnect (LSC) before accessing any PHY registers1762 */1763 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) &&1764 (!(er32(STATUS) & E1000_STATUS_LU)))1765 schedule_work(&adapter->downshift_task);1766 1767 /* 80003ES2LAN workaround-- For packet buffer work-around on1768 * link down event; disable receives here in the ISR and reset1769 * adapter in watchdog1770 */1771 if (netif_carrier_ok(netdev) &&1772 adapter->flags & FLAG_RX_NEEDS_RESTART) {1773 /* disable receives */1774 u32 rctl = er32(RCTL);1775 1776 ew32(RCTL, rctl & ~E1000_RCTL_EN);1777 adapter->flags |= FLAG_RESTART_NOW;1778 }1779 /* guard against interrupt when we're going down */1780 if (!test_bit(__E1000_DOWN, &adapter->state))1781 mod_timer(&adapter->watchdog_timer, jiffies + 1);1782 }1783 1784 /* Reset on uncorrectable ECC error */1785 if ((icr & E1000_ICR_ECCER) && (hw->mac.type >= e1000_pch_lpt)) {1786 u32 pbeccsts = er32(PBECCSTS);1787 1788 adapter->corr_errors +=1789 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK;1790 adapter->uncorr_errors +=1791 FIELD_GET(E1000_PBECCSTS_UNCORR_ERR_CNT_MASK, pbeccsts);1792 1793 /* Do the reset outside of interrupt context */1794 schedule_work(&adapter->reset_task);1795 1796 /* return immediately since reset is imminent */1797 return IRQ_HANDLED;1798 }1799 1800 if (napi_schedule_prep(&adapter->napi)) {1801 adapter->total_tx_bytes = 0;1802 adapter->total_tx_packets = 0;1803 adapter->total_rx_bytes = 0;1804 adapter->total_rx_packets = 0;1805 __napi_schedule(&adapter->napi);1806 }1807 1808 return IRQ_HANDLED;1809}1810 1811/**1812 * e1000_intr - Interrupt Handler1813 * @irq: interrupt number1814 * @data: pointer to a network interface device structure1815 **/1816static irqreturn_t e1000_intr(int __always_unused irq, void *data)1817{1818 struct net_device *netdev = data;1819 struct e1000_adapter *adapter = netdev_priv(netdev);1820 struct e1000_hw *hw = &adapter->hw;1821 u32 rctl, icr = er32(ICR);1822 1823 if (!icr || test_bit(__E1000_DOWN, &adapter->state))1824 return IRQ_NONE; /* Not our interrupt */1825 1826 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is1827 * not set, then the adapter didn't send an interrupt1828 */1829 if (!(icr & E1000_ICR_INT_ASSERTED))1830 return IRQ_NONE;1831 1832 /* Interrupt Auto-Mask...upon reading ICR,1833 * interrupts are masked. No need for the1834 * IMC write1835 */1836 1837 if (icr & E1000_ICR_LSC) {1838 hw->mac.get_link_status = true;1839 /* ICH8 workaround-- Call gig speed drop workaround on cable1840 * disconnect (LSC) before accessing any PHY registers1841 */1842 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) &&1843 (!(er32(STATUS) & E1000_STATUS_LU)))1844 schedule_work(&adapter->downshift_task);1845 1846 /* 80003ES2LAN workaround--1847 * For packet buffer work-around on link down event;1848 * disable receives here in the ISR and1849 * reset adapter in watchdog1850 */1851 if (netif_carrier_ok(netdev) &&1852 (adapter->flags & FLAG_RX_NEEDS_RESTART)) {1853 /* disable receives */1854 rctl = er32(RCTL);1855 ew32(RCTL, rctl & ~E1000_RCTL_EN);1856 adapter->flags |= FLAG_RESTART_NOW;1857 }1858 /* guard against interrupt when we're going down */1859 if (!test_bit(__E1000_DOWN, &adapter->state))1860 mod_timer(&adapter->watchdog_timer, jiffies + 1);1861 }1862 1863 /* Reset on uncorrectable ECC error */1864 if ((icr & E1000_ICR_ECCER) && (hw->mac.type >= e1000_pch_lpt)) {1865 u32 pbeccsts = er32(PBECCSTS);1866 1867 adapter->corr_errors +=1868 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK;1869 adapter->uncorr_errors +=1870 FIELD_GET(E1000_PBECCSTS_UNCORR_ERR_CNT_MASK, pbeccsts);1871 1872 /* Do the reset outside of interrupt context */1873 schedule_work(&adapter->reset_task);1874 1875 /* return immediately since reset is imminent */1876 return IRQ_HANDLED;1877 }1878 1879 if (napi_schedule_prep(&adapter->napi)) {1880 adapter->total_tx_bytes = 0;1881 adapter->total_tx_packets = 0;1882 adapter->total_rx_bytes = 0;1883 adapter->total_rx_packets = 0;1884 __napi_schedule(&adapter->napi);1885 }1886 1887 return IRQ_HANDLED;1888}1889 1890static irqreturn_t e1000_msix_other(int __always_unused irq, void *data)1891{1892 struct net_device *netdev = data;1893 struct e1000_adapter *adapter = netdev_priv(netdev);1894 struct e1000_hw *hw = &adapter->hw;1895 u32 icr = er32(ICR);1896 1897 if (icr & adapter->eiac_mask)1898 ew32(ICS, (icr & adapter->eiac_mask));1899 1900 if (icr & E1000_ICR_LSC) {1901 hw->mac.get_link_status = true;1902 /* guard against interrupt when we're going down */1903 if (!test_bit(__E1000_DOWN, &adapter->state))1904 mod_timer(&adapter->watchdog_timer, jiffies + 1);1905 }1906 1907 if (!test_bit(__E1000_DOWN, &adapter->state))1908 ew32(IMS, E1000_IMS_OTHER | IMS_OTHER_MASK);1909 1910 return IRQ_HANDLED;1911}1912 1913static irqreturn_t e1000_intr_msix_tx(int __always_unused irq, void *data)1914{1915 struct net_device *netdev = data;1916 struct e1000_adapter *adapter = netdev_priv(netdev);1917 struct e1000_hw *hw = &adapter->hw;1918 struct e1000_ring *tx_ring = adapter->tx_ring;1919 1920 adapter->total_tx_bytes = 0;1921 adapter->total_tx_packets = 0;1922 1923 if (!e1000_clean_tx_irq(tx_ring))1924 /* Ring was not completely cleaned, so fire another interrupt */1925 ew32(ICS, tx_ring->ims_val);1926 1927 if (!test_bit(__E1000_DOWN, &adapter->state))1928 ew32(IMS, adapter->tx_ring->ims_val);1929 1930 return IRQ_HANDLED;1931}1932 1933static irqreturn_t e1000_intr_msix_rx(int __always_unused irq, void *data)1934{1935 struct net_device *netdev = data;1936 struct e1000_adapter *adapter = netdev_priv(netdev);1937 struct e1000_ring *rx_ring = adapter->rx_ring;1938 1939 /* Write the ITR value calculated at the end of the1940 * previous interrupt.1941 */1942 if (rx_ring->set_itr) {1943 u32 itr = rx_ring->itr_val ?1944 1000000000 / (rx_ring->itr_val * 256) : 0;1945 1946 writel(itr, rx_ring->itr_register);1947 rx_ring->set_itr = 0;1948 }1949 1950 if (napi_schedule_prep(&adapter->napi)) {1951 adapter->total_rx_bytes = 0;1952 adapter->total_rx_packets = 0;1953 __napi_schedule(&adapter->napi);1954 }1955 return IRQ_HANDLED;1956}1957 1958/**1959 * e1000_configure_msix - Configure MSI-X hardware1960 * @adapter: board private structure1961 *1962 * e1000_configure_msix sets up the hardware to properly1963 * generate MSI-X interrupts.1964 **/1965static void e1000_configure_msix(struct e1000_adapter *adapter)1966{1967 struct e1000_hw *hw = &adapter->hw;1968 struct e1000_ring *rx_ring = adapter->rx_ring;1969 struct e1000_ring *tx_ring = adapter->tx_ring;1970 int vector = 0;1971 u32 ctrl_ext, ivar = 0;1972 1973 adapter->eiac_mask = 0;1974 1975 /* Workaround issue with spurious interrupts on 82574 in MSI-X mode */1976 if (hw->mac.type == e1000_82574) {1977 u32 rfctl = er32(RFCTL);1978 1979 rfctl |= E1000_RFCTL_ACK_DIS;1980 ew32(RFCTL, rfctl);1981 }1982 1983 /* Configure Rx vector */1984 rx_ring->ims_val = E1000_IMS_RXQ0;1985 adapter->eiac_mask |= rx_ring->ims_val;1986 if (rx_ring->itr_val)1987 writel(1000000000 / (rx_ring->itr_val * 256),1988 rx_ring->itr_register);1989 else1990 writel(1, rx_ring->itr_register);1991 ivar = E1000_IVAR_INT_ALLOC_VALID | vector;1992 1993 /* Configure Tx vector */1994 tx_ring->ims_val = E1000_IMS_TXQ0;1995 vector++;1996 if (tx_ring->itr_val)1997 writel(1000000000 / (tx_ring->itr_val * 256),1998 tx_ring->itr_register);1999 else2000 writel(1, tx_ring->itr_register);2001 adapter->eiac_mask |= tx_ring->ims_val;2002 ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 8);2003 2004 /* set vector for Other Causes, e.g. link changes */2005 vector++;2006 ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 16);2007 if (rx_ring->itr_val)2008 writel(1000000000 / (rx_ring->itr_val * 256),2009 hw->hw_addr + E1000_EITR_82574(vector));2010 else2011 writel(1, hw->hw_addr + E1000_EITR_82574(vector));2012 2013 /* Cause Tx interrupts on every write back */2014 ivar |= BIT(31);2015 2016 ew32(IVAR, ivar);2017 2018 /* enable MSI-X PBA support */2019 ctrl_ext = er32(CTRL_EXT) & ~E1000_CTRL_EXT_IAME;2020 ctrl_ext |= E1000_CTRL_EXT_PBA_CLR | E1000_CTRL_EXT_EIAME;2021 ew32(CTRL_EXT, ctrl_ext);2022 e1e_flush();2023}2024 2025void e1000e_reset_interrupt_capability(struct e1000_adapter *adapter)2026{2027 if (adapter->msix_entries) {2028 pci_disable_msix(adapter->pdev);2029 kfree(adapter->msix_entries);2030 adapter->msix_entries = NULL;2031 } else if (adapter->flags & FLAG_MSI_ENABLED) {2032 pci_disable_msi(adapter->pdev);2033 adapter->flags &= ~FLAG_MSI_ENABLED;2034 }2035}2036 2037/**2038 * e1000e_set_interrupt_capability - set MSI or MSI-X if supported2039 * @adapter: board private structure2040 *2041 * Attempt to configure interrupts using the best available2042 * capabilities of the hardware and kernel.2043 **/2044void e1000e_set_interrupt_capability(struct e1000_adapter *adapter)2045{2046 int err;2047 int i;2048 2049 switch (adapter->int_mode) {2050 case E1000E_INT_MODE_MSIX:2051 if (adapter->flags & FLAG_HAS_MSIX) {2052 adapter->num_vectors = 3; /* RxQ0, TxQ0 and other */2053 adapter->msix_entries = kcalloc(adapter->num_vectors,2054 sizeof(struct2055 msix_entry),2056 GFP_KERNEL);2057 if (adapter->msix_entries) {2058 struct e1000_adapter *a = adapter;2059 2060 for (i = 0; i < adapter->num_vectors; i++)2061 adapter->msix_entries[i].entry = i;2062 2063 err = pci_enable_msix_range(a->pdev,2064 a->msix_entries,2065 a->num_vectors,2066 a->num_vectors);2067 if (err > 0)2068 return;2069 }2070 /* MSI-X failed, so fall through and try MSI */2071 e_err("Failed to initialize MSI-X interrupts. Falling back to MSI interrupts.\n");2072 e1000e_reset_interrupt_capability(adapter);2073 }2074 adapter->int_mode = E1000E_INT_MODE_MSI;2075 fallthrough;2076 case E1000E_INT_MODE_MSI:2077 if (!pci_enable_msi(adapter->pdev)) {2078 adapter->flags |= FLAG_MSI_ENABLED;2079 } else {2080 adapter->int_mode = E1000E_INT_MODE_LEGACY;2081 e_err("Failed to initialize MSI interrupts. Falling back to legacy interrupts.\n");2082 }2083 fallthrough;2084 case E1000E_INT_MODE_LEGACY:2085 /* Don't do anything; this is the system default */2086 break;2087 }2088 2089 /* store the number of vectors being used */2090 adapter->num_vectors = 1;2091}2092 2093/**2094 * e1000_request_msix - Initialize MSI-X interrupts2095 * @adapter: board private structure2096 *2097 * e1000_request_msix allocates MSI-X vectors and requests interrupts from the2098 * kernel.2099 **/2100static int e1000_request_msix(struct e1000_adapter *adapter)2101{2102 struct net_device *netdev = adapter->netdev;2103 int err = 0, vector = 0;2104 2105 if (strlen(netdev->name) < (IFNAMSIZ - 5))2106 snprintf(adapter->rx_ring->name,2107 sizeof(adapter->rx_ring->name) - 1,2108 "%.14s-rx-0", netdev->name);2109 else2110 memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ);2111 err = request_irq(adapter->msix_entries[vector].vector,2112 e1000_intr_msix_rx, 0, adapter->rx_ring->name,2113 netdev);2114 if (err)2115 return err;2116 adapter->rx_ring->itr_register = adapter->hw.hw_addr +2117 E1000_EITR_82574(vector);2118 adapter->rx_ring->itr_val = adapter->itr;2119 vector++;2120 2121 if (strlen(netdev->name) < (IFNAMSIZ - 5))2122 snprintf(adapter->tx_ring->name,2123 sizeof(adapter->tx_ring->name) - 1,2124 "%.14s-tx-0", netdev->name);2125 else2126 memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ);2127 err = request_irq(adapter->msix_entries[vector].vector,2128 e1000_intr_msix_tx, 0, adapter->tx_ring->name,2129 netdev);2130 if (err)2131 return err;2132 adapter->tx_ring->itr_register = adapter->hw.hw_addr +2133 E1000_EITR_82574(vector);2134 adapter->tx_ring->itr_val = adapter->itr;2135 vector++;2136 2137 err = request_irq(adapter->msix_entries[vector].vector,2138 e1000_msix_other, 0, netdev->name, netdev);2139 if (err)2140 return err;2141 2142 e1000_configure_msix(adapter);2143 2144 return 0;2145}2146 2147/**2148 * e1000_request_irq - initialize interrupts2149 * @adapter: board private structure2150 *2151 * Attempts to configure interrupts using the best available2152 * capabilities of the hardware and kernel.2153 **/2154static int e1000_request_irq(struct e1000_adapter *adapter)2155{2156 struct net_device *netdev = adapter->netdev;2157 int err;2158 2159 if (adapter->msix_entries) {2160 err = e1000_request_msix(adapter);2161 if (!err)2162 return err;2163 /* fall back to MSI */2164 e1000e_reset_interrupt_capability(adapter);2165 adapter->int_mode = E1000E_INT_MODE_MSI;2166 e1000e_set_interrupt_capability(adapter);2167 }2168 if (adapter->flags & FLAG_MSI_ENABLED) {2169 err = request_irq(adapter->pdev->irq, e1000_intr_msi, 0,2170 netdev->name, netdev);2171 if (!err)2172 return err;2173 2174 /* fall back to legacy interrupt */2175 e1000e_reset_interrupt_capability(adapter);2176 adapter->int_mode = E1000E_INT_MODE_LEGACY;2177 }2178 2179 err = request_irq(adapter->pdev->irq, e1000_intr, IRQF_SHARED,2180 netdev->name, netdev);2181 if (err)2182 e_err("Unable to allocate interrupt, Error: %d\n", err);2183 2184 return err;2185}2186 2187static void e1000_free_irq(struct e1000_adapter *adapter)2188{2189 struct net_device *netdev = adapter->netdev;2190 2191 if (adapter->msix_entries) {2192 int vector = 0;2193 2194 free_irq(adapter->msix_entries[vector].vector, netdev);2195 vector++;2196 2197 free_irq(adapter->msix_entries[vector].vector, netdev);2198 vector++;2199 2200 /* Other Causes interrupt vector */2201 free_irq(adapter->msix_entries[vector].vector, netdev);2202 return;2203 }2204 2205 free_irq(adapter->pdev->irq, netdev);2206}2207 2208/**2209 * e1000_irq_disable - Mask off interrupt generation on the NIC2210 * @adapter: board private structure2211 **/2212static void e1000_irq_disable(struct e1000_adapter *adapter)2213{2214 struct e1000_hw *hw = &adapter->hw;2215 2216 ew32(IMC, ~0);2217 if (adapter->msix_entries)2218 ew32(EIAC_82574, 0);2219 e1e_flush();2220 2221 if (adapter->msix_entries) {2222 int i;2223 2224 for (i = 0; i < adapter->num_vectors; i++)2225 synchronize_irq(adapter->msix_entries[i].vector);2226 } else {2227 synchronize_irq(adapter->pdev->irq);2228 }2229}2230 2231/**2232 * e1000_irq_enable - Enable default interrupt generation settings2233 * @adapter: board private structure2234 **/2235static void e1000_irq_enable(struct e1000_adapter *adapter)2236{2237 struct e1000_hw *hw = &adapter->hw;2238 2239 if (adapter->msix_entries) {2240 ew32(EIAC_82574, adapter->eiac_mask & E1000_EIAC_MASK_82574);2241 ew32(IMS, adapter->eiac_mask | E1000_IMS_OTHER |2242 IMS_OTHER_MASK);2243 } else if (hw->mac.type >= e1000_pch_lpt) {2244 ew32(IMS, IMS_ENABLE_MASK | E1000_IMS_ECCER);2245 } else {2246 ew32(IMS, IMS_ENABLE_MASK);2247 }2248 e1e_flush();2249}2250 2251/**2252 * e1000e_get_hw_control - get control of the h/w from f/w2253 * @adapter: address of board private structure2254 *2255 * e1000e_get_hw_control sets {CTRL_EXT|SWSM}:DRV_LOAD bit.2256 * For ASF and Pass Through versions of f/w this means that2257 * the driver is loaded. For AMT version (only with 82573)2258 * of the f/w this means that the network i/f is open.2259 **/2260void e1000e_get_hw_control(struct e1000_adapter *adapter)2261{2262 struct e1000_hw *hw = &adapter->hw;2263 u32 ctrl_ext;2264 u32 swsm;2265 2266 /* Let firmware know the driver has taken over */2267 if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) {2268 swsm = er32(SWSM);2269 ew32(SWSM, swsm | E1000_SWSM_DRV_LOAD);2270 } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) {2271 ctrl_ext = er32(CTRL_EXT);2272 ew32(CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);2273 }2274}2275 2276/**2277 * e1000e_release_hw_control - release control of the h/w to f/w2278 * @adapter: address of board private structure2279 *2280 * e1000e_release_hw_control resets {CTRL_EXT|SWSM}:DRV_LOAD bit.2281 * For ASF and Pass Through versions of f/w this means that the2282 * driver is no longer loaded. For AMT version (only with 82573) i2283 * of the f/w this means that the network i/f is closed.2284 *2285 **/2286void e1000e_release_hw_control(struct e1000_adapter *adapter)2287{2288 struct e1000_hw *hw = &adapter->hw;2289 u32 ctrl_ext;2290 u32 swsm;2291 2292 /* Let firmware taken over control of h/w */2293 if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) {2294 swsm = er32(SWSM);2295 ew32(SWSM, swsm & ~E1000_SWSM_DRV_LOAD);2296 } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) {2297 ctrl_ext = er32(CTRL_EXT);2298 ew32(CTRL_EXT, ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);2299 }2300}2301 2302/**2303 * e1000_alloc_ring_dma - allocate memory for a ring structure2304 * @adapter: board private structure2305 * @ring: ring struct for which to allocate dma2306 **/2307static int e1000_alloc_ring_dma(struct e1000_adapter *adapter,2308 struct e1000_ring *ring)2309{2310 struct pci_dev *pdev = adapter->pdev;2311 2312 ring->desc = dma_alloc_coherent(&pdev->dev, ring->size, &ring->dma,2313 GFP_KERNEL);2314 if (!ring->desc)2315 return -ENOMEM;2316 2317 return 0;2318}2319 2320/**2321 * e1000e_setup_tx_resources - allocate Tx resources (Descriptors)2322 * @tx_ring: Tx descriptor ring2323 *2324 * Return 0 on success, negative on failure2325 **/2326int e1000e_setup_tx_resources(struct e1000_ring *tx_ring)2327{2328 struct e1000_adapter *adapter = tx_ring->adapter;2329 int err = -ENOMEM, size;2330 2331 size = sizeof(struct e1000_buffer) * tx_ring->count;2332 tx_ring->buffer_info = vzalloc(size);2333 if (!tx_ring->buffer_info)2334 goto err;2335 2336 /* round up to nearest 4K */2337 tx_ring->size = tx_ring->count * sizeof(struct e1000_tx_desc);2338 tx_ring->size = ALIGN(tx_ring->size, 4096);2339 2340 err = e1000_alloc_ring_dma(adapter, tx_ring);2341 if (err)2342 goto err;2343 2344 tx_ring->next_to_use = 0;2345 tx_ring->next_to_clean = 0;2346 2347 return 0;2348err:2349 vfree(tx_ring->buffer_info);2350 e_err("Unable to allocate memory for the transmit descriptor ring\n");2351 return err;2352}2353 2354/**2355 * e1000e_setup_rx_resources - allocate Rx resources (Descriptors)2356 * @rx_ring: Rx descriptor ring2357 *2358 * Returns 0 on success, negative on failure2359 **/2360int e1000e_setup_rx_resources(struct e1000_ring *rx_ring)2361{2362 struct e1000_adapter *adapter = rx_ring->adapter;2363 struct e1000_buffer *buffer_info;2364 int i, size, desc_len, err = -ENOMEM;2365 2366 size = sizeof(struct e1000_buffer) * rx_ring->count;2367 rx_ring->buffer_info = vzalloc(size);2368 if (!rx_ring->buffer_info)2369 goto err;2370 2371 for (i = 0; i < rx_ring->count; i++) {2372 buffer_info = &rx_ring->buffer_info[i];2373 buffer_info->ps_pages = kcalloc(PS_PAGE_BUFFERS,2374 sizeof(struct e1000_ps_page),2375 GFP_KERNEL);2376 if (!buffer_info->ps_pages)2377 goto err_pages;2378 }2379 2380 desc_len = sizeof(union e1000_rx_desc_packet_split);2381 2382 /* Round up to nearest 4K */2383 rx_ring->size = rx_ring->count * desc_len;2384 rx_ring->size = ALIGN(rx_ring->size, 4096);2385 2386 err = e1000_alloc_ring_dma(adapter, rx_ring);2387 if (err)2388 goto err_pages;2389 2390 rx_ring->next_to_clean = 0;2391 rx_ring->next_to_use = 0;2392 rx_ring->rx_skb_top = NULL;2393 2394 return 0;2395 2396err_pages:2397 for (i = 0; i < rx_ring->count; i++) {2398 buffer_info = &rx_ring->buffer_info[i];2399 kfree(buffer_info->ps_pages);2400 }2401err:2402 vfree(rx_ring->buffer_info);2403 e_err("Unable to allocate memory for the receive descriptor ring\n");2404 return err;2405}2406 2407/**2408 * e1000_clean_tx_ring - Free Tx Buffers2409 * @tx_ring: Tx descriptor ring2410 **/2411static void e1000_clean_tx_ring(struct e1000_ring *tx_ring)2412{2413 struct e1000_adapter *adapter = tx_ring->adapter;2414 struct e1000_buffer *buffer_info;2415 unsigned long size;2416 unsigned int i;2417 2418 for (i = 0; i < tx_ring->count; i++) {2419 buffer_info = &tx_ring->buffer_info[i];2420 e1000_put_txbuf(tx_ring, buffer_info, false);2421 }2422 2423 netdev_reset_queue(adapter->netdev);2424 size = sizeof(struct e1000_buffer) * tx_ring->count;2425 memset(tx_ring->buffer_info, 0, size);2426 2427 memset(tx_ring->desc, 0, tx_ring->size);2428 2429 tx_ring->next_to_use = 0;2430 tx_ring->next_to_clean = 0;2431}2432 2433/**2434 * e1000e_free_tx_resources - Free Tx Resources per Queue2435 * @tx_ring: Tx descriptor ring2436 *2437 * Free all transmit software resources2438 **/2439void e1000e_free_tx_resources(struct e1000_ring *tx_ring)2440{2441 struct e1000_adapter *adapter = tx_ring->adapter;2442 struct pci_dev *pdev = adapter->pdev;2443 2444 e1000_clean_tx_ring(tx_ring);2445 2446 vfree(tx_ring->buffer_info);2447 tx_ring->buffer_info = NULL;2448 2449 dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,2450 tx_ring->dma);2451 tx_ring->desc = NULL;2452}2453 2454/**2455 * e1000e_free_rx_resources - Free Rx Resources2456 * @rx_ring: Rx descriptor ring2457 *2458 * Free all receive software resources2459 **/2460void e1000e_free_rx_resources(struct e1000_ring *rx_ring)2461{2462 struct e1000_adapter *adapter = rx_ring->adapter;2463 struct pci_dev *pdev = adapter->pdev;2464 int i;2465 2466 e1000_clean_rx_ring(rx_ring);2467 2468 for (i = 0; i < rx_ring->count; i++)2469 kfree(rx_ring->buffer_info[i].ps_pages);2470 2471 vfree(rx_ring->buffer_info);2472 rx_ring->buffer_info = NULL;2473 2474 dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,2475 rx_ring->dma);2476 rx_ring->desc = NULL;2477}2478 2479/**2480 * e1000_update_itr - update the dynamic ITR value based on statistics2481 * @itr_setting: current adapter->itr2482 * @packets: the number of packets during this measurement interval2483 * @bytes: the number of bytes during this measurement interval2484 *2485 * Stores a new ITR value based on packets and byte2486 * counts during the last interrupt. The advantage of per interrupt2487 * computation is faster updates and more accurate ITR for the current2488 * traffic pattern. Constants in this function were computed2489 * based on theoretical maximum wire speed and thresholds were set based2490 * on testing data as well as attempting to minimize response time2491 * while increasing bulk throughput. This functionality is controlled2492 * by the InterruptThrottleRate module parameter.2493 **/2494static unsigned int e1000_update_itr(u16 itr_setting, int packets, int bytes)2495{2496 unsigned int retval = itr_setting;2497 2498 if (packets == 0)2499 return itr_setting;2500 2501 switch (itr_setting) {2502 case lowest_latency:2503 /* handle TSO and jumbo frames */2504 if (bytes / packets > 8000)2505 retval = bulk_latency;2506 else if ((packets < 5) && (bytes > 512))2507 retval = low_latency;2508 break;2509 case low_latency: /* 50 usec aka 20000 ints/s */2510 if (bytes > 10000) {2511 /* this if handles the TSO accounting */2512 if (bytes / packets > 8000)2513 retval = bulk_latency;2514 else if ((packets < 10) || ((bytes / packets) > 1200))2515 retval = bulk_latency;2516 else if ((packets > 35))2517 retval = lowest_latency;2518 } else if (bytes / packets > 2000) {2519 retval = bulk_latency;2520 } else if (packets <= 2 && bytes < 512) {2521 retval = lowest_latency;2522 }2523 break;2524 case bulk_latency: /* 250 usec aka 4000 ints/s */2525 if (bytes > 25000) {2526 if (packets > 35)2527 retval = low_latency;2528 } else if (bytes < 6000) {2529 retval = low_latency;2530 }2531 break;2532 }2533 2534 return retval;2535}2536 2537static void e1000_set_itr(struct e1000_adapter *adapter)2538{2539 u16 current_itr;2540 u32 new_itr = adapter->itr;2541 2542 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */2543 if (adapter->link_speed != SPEED_1000) {2544 new_itr = 4000;2545 goto set_itr_now;2546 }2547 2548 if (adapter->flags2 & FLAG2_DISABLE_AIM) {2549 new_itr = 0;2550 goto set_itr_now;2551 }2552 2553 adapter->tx_itr = e1000_update_itr(adapter->tx_itr,2554 adapter->total_tx_packets,2555 adapter->total_tx_bytes);2556 /* conservative mode (itr 3) eliminates the lowest_latency setting */2557 if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency)2558 adapter->tx_itr = low_latency;2559 2560 adapter->rx_itr = e1000_update_itr(adapter->rx_itr,2561 adapter->total_rx_packets,2562 adapter->total_rx_bytes);2563 /* conservative mode (itr 3) eliminates the lowest_latency setting */2564 if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)2565 adapter->rx_itr = low_latency;2566 2567 current_itr = max(adapter->rx_itr, adapter->tx_itr);2568 2569 /* counts and packets in update_itr are dependent on these numbers */2570 switch (current_itr) {2571 case lowest_latency:2572 new_itr = 70000;2573 break;2574 case low_latency:2575 new_itr = 20000; /* aka hwitr = ~200 */2576 break;2577 case bulk_latency:2578 new_itr = 4000;2579 break;2580 default:2581 break;2582 }2583 2584set_itr_now:2585 if (new_itr != adapter->itr) {2586 /* this attempts to bias the interrupt rate towards Bulk2587 * by adding intermediate steps when interrupt rate is2588 * increasing2589 */2590 new_itr = new_itr > adapter->itr ?2591 min(adapter->itr + (new_itr >> 2), new_itr) : new_itr;2592 adapter->itr = new_itr;2593 adapter->rx_ring->itr_val = new_itr;2594 if (adapter->msix_entries)2595 adapter->rx_ring->set_itr = 1;2596 else2597 e1000e_write_itr(adapter, new_itr);2598 }2599}2600 2601/**2602 * e1000e_write_itr - write the ITR value to the appropriate registers2603 * @adapter: address of board private structure2604 * @itr: new ITR value to program2605 *2606 * e1000e_write_itr determines if the adapter is in MSI-X mode2607 * and, if so, writes the EITR registers with the ITR value.2608 * Otherwise, it writes the ITR value into the ITR register.2609 **/2610void e1000e_write_itr(struct e1000_adapter *adapter, u32 itr)2611{2612 struct e1000_hw *hw = &adapter->hw;2613 u32 new_itr = itr ? 1000000000 / (itr * 256) : 0;2614 2615 if (adapter->msix_entries) {2616 int vector;2617 2618 for (vector = 0; vector < adapter->num_vectors; vector++)2619 writel(new_itr, hw->hw_addr + E1000_EITR_82574(vector));2620 } else {2621 ew32(ITR, new_itr);2622 }2623}2624 2625/**2626 * e1000_alloc_queues - Allocate memory for all rings2627 * @adapter: board private structure to initialize2628 **/2629static int e1000_alloc_queues(struct e1000_adapter *adapter)2630{2631 int size = sizeof(struct e1000_ring);2632 2633 adapter->tx_ring = kzalloc(size, GFP_KERNEL);2634 if (!adapter->tx_ring)2635 goto err;2636 adapter->tx_ring->count = adapter->tx_ring_count;2637 adapter->tx_ring->adapter = adapter;2638 2639 adapter->rx_ring = kzalloc(size, GFP_KERNEL);2640 if (!adapter->rx_ring)2641 goto err;2642 adapter->rx_ring->count = adapter->rx_ring_count;2643 adapter->rx_ring->adapter = adapter;2644 2645 return 0;2646err:2647 e_err("Unable to allocate memory for queues\n");2648 kfree(adapter->rx_ring);2649 kfree(adapter->tx_ring);2650 return -ENOMEM;2651}2652 2653/**2654 * e1000e_poll - NAPI Rx polling callback2655 * @napi: struct associated with this polling callback2656 * @budget: number of packets driver is allowed to process this poll2657 **/2658static int e1000e_poll(struct napi_struct *napi, int budget)2659{2660 struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter,2661 napi);2662 struct e1000_hw *hw = &adapter->hw;2663 struct net_device *poll_dev = adapter->netdev;2664 int tx_cleaned = 1, work_done = 0;2665 2666 adapter = netdev_priv(poll_dev);2667 2668 if (!adapter->msix_entries ||2669 (adapter->rx_ring->ims_val & adapter->tx_ring->ims_val))2670 tx_cleaned = e1000_clean_tx_irq(adapter->tx_ring);2671 2672 adapter->clean_rx(adapter->rx_ring, &work_done, budget);2673 2674 if (!tx_cleaned || work_done == budget)2675 return budget;2676 2677 /* Exit the polling mode, but don't re-enable interrupts if stack might2678 * poll us due to busy-polling2679 */2680 if (likely(napi_complete_done(napi, work_done))) {2681 if (adapter->itr_setting & 3)2682 e1000_set_itr(adapter);2683 if (!test_bit(__E1000_DOWN, &adapter->state)) {2684 if (adapter->msix_entries)2685 ew32(IMS, adapter->rx_ring->ims_val);2686 else2687 e1000_irq_enable(adapter);2688 }2689 }2690 2691 return work_done;2692}2693 2694static int e1000_vlan_rx_add_vid(struct net_device *netdev,2695 __always_unused __be16 proto, u16 vid)2696{2697 struct e1000_adapter *adapter = netdev_priv(netdev);2698 struct e1000_hw *hw = &adapter->hw;2699 u32 vfta, index;2700 2701 /* don't update vlan cookie if already programmed */2702 if ((adapter->hw.mng_cookie.status &2703 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&2704 (vid == adapter->mng_vlan_id))2705 return 0;2706 2707 /* add VID to filter table */2708 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {2709 index = (vid >> 5) & 0x7F;2710 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index);2711 vfta |= BIT((vid & 0x1F));2712 hw->mac.ops.write_vfta(hw, index, vfta);2713 }2714 2715 set_bit(vid, adapter->active_vlans);2716 2717 return 0;2718}2719 2720static int e1000_vlan_rx_kill_vid(struct net_device *netdev,2721 __always_unused __be16 proto, u16 vid)2722{2723 struct e1000_adapter *adapter = netdev_priv(netdev);2724 struct e1000_hw *hw = &adapter->hw;2725 u32 vfta, index;2726 2727 if ((adapter->hw.mng_cookie.status &2728 E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&2729 (vid == adapter->mng_vlan_id)) {2730 /* release control to f/w */2731 e1000e_release_hw_control(adapter);2732 return 0;2733 }2734 2735 /* remove VID from filter table */2736 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {2737 index = (vid >> 5) & 0x7F;2738 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index);2739 vfta &= ~BIT((vid & 0x1F));2740 hw->mac.ops.write_vfta(hw, index, vfta);2741 }2742 2743 clear_bit(vid, adapter->active_vlans);2744 2745 return 0;2746}2747 2748/**2749 * e1000e_vlan_filter_disable - helper to disable hw VLAN filtering2750 * @adapter: board private structure to initialize2751 **/2752static void e1000e_vlan_filter_disable(struct e1000_adapter *adapter)2753{2754 struct net_device *netdev = adapter->netdev;2755 struct e1000_hw *hw = &adapter->hw;2756 u32 rctl;2757 2758 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {2759 /* disable VLAN receive filtering */2760 rctl = er32(RCTL);2761 rctl &= ~(E1000_RCTL_VFE | E1000_RCTL_CFIEN);2762 ew32(RCTL, rctl);2763 2764 if (adapter->mng_vlan_id != (u16)E1000_MNG_VLAN_NONE) {2765 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q),2766 adapter->mng_vlan_id);2767 adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;2768 }2769 }2770}2771 2772/**2773 * e1000e_vlan_filter_enable - helper to enable HW VLAN filtering2774 * @adapter: board private structure to initialize2775 **/2776static void e1000e_vlan_filter_enable(struct e1000_adapter *adapter)2777{2778 struct e1000_hw *hw = &adapter->hw;2779 u32 rctl;2780 2781 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {2782 /* enable VLAN receive filtering */2783 rctl = er32(RCTL);2784 rctl |= E1000_RCTL_VFE;2785 rctl &= ~E1000_RCTL_CFIEN;2786 ew32(RCTL, rctl);2787 }2788}2789 2790/**2791 * e1000e_vlan_strip_disable - helper to disable HW VLAN stripping2792 * @adapter: board private structure to initialize2793 **/2794static void e1000e_vlan_strip_disable(struct e1000_adapter *adapter)2795{2796 struct e1000_hw *hw = &adapter->hw;2797 u32 ctrl;2798 2799 /* disable VLAN tag insert/strip */2800 ctrl = er32(CTRL);2801 ctrl &= ~E1000_CTRL_VME;2802 ew32(CTRL, ctrl);2803}2804 2805/**2806 * e1000e_vlan_strip_enable - helper to enable HW VLAN stripping2807 * @adapter: board private structure to initialize2808 **/2809static void e1000e_vlan_strip_enable(struct e1000_adapter *adapter)2810{2811 struct e1000_hw *hw = &adapter->hw;2812 u32 ctrl;2813 2814 /* enable VLAN tag insert/strip */2815 ctrl = er32(CTRL);2816 ctrl |= E1000_CTRL_VME;2817 ew32(CTRL, ctrl);2818}2819 2820static void e1000_update_mng_vlan(struct e1000_adapter *adapter)2821{2822 struct net_device *netdev = adapter->netdev;2823 u16 vid = adapter->hw.mng_cookie.vlan_id;2824 u16 old_vid = adapter->mng_vlan_id;2825 2826 if (adapter->hw.mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {2827 e1000_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);2828 adapter->mng_vlan_id = vid;2829 }2830 2831 if ((old_vid != (u16)E1000_MNG_VLAN_NONE) && (vid != old_vid))2832 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q), old_vid);2833}2834 2835static void e1000_restore_vlan(struct e1000_adapter *adapter)2836{2837 u16 vid;2838 2839 e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0);2840 2841 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)2842 e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);2843}2844 2845static void e1000_init_manageability_pt(struct e1000_adapter *adapter)2846{2847 struct e1000_hw *hw = &adapter->hw;2848 u32 manc, manc2h, mdef, i, j;2849 2850 if (!(adapter->flags & FLAG_MNG_PT_ENABLED))2851 return;2852 2853 manc = er32(MANC);2854 2855 /* enable receiving management packets to the host. this will probably2856 * generate destination unreachable messages from the host OS, but2857 * the packets will be handled on SMBUS2858 */2859 manc |= E1000_MANC_EN_MNG2HOST;2860 manc2h = er32(MANC2H);2861 2862 switch (hw->mac.type) {2863 default:2864 manc2h |= (E1000_MANC2H_PORT_623 | E1000_MANC2H_PORT_664);2865 break;2866 case e1000_82574:2867 case e1000_82583:2868 /* Check if IPMI pass-through decision filter already exists;2869 * if so, enable it.2870 */2871 for (i = 0, j = 0; i < 8; i++) {2872 mdef = er32(MDEF(i));2873 2874 /* Ignore filters with anything other than IPMI ports */2875 if (mdef & ~(E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664))2876 continue;2877 2878 /* Enable this decision filter in MANC2H */2879 if (mdef)2880 manc2h |= BIT(i);2881 2882 j |= mdef;2883 }2884 2885 if (j == (E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664))2886 break;2887 2888 /* Create new decision filter in an empty filter */2889 for (i = 0, j = 0; i < 8; i++)2890 if (er32(MDEF(i)) == 0) {2891 ew32(MDEF(i), (E1000_MDEF_PORT_623 |2892 E1000_MDEF_PORT_664));2893 manc2h |= BIT(1);2894 j++;2895 break;2896 }2897 2898 if (!j)2899 e_warn("Unable to create IPMI pass-through filter\n");2900 break;2901 }2902 2903 ew32(MANC2H, manc2h);2904 ew32(MANC, manc);2905}2906 2907/**2908 * e1000_configure_tx - Configure Transmit Unit after Reset2909 * @adapter: board private structure2910 *2911 * Configure the Tx unit of the MAC after a reset.2912 **/2913static void e1000_configure_tx(struct e1000_adapter *adapter)2914{2915 struct e1000_hw *hw = &adapter->hw;2916 struct e1000_ring *tx_ring = adapter->tx_ring;2917 u64 tdba;2918 u32 tdlen, tctl, tarc;2919 2920 /* Setup the HW Tx Head and Tail descriptor pointers */2921 tdba = tx_ring->dma;2922 tdlen = tx_ring->count * sizeof(struct e1000_tx_desc);2923 ew32(TDBAL(0), (tdba & DMA_BIT_MASK(32)));2924 ew32(TDBAH(0), (tdba >> 32));2925 ew32(TDLEN(0), tdlen);2926 ew32(TDH(0), 0);2927 ew32(TDT(0), 0);2928 tx_ring->head = adapter->hw.hw_addr + E1000_TDH(0);2929 tx_ring->tail = adapter->hw.hw_addr + E1000_TDT(0);2930 2931 writel(0, tx_ring->head);2932 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)2933 e1000e_update_tdt_wa(tx_ring, 0);2934 else2935 writel(0, tx_ring->tail);2936 2937 /* Set the Tx Interrupt Delay register */2938 ew32(TIDV, adapter->tx_int_delay);2939 /* Tx irq moderation */2940 ew32(TADV, adapter->tx_abs_int_delay);2941 2942 if (adapter->flags2 & FLAG2_DMA_BURST) {2943 u32 txdctl = er32(TXDCTL(0));2944 2945 txdctl &= ~(E1000_TXDCTL_PTHRESH | E1000_TXDCTL_HTHRESH |2946 E1000_TXDCTL_WTHRESH);2947 /* set up some performance related parameters to encourage the2948 * hardware to use the bus more efficiently in bursts, depends2949 * on the tx_int_delay to be enabled,2950 * wthresh = 1 ==> burst write is disabled to avoid Tx stalls2951 * hthresh = 1 ==> prefetch when one or more available2952 * pthresh = 0x1f ==> prefetch if internal cache 31 or less2953 * BEWARE: this seems to work but should be considered first if2954 * there are Tx hangs or other Tx related bugs2955 */2956 txdctl |= E1000_TXDCTL_DMA_BURST_ENABLE;2957 ew32(TXDCTL(0), txdctl);2958 }2959 /* erratum work around: set txdctl the same for both queues */2960 ew32(TXDCTL(1), er32(TXDCTL(0)));2961 2962 /* Program the Transmit Control Register */2963 tctl = er32(TCTL);2964 tctl &= ~E1000_TCTL_CT;2965 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |2966 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);2967 2968 if (adapter->flags & FLAG_TARC_SPEED_MODE_BIT) {2969 tarc = er32(TARC(0));2970 /* set the speed mode bit, we'll clear it if we're not at2971 * gigabit link later2972 */2973#define SPEED_MODE_BIT BIT(21)2974 tarc |= SPEED_MODE_BIT;2975 ew32(TARC(0), tarc);2976 }2977 2978 /* errata: program both queues to unweighted RR */2979 if (adapter->flags & FLAG_TARC_SET_BIT_ZERO) {2980 tarc = er32(TARC(0));2981 tarc |= 1;2982 ew32(TARC(0), tarc);2983 tarc = er32(TARC(1));2984 tarc |= 1;2985 ew32(TARC(1), tarc);2986 }2987 2988 /* Setup Transmit Descriptor Settings for eop descriptor */2989 adapter->txd_cmd = E1000_TXD_CMD_EOP | E1000_TXD_CMD_IFCS;2990 2991 /* only set IDE if we are delaying interrupts using the timers */2992 if (adapter->tx_int_delay)2993 adapter->txd_cmd |= E1000_TXD_CMD_IDE;2994 2995 /* enable Report Status bit */2996 adapter->txd_cmd |= E1000_TXD_CMD_RS;2997 2998 ew32(TCTL, tctl);2999 3000 hw->mac.ops.config_collision_dist(hw);3001 3002 /* SPT and KBL Si errata workaround to avoid data corruption */3003 if (hw->mac.type == e1000_pch_spt) {3004 u32 reg_val;3005 3006 reg_val = er32(IOSFPC);3007 reg_val |= E1000_RCTL_RDMTS_HEX;3008 ew32(IOSFPC, reg_val);3009 3010 reg_val = er32(TARC(0));3011 /* SPT and KBL Si errata workaround to avoid Tx hang.3012 * Dropping the number of outstanding requests from3013 * 3 to 2 in order to avoid a buffer overrun.3014 */3015 reg_val &= ~E1000_TARC0_CB_MULTIQ_3_REQ;3016 reg_val |= E1000_TARC0_CB_MULTIQ_2_REQ;3017 ew32(TARC(0), reg_val);3018 }3019}3020 3021#define PAGE_USE_COUNT(S) (((S) >> PAGE_SHIFT) + \3022 (((S) & (PAGE_SIZE - 1)) ? 1 : 0))3023 3024/**3025 * e1000_setup_rctl - configure the receive control registers3026 * @adapter: Board private structure3027 **/3028static void e1000_setup_rctl(struct e1000_adapter *adapter)3029{3030 struct e1000_hw *hw = &adapter->hw;3031 u32 rctl, rfctl;3032 u32 pages = 0;3033 3034 /* Workaround Si errata on PCHx - configure jumbo frame flow.3035 * If jumbo frames not set, program related MAC/PHY registers3036 * to h/w defaults3037 */3038 if (hw->mac.type >= e1000_pch2lan) {3039 s32 ret_val;3040 3041 if (adapter->netdev->mtu > ETH_DATA_LEN)3042 ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, true);3043 else3044 ret_val = e1000_lv_jumbo_workaround_ich8lan(hw, false);3045 3046 if (ret_val)3047 e_dbg("failed to enable|disable jumbo frame workaround mode\n");3048 }3049 3050 /* Program MC offset vector base */3051 rctl = er32(RCTL);3052 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);3053 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM |3054 E1000_RCTL_LBM_NO | E1000_RCTL_RDMTS_HALF |3055 (adapter->hw.mac.mc_filter_type << E1000_RCTL_MO_SHIFT);3056 3057 /* Do not Store bad packets */3058 rctl &= ~E1000_RCTL_SBP;3059 3060 /* Enable Long Packet receive */3061 if (adapter->netdev->mtu <= ETH_DATA_LEN)3062 rctl &= ~E1000_RCTL_LPE;3063 else3064 rctl |= E1000_RCTL_LPE;3065 3066 /* Some systems expect that the CRC is included in SMBUS traffic. The3067 * hardware strips the CRC before sending to both SMBUS (BMC) and to3068 * host memory when this is enabled3069 */3070 if (adapter->flags2 & FLAG2_CRC_STRIPPING)3071 rctl |= E1000_RCTL_SECRC;3072 3073 /* Workaround Si errata on 82577 PHY - configure IPG for jumbos */3074 if ((hw->phy.type == e1000_phy_82577) && (rctl & E1000_RCTL_LPE)) {3075 u16 phy_data;3076 3077 e1e_rphy(hw, PHY_REG(770, 26), &phy_data);3078 phy_data &= 0xfff8;3079 phy_data |= BIT(2);3080 e1e_wphy(hw, PHY_REG(770, 26), phy_data);3081 3082 e1e_rphy(hw, 22, &phy_data);3083 phy_data &= 0x0fff;3084 phy_data |= BIT(14);3085 e1e_wphy(hw, 0x10, 0x2823);3086 e1e_wphy(hw, 0x11, 0x0003);3087 e1e_wphy(hw, 22, phy_data);3088 }3089 3090 /* Setup buffer sizes */3091 rctl &= ~E1000_RCTL_SZ_4096;3092 rctl |= E1000_RCTL_BSEX;3093 switch (adapter->rx_buffer_len) {3094 case 2048:3095 default:3096 rctl |= E1000_RCTL_SZ_2048;3097 rctl &= ~E1000_RCTL_BSEX;3098 break;3099 case 4096:3100 rctl |= E1000_RCTL_SZ_4096;3101 break;3102 case 8192:3103 rctl |= E1000_RCTL_SZ_8192;3104 break;3105 case 16384:3106 rctl |= E1000_RCTL_SZ_16384;3107 break;3108 }3109 3110 /* Enable Extended Status in all Receive Descriptors */3111 rfctl = er32(RFCTL);3112 rfctl |= E1000_RFCTL_EXTEN;3113 ew32(RFCTL, rfctl);3114 3115 /* 82571 and greater support packet-split where the protocol3116 * header is placed in skb->data and the packet data is3117 * placed in pages hanging off of skb_shinfo(skb)->nr_frags.3118 * In the case of a non-split, skb->data is linearly filled,3119 * followed by the page buffers. Therefore, skb->data is3120 * sized to hold the largest protocol header.3121 *3122 * allocations using alloc_page take too long for regular MTU3123 * so only enable packet split for jumbo frames3124 *3125 * Using pages when the page size is greater than 16k wastes3126 * a lot of memory, since we allocate 3 pages at all times3127 * per packet.3128 */3129 pages = PAGE_USE_COUNT(adapter->netdev->mtu);3130 if ((pages <= 3) && (PAGE_SIZE <= 16384) && (rctl & E1000_RCTL_LPE))3131 adapter->rx_ps_pages = pages;3132 else3133 adapter->rx_ps_pages = 0;3134 3135 if (adapter->rx_ps_pages) {3136 u32 psrctl = 0;3137 3138 /* Enable Packet split descriptors */3139 rctl |= E1000_RCTL_DTYP_PS;3140 3141 psrctl |= adapter->rx_ps_bsize0 >> E1000_PSRCTL_BSIZE0_SHIFT;3142 3143 switch (adapter->rx_ps_pages) {3144 case 3:3145 psrctl |= PAGE_SIZE << E1000_PSRCTL_BSIZE3_SHIFT;3146 fallthrough;3147 case 2:3148 psrctl |= PAGE_SIZE << E1000_PSRCTL_BSIZE2_SHIFT;3149 fallthrough;3150 case 1:3151 psrctl |= PAGE_SIZE >> E1000_PSRCTL_BSIZE1_SHIFT;3152 break;3153 }3154 3155 ew32(PSRCTL, psrctl);3156 }3157 3158 /* This is useful for sniffing bad packets. */3159 if (adapter->netdev->features & NETIF_F_RXALL) {3160 /* UPE and MPE will be handled by normal PROMISC logic3161 * in e1000e_set_rx_mode3162 */3163 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */3164 E1000_RCTL_BAM | /* RX All Bcast Pkts */3165 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */3166 3167 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */3168 E1000_RCTL_DPF | /* Allow filtered pause */3169 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */3170 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,3171 * and that breaks VLANs.3172 */3173 }3174 3175 ew32(RCTL, rctl);3176 /* just started the receive unit, no need to restart */3177 adapter->flags &= ~FLAG_RESTART_NOW;3178}3179 3180/**3181 * e1000_configure_rx - Configure Receive Unit after Reset3182 * @adapter: board private structure3183 *3184 * Configure the Rx unit of the MAC after a reset.3185 **/3186static void e1000_configure_rx(struct e1000_adapter *adapter)3187{3188 struct e1000_hw *hw = &adapter->hw;3189 struct e1000_ring *rx_ring = adapter->rx_ring;3190 u64 rdba;3191 u32 rdlen, rctl, rxcsum, ctrl_ext;3192 3193 if (adapter->rx_ps_pages) {3194 /* this is a 32 byte descriptor */3195 rdlen = rx_ring->count *3196 sizeof(union e1000_rx_desc_packet_split);3197 adapter->clean_rx = e1000_clean_rx_irq_ps;3198 adapter->alloc_rx_buf = e1000_alloc_rx_buffers_ps;3199 } else if (adapter->netdev->mtu > ETH_FRAME_LEN + ETH_FCS_LEN) {3200 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended);3201 adapter->clean_rx = e1000_clean_jumbo_rx_irq;3202 adapter->alloc_rx_buf = e1000_alloc_jumbo_rx_buffers;3203 } else {3204 rdlen = rx_ring->count * sizeof(union e1000_rx_desc_extended);3205 adapter->clean_rx = e1000_clean_rx_irq;3206 adapter->alloc_rx_buf = e1000_alloc_rx_buffers;3207 }3208 3209 /* disable receives while setting up the descriptors */3210 rctl = er32(RCTL);3211 if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX))3212 ew32(RCTL, rctl & ~E1000_RCTL_EN);3213 e1e_flush();3214 usleep_range(10000, 11000);3215 3216 if (adapter->flags2 & FLAG2_DMA_BURST) {3217 /* set the writeback threshold (only takes effect if the RDTR3218 * is set). set GRAN=1 and write back up to 0x4 worth, and3219 * enable prefetching of 0x20 Rx descriptors3220 * granularity = 013221 * wthresh = 04,3222 * hthresh = 04,3223 * pthresh = 0x203224 */3225 ew32(RXDCTL(0), E1000_RXDCTL_DMA_BURST_ENABLE);3226 ew32(RXDCTL(1), E1000_RXDCTL_DMA_BURST_ENABLE);3227 }3228 3229 /* set the Receive Delay Timer Register */3230 ew32(RDTR, adapter->rx_int_delay);3231 3232 /* irq moderation */3233 ew32(RADV, adapter->rx_abs_int_delay);3234 if ((adapter->itr_setting != 0) && (adapter->itr != 0))3235 e1000e_write_itr(adapter, adapter->itr);3236 3237 ctrl_ext = er32(CTRL_EXT);3238 /* Auto-Mask interrupts upon ICR access */3239 ctrl_ext |= E1000_CTRL_EXT_IAME;3240 ew32(IAM, 0xffffffff);3241 ew32(CTRL_EXT, ctrl_ext);3242 e1e_flush();3243 3244 /* Setup the HW Rx Head and Tail Descriptor Pointers and3245 * the Base and Length of the Rx Descriptor Ring3246 */3247 rdba = rx_ring->dma;3248 ew32(RDBAL(0), (rdba & DMA_BIT_MASK(32)));3249 ew32(RDBAH(0), (rdba >> 32));3250 ew32(RDLEN(0), rdlen);3251 ew32(RDH(0), 0);3252 ew32(RDT(0), 0);3253 rx_ring->head = adapter->hw.hw_addr + E1000_RDH(0);3254 rx_ring->tail = adapter->hw.hw_addr + E1000_RDT(0);3255 3256 writel(0, rx_ring->head);3257 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)3258 e1000e_update_rdt_wa(rx_ring, 0);3259 else3260 writel(0, rx_ring->tail);3261 3262 /* Enable Receive Checksum Offload for TCP and UDP */3263 rxcsum = er32(RXCSUM);3264 if (adapter->netdev->features & NETIF_F_RXCSUM)3265 rxcsum |= E1000_RXCSUM_TUOFL;3266 else3267 rxcsum &= ~E1000_RXCSUM_TUOFL;3268 ew32(RXCSUM, rxcsum);3269 3270 /* With jumbo frames, excessive C-state transition latencies result3271 * in dropped transactions.3272 */3273 if (adapter->netdev->mtu > ETH_DATA_LEN) {3274 u32 lat =3275 ((er32(PBA) & E1000_PBA_RXA_MASK) * 1024 -3276 adapter->max_frame_size) * 8 / 1000;3277 3278 if (adapter->flags & FLAG_IS_ICH) {3279 u32 rxdctl = er32(RXDCTL(0));3280 3281 ew32(RXDCTL(0), rxdctl | 0x3 | BIT(8));3282 }3283 3284 dev_info(&adapter->pdev->dev,3285 "Some CPU C-states have been disabled in order to enable jumbo frames\n");3286 cpu_latency_qos_update_request(&adapter->pm_qos_req, lat);3287 } else {3288 cpu_latency_qos_update_request(&adapter->pm_qos_req,3289 PM_QOS_DEFAULT_VALUE);3290 }3291 3292 /* Enable Receives */3293 ew32(RCTL, rctl);3294}3295 3296/**3297 * e1000e_write_mc_addr_list - write multicast addresses to MTA3298 * @netdev: network interface device structure3299 *3300 * Writes multicast address list to the MTA hash table.3301 * Returns: -ENOMEM on failure3302 * 0 on no addresses written3303 * X on writing X addresses to MTA3304 */3305static int e1000e_write_mc_addr_list(struct net_device *netdev)3306{3307 struct e1000_adapter *adapter = netdev_priv(netdev);3308 struct e1000_hw *hw = &adapter->hw;3309 struct netdev_hw_addr *ha;3310 u8 *mta_list;3311 int i;3312 3313 if (netdev_mc_empty(netdev)) {3314 /* nothing to program, so clear mc list */3315 hw->mac.ops.update_mc_addr_list(hw, NULL, 0);3316 return 0;3317 }3318 3319 mta_list = kcalloc(netdev_mc_count(netdev), ETH_ALEN, GFP_ATOMIC);3320 if (!mta_list)3321 return -ENOMEM;3322 3323 /* update_mc_addr_list expects a packed array of only addresses. */3324 i = 0;3325 netdev_for_each_mc_addr(ha, netdev)3326 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);3327 3328 hw->mac.ops.update_mc_addr_list(hw, mta_list, i);3329 kfree(mta_list);3330 3331 return netdev_mc_count(netdev);3332}3333 3334/**3335 * e1000e_write_uc_addr_list - write unicast addresses to RAR table3336 * @netdev: network interface device structure3337 *3338 * Writes unicast address list to the RAR table.3339 * Returns: -ENOMEM on failure/insufficient address space3340 * 0 on no addresses written3341 * X on writing X addresses to the RAR table3342 **/3343static int e1000e_write_uc_addr_list(struct net_device *netdev)3344{3345 struct e1000_adapter *adapter = netdev_priv(netdev);3346 struct e1000_hw *hw = &adapter->hw;3347 unsigned int rar_entries;3348 int count = 0;3349 3350 rar_entries = hw->mac.ops.rar_get_count(hw);3351 3352 /* save a rar entry for our hardware address */3353 rar_entries--;3354 3355 /* save a rar entry for the LAA workaround */3356 if (adapter->flags & FLAG_RESET_OVERWRITES_LAA)3357 rar_entries--;3358 3359 /* return ENOMEM indicating insufficient memory for addresses */3360 if (netdev_uc_count(netdev) > rar_entries)3361 return -ENOMEM;3362 3363 if (!netdev_uc_empty(netdev) && rar_entries) {3364 struct netdev_hw_addr *ha;3365 3366 /* write the addresses in reverse order to avoid write3367 * combining3368 */3369 netdev_for_each_uc_addr(ha, netdev) {3370 int ret_val;3371 3372 if (!rar_entries)3373 break;3374 ret_val = hw->mac.ops.rar_set(hw, ha->addr, rar_entries--);3375 if (ret_val < 0)3376 return -ENOMEM;3377 count++;3378 }3379 }3380 3381 /* zero out the remaining RAR entries not used above */3382 for (; rar_entries > 0; rar_entries--) {3383 ew32(RAH(rar_entries), 0);3384 ew32(RAL(rar_entries), 0);3385 }3386 e1e_flush();3387 3388 return count;3389}3390 3391/**3392 * e1000e_set_rx_mode - secondary unicast, Multicast and Promiscuous mode set3393 * @netdev: network interface device structure3394 *3395 * The ndo_set_rx_mode entry point is called whenever the unicast or multicast3396 * address list or the network interface flags are updated. This routine is3397 * responsible for configuring the hardware for proper unicast, multicast,3398 * promiscuous mode, and all-multi behavior.3399 **/3400static void e1000e_set_rx_mode(struct net_device *netdev)3401{3402 struct e1000_adapter *adapter = netdev_priv(netdev);3403 struct e1000_hw *hw = &adapter->hw;3404 u32 rctl;3405 3406 if (pm_runtime_suspended(netdev->dev.parent))3407 return;3408 3409 /* Check for Promiscuous and All Multicast modes */3410 rctl = er32(RCTL);3411 3412 /* clear the affected bits */3413 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE);3414 3415 if (netdev->flags & IFF_PROMISC) {3416 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);3417 /* Do not hardware filter VLANs in promisc mode */3418 e1000e_vlan_filter_disable(adapter);3419 } else {3420 int count;3421 3422 if (netdev->flags & IFF_ALLMULTI) {3423 rctl |= E1000_RCTL_MPE;3424 } else {3425 /* Write addresses to the MTA, if the attempt fails3426 * then we should just turn on promiscuous mode so3427 * that we can at least receive multicast traffic3428 */3429 count = e1000e_write_mc_addr_list(netdev);3430 if (count < 0)3431 rctl |= E1000_RCTL_MPE;3432 }3433 e1000e_vlan_filter_enable(adapter);3434 /* Write addresses to available RAR registers, if there is not3435 * sufficient space to store all the addresses then enable3436 * unicast promiscuous mode3437 */3438 count = e1000e_write_uc_addr_list(netdev);3439 if (count < 0)3440 rctl |= E1000_RCTL_UPE;3441 }3442 3443 ew32(RCTL, rctl);3444 3445 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX)3446 e1000e_vlan_strip_enable(adapter);3447 else3448 e1000e_vlan_strip_disable(adapter);3449}3450 3451static void e1000e_setup_rss_hash(struct e1000_adapter *adapter)3452{3453 struct e1000_hw *hw = &adapter->hw;3454 u32 mrqc, rxcsum;3455 u32 rss_key[10];3456 int i;3457 3458 netdev_rss_key_fill(rss_key, sizeof(rss_key));3459 for (i = 0; i < 10; i++)3460 ew32(RSSRK(i), rss_key[i]);3461 3462 /* Direct all traffic to queue 0 */3463 for (i = 0; i < 32; i++)3464 ew32(RETA(i), 0);3465 3466 /* Disable raw packet checksumming so that RSS hash is placed in3467 * descriptor on writeback.3468 */3469 rxcsum = er32(RXCSUM);3470 rxcsum |= E1000_RXCSUM_PCSD;3471 3472 ew32(RXCSUM, rxcsum);3473 3474 mrqc = (E1000_MRQC_RSS_FIELD_IPV4 |3475 E1000_MRQC_RSS_FIELD_IPV4_TCP |3476 E1000_MRQC_RSS_FIELD_IPV6 |3477 E1000_MRQC_RSS_FIELD_IPV6_TCP |3478 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX);3479 3480 ew32(MRQC, mrqc);3481}3482 3483/**3484 * e1000e_get_base_timinca - get default SYSTIM time increment attributes3485 * @adapter: board private structure3486 * @timinca: pointer to returned time increment attributes3487 *3488 * Get attributes for incrementing the System Time Register SYSTIML/H at3489 * the default base frequency, and set the cyclecounter shift value.3490 **/3491s32 e1000e_get_base_timinca(struct e1000_adapter *adapter, u32 *timinca)3492{3493 struct e1000_hw *hw = &adapter->hw;3494 u32 incvalue, incperiod, shift;3495 3496 /* Make sure clock is enabled on I217/I218/I219 before checking3497 * the frequency3498 */3499 if ((hw->mac.type >= e1000_pch_lpt) &&3500 !(er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_ENABLED) &&3501 !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_ENABLED)) {3502 u32 fextnvm7 = er32(FEXTNVM7);3503 3504 if (!(fextnvm7 & BIT(0))) {3505 ew32(FEXTNVM7, fextnvm7 | BIT(0));3506 e1e_flush();3507 }3508 }3509 3510 switch (hw->mac.type) {3511 case e1000_pch2lan:3512 /* Stable 96MHz frequency */3513 incperiod = INCPERIOD_96MHZ;3514 incvalue = INCVALUE_96MHZ;3515 shift = INCVALUE_SHIFT_96MHZ;3516 adapter->cc.shift = shift + INCPERIOD_SHIFT_96MHZ;3517 break;3518 case e1000_pch_lpt:3519 if (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_SYSCFI) {3520 /* Stable 96MHz frequency */3521 incperiod = INCPERIOD_96MHZ;3522 incvalue = INCVALUE_96MHZ;3523 shift = INCVALUE_SHIFT_96MHZ;3524 adapter->cc.shift = shift + INCPERIOD_SHIFT_96MHZ;3525 } else {3526 /* Stable 25MHz frequency */3527 incperiod = INCPERIOD_25MHZ;3528 incvalue = INCVALUE_25MHZ;3529 shift = INCVALUE_SHIFT_25MHZ;3530 adapter->cc.shift = shift;3531 }3532 break;3533 case e1000_pch_spt:3534 /* Stable 24MHz frequency */3535 incperiod = INCPERIOD_24MHZ;3536 incvalue = INCVALUE_24MHZ;3537 shift = INCVALUE_SHIFT_24MHZ;3538 adapter->cc.shift = shift;3539 break;3540 case e1000_pch_cnp:3541 case e1000_pch_tgp:3542 case e1000_pch_adp:3543 case e1000_pch_mtp:3544 case e1000_pch_lnp:3545 case e1000_pch_ptp:3546 case e1000_pch_nvp:3547 if (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_SYSCFI) {3548 /* Stable 24MHz frequency */3549 incperiod = INCPERIOD_24MHZ;3550 incvalue = INCVALUE_24MHZ;3551 shift = INCVALUE_SHIFT_24MHZ;3552 adapter->cc.shift = shift;3553 } else {3554 /* Stable 38400KHz frequency */3555 incperiod = INCPERIOD_38400KHZ;3556 incvalue = INCVALUE_38400KHZ;3557 shift = INCVALUE_SHIFT_38400KHZ;3558 adapter->cc.shift = shift;3559 }3560 break;3561 case e1000_82574:3562 case e1000_82583:3563 /* Stable 25MHz frequency */3564 incperiod = INCPERIOD_25MHZ;3565 incvalue = INCVALUE_25MHZ;3566 shift = INCVALUE_SHIFT_25MHZ;3567 adapter->cc.shift = shift;3568 break;3569 default:3570 return -EINVAL;3571 }3572 3573 *timinca = ((incperiod << E1000_TIMINCA_INCPERIOD_SHIFT) |3574 ((incvalue << shift) & E1000_TIMINCA_INCVALUE_MASK));3575 3576 return 0;3577}3578 3579/**3580 * e1000e_config_hwtstamp - configure the hwtstamp registers and enable/disable3581 * @adapter: board private structure3582 * @config: timestamp configuration3583 *3584 * Outgoing time stamping can be enabled and disabled. Play nice and3585 * disable it when requested, although it shouldn't cause any overhead3586 * when no packet needs it. At most one packet in the queue may be3587 * marked for time stamping, otherwise it would be impossible to tell3588 * for sure to which packet the hardware time stamp belongs.3589 *3590 * Incoming time stamping has to be configured via the hardware filters.3591 * Not all combinations are supported, in particular event type has to be3592 * specified. Matching the kind of event packet is not supported, with the3593 * exception of "all V2 events regardless of level 2 or 4".3594 **/3595static int e1000e_config_hwtstamp(struct e1000_adapter *adapter,3596 struct hwtstamp_config *config)3597{3598 struct e1000_hw *hw = &adapter->hw;3599 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;3600 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;3601 u32 rxmtrl = 0;3602 u16 rxudp = 0;3603 bool is_l4 = false;3604 bool is_l2 = false;3605 u32 regval;3606 3607 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP))3608 return -EINVAL;3609 3610 switch (config->tx_type) {3611 case HWTSTAMP_TX_OFF:3612 tsync_tx_ctl = 0;3613 break;3614 case HWTSTAMP_TX_ON:3615 break;3616 default:3617 return -ERANGE;3618 }3619 3620 switch (config->rx_filter) {3621 case HWTSTAMP_FILTER_NONE:3622 tsync_rx_ctl = 0;3623 break;3624 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:3625 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;3626 rxmtrl = E1000_RXMTRL_PTP_V1_SYNC_MESSAGE;3627 is_l4 = true;3628 break;3629 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:3630 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;3631 rxmtrl = E1000_RXMTRL_PTP_V1_DELAY_REQ_MESSAGE;3632 is_l4 = true;3633 break;3634 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:3635 /* Also time stamps V2 L2 Path Delay Request/Response */3636 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_V2;3637 rxmtrl = E1000_RXMTRL_PTP_V2_SYNC_MESSAGE;3638 is_l2 = true;3639 break;3640 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:3641 /* Also time stamps V2 L2 Path Delay Request/Response. */3642 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_V2;3643 rxmtrl = E1000_RXMTRL_PTP_V2_DELAY_REQ_MESSAGE;3644 is_l2 = true;3645 break;3646 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:3647 /* Hardware cannot filter just V2 L4 Sync messages */3648 fallthrough;3649 case HWTSTAMP_FILTER_PTP_V2_SYNC:3650 /* Also time stamps V2 Path Delay Request/Response. */3651 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;3652 rxmtrl = E1000_RXMTRL_PTP_V2_SYNC_MESSAGE;3653 is_l2 = true;3654 is_l4 = true;3655 break;3656 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:3657 /* Hardware cannot filter just V2 L4 Delay Request messages */3658 fallthrough;3659 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:3660 /* Also time stamps V2 Path Delay Request/Response. */3661 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;3662 rxmtrl = E1000_RXMTRL_PTP_V2_DELAY_REQ_MESSAGE;3663 is_l2 = true;3664 is_l4 = true;3665 break;3666 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:3667 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:3668 /* Hardware cannot filter just V2 L4 or L2 Event messages */3669 fallthrough;3670 case HWTSTAMP_FILTER_PTP_V2_EVENT:3671 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;3672 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;3673 is_l2 = true;3674 is_l4 = true;3675 break;3676 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:3677 /* For V1, the hardware can only filter Sync messages or3678 * Delay Request messages but not both so fall-through to3679 * time stamp all packets.3680 */3681 fallthrough;3682 case HWTSTAMP_FILTER_NTP_ALL:3683 case HWTSTAMP_FILTER_ALL:3684 is_l2 = true;3685 is_l4 = true;3686 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;3687 config->rx_filter = HWTSTAMP_FILTER_ALL;3688 break;3689 default:3690 return -ERANGE;3691 }3692 3693 adapter->hwtstamp_config = *config;3694 3695 /* enable/disable Tx h/w time stamping */3696 regval = er32(TSYNCTXCTL);3697 regval &= ~E1000_TSYNCTXCTL_ENABLED;3698 regval |= tsync_tx_ctl;3699 ew32(TSYNCTXCTL, regval);3700 if ((er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_ENABLED) !=3701 (regval & E1000_TSYNCTXCTL_ENABLED)) {3702 e_err("Timesync Tx Control register not set as expected\n");3703 return -EAGAIN;3704 }3705 3706 /* enable/disable Rx h/w time stamping */3707 regval = er32(TSYNCRXCTL);3708 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);3709 regval |= tsync_rx_ctl;3710 ew32(TSYNCRXCTL, regval);3711 if ((er32(TSYNCRXCTL) & (E1000_TSYNCRXCTL_ENABLED |3712 E1000_TSYNCRXCTL_TYPE_MASK)) !=3713 (regval & (E1000_TSYNCRXCTL_ENABLED |3714 E1000_TSYNCRXCTL_TYPE_MASK))) {3715 e_err("Timesync Rx Control register not set as expected\n");3716 return -EAGAIN;3717 }3718 3719 /* L2: define ethertype filter for time stamped packets */3720 if (is_l2)3721 rxmtrl |= ETH_P_1588;3722 3723 /* define which PTP packets get time stamped */3724 ew32(RXMTRL, rxmtrl);3725 3726 /* Filter by destination port */3727 if (is_l4) {3728 rxudp = PTP_EV_PORT;3729 cpu_to_be16s(&rxudp);3730 }3731 ew32(RXUDP, rxudp);3732 3733 e1e_flush();3734 3735 /* Clear TSYNCRXCTL_VALID & TSYNCTXCTL_VALID bit */3736 er32(RXSTMPH);3737 er32(TXSTMPH);3738 3739 return 0;3740}3741 3742/**3743 * e1000_configure - configure the hardware for Rx and Tx3744 * @adapter: private board structure3745 **/3746static void e1000_configure(struct e1000_adapter *adapter)3747{3748 struct e1000_ring *rx_ring = adapter->rx_ring;3749 3750 e1000e_set_rx_mode(adapter->netdev);3751 3752 e1000_restore_vlan(adapter);3753 e1000_init_manageability_pt(adapter);3754 3755 e1000_configure_tx(adapter);3756 3757 if (adapter->netdev->features & NETIF_F_RXHASH)3758 e1000e_setup_rss_hash(adapter);3759 e1000_setup_rctl(adapter);3760 e1000_configure_rx(adapter);3761 adapter->alloc_rx_buf(rx_ring, e1000_desc_unused(rx_ring), GFP_KERNEL);3762}3763 3764/**3765 * e1000e_power_up_phy - restore link in case the phy was powered down3766 * @adapter: address of board private structure3767 *3768 * The phy may be powered down to save power and turn off link when the3769 * driver is unloaded and wake on lan is not enabled (among others)3770 * *** this routine MUST be followed by a call to e1000e_reset ***3771 **/3772void e1000e_power_up_phy(struct e1000_adapter *adapter)3773{3774 if (adapter->hw.phy.ops.power_up)3775 adapter->hw.phy.ops.power_up(&adapter->hw);3776 3777 adapter->hw.mac.ops.setup_link(&adapter->hw);3778}3779 3780/**3781 * e1000_power_down_phy - Power down the PHY3782 * @adapter: board private structure3783 *3784 * Power down the PHY so no link is implied when interface is down.3785 * The PHY cannot be powered down if management or WoL is active.3786 */3787static void e1000_power_down_phy(struct e1000_adapter *adapter)3788{3789 if (adapter->hw.phy.ops.power_down)3790 adapter->hw.phy.ops.power_down(&adapter->hw);3791}3792 3793/**3794 * e1000_flush_tx_ring - remove all descriptors from the tx_ring3795 * @adapter: board private structure3796 *3797 * We want to clear all pending descriptors from the TX ring.3798 * zeroing happens when the HW reads the regs. We assign the ring itself as3799 * the data of the next descriptor. We don't care about the data we are about3800 * to reset the HW.3801 */3802static void e1000_flush_tx_ring(struct e1000_adapter *adapter)3803{3804 struct e1000_hw *hw = &adapter->hw;3805 struct e1000_ring *tx_ring = adapter->tx_ring;3806 struct e1000_tx_desc *tx_desc = NULL;3807 u32 tdt, tctl, txd_lower = E1000_TXD_CMD_IFCS;3808 u16 size = 512;3809 3810 tctl = er32(TCTL);3811 ew32(TCTL, tctl | E1000_TCTL_EN);3812 tdt = er32(TDT(0));3813 BUG_ON(tdt != tx_ring->next_to_use);3814 tx_desc = E1000_TX_DESC(*tx_ring, tx_ring->next_to_use);3815 tx_desc->buffer_addr = cpu_to_le64(tx_ring->dma);3816 3817 tx_desc->lower.data = cpu_to_le32(txd_lower | size);3818 tx_desc->upper.data = 0;3819 /* flush descriptors to memory before notifying the HW */3820 wmb();3821 tx_ring->next_to_use++;3822 if (tx_ring->next_to_use == tx_ring->count)3823 tx_ring->next_to_use = 0;3824 ew32(TDT(0), tx_ring->next_to_use);3825 usleep_range(200, 250);3826}3827 3828/**3829 * e1000_flush_rx_ring - remove all descriptors from the rx_ring3830 * @adapter: board private structure3831 *3832 * Mark all descriptors in the RX ring as consumed and disable the rx ring3833 */3834static void e1000_flush_rx_ring(struct e1000_adapter *adapter)3835{3836 u32 rctl, rxdctl;3837 struct e1000_hw *hw = &adapter->hw;3838 3839 rctl = er32(RCTL);3840 ew32(RCTL, rctl & ~E1000_RCTL_EN);3841 e1e_flush();3842 usleep_range(100, 150);3843 3844 rxdctl = er32(RXDCTL(0));3845 /* zero the lower 14 bits (prefetch and host thresholds) */3846 rxdctl &= 0xffffc000;3847 3848 /* update thresholds: prefetch threshold to 31, host threshold to 13849 * and make sure the granularity is "descriptors" and not "cache lines"3850 */3851 rxdctl |= (0x1F | BIT(8) | E1000_RXDCTL_THRESH_UNIT_DESC);3852 3853 ew32(RXDCTL(0), rxdctl);3854 /* momentarily enable the RX ring for the changes to take effect */3855 ew32(RCTL, rctl | E1000_RCTL_EN);3856 e1e_flush();3857 usleep_range(100, 150);3858 ew32(RCTL, rctl & ~E1000_RCTL_EN);3859}3860 3861/**3862 * e1000_flush_desc_rings - remove all descriptors from the descriptor rings3863 * @adapter: board private structure3864 *3865 * In i219, the descriptor rings must be emptied before resetting the HW3866 * or before changing the device state to D3 during runtime (runtime PM).3867 *3868 * Failure to do this will cause the HW to enter a unit hang state which can3869 * only be released by PCI reset on the device3870 *3871 */3872 3873static void e1000_flush_desc_rings(struct e1000_adapter *adapter)3874{3875 u16 hang_state;3876 u32 fext_nvm11, tdlen;3877 struct e1000_hw *hw = &adapter->hw;3878 3879 /* First, disable MULR fix in FEXTNVM11 */3880 fext_nvm11 = er32(FEXTNVM11);3881 fext_nvm11 |= E1000_FEXTNVM11_DISABLE_MULR_FIX;3882 ew32(FEXTNVM11, fext_nvm11);3883 /* do nothing if we're not in faulty state, or if the queue is empty */3884 tdlen = er32(TDLEN(0));3885 pci_read_config_word(adapter->pdev, PCICFG_DESC_RING_STATUS,3886 &hang_state);3887 if (!(hang_state & FLUSH_DESC_REQUIRED) || !tdlen)3888 return;3889 e1000_flush_tx_ring(adapter);3890 /* recheck, maybe the fault is caused by the rx ring */3891 pci_read_config_word(adapter->pdev, PCICFG_DESC_RING_STATUS,3892 &hang_state);3893 if (hang_state & FLUSH_DESC_REQUIRED)3894 e1000_flush_rx_ring(adapter);3895}3896 3897/**3898 * e1000e_systim_reset - reset the timesync registers after a hardware reset3899 * @adapter: board private structure3900 *3901 * When the MAC is reset, all hardware bits for timesync will be reset to the3902 * default values. This function will restore the settings last in place.3903 * Since the clock SYSTIME registers are reset, we will simply restore the3904 * cyclecounter to the kernel real clock time.3905 **/3906static void e1000e_systim_reset(struct e1000_adapter *adapter)3907{3908 struct ptp_clock_info *info = &adapter->ptp_clock_info;3909 struct e1000_hw *hw = &adapter->hw;3910 unsigned long flags;3911 u32 timinca;3912 s32 ret_val;3913 3914 if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP))3915 return;3916 3917 if (info->adjfine) {3918 /* restore the previous ptp frequency delta */3919 ret_val = info->adjfine(info, adapter->ptp_delta);3920 } else {3921 /* set the default base frequency if no adjustment possible */3922 ret_val = e1000e_get_base_timinca(adapter, &timinca);3923 if (!ret_val)3924 ew32(TIMINCA, timinca);3925 }3926 3927 if (ret_val) {3928 dev_warn(&adapter->pdev->dev,3929 "Failed to restore TIMINCA clock rate delta: %d\n",3930 ret_val);3931 return;3932 }3933 3934 /* reset the systim ns time counter */3935 spin_lock_irqsave(&adapter->systim_lock, flags);3936 timecounter_init(&adapter->tc, &adapter->cc,3937 ktime_to_ns(ktime_get_real()));3938 spin_unlock_irqrestore(&adapter->systim_lock, flags);3939 3940 /* restore the previous hwtstamp configuration settings */3941 e1000e_config_hwtstamp(adapter, &adapter->hwtstamp_config);3942}3943 3944/**3945 * e1000e_reset - bring the hardware into a known good state3946 * @adapter: board private structure3947 *3948 * This function boots the hardware and enables some settings that3949 * require a configuration cycle of the hardware - those cannot be3950 * set/changed during runtime. After reset the device needs to be3951 * properly configured for Rx, Tx etc.3952 */3953void e1000e_reset(struct e1000_adapter *adapter)3954{3955 struct e1000_mac_info *mac = &adapter->hw.mac;3956 struct e1000_fc_info *fc = &adapter->hw.fc;3957 struct e1000_hw *hw = &adapter->hw;3958 u32 tx_space, min_tx_space, min_rx_space;3959 u32 pba = adapter->pba;3960 u16 hwm;3961 3962 /* reset Packet Buffer Allocation to default */3963 ew32(PBA, pba);3964 3965 if (adapter->max_frame_size > (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN)) {3966 /* To maintain wire speed transmits, the Tx FIFO should be3967 * large enough to accommodate two full transmit packets,3968 * rounded up to the next 1KB and expressed in KB. Likewise,3969 * the Rx FIFO should be large enough to accommodate at least3970 * one full receive packet and is similarly rounded up and3971 * expressed in KB.3972 */3973 pba = er32(PBA);3974 /* upper 16 bits has Tx packet buffer allocation size in KB */3975 tx_space = pba >> 16;3976 /* lower 16 bits has Rx packet buffer allocation size in KB */3977 pba &= 0xffff;3978 /* the Tx fifo also stores 16 bytes of information about the Tx3979 * but don't include ethernet FCS because hardware appends it3980 */3981 min_tx_space = (adapter->max_frame_size +3982 sizeof(struct e1000_tx_desc) - ETH_FCS_LEN) * 2;3983 min_tx_space = ALIGN(min_tx_space, 1024);3984 min_tx_space >>= 10;3985 /* software strips receive CRC, so leave room for it */3986 min_rx_space = adapter->max_frame_size;3987 min_rx_space = ALIGN(min_rx_space, 1024);3988 min_rx_space >>= 10;3989 3990 /* If current Tx allocation is less than the min Tx FIFO size,3991 * and the min Tx FIFO size is less than the current Rx FIFO3992 * allocation, take space away from current Rx allocation3993 */3994 if ((tx_space < min_tx_space) &&3995 ((min_tx_space - tx_space) < pba)) {3996 pba -= min_tx_space - tx_space;3997 3998 /* if short on Rx space, Rx wins and must trump Tx3999 * adjustment4000 */4001 if (pba < min_rx_space)4002 pba = min_rx_space;4003 }4004 4005 ew32(PBA, pba);4006 }4007 4008 /* flow control settings4009 *4010 * The high water mark must be low enough to fit one full frame4011 * (or the size used for early receive) above it in the Rx FIFO.4012 * Set it to the lower of:4013 * - 90% of the Rx FIFO size, and4014 * - the full Rx FIFO size minus one full frame4015 */4016 if (adapter->flags & FLAG_DISABLE_FC_PAUSE_TIME)4017 fc->pause_time = 0xFFFF;4018 else4019 fc->pause_time = E1000_FC_PAUSE_TIME;4020 fc->send_xon = true;4021 fc->current_mode = fc->requested_mode;4022 4023 switch (hw->mac.type) {4024 case e1000_ich9lan:4025 case e1000_ich10lan:4026 if (adapter->netdev->mtu > ETH_DATA_LEN) {4027 pba = 14;4028 ew32(PBA, pba);4029 fc->high_water = 0x2800;4030 fc->low_water = fc->high_water - 8;4031 break;4032 }4033 fallthrough;4034 default:4035 hwm = min(((pba << 10) * 9 / 10),4036 ((pba << 10) - adapter->max_frame_size));4037 4038 fc->high_water = hwm & E1000_FCRTH_RTH; /* 8-byte granularity */4039 fc->low_water = fc->high_water - 8;4040 break;4041 case e1000_pchlan:4042 /* Workaround PCH LOM adapter hangs with certain network4043 * loads. If hangs persist, try disabling Tx flow control.4044 */4045 if (adapter->netdev->mtu > ETH_DATA_LEN) {4046 fc->high_water = 0x3500;4047 fc->low_water = 0x1500;4048 } else {4049 fc->high_water = 0x5000;4050 fc->low_water = 0x3000;4051 }4052 fc->refresh_time = 0x1000;4053 break;4054 case e1000_pch2lan:4055 case e1000_pch_lpt:4056 case e1000_pch_spt:4057 case e1000_pch_cnp:4058 case e1000_pch_tgp:4059 case e1000_pch_adp:4060 case e1000_pch_mtp:4061 case e1000_pch_lnp:4062 case e1000_pch_ptp:4063 case e1000_pch_nvp:4064 fc->refresh_time = 0xFFFF;4065 fc->pause_time = 0xFFFF;4066 4067 if (adapter->netdev->mtu <= ETH_DATA_LEN) {4068 fc->high_water = 0x05C20;4069 fc->low_water = 0x05048;4070 break;4071 }4072 4073 pba = 14;4074 ew32(PBA, pba);4075 fc->high_water = ((pba << 10) * 9 / 10) & E1000_FCRTH_RTH;4076 fc->low_water = ((pba << 10) * 8 / 10) & E1000_FCRTL_RTL;4077 break;4078 }4079 4080 /* Alignment of Tx data is on an arbitrary byte boundary with the4081 * maximum size per Tx descriptor limited only to the transmit4082 * allocation of the packet buffer minus 96 bytes with an upper4083 * limit of 24KB due to receive synchronization limitations.4084 */4085 adapter->tx_fifo_limit = min_t(u32, ((er32(PBA) >> 16) << 10) - 96,4086 24 << 10);4087 4088 /* Disable Adaptive Interrupt Moderation if 2 full packets cannot4089 * fit in receive buffer.4090 */4091 if (adapter->itr_setting & 0x3) {4092 if ((adapter->max_frame_size * 2) > (pba << 10)) {4093 if (!(adapter->flags2 & FLAG2_DISABLE_AIM)) {4094 dev_info(&adapter->pdev->dev,4095 "Interrupt Throttle Rate off\n");4096 adapter->flags2 |= FLAG2_DISABLE_AIM;4097 e1000e_write_itr(adapter, 0);4098 }4099 } else if (adapter->flags2 & FLAG2_DISABLE_AIM) {4100 dev_info(&adapter->pdev->dev,4101 "Interrupt Throttle Rate on\n");4102 adapter->flags2 &= ~FLAG2_DISABLE_AIM;4103 adapter->itr = 20000;4104 e1000e_write_itr(adapter, adapter->itr);4105 }4106 }4107 4108 if (hw->mac.type >= e1000_pch_spt)4109 e1000_flush_desc_rings(adapter);4110 /* Allow time for pending master requests to run */4111 mac->ops.reset_hw(hw);4112 4113 /* For parts with AMT enabled, let the firmware know4114 * that the network interface is in control4115 */4116 if (adapter->flags & FLAG_HAS_AMT)4117 e1000e_get_hw_control(adapter);4118 4119 ew32(WUC, 0);4120 4121 if (mac->ops.init_hw(hw))4122 e_err("Hardware Error\n");4123 4124 e1000_update_mng_vlan(adapter);4125 4126 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */4127 ew32(VET, ETH_P_8021Q);4128 4129 e1000e_reset_adaptive(hw);4130 4131 /* restore systim and hwtstamp settings */4132 e1000e_systim_reset(adapter);4133 4134 /* Set EEE advertisement as appropriate */4135 if (adapter->flags2 & FLAG2_HAS_EEE) {4136 s32 ret_val;4137 u16 adv_addr;4138 4139 switch (hw->phy.type) {4140 case e1000_phy_82579:4141 adv_addr = I82579_EEE_ADVERTISEMENT;4142 break;4143 case e1000_phy_i217:4144 adv_addr = I217_EEE_ADVERTISEMENT;4145 break;4146 default:4147 dev_err(&adapter->pdev->dev,4148 "Invalid PHY type setting EEE advertisement\n");4149 return;4150 }4151 4152 ret_val = hw->phy.ops.acquire(hw);4153 if (ret_val) {4154 dev_err(&adapter->pdev->dev,4155 "EEE advertisement - unable to acquire PHY\n");4156 return;4157 }4158 4159 e1000_write_emi_reg_locked(hw, adv_addr,4160 hw->dev_spec.ich8lan.eee_disable ?4161 0 : adapter->eee_advert);4162 4163 hw->phy.ops.release(hw);4164 }4165 4166 if (!netif_running(adapter->netdev) &&4167 !test_bit(__E1000_TESTING, &adapter->state))4168 e1000_power_down_phy(adapter);4169 4170 e1000_get_phy_info(hw);4171 4172 if ((adapter->flags & FLAG_HAS_SMART_POWER_DOWN) &&4173 !(adapter->flags & FLAG_SMART_POWER_DOWN)) {4174 u16 phy_data = 0;4175 /* speed up time to link by disabling smart power down, ignore4176 * the return value of this function because there is nothing4177 * different we would do if it failed4178 */4179 e1e_rphy(hw, IGP02E1000_PHY_POWER_MGMT, &phy_data);4180 phy_data &= ~IGP02E1000_PM_SPD;4181 e1e_wphy(hw, IGP02E1000_PHY_POWER_MGMT, phy_data);4182 }4183 if (hw->mac.type >= e1000_pch_spt && adapter->int_mode == 0) {4184 u32 reg;4185 4186 /* Fextnvm7 @ 0xe4[2] = 1 */4187 reg = er32(FEXTNVM7);4188 reg |= E1000_FEXTNVM7_SIDE_CLK_UNGATE;4189 ew32(FEXTNVM7, reg);4190 /* Fextnvm9 @ 0x5bb4[13:12] = 11 */4191 reg = er32(FEXTNVM9);4192 reg |= E1000_FEXTNVM9_IOSFSB_CLKGATE_DIS |4193 E1000_FEXTNVM9_IOSFSB_CLKREQ_DIS;4194 ew32(FEXTNVM9, reg);4195 }4196 4197}4198 4199/**4200 * e1000e_trigger_lsc - trigger an LSC interrupt4201 * @adapter: board private structure4202 *4203 * Fire a link status change interrupt to start the watchdog.4204 **/4205static void e1000e_trigger_lsc(struct e1000_adapter *adapter)4206{4207 struct e1000_hw *hw = &adapter->hw;4208 4209 if (adapter->msix_entries)4210 ew32(ICS, E1000_ICS_LSC | E1000_ICS_OTHER);4211 else4212 ew32(ICS, E1000_ICS_LSC);4213}4214 4215void e1000e_up(struct e1000_adapter *adapter)4216{4217 /* hardware has been reset, we need to reload some things */4218 e1000_configure(adapter);4219 4220 clear_bit(__E1000_DOWN, &adapter->state);4221 4222 if (adapter->msix_entries)4223 e1000_configure_msix(adapter);4224 e1000_irq_enable(adapter);4225 4226 /* Tx queue started by watchdog timer when link is up */4227 4228 e1000e_trigger_lsc(adapter);4229}4230 4231static void e1000e_flush_descriptors(struct e1000_adapter *adapter)4232{4233 struct e1000_hw *hw = &adapter->hw;4234 4235 if (!(adapter->flags2 & FLAG2_DMA_BURST))4236 return;4237 4238 /* flush pending descriptor writebacks to memory */4239 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);4240 ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD);4241 4242 /* execute the writes immediately */4243 e1e_flush();4244 4245 /* due to rare timing issues, write to TIDV/RDTR again to ensure the4246 * write is successful4247 */4248 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);4249 ew32(RDTR, adapter->rx_int_delay | E1000_RDTR_FPD);4250 4251 /* execute the writes immediately */4252 e1e_flush();4253}4254 4255static void e1000e_update_stats(struct e1000_adapter *adapter);4256 4257/**4258 * e1000e_down - quiesce the device and optionally reset the hardware4259 * @adapter: board private structure4260 * @reset: boolean flag to reset the hardware or not4261 */4262void e1000e_down(struct e1000_adapter *adapter, bool reset)4263{4264 struct net_device *netdev = adapter->netdev;4265 struct e1000_hw *hw = &adapter->hw;4266 u32 tctl, rctl;4267 4268 /* signal that we're down so the interrupt handler does not4269 * reschedule our watchdog timer4270 */4271 set_bit(__E1000_DOWN, &adapter->state);4272 4273 netif_carrier_off(netdev);4274 4275 /* disable receives in the hardware */4276 rctl = er32(RCTL);4277 if (!(adapter->flags2 & FLAG2_NO_DISABLE_RX))4278 ew32(RCTL, rctl & ~E1000_RCTL_EN);4279 /* flush and sleep below */4280 4281 netif_stop_queue(netdev);4282 4283 /* disable transmits in the hardware */4284 tctl = er32(TCTL);4285 tctl &= ~E1000_TCTL_EN;4286 ew32(TCTL, tctl);4287 4288 /* flush both disables and wait for them to finish */4289 e1e_flush();4290 usleep_range(10000, 11000);4291 4292 e1000_irq_disable(adapter);4293 4294 napi_synchronize(&adapter->napi);4295 4296 del_timer_sync(&adapter->watchdog_timer);4297 del_timer_sync(&adapter->phy_info_timer);4298 4299 spin_lock(&adapter->stats64_lock);4300 e1000e_update_stats(adapter);4301 spin_unlock(&adapter->stats64_lock);4302 4303 e1000e_flush_descriptors(adapter);4304 4305 adapter->link_speed = 0;4306 adapter->link_duplex = 0;4307 4308 /* Disable Si errata workaround on PCHx for jumbo frame flow */4309 if ((hw->mac.type >= e1000_pch2lan) &&4310 (adapter->netdev->mtu > ETH_DATA_LEN) &&4311 e1000_lv_jumbo_workaround_ich8lan(hw, false))4312 e_dbg("failed to disable jumbo frame workaround mode\n");4313 4314 if (!pci_channel_offline(adapter->pdev)) {4315 if (reset)4316 e1000e_reset(adapter);4317 else if (hw->mac.type >= e1000_pch_spt)4318 e1000_flush_desc_rings(adapter);4319 }4320 e1000_clean_tx_ring(adapter->tx_ring);4321 e1000_clean_rx_ring(adapter->rx_ring);4322}4323 4324void e1000e_reinit_locked(struct e1000_adapter *adapter)4325{4326 might_sleep();4327 while (test_and_set_bit(__E1000_RESETTING, &adapter->state))4328 usleep_range(1000, 1100);4329 e1000e_down(adapter, true);4330 e1000e_up(adapter);4331 clear_bit(__E1000_RESETTING, &adapter->state);4332}4333 4334/**4335 * e1000e_sanitize_systim - sanitize raw cycle counter reads4336 * @hw: pointer to the HW structure4337 * @systim: PHC time value read, sanitized and returned4338 * @sts: structure to hold system time before and after reading SYSTIML,4339 * may be NULL4340 *4341 * Errata for 82574/82583 possible bad bits read from SYSTIMH/L:4342 * check to see that the time is incrementing at a reasonable4343 * rate and is a multiple of incvalue.4344 **/4345static u64 e1000e_sanitize_systim(struct e1000_hw *hw, u64 systim,4346 struct ptp_system_timestamp *sts)4347{4348 u64 time_delta, rem, temp;4349 u64 systim_next;4350 u32 incvalue;4351 int i;4352 4353 incvalue = er32(TIMINCA) & E1000_TIMINCA_INCVALUE_MASK;4354 for (i = 0; i < E1000_MAX_82574_SYSTIM_REREADS; i++) {4355 /* latch SYSTIMH on read of SYSTIML */4356 ptp_read_system_prets(sts);4357 systim_next = (u64)er32(SYSTIML);4358 ptp_read_system_postts(sts);4359 systim_next |= (u64)er32(SYSTIMH) << 32;4360 4361 time_delta = systim_next - systim;4362 temp = time_delta;4363 /* VMWare users have seen incvalue of zero, don't div / 0 */4364 rem = incvalue ? do_div(temp, incvalue) : (time_delta != 0);4365 4366 systim = systim_next;4367 4368 if ((time_delta < E1000_82574_SYSTIM_EPSILON) && (rem == 0))4369 break;4370 }4371 4372 return systim;4373}4374 4375/**4376 * e1000e_read_systim - read SYSTIM register4377 * @adapter: board private structure4378 * @sts: structure which will contain system time before and after reading4379 * SYSTIML, may be NULL4380 **/4381u64 e1000e_read_systim(struct e1000_adapter *adapter,4382 struct ptp_system_timestamp *sts)4383{4384 struct e1000_hw *hw = &adapter->hw;4385 u32 systimel, systimel_2, systimeh;4386 u64 systim;4387 /* SYSTIMH latching upon SYSTIML read does not work well.4388 * This means that if SYSTIML overflows after we read it but before4389 * we read SYSTIMH, the value of SYSTIMH has been incremented and we4390 * will experience a huge non linear increment in the systime value4391 * to fix that we test for overflow and if true, we re-read systime.4392 */4393 ptp_read_system_prets(sts);4394 systimel = er32(SYSTIML);4395 ptp_read_system_postts(sts);4396 systimeh = er32(SYSTIMH);4397 /* Is systimel is so large that overflow is possible? */4398 if (systimel >= (u32)0xffffffff - E1000_TIMINCA_INCVALUE_MASK) {4399 ptp_read_system_prets(sts);4400 systimel_2 = er32(SYSTIML);4401 ptp_read_system_postts(sts);4402 if (systimel > systimel_2) {4403 /* There was an overflow, read again SYSTIMH, and use4404 * systimel_24405 */4406 systimeh = er32(SYSTIMH);4407 systimel = systimel_2;4408 }4409 }4410 systim = (u64)systimel;4411 systim |= (u64)systimeh << 32;4412 4413 if (adapter->flags2 & FLAG2_CHECK_SYSTIM_OVERFLOW)4414 systim = e1000e_sanitize_systim(hw, systim, sts);4415 4416 return systim;4417}4418 4419/**4420 * e1000e_cyclecounter_read - read raw cycle counter (used by time counter)4421 * @cc: cyclecounter structure4422 **/4423static u64 e1000e_cyclecounter_read(const struct cyclecounter *cc)4424{4425 struct e1000_adapter *adapter = container_of(cc, struct e1000_adapter,4426 cc);4427 4428 return e1000e_read_systim(adapter, NULL);4429}4430 4431/**4432 * e1000_sw_init - Initialize general software structures (struct e1000_adapter)4433 * @adapter: board private structure to initialize4434 *4435 * e1000_sw_init initializes the Adapter private data structure.4436 * Fields are initialized based on PCI device information and4437 * OS network device settings (MTU size).4438 **/4439static int e1000_sw_init(struct e1000_adapter *adapter)4440{4441 struct net_device *netdev = adapter->netdev;4442 4443 adapter->rx_buffer_len = VLAN_ETH_FRAME_LEN + ETH_FCS_LEN;4444 adapter->rx_ps_bsize0 = 128;4445 adapter->max_frame_size = netdev->mtu + VLAN_ETH_HLEN + ETH_FCS_LEN;4446 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;4447 adapter->tx_ring_count = E1000_DEFAULT_TXD;4448 adapter->rx_ring_count = E1000_DEFAULT_RXD;4449 4450 spin_lock_init(&adapter->stats64_lock);4451 4452 e1000e_set_interrupt_capability(adapter);4453 4454 if (e1000_alloc_queues(adapter))4455 return -ENOMEM;4456 4457 /* Setup hardware time stamping cyclecounter */4458 if (adapter->flags & FLAG_HAS_HW_TIMESTAMP) {4459 adapter->cc.read = e1000e_cyclecounter_read;4460 adapter->cc.mask = CYCLECOUNTER_MASK(64);4461 adapter->cc.mult = 1;4462 /* cc.shift set in e1000e_get_base_tininca() */4463 4464 spin_lock_init(&adapter->systim_lock);4465 INIT_WORK(&adapter->tx_hwtstamp_work, e1000e_tx_hwtstamp_work);4466 }4467 4468 /* Explicitly disable IRQ since the NIC can be in any state. */4469 e1000_irq_disable(adapter);4470 4471 set_bit(__E1000_DOWN, &adapter->state);4472 return 0;4473}4474 4475/**4476 * e1000_intr_msi_test - Interrupt Handler4477 * @irq: interrupt number4478 * @data: pointer to a network interface device structure4479 **/4480static irqreturn_t e1000_intr_msi_test(int __always_unused irq, void *data)4481{4482 struct net_device *netdev = data;4483 struct e1000_adapter *adapter = netdev_priv(netdev);4484 struct e1000_hw *hw = &adapter->hw;4485 u32 icr = er32(ICR);4486 4487 e_dbg("icr is %08X\n", icr);4488 if (icr & E1000_ICR_RXSEQ) {4489 adapter->flags &= ~FLAG_MSI_TEST_FAILED;4490 /* Force memory writes to complete before acknowledging the4491 * interrupt is handled.4492 */4493 wmb();4494 }4495 4496 return IRQ_HANDLED;4497}4498 4499/**4500 * e1000_test_msi_interrupt - Returns 0 for successful test4501 * @adapter: board private struct4502 *4503 * code flow taken from tg3.c4504 **/4505static int e1000_test_msi_interrupt(struct e1000_adapter *adapter)4506{4507 struct net_device *netdev = adapter->netdev;4508 struct e1000_hw *hw = &adapter->hw;4509 int err;4510 4511 /* poll_enable hasn't been called yet, so don't need disable */4512 /* clear any pending events */4513 er32(ICR);4514 4515 /* free the real vector and request a test handler */4516 e1000_free_irq(adapter);4517 e1000e_reset_interrupt_capability(adapter);4518 4519 /* Assume that the test fails, if it succeeds then the test4520 * MSI irq handler will unset this flag4521 */4522 adapter->flags |= FLAG_MSI_TEST_FAILED;4523 4524 err = pci_enable_msi(adapter->pdev);4525 if (err)4526 goto msi_test_failed;4527 4528 err = request_irq(adapter->pdev->irq, e1000_intr_msi_test, 0,4529 netdev->name, netdev);4530 if (err) {4531 pci_disable_msi(adapter->pdev);4532 goto msi_test_failed;4533 }4534 4535 /* Force memory writes to complete before enabling and firing an4536 * interrupt.4537 */4538 wmb();4539 4540 e1000_irq_enable(adapter);4541 4542 /* fire an unusual interrupt on the test handler */4543 ew32(ICS, E1000_ICS_RXSEQ);4544 e1e_flush();4545 msleep(100);4546 4547 e1000_irq_disable(adapter);4548 4549 rmb(); /* read flags after interrupt has been fired */4550 4551 if (adapter->flags & FLAG_MSI_TEST_FAILED) {4552 adapter->int_mode = E1000E_INT_MODE_LEGACY;4553 e_info("MSI interrupt test failed, using legacy interrupt.\n");4554 } else {4555 e_dbg("MSI interrupt test succeeded!\n");4556 }4557 4558 free_irq(adapter->pdev->irq, netdev);4559 pci_disable_msi(adapter->pdev);4560 4561msi_test_failed:4562 e1000e_set_interrupt_capability(adapter);4563 return e1000_request_irq(adapter);4564}4565 4566/**4567 * e1000_test_msi - Returns 0 if MSI test succeeds or INTx mode is restored4568 * @adapter: board private struct4569 *4570 * code flow taken from tg3.c, called with e1000 interrupts disabled.4571 **/4572static int e1000_test_msi(struct e1000_adapter *adapter)4573{4574 int err;4575 u16 pci_cmd;4576 4577 if (!(adapter->flags & FLAG_MSI_ENABLED))4578 return 0;4579 4580 /* disable SERR in case the MSI write causes a master abort */4581 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd);4582 if (pci_cmd & PCI_COMMAND_SERR)4583 pci_write_config_word(adapter->pdev, PCI_COMMAND,4584 pci_cmd & ~PCI_COMMAND_SERR);4585 4586 err = e1000_test_msi_interrupt(adapter);4587 4588 /* re-enable SERR */4589 if (pci_cmd & PCI_COMMAND_SERR) {4590 pci_read_config_word(adapter->pdev, PCI_COMMAND, &pci_cmd);4591 pci_cmd |= PCI_COMMAND_SERR;4592 pci_write_config_word(adapter->pdev, PCI_COMMAND, pci_cmd);4593 }4594 4595 return err;4596}4597 4598/**4599 * e1000e_open - Called when a network interface is made active4600 * @netdev: network interface device structure4601 *4602 * Returns 0 on success, negative value on failure4603 *4604 * The open entry point is called when a network interface is made4605 * active by the system (IFF_UP). At this point all resources needed4606 * for transmit and receive operations are allocated, the interrupt4607 * handler is registered with the OS, the watchdog timer is started,4608 * and the stack is notified that the interface is ready.4609 **/4610int e1000e_open(struct net_device *netdev)4611{4612 struct e1000_adapter *adapter = netdev_priv(netdev);4613 struct e1000_hw *hw = &adapter->hw;4614 struct pci_dev *pdev = adapter->pdev;4615 int err;4616 4617 /* disallow open during test */4618 if (test_bit(__E1000_TESTING, &adapter->state))4619 return -EBUSY;4620 4621 pm_runtime_get_sync(&pdev->dev);4622 4623 netif_carrier_off(netdev);4624 netif_stop_queue(netdev);4625 4626 /* allocate transmit descriptors */4627 err = e1000e_setup_tx_resources(adapter->tx_ring);4628 if (err)4629 goto err_setup_tx;4630 4631 /* allocate receive descriptors */4632 err = e1000e_setup_rx_resources(adapter->rx_ring);4633 if (err)4634 goto err_setup_rx;4635 4636 /* If AMT is enabled, let the firmware know that the network4637 * interface is now open and reset the part to a known state.4638 */4639 if (adapter->flags & FLAG_HAS_AMT) {4640 e1000e_get_hw_control(adapter);4641 e1000e_reset(adapter);4642 }4643 4644 e1000e_power_up_phy(adapter);4645 4646 adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;4647 if ((adapter->hw.mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN))4648 e1000_update_mng_vlan(adapter);4649 4650 /* DMA latency requirement to workaround jumbo issue */4651 cpu_latency_qos_add_request(&adapter->pm_qos_req, PM_QOS_DEFAULT_VALUE);4652 4653 /* before we allocate an interrupt, we must be ready to handle it.4654 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt4655 * as soon as we call pci_request_irq, so we have to setup our4656 * clean_rx handler before we do so.4657 */4658 e1000_configure(adapter);4659 4660 err = e1000_request_irq(adapter);4661 if (err)4662 goto err_req_irq;4663 4664 /* Work around PCIe errata with MSI interrupts causing some chipsets to4665 * ignore e1000e MSI messages, which means we need to test our MSI4666 * interrupt now4667 */4668 if (adapter->int_mode != E1000E_INT_MODE_LEGACY) {4669 err = e1000_test_msi(adapter);4670 if (err) {4671 e_err("Interrupt allocation failed\n");4672 goto err_req_irq;4673 }4674 }4675 4676 /* From here on the code is the same as e1000e_up() */4677 clear_bit(__E1000_DOWN, &adapter->state);4678 4679 napi_enable(&adapter->napi);4680 4681 e1000_irq_enable(adapter);4682 4683 adapter->tx_hang_recheck = false;4684 4685 hw->mac.get_link_status = true;4686 pm_runtime_put(&pdev->dev);4687 4688 e1000e_trigger_lsc(adapter);4689 4690 return 0;4691 4692err_req_irq:4693 cpu_latency_qos_remove_request(&adapter->pm_qos_req);4694 e1000e_release_hw_control(adapter);4695 e1000_power_down_phy(adapter);4696 e1000e_free_rx_resources(adapter->rx_ring);4697err_setup_rx:4698 e1000e_free_tx_resources(adapter->tx_ring);4699err_setup_tx:4700 e1000e_reset(adapter);4701 pm_runtime_put_sync(&pdev->dev);4702 4703 return err;4704}4705 4706/**4707 * e1000e_close - Disables a network interface4708 * @netdev: network interface device structure4709 *4710 * Returns 0, this is not allowed to fail4711 *4712 * The close entry point is called when an interface is de-activated4713 * by the OS. The hardware is still under the drivers control, but4714 * needs to be disabled. A global MAC reset is issued to stop the4715 * hardware, and all transmit and receive resources are freed.4716 **/4717int e1000e_close(struct net_device *netdev)4718{4719 struct e1000_adapter *adapter = netdev_priv(netdev);4720 struct pci_dev *pdev = adapter->pdev;4721 int count = E1000_CHECK_RESET_COUNT;4722 4723 while (test_bit(__E1000_RESETTING, &adapter->state) && count--)4724 usleep_range(10000, 11000);4725 4726 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state));4727 4728 pm_runtime_get_sync(&pdev->dev);4729 4730 if (netif_device_present(netdev)) {4731 e1000e_down(adapter, true);4732 e1000_free_irq(adapter);4733 4734 /* Link status message must follow this format */4735 netdev_info(netdev, "NIC Link is Down\n");4736 }4737 4738 napi_disable(&adapter->napi);4739 4740 e1000e_free_tx_resources(adapter->tx_ring);4741 e1000e_free_rx_resources(adapter->rx_ring);4742 4743 /* kill manageability vlan ID if supported, but not if a vlan with4744 * the same ID is registered on the host OS (let 8021q kill it)4745 */4746 if (adapter->hw.mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN)4747 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q),4748 adapter->mng_vlan_id);4749 4750 /* If AMT is enabled, let the firmware know that the network4751 * interface is now closed4752 */4753 if ((adapter->flags & FLAG_HAS_AMT) &&4754 !test_bit(__E1000_TESTING, &adapter->state))4755 e1000e_release_hw_control(adapter);4756 4757 cpu_latency_qos_remove_request(&adapter->pm_qos_req);4758 4759 pm_runtime_put_sync(&pdev->dev);4760 4761 return 0;4762}4763 4764/**4765 * e1000_set_mac - Change the Ethernet Address of the NIC4766 * @netdev: network interface device structure4767 * @p: pointer to an address structure4768 *4769 * Returns 0 on success, negative on failure4770 **/4771static int e1000_set_mac(struct net_device *netdev, void *p)4772{4773 struct e1000_adapter *adapter = netdev_priv(netdev);4774 struct e1000_hw *hw = &adapter->hw;4775 struct sockaddr *addr = p;4776 4777 if (!is_valid_ether_addr(addr->sa_data))4778 return -EADDRNOTAVAIL;4779 4780 eth_hw_addr_set(netdev, addr->sa_data);4781 memcpy(adapter->hw.mac.addr, addr->sa_data, netdev->addr_len);4782 4783 hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr, 0);4784 4785 if (adapter->flags & FLAG_RESET_OVERWRITES_LAA) {4786 /* activate the work around */4787 e1000e_set_laa_state_82571(&adapter->hw, 1);4788 4789 /* Hold a copy of the LAA in RAR[14] This is done so that4790 * between the time RAR[0] gets clobbered and the time it4791 * gets fixed (in e1000_watchdog), the actual LAA is in one4792 * of the RARs and no incoming packets directed to this port4793 * are dropped. Eventually the LAA will be in RAR[0] and4794 * RAR[14]4795 */4796 hw->mac.ops.rar_set(&adapter->hw, adapter->hw.mac.addr,4797 adapter->hw.mac.rar_entry_count - 1);4798 }4799 4800 return 0;4801}4802 4803/**4804 * e1000e_update_phy_task - work thread to update phy4805 * @work: pointer to our work struct4806 *4807 * this worker thread exists because we must acquire a4808 * semaphore to read the phy, which we could msleep while4809 * waiting for it, and we can't msleep in a timer.4810 **/4811static void e1000e_update_phy_task(struct work_struct *work)4812{4813 struct e1000_adapter *adapter = container_of(work,4814 struct e1000_adapter,4815 update_phy_task);4816 struct e1000_hw *hw = &adapter->hw;4817 4818 if (test_bit(__E1000_DOWN, &adapter->state))4819 return;4820 4821 e1000_get_phy_info(hw);4822 4823 /* Enable EEE on 82579 after link up */4824 if (hw->phy.type >= e1000_phy_82579)4825 e1000_set_eee_pchlan(hw);4826}4827 4828/**4829 * e1000_update_phy_info - timre call-back to update PHY info4830 * @t: pointer to timer_list containing private info adapter4831 *4832 * Need to wait a few seconds after link up to get diagnostic information from4833 * the phy4834 **/4835static void e1000_update_phy_info(struct timer_list *t)4836{4837 struct e1000_adapter *adapter = from_timer(adapter, t, phy_info_timer);4838 4839 if (test_bit(__E1000_DOWN, &adapter->state))4840 return;4841 4842 schedule_work(&adapter->update_phy_task);4843}4844 4845/**4846 * e1000e_update_phy_stats - Update the PHY statistics counters4847 * @adapter: board private structure4848 *4849 * Read/clear the upper 16-bit PHY registers and read/accumulate lower4850 **/4851static void e1000e_update_phy_stats(struct e1000_adapter *adapter)4852{4853 struct e1000_hw *hw = &adapter->hw;4854 s32 ret_val;4855 u16 phy_data;4856 4857 ret_val = hw->phy.ops.acquire(hw);4858 if (ret_val)4859 return;4860 4861 /* A page set is expensive so check if already on desired page.4862 * If not, set to the page with the PHY status registers.4863 */4864 hw->phy.addr = 1;4865 ret_val = e1000e_read_phy_reg_mdic(hw, IGP01E1000_PHY_PAGE_SELECT,4866 &phy_data);4867 if (ret_val)4868 goto release;4869 if (phy_data != (HV_STATS_PAGE << IGP_PAGE_SHIFT)) {4870 ret_val = hw->phy.ops.set_page(hw,4871 HV_STATS_PAGE << IGP_PAGE_SHIFT);4872 if (ret_val)4873 goto release;4874 }4875 4876 /* Single Collision Count */4877 hw->phy.ops.read_reg_page(hw, HV_SCC_UPPER, &phy_data);4878 ret_val = hw->phy.ops.read_reg_page(hw, HV_SCC_LOWER, &phy_data);4879 if (!ret_val)4880 adapter->stats.scc += phy_data;4881 4882 /* Excessive Collision Count */4883 hw->phy.ops.read_reg_page(hw, HV_ECOL_UPPER, &phy_data);4884 ret_val = hw->phy.ops.read_reg_page(hw, HV_ECOL_LOWER, &phy_data);4885 if (!ret_val)4886 adapter->stats.ecol += phy_data;4887 4888 /* Multiple Collision Count */4889 hw->phy.ops.read_reg_page(hw, HV_MCC_UPPER, &phy_data);4890 ret_val = hw->phy.ops.read_reg_page(hw, HV_MCC_LOWER, &phy_data);4891 if (!ret_val)4892 adapter->stats.mcc += phy_data;4893 4894 /* Late Collision Count */4895 hw->phy.ops.read_reg_page(hw, HV_LATECOL_UPPER, &phy_data);4896 ret_val = hw->phy.ops.read_reg_page(hw, HV_LATECOL_LOWER, &phy_data);4897 if (!ret_val)4898 adapter->stats.latecol += phy_data;4899 4900 /* Collision Count - also used for adaptive IFS */4901 hw->phy.ops.read_reg_page(hw, HV_COLC_UPPER, &phy_data);4902 ret_val = hw->phy.ops.read_reg_page(hw, HV_COLC_LOWER, &phy_data);4903 if (!ret_val)4904 hw->mac.collision_delta = phy_data;4905 4906 /* Defer Count */4907 hw->phy.ops.read_reg_page(hw, HV_DC_UPPER, &phy_data);4908 ret_val = hw->phy.ops.read_reg_page(hw, HV_DC_LOWER, &phy_data);4909 if (!ret_val)4910 adapter->stats.dc += phy_data;4911 4912 /* Transmit with no CRS */4913 hw->phy.ops.read_reg_page(hw, HV_TNCRS_UPPER, &phy_data);4914 ret_val = hw->phy.ops.read_reg_page(hw, HV_TNCRS_LOWER, &phy_data);4915 if (!ret_val)4916 adapter->stats.tncrs += phy_data;4917 4918release:4919 hw->phy.ops.release(hw);4920}4921 4922/**4923 * e1000e_update_stats - Update the board statistics counters4924 * @adapter: board private structure4925 **/4926static void e1000e_update_stats(struct e1000_adapter *adapter)4927{4928 struct net_device *netdev = adapter->netdev;4929 struct e1000_hw *hw = &adapter->hw;4930 struct pci_dev *pdev = adapter->pdev;4931 4932 /* Prevent stats update while adapter is being reset, or if the pci4933 * connection is down.4934 */4935 if (adapter->link_speed == 0)4936 return;4937 if (pci_channel_offline(pdev))4938 return;4939 4940 adapter->stats.crcerrs += er32(CRCERRS);4941 adapter->stats.gprc += er32(GPRC);4942 adapter->stats.gorc += er32(GORCL);4943 er32(GORCH); /* Clear gorc */4944 adapter->stats.bprc += er32(BPRC);4945 adapter->stats.mprc += er32(MPRC);4946 adapter->stats.roc += er32(ROC);4947 4948 adapter->stats.mpc += er32(MPC);4949 4950 /* Half-duplex statistics */4951 if (adapter->link_duplex == HALF_DUPLEX) {4952 if (adapter->flags2 & FLAG2_HAS_PHY_STATS) {4953 e1000e_update_phy_stats(adapter);4954 } else {4955 adapter->stats.scc += er32(SCC);4956 adapter->stats.ecol += er32(ECOL);4957 adapter->stats.mcc += er32(MCC);4958 adapter->stats.latecol += er32(LATECOL);4959 adapter->stats.dc += er32(DC);4960 4961 hw->mac.collision_delta = er32(COLC);4962 4963 if ((hw->mac.type != e1000_82574) &&4964 (hw->mac.type != e1000_82583))4965 adapter->stats.tncrs += er32(TNCRS);4966 }4967 adapter->stats.colc += hw->mac.collision_delta;4968 }4969 4970 adapter->stats.xonrxc += er32(XONRXC);4971 adapter->stats.xontxc += er32(XONTXC);4972 adapter->stats.xoffrxc += er32(XOFFRXC);4973 adapter->stats.xofftxc += er32(XOFFTXC);4974 adapter->stats.gptc += er32(GPTC);4975 adapter->stats.gotc += er32(GOTCL);4976 er32(GOTCH); /* Clear gotc */4977 adapter->stats.rnbc += er32(RNBC);4978 adapter->stats.ruc += er32(RUC);4979 4980 adapter->stats.mptc += er32(MPTC);4981 adapter->stats.bptc += er32(BPTC);4982 4983 /* used for adaptive IFS */4984 4985 hw->mac.tx_packet_delta = er32(TPT);4986 adapter->stats.tpt += hw->mac.tx_packet_delta;4987 4988 adapter->stats.algnerrc += er32(ALGNERRC);4989 adapter->stats.rxerrc += er32(RXERRC);4990 adapter->stats.cexterr += er32(CEXTERR);4991 adapter->stats.tsctc += er32(TSCTC);4992 adapter->stats.tsctfc += er32(TSCTFC);4993 4994 /* Fill out the OS statistics structure */4995 netdev->stats.multicast = adapter->stats.mprc;4996 netdev->stats.collisions = adapter->stats.colc;4997 4998 /* Rx Errors */4999 5000 /* RLEC on some newer hardware can be incorrect so build5001 * our own version based on RUC and ROC5002 */5003 netdev->stats.rx_errors = adapter->stats.rxerrc +5004 adapter->stats.crcerrs + adapter->stats.algnerrc +5005 adapter->stats.ruc + adapter->stats.roc + adapter->stats.cexterr;5006 netdev->stats.rx_length_errors = adapter->stats.ruc +5007 adapter->stats.roc;5008 netdev->stats.rx_crc_errors = adapter->stats.crcerrs;5009 netdev->stats.rx_frame_errors = adapter->stats.algnerrc;5010 netdev->stats.rx_missed_errors = adapter->stats.mpc;5011 5012 /* Tx Errors */5013 netdev->stats.tx_errors = adapter->stats.ecol + adapter->stats.latecol;5014 netdev->stats.tx_aborted_errors = adapter->stats.ecol;5015 netdev->stats.tx_window_errors = adapter->stats.latecol;5016 netdev->stats.tx_carrier_errors = adapter->stats.tncrs;5017 5018 /* Tx Dropped needs to be maintained elsewhere */5019 5020 /* Management Stats */5021 adapter->stats.mgptc += er32(MGTPTC);5022 adapter->stats.mgprc += er32(MGTPRC);5023 adapter->stats.mgpdc += er32(MGTPDC);5024 5025 /* Correctable ECC Errors */5026 if (hw->mac.type >= e1000_pch_lpt) {5027 u32 pbeccsts = er32(PBECCSTS);5028 5029 adapter->corr_errors +=5030 pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK;5031 adapter->uncorr_errors +=5032 FIELD_GET(E1000_PBECCSTS_UNCORR_ERR_CNT_MASK, pbeccsts);5033 }5034}5035 5036/**5037 * e1000_phy_read_status - Update the PHY register status snapshot5038 * @adapter: board private structure5039 **/5040static void e1000_phy_read_status(struct e1000_adapter *adapter)5041{5042 struct e1000_hw *hw = &adapter->hw;5043 struct e1000_phy_regs *phy = &adapter->phy_regs;5044 5045 if (!pm_runtime_suspended((&adapter->pdev->dev)->parent) &&5046 (er32(STATUS) & E1000_STATUS_LU) &&5047 (adapter->hw.phy.media_type == e1000_media_type_copper)) {5048 int ret_val;5049 5050 ret_val = e1e_rphy(hw, MII_BMCR, &phy->bmcr);5051 ret_val |= e1e_rphy(hw, MII_BMSR, &phy->bmsr);5052 ret_val |= e1e_rphy(hw, MII_ADVERTISE, &phy->advertise);5053 ret_val |= e1e_rphy(hw, MII_LPA, &phy->lpa);5054 ret_val |= e1e_rphy(hw, MII_EXPANSION, &phy->expansion);5055 ret_val |= e1e_rphy(hw, MII_CTRL1000, &phy->ctrl1000);5056 ret_val |= e1e_rphy(hw, MII_STAT1000, &phy->stat1000);5057 ret_val |= e1e_rphy(hw, MII_ESTATUS, &phy->estatus);5058 if (ret_val)5059 e_warn("Error reading PHY register\n");5060 } else {5061 /* Do not read PHY registers if link is not up5062 * Set values to typical power-on defaults5063 */5064 phy->bmcr = (BMCR_SPEED1000 | BMCR_ANENABLE | BMCR_FULLDPLX);5065 phy->bmsr = (BMSR_100FULL | BMSR_100HALF | BMSR_10FULL |5066 BMSR_10HALF | BMSR_ESTATEN | BMSR_ANEGCAPABLE |5067 BMSR_ERCAP);5068 phy->advertise = (ADVERTISE_PAUSE_ASYM | ADVERTISE_PAUSE_CAP |5069 ADVERTISE_ALL | ADVERTISE_CSMA);5070 phy->lpa = 0;5071 phy->expansion = EXPANSION_ENABLENPAGE;5072 phy->ctrl1000 = ADVERTISE_1000FULL;5073 phy->stat1000 = 0;5074 phy->estatus = (ESTATUS_1000_TFULL | ESTATUS_1000_THALF);5075 }5076}5077 5078static void e1000_print_link_info(struct e1000_adapter *adapter)5079{5080 struct e1000_hw *hw = &adapter->hw;5081 u32 ctrl = er32(CTRL);5082 5083 /* Link status message must follow this format for user tools */5084 netdev_info(adapter->netdev,5085 "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",5086 adapter->link_speed,5087 adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half",5088 (ctrl & E1000_CTRL_TFCE) && (ctrl & E1000_CTRL_RFCE) ? "Rx/Tx" :5089 (ctrl & E1000_CTRL_RFCE) ? "Rx" :5090 (ctrl & E1000_CTRL_TFCE) ? "Tx" : "None");5091}5092 5093static bool e1000e_has_link(struct e1000_adapter *adapter)5094{5095 struct e1000_hw *hw = &adapter->hw;5096 bool link_active = false;5097 s32 ret_val = 0;5098 5099 /* get_link_status is set on LSC (link status) interrupt or5100 * Rx sequence error interrupt. get_link_status will stay5101 * true until the check_for_link establishes link5102 * for copper adapters ONLY5103 */5104 switch (hw->phy.media_type) {5105 case e1000_media_type_copper:5106 if (hw->mac.get_link_status) {5107 ret_val = hw->mac.ops.check_for_link(hw);5108 link_active = !hw->mac.get_link_status;5109 } else {5110 link_active = true;5111 }5112 break;5113 case e1000_media_type_fiber:5114 ret_val = hw->mac.ops.check_for_link(hw);5115 link_active = !!(er32(STATUS) & E1000_STATUS_LU);5116 break;5117 case e1000_media_type_internal_serdes:5118 ret_val = hw->mac.ops.check_for_link(hw);5119 link_active = hw->mac.serdes_has_link;5120 break;5121 default:5122 case e1000_media_type_unknown:5123 break;5124 }5125 5126 if ((ret_val == -E1000_ERR_PHY) && (hw->phy.type == e1000_phy_igp_3) &&5127 (er32(CTRL) & E1000_PHY_CTRL_GBE_DISABLE)) {5128 /* See e1000_kmrn_lock_loss_workaround_ich8lan() */5129 e_info("Gigabit has been disabled, downgrading speed\n");5130 }5131 5132 return link_active;5133}5134 5135static void e1000e_enable_receives(struct e1000_adapter *adapter)5136{5137 /* make sure the receive unit is started */5138 if ((adapter->flags & FLAG_RX_NEEDS_RESTART) &&5139 (adapter->flags & FLAG_RESTART_NOW)) {5140 struct e1000_hw *hw = &adapter->hw;5141 u32 rctl = er32(RCTL);5142 5143 ew32(RCTL, rctl | E1000_RCTL_EN);5144 adapter->flags &= ~FLAG_RESTART_NOW;5145 }5146}5147 5148static void e1000e_check_82574_phy_workaround(struct e1000_adapter *adapter)5149{5150 struct e1000_hw *hw = &adapter->hw;5151 5152 /* With 82574 controllers, PHY needs to be checked periodically5153 * for hung state and reset, if two calls return true5154 */5155 if (e1000_check_phy_82574(hw))5156 adapter->phy_hang_count++;5157 else5158 adapter->phy_hang_count = 0;5159 5160 if (adapter->phy_hang_count > 1) {5161 adapter->phy_hang_count = 0;5162 e_dbg("PHY appears hung - resetting\n");5163 schedule_work(&adapter->reset_task);5164 }5165}5166 5167/**5168 * e1000_watchdog - Timer Call-back5169 * @t: pointer to timer_list containing private info adapter5170 **/5171static void e1000_watchdog(struct timer_list *t)5172{5173 struct e1000_adapter *adapter = from_timer(adapter, t, watchdog_timer);5174 5175 /* Do the rest outside of interrupt context */5176 schedule_work(&adapter->watchdog_task);5177 5178 /* TODO: make this use queue_delayed_work() */5179}5180 5181static void e1000_watchdog_task(struct work_struct *work)5182{5183 struct e1000_adapter *adapter = container_of(work,5184 struct e1000_adapter,5185 watchdog_task);5186 struct net_device *netdev = adapter->netdev;5187 struct e1000_mac_info *mac = &adapter->hw.mac;5188 struct e1000_phy_info *phy = &adapter->hw.phy;5189 struct e1000_ring *tx_ring = adapter->tx_ring;5190 u32 dmoff_exit_timeout = 100, tries = 0;5191 struct e1000_hw *hw = &adapter->hw;5192 u32 link, tctl, pcim_state;5193 5194 if (test_bit(__E1000_DOWN, &adapter->state))5195 return;5196 5197 link = e1000e_has_link(adapter);5198 if ((netif_carrier_ok(netdev)) && link) {5199 /* Cancel scheduled suspend requests. */5200 pm_runtime_resume(netdev->dev.parent);5201 5202 e1000e_enable_receives(adapter);5203 goto link_up;5204 }5205 5206 if ((e1000e_enable_tx_pkt_filtering(hw)) &&5207 (adapter->mng_vlan_id != adapter->hw.mng_cookie.vlan_id))5208 e1000_update_mng_vlan(adapter);5209 5210 if (link) {5211 if (!netif_carrier_ok(netdev)) {5212 bool txb2b = true;5213 5214 /* Cancel scheduled suspend requests. */5215 pm_runtime_resume(netdev->dev.parent);5216 5217 /* Checking if MAC is in DMoff state*/5218 if (er32(FWSM) & E1000_ICH_FWSM_FW_VALID) {5219 pcim_state = er32(STATUS);5220 while (pcim_state & E1000_STATUS_PCIM_STATE) {5221 if (tries++ == dmoff_exit_timeout) {5222 e_dbg("Error in exiting dmoff\n");5223 break;5224 }5225 usleep_range(10000, 20000);5226 pcim_state = er32(STATUS);5227 5228 /* Checking if MAC exited DMoff state */5229 if (!(pcim_state & E1000_STATUS_PCIM_STATE))5230 e1000_phy_hw_reset(&adapter->hw);5231 }5232 }5233 5234 /* update snapshot of PHY registers on LSC */5235 e1000_phy_read_status(adapter);5236 mac->ops.get_link_up_info(&adapter->hw,5237 &adapter->link_speed,5238 &adapter->link_duplex);5239 e1000_print_link_info(adapter);5240 5241 /* check if SmartSpeed worked */5242 e1000e_check_downshift(hw);5243 if (phy->speed_downgraded)5244 netdev_warn(netdev,5245 "Link Speed was downgraded by SmartSpeed\n");5246 5247 /* On supported PHYs, check for duplex mismatch only5248 * if link has autonegotiated at 10/100 half5249 */5250 if ((hw->phy.type == e1000_phy_igp_3 ||5251 hw->phy.type == e1000_phy_bm) &&5252 hw->mac.autoneg &&5253 (adapter->link_speed == SPEED_10 ||5254 adapter->link_speed == SPEED_100) &&5255 (adapter->link_duplex == HALF_DUPLEX)) {5256 u16 autoneg_exp;5257 5258 e1e_rphy(hw, MII_EXPANSION, &autoneg_exp);5259 5260 if (!(autoneg_exp & EXPANSION_NWAY))5261 e_info("Autonegotiated half duplex but link partner cannot autoneg. Try forcing full duplex if link gets many collisions.\n");5262 }5263 5264 /* adjust timeout factor according to speed/duplex */5265 adapter->tx_timeout_factor = 1;5266 switch (adapter->link_speed) {5267 case SPEED_10:5268 txb2b = false;5269 adapter->tx_timeout_factor = 16;5270 break;5271 case SPEED_100:5272 txb2b = false;5273 adapter->tx_timeout_factor = 10;5274 break;5275 }5276 5277 /* workaround: re-program speed mode bit after5278 * link-up event5279 */5280 if ((adapter->flags & FLAG_TARC_SPEED_MODE_BIT) &&5281 !txb2b) {5282 u32 tarc0;5283 5284 tarc0 = er32(TARC(0));5285 tarc0 &= ~SPEED_MODE_BIT;5286 ew32(TARC(0), tarc0);5287 }5288 5289 /* enable transmits in the hardware, need to do this5290 * after setting TARC(0)5291 */5292 tctl = er32(TCTL);5293 tctl |= E1000_TCTL_EN;5294 ew32(TCTL, tctl);5295 5296 /* Perform any post-link-up configuration before5297 * reporting link up.5298 */5299 if (phy->ops.cfg_on_link_up)5300 phy->ops.cfg_on_link_up(hw);5301 5302 netif_wake_queue(netdev);5303 netif_carrier_on(netdev);5304 5305 if (!test_bit(__E1000_DOWN, &adapter->state))5306 mod_timer(&adapter->phy_info_timer,5307 round_jiffies(jiffies + 2 * HZ));5308 }5309 } else {5310 if (netif_carrier_ok(netdev)) {5311 adapter->link_speed = 0;5312 adapter->link_duplex = 0;5313 /* Link status message must follow this format */5314 netdev_info(netdev, "NIC Link is Down\n");5315 netif_carrier_off(netdev);5316 netif_stop_queue(netdev);5317 if (!test_bit(__E1000_DOWN, &adapter->state))5318 mod_timer(&adapter->phy_info_timer,5319 round_jiffies(jiffies + 2 * HZ));5320 5321 /* 8000ES2LAN requires a Rx packet buffer work-around5322 * on link down event; reset the controller to flush5323 * the Rx packet buffer.5324 */5325 if (adapter->flags & FLAG_RX_NEEDS_RESTART)5326 adapter->flags |= FLAG_RESTART_NOW;5327 else5328 pm_schedule_suspend(netdev->dev.parent,5329 LINK_TIMEOUT);5330 }5331 }5332 5333link_up:5334 spin_lock(&adapter->stats64_lock);5335 e1000e_update_stats(adapter);5336 5337 mac->tx_packet_delta = adapter->stats.tpt - adapter->tpt_old;5338 adapter->tpt_old = adapter->stats.tpt;5339 mac->collision_delta = adapter->stats.colc - adapter->colc_old;5340 adapter->colc_old = adapter->stats.colc;5341 5342 adapter->gorc = adapter->stats.gorc - adapter->gorc_old;5343 adapter->gorc_old = adapter->stats.gorc;5344 adapter->gotc = adapter->stats.gotc - adapter->gotc_old;5345 adapter->gotc_old = adapter->stats.gotc;5346 spin_unlock(&adapter->stats64_lock);5347 5348 /* If the link is lost the controller stops DMA, but5349 * if there is queued Tx work it cannot be done. So5350 * reset the controller to flush the Tx packet buffers.5351 */5352 if (!netif_carrier_ok(netdev) &&5353 (e1000_desc_unused(tx_ring) + 1 < tx_ring->count))5354 adapter->flags |= FLAG_RESTART_NOW;5355 5356 /* If reset is necessary, do it outside of interrupt context. */5357 if (adapter->flags & FLAG_RESTART_NOW) {5358 schedule_work(&adapter->reset_task);5359 /* return immediately since reset is imminent */5360 return;5361 }5362 5363 e1000e_update_adaptive(&adapter->hw);5364 5365 /* Simple mode for Interrupt Throttle Rate (ITR) */5366 if (adapter->itr_setting == 4) {5367 /* Symmetric Tx/Rx gets a reduced ITR=2000;5368 * Total asymmetrical Tx or Rx gets ITR=8000;5369 * everyone else is between 2000-8000.5370 */5371 u32 goc = (adapter->gotc + adapter->gorc) / 10000;5372 u32 dif = (adapter->gotc > adapter->gorc ?5373 adapter->gotc - adapter->gorc :5374 adapter->gorc - adapter->gotc) / 10000;5375 u32 itr = goc > 0 ? (dif * 6000 / goc + 2000) : 8000;5376 5377 e1000e_write_itr(adapter, itr);5378 }5379 5380 /* Cause software interrupt to ensure Rx ring is cleaned */5381 if (adapter->msix_entries)5382 ew32(ICS, adapter->rx_ring->ims_val);5383 else5384 ew32(ICS, E1000_ICS_RXDMT0);5385 5386 /* flush pending descriptors to memory before detecting Tx hang */5387 e1000e_flush_descriptors(adapter);5388 5389 /* Force detection of hung controller every watchdog period */5390 adapter->detect_tx_hung = true;5391 5392 /* With 82571 controllers, LAA may be overwritten due to controller5393 * reset from the other port. Set the appropriate LAA in RAR[0]5394 */5395 if (e1000e_get_laa_state_82571(hw))5396 hw->mac.ops.rar_set(hw, adapter->hw.mac.addr, 0);5397 5398 if (adapter->flags2 & FLAG2_CHECK_PHY_HANG)5399 e1000e_check_82574_phy_workaround(adapter);5400 5401 /* Clear valid timestamp stuck in RXSTMPL/H due to a Rx error */5402 if (adapter->hwtstamp_config.rx_filter != HWTSTAMP_FILTER_NONE) {5403 if ((adapter->flags2 & FLAG2_CHECK_RX_HWTSTAMP) &&5404 (er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID)) {5405 er32(RXSTMPH);5406 adapter->rx_hwtstamp_cleared++;5407 } else {5408 adapter->flags2 |= FLAG2_CHECK_RX_HWTSTAMP;5409 }5410 }5411 5412 /* Reset the timer */5413 if (!test_bit(__E1000_DOWN, &adapter->state))5414 mod_timer(&adapter->watchdog_timer,5415 round_jiffies(jiffies + 2 * HZ));5416}5417 5418#define E1000_TX_FLAGS_CSUM 0x000000015419#define E1000_TX_FLAGS_VLAN 0x000000025420#define E1000_TX_FLAGS_TSO 0x000000045421#define E1000_TX_FLAGS_IPV4 0x000000085422#define E1000_TX_FLAGS_NO_FCS 0x000000105423#define E1000_TX_FLAGS_HWTSTAMP 0x000000205424#define E1000_TX_FLAGS_VLAN_MASK 0xffff00005425#define E1000_TX_FLAGS_VLAN_SHIFT 165426 5427static int e1000_tso(struct e1000_ring *tx_ring, struct sk_buff *skb,5428 __be16 protocol)5429{5430 struct e1000_context_desc *context_desc;5431 struct e1000_buffer *buffer_info;5432 unsigned int i;5433 u32 cmd_length = 0;5434 u16 ipcse = 0, mss;5435 u8 ipcss, ipcso, tucss, tucso, hdr_len;5436 int err;5437 5438 if (!skb_is_gso(skb))5439 return 0;5440 5441 err = skb_cow_head(skb, 0);5442 if (err < 0)5443 return err;5444 5445 hdr_len = skb_tcp_all_headers(skb);5446 mss = skb_shinfo(skb)->gso_size;5447 if (protocol == htons(ETH_P_IP)) {5448 struct iphdr *iph = ip_hdr(skb);5449 iph->tot_len = 0;5450 iph->check = 0;5451 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr,5452 0, IPPROTO_TCP, 0);5453 cmd_length = E1000_TXD_CMD_IP;5454 ipcse = skb_transport_offset(skb) - 1;5455 } else if (skb_is_gso_v6(skb)) {5456 tcp_v6_gso_csum_prep(skb);5457 ipcse = 0;5458 }5459 ipcss = skb_network_offset(skb);5460 ipcso = (void *)&(ip_hdr(skb)->check) - (void *)skb->data;5461 tucss = skb_transport_offset(skb);5462 tucso = (void *)&(tcp_hdr(skb)->check) - (void *)skb->data;5463 5464 cmd_length |= (E1000_TXD_CMD_DEXT | E1000_TXD_CMD_TSE |5465 E1000_TXD_CMD_TCP | (skb->len - (hdr_len)));5466 5467 i = tx_ring->next_to_use;5468 context_desc = E1000_CONTEXT_DESC(*tx_ring, i);5469 buffer_info = &tx_ring->buffer_info[i];5470 5471 context_desc->lower_setup.ip_fields.ipcss = ipcss;5472 context_desc->lower_setup.ip_fields.ipcso = ipcso;5473 context_desc->lower_setup.ip_fields.ipcse = cpu_to_le16(ipcse);5474 context_desc->upper_setup.tcp_fields.tucss = tucss;5475 context_desc->upper_setup.tcp_fields.tucso = tucso;5476 context_desc->upper_setup.tcp_fields.tucse = 0;5477 context_desc->tcp_seg_setup.fields.mss = cpu_to_le16(mss);5478 context_desc->tcp_seg_setup.fields.hdr_len = hdr_len;5479 context_desc->cmd_and_length = cpu_to_le32(cmd_length);5480 5481 buffer_info->time_stamp = jiffies;5482 buffer_info->next_to_watch = i;5483 5484 i++;5485 if (i == tx_ring->count)5486 i = 0;5487 tx_ring->next_to_use = i;5488 5489 return 1;5490}5491 5492static bool e1000_tx_csum(struct e1000_ring *tx_ring, struct sk_buff *skb,5493 __be16 protocol)5494{5495 struct e1000_adapter *adapter = tx_ring->adapter;5496 struct e1000_context_desc *context_desc;5497 struct e1000_buffer *buffer_info;5498 unsigned int i;5499 u8 css;5500 u32 cmd_len = E1000_TXD_CMD_DEXT;5501 5502 if (skb->ip_summed != CHECKSUM_PARTIAL)5503 return false;5504 5505 switch (protocol) {5506 case cpu_to_be16(ETH_P_IP):5507 if (ip_hdr(skb)->protocol == IPPROTO_TCP)5508 cmd_len |= E1000_TXD_CMD_TCP;5509 break;5510 case cpu_to_be16(ETH_P_IPV6):5511 /* XXX not handling all IPV6 headers */5512 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)5513 cmd_len |= E1000_TXD_CMD_TCP;5514 break;5515 default:5516 if (unlikely(net_ratelimit()))5517 e_warn("checksum_partial proto=%x!\n",5518 be16_to_cpu(protocol));5519 break;5520 }5521 5522 css = skb_checksum_start_offset(skb);5523 5524 i = tx_ring->next_to_use;5525 buffer_info = &tx_ring->buffer_info[i];5526 context_desc = E1000_CONTEXT_DESC(*tx_ring, i);5527 5528 context_desc->lower_setup.ip_config = 0;5529 context_desc->upper_setup.tcp_fields.tucss = css;5530 context_desc->upper_setup.tcp_fields.tucso = css + skb->csum_offset;5531 context_desc->upper_setup.tcp_fields.tucse = 0;5532 context_desc->tcp_seg_setup.data = 0;5533 context_desc->cmd_and_length = cpu_to_le32(cmd_len);5534 5535 buffer_info->time_stamp = jiffies;5536 buffer_info->next_to_watch = i;5537 5538 i++;5539 if (i == tx_ring->count)5540 i = 0;5541 tx_ring->next_to_use = i;5542 5543 return true;5544}5545 5546static int e1000_tx_map(struct e1000_ring *tx_ring, struct sk_buff *skb,5547 unsigned int first, unsigned int max_per_txd,5548 unsigned int nr_frags)5549{5550 struct e1000_adapter *adapter = tx_ring->adapter;5551 struct pci_dev *pdev = adapter->pdev;5552 struct e1000_buffer *buffer_info;5553 unsigned int len = skb_headlen(skb);5554 unsigned int offset = 0, size, count = 0, i;5555 unsigned int f, bytecount, segs;5556 5557 i = tx_ring->next_to_use;5558 5559 while (len) {5560 buffer_info = &tx_ring->buffer_info[i];5561 size = min(len, max_per_txd);5562 5563 buffer_info->length = size;5564 buffer_info->time_stamp = jiffies;5565 buffer_info->next_to_watch = i;5566 buffer_info->dma = dma_map_single(&pdev->dev,5567 skb->data + offset,5568 size, DMA_TO_DEVICE);5569 buffer_info->mapped_as_page = false;5570 if (dma_mapping_error(&pdev->dev, buffer_info->dma))5571 goto dma_error;5572 5573 len -= size;5574 offset += size;5575 count++;5576 5577 if (len) {5578 i++;5579 if (i == tx_ring->count)5580 i = 0;5581 }5582 }5583 5584 for (f = 0; f < nr_frags; f++) {5585 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];5586 5587 len = skb_frag_size(frag);5588 offset = 0;5589 5590 while (len) {5591 i++;5592 if (i == tx_ring->count)5593 i = 0;5594 5595 buffer_info = &tx_ring->buffer_info[i];5596 size = min(len, max_per_txd);5597 5598 buffer_info->length = size;5599 buffer_info->time_stamp = jiffies;5600 buffer_info->next_to_watch = i;5601 buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag,5602 offset, size,5603 DMA_TO_DEVICE);5604 buffer_info->mapped_as_page = true;5605 if (dma_mapping_error(&pdev->dev, buffer_info->dma))5606 goto dma_error;5607 5608 len -= size;5609 offset += size;5610 count++;5611 }5612 }5613 5614 segs = skb_shinfo(skb)->gso_segs ? : 1;5615 /* multiply data chunks by size of headers */5616 bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len;5617 5618 tx_ring->buffer_info[i].skb = skb;5619 tx_ring->buffer_info[i].segs = segs;5620 tx_ring->buffer_info[i].bytecount = bytecount;5621 tx_ring->buffer_info[first].next_to_watch = i;5622 5623 return count;5624 5625dma_error:5626 dev_err(&pdev->dev, "Tx DMA map failed\n");5627 buffer_info->dma = 0;5628 if (count)5629 count--;5630 5631 while (count--) {5632 if (i == 0)5633 i += tx_ring->count;5634 i--;5635 buffer_info = &tx_ring->buffer_info[i];5636 e1000_put_txbuf(tx_ring, buffer_info, true);5637 }5638 5639 return 0;5640}5641 5642static void e1000_tx_queue(struct e1000_ring *tx_ring, int tx_flags, int count)5643{5644 struct e1000_adapter *adapter = tx_ring->adapter;5645 struct e1000_tx_desc *tx_desc = NULL;5646 struct e1000_buffer *buffer_info;5647 u32 txd_upper = 0, txd_lower = E1000_TXD_CMD_IFCS;5648 unsigned int i;5649 5650 if (tx_flags & E1000_TX_FLAGS_TSO) {5651 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D |5652 E1000_TXD_CMD_TSE;5653 txd_upper |= E1000_TXD_POPTS_TXSM << 8;5654 5655 if (tx_flags & E1000_TX_FLAGS_IPV4)5656 txd_upper |= E1000_TXD_POPTS_IXSM << 8;5657 }5658 5659 if (tx_flags & E1000_TX_FLAGS_CSUM) {5660 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;5661 txd_upper |= E1000_TXD_POPTS_TXSM << 8;5662 }5663 5664 if (tx_flags & E1000_TX_FLAGS_VLAN) {5665 txd_lower |= E1000_TXD_CMD_VLE;5666 txd_upper |= (tx_flags & E1000_TX_FLAGS_VLAN_MASK);5667 }5668 5669 if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))5670 txd_lower &= ~(E1000_TXD_CMD_IFCS);5671 5672 if (unlikely(tx_flags & E1000_TX_FLAGS_HWTSTAMP)) {5673 txd_lower |= E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;5674 txd_upper |= E1000_TXD_EXTCMD_TSTAMP;5675 }5676 5677 i = tx_ring->next_to_use;5678 5679 do {5680 buffer_info = &tx_ring->buffer_info[i];5681 tx_desc = E1000_TX_DESC(*tx_ring, i);5682 tx_desc->buffer_addr = cpu_to_le64(buffer_info->dma);5683 tx_desc->lower.data = cpu_to_le32(txd_lower |5684 buffer_info->length);5685 tx_desc->upper.data = cpu_to_le32(txd_upper);5686 5687 i++;5688 if (i == tx_ring->count)5689 i = 0;5690 } while (--count > 0);5691 5692 tx_desc->lower.data |= cpu_to_le32(adapter->txd_cmd);5693 5694 /* txd_cmd re-enables FCS, so we'll re-disable it here as desired. */5695 if (unlikely(tx_flags & E1000_TX_FLAGS_NO_FCS))5696 tx_desc->lower.data &= ~(cpu_to_le32(E1000_TXD_CMD_IFCS));5697 5698 /* Force memory writes to complete before letting h/w5699 * know there are new descriptors to fetch. (Only5700 * applicable for weak-ordered memory model archs,5701 * such as IA-64).5702 */5703 wmb();5704 5705 tx_ring->next_to_use = i;5706}5707 5708#define MINIMUM_DHCP_PACKET_SIZE 2825709static int e1000_transfer_dhcp_info(struct e1000_adapter *adapter,5710 struct sk_buff *skb)5711{5712 struct e1000_hw *hw = &adapter->hw;5713 u16 length, offset;5714 5715 if (skb_vlan_tag_present(skb) &&5716 !((skb_vlan_tag_get(skb) == adapter->hw.mng_cookie.vlan_id) &&5717 (adapter->hw.mng_cookie.status &5718 E1000_MNG_DHCP_COOKIE_STATUS_VLAN)))5719 return 0;5720 5721 if (skb->len <= MINIMUM_DHCP_PACKET_SIZE)5722 return 0;5723 5724 if (((struct ethhdr *)skb->data)->h_proto != htons(ETH_P_IP))5725 return 0;5726 5727 {5728 const struct iphdr *ip = (struct iphdr *)((u8 *)skb->data + 14);5729 struct udphdr *udp;5730 5731 if (ip->protocol != IPPROTO_UDP)5732 return 0;5733 5734 udp = (struct udphdr *)((u8 *)ip + (ip->ihl << 2));5735 if (ntohs(udp->dest) != 67)5736 return 0;5737 5738 offset = (u8 *)udp + 8 - skb->data;5739 length = skb->len - offset;5740 return e1000e_mng_write_dhcp_info(hw, (u8 *)udp + 8, length);5741 }5742 5743 return 0;5744}5745 5746static int __e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size)5747{5748 struct e1000_adapter *adapter = tx_ring->adapter;5749 5750 netif_stop_queue(adapter->netdev);5751 /* Herbert's original patch had:5752 * smp_mb__after_netif_stop_queue();5753 * but since that doesn't exist yet, just open code it.5754 */5755 smp_mb();5756 5757 /* We need to check again in a case another CPU has just5758 * made room available.5759 */5760 if (e1000_desc_unused(tx_ring) < size)5761 return -EBUSY;5762 5763 /* A reprieve! */5764 netif_start_queue(adapter->netdev);5765 ++adapter->restart_queue;5766 return 0;5767}5768 5769static int e1000_maybe_stop_tx(struct e1000_ring *tx_ring, int size)5770{5771 BUG_ON(size > tx_ring->count);5772 5773 if (e1000_desc_unused(tx_ring) >= size)5774 return 0;5775 return __e1000_maybe_stop_tx(tx_ring, size);5776}5777 5778static netdev_tx_t e1000_xmit_frame(struct sk_buff *skb,5779 struct net_device *netdev)5780{5781 struct e1000_adapter *adapter = netdev_priv(netdev);5782 struct e1000_ring *tx_ring = adapter->tx_ring;5783 unsigned int first;5784 unsigned int tx_flags = 0;5785 unsigned int len = skb_headlen(skb);5786 unsigned int nr_frags;5787 unsigned int mss;5788 int count = 0;5789 int tso;5790 unsigned int f;5791 __be16 protocol = vlan_get_protocol(skb);5792 5793 if (test_bit(__E1000_DOWN, &adapter->state)) {5794 dev_kfree_skb_any(skb);5795 return NETDEV_TX_OK;5796 }5797 5798 if (skb->len <= 0) {5799 dev_kfree_skb_any(skb);5800 return NETDEV_TX_OK;5801 }5802 5803 /* The minimum packet size with TCTL.PSP set is 17 bytes so5804 * pad skb in order to meet this minimum size requirement5805 */5806 if (skb_put_padto(skb, 17))5807 return NETDEV_TX_OK;5808 5809 mss = skb_shinfo(skb)->gso_size;5810 if (mss) {5811 u8 hdr_len;5812 5813 /* TSO Workaround for 82571/2/3 Controllers -- if skb->data5814 * points to just header, pull a few bytes of payload from5815 * frags into skb->data5816 */5817 hdr_len = skb_tcp_all_headers(skb);5818 /* we do this workaround for ES2LAN, but it is un-necessary,5819 * avoiding it could save a lot of cycles5820 */5821 if (skb->data_len && (hdr_len == len)) {5822 unsigned int pull_size;5823 5824 pull_size = min_t(unsigned int, 4, skb->data_len);5825 if (!__pskb_pull_tail(skb, pull_size)) {5826 e_err("__pskb_pull_tail failed.\n");5827 dev_kfree_skb_any(skb);5828 return NETDEV_TX_OK;5829 }5830 len = skb_headlen(skb);5831 }5832 }5833 5834 /* reserve a descriptor for the offload context */5835 if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL))5836 count++;5837 count++;5838 5839 count += DIV_ROUND_UP(len, adapter->tx_fifo_limit);5840 5841 nr_frags = skb_shinfo(skb)->nr_frags;5842 for (f = 0; f < nr_frags; f++)5843 count += DIV_ROUND_UP(skb_frag_size(&skb_shinfo(skb)->frags[f]),5844 adapter->tx_fifo_limit);5845 5846 if (adapter->hw.mac.tx_pkt_filtering)5847 e1000_transfer_dhcp_info(adapter, skb);5848 5849 /* need: count + 2 desc gap to keep tail from touching5850 * head, otherwise try next time5851 */5852 if (e1000_maybe_stop_tx(tx_ring, count + 2))5853 return NETDEV_TX_BUSY;5854 5855 if (skb_vlan_tag_present(skb)) {5856 tx_flags |= E1000_TX_FLAGS_VLAN;5857 tx_flags |= (skb_vlan_tag_get(skb) <<5858 E1000_TX_FLAGS_VLAN_SHIFT);5859 }5860 5861 first = tx_ring->next_to_use;5862 5863 tso = e1000_tso(tx_ring, skb, protocol);5864 if (tso < 0) {5865 dev_kfree_skb_any(skb);5866 return NETDEV_TX_OK;5867 }5868 5869 if (tso)5870 tx_flags |= E1000_TX_FLAGS_TSO;5871 else if (e1000_tx_csum(tx_ring, skb, protocol))5872 tx_flags |= E1000_TX_FLAGS_CSUM;5873 5874 /* Old method was to assume IPv4 packet by default if TSO was enabled.5875 * 82571 hardware supports TSO capabilities for IPv6 as well...5876 * no longer assume, we must.5877 */5878 if (protocol == htons(ETH_P_IP))5879 tx_flags |= E1000_TX_FLAGS_IPV4;5880 5881 if (unlikely(skb->no_fcs))5882 tx_flags |= E1000_TX_FLAGS_NO_FCS;5883 5884 /* if count is 0 then mapping error has occurred */5885 count = e1000_tx_map(tx_ring, skb, first, adapter->tx_fifo_limit,5886 nr_frags);5887 if (count) {5888 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&5889 (adapter->flags & FLAG_HAS_HW_TIMESTAMP)) {5890 if (!adapter->tx_hwtstamp_skb) {5891 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;5892 tx_flags |= E1000_TX_FLAGS_HWTSTAMP;5893 adapter->tx_hwtstamp_skb = skb_get(skb);5894 adapter->tx_hwtstamp_start = jiffies;5895 schedule_work(&adapter->tx_hwtstamp_work);5896 } else {5897 adapter->tx_hwtstamp_skipped++;5898 }5899 }5900 5901 skb_tx_timestamp(skb);5902 5903 netdev_sent_queue(netdev, skb->len);5904 e1000_tx_queue(tx_ring, tx_flags, count);5905 /* Make sure there is space in the ring for the next send. */5906 e1000_maybe_stop_tx(tx_ring,5907 ((MAX_SKB_FRAGS + 1) *5908 DIV_ROUND_UP(PAGE_SIZE,5909 adapter->tx_fifo_limit) + 4));5910 5911 if (!netdev_xmit_more() ||5912 netif_xmit_stopped(netdev_get_tx_queue(netdev, 0))) {5913 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)5914 e1000e_update_tdt_wa(tx_ring,5915 tx_ring->next_to_use);5916 else5917 writel(tx_ring->next_to_use, tx_ring->tail);5918 }5919 } else {5920 dev_kfree_skb_any(skb);5921 tx_ring->buffer_info[first].time_stamp = 0;5922 tx_ring->next_to_use = first;5923 }5924 5925 return NETDEV_TX_OK;5926}5927 5928/**5929 * e1000_tx_timeout - Respond to a Tx Hang5930 * @netdev: network interface device structure5931 * @txqueue: index of the hung queue (unused)5932 **/5933static void e1000_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)5934{5935 struct e1000_adapter *adapter = netdev_priv(netdev);5936 5937 /* Do the reset outside of interrupt context */5938 adapter->tx_timeout_count++;5939 schedule_work(&adapter->reset_task);5940}5941 5942static void e1000_reset_task(struct work_struct *work)5943{5944 struct e1000_adapter *adapter;5945 adapter = container_of(work, struct e1000_adapter, reset_task);5946 5947 rtnl_lock();5948 /* don't run the task if already down */5949 if (test_bit(__E1000_DOWN, &adapter->state)) {5950 rtnl_unlock();5951 return;5952 }5953 5954 if (!(adapter->flags & FLAG_RESTART_NOW)) {5955 e1000e_dump(adapter);5956 e_err("Reset adapter unexpectedly\n");5957 }5958 e1000e_reinit_locked(adapter);5959 rtnl_unlock();5960}5961 5962/**5963 * e1000e_get_stats64 - Get System Network Statistics5964 * @netdev: network interface device structure5965 * @stats: rtnl_link_stats64 pointer5966 *5967 * Returns the address of the device statistics structure.5968 **/5969void e1000e_get_stats64(struct net_device *netdev,5970 struct rtnl_link_stats64 *stats)5971{5972 struct e1000_adapter *adapter = netdev_priv(netdev);5973 5974 spin_lock(&adapter->stats64_lock);5975 e1000e_update_stats(adapter);5976 /* Fill out the OS statistics structure */5977 stats->rx_bytes = adapter->stats.gorc;5978 stats->rx_packets = adapter->stats.gprc;5979 stats->tx_bytes = adapter->stats.gotc;5980 stats->tx_packets = adapter->stats.gptc;5981 stats->multicast = adapter->stats.mprc;5982 stats->collisions = adapter->stats.colc;5983 5984 /* Rx Errors */5985 5986 /* RLEC on some newer hardware can be incorrect so build5987 * our own version based on RUC and ROC5988 */5989 stats->rx_errors = adapter->stats.rxerrc +5990 adapter->stats.crcerrs + adapter->stats.algnerrc +5991 adapter->stats.ruc + adapter->stats.roc + adapter->stats.cexterr;5992 stats->rx_length_errors = adapter->stats.ruc + adapter->stats.roc;5993 stats->rx_crc_errors = adapter->stats.crcerrs;5994 stats->rx_frame_errors = adapter->stats.algnerrc;5995 stats->rx_missed_errors = adapter->stats.mpc;5996 5997 /* Tx Errors */5998 stats->tx_errors = adapter->stats.ecol + adapter->stats.latecol;5999 stats->tx_aborted_errors = adapter->stats.ecol;6000 stats->tx_window_errors = adapter->stats.latecol;6001 stats->tx_carrier_errors = adapter->stats.tncrs;6002 6003 /* Tx Dropped needs to be maintained elsewhere */6004 6005 spin_unlock(&adapter->stats64_lock);6006}6007 6008/**6009 * e1000_change_mtu - Change the Maximum Transfer Unit6010 * @netdev: network interface device structure6011 * @new_mtu: new value for maximum frame size6012 *6013 * Returns 0 on success, negative on failure6014 **/6015static int e1000_change_mtu(struct net_device *netdev, int new_mtu)6016{6017 struct e1000_adapter *adapter = netdev_priv(netdev);6018 int max_frame = new_mtu + VLAN_ETH_HLEN + ETH_FCS_LEN;6019 6020 /* Jumbo frame support */6021 if ((new_mtu > ETH_DATA_LEN) &&6022 !(adapter->flags & FLAG_HAS_JUMBO_FRAMES)) {6023 e_err("Jumbo Frames not supported.\n");6024 return -EINVAL;6025 }6026 6027 /* Jumbo frame workaround on 82579 and newer requires CRC be stripped */6028 if ((adapter->hw.mac.type >= e1000_pch2lan) &&6029 !(adapter->flags2 & FLAG2_CRC_STRIPPING) &&6030 (new_mtu > ETH_DATA_LEN)) {6031 e_err("Jumbo Frames not supported on this device when CRC stripping is disabled.\n");6032 return -EINVAL;6033 }6034 6035 while (test_and_set_bit(__E1000_RESETTING, &adapter->state))6036 usleep_range(1000, 1100);6037 /* e1000e_down -> e1000e_reset dependent on max_frame_size & mtu */6038 adapter->max_frame_size = max_frame;6039 netdev_dbg(netdev, "changing MTU from %d to %d\n",6040 netdev->mtu, new_mtu);6041 WRITE_ONCE(netdev->mtu, new_mtu);6042 6043 pm_runtime_get_sync(netdev->dev.parent);6044 6045 if (netif_running(netdev))6046 e1000e_down(adapter, true);6047 6048 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN6049 * means we reserve 2 more, this pushes us to allocate from the next6050 * larger slab size.6051 * i.e. RXBUFFER_2048 --> size-4096 slab6052 * However with the new *_jumbo_rx* routines, jumbo receives will use6053 * fragmented skbs6054 */6055 6056 if (max_frame <= 2048)6057 adapter->rx_buffer_len = 2048;6058 else6059 adapter->rx_buffer_len = 4096;6060 6061 /* adjust allocation if LPE protects us, and we aren't using SBP */6062 if (max_frame <= (VLAN_ETH_FRAME_LEN + ETH_FCS_LEN))6063 adapter->rx_buffer_len = VLAN_ETH_FRAME_LEN + ETH_FCS_LEN;6064 6065 if (netif_running(netdev))6066 e1000e_up(adapter);6067 else6068 e1000e_reset(adapter);6069 6070 pm_runtime_put_sync(netdev->dev.parent);6071 6072 clear_bit(__E1000_RESETTING, &adapter->state);6073 6074 return 0;6075}6076 6077static int e1000_mii_ioctl(struct net_device *netdev, struct ifreq *ifr,6078 int cmd)6079{6080 struct e1000_adapter *adapter = netdev_priv(netdev);6081 struct mii_ioctl_data *data = if_mii(ifr);6082 6083 if (adapter->hw.phy.media_type != e1000_media_type_copper)6084 return -EOPNOTSUPP;6085 6086 switch (cmd) {6087 case SIOCGMIIPHY:6088 data->phy_id = adapter->hw.phy.addr;6089 break;6090 case SIOCGMIIREG:6091 e1000_phy_read_status(adapter);6092 6093 switch (data->reg_num & 0x1F) {6094 case MII_BMCR:6095 data->val_out = adapter->phy_regs.bmcr;6096 break;6097 case MII_BMSR:6098 data->val_out = adapter->phy_regs.bmsr;6099 break;6100 case MII_PHYSID1:6101 data->val_out = (adapter->hw.phy.id >> 16);6102 break;6103 case MII_PHYSID2:6104 data->val_out = (adapter->hw.phy.id & 0xFFFF);6105 break;6106 case MII_ADVERTISE:6107 data->val_out = adapter->phy_regs.advertise;6108 break;6109 case MII_LPA:6110 data->val_out = adapter->phy_regs.lpa;6111 break;6112 case MII_EXPANSION:6113 data->val_out = adapter->phy_regs.expansion;6114 break;6115 case MII_CTRL1000:6116 data->val_out = adapter->phy_regs.ctrl1000;6117 break;6118 case MII_STAT1000:6119 data->val_out = adapter->phy_regs.stat1000;6120 break;6121 case MII_ESTATUS:6122 data->val_out = adapter->phy_regs.estatus;6123 break;6124 default:6125 return -EIO;6126 }6127 break;6128 case SIOCSMIIREG:6129 default:6130 return -EOPNOTSUPP;6131 }6132 return 0;6133}6134 6135/**6136 * e1000e_hwtstamp_set - control hardware time stamping6137 * @netdev: network interface device structure6138 * @ifr: interface request6139 *6140 * Outgoing time stamping can be enabled and disabled. Play nice and6141 * disable it when requested, although it shouldn't cause any overhead6142 * when no packet needs it. At most one packet in the queue may be6143 * marked for time stamping, otherwise it would be impossible to tell6144 * for sure to which packet the hardware time stamp belongs.6145 *6146 * Incoming time stamping has to be configured via the hardware filters.6147 * Not all combinations are supported, in particular event type has to be6148 * specified. Matching the kind of event packet is not supported, with the6149 * exception of "all V2 events regardless of level 2 or 4".6150 **/6151static int e1000e_hwtstamp_set(struct net_device *netdev, struct ifreq *ifr)6152{6153 struct e1000_adapter *adapter = netdev_priv(netdev);6154 struct hwtstamp_config config;6155 int ret_val;6156 6157 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))6158 return -EFAULT;6159 6160 ret_val = e1000e_config_hwtstamp(adapter, &config);6161 if (ret_val)6162 return ret_val;6163 6164 switch (config.rx_filter) {6165 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:6166 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:6167 case HWTSTAMP_FILTER_PTP_V2_SYNC:6168 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:6169 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:6170 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:6171 /* With V2 type filters which specify a Sync or Delay Request,6172 * Path Delay Request/Response messages are also time stamped6173 * by hardware so notify the caller the requested packets plus6174 * some others are time stamped.6175 */6176 config.rx_filter = HWTSTAMP_FILTER_SOME;6177 break;6178 default:6179 break;6180 }6181 6182 return copy_to_user(ifr->ifr_data, &config,6183 sizeof(config)) ? -EFAULT : 0;6184}6185 6186static int e1000e_hwtstamp_get(struct net_device *netdev, struct ifreq *ifr)6187{6188 struct e1000_adapter *adapter = netdev_priv(netdev);6189 6190 return copy_to_user(ifr->ifr_data, &adapter->hwtstamp_config,6191 sizeof(adapter->hwtstamp_config)) ? -EFAULT : 0;6192}6193 6194static int e1000_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)6195{6196 switch (cmd) {6197 case SIOCGMIIPHY:6198 case SIOCGMIIREG:6199 case SIOCSMIIREG:6200 return e1000_mii_ioctl(netdev, ifr, cmd);6201 case SIOCSHWTSTAMP:6202 return e1000e_hwtstamp_set(netdev, ifr);6203 case SIOCGHWTSTAMP:6204 return e1000e_hwtstamp_get(netdev, ifr);6205 default:6206 return -EOPNOTSUPP;6207 }6208}6209 6210static int e1000_init_phy_wakeup(struct e1000_adapter *adapter, u32 wufc)6211{6212 struct e1000_hw *hw = &adapter->hw;6213 u32 i, mac_reg, wuc;6214 u16 phy_reg, wuc_enable;6215 int retval;6216 6217 /* copy MAC RARs to PHY RARs */6218 e1000_copy_rx_addrs_to_phy_ich8lan(hw);6219 6220 retval = hw->phy.ops.acquire(hw);6221 if (retval) {6222 e_err("Could not acquire PHY\n");6223 return retval;6224 }6225 6226 /* Enable access to wakeup registers on and set page to BM_WUC_PAGE */6227 retval = e1000_enable_phy_wakeup_reg_access_bm(hw, &wuc_enable);6228 if (retval)6229 goto release;6230 6231 /* copy MAC MTA to PHY MTA - only needed for pchlan */6232 for (i = 0; i < adapter->hw.mac.mta_reg_count; i++) {6233 mac_reg = E1000_READ_REG_ARRAY(hw, E1000_MTA, i);6234 hw->phy.ops.write_reg_page(hw, BM_MTA(i),6235 (u16)(mac_reg & 0xFFFF));6236 hw->phy.ops.write_reg_page(hw, BM_MTA(i) + 1,6237 (u16)((mac_reg >> 16) & 0xFFFF));6238 }6239 6240 /* configure PHY Rx Control register */6241 hw->phy.ops.read_reg_page(&adapter->hw, BM_RCTL, &phy_reg);6242 mac_reg = er32(RCTL);6243 if (mac_reg & E1000_RCTL_UPE)6244 phy_reg |= BM_RCTL_UPE;6245 if (mac_reg & E1000_RCTL_MPE)6246 phy_reg |= BM_RCTL_MPE;6247 phy_reg &= ~(BM_RCTL_MO_MASK);6248 if (mac_reg & E1000_RCTL_MO_3)6249 phy_reg |= (FIELD_GET(E1000_RCTL_MO_3, mac_reg)6250 << BM_RCTL_MO_SHIFT);6251 if (mac_reg & E1000_RCTL_BAM)6252 phy_reg |= BM_RCTL_BAM;6253 if (mac_reg & E1000_RCTL_PMCF)6254 phy_reg |= BM_RCTL_PMCF;6255 mac_reg = er32(CTRL);6256 if (mac_reg & E1000_CTRL_RFCE)6257 phy_reg |= BM_RCTL_RFCE;6258 hw->phy.ops.write_reg_page(&adapter->hw, BM_RCTL, phy_reg);6259 6260 wuc = E1000_WUC_PME_EN;6261 if (wufc & (E1000_WUFC_MAG | E1000_WUFC_LNKC))6262 wuc |= E1000_WUC_APME;6263 6264 /* enable PHY wakeup in MAC register */6265 ew32(WUFC, wufc);6266 ew32(WUC, (E1000_WUC_PHY_WAKE | E1000_WUC_APMPME |6267 E1000_WUC_PME_STATUS | wuc));6268 6269 /* configure and enable PHY wakeup in PHY registers */6270 hw->phy.ops.write_reg_page(&adapter->hw, BM_WUFC, wufc);6271 hw->phy.ops.write_reg_page(&adapter->hw, BM_WUC, wuc);6272 6273 /* activate PHY wakeup */6274 wuc_enable |= BM_WUC_ENABLE_BIT | BM_WUC_HOST_WU_BIT;6275 retval = e1000_disable_phy_wakeup_reg_access_bm(hw, &wuc_enable);6276 if (retval)6277 e_err("Could not set PHY Host Wakeup bit\n");6278release:6279 hw->phy.ops.release(hw);6280 6281 return retval;6282}6283 6284static void e1000e_flush_lpic(struct pci_dev *pdev)6285{6286 struct net_device *netdev = pci_get_drvdata(pdev);6287 struct e1000_adapter *adapter = netdev_priv(netdev);6288 struct e1000_hw *hw = &adapter->hw;6289 u32 ret_val;6290 6291 pm_runtime_get_sync(netdev->dev.parent);6292 6293 ret_val = hw->phy.ops.acquire(hw);6294 if (ret_val)6295 goto fl_out;6296 6297 pr_info("EEE TX LPI TIMER: %08X\n",6298 er32(LPIC) >> E1000_LPIC_LPIET_SHIFT);6299 6300 hw->phy.ops.release(hw);6301 6302fl_out:6303 pm_runtime_put_sync(netdev->dev.parent);6304}6305 6306/* S0ix implementation */6307static void e1000e_s0ix_entry_flow(struct e1000_adapter *adapter)6308{6309 struct e1000_hw *hw = &adapter->hw;6310 u32 mac_data;6311 u16 phy_data;6312 6313 if (er32(FWSM) & E1000_ICH_FWSM_FW_VALID &&6314 hw->mac.type >= e1000_pch_adp) {6315 /* Request ME configure the device for S0ix */6316 mac_data = er32(H2ME);6317 mac_data |= E1000_H2ME_START_DPG;6318 mac_data &= ~E1000_H2ME_EXIT_DPG;6319 trace_e1000e_trace_mac_register(mac_data);6320 ew32(H2ME, mac_data);6321 } else {6322 /* Request driver configure the device to S0ix */6323 /* Disable the periodic inband message,6324 * don't request PCIe clock in K1 page770_17[10:9] = 10b6325 */6326 e1e_rphy(hw, HV_PM_CTRL, &phy_data);6327 phy_data &= ~HV_PM_CTRL_K1_CLK_REQ;6328 phy_data |= BIT(10);6329 e1e_wphy(hw, HV_PM_CTRL, phy_data);6330 6331 /* Make sure we don't exit K1 every time a new packet arrives6332 * 772_29[5] = 1 CS_Mode_Stay_In_K16333 */6334 e1e_rphy(hw, I217_CGFREG, &phy_data);6335 phy_data |= BIT(5);6336 e1e_wphy(hw, I217_CGFREG, phy_data);6337 6338 /* Change the MAC/PHY interface to SMBus6339 * Force the SMBus in PHY page769_23[0] = 16340 * Force the SMBus in MAC CTRL_EXT[11] = 16341 */6342 e1e_rphy(hw, CV_SMB_CTRL, &phy_data);6343 phy_data |= CV_SMB_CTRL_FORCE_SMBUS;6344 e1e_wphy(hw, CV_SMB_CTRL, phy_data);6345 mac_data = er32(CTRL_EXT);6346 mac_data |= E1000_CTRL_EXT_FORCE_SMBUS;6347 ew32(CTRL_EXT, mac_data);6348 6349 /* DFT control: PHY bit: page769_20[0] = 16350 * page769_20[7] - PHY PLL stop6351 * page769_20[8] - PHY go to the electrical idle6352 * page769_20[9] - PHY serdes disable6353 * Gate PPW via EXTCNF_CTRL - set 0x0F00[7] = 16354 */6355 e1e_rphy(hw, I82579_DFT_CTRL, &phy_data);6356 phy_data |= BIT(0);6357 phy_data |= BIT(7);6358 phy_data |= BIT(8);6359 phy_data |= BIT(9);6360 e1e_wphy(hw, I82579_DFT_CTRL, phy_data);6361 6362 mac_data = er32(EXTCNF_CTRL);6363 mac_data |= E1000_EXTCNF_CTRL_GATE_PHY_CFG;6364 ew32(EXTCNF_CTRL, mac_data);6365 6366 /* Disable disconnected cable conditioning for Power Gating */6367 mac_data = er32(DPGFR);6368 mac_data |= BIT(2);6369 ew32(DPGFR, mac_data);6370 6371 /* Enable the Dynamic Clock Gating in the DMA and MAC */6372 mac_data = er32(CTRL_EXT);6373 mac_data |= E1000_CTRL_EXT_DMA_DYN_CLK_EN;6374 ew32(CTRL_EXT, mac_data);6375 }6376 6377 /* Enable the Dynamic Power Gating in the MAC */6378 mac_data = er32(FEXTNVM7);6379 mac_data |= BIT(22);6380 ew32(FEXTNVM7, mac_data);6381 6382 /* Don't wake from dynamic Power Gating with clock request */6383 mac_data = er32(FEXTNVM12);6384 mac_data |= BIT(12);6385 ew32(FEXTNVM12, mac_data);6386 6387 /* Ungate PGCB clock */6388 mac_data = er32(FEXTNVM9);6389 mac_data &= ~BIT(28);6390 ew32(FEXTNVM9, mac_data);6391 6392 /* Enable K1 off to enable mPHY Power Gating */6393 mac_data = er32(FEXTNVM6);6394 mac_data |= BIT(31);6395 ew32(FEXTNVM6, mac_data);6396 6397 /* Enable mPHY power gating for any link and speed */6398 mac_data = er32(FEXTNVM8);6399 mac_data |= BIT(9);6400 ew32(FEXTNVM8, mac_data);6401 6402 /* No MAC DPG gating SLP_S0 in modern standby6403 * Switch the logic of the lanphypc to use PMC counter6404 */6405 mac_data = er32(FEXTNVM5);6406 mac_data |= BIT(7);6407 ew32(FEXTNVM5, mac_data);6408 6409 /* Disable the time synchronization clock */6410 mac_data = er32(FEXTNVM7);6411 mac_data |= BIT(31);6412 mac_data &= ~BIT(0);6413 ew32(FEXTNVM7, mac_data);6414 6415 /* Dynamic Power Gating Enable */6416 mac_data = er32(CTRL_EXT);6417 mac_data |= BIT(3);6418 ew32(CTRL_EXT, mac_data);6419 6420 /* Check MAC Tx/Rx packet buffer pointers.6421 * Reset MAC Tx/Rx packet buffer pointers to suppress any6422 * pending traffic indication that would prevent power gating.6423 */6424 mac_data = er32(TDFH);6425 if (mac_data)6426 ew32(TDFH, 0);6427 mac_data = er32(TDFT);6428 if (mac_data)6429 ew32(TDFT, 0);6430 mac_data = er32(TDFHS);6431 if (mac_data)6432 ew32(TDFHS, 0);6433 mac_data = er32(TDFTS);6434 if (mac_data)6435 ew32(TDFTS, 0);6436 mac_data = er32(TDFPC);6437 if (mac_data)6438 ew32(TDFPC, 0);6439 mac_data = er32(RDFH);6440 if (mac_data)6441 ew32(RDFH, 0);6442 mac_data = er32(RDFT);6443 if (mac_data)6444 ew32(RDFT, 0);6445 mac_data = er32(RDFHS);6446 if (mac_data)6447 ew32(RDFHS, 0);6448 mac_data = er32(RDFTS);6449 if (mac_data)6450 ew32(RDFTS, 0);6451 mac_data = er32(RDFPC);6452 if (mac_data)6453 ew32(RDFPC, 0);6454}6455 6456static void e1000e_s0ix_exit_flow(struct e1000_adapter *adapter)6457{6458 struct e1000_hw *hw = &adapter->hw;6459 bool firmware_bug = false;6460 u32 mac_data;6461 u16 phy_data;6462 u32 i = 0;6463 6464 if (er32(FWSM) & E1000_ICH_FWSM_FW_VALID &&6465 hw->mac.type >= e1000_pch_adp) {6466 /* Keep the GPT clock enabled for CSME */6467 mac_data = er32(FEXTNVM);6468 mac_data |= BIT(3);6469 ew32(FEXTNVM, mac_data);6470 /* Request ME unconfigure the device from S0ix */6471 mac_data = er32(H2ME);6472 mac_data &= ~E1000_H2ME_START_DPG;6473 mac_data |= E1000_H2ME_EXIT_DPG;6474 trace_e1000e_trace_mac_register(mac_data);6475 ew32(H2ME, mac_data);6476 6477 /* Poll up to 2.5 seconds for ME to unconfigure DPG.6478 * If this takes more than 1 second, show a warning indicating a6479 * firmware bug6480 */6481 while (!(er32(EXFWSM) & E1000_EXFWSM_DPG_EXIT_DONE)) {6482 if (i > 100 && !firmware_bug)6483 firmware_bug = true;6484 6485 if (i++ == 250) {6486 e_dbg("Timeout (firmware bug): %d msec\n",6487 i * 10);6488 break;6489 }6490 6491 usleep_range(10000, 11000);6492 }6493 if (firmware_bug)6494 e_warn("DPG_EXIT_DONE took %d msec. This is a firmware bug\n",6495 i * 10);6496 else6497 e_dbg("DPG_EXIT_DONE cleared after %d msec\n", i * 10);6498 } else {6499 /* Request driver unconfigure the device from S0ix */6500 6501 /* Cancel disable disconnected cable conditioning6502 * for Power Gating6503 */6504 mac_data = er32(DPGFR);6505 mac_data &= ~BIT(2);6506 ew32(DPGFR, mac_data);6507 6508 /* Disable the Dynamic Clock Gating in the DMA and MAC */6509 mac_data = er32(CTRL_EXT);6510 mac_data &= 0xFFF7FFFF;6511 ew32(CTRL_EXT, mac_data);6512 6513 /* Enable the periodic inband message,6514 * Request PCIe clock in K1 page770_17[10:9] =01b6515 */6516 e1e_rphy(hw, HV_PM_CTRL, &phy_data);6517 phy_data &= 0xFBFF;6518 phy_data |= HV_PM_CTRL_K1_CLK_REQ;6519 e1e_wphy(hw, HV_PM_CTRL, phy_data);6520 6521 /* Return back configuration6522 * 772_29[5] = 0 CS_Mode_Stay_In_K16523 */6524 e1e_rphy(hw, I217_CGFREG, &phy_data);6525 phy_data &= 0xFFDF;6526 e1e_wphy(hw, I217_CGFREG, phy_data);6527 6528 /* Change the MAC/PHY interface to Kumeran6529 * Unforce the SMBus in PHY page769_23[0] = 06530 * Unforce the SMBus in MAC CTRL_EXT[11] = 06531 */6532 e1e_rphy(hw, CV_SMB_CTRL, &phy_data);6533 phy_data &= ~CV_SMB_CTRL_FORCE_SMBUS;6534 e1e_wphy(hw, CV_SMB_CTRL, phy_data);6535 mac_data = er32(CTRL_EXT);6536 mac_data &= ~E1000_CTRL_EXT_FORCE_SMBUS;6537 ew32(CTRL_EXT, mac_data);6538 }6539 6540 /* Disable Dynamic Power Gating */6541 mac_data = er32(CTRL_EXT);6542 mac_data &= 0xFFFFFFF7;6543 ew32(CTRL_EXT, mac_data);6544 6545 /* Enable the time synchronization clock */6546 mac_data = er32(FEXTNVM7);6547 mac_data &= ~BIT(31);6548 mac_data |= BIT(0);6549 ew32(FEXTNVM7, mac_data);6550 6551 /* Disable the Dynamic Power Gating in the MAC */6552 mac_data = er32(FEXTNVM7);6553 mac_data &= 0xFFBFFFFF;6554 ew32(FEXTNVM7, mac_data);6555 6556 /* Disable mPHY power gating for any link and speed */6557 mac_data = er32(FEXTNVM8);6558 mac_data &= ~BIT(9);6559 ew32(FEXTNVM8, mac_data);6560 6561 /* Disable K1 off */6562 mac_data = er32(FEXTNVM6);6563 mac_data &= ~BIT(31);6564 ew32(FEXTNVM6, mac_data);6565 6566 /* Disable Ungate PGCB clock */6567 mac_data = er32(FEXTNVM9);6568 mac_data |= BIT(28);6569 ew32(FEXTNVM9, mac_data);6570 6571 /* Cancel not waking from dynamic6572 * Power Gating with clock request6573 */6574 mac_data = er32(FEXTNVM12);6575 mac_data &= ~BIT(12);6576 ew32(FEXTNVM12, mac_data);6577 6578 /* Revert the lanphypc logic to use the internal Gbe counter6579 * and not the PMC counter6580 */6581 mac_data = er32(FEXTNVM5);6582 mac_data &= 0xFFFFFF7F;6583 ew32(FEXTNVM5, mac_data);6584}6585 6586static int e1000e_pm_freeze(struct device *dev)6587{6588 struct net_device *netdev = dev_get_drvdata(dev);6589 struct e1000_adapter *adapter = netdev_priv(netdev);6590 bool present;6591 6592 rtnl_lock();6593 6594 present = netif_device_present(netdev);6595 netif_device_detach(netdev);6596 6597 if (present && netif_running(netdev)) {6598 int count = E1000_CHECK_RESET_COUNT;6599 6600 while (test_bit(__E1000_RESETTING, &adapter->state) && count--)6601 usleep_range(10000, 11000);6602 6603 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state));6604 6605 /* Quiesce the device without resetting the hardware */6606 e1000e_down(adapter, false);6607 e1000_free_irq(adapter);6608 }6609 rtnl_unlock();6610 6611 e1000e_reset_interrupt_capability(adapter);6612 6613 /* Allow time for pending master requests to run */6614 e1000e_disable_pcie_master(&adapter->hw);6615 6616 return 0;6617}6618 6619static int __e1000_shutdown(struct pci_dev *pdev, bool runtime)6620{6621 struct net_device *netdev = pci_get_drvdata(pdev);6622 struct e1000_adapter *adapter = netdev_priv(netdev);6623 struct e1000_hw *hw = &adapter->hw;6624 u32 ctrl, ctrl_ext, rctl, status, wufc;6625 int retval = 0;6626 6627 /* Runtime suspend should only enable wakeup for link changes */6628 if (runtime)6629 wufc = E1000_WUFC_LNKC;6630 else if (device_may_wakeup(&pdev->dev))6631 wufc = adapter->wol;6632 else6633 wufc = 0;6634 6635 status = er32(STATUS);6636 if (status & E1000_STATUS_LU)6637 wufc &= ~E1000_WUFC_LNKC;6638 6639 if (wufc) {6640 e1000_setup_rctl(adapter);6641 e1000e_set_rx_mode(netdev);6642 6643 /* turn on all-multi mode if wake on multicast is enabled */6644 if (wufc & E1000_WUFC_MC) {6645 rctl = er32(RCTL);6646 rctl |= E1000_RCTL_MPE;6647 ew32(RCTL, rctl);6648 }6649 6650 ctrl = er32(CTRL);6651 ctrl |= E1000_CTRL_ADVD3WUC;6652 if (!(adapter->flags2 & FLAG2_HAS_PHY_WAKEUP))6653 ctrl |= E1000_CTRL_EN_PHY_PWR_MGMT;6654 ew32(CTRL, ctrl);6655 6656 if (adapter->hw.phy.media_type == e1000_media_type_fiber ||6657 adapter->hw.phy.media_type ==6658 e1000_media_type_internal_serdes) {6659 /* keep the laser running in D3 */6660 ctrl_ext = er32(CTRL_EXT);6661 ctrl_ext |= E1000_CTRL_EXT_SDP3_DATA;6662 ew32(CTRL_EXT, ctrl_ext);6663 }6664 6665 if (!runtime)6666 e1000e_power_up_phy(adapter);6667 6668 if (adapter->flags & FLAG_IS_ICH)6669 e1000_suspend_workarounds_ich8lan(&adapter->hw);6670 6671 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) {6672 /* enable wakeup by the PHY */6673 retval = e1000_init_phy_wakeup(adapter, wufc);6674 if (retval) {6675 e_err("Failed to enable wakeup\n");6676 goto skip_phy_configurations;6677 }6678 } else {6679 /* enable wakeup by the MAC */6680 ew32(WUFC, wufc);6681 ew32(WUC, E1000_WUC_PME_EN);6682 }6683 } else {6684 ew32(WUC, 0);6685 ew32(WUFC, 0);6686 6687 e1000_power_down_phy(adapter);6688 }6689 6690 if (adapter->hw.phy.type == e1000_phy_igp_3) {6691 e1000e_igp3_phy_powerdown_workaround_ich8lan(&adapter->hw);6692 } else if (hw->mac.type >= e1000_pch_lpt) {6693 if (wufc && !(wufc & (E1000_WUFC_EX | E1000_WUFC_MC | E1000_WUFC_BC))) {6694 /* ULP does not support wake from unicast, multicast6695 * or broadcast.6696 */6697 retval = e1000_enable_ulp_lpt_lp(hw, !runtime);6698 if (retval) {6699 e_err("Failed to enable ULP\n");6700 goto skip_phy_configurations;6701 }6702 }6703 }6704 6705 /* Ensure that the appropriate bits are set in LPI_CTRL6706 * for EEE in Sx6707 */6708 if ((hw->phy.type >= e1000_phy_i217) &&6709 adapter->eee_advert && hw->dev_spec.ich8lan.eee_lp_ability) {6710 u16 lpi_ctrl = 0;6711 6712 retval = hw->phy.ops.acquire(hw);6713 if (!retval) {6714 retval = e1e_rphy_locked(hw, I82579_LPI_CTRL,6715 &lpi_ctrl);6716 if (!retval) {6717 if (adapter->eee_advert &6718 hw->dev_spec.ich8lan.eee_lp_ability &6719 I82579_EEE_100_SUPPORTED)6720 lpi_ctrl |= I82579_LPI_CTRL_100_ENABLE;6721 if (adapter->eee_advert &6722 hw->dev_spec.ich8lan.eee_lp_ability &6723 I82579_EEE_1000_SUPPORTED)6724 lpi_ctrl |= I82579_LPI_CTRL_1000_ENABLE;6725 6726 retval = e1e_wphy_locked(hw, I82579_LPI_CTRL,6727 lpi_ctrl);6728 }6729 }6730 hw->phy.ops.release(hw);6731 }6732 6733skip_phy_configurations:6734 /* Release control of h/w to f/w. If f/w is AMT enabled, this6735 * would have already happened in close and is redundant.6736 */6737 e1000e_release_hw_control(adapter);6738 6739 pci_clear_master(pdev);6740 6741 /* The pci-e switch on some quad port adapters will report a6742 * correctable error when the MAC transitions from D0 to D3. To6743 * prevent this we need to mask off the correctable errors on the6744 * downstream port of the pci-e switch.6745 *6746 * We don't have the associated upstream bridge while assigning6747 * the PCI device into guest. For example, the KVM on power is6748 * one of the cases.6749 */6750 if (adapter->flags & FLAG_IS_QUAD_PORT) {6751 struct pci_dev *us_dev = pdev->bus->self;6752 u16 devctl;6753 6754 if (!us_dev)6755 return 0;6756 6757 pcie_capability_read_word(us_dev, PCI_EXP_DEVCTL, &devctl);6758 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL,6759 (devctl & ~PCI_EXP_DEVCTL_CERE));6760 6761 pci_save_state(pdev);6762 pci_prepare_to_sleep(pdev);6763 6764 pcie_capability_write_word(us_dev, PCI_EXP_DEVCTL, devctl);6765 }6766 6767 return 0;6768}6769 6770/**6771 * __e1000e_disable_aspm - Disable ASPM states6772 * @pdev: pointer to PCI device struct6773 * @state: bit-mask of ASPM states to disable6774 * @locked: indication if this context holds pci_bus_sem locked.6775 *6776 * Some devices *must* have certain ASPM states disabled per hardware errata.6777 **/6778static void __e1000e_disable_aspm(struct pci_dev *pdev, u16 state, int locked)6779{6780 struct pci_dev *parent = pdev->bus->self;6781 u16 aspm_dis_mask = 0;6782 u16 pdev_aspmc, parent_aspmc;6783 6784 switch (state) {6785 case PCIE_LINK_STATE_L0S:6786 case PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1:6787 aspm_dis_mask |= PCI_EXP_LNKCTL_ASPM_L0S;6788 fallthrough; /* can't have L1 without L0s */6789 case PCIE_LINK_STATE_L1:6790 aspm_dis_mask |= PCI_EXP_LNKCTL_ASPM_L1;6791 break;6792 default:6793 return;6794 }6795 6796 pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &pdev_aspmc);6797 pdev_aspmc &= PCI_EXP_LNKCTL_ASPMC;6798 6799 if (parent) {6800 pcie_capability_read_word(parent, PCI_EXP_LNKCTL,6801 &parent_aspmc);6802 parent_aspmc &= PCI_EXP_LNKCTL_ASPMC;6803 }6804 6805 /* Nothing to do if the ASPM states to be disabled already are */6806 if (!(pdev_aspmc & aspm_dis_mask) &&6807 (!parent || !(parent_aspmc & aspm_dis_mask)))6808 return;6809 6810 dev_info(&pdev->dev, "Disabling ASPM %s %s\n",6811 (aspm_dis_mask & pdev_aspmc & PCI_EXP_LNKCTL_ASPM_L0S) ?6812 "L0s" : "",6813 (aspm_dis_mask & pdev_aspmc & PCI_EXP_LNKCTL_ASPM_L1) ?6814 "L1" : "");6815 6816#ifdef CONFIG_PCIEASPM6817 if (locked)6818 pci_disable_link_state_locked(pdev, state);6819 else6820 pci_disable_link_state(pdev, state);6821 6822 /* Double-check ASPM control. If not disabled by the above, the6823 * BIOS is preventing that from happening (or CONFIG_PCIEASPM is6824 * not enabled); override by writing PCI config space directly.6825 */6826 pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &pdev_aspmc);6827 pdev_aspmc &= PCI_EXP_LNKCTL_ASPMC;6828 6829 if (!(aspm_dis_mask & pdev_aspmc))6830 return;6831#endif6832 6833 /* Both device and parent should have the same ASPM setting.6834 * Disable ASPM in downstream component first and then upstream.6835 */6836 pcie_capability_clear_word(pdev, PCI_EXP_LNKCTL, aspm_dis_mask);6837 6838 if (parent)6839 pcie_capability_clear_word(parent, PCI_EXP_LNKCTL,6840 aspm_dis_mask);6841}6842 6843/**6844 * e1000e_disable_aspm - Disable ASPM states.6845 * @pdev: pointer to PCI device struct6846 * @state: bit-mask of ASPM states to disable6847 *6848 * This function acquires the pci_bus_sem!6849 * Some devices *must* have certain ASPM states disabled per hardware errata.6850 **/6851static void e1000e_disable_aspm(struct pci_dev *pdev, u16 state)6852{6853 __e1000e_disable_aspm(pdev, state, 0);6854}6855 6856/**6857 * e1000e_disable_aspm_locked - Disable ASPM states.6858 * @pdev: pointer to PCI device struct6859 * @state: bit-mask of ASPM states to disable6860 *6861 * This function must be called with pci_bus_sem acquired!6862 * Some devices *must* have certain ASPM states disabled per hardware errata.6863 **/6864static void e1000e_disable_aspm_locked(struct pci_dev *pdev, u16 state)6865{6866 __e1000e_disable_aspm(pdev, state, 1);6867}6868 6869static int e1000e_pm_thaw(struct device *dev)6870{6871 struct net_device *netdev = dev_get_drvdata(dev);6872 struct e1000_adapter *adapter = netdev_priv(netdev);6873 int rc = 0;6874 6875 e1000e_set_interrupt_capability(adapter);6876 6877 rtnl_lock();6878 if (netif_running(netdev)) {6879 rc = e1000_request_irq(adapter);6880 if (rc)6881 goto err_irq;6882 6883 e1000e_up(adapter);6884 }6885 6886 netif_device_attach(netdev);6887err_irq:6888 rtnl_unlock();6889 6890 return rc;6891}6892 6893static int __e1000_resume(struct pci_dev *pdev)6894{6895 struct net_device *netdev = pci_get_drvdata(pdev);6896 struct e1000_adapter *adapter = netdev_priv(netdev);6897 struct e1000_hw *hw = &adapter->hw;6898 u16 aspm_disable_flag = 0;6899 6900 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S)6901 aspm_disable_flag = PCIE_LINK_STATE_L0S;6902 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1)6903 aspm_disable_flag |= PCIE_LINK_STATE_L1;6904 if (aspm_disable_flag)6905 e1000e_disable_aspm(pdev, aspm_disable_flag);6906 6907 pci_set_master(pdev);6908 6909 if (hw->mac.type >= e1000_pch2lan)6910 e1000_resume_workarounds_pchlan(&adapter->hw);6911 6912 e1000e_power_up_phy(adapter);6913 6914 /* report the system wakeup cause from S3/S4 */6915 if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) {6916 u16 phy_data;6917 6918 e1e_rphy(&adapter->hw, BM_WUS, &phy_data);6919 if (phy_data) {6920 e_info("PHY Wakeup cause - %s\n",6921 phy_data & E1000_WUS_EX ? "Unicast Packet" :6922 phy_data & E1000_WUS_MC ? "Multicast Packet" :6923 phy_data & E1000_WUS_BC ? "Broadcast Packet" :6924 phy_data & E1000_WUS_MAG ? "Magic Packet" :6925 phy_data & E1000_WUS_LNKC ?6926 "Link Status Change" : "other");6927 }6928 e1e_wphy(&adapter->hw, BM_WUS, ~0);6929 } else {6930 u32 wus = er32(WUS);6931 6932 if (wus) {6933 e_info("MAC Wakeup cause - %s\n",6934 wus & E1000_WUS_EX ? "Unicast Packet" :6935 wus & E1000_WUS_MC ? "Multicast Packet" :6936 wus & E1000_WUS_BC ? "Broadcast Packet" :6937 wus & E1000_WUS_MAG ? "Magic Packet" :6938 wus & E1000_WUS_LNKC ? "Link Status Change" :6939 "other");6940 }6941 ew32(WUS, ~0);6942 }6943 6944 e1000e_reset(adapter);6945 6946 e1000_init_manageability_pt(adapter);6947 6948 /* If the controller has AMT, do not set DRV_LOAD until the interface6949 * is up. For all other cases, let the f/w know that the h/w is now6950 * under the control of the driver.6951 */6952 if (!(adapter->flags & FLAG_HAS_AMT))6953 e1000e_get_hw_control(adapter);6954 6955 return 0;6956}6957 6958static int e1000e_pm_prepare(struct device *dev)6959{6960 return pm_runtime_suspended(dev) &&6961 pm_suspend_via_firmware();6962}6963 6964static int e1000e_pm_suspend(struct device *dev)6965{6966 struct net_device *netdev = pci_get_drvdata(to_pci_dev(dev));6967 struct e1000_adapter *adapter = netdev_priv(netdev);6968 struct pci_dev *pdev = to_pci_dev(dev);6969 int rc;6970 6971 e1000e_flush_lpic(pdev);6972 6973 e1000e_pm_freeze(dev);6974 6975 rc = __e1000_shutdown(pdev, false);6976 if (!rc) {6977 /* Introduce S0ix implementation */6978 if (adapter->flags2 & FLAG2_ENABLE_S0IX_FLOWS)6979 e1000e_s0ix_entry_flow(adapter);6980 }6981 6982 return 0;6983}6984 6985static int e1000e_pm_resume(struct device *dev)6986{6987 struct net_device *netdev = pci_get_drvdata(to_pci_dev(dev));6988 struct e1000_adapter *adapter = netdev_priv(netdev);6989 struct pci_dev *pdev = to_pci_dev(dev);6990 int rc;6991 6992 /* Introduce S0ix implementation */6993 if (adapter->flags2 & FLAG2_ENABLE_S0IX_FLOWS)6994 e1000e_s0ix_exit_flow(adapter);6995 6996 rc = __e1000_resume(pdev);6997 if (rc)6998 return rc;6999 7000 return e1000e_pm_thaw(dev);7001}7002 7003static __maybe_unused int e1000e_pm_runtime_idle(struct device *dev)7004{7005 struct net_device *netdev = dev_get_drvdata(dev);7006 struct e1000_adapter *adapter = netdev_priv(netdev);7007 u16 eee_lp;7008 7009 eee_lp = adapter->hw.dev_spec.ich8lan.eee_lp_ability;7010 7011 if (!e1000e_has_link(adapter)) {7012 adapter->hw.dev_spec.ich8lan.eee_lp_ability = eee_lp;7013 pm_schedule_suspend(dev, 5 * MSEC_PER_SEC);7014 }7015 7016 return -EBUSY;7017}7018 7019static int e1000e_pm_runtime_resume(struct device *dev)7020{7021 struct pci_dev *pdev = to_pci_dev(dev);7022 struct net_device *netdev = pci_get_drvdata(pdev);7023 struct e1000_adapter *adapter = netdev_priv(netdev);7024 int rc;7025 7026 pdev->pme_poll = true;7027 7028 rc = __e1000_resume(pdev);7029 if (rc)7030 return rc;7031 7032 if (netdev->flags & IFF_UP)7033 e1000e_up(adapter);7034 7035 return rc;7036}7037 7038static int e1000e_pm_runtime_suspend(struct device *dev)7039{7040 struct pci_dev *pdev = to_pci_dev(dev);7041 struct net_device *netdev = pci_get_drvdata(pdev);7042 struct e1000_adapter *adapter = netdev_priv(netdev);7043 7044 if (netdev->flags & IFF_UP) {7045 int count = E1000_CHECK_RESET_COUNT;7046 7047 while (test_bit(__E1000_RESETTING, &adapter->state) && count--)7048 usleep_range(10000, 11000);7049 7050 WARN_ON(test_bit(__E1000_RESETTING, &adapter->state));7051 7052 /* Down the device without resetting the hardware */7053 e1000e_down(adapter, false);7054 }7055 7056 if (__e1000_shutdown(pdev, true)) {7057 e1000e_pm_runtime_resume(dev);7058 return -EBUSY;7059 }7060 7061 return 0;7062}7063 7064static void e1000_shutdown(struct pci_dev *pdev)7065{7066 e1000e_flush_lpic(pdev);7067 7068 e1000e_pm_freeze(&pdev->dev);7069 7070 __e1000_shutdown(pdev, false);7071}7072 7073#ifdef CONFIG_NET_POLL_CONTROLLER7074 7075static irqreturn_t e1000_intr_msix(int __always_unused irq, void *data)7076{7077 struct net_device *netdev = data;7078 struct e1000_adapter *adapter = netdev_priv(netdev);7079 7080 if (adapter->msix_entries) {7081 int vector, msix_irq;7082 7083 vector = 0;7084 msix_irq = adapter->msix_entries[vector].vector;7085 if (disable_hardirq(msix_irq))7086 e1000_intr_msix_rx(msix_irq, netdev);7087 enable_irq(msix_irq);7088 7089 vector++;7090 msix_irq = adapter->msix_entries[vector].vector;7091 if (disable_hardirq(msix_irq))7092 e1000_intr_msix_tx(msix_irq, netdev);7093 enable_irq(msix_irq);7094 7095 vector++;7096 msix_irq = adapter->msix_entries[vector].vector;7097 if (disable_hardirq(msix_irq))7098 e1000_msix_other(msix_irq, netdev);7099 enable_irq(msix_irq);7100 }7101 7102 return IRQ_HANDLED;7103}7104 7105/**7106 * e1000_netpoll7107 * @netdev: network interface device structure7108 *7109 * Polling 'interrupt' - used by things like netconsole to send skbs7110 * without having to re-enable interrupts. It's not called while7111 * the interrupt routine is executing.7112 */7113static void e1000_netpoll(struct net_device *netdev)7114{7115 struct e1000_adapter *adapter = netdev_priv(netdev);7116 7117 switch (adapter->int_mode) {7118 case E1000E_INT_MODE_MSIX:7119 e1000_intr_msix(adapter->pdev->irq, netdev);7120 break;7121 case E1000E_INT_MODE_MSI:7122 if (disable_hardirq(adapter->pdev->irq))7123 e1000_intr_msi(adapter->pdev->irq, netdev);7124 enable_irq(adapter->pdev->irq);7125 break;7126 default: /* E1000E_INT_MODE_LEGACY */7127 if (disable_hardirq(adapter->pdev->irq))7128 e1000_intr(adapter->pdev->irq, netdev);7129 enable_irq(adapter->pdev->irq);7130 break;7131 }7132}7133#endif7134 7135/**7136 * e1000_io_error_detected - called when PCI error is detected7137 * @pdev: Pointer to PCI device7138 * @state: The current pci connection state7139 *7140 * This function is called after a PCI bus error affecting7141 * this device has been detected.7142 */7143static pci_ers_result_t e1000_io_error_detected(struct pci_dev *pdev,7144 pci_channel_state_t state)7145{7146 e1000e_pm_freeze(&pdev->dev);7147 7148 if (state == pci_channel_io_perm_failure)7149 return PCI_ERS_RESULT_DISCONNECT;7150 7151 pci_disable_device(pdev);7152 7153 /* Request a slot reset. */7154 return PCI_ERS_RESULT_NEED_RESET;7155}7156 7157/**7158 * e1000_io_slot_reset - called after the pci bus has been reset.7159 * @pdev: Pointer to PCI device7160 *7161 * Restart the card from scratch, as if from a cold-boot. Implementation7162 * resembles the first-half of the e1000e_pm_resume routine.7163 */7164static pci_ers_result_t e1000_io_slot_reset(struct pci_dev *pdev)7165{7166 struct net_device *netdev = pci_get_drvdata(pdev);7167 struct e1000_adapter *adapter = netdev_priv(netdev);7168 struct e1000_hw *hw = &adapter->hw;7169 u16 aspm_disable_flag = 0;7170 int err;7171 pci_ers_result_t result;7172 7173 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L0S)7174 aspm_disable_flag = PCIE_LINK_STATE_L0S;7175 if (adapter->flags2 & FLAG2_DISABLE_ASPM_L1)7176 aspm_disable_flag |= PCIE_LINK_STATE_L1;7177 if (aspm_disable_flag)7178 e1000e_disable_aspm_locked(pdev, aspm_disable_flag);7179 7180 err = pci_enable_device_mem(pdev);7181 if (err) {7182 dev_err(&pdev->dev,7183 "Cannot re-enable PCI device after reset.\n");7184 result = PCI_ERS_RESULT_DISCONNECT;7185 } else {7186 pdev->state_saved = true;7187 pci_restore_state(pdev);7188 pci_set_master(pdev);7189 7190 pci_enable_wake(pdev, PCI_D3hot, 0);7191 pci_enable_wake(pdev, PCI_D3cold, 0);7192 7193 e1000e_reset(adapter);7194 ew32(WUS, ~0);7195 result = PCI_ERS_RESULT_RECOVERED;7196 }7197 7198 return result;7199}7200 7201/**7202 * e1000_io_resume - called when traffic can start flowing again.7203 * @pdev: Pointer to PCI device7204 *7205 * This callback is called when the error recovery driver tells us that7206 * its OK to resume normal operation. Implementation resembles the7207 * second-half of the e1000e_pm_resume routine.7208 */7209static void e1000_io_resume(struct pci_dev *pdev)7210{7211 struct net_device *netdev = pci_get_drvdata(pdev);7212 struct e1000_adapter *adapter = netdev_priv(netdev);7213 7214 e1000_init_manageability_pt(adapter);7215 7216 e1000e_pm_thaw(&pdev->dev);7217 7218 /* If the controller has AMT, do not set DRV_LOAD until the interface7219 * is up. For all other cases, let the f/w know that the h/w is now7220 * under the control of the driver.7221 */7222 if (!(adapter->flags & FLAG_HAS_AMT))7223 e1000e_get_hw_control(adapter);7224}7225 7226static void e1000_print_device_info(struct e1000_adapter *adapter)7227{7228 struct e1000_hw *hw = &adapter->hw;7229 struct net_device *netdev = adapter->netdev;7230 u32 ret_val;7231 u8 pba_str[E1000_PBANUM_LENGTH];7232 7233 /* print bus type/speed/width info */7234 e_info("(PCI Express:2.5GT/s:%s) %pM\n",7235 /* bus width */7236 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :7237 "Width x1"),7238 /* MAC address */7239 netdev->dev_addr);7240 e_info("Intel(R) PRO/%s Network Connection\n",7241 (hw->phy.type == e1000_phy_ife) ? "10/100" : "1000");7242 ret_val = e1000_read_pba_string_generic(hw, pba_str,7243 E1000_PBANUM_LENGTH);7244 if (ret_val)7245 strscpy((char *)pba_str, "Unknown", sizeof(pba_str));7246 e_info("MAC: %d, PHY: %d, PBA No: %s\n",7247 hw->mac.type, hw->phy.type, pba_str);7248}7249 7250static void e1000_eeprom_checks(struct e1000_adapter *adapter)7251{7252 struct e1000_hw *hw = &adapter->hw;7253 int ret_val;7254 u16 buf = 0;7255 7256 if (hw->mac.type != e1000_82573)7257 return;7258 7259 ret_val = e1000_read_nvm(hw, NVM_INIT_CONTROL2_REG, 1, &buf);7260 le16_to_cpus(&buf);7261 if (!ret_val && (!(buf & BIT(0)))) {7262 /* Deep Smart Power Down (DSPD) */7263 dev_warn(&adapter->pdev->dev,7264 "Warning: detected DSPD enabled in EEPROM\n");7265 }7266}7267 7268static netdev_features_t e1000_fix_features(struct net_device *netdev,7269 netdev_features_t features)7270{7271 struct e1000_adapter *adapter = netdev_priv(netdev);7272 struct e1000_hw *hw = &adapter->hw;7273 7274 /* Jumbo frame workaround on 82579 and newer requires CRC be stripped */7275 if ((hw->mac.type >= e1000_pch2lan) && (netdev->mtu > ETH_DATA_LEN))7276 features &= ~NETIF_F_RXFCS;7277 7278 /* Since there is no support for separate Rx/Tx vlan accel7279 * enable/disable make sure Tx flag is always in same state as Rx.7280 */7281 if (features & NETIF_F_HW_VLAN_CTAG_RX)7282 features |= NETIF_F_HW_VLAN_CTAG_TX;7283 else7284 features &= ~NETIF_F_HW_VLAN_CTAG_TX;7285 7286 return features;7287}7288 7289static int e1000_set_features(struct net_device *netdev,7290 netdev_features_t features)7291{7292 struct e1000_adapter *adapter = netdev_priv(netdev);7293 netdev_features_t changed = features ^ netdev->features;7294 7295 if (changed & (NETIF_F_TSO | NETIF_F_TSO6))7296 adapter->flags |= FLAG_TSO_FORCE;7297 7298 if (!(changed & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX |7299 NETIF_F_RXCSUM | NETIF_F_RXHASH | NETIF_F_RXFCS |7300 NETIF_F_RXALL)))7301 return 0;7302 7303 if (changed & NETIF_F_RXFCS) {7304 if (features & NETIF_F_RXFCS) {7305 adapter->flags2 &= ~FLAG2_CRC_STRIPPING;7306 } else {7307 /* We need to take it back to defaults, which might mean7308 * stripping is still disabled at the adapter level.7309 */7310 if (adapter->flags2 & FLAG2_DFLT_CRC_STRIPPING)7311 adapter->flags2 |= FLAG2_CRC_STRIPPING;7312 else7313 adapter->flags2 &= ~FLAG2_CRC_STRIPPING;7314 }7315 }7316 7317 netdev->features = features;7318 7319 if (netif_running(netdev))7320 e1000e_reinit_locked(adapter);7321 else7322 e1000e_reset(adapter);7323 7324 return 1;7325}7326 7327static const struct net_device_ops e1000e_netdev_ops = {7328 .ndo_open = e1000e_open,7329 .ndo_stop = e1000e_close,7330 .ndo_start_xmit = e1000_xmit_frame,7331 .ndo_get_stats64 = e1000e_get_stats64,7332 .ndo_set_rx_mode = e1000e_set_rx_mode,7333 .ndo_set_mac_address = e1000_set_mac,7334 .ndo_change_mtu = e1000_change_mtu,7335 .ndo_eth_ioctl = e1000_ioctl,7336 .ndo_tx_timeout = e1000_tx_timeout,7337 .ndo_validate_addr = eth_validate_addr,7338 7339 .ndo_vlan_rx_add_vid = e1000_vlan_rx_add_vid,7340 .ndo_vlan_rx_kill_vid = e1000_vlan_rx_kill_vid,7341#ifdef CONFIG_NET_POLL_CONTROLLER7342 .ndo_poll_controller = e1000_netpoll,7343#endif7344 .ndo_set_features = e1000_set_features,7345 .ndo_fix_features = e1000_fix_features,7346 .ndo_features_check = passthru_features_check,7347};7348 7349/**7350 * e1000_probe - Device Initialization Routine7351 * @pdev: PCI device information struct7352 * @ent: entry in e1000_pci_tbl7353 *7354 * Returns 0 on success, negative on failure7355 *7356 * e1000_probe initializes an adapter identified by a pci_dev structure.7357 * The OS initialization, configuring of the adapter private structure,7358 * and a hardware reset occur.7359 **/7360static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent)7361{7362 struct net_device *netdev;7363 struct e1000_adapter *adapter;7364 struct e1000_hw *hw;7365 const struct e1000_info *ei = e1000_info_tbl[ent->driver_data];7366 resource_size_t mmio_start, mmio_len;7367 resource_size_t flash_start, flash_len;7368 static int cards_found;7369 u16 aspm_disable_flag = 0;7370 u16 eeprom_data = 0;7371 u16 eeprom_apme_mask = E1000_EEPROM_APME;7372 int bars, i, err;7373 s32 ret_val = 0;7374 7375 if (ei->flags2 & FLAG2_DISABLE_ASPM_L0S)7376 aspm_disable_flag = PCIE_LINK_STATE_L0S;7377 if (ei->flags2 & FLAG2_DISABLE_ASPM_L1)7378 aspm_disable_flag |= PCIE_LINK_STATE_L1;7379 if (aspm_disable_flag)7380 e1000e_disable_aspm(pdev, aspm_disable_flag);7381 7382 err = pci_enable_device_mem(pdev);7383 if (err)7384 return err;7385 7386 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));7387 if (err) {7388 dev_err(&pdev->dev,7389 "No usable DMA configuration, aborting\n");7390 goto err_dma;7391 }7392 7393 bars = pci_select_bars(pdev, IORESOURCE_MEM);7394 err = pci_request_selected_regions_exclusive(pdev, bars,7395 e1000e_driver_name);7396 if (err)7397 goto err_pci_reg;7398 7399 pci_set_master(pdev);7400 /* PCI config space info */7401 err = pci_save_state(pdev);7402 if (err)7403 goto err_alloc_etherdev;7404 7405 err = -ENOMEM;7406 netdev = alloc_etherdev(sizeof(struct e1000_adapter));7407 if (!netdev)7408 goto err_alloc_etherdev;7409 7410 SET_NETDEV_DEV(netdev, &pdev->dev);7411 7412 netdev->irq = pdev->irq;7413 7414 pci_set_drvdata(pdev, netdev);7415 adapter = netdev_priv(netdev);7416 hw = &adapter->hw;7417 adapter->netdev = netdev;7418 adapter->pdev = pdev;7419 adapter->ei = ei;7420 adapter->pba = ei->pba;7421 adapter->flags = ei->flags;7422 adapter->flags2 = ei->flags2;7423 adapter->hw.adapter = adapter;7424 adapter->hw.mac.type = ei->mac;7425 adapter->max_hw_frame_size = ei->max_hw_frame_size;7426 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);7427 7428 mmio_start = pci_resource_start(pdev, 0);7429 mmio_len = pci_resource_len(pdev, 0);7430 7431 err = -EIO;7432 adapter->hw.hw_addr = ioremap(mmio_start, mmio_len);7433 if (!adapter->hw.hw_addr)7434 goto err_ioremap;7435 7436 if ((adapter->flags & FLAG_HAS_FLASH) &&7437 (pci_resource_flags(pdev, 1) & IORESOURCE_MEM) &&7438 (hw->mac.type < e1000_pch_spt)) {7439 flash_start = pci_resource_start(pdev, 1);7440 flash_len = pci_resource_len(pdev, 1);7441 adapter->hw.flash_address = ioremap(flash_start, flash_len);7442 if (!adapter->hw.flash_address)7443 goto err_flashmap;7444 }7445 7446 /* Set default EEE advertisement */7447 if (adapter->flags2 & FLAG2_HAS_EEE)7448 adapter->eee_advert = MDIO_EEE_100TX | MDIO_EEE_1000T;7449 7450 /* construct the net_device struct */7451 netdev->netdev_ops = &e1000e_netdev_ops;7452 e1000e_set_ethtool_ops(netdev);7453 netdev->watchdog_timeo = 5 * HZ;7454 netif_napi_add(netdev, &adapter->napi, e1000e_poll);7455 strscpy(netdev->name, pci_name(pdev), sizeof(netdev->name));7456 7457 netdev->mem_start = mmio_start;7458 netdev->mem_end = mmio_start + mmio_len;7459 7460 adapter->bd_number = cards_found++;7461 7462 e1000e_check_options(adapter);7463 7464 /* setup adapter struct */7465 err = e1000_sw_init(adapter);7466 if (err)7467 goto err_sw_init;7468 7469 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));7470 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));7471 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));7472 7473 err = ei->get_variants(adapter);7474 if (err)7475 goto err_hw_init;7476 7477 if ((adapter->flags & FLAG_IS_ICH) &&7478 (adapter->flags & FLAG_READ_ONLY_NVM) &&7479 (hw->mac.type < e1000_pch_spt))7480 e1000e_write_protect_nvm_ich8lan(&adapter->hw);7481 7482 hw->mac.ops.get_bus_info(&adapter->hw);7483 7484 adapter->hw.phy.autoneg_wait_to_complete = 0;7485 7486 /* Copper options */7487 if (adapter->hw.phy.media_type == e1000_media_type_copper) {7488 adapter->hw.phy.mdix = AUTO_ALL_MODES;7489 adapter->hw.phy.disable_polarity_correction = 0;7490 adapter->hw.phy.ms_type = e1000_ms_hw_default;7491 }7492 7493 if (hw->phy.ops.check_reset_block && hw->phy.ops.check_reset_block(hw))7494 dev_info(&pdev->dev,7495 "PHY reset is blocked due to SOL/IDER session.\n");7496 7497 /* Set initial default active device features */7498 netdev->features = (NETIF_F_SG |7499 NETIF_F_HW_VLAN_CTAG_RX |7500 NETIF_F_HW_VLAN_CTAG_TX |7501 NETIF_F_TSO |7502 NETIF_F_TSO6 |7503 NETIF_F_RXHASH |7504 NETIF_F_RXCSUM |7505 NETIF_F_HW_CSUM);7506 7507 /* disable TSO for pcie and 10/100 speeds to avoid7508 * some hardware issues and for i219 to fix transfer7509 * speed being capped at 60%7510 */7511 if (!(adapter->flags & FLAG_TSO_FORCE)) {7512 switch (adapter->link_speed) {7513 case SPEED_10:7514 case SPEED_100:7515 e_info("10/100 speed: disabling TSO\n");7516 netdev->features &= ~NETIF_F_TSO;7517 netdev->features &= ~NETIF_F_TSO6;7518 break;7519 case SPEED_1000:7520 netdev->features |= NETIF_F_TSO;7521 netdev->features |= NETIF_F_TSO6;7522 break;7523 default:7524 /* oops */7525 break;7526 }7527 if (hw->mac.type == e1000_pch_spt) {7528 netdev->features &= ~NETIF_F_TSO;7529 netdev->features &= ~NETIF_F_TSO6;7530 }7531 }7532 7533 /* Set user-changeable features (subset of all device features) */7534 netdev->hw_features = netdev->features;7535 netdev->hw_features |= NETIF_F_RXFCS;7536 netdev->priv_flags |= IFF_SUPP_NOFCS;7537 netdev->hw_features |= NETIF_F_RXALL;7538 7539 if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER)7540 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;7541 7542 netdev->vlan_features |= (NETIF_F_SG |7543 NETIF_F_TSO |7544 NETIF_F_TSO6 |7545 NETIF_F_HW_CSUM);7546 7547 netdev->priv_flags |= IFF_UNICAST_FLT;7548 7549 netdev->features |= NETIF_F_HIGHDMA;7550 netdev->vlan_features |= NETIF_F_HIGHDMA;7551 7552 /* MTU range: 68 - max_hw_frame_size */7553 netdev->min_mtu = ETH_MIN_MTU;7554 netdev->max_mtu = adapter->max_hw_frame_size -7555 (VLAN_ETH_HLEN + ETH_FCS_LEN);7556 7557 if (e1000e_enable_mng_pass_thru(&adapter->hw))7558 adapter->flags |= FLAG_MNG_PT_ENABLED;7559 7560 /* before reading the NVM, reset the controller to7561 * put the device in a known good starting state7562 */7563 adapter->hw.mac.ops.reset_hw(&adapter->hw);7564 7565 /* systems with ASPM and others may see the checksum fail on the first7566 * attempt. Let's give it a few tries7567 */7568 for (i = 0;; i++) {7569 if (e1000_validate_nvm_checksum(&adapter->hw) >= 0)7570 break;7571 if (i == 2) {7572 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");7573 err = -EIO;7574 goto err_eeprom;7575 }7576 }7577 7578 e1000_eeprom_checks(adapter);7579 7580 /* copy the MAC address */7581 if (e1000e_read_mac_addr(&adapter->hw))7582 dev_err(&pdev->dev,7583 "NVM Read Error while reading MAC address\n");7584 7585 eth_hw_addr_set(netdev, adapter->hw.mac.addr);7586 7587 if (!is_valid_ether_addr(netdev->dev_addr)) {7588 dev_err(&pdev->dev, "Invalid MAC Address: %pM\n",7589 netdev->dev_addr);7590 err = -EIO;7591 goto err_eeprom;7592 }7593 7594 timer_setup(&adapter->watchdog_timer, e1000_watchdog, 0);7595 timer_setup(&adapter->phy_info_timer, e1000_update_phy_info, 0);7596 7597 INIT_WORK(&adapter->reset_task, e1000_reset_task);7598 INIT_WORK(&adapter->watchdog_task, e1000_watchdog_task);7599 INIT_WORK(&adapter->downshift_task, e1000e_downshift_workaround);7600 INIT_WORK(&adapter->update_phy_task, e1000e_update_phy_task);7601 INIT_WORK(&adapter->print_hang_task, e1000_print_hw_hang);7602 7603 /* Initialize link parameters. User can change them with ethtool */7604 adapter->hw.mac.autoneg = 1;7605 adapter->fc_autoneg = true;7606 adapter->hw.fc.requested_mode = e1000_fc_default;7607 adapter->hw.fc.current_mode = e1000_fc_default;7608 adapter->hw.phy.autoneg_advertised = 0x2f;7609 7610 /* Initial Wake on LAN setting - If APM wake is enabled in7611 * the EEPROM, enable the ACPI Magic Packet filter7612 */7613 if (adapter->flags & FLAG_APME_IN_WUC) {7614 /* APME bit in EEPROM is mapped to WUC.APME */7615 eeprom_data = er32(WUC);7616 eeprom_apme_mask = E1000_WUC_APME;7617 if ((hw->mac.type > e1000_ich10lan) &&7618 (eeprom_data & E1000_WUC_PHY_WAKE))7619 adapter->flags2 |= FLAG2_HAS_PHY_WAKEUP;7620 } else if (adapter->flags & FLAG_APME_IN_CTRL3) {7621 if (adapter->flags & FLAG_APME_CHECK_PORT_B &&7622 (adapter->hw.bus.func == 1))7623 ret_val = e1000_read_nvm(&adapter->hw,7624 NVM_INIT_CONTROL3_PORT_B,7625 1, &eeprom_data);7626 else7627 ret_val = e1000_read_nvm(&adapter->hw,7628 NVM_INIT_CONTROL3_PORT_A,7629 1, &eeprom_data);7630 }7631 7632 /* fetch WoL from EEPROM */7633 if (ret_val)7634 e_dbg("NVM read error getting WoL initial values: %d\n", ret_val);7635 else if (eeprom_data & eeprom_apme_mask)7636 adapter->eeprom_wol |= E1000_WUFC_MAG;7637 7638 /* now that we have the eeprom settings, apply the special cases7639 * where the eeprom may be wrong or the board simply won't support7640 * wake on lan on a particular port7641 */7642 if (!(adapter->flags & FLAG_HAS_WOL))7643 adapter->eeprom_wol = 0;7644 7645 /* initialize the wol settings based on the eeprom settings */7646 adapter->wol = adapter->eeprom_wol;7647 7648 /* make sure adapter isn't asleep if manageability is enabled */7649 if (adapter->wol || (adapter->flags & FLAG_MNG_PT_ENABLED) ||7650 (hw->mac.ops.check_mng_mode(hw)))7651 device_wakeup_enable(&pdev->dev);7652 7653 /* save off EEPROM version number */7654 ret_val = e1000_read_nvm(&adapter->hw, 5, 1, &adapter->eeprom_vers);7655 7656 if (ret_val) {7657 e_dbg("NVM read error getting EEPROM version: %d\n", ret_val);7658 adapter->eeprom_vers = 0;7659 }7660 7661 /* init PTP hardware clock */7662 e1000e_ptp_init(adapter);7663 7664 /* reset the hardware with the new settings */7665 e1000e_reset(adapter);7666 7667 /* If the controller has AMT, do not set DRV_LOAD until the interface7668 * is up. For all other cases, let the f/w know that the h/w is now7669 * under the control of the driver.7670 */7671 if (!(adapter->flags & FLAG_HAS_AMT))7672 e1000e_get_hw_control(adapter);7673 7674 if (hw->mac.type >= e1000_pch_cnp)7675 adapter->flags2 |= FLAG2_ENABLE_S0IX_FLOWS;7676 7677 strscpy(netdev->name, "eth%d", sizeof(netdev->name));7678 err = register_netdev(netdev);7679 if (err)7680 goto err_register;7681 7682 /* carrier off reporting is important to ethtool even BEFORE open */7683 netif_carrier_off(netdev);7684 7685 e1000_print_device_info(adapter);7686 7687 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_SMART_PREPARE);7688 7689 if (pci_dev_run_wake(pdev))7690 pm_runtime_put_noidle(&pdev->dev);7691 7692 return 0;7693 7694err_register:7695 if (!(adapter->flags & FLAG_HAS_AMT))7696 e1000e_release_hw_control(adapter);7697err_eeprom:7698 if (hw->phy.ops.check_reset_block && !hw->phy.ops.check_reset_block(hw))7699 e1000_phy_hw_reset(&adapter->hw);7700err_hw_init:7701 kfree(adapter->tx_ring);7702 kfree(adapter->rx_ring);7703err_sw_init:7704 if ((adapter->hw.flash_address) && (hw->mac.type < e1000_pch_spt))7705 iounmap(adapter->hw.flash_address);7706 e1000e_reset_interrupt_capability(adapter);7707err_flashmap:7708 iounmap(adapter->hw.hw_addr);7709err_ioremap:7710 free_netdev(netdev);7711err_alloc_etherdev:7712 pci_release_mem_regions(pdev);7713err_pci_reg:7714err_dma:7715 pci_disable_device(pdev);7716 return err;7717}7718 7719/**7720 * e1000_remove - Device Removal Routine7721 * @pdev: PCI device information struct7722 *7723 * e1000_remove is called by the PCI subsystem to alert the driver7724 * that it should release a PCI device. This could be caused by a7725 * Hot-Plug event, or because the driver is going to be removed from7726 * memory.7727 **/7728static void e1000_remove(struct pci_dev *pdev)7729{7730 struct net_device *netdev = pci_get_drvdata(pdev);7731 struct e1000_adapter *adapter = netdev_priv(netdev);7732 7733 e1000e_ptp_remove(adapter);7734 7735 /* The timers may be rescheduled, so explicitly disable them7736 * from being rescheduled.7737 */7738 set_bit(__E1000_DOWN, &adapter->state);7739 del_timer_sync(&adapter->watchdog_timer);7740 del_timer_sync(&adapter->phy_info_timer);7741 7742 cancel_work_sync(&adapter->reset_task);7743 cancel_work_sync(&adapter->watchdog_task);7744 cancel_work_sync(&adapter->downshift_task);7745 cancel_work_sync(&adapter->update_phy_task);7746 cancel_work_sync(&adapter->print_hang_task);7747 7748 if (adapter->flags & FLAG_HAS_HW_TIMESTAMP) {7749 cancel_work_sync(&adapter->tx_hwtstamp_work);7750 if (adapter->tx_hwtstamp_skb) {7751 dev_consume_skb_any(adapter->tx_hwtstamp_skb);7752 adapter->tx_hwtstamp_skb = NULL;7753 }7754 }7755 7756 unregister_netdev(netdev);7757 7758 if (pci_dev_run_wake(pdev))7759 pm_runtime_get_noresume(&pdev->dev);7760 7761 /* Release control of h/w to f/w. If f/w is AMT enabled, this7762 * would have already happened in close and is redundant.7763 */7764 e1000e_release_hw_control(adapter);7765 7766 e1000e_reset_interrupt_capability(adapter);7767 kfree(adapter->tx_ring);7768 kfree(adapter->rx_ring);7769 7770 iounmap(adapter->hw.hw_addr);7771 if ((adapter->hw.flash_address) &&7772 (adapter->hw.mac.type < e1000_pch_spt))7773 iounmap(adapter->hw.flash_address);7774 pci_release_mem_regions(pdev);7775 7776 free_netdev(netdev);7777 7778 pci_disable_device(pdev);7779}7780 7781/* PCI Error Recovery (ERS) */7782static const struct pci_error_handlers e1000_err_handler = {7783 .error_detected = e1000_io_error_detected,7784 .slot_reset = e1000_io_slot_reset,7785 .resume = e1000_io_resume,7786};7787 7788static const struct pci_device_id e1000_pci_tbl[] = {7789 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_COPPER), board_82571 },7790 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_FIBER), board_82571 },7791 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER), board_82571 },7792 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER_LP),7793 board_82571 },7794 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_FIBER), board_82571 },7795 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES), board_82571 },7796 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_DUAL), board_82571 },7797 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_QUAD), board_82571 },7798 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82571PT_QUAD_COPPER), board_82571 },7799 7800 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI), board_82572 },7801 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_COPPER), board_82572 },7802 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_FIBER), board_82572 },7803 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_SERDES), board_82572 },7804 7805 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E), board_82573 },7806 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E_IAMT), board_82573 },7807 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82573L), board_82573 },7808 7809 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574L), board_82574 },7810 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82574LA), board_82574 },7811 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82583V), board_82583 },7812 7813 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_DPT),7814 board_80003es2lan },7815 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_SPT),7816 board_80003es2lan },7817 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_DPT),7818 board_80003es2lan },7819 { PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_SPT),7820 board_80003es2lan },7821 7822 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE), board_ich8lan },7823 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_G), board_ich8lan },7824 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_GT), board_ich8lan },7825 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_AMT), board_ich8lan },7826 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_C), board_ich8lan },7827 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M), board_ich8lan },7828 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M_AMT), board_ich8lan },7829 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_82567V_3), board_ich8lan },7830 7831 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE), board_ich9lan },7832 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_G), board_ich9lan },7833 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_GT), board_ich9lan },7834 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_AMT), board_ich9lan },7835 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_C), board_ich9lan },7836 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_BM), board_ich9lan },7837 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M), board_ich9lan },7838 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_AMT), board_ich9lan },7839 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_V), board_ich9lan },7840 7841 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LM), board_ich9lan },7842 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LF), board_ich9lan },7843 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_V), board_ich9lan },7844 7845 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LM), board_ich10lan },7846 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LF), board_ich10lan },7847 { PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_V), board_ich10lan },7848 7849 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LM), board_pchlan },7850 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LC), board_pchlan },7851 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DM), board_pchlan },7852 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DC), board_pchlan },7853 7854 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_LM), board_pch2lan },7855 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_V), board_pch2lan },7856 7857 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_LM), board_pch_lpt },7858 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_V), board_pch_lpt },7859 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_LM), board_pch_lpt },7860 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_V), board_pch_lpt },7861 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_LM2), board_pch_lpt },7862 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_V2), board_pch_lpt },7863 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_LM3), board_pch_lpt },7864 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_V3), board_pch_lpt },7865 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM), board_pch_spt },7866 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V), board_pch_spt },7867 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM2), board_pch_spt },7868 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V2), board_pch_spt },7869 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LBG_I219_LM3), board_pch_spt },7870 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM4), board_pch_spt },7871 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V4), board_pch_spt },7872 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM5), board_pch_spt },7873 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V5), board_pch_spt },7874 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_LM6), board_pch_cnp },7875 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_V6), board_pch_cnp },7876 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_LM7), board_pch_cnp },7877 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_V7), board_pch_cnp },7878 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_LM8), board_pch_cnp },7879 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_V8), board_pch_cnp },7880 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_LM9), board_pch_cnp },7881 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_V9), board_pch_cnp },7882 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM10), board_pch_cnp },7883 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V10), board_pch_cnp },7884 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM11), board_pch_cnp },7885 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V11), board_pch_cnp },7886 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM12), board_pch_spt },7887 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V12), board_pch_spt },7888 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM13), board_pch_tgp },7889 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V13), board_pch_tgp },7890 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM14), board_pch_tgp },7891 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V14), board_pch_tgp },7892 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM15), board_pch_tgp },7893 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V15), board_pch_tgp },7894 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_LM23), board_pch_adp },7895 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_V23), board_pch_adp },7896 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM16), board_pch_adp },7897 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V16), board_pch_adp },7898 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM17), board_pch_adp },7899 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V17), board_pch_adp },7900 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_LM22), board_pch_adp },7901 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_V22), board_pch_adp },7902 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM19), board_pch_adp },7903 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V19), board_pch_adp },7904 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_LM18), board_pch_mtp },7905 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_V18), board_pch_mtp },7906 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_LM20), board_pch_mtp },7907 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_V20), board_pch_mtp },7908 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_LM21), board_pch_mtp },7909 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_V21), board_pch_mtp },7910 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ARL_I219_LM24), board_pch_mtp },7911 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ARL_I219_V24), board_pch_mtp },7912 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_PTP_I219_LM25), board_pch_mtp },7913 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_PTP_I219_V25), board_pch_mtp },7914 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_PTP_I219_LM26), board_pch_mtp },7915 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_PTP_I219_V26), board_pch_mtp },7916 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_PTP_I219_LM27), board_pch_mtp },7917 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_PTP_I219_V27), board_pch_mtp },7918 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_NVL_I219_LM29), board_pch_mtp },7919 { PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_NVL_I219_V29), board_pch_mtp },7920 7921 { 0, 0, 0, 0, 0, 0, 0 } /* terminate list */7922};7923MODULE_DEVICE_TABLE(pci, e1000_pci_tbl);7924 7925static const struct dev_pm_ops e1000e_pm_ops = {7926 .prepare = e1000e_pm_prepare,7927 .suspend = e1000e_pm_suspend,7928 .resume = e1000e_pm_resume,7929 .freeze = e1000e_pm_freeze,7930 .thaw = e1000e_pm_thaw,7931 .poweroff = e1000e_pm_suspend,7932 .restore = e1000e_pm_resume,7933 RUNTIME_PM_OPS(e1000e_pm_runtime_suspend, e1000e_pm_runtime_resume,7934 e1000e_pm_runtime_idle)7935};7936 7937/* PCI Device API Driver */7938static struct pci_driver e1000_driver = {7939 .name = e1000e_driver_name,7940 .id_table = e1000_pci_tbl,7941 .probe = e1000_probe,7942 .remove = e1000_remove,7943 .driver.pm = pm_ptr(&e1000e_pm_ops),7944 .shutdown = e1000_shutdown,7945 .err_handler = &e1000_err_handler7946};7947 7948/**7949 * e1000_init_module - Driver Registration Routine7950 *7951 * e1000_init_module is the first routine called when the driver is7952 * loaded. All it does is register with the PCI subsystem.7953 **/7954static int __init e1000_init_module(void)7955{7956 pr_info("Intel(R) PRO/1000 Network Driver\n");7957 pr_info("Copyright(c) 1999 - 2015 Intel Corporation.\n");7958 7959 return pci_register_driver(&e1000_driver);7960}7961module_init(e1000_init_module);7962 7963/**7964 * e1000_exit_module - Driver Exit Cleanup Routine7965 *7966 * e1000_exit_module is called just before the driver is removed7967 * from memory.7968 **/7969static void __exit e1000_exit_module(void)7970{7971 pci_unregister_driver(&e1000_driver);7972}7973module_exit(e1000_exit_module);7974 7975MODULE_DESCRIPTION("Intel(R) PRO/1000 Network Driver");7976MODULE_LICENSE("GPL v2");7977 7978/* netdev.c */7979