brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.02/* Copyright(c) 2009 - 2018 Intel Corporation. */3 4#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt5 6#include <linux/bitfield.h>7#include <linux/delay.h>8#include <linux/ethtool.h>9#include <linux/if_vlan.h>10#include <linux/init.h>11#include <linux/ipv6.h>12#include <linux/mii.h>13#include <linux/module.h>14#include <linux/netdevice.h>15#include <linux/pagemap.h>16#include <linux/pci.h>17#include <linux/prefetch.h>18#include <linux/sctp.h>19#include <linux/slab.h>20#include <linux/tcp.h>21#include <linux/types.h>22#include <linux/vmalloc.h>23#include <net/checksum.h>24#include <net/ip6_checksum.h>25#include "igbvf.h"26 27char igbvf_driver_name[] = "igbvf";28static const char igbvf_driver_string[] =29		  "Intel(R) Gigabit Virtual Function Network Driver";30static const char igbvf_copyright[] =31		  "Copyright (c) 2009 - 2012 Intel Corporation.";32 33#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)34static int debug = -1;35module_param(debug, int, 0);36MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");37 38static int igbvf_poll(struct napi_struct *napi, int budget);39static void igbvf_reset(struct igbvf_adapter *);40static void igbvf_set_interrupt_capability(struct igbvf_adapter *);41static void igbvf_reset_interrupt_capability(struct igbvf_adapter *);42 43static struct igbvf_info igbvf_vf_info = {44	.mac		= e1000_vfadapt,45	.flags		= 0,46	.pba		= 10,47	.init_ops	= e1000_init_function_pointers_vf,48};49 50static struct igbvf_info igbvf_i350_vf_info = {51	.mac		= e1000_vfadapt_i350,52	.flags		= 0,53	.pba		= 10,54	.init_ops	= e1000_init_function_pointers_vf,55};56 57static const struct igbvf_info *igbvf_info_tbl[] = {58	[board_vf]	= &igbvf_vf_info,59	[board_i350_vf]	= &igbvf_i350_vf_info,60};61 62/**63 * igbvf_desc_unused - calculate if we have unused descriptors64 * @ring: address of receive ring structure65 **/66static int igbvf_desc_unused(struct igbvf_ring *ring)67{68	if (ring->next_to_clean > ring->next_to_use)69		return ring->next_to_clean - ring->next_to_use - 1;70 71	return ring->count + ring->next_to_clean - ring->next_to_use - 1;72}73 74/**75 * igbvf_receive_skb - helper function to handle Rx indications76 * @adapter: board private structure77 * @netdev: pointer to netdev struct78 * @skb: skb to indicate to stack79 * @status: descriptor status field as written by hardware80 * @vlan: descriptor vlan field as written by hardware (no le/be conversion)81 * @skb: pointer to sk_buff to be indicated to stack82 **/83static void igbvf_receive_skb(struct igbvf_adapter *adapter,84			      struct net_device *netdev,85			      struct sk_buff *skb,86			      u32 status, __le16 vlan)87{88	u16 vid;89 90	if (status & E1000_RXD_STAT_VP) {91		if ((adapter->flags & IGBVF_FLAG_RX_LB_VLAN_BSWAP) &&92		    (status & E1000_RXDEXT_STATERR_LB))93			vid = be16_to_cpu((__force __be16)vlan) & E1000_RXD_SPC_VLAN_MASK;94		else95			vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;96		if (test_bit(vid, adapter->active_vlans))97			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);98	}99 100	napi_gro_receive(&adapter->rx_ring->napi, skb);101}102 103static inline void igbvf_rx_checksum_adv(struct igbvf_adapter *adapter,104					 u32 status_err, struct sk_buff *skb)105{106	skb_checksum_none_assert(skb);107 108	/* Ignore Checksum bit is set or checksum is disabled through ethtool */109	if ((status_err & E1000_RXD_STAT_IXSM) ||110	    (adapter->flags & IGBVF_FLAG_RX_CSUM_DISABLED))111		return;112 113	/* TCP/UDP checksum error bit is set */114	if (status_err &115	    (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {116		/* let the stack verify checksum errors */117		adapter->hw_csum_err++;118		return;119	}120 121	/* It must be a TCP or UDP packet with a valid checksum */122	if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))123		skb->ip_summed = CHECKSUM_UNNECESSARY;124 125	adapter->hw_csum_good++;126}127 128/**129 * igbvf_alloc_rx_buffers - Replace used receive buffers; packet split130 * @rx_ring: address of ring structure to repopulate131 * @cleaned_count: number of buffers to repopulate132 **/133static void igbvf_alloc_rx_buffers(struct igbvf_ring *rx_ring,134				   int cleaned_count)135{136	struct igbvf_adapter *adapter = rx_ring->adapter;137	struct net_device *netdev = adapter->netdev;138	struct pci_dev *pdev = adapter->pdev;139	union e1000_adv_rx_desc *rx_desc;140	struct igbvf_buffer *buffer_info;141	struct sk_buff *skb;142	unsigned int i;143	int bufsz;144 145	i = rx_ring->next_to_use;146	buffer_info = &rx_ring->buffer_info[i];147 148	if (adapter->rx_ps_hdr_size)149		bufsz = adapter->rx_ps_hdr_size;150	else151		bufsz = adapter->rx_buffer_len;152 153	while (cleaned_count--) {154		rx_desc = IGBVF_RX_DESC_ADV(*rx_ring, i);155 156		if (adapter->rx_ps_hdr_size && !buffer_info->page_dma) {157			if (!buffer_info->page) {158				buffer_info->page = alloc_page(GFP_ATOMIC);159				if (!buffer_info->page) {160					adapter->alloc_rx_buff_failed++;161					goto no_buffers;162				}163				buffer_info->page_offset = 0;164			} else {165				buffer_info->page_offset ^= PAGE_SIZE / 2;166			}167			buffer_info->page_dma =168				dma_map_page(&pdev->dev, buffer_info->page,169					     buffer_info->page_offset,170					     PAGE_SIZE / 2,171					     DMA_FROM_DEVICE);172			if (dma_mapping_error(&pdev->dev,173					      buffer_info->page_dma)) {174				__free_page(buffer_info->page);175				buffer_info->page = NULL;176				dev_err(&pdev->dev, "RX DMA map failed\n");177				break;178			}179		}180 181		if (!buffer_info->skb) {182			skb = netdev_alloc_skb_ip_align(netdev, bufsz);183			if (!skb) {184				adapter->alloc_rx_buff_failed++;185				goto no_buffers;186			}187 188			buffer_info->skb = skb;189			buffer_info->dma = dma_map_single(&pdev->dev, skb->data,190							  bufsz,191							  DMA_FROM_DEVICE);192			if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {193				dev_kfree_skb(buffer_info->skb);194				buffer_info->skb = NULL;195				dev_err(&pdev->dev, "RX DMA map failed\n");196				goto no_buffers;197			}198		}199		/* Refresh the desc even if buffer_addrs didn't change because200		 * each write-back erases this info.201		 */202		if (adapter->rx_ps_hdr_size) {203			rx_desc->read.pkt_addr =204			     cpu_to_le64(buffer_info->page_dma);205			rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);206		} else {207			rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);208			rx_desc->read.hdr_addr = 0;209		}210 211		i++;212		if (i == rx_ring->count)213			i = 0;214		buffer_info = &rx_ring->buffer_info[i];215	}216 217no_buffers:218	if (rx_ring->next_to_use != i) {219		rx_ring->next_to_use = i;220		if (i == 0)221			i = (rx_ring->count - 1);222		else223			i--;224 225		/* Force memory writes to complete before letting h/w226		 * know there are new descriptors to fetch.  (Only227		 * applicable for weak-ordered memory model archs,228		 * such as IA-64).229		*/230		wmb();231		writel(i, adapter->hw.hw_addr + rx_ring->tail);232	}233}234 235/**236 * igbvf_clean_rx_irq - Send received data up the network stack; legacy237 * @adapter: board private structure238 * @work_done: output parameter used to indicate completed work239 * @work_to_do: input parameter setting limit of work240 *241 * the return value indicates whether actual cleaning was done, there242 * is no guarantee that everything was cleaned243 **/244static bool igbvf_clean_rx_irq(struct igbvf_adapter *adapter,245			       int *work_done, int work_to_do)246{247	struct igbvf_ring *rx_ring = adapter->rx_ring;248	struct net_device *netdev = adapter->netdev;249	struct pci_dev *pdev = adapter->pdev;250	union e1000_adv_rx_desc *rx_desc, *next_rxd;251	struct igbvf_buffer *buffer_info, *next_buffer;252	struct sk_buff *skb;253	bool cleaned = false;254	int cleaned_count = 0;255	unsigned int total_bytes = 0, total_packets = 0;256	unsigned int i;257	u32 length, hlen, staterr;258 259	i = rx_ring->next_to_clean;260	rx_desc = IGBVF_RX_DESC_ADV(*rx_ring, i);261	staterr = le32_to_cpu(rx_desc->wb.upper.status_error);262 263	while (staterr & E1000_RXD_STAT_DD) {264		if (*work_done >= work_to_do)265			break;266		(*work_done)++;267		rmb(); /* read descriptor and rx_buffer_info after status DD */268 269		buffer_info = &rx_ring->buffer_info[i];270 271		/* HW will not DMA in data larger than the given buffer, even272		 * if it parses the (NFS, of course) header to be larger.  In273		 * that case, it fills the header buffer and spills the rest274		 * into the page.275		 */276		hlen = le16_get_bits(rx_desc->wb.lower.lo_dword.hs_rss.hdr_info,277				     E1000_RXDADV_HDRBUFLEN_MASK);278		if (hlen > adapter->rx_ps_hdr_size)279			hlen = adapter->rx_ps_hdr_size;280 281		length = le16_to_cpu(rx_desc->wb.upper.length);282		cleaned = true;283		cleaned_count++;284 285		skb = buffer_info->skb;286		prefetch(skb->data - NET_IP_ALIGN);287		buffer_info->skb = NULL;288		if (!adapter->rx_ps_hdr_size) {289			dma_unmap_single(&pdev->dev, buffer_info->dma,290					 adapter->rx_buffer_len,291					 DMA_FROM_DEVICE);292			buffer_info->dma = 0;293			skb_put(skb, length);294			goto send_up;295		}296 297		if (!skb_shinfo(skb)->nr_frags) {298			dma_unmap_single(&pdev->dev, buffer_info->dma,299					 adapter->rx_ps_hdr_size,300					 DMA_FROM_DEVICE);301			buffer_info->dma = 0;302			skb_put(skb, hlen);303		}304 305		if (length) {306			dma_unmap_page(&pdev->dev, buffer_info->page_dma,307				       PAGE_SIZE / 2,308				       DMA_FROM_DEVICE);309			buffer_info->page_dma = 0;310 311			skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,312					   buffer_info->page,313					   buffer_info->page_offset,314					   length);315 316			if ((adapter->rx_buffer_len > (PAGE_SIZE / 2)) ||317			    (page_count(buffer_info->page) != 1))318				buffer_info->page = NULL;319			else320				get_page(buffer_info->page);321 322			skb->len += length;323			skb->data_len += length;324			skb->truesize += PAGE_SIZE / 2;325		}326send_up:327		i++;328		if (i == rx_ring->count)329			i = 0;330		next_rxd = IGBVF_RX_DESC_ADV(*rx_ring, i);331		prefetch(next_rxd);332		next_buffer = &rx_ring->buffer_info[i];333 334		if (!(staterr & E1000_RXD_STAT_EOP)) {335			buffer_info->skb = next_buffer->skb;336			buffer_info->dma = next_buffer->dma;337			next_buffer->skb = skb;338			next_buffer->dma = 0;339			goto next_desc;340		}341 342		if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {343			dev_kfree_skb_irq(skb);344			goto next_desc;345		}346 347		total_bytes += skb->len;348		total_packets++;349 350		igbvf_rx_checksum_adv(adapter, staterr, skb);351 352		skb->protocol = eth_type_trans(skb, netdev);353 354		igbvf_receive_skb(adapter, netdev, skb, staterr,355				  rx_desc->wb.upper.vlan);356 357next_desc:358		rx_desc->wb.upper.status_error = 0;359 360		/* return some buffers to hardware, one at a time is too slow */361		if (cleaned_count >= IGBVF_RX_BUFFER_WRITE) {362			igbvf_alloc_rx_buffers(rx_ring, cleaned_count);363			cleaned_count = 0;364		}365 366		/* use prefetched values */367		rx_desc = next_rxd;368		buffer_info = next_buffer;369 370		staterr = le32_to_cpu(rx_desc->wb.upper.status_error);371	}372 373	rx_ring->next_to_clean = i;374	cleaned_count = igbvf_desc_unused(rx_ring);375 376	if (cleaned_count)377		igbvf_alloc_rx_buffers(rx_ring, cleaned_count);378 379	adapter->total_rx_packets += total_packets;380	adapter->total_rx_bytes += total_bytes;381	netdev->stats.rx_bytes += total_bytes;382	netdev->stats.rx_packets += total_packets;383	return cleaned;384}385 386static void igbvf_put_txbuf(struct igbvf_adapter *adapter,387			    struct igbvf_buffer *buffer_info)388{389	if (buffer_info->dma) {390		if (buffer_info->mapped_as_page)391			dma_unmap_page(&adapter->pdev->dev,392				       buffer_info->dma,393				       buffer_info->length,394				       DMA_TO_DEVICE);395		else396			dma_unmap_single(&adapter->pdev->dev,397					 buffer_info->dma,398					 buffer_info->length,399					 DMA_TO_DEVICE);400		buffer_info->dma = 0;401	}402	if (buffer_info->skb) {403		dev_kfree_skb_any(buffer_info->skb);404		buffer_info->skb = NULL;405	}406	buffer_info->time_stamp = 0;407}408 409/**410 * igbvf_setup_tx_resources - allocate Tx resources (Descriptors)411 * @adapter: board private structure412 * @tx_ring: ring being initialized413 *414 * Return 0 on success, negative on failure415 **/416int igbvf_setup_tx_resources(struct igbvf_adapter *adapter,417			     struct igbvf_ring *tx_ring)418{419	struct pci_dev *pdev = adapter->pdev;420	int size;421 422	size = sizeof(struct igbvf_buffer) * tx_ring->count;423	tx_ring->buffer_info = vzalloc(size);424	if (!tx_ring->buffer_info)425		goto err;426 427	/* round up to nearest 4K */428	tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);429	tx_ring->size = ALIGN(tx_ring->size, 4096);430 431	tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size,432					   &tx_ring->dma, GFP_KERNEL);433	if (!tx_ring->desc)434		goto err;435 436	tx_ring->adapter = adapter;437	tx_ring->next_to_use = 0;438	tx_ring->next_to_clean = 0;439 440	return 0;441err:442	vfree(tx_ring->buffer_info);443	dev_err(&adapter->pdev->dev,444		"Unable to allocate memory for the transmit descriptor ring\n");445	return -ENOMEM;446}447 448/**449 * igbvf_setup_rx_resources - allocate Rx resources (Descriptors)450 * @adapter: board private structure451 * @rx_ring: ring being initialized452 *453 * Returns 0 on success, negative on failure454 **/455int igbvf_setup_rx_resources(struct igbvf_adapter *adapter,456			     struct igbvf_ring *rx_ring)457{458	struct pci_dev *pdev = adapter->pdev;459	int size, desc_len;460 461	size = sizeof(struct igbvf_buffer) * rx_ring->count;462	rx_ring->buffer_info = vzalloc(size);463	if (!rx_ring->buffer_info)464		goto err;465 466	desc_len = sizeof(union e1000_adv_rx_desc);467 468	/* Round up to nearest 4K */469	rx_ring->size = rx_ring->count * desc_len;470	rx_ring->size = ALIGN(rx_ring->size, 4096);471 472	rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size,473					   &rx_ring->dma, GFP_KERNEL);474	if (!rx_ring->desc)475		goto err;476 477	rx_ring->next_to_clean = 0;478	rx_ring->next_to_use = 0;479 480	rx_ring->adapter = adapter;481 482	return 0;483 484err:485	vfree(rx_ring->buffer_info);486	rx_ring->buffer_info = NULL;487	dev_err(&adapter->pdev->dev,488		"Unable to allocate memory for the receive descriptor ring\n");489	return -ENOMEM;490}491 492/**493 * igbvf_clean_tx_ring - Free Tx Buffers494 * @tx_ring: ring to be cleaned495 **/496static void igbvf_clean_tx_ring(struct igbvf_ring *tx_ring)497{498	struct igbvf_adapter *adapter = tx_ring->adapter;499	struct igbvf_buffer *buffer_info;500	unsigned long size;501	unsigned int i;502 503	if (!tx_ring->buffer_info)504		return;505 506	/* Free all the Tx ring sk_buffs */507	for (i = 0; i < tx_ring->count; i++) {508		buffer_info = &tx_ring->buffer_info[i];509		igbvf_put_txbuf(adapter, buffer_info);510	}511 512	size = sizeof(struct igbvf_buffer) * tx_ring->count;513	memset(tx_ring->buffer_info, 0, size);514 515	/* Zero out the descriptor ring */516	memset(tx_ring->desc, 0, tx_ring->size);517 518	tx_ring->next_to_use = 0;519	tx_ring->next_to_clean = 0;520 521	writel(0, adapter->hw.hw_addr + tx_ring->head);522	writel(0, adapter->hw.hw_addr + tx_ring->tail);523}524 525/**526 * igbvf_free_tx_resources - Free Tx Resources per Queue527 * @tx_ring: ring to free resources from528 *529 * Free all transmit software resources530 **/531void igbvf_free_tx_resources(struct igbvf_ring *tx_ring)532{533	struct pci_dev *pdev = tx_ring->adapter->pdev;534 535	igbvf_clean_tx_ring(tx_ring);536 537	vfree(tx_ring->buffer_info);538	tx_ring->buffer_info = NULL;539 540	dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,541			  tx_ring->dma);542 543	tx_ring->desc = NULL;544}545 546/**547 * igbvf_clean_rx_ring - Free Rx Buffers per Queue548 * @rx_ring: ring structure pointer to free buffers from549 **/550static void igbvf_clean_rx_ring(struct igbvf_ring *rx_ring)551{552	struct igbvf_adapter *adapter = rx_ring->adapter;553	struct igbvf_buffer *buffer_info;554	struct pci_dev *pdev = adapter->pdev;555	unsigned long size;556	unsigned int i;557 558	if (!rx_ring->buffer_info)559		return;560 561	/* Free all the Rx ring sk_buffs */562	for (i = 0; i < rx_ring->count; i++) {563		buffer_info = &rx_ring->buffer_info[i];564		if (buffer_info->dma) {565			if (adapter->rx_ps_hdr_size) {566				dma_unmap_single(&pdev->dev, buffer_info->dma,567						 adapter->rx_ps_hdr_size,568						 DMA_FROM_DEVICE);569			} else {570				dma_unmap_single(&pdev->dev, buffer_info->dma,571						 adapter->rx_buffer_len,572						 DMA_FROM_DEVICE);573			}574			buffer_info->dma = 0;575		}576 577		if (buffer_info->skb) {578			dev_kfree_skb(buffer_info->skb);579			buffer_info->skb = NULL;580		}581 582		if (buffer_info->page) {583			if (buffer_info->page_dma)584				dma_unmap_page(&pdev->dev,585					       buffer_info->page_dma,586					       PAGE_SIZE / 2,587					       DMA_FROM_DEVICE);588			put_page(buffer_info->page);589			buffer_info->page = NULL;590			buffer_info->page_dma = 0;591			buffer_info->page_offset = 0;592		}593	}594 595	size = sizeof(struct igbvf_buffer) * rx_ring->count;596	memset(rx_ring->buffer_info, 0, size);597 598	/* Zero out the descriptor ring */599	memset(rx_ring->desc, 0, rx_ring->size);600 601	rx_ring->next_to_clean = 0;602	rx_ring->next_to_use = 0;603 604	writel(0, adapter->hw.hw_addr + rx_ring->head);605	writel(0, adapter->hw.hw_addr + rx_ring->tail);606}607 608/**609 * igbvf_free_rx_resources - Free Rx Resources610 * @rx_ring: ring to clean the resources from611 *612 * Free all receive software resources613 **/614 615void igbvf_free_rx_resources(struct igbvf_ring *rx_ring)616{617	struct pci_dev *pdev = rx_ring->adapter->pdev;618 619	igbvf_clean_rx_ring(rx_ring);620 621	vfree(rx_ring->buffer_info);622	rx_ring->buffer_info = NULL;623 624	dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,625			  rx_ring->dma);626	rx_ring->desc = NULL;627}628 629/**630 * igbvf_update_itr - update the dynamic ITR value based on statistics631 * @adapter: pointer to adapter632 * @itr_setting: current adapter->itr633 * @packets: the number of packets during this measurement interval634 * @bytes: the number of bytes during this measurement interval635 *636 * Stores a new ITR value based on packets and byte counts during the last637 * interrupt.  The advantage of per interrupt computation is faster updates638 * and more accurate ITR for the current traffic pattern.  Constants in this639 * function were computed based on theoretical maximum wire speed and thresholds640 * were set based on testing data as well as attempting to minimize response641 * time while increasing bulk throughput.642 **/643static enum latency_range igbvf_update_itr(struct igbvf_adapter *adapter,644					   enum latency_range itr_setting,645					   int packets, int bytes)646{647	enum latency_range retval = itr_setting;648 649	if (packets == 0)650		goto update_itr_done;651 652	switch (itr_setting) {653	case lowest_latency:654		/* handle TSO and jumbo frames */655		if (bytes/packets > 8000)656			retval = bulk_latency;657		else if ((packets < 5) && (bytes > 512))658			retval = low_latency;659		break;660	case low_latency:  /* 50 usec aka 20000 ints/s */661		if (bytes > 10000) {662			/* this if handles the TSO accounting */663			if (bytes/packets > 8000)664				retval = bulk_latency;665			else if ((packets < 10) || ((bytes/packets) > 1200))666				retval = bulk_latency;667			else if ((packets > 35))668				retval = lowest_latency;669		} else if (bytes/packets > 2000) {670			retval = bulk_latency;671		} else if (packets <= 2 && bytes < 512) {672			retval = lowest_latency;673		}674		break;675	case bulk_latency: /* 250 usec aka 4000 ints/s */676		if (bytes > 25000) {677			if (packets > 35)678				retval = low_latency;679		} else if (bytes < 6000) {680			retval = low_latency;681		}682		break;683	default:684		break;685	}686 687update_itr_done:688	return retval;689}690 691static int igbvf_range_to_itr(enum latency_range current_range)692{693	int new_itr;694 695	switch (current_range) {696	/* counts and packets in update_itr are dependent on these numbers */697	case lowest_latency:698		new_itr = IGBVF_70K_ITR;699		break;700	case low_latency:701		new_itr = IGBVF_20K_ITR;702		break;703	case bulk_latency:704		new_itr = IGBVF_4K_ITR;705		break;706	default:707		new_itr = IGBVF_START_ITR;708		break;709	}710	return new_itr;711}712 713static void igbvf_set_itr(struct igbvf_adapter *adapter)714{715	u32 new_itr;716 717	adapter->tx_ring->itr_range =718			igbvf_update_itr(adapter,719					 adapter->tx_ring->itr_val,720					 adapter->total_tx_packets,721					 adapter->total_tx_bytes);722 723	/* conservative mode (itr 3) eliminates the lowest_latency setting */724	if (adapter->requested_itr == 3 &&725	    adapter->tx_ring->itr_range == lowest_latency)726		adapter->tx_ring->itr_range = low_latency;727 728	new_itr = igbvf_range_to_itr(adapter->tx_ring->itr_range);729 730	if (new_itr != adapter->tx_ring->itr_val) {731		u32 current_itr = adapter->tx_ring->itr_val;732		/* this attempts to bias the interrupt rate towards Bulk733		 * by adding intermediate steps when interrupt rate is734		 * increasing735		 */736		new_itr = new_itr > current_itr ?737			  min(current_itr + (new_itr >> 2), new_itr) :738			  new_itr;739		adapter->tx_ring->itr_val = new_itr;740 741		adapter->tx_ring->set_itr = 1;742	}743 744	adapter->rx_ring->itr_range =745			igbvf_update_itr(adapter, adapter->rx_ring->itr_val,746					 adapter->total_rx_packets,747					 adapter->total_rx_bytes);748	if (adapter->requested_itr == 3 &&749	    adapter->rx_ring->itr_range == lowest_latency)750		adapter->rx_ring->itr_range = low_latency;751 752	new_itr = igbvf_range_to_itr(adapter->rx_ring->itr_range);753 754	if (new_itr != adapter->rx_ring->itr_val) {755		u32 current_itr = adapter->rx_ring->itr_val;756 757		new_itr = new_itr > current_itr ?758			  min(current_itr + (new_itr >> 2), new_itr) :759			  new_itr;760		adapter->rx_ring->itr_val = new_itr;761 762		adapter->rx_ring->set_itr = 1;763	}764}765 766/**767 * igbvf_clean_tx_irq - Reclaim resources after transmit completes768 * @tx_ring: ring structure to clean descriptors from769 *770 * returns true if ring is completely cleaned771 **/772static bool igbvf_clean_tx_irq(struct igbvf_ring *tx_ring)773{774	struct igbvf_adapter *adapter = tx_ring->adapter;775	struct net_device *netdev = adapter->netdev;776	struct igbvf_buffer *buffer_info;777	struct sk_buff *skb;778	union e1000_adv_tx_desc *tx_desc, *eop_desc;779	unsigned int total_bytes = 0, total_packets = 0;780	unsigned int i, count = 0;781	bool cleaned = false;782 783	i = tx_ring->next_to_clean;784	buffer_info = &tx_ring->buffer_info[i];785	eop_desc = buffer_info->next_to_watch;786 787	do {788		/* if next_to_watch is not set then there is no work pending */789		if (!eop_desc)790			break;791 792		/* prevent any other reads prior to eop_desc */793		smp_rmb();794 795		/* if DD is not set pending work has not been completed */796		if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))797			break;798 799		/* clear next_to_watch to prevent false hangs */800		buffer_info->next_to_watch = NULL;801 802		for (cleaned = false; !cleaned; count++) {803			tx_desc = IGBVF_TX_DESC_ADV(*tx_ring, i);804			cleaned = (tx_desc == eop_desc);805			skb = buffer_info->skb;806 807			if (skb) {808				unsigned int segs, bytecount;809 810				/* gso_segs is currently only valid for tcp */811				segs = skb_shinfo(skb)->gso_segs ?: 1;812				/* multiply data chunks by size of headers */813				bytecount = ((segs - 1) * skb_headlen(skb)) +814					    skb->len;815				total_packets += segs;816				total_bytes += bytecount;817			}818 819			igbvf_put_txbuf(adapter, buffer_info);820			tx_desc->wb.status = 0;821 822			i++;823			if (i == tx_ring->count)824				i = 0;825 826			buffer_info = &tx_ring->buffer_info[i];827		}828 829		eop_desc = buffer_info->next_to_watch;830	} while (count < tx_ring->count);831 832	tx_ring->next_to_clean = i;833 834	if (unlikely(count && netif_carrier_ok(netdev) &&835	    igbvf_desc_unused(tx_ring) >= IGBVF_TX_QUEUE_WAKE)) {836		/* Make sure that anybody stopping the queue after this837		 * sees the new next_to_clean.838		 */839		smp_mb();840		if (netif_queue_stopped(netdev) &&841		    !(test_bit(__IGBVF_DOWN, &adapter->state))) {842			netif_wake_queue(netdev);843			++adapter->restart_queue;844		}845	}846 847	netdev->stats.tx_bytes += total_bytes;848	netdev->stats.tx_packets += total_packets;849	return count < tx_ring->count;850}851 852static irqreturn_t igbvf_msix_other(int irq, void *data)853{854	struct net_device *netdev = data;855	struct igbvf_adapter *adapter = netdev_priv(netdev);856	struct e1000_hw *hw = &adapter->hw;857 858	adapter->int_counter1++;859 860	hw->mac.get_link_status = 1;861	if (!test_bit(__IGBVF_DOWN, &adapter->state))862		mod_timer(&adapter->watchdog_timer, jiffies + 1);863 864	ew32(EIMS, adapter->eims_other);865 866	return IRQ_HANDLED;867}868 869static irqreturn_t igbvf_intr_msix_tx(int irq, void *data)870{871	struct net_device *netdev = data;872	struct igbvf_adapter *adapter = netdev_priv(netdev);873	struct e1000_hw *hw = &adapter->hw;874	struct igbvf_ring *tx_ring = adapter->tx_ring;875 876	if (tx_ring->set_itr) {877		writel(tx_ring->itr_val,878		       adapter->hw.hw_addr + tx_ring->itr_register);879		adapter->tx_ring->set_itr = 0;880	}881 882	adapter->total_tx_bytes = 0;883	adapter->total_tx_packets = 0;884 885	/* auto mask will automatically re-enable the interrupt when we write886	 * EICS887	 */888	if (!igbvf_clean_tx_irq(tx_ring))889		/* Ring was not completely cleaned, so fire another interrupt */890		ew32(EICS, tx_ring->eims_value);891	else892		ew32(EIMS, tx_ring->eims_value);893 894	return IRQ_HANDLED;895}896 897static irqreturn_t igbvf_intr_msix_rx(int irq, void *data)898{899	struct net_device *netdev = data;900	struct igbvf_adapter *adapter = netdev_priv(netdev);901 902	adapter->int_counter0++;903 904	/* Write the ITR value calculated at the end of the905	 * previous interrupt.906	 */907	if (adapter->rx_ring->set_itr) {908		writel(adapter->rx_ring->itr_val,909		       adapter->hw.hw_addr + adapter->rx_ring->itr_register);910		adapter->rx_ring->set_itr = 0;911	}912 913	if (napi_schedule_prep(&adapter->rx_ring->napi)) {914		adapter->total_rx_bytes = 0;915		adapter->total_rx_packets = 0;916		__napi_schedule(&adapter->rx_ring->napi);917	}918 919	return IRQ_HANDLED;920}921 922#define IGBVF_NO_QUEUE -1923 924static void igbvf_assign_vector(struct igbvf_adapter *adapter, int rx_queue,925				int tx_queue, int msix_vector)926{927	struct e1000_hw *hw = &adapter->hw;928	u32 ivar, index;929 930	/* 82576 uses a table-based method for assigning vectors.931	 * Each queue has a single entry in the table to which we write932	 * a vector number along with a "valid" bit.  Sadly, the layout933	 * of the table is somewhat counterintuitive.934	 */935	if (rx_queue > IGBVF_NO_QUEUE) {936		index = (rx_queue >> 1);937		ivar = array_er32(IVAR0, index);938		if (rx_queue & 0x1) {939			/* vector goes into third byte of register */940			ivar = ivar & 0xFF00FFFF;941			ivar |= (msix_vector | E1000_IVAR_VALID) << 16;942		} else {943			/* vector goes into low byte of register */944			ivar = ivar & 0xFFFFFF00;945			ivar |= msix_vector | E1000_IVAR_VALID;946		}947		adapter->rx_ring[rx_queue].eims_value = BIT(msix_vector);948		array_ew32(IVAR0, index, ivar);949	}950	if (tx_queue > IGBVF_NO_QUEUE) {951		index = (tx_queue >> 1);952		ivar = array_er32(IVAR0, index);953		if (tx_queue & 0x1) {954			/* vector goes into high byte of register */955			ivar = ivar & 0x00FFFFFF;956			ivar |= (msix_vector | E1000_IVAR_VALID) << 24;957		} else {958			/* vector goes into second byte of register */959			ivar = ivar & 0xFFFF00FF;960			ivar |= (msix_vector | E1000_IVAR_VALID) << 8;961		}962		adapter->tx_ring[tx_queue].eims_value = BIT(msix_vector);963		array_ew32(IVAR0, index, ivar);964	}965}966 967/**968 * igbvf_configure_msix - Configure MSI-X hardware969 * @adapter: board private structure970 *971 * igbvf_configure_msix sets up the hardware to properly972 * generate MSI-X interrupts.973 **/974static void igbvf_configure_msix(struct igbvf_adapter *adapter)975{976	u32 tmp;977	struct e1000_hw *hw = &adapter->hw;978	struct igbvf_ring *tx_ring = adapter->tx_ring;979	struct igbvf_ring *rx_ring = adapter->rx_ring;980	int vector = 0;981 982	adapter->eims_enable_mask = 0;983 984	igbvf_assign_vector(adapter, IGBVF_NO_QUEUE, 0, vector++);985	adapter->eims_enable_mask |= tx_ring->eims_value;986	writel(tx_ring->itr_val, hw->hw_addr + tx_ring->itr_register);987	igbvf_assign_vector(adapter, 0, IGBVF_NO_QUEUE, vector++);988	adapter->eims_enable_mask |= rx_ring->eims_value;989	writel(rx_ring->itr_val, hw->hw_addr + rx_ring->itr_register);990 991	/* set vector for other causes, i.e. link changes */992 993	tmp = (vector++ | E1000_IVAR_VALID);994 995	ew32(IVAR_MISC, tmp);996 997	adapter->eims_enable_mask = GENMASK(vector - 1, 0);998	adapter->eims_other = BIT(vector - 1);999	e1e_flush();1000}1001 1002static void igbvf_reset_interrupt_capability(struct igbvf_adapter *adapter)1003{1004	if (adapter->msix_entries) {1005		pci_disable_msix(adapter->pdev);1006		kfree(adapter->msix_entries);1007		adapter->msix_entries = NULL;1008	}1009}1010 1011/**1012 * igbvf_set_interrupt_capability - set MSI or MSI-X if supported1013 * @adapter: board private structure1014 *1015 * Attempt to configure interrupts using the best available1016 * capabilities of the hardware and kernel.1017 **/1018static void igbvf_set_interrupt_capability(struct igbvf_adapter *adapter)1019{1020	int err = -ENOMEM;1021	int i;1022 1023	/* we allocate 3 vectors, 1 for Tx, 1 for Rx, one for PF messages */1024	adapter->msix_entries = kcalloc(3, sizeof(struct msix_entry),1025					GFP_KERNEL);1026	if (adapter->msix_entries) {1027		for (i = 0; i < 3; i++)1028			adapter->msix_entries[i].entry = i;1029 1030		err = pci_enable_msix_range(adapter->pdev,1031					    adapter->msix_entries, 3, 3);1032	}1033 1034	if (err < 0) {1035		/* MSI-X failed */1036		dev_err(&adapter->pdev->dev,1037			"Failed to initialize MSI-X interrupts.\n");1038		igbvf_reset_interrupt_capability(adapter);1039	}1040}1041 1042/**1043 * igbvf_request_msix - Initialize MSI-X interrupts1044 * @adapter: board private structure1045 *1046 * igbvf_request_msix allocates MSI-X vectors and requests interrupts from the1047 * kernel.1048 **/1049static int igbvf_request_msix(struct igbvf_adapter *adapter)1050{1051	struct net_device *netdev = adapter->netdev;1052	int err = 0, vector = 0;1053 1054	if (strlen(netdev->name) < (IFNAMSIZ - 5)) {1055		sprintf(adapter->tx_ring->name, "%s-tx-0", netdev->name);1056		sprintf(adapter->rx_ring->name, "%s-rx-0", netdev->name);1057	} else {1058		memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ);1059		memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ);1060	}1061 1062	err = request_irq(adapter->msix_entries[vector].vector,1063			  igbvf_intr_msix_tx, 0, adapter->tx_ring->name,1064			  netdev);1065	if (err)1066		goto out;1067 1068	adapter->tx_ring->itr_register = E1000_EITR(vector);1069	adapter->tx_ring->itr_val = adapter->current_itr;1070	vector++;1071 1072	err = request_irq(adapter->msix_entries[vector].vector,1073			  igbvf_intr_msix_rx, 0, adapter->rx_ring->name,1074			  netdev);1075	if (err)1076		goto free_irq_tx;1077 1078	adapter->rx_ring->itr_register = E1000_EITR(vector);1079	adapter->rx_ring->itr_val = adapter->current_itr;1080	vector++;1081 1082	err = request_irq(adapter->msix_entries[vector].vector,1083			  igbvf_msix_other, 0, netdev->name, netdev);1084	if (err)1085		goto free_irq_rx;1086 1087	igbvf_configure_msix(adapter);1088	return 0;1089free_irq_rx:1090	free_irq(adapter->msix_entries[--vector].vector, netdev);1091free_irq_tx:1092	free_irq(adapter->msix_entries[--vector].vector, netdev);1093out:1094	return err;1095}1096 1097/**1098 * igbvf_alloc_queues - Allocate memory for all rings1099 * @adapter: board private structure to initialize1100 **/1101static int igbvf_alloc_queues(struct igbvf_adapter *adapter)1102{1103	struct net_device *netdev = adapter->netdev;1104 1105	adapter->tx_ring = kzalloc(sizeof(struct igbvf_ring), GFP_KERNEL);1106	if (!adapter->tx_ring)1107		return -ENOMEM;1108 1109	adapter->rx_ring = kzalloc(sizeof(struct igbvf_ring), GFP_KERNEL);1110	if (!adapter->rx_ring) {1111		kfree(adapter->tx_ring);1112		return -ENOMEM;1113	}1114 1115	netif_napi_add(netdev, &adapter->rx_ring->napi, igbvf_poll);1116 1117	return 0;1118}1119 1120/**1121 * igbvf_request_irq - initialize interrupts1122 * @adapter: board private structure1123 *1124 * Attempts to configure interrupts using the best available1125 * capabilities of the hardware and kernel.1126 **/1127static int igbvf_request_irq(struct igbvf_adapter *adapter)1128{1129	int err = -1;1130 1131	/* igbvf supports msi-x only */1132	if (adapter->msix_entries)1133		err = igbvf_request_msix(adapter);1134 1135	if (!err)1136		return err;1137 1138	dev_err(&adapter->pdev->dev,1139		"Unable to allocate interrupt, Error: %d\n", err);1140 1141	return err;1142}1143 1144static void igbvf_free_irq(struct igbvf_adapter *adapter)1145{1146	struct net_device *netdev = adapter->netdev;1147	int vector;1148 1149	if (adapter->msix_entries) {1150		for (vector = 0; vector < 3; vector++)1151			free_irq(adapter->msix_entries[vector].vector, netdev);1152	}1153}1154 1155/**1156 * igbvf_irq_disable - Mask off interrupt generation on the NIC1157 * @adapter: board private structure1158 **/1159static void igbvf_irq_disable(struct igbvf_adapter *adapter)1160{1161	struct e1000_hw *hw = &adapter->hw;1162 1163	ew32(EIMC, ~0);1164 1165	if (adapter->msix_entries)1166		ew32(EIAC, 0);1167}1168 1169/**1170 * igbvf_irq_enable - Enable default interrupt generation settings1171 * @adapter: board private structure1172 **/1173static void igbvf_irq_enable(struct igbvf_adapter *adapter)1174{1175	struct e1000_hw *hw = &adapter->hw;1176 1177	ew32(EIAC, adapter->eims_enable_mask);1178	ew32(EIAM, adapter->eims_enable_mask);1179	ew32(EIMS, adapter->eims_enable_mask);1180}1181 1182/**1183 * igbvf_poll - NAPI Rx polling callback1184 * @napi: struct associated with this polling callback1185 * @budget: amount of packets driver is allowed to process this poll1186 **/1187static int igbvf_poll(struct napi_struct *napi, int budget)1188{1189	struct igbvf_ring *rx_ring = container_of(napi, struct igbvf_ring, napi);1190	struct igbvf_adapter *adapter = rx_ring->adapter;1191	struct e1000_hw *hw = &adapter->hw;1192	int work_done = 0;1193 1194	igbvf_clean_rx_irq(adapter, &work_done, budget);1195 1196	if (work_done == budget)1197		return budget;1198 1199	/* Exit the polling mode, but don't re-enable interrupts if stack might1200	 * poll us due to busy-polling1201	 */1202	if (likely(napi_complete_done(napi, work_done))) {1203		if (adapter->requested_itr & 3)1204			igbvf_set_itr(adapter);1205 1206		if (!test_bit(__IGBVF_DOWN, &adapter->state))1207			ew32(EIMS, adapter->rx_ring->eims_value);1208	}1209 1210	return work_done;1211}1212 1213/**1214 * igbvf_set_rlpml - set receive large packet maximum length1215 * @adapter: board private structure1216 *1217 * Configure the maximum size of packets that will be received1218 */1219static void igbvf_set_rlpml(struct igbvf_adapter *adapter)1220{1221	int max_frame_size;1222	struct e1000_hw *hw = &adapter->hw;1223 1224	max_frame_size = adapter->max_frame_size + VLAN_TAG_SIZE;1225 1226	spin_lock_bh(&hw->mbx_lock);1227 1228	e1000_rlpml_set_vf(hw, max_frame_size);1229 1230	spin_unlock_bh(&hw->mbx_lock);1231}1232 1233static int igbvf_vlan_rx_add_vid(struct net_device *netdev,1234				 __be16 proto, u16 vid)1235{1236	struct igbvf_adapter *adapter = netdev_priv(netdev);1237	struct e1000_hw *hw = &adapter->hw;1238 1239	spin_lock_bh(&hw->mbx_lock);1240 1241	if (hw->mac.ops.set_vfta(hw, vid, true)) {1242		dev_warn(&adapter->pdev->dev, "Vlan id %d\n is not added", vid);1243		spin_unlock_bh(&hw->mbx_lock);1244		return -EINVAL;1245	}1246 1247	spin_unlock_bh(&hw->mbx_lock);1248 1249	set_bit(vid, adapter->active_vlans);1250	return 0;1251}1252 1253static int igbvf_vlan_rx_kill_vid(struct net_device *netdev,1254				  __be16 proto, u16 vid)1255{1256	struct igbvf_adapter *adapter = netdev_priv(netdev);1257	struct e1000_hw *hw = &adapter->hw;1258 1259	spin_lock_bh(&hw->mbx_lock);1260 1261	if (hw->mac.ops.set_vfta(hw, vid, false)) {1262		dev_err(&adapter->pdev->dev,1263			"Failed to remove vlan id %d\n", vid);1264		spin_unlock_bh(&hw->mbx_lock);1265		return -EINVAL;1266	}1267 1268	spin_unlock_bh(&hw->mbx_lock);1269 1270	clear_bit(vid, adapter->active_vlans);1271	return 0;1272}1273 1274static void igbvf_restore_vlan(struct igbvf_adapter *adapter)1275{1276	u16 vid;1277 1278	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)1279		igbvf_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);1280}1281 1282/**1283 * igbvf_configure_tx - Configure Transmit Unit after Reset1284 * @adapter: board private structure1285 *1286 * Configure the Tx unit of the MAC after a reset.1287 **/1288static void igbvf_configure_tx(struct igbvf_adapter *adapter)1289{1290	struct e1000_hw *hw = &adapter->hw;1291	struct igbvf_ring *tx_ring = adapter->tx_ring;1292	u64 tdba;1293	u32 txdctl, dca_txctrl;1294 1295	/* disable transmits */1296	txdctl = er32(TXDCTL(0));1297	ew32(TXDCTL(0), txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);1298	e1e_flush();1299	msleep(10);1300 1301	/* Setup the HW Tx Head and Tail descriptor pointers */1302	ew32(TDLEN(0), tx_ring->count * sizeof(union e1000_adv_tx_desc));1303	tdba = tx_ring->dma;1304	ew32(TDBAL(0), (tdba & DMA_BIT_MASK(32)));1305	ew32(TDBAH(0), (tdba >> 32));1306	ew32(TDH(0), 0);1307	ew32(TDT(0), 0);1308	tx_ring->head = E1000_TDH(0);1309	tx_ring->tail = E1000_TDT(0);1310 1311	/* Turn off Relaxed Ordering on head write-backs.  The writebacks1312	 * MUST be delivered in order or it will completely screw up1313	 * our bookkeeping.1314	 */1315	dca_txctrl = er32(DCA_TXCTRL(0));1316	dca_txctrl &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN;1317	ew32(DCA_TXCTRL(0), dca_txctrl);1318 1319	/* enable transmits */1320	txdctl |= E1000_TXDCTL_QUEUE_ENABLE;1321	ew32(TXDCTL(0), txdctl);1322 1323	/* Setup Transmit Descriptor Settings for eop descriptor */1324	adapter->txd_cmd = E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_IFCS;1325 1326	/* enable Report Status bit */1327	adapter->txd_cmd |= E1000_ADVTXD_DCMD_RS;1328}1329 1330/**1331 * igbvf_setup_srrctl - configure the receive control registers1332 * @adapter: Board private structure1333 **/1334static void igbvf_setup_srrctl(struct igbvf_adapter *adapter)1335{1336	struct e1000_hw *hw = &adapter->hw;1337	u32 srrctl = 0;1338 1339	srrctl &= ~(E1000_SRRCTL_DESCTYPE_MASK |1340		    E1000_SRRCTL_BSIZEHDR_MASK |1341		    E1000_SRRCTL_BSIZEPKT_MASK);1342 1343	/* Enable queue drop to avoid head of line blocking */1344	srrctl |= E1000_SRRCTL_DROP_EN;1345 1346	/* Setup buffer sizes */1347	srrctl |= ALIGN(adapter->rx_buffer_len, 1024) >>1348		  E1000_SRRCTL_BSIZEPKT_SHIFT;1349 1350	if (adapter->rx_buffer_len < 2048) {1351		adapter->rx_ps_hdr_size = 0;1352		srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;1353	} else {1354		adapter->rx_ps_hdr_size = 128;1355		srrctl |= adapter->rx_ps_hdr_size <<1356			  E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;1357		srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;1358	}1359 1360	ew32(SRRCTL(0), srrctl);1361}1362 1363/**1364 * igbvf_configure_rx - Configure Receive Unit after Reset1365 * @adapter: board private structure1366 *1367 * Configure the Rx unit of the MAC after a reset.1368 **/1369static void igbvf_configure_rx(struct igbvf_adapter *adapter)1370{1371	struct e1000_hw *hw = &adapter->hw;1372	struct igbvf_ring *rx_ring = adapter->rx_ring;1373	u64 rdba;1374	u32 rxdctl;1375 1376	/* disable receives */1377	rxdctl = er32(RXDCTL(0));1378	ew32(RXDCTL(0), rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);1379	e1e_flush();1380	msleep(10);1381 1382	/* Setup the HW Rx Head and Tail Descriptor Pointers and1383	 * the Base and Length of the Rx Descriptor Ring1384	 */1385	rdba = rx_ring->dma;1386	ew32(RDBAL(0), (rdba & DMA_BIT_MASK(32)));1387	ew32(RDBAH(0), (rdba >> 32));1388	ew32(RDLEN(0), rx_ring->count * sizeof(union e1000_adv_rx_desc));1389	rx_ring->head = E1000_RDH(0);1390	rx_ring->tail = E1000_RDT(0);1391	ew32(RDH(0), 0);1392	ew32(RDT(0), 0);1393 1394	rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;1395	rxdctl &= 0xFFF00000;1396	rxdctl |= IGBVF_RX_PTHRESH;1397	rxdctl |= IGBVF_RX_HTHRESH << 8;1398	rxdctl |= IGBVF_RX_WTHRESH << 16;1399 1400	igbvf_set_rlpml(adapter);1401 1402	/* enable receives */1403	ew32(RXDCTL(0), rxdctl);1404}1405 1406/**1407 * igbvf_set_multi - Multicast and Promiscuous mode set1408 * @netdev: network interface device structure1409 *1410 * The set_multi entry point is called whenever the multicast address1411 * list or the network interface flags are updated.  This routine is1412 * responsible for configuring the hardware for proper multicast,1413 * promiscuous mode, and all-multi behavior.1414 **/1415static void igbvf_set_multi(struct net_device *netdev)1416{1417	struct igbvf_adapter *adapter = netdev_priv(netdev);1418	struct e1000_hw *hw = &adapter->hw;1419	struct netdev_hw_addr *ha;1420	u8  *mta_list = NULL;1421	int i;1422 1423	if (!netdev_mc_empty(netdev)) {1424		mta_list = kmalloc_array(netdev_mc_count(netdev), ETH_ALEN,1425					 GFP_ATOMIC);1426		if (!mta_list)1427			return;1428	}1429 1430	/* prepare a packed array of only addresses. */1431	i = 0;1432	netdev_for_each_mc_addr(ha, netdev)1433		memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);1434 1435	spin_lock_bh(&hw->mbx_lock);1436 1437	hw->mac.ops.update_mc_addr_list(hw, mta_list, i, 0, 0);1438 1439	spin_unlock_bh(&hw->mbx_lock);1440	kfree(mta_list);1441}1442 1443/**1444 * igbvf_set_uni - Configure unicast MAC filters1445 * @netdev: network interface device structure1446 *1447 * This routine is responsible for configuring the hardware for proper1448 * unicast filters.1449 **/1450static int igbvf_set_uni(struct net_device *netdev)1451{1452	struct igbvf_adapter *adapter = netdev_priv(netdev);1453	struct e1000_hw *hw = &adapter->hw;1454 1455	if (netdev_uc_count(netdev) > IGBVF_MAX_MAC_FILTERS) {1456		pr_err("Too many unicast filters - No Space\n");1457		return -ENOSPC;1458	}1459 1460	spin_lock_bh(&hw->mbx_lock);1461 1462	/* Clear all unicast MAC filters */1463	hw->mac.ops.set_uc_addr(hw, E1000_VF_MAC_FILTER_CLR, NULL);1464 1465	spin_unlock_bh(&hw->mbx_lock);1466 1467	if (!netdev_uc_empty(netdev)) {1468		struct netdev_hw_addr *ha;1469 1470		/* Add MAC filters one by one */1471		netdev_for_each_uc_addr(ha, netdev) {1472			spin_lock_bh(&hw->mbx_lock);1473 1474			hw->mac.ops.set_uc_addr(hw, E1000_VF_MAC_FILTER_ADD,1475						ha->addr);1476 1477			spin_unlock_bh(&hw->mbx_lock);1478			udelay(200);1479		}1480	}1481 1482	return 0;1483}1484 1485static void igbvf_set_rx_mode(struct net_device *netdev)1486{1487	igbvf_set_multi(netdev);1488	igbvf_set_uni(netdev);1489}1490 1491/**1492 * igbvf_configure - configure the hardware for Rx and Tx1493 * @adapter: private board structure1494 **/1495static void igbvf_configure(struct igbvf_adapter *adapter)1496{1497	igbvf_set_rx_mode(adapter->netdev);1498 1499	igbvf_restore_vlan(adapter);1500 1501	igbvf_configure_tx(adapter);1502	igbvf_setup_srrctl(adapter);1503	igbvf_configure_rx(adapter);1504	igbvf_alloc_rx_buffers(adapter->rx_ring,1505			       igbvf_desc_unused(adapter->rx_ring));1506}1507 1508/* igbvf_reset - bring the hardware into a known good state1509 * @adapter: private board structure1510 *1511 * This function boots the hardware and enables some settings that1512 * require a configuration cycle of the hardware - those cannot be1513 * set/changed during runtime. After reset the device needs to be1514 * properly configured for Rx, Tx etc.1515 */1516static void igbvf_reset(struct igbvf_adapter *adapter)1517{1518	struct e1000_mac_info *mac = &adapter->hw.mac;1519	struct net_device *netdev = adapter->netdev;1520	struct e1000_hw *hw = &adapter->hw;1521 1522	spin_lock_bh(&hw->mbx_lock);1523 1524	/* Allow time for pending master requests to run */1525	if (mac->ops.reset_hw(hw))1526		dev_info(&adapter->pdev->dev, "PF still resetting\n");1527 1528	mac->ops.init_hw(hw);1529 1530	spin_unlock_bh(&hw->mbx_lock);1531 1532	if (is_valid_ether_addr(adapter->hw.mac.addr)) {1533		eth_hw_addr_set(netdev, adapter->hw.mac.addr);1534		memcpy(netdev->perm_addr, adapter->hw.mac.addr,1535		       netdev->addr_len);1536	}1537 1538	adapter->last_reset = jiffies;1539}1540 1541int igbvf_up(struct igbvf_adapter *adapter)1542{1543	struct e1000_hw *hw = &adapter->hw;1544 1545	/* hardware has been reset, we need to reload some things */1546	igbvf_configure(adapter);1547 1548	clear_bit(__IGBVF_DOWN, &adapter->state);1549 1550	napi_enable(&adapter->rx_ring->napi);1551	if (adapter->msix_entries)1552		igbvf_configure_msix(adapter);1553 1554	/* Clear any pending interrupts. */1555	er32(EICR);1556	igbvf_irq_enable(adapter);1557 1558	/* start the watchdog */1559	hw->mac.get_link_status = 1;1560	mod_timer(&adapter->watchdog_timer, jiffies + 1);1561 1562	return 0;1563}1564 1565void igbvf_down(struct igbvf_adapter *adapter)1566{1567	struct net_device *netdev = adapter->netdev;1568	struct e1000_hw *hw = &adapter->hw;1569	u32 rxdctl, txdctl;1570 1571	/* signal that we're down so the interrupt handler does not1572	 * reschedule our watchdog timer1573	 */1574	set_bit(__IGBVF_DOWN, &adapter->state);1575 1576	/* disable receives in the hardware */1577	rxdctl = er32(RXDCTL(0));1578	ew32(RXDCTL(0), rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);1579 1580	netif_carrier_off(netdev);1581	netif_stop_queue(netdev);1582 1583	/* disable transmits in the hardware */1584	txdctl = er32(TXDCTL(0));1585	ew32(TXDCTL(0), txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);1586 1587	/* flush both disables and wait for them to finish */1588	e1e_flush();1589	msleep(10);1590 1591	napi_disable(&adapter->rx_ring->napi);1592 1593	igbvf_irq_disable(adapter);1594 1595	del_timer_sync(&adapter->watchdog_timer);1596 1597	/* record the stats before reset*/1598	igbvf_update_stats(adapter);1599 1600	adapter->link_speed = 0;1601	adapter->link_duplex = 0;1602 1603	igbvf_reset(adapter);1604	igbvf_clean_tx_ring(adapter->tx_ring);1605	igbvf_clean_rx_ring(adapter->rx_ring);1606}1607 1608void igbvf_reinit_locked(struct igbvf_adapter *adapter)1609{1610	might_sleep();1611	while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))1612		usleep_range(1000, 2000);1613	igbvf_down(adapter);1614	igbvf_up(adapter);1615	clear_bit(__IGBVF_RESETTING, &adapter->state);1616}1617 1618/**1619 * igbvf_sw_init - Initialize general software structures (struct igbvf_adapter)1620 * @adapter: board private structure to initialize1621 *1622 * igbvf_sw_init initializes the Adapter private data structure.1623 * Fields are initialized based on PCI device information and1624 * OS network device settings (MTU size).1625 **/1626static int igbvf_sw_init(struct igbvf_adapter *adapter)1627{1628	struct net_device *netdev = adapter->netdev;1629	s32 rc;1630 1631	adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN;1632	adapter->rx_ps_hdr_size = 0;1633	adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;1634	adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;1635 1636	adapter->tx_int_delay = 8;1637	adapter->tx_abs_int_delay = 32;1638	adapter->rx_int_delay = 0;1639	adapter->rx_abs_int_delay = 8;1640	adapter->requested_itr = 3;1641	adapter->current_itr = IGBVF_START_ITR;1642 1643	/* Set various function pointers */1644	adapter->ei->init_ops(&adapter->hw);1645 1646	rc = adapter->hw.mac.ops.init_params(&adapter->hw);1647	if (rc)1648		return rc;1649 1650	rc = adapter->hw.mbx.ops.init_params(&adapter->hw);1651	if (rc)1652		return rc;1653 1654	igbvf_set_interrupt_capability(adapter);1655 1656	if (igbvf_alloc_queues(adapter))1657		return -ENOMEM;1658 1659	spin_lock_init(&adapter->tx_queue_lock);1660 1661	/* Explicitly disable IRQ since the NIC can be in any state. */1662	igbvf_irq_disable(adapter);1663 1664	spin_lock_init(&adapter->stats_lock);1665	spin_lock_init(&adapter->hw.mbx_lock);1666 1667	set_bit(__IGBVF_DOWN, &adapter->state);1668	return 0;1669}1670 1671static void igbvf_initialize_last_counter_stats(struct igbvf_adapter *adapter)1672{1673	struct e1000_hw *hw = &adapter->hw;1674 1675	adapter->stats.last_gprc = er32(VFGPRC);1676	adapter->stats.last_gorc = er32(VFGORC);1677	adapter->stats.last_gptc = er32(VFGPTC);1678	adapter->stats.last_gotc = er32(VFGOTC);1679	adapter->stats.last_mprc = er32(VFMPRC);1680	adapter->stats.last_gotlbc = er32(VFGOTLBC);1681	adapter->stats.last_gptlbc = er32(VFGPTLBC);1682	adapter->stats.last_gorlbc = er32(VFGORLBC);1683	adapter->stats.last_gprlbc = er32(VFGPRLBC);1684 1685	adapter->stats.base_gprc = er32(VFGPRC);1686	adapter->stats.base_gorc = er32(VFGORC);1687	adapter->stats.base_gptc = er32(VFGPTC);1688	adapter->stats.base_gotc = er32(VFGOTC);1689	adapter->stats.base_mprc = er32(VFMPRC);1690	adapter->stats.base_gotlbc = er32(VFGOTLBC);1691	adapter->stats.base_gptlbc = er32(VFGPTLBC);1692	adapter->stats.base_gorlbc = er32(VFGORLBC);1693	adapter->stats.base_gprlbc = er32(VFGPRLBC);1694}1695 1696/**1697 * igbvf_open - Called when a network interface is made active1698 * @netdev: network interface device structure1699 *1700 * Returns 0 on success, negative value on failure1701 *1702 * The open entry point is called when a network interface is made1703 * active by the system (IFF_UP).  At this point all resources needed1704 * for transmit and receive operations are allocated, the interrupt1705 * handler is registered with the OS, the watchdog timer is started,1706 * and the stack is notified that the interface is ready.1707 **/1708static int igbvf_open(struct net_device *netdev)1709{1710	struct igbvf_adapter *adapter = netdev_priv(netdev);1711	struct e1000_hw *hw = &adapter->hw;1712	int err;1713 1714	/* disallow open during test */1715	if (test_bit(__IGBVF_TESTING, &adapter->state))1716		return -EBUSY;1717 1718	/* allocate transmit descriptors */1719	err = igbvf_setup_tx_resources(adapter, adapter->tx_ring);1720	if (err)1721		goto err_setup_tx;1722 1723	/* allocate receive descriptors */1724	err = igbvf_setup_rx_resources(adapter, adapter->rx_ring);1725	if (err)1726		goto err_setup_rx;1727 1728	/* before we allocate an interrupt, we must be ready to handle it.1729	 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt1730	 * as soon as we call pci_request_irq, so we have to setup our1731	 * clean_rx handler before we do so.1732	 */1733	igbvf_configure(adapter);1734 1735	err = igbvf_request_irq(adapter);1736	if (err)1737		goto err_req_irq;1738 1739	/* From here on the code is the same as igbvf_up() */1740	clear_bit(__IGBVF_DOWN, &adapter->state);1741 1742	napi_enable(&adapter->rx_ring->napi);1743 1744	/* clear any pending interrupts */1745	er32(EICR);1746 1747	igbvf_irq_enable(adapter);1748 1749	/* start the watchdog */1750	hw->mac.get_link_status = 1;1751	mod_timer(&adapter->watchdog_timer, jiffies + 1);1752 1753	return 0;1754 1755err_req_irq:1756	igbvf_free_rx_resources(adapter->rx_ring);1757err_setup_rx:1758	igbvf_free_tx_resources(adapter->tx_ring);1759err_setup_tx:1760	igbvf_reset(adapter);1761 1762	return err;1763}1764 1765/**1766 * igbvf_close - Disables a network interface1767 * @netdev: network interface device structure1768 *1769 * Returns 0, this is not allowed to fail1770 *1771 * The close entry point is called when an interface is de-activated1772 * by the OS.  The hardware is still under the drivers control, but1773 * needs to be disabled.  A global MAC reset is issued to stop the1774 * hardware, and all transmit and receive resources are freed.1775 **/1776static int igbvf_close(struct net_device *netdev)1777{1778	struct igbvf_adapter *adapter = netdev_priv(netdev);1779 1780	WARN_ON(test_bit(__IGBVF_RESETTING, &adapter->state));1781	igbvf_down(adapter);1782 1783	igbvf_free_irq(adapter);1784 1785	igbvf_free_tx_resources(adapter->tx_ring);1786	igbvf_free_rx_resources(adapter->rx_ring);1787 1788	return 0;1789}1790 1791/**1792 * igbvf_set_mac - Change the Ethernet Address of the NIC1793 * @netdev: network interface device structure1794 * @p: pointer to an address structure1795 *1796 * Returns 0 on success, negative on failure1797 **/1798static int igbvf_set_mac(struct net_device *netdev, void *p)1799{1800	struct igbvf_adapter *adapter = netdev_priv(netdev);1801	struct e1000_hw *hw = &adapter->hw;1802	struct sockaddr *addr = p;1803 1804	if (!is_valid_ether_addr(addr->sa_data))1805		return -EADDRNOTAVAIL;1806 1807	memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);1808 1809	spin_lock_bh(&hw->mbx_lock);1810 1811	hw->mac.ops.rar_set(hw, hw->mac.addr, 0);1812 1813	spin_unlock_bh(&hw->mbx_lock);1814 1815	if (!ether_addr_equal(addr->sa_data, hw->mac.addr))1816		return -EADDRNOTAVAIL;1817 1818	eth_hw_addr_set(netdev, addr->sa_data);1819 1820	return 0;1821}1822 1823#define UPDATE_VF_COUNTER(reg, name) \1824{ \1825	u32 current_counter = er32(reg); \1826	if (current_counter < adapter->stats.last_##name) \1827		adapter->stats.name += 0x100000000LL; \1828	adapter->stats.last_##name = current_counter; \1829	adapter->stats.name &= 0xFFFFFFFF00000000LL; \1830	adapter->stats.name |= current_counter; \1831}1832 1833/**1834 * igbvf_update_stats - Update the board statistics counters1835 * @adapter: board private structure1836**/1837void igbvf_update_stats(struct igbvf_adapter *adapter)1838{1839	struct e1000_hw *hw = &adapter->hw;1840	struct pci_dev *pdev = adapter->pdev;1841 1842	/* Prevent stats update while adapter is being reset, link is down1843	 * or if the pci connection is down.1844	 */1845	if (adapter->link_speed == 0)1846		return;1847 1848	if (test_bit(__IGBVF_RESETTING, &adapter->state))1849		return;1850 1851	if (pci_channel_offline(pdev))1852		return;1853 1854	UPDATE_VF_COUNTER(VFGPRC, gprc);1855	UPDATE_VF_COUNTER(VFGORC, gorc);1856	UPDATE_VF_COUNTER(VFGPTC, gptc);1857	UPDATE_VF_COUNTER(VFGOTC, gotc);1858	UPDATE_VF_COUNTER(VFMPRC, mprc);1859	UPDATE_VF_COUNTER(VFGOTLBC, gotlbc);1860	UPDATE_VF_COUNTER(VFGPTLBC, gptlbc);1861	UPDATE_VF_COUNTER(VFGORLBC, gorlbc);1862	UPDATE_VF_COUNTER(VFGPRLBC, gprlbc);1863 1864	/* Fill out the OS statistics structure */1865	adapter->netdev->stats.multicast = adapter->stats.mprc;1866}1867 1868static void igbvf_print_link_info(struct igbvf_adapter *adapter)1869{1870	dev_info(&adapter->pdev->dev, "Link is Up %d Mbps %s Duplex\n",1871		 adapter->link_speed,1872		 adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half");1873}1874 1875static bool igbvf_has_link(struct igbvf_adapter *adapter)1876{1877	struct e1000_hw *hw = &adapter->hw;1878	s32 ret_val = E1000_SUCCESS;1879	bool link_active;1880 1881	/* If interface is down, stay link down */1882	if (test_bit(__IGBVF_DOWN, &adapter->state))1883		return false;1884 1885	spin_lock_bh(&hw->mbx_lock);1886 1887	ret_val = hw->mac.ops.check_for_link(hw);1888 1889	spin_unlock_bh(&hw->mbx_lock);1890 1891	link_active = !hw->mac.get_link_status;1892 1893	/* if check for link returns error we will need to reset */1894	if (ret_val && time_after(jiffies, adapter->last_reset + (10 * HZ)))1895		schedule_work(&adapter->reset_task);1896 1897	return link_active;1898}1899 1900/**1901 * igbvf_watchdog - Timer Call-back1902 * @t: timer list pointer containing private struct1903 **/1904static void igbvf_watchdog(struct timer_list *t)1905{1906	struct igbvf_adapter *adapter = from_timer(adapter, t, watchdog_timer);1907 1908	/* Do the rest outside of interrupt context */1909	schedule_work(&adapter->watchdog_task);1910}1911 1912static void igbvf_watchdog_task(struct work_struct *work)1913{1914	struct igbvf_adapter *adapter = container_of(work,1915						     struct igbvf_adapter,1916						     watchdog_task);1917	struct net_device *netdev = adapter->netdev;1918	struct e1000_mac_info *mac = &adapter->hw.mac;1919	struct igbvf_ring *tx_ring = adapter->tx_ring;1920	struct e1000_hw *hw = &adapter->hw;1921	u32 link;1922	int tx_pending = 0;1923 1924	link = igbvf_has_link(adapter);1925 1926	if (link) {1927		if (!netif_carrier_ok(netdev)) {1928			mac->ops.get_link_up_info(&adapter->hw,1929						  &adapter->link_speed,1930						  &adapter->link_duplex);1931			igbvf_print_link_info(adapter);1932 1933			netif_carrier_on(netdev);1934			netif_wake_queue(netdev);1935		}1936	} else {1937		if (netif_carrier_ok(netdev)) {1938			adapter->link_speed = 0;1939			adapter->link_duplex = 0;1940			dev_info(&adapter->pdev->dev, "Link is Down\n");1941			netif_carrier_off(netdev);1942			netif_stop_queue(netdev);1943		}1944	}1945 1946	if (netif_carrier_ok(netdev)) {1947		igbvf_update_stats(adapter);1948	} else {1949		tx_pending = (igbvf_desc_unused(tx_ring) + 1 <1950			      tx_ring->count);1951		if (tx_pending) {1952			/* We've lost link, so the controller stops DMA,1953			 * but we've got queued Tx work that's never going1954			 * to get done, so reset controller to flush Tx.1955			 * (Do the reset outside of interrupt context).1956			 */1957			adapter->tx_timeout_count++;1958			schedule_work(&adapter->reset_task);1959		}1960	}1961 1962	/* Cause software interrupt to ensure Rx ring is cleaned */1963	ew32(EICS, adapter->rx_ring->eims_value);1964 1965	/* Reset the timer */1966	if (!test_bit(__IGBVF_DOWN, &adapter->state))1967		mod_timer(&adapter->watchdog_timer,1968			  round_jiffies(jiffies + (2 * HZ)));1969}1970 1971#define IGBVF_TX_FLAGS_CSUM		0x000000011972#define IGBVF_TX_FLAGS_VLAN		0x000000021973#define IGBVF_TX_FLAGS_TSO		0x000000041974#define IGBVF_TX_FLAGS_IPV4		0x000000081975#define IGBVF_TX_FLAGS_VLAN_MASK	0xffff00001976#define IGBVF_TX_FLAGS_VLAN_SHIFT	161977 1978static void igbvf_tx_ctxtdesc(struct igbvf_ring *tx_ring, u32 vlan_macip_lens,1979			      u32 type_tucmd, u32 mss_l4len_idx)1980{1981	struct e1000_adv_tx_context_desc *context_desc;1982	struct igbvf_buffer *buffer_info;1983	u16 i = tx_ring->next_to_use;1984 1985	context_desc = IGBVF_TX_CTXTDESC_ADV(*tx_ring, i);1986	buffer_info = &tx_ring->buffer_info[i];1987 1988	i++;1989	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;1990 1991	/* set bits to identify this as an advanced context descriptor */1992	type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;1993 1994	context_desc->vlan_macip_lens	= cpu_to_le32(vlan_macip_lens);1995	context_desc->seqnum_seed	= 0;1996	context_desc->type_tucmd_mlhl	= cpu_to_le32(type_tucmd);1997	context_desc->mss_l4len_idx	= cpu_to_le32(mss_l4len_idx);1998 1999	buffer_info->time_stamp = jiffies;2000	buffer_info->dma = 0;2001}2002 2003static int igbvf_tso(struct igbvf_ring *tx_ring,2004		     struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)2005{2006	u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;2007	union {2008		struct iphdr *v4;2009		struct ipv6hdr *v6;2010		unsigned char *hdr;2011	} ip;2012	union {2013		struct tcphdr *tcp;2014		unsigned char *hdr;2015	} l4;2016	u32 paylen, l4_offset;2017	int err;2018 2019	if (skb->ip_summed != CHECKSUM_PARTIAL)2020		return 0;2021 2022	if (!skb_is_gso(skb))2023		return 0;2024 2025	err = skb_cow_head(skb, 0);2026	if (err < 0)2027		return err;2028 2029	ip.hdr = skb_network_header(skb);2030	l4.hdr = skb_checksum_start(skb);2031 2032	/* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */2033	type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;2034 2035	/* initialize outer IP header fields */2036	if (ip.v4->version == 4) {2037		unsigned char *csum_start = skb_checksum_start(skb);2038		unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);2039 2040		/* IP header will have to cancel out any data that2041		 * is not a part of the outer IP header2042		 */2043		ip.v4->check = csum_fold(csum_partial(trans_start,2044						      csum_start - trans_start,2045						      0));2046		type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;2047 2048		ip.v4->tot_len = 0;2049	} else {2050		ip.v6->payload_len = 0;2051	}2052 2053	/* determine offset of inner transport header */2054	l4_offset = l4.hdr - skb->data;2055 2056	/* compute length of segmentation header */2057	*hdr_len = (l4.tcp->doff * 4) + l4_offset;2058 2059	/* remove payload length from inner checksum */2060	paylen = skb->len - l4_offset;2061	csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen));2062 2063	/* MSS L4LEN IDX */2064	mss_l4len_idx = (*hdr_len - l4_offset) << E1000_ADVTXD_L4LEN_SHIFT;2065	mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;2066 2067	/* VLAN MACLEN IPLEN */2068	vlan_macip_lens = l4.hdr - ip.hdr;2069	vlan_macip_lens |= (ip.hdr - skb->data) << E1000_ADVTXD_MACLEN_SHIFT;2070	vlan_macip_lens |= tx_flags & IGBVF_TX_FLAGS_VLAN_MASK;2071 2072	igbvf_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);2073 2074	return 1;2075}2076 2077static bool igbvf_tx_csum(struct igbvf_ring *tx_ring, struct sk_buff *skb,2078			  u32 tx_flags, __be16 protocol)2079{2080	u32 vlan_macip_lens = 0;2081	u32 type_tucmd = 0;2082 2083	if (skb->ip_summed != CHECKSUM_PARTIAL) {2084csum_failed:2085		if (!(tx_flags & IGBVF_TX_FLAGS_VLAN))2086			return false;2087		goto no_csum;2088	}2089 2090	switch (skb->csum_offset) {2091	case offsetof(struct tcphdr, check):2092		type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;2093		fallthrough;2094	case offsetof(struct udphdr, check):2095		break;2096	case offsetof(struct sctphdr, checksum):2097		/* validate that this is actually an SCTP request */2098		if (skb_csum_is_sctp(skb)) {2099			type_tucmd = E1000_ADVTXD_TUCMD_L4T_SCTP;2100			break;2101		}2102		fallthrough;2103	default:2104		skb_checksum_help(skb);2105		goto csum_failed;2106	}2107 2108	vlan_macip_lens = skb_checksum_start_offset(skb) -2109			  skb_network_offset(skb);2110no_csum:2111	vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;2112	vlan_macip_lens |= tx_flags & IGBVF_TX_FLAGS_VLAN_MASK;2113 2114	igbvf_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, 0);2115	return true;2116}2117 2118static int igbvf_maybe_stop_tx(struct net_device *netdev, int size)2119{2120	struct igbvf_adapter *adapter = netdev_priv(netdev);2121 2122	/* there is enough descriptors then we don't need to worry  */2123	if (igbvf_desc_unused(adapter->tx_ring) >= size)2124		return 0;2125 2126	netif_stop_queue(netdev);2127 2128	/* Herbert's original patch had:2129	 *  smp_mb__after_netif_stop_queue();2130	 * but since that doesn't exist yet, just open code it.2131	 */2132	smp_mb();2133 2134	/* We need to check again just in case room has been made available */2135	if (igbvf_desc_unused(adapter->tx_ring) < size)2136		return -EBUSY;2137 2138	netif_wake_queue(netdev);2139 2140	++adapter->restart_queue;2141	return 0;2142}2143 2144#define IGBVF_MAX_TXD_PWR	162145#define IGBVF_MAX_DATA_PER_TXD	(1u << IGBVF_MAX_TXD_PWR)2146 2147static inline int igbvf_tx_map_adv(struct igbvf_adapter *adapter,2148				   struct igbvf_ring *tx_ring,2149				   struct sk_buff *skb)2150{2151	struct igbvf_buffer *buffer_info;2152	struct pci_dev *pdev = adapter->pdev;2153	unsigned int len = skb_headlen(skb);2154	unsigned int count = 0, i;2155	unsigned int f;2156 2157	i = tx_ring->next_to_use;2158 2159	buffer_info = &tx_ring->buffer_info[i];2160	BUG_ON(len >= IGBVF_MAX_DATA_PER_TXD);2161	buffer_info->length = len;2162	/* set time_stamp *before* dma to help avoid a possible race */2163	buffer_info->time_stamp = jiffies;2164	buffer_info->mapped_as_page = false;2165	buffer_info->dma = dma_map_single(&pdev->dev, skb->data, len,2166					  DMA_TO_DEVICE);2167	if (dma_mapping_error(&pdev->dev, buffer_info->dma))2168		goto dma_error;2169 2170	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {2171		const skb_frag_t *frag;2172 2173		count++;2174		i++;2175		if (i == tx_ring->count)2176			i = 0;2177 2178		frag = &skb_shinfo(skb)->frags[f];2179		len = skb_frag_size(frag);2180 2181		buffer_info = &tx_ring->buffer_info[i];2182		BUG_ON(len >= IGBVF_MAX_DATA_PER_TXD);2183		buffer_info->length = len;2184		buffer_info->time_stamp = jiffies;2185		buffer_info->mapped_as_page = true;2186		buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag, 0, len,2187						    DMA_TO_DEVICE);2188		if (dma_mapping_error(&pdev->dev, buffer_info->dma))2189			goto dma_error;2190	}2191 2192	tx_ring->buffer_info[i].skb = skb;2193 2194	return ++count;2195 2196dma_error:2197	dev_err(&pdev->dev, "TX DMA map failed\n");2198 2199	/* clear timestamp and dma mappings for failed buffer_info mapping */2200	buffer_info->dma = 0;2201	buffer_info->time_stamp = 0;2202	buffer_info->length = 0;2203	buffer_info->mapped_as_page = false;2204	if (count)2205		count--;2206 2207	/* clear timestamp and dma mappings for remaining portion of packet */2208	while (count--) {2209		if (i == 0)2210			i += tx_ring->count;2211		i--;2212		buffer_info = &tx_ring->buffer_info[i];2213		igbvf_put_txbuf(adapter, buffer_info);2214	}2215 2216	return 0;2217}2218 2219static inline void igbvf_tx_queue_adv(struct igbvf_adapter *adapter,2220				      struct igbvf_ring *tx_ring,2221				      int tx_flags, int count,2222				      unsigned int first, u32 paylen,2223				      u8 hdr_len)2224{2225	union e1000_adv_tx_desc *tx_desc = NULL;2226	struct igbvf_buffer *buffer_info;2227	u32 olinfo_status = 0, cmd_type_len;2228	unsigned int i;2229 2230	cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |2231			E1000_ADVTXD_DCMD_DEXT);2232 2233	if (tx_flags & IGBVF_TX_FLAGS_VLAN)2234		cmd_type_len |= E1000_ADVTXD_DCMD_VLE;2235 2236	if (tx_flags & IGBVF_TX_FLAGS_TSO) {2237		cmd_type_len |= E1000_ADVTXD_DCMD_TSE;2238 2239		/* insert tcp checksum */2240		olinfo_status |= E1000_TXD_POPTS_TXSM << 8;2241 2242		/* insert ip checksum */2243		if (tx_flags & IGBVF_TX_FLAGS_IPV4)2244			olinfo_status |= E1000_TXD_POPTS_IXSM << 8;2245 2246	} else if (tx_flags & IGBVF_TX_FLAGS_CSUM) {2247		olinfo_status |= E1000_TXD_POPTS_TXSM << 8;2248	}2249 2250	olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);2251 2252	i = tx_ring->next_to_use;2253	while (count--) {2254		buffer_info = &tx_ring->buffer_info[i];2255		tx_desc = IGBVF_TX_DESC_ADV(*tx_ring, i);2256		tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);2257		tx_desc->read.cmd_type_len =2258			 cpu_to_le32(cmd_type_len | buffer_info->length);2259		tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);2260		i++;2261		if (i == tx_ring->count)2262			i = 0;2263	}2264 2265	tx_desc->read.cmd_type_len |= cpu_to_le32(adapter->txd_cmd);2266	/* Force memory writes to complete before letting h/w2267	 * know there are new descriptors to fetch.  (Only2268	 * applicable for weak-ordered memory model archs,2269	 * such as IA-64).2270	 */2271	wmb();2272 2273	tx_ring->buffer_info[first].next_to_watch = tx_desc;2274	tx_ring->next_to_use = i;2275	writel(i, adapter->hw.hw_addr + tx_ring->tail);2276}2277 2278static netdev_tx_t igbvf_xmit_frame_ring_adv(struct sk_buff *skb,2279					     struct net_device *netdev,2280					     struct igbvf_ring *tx_ring)2281{2282	struct igbvf_adapter *adapter = netdev_priv(netdev);2283	unsigned int first, tx_flags = 0;2284	u8 hdr_len = 0;2285	int count = 0;2286	int tso = 0;2287	__be16 protocol = vlan_get_protocol(skb);2288 2289	if (test_bit(__IGBVF_DOWN, &adapter->state)) {2290		dev_kfree_skb_any(skb);2291		return NETDEV_TX_OK;2292	}2293 2294	if (skb->len <= 0) {2295		dev_kfree_skb_any(skb);2296		return NETDEV_TX_OK;2297	}2298 2299	/* need: count + 4 desc gap to keep tail from touching2300	 *       + 2 desc gap to keep tail from touching head,2301	 *       + 1 desc for skb->data,2302	 *       + 1 desc for context descriptor,2303	 * head, otherwise try next time2304	 */2305	if (igbvf_maybe_stop_tx(netdev, skb_shinfo(skb)->nr_frags + 4)) {2306		/* this is a hard error */2307		return NETDEV_TX_BUSY;2308	}2309 2310	if (skb_vlan_tag_present(skb)) {2311		tx_flags |= IGBVF_TX_FLAGS_VLAN;2312		tx_flags |= (skb_vlan_tag_get(skb) <<2313			     IGBVF_TX_FLAGS_VLAN_SHIFT);2314	}2315 2316	if (protocol == htons(ETH_P_IP))2317		tx_flags |= IGBVF_TX_FLAGS_IPV4;2318 2319	first = tx_ring->next_to_use;2320 2321	tso = igbvf_tso(tx_ring, skb, tx_flags, &hdr_len);2322	if (unlikely(tso < 0)) {2323		dev_kfree_skb_any(skb);2324		return NETDEV_TX_OK;2325	}2326 2327	if (tso)2328		tx_flags |= IGBVF_TX_FLAGS_TSO;2329	else if (igbvf_tx_csum(tx_ring, skb, tx_flags, protocol) &&2330		 (skb->ip_summed == CHECKSUM_PARTIAL))2331		tx_flags |= IGBVF_TX_FLAGS_CSUM;2332 2333	/* count reflects descriptors mapped, if 0 then mapping error2334	 * has occurred and we need to rewind the descriptor queue2335	 */2336	count = igbvf_tx_map_adv(adapter, tx_ring, skb);2337 2338	if (count) {2339		igbvf_tx_queue_adv(adapter, tx_ring, tx_flags, count,2340				   first, skb->len, hdr_len);2341		/* Make sure there is space in the ring for the next send. */2342		igbvf_maybe_stop_tx(netdev, MAX_SKB_FRAGS + 4);2343	} else {2344		dev_kfree_skb_any(skb);2345		tx_ring->buffer_info[first].time_stamp = 0;2346		tx_ring->next_to_use = first;2347	}2348 2349	return NETDEV_TX_OK;2350}2351 2352static netdev_tx_t igbvf_xmit_frame(struct sk_buff *skb,2353				    struct net_device *netdev)2354{2355	struct igbvf_adapter *adapter = netdev_priv(netdev);2356	struct igbvf_ring *tx_ring;2357 2358	if (test_bit(__IGBVF_DOWN, &adapter->state)) {2359		dev_kfree_skb_any(skb);2360		return NETDEV_TX_OK;2361	}2362 2363	tx_ring = &adapter->tx_ring[0];2364 2365	return igbvf_xmit_frame_ring_adv(skb, netdev, tx_ring);2366}2367 2368/**2369 * igbvf_tx_timeout - Respond to a Tx Hang2370 * @netdev: network interface device structure2371 * @txqueue: queue timing out (unused)2372 **/2373static void igbvf_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)2374{2375	struct igbvf_adapter *adapter = netdev_priv(netdev);2376 2377	/* Do the reset outside of interrupt context */2378	adapter->tx_timeout_count++;2379	schedule_work(&adapter->reset_task);2380}2381 2382static void igbvf_reset_task(struct work_struct *work)2383{2384	struct igbvf_adapter *adapter;2385 2386	adapter = container_of(work, struct igbvf_adapter, reset_task);2387 2388	igbvf_reinit_locked(adapter);2389}2390 2391/**2392 * igbvf_change_mtu - Change the Maximum Transfer Unit2393 * @netdev: network interface device structure2394 * @new_mtu: new value for maximum frame size2395 *2396 * Returns 0 on success, negative on failure2397 **/2398static int igbvf_change_mtu(struct net_device *netdev, int new_mtu)2399{2400	struct igbvf_adapter *adapter = netdev_priv(netdev);2401	int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;2402 2403	while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))2404		usleep_range(1000, 2000);2405	/* igbvf_down has a dependency on max_frame_size */2406	adapter->max_frame_size = max_frame;2407	if (netif_running(netdev))2408		igbvf_down(adapter);2409 2410	/* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN2411	 * means we reserve 2 more, this pushes us to allocate from the next2412	 * larger slab size.2413	 * i.e. RXBUFFER_2048 --> size-4096 slab2414	 * However with the new *_jumbo_rx* routines, jumbo receives will use2415	 * fragmented skbs2416	 */2417 2418	if (max_frame <= 1024)2419		adapter->rx_buffer_len = 1024;2420	else if (max_frame <= 2048)2421		adapter->rx_buffer_len = 2048;2422	else2423#if (PAGE_SIZE / 2) > 163842424		adapter->rx_buffer_len = 16384;2425#else2426		adapter->rx_buffer_len = PAGE_SIZE / 2;2427#endif2428 2429	/* adjust allocation if LPE protects us, and we aren't using SBP */2430	if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN) ||2431	    (max_frame == ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN))2432		adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN +2433					 ETH_FCS_LEN;2434 2435	netdev_dbg(netdev, "changing MTU from %d to %d\n",2436		   netdev->mtu, new_mtu);2437	WRITE_ONCE(netdev->mtu, new_mtu);2438 2439	if (netif_running(netdev))2440		igbvf_up(adapter);2441	else2442		igbvf_reset(adapter);2443 2444	clear_bit(__IGBVF_RESETTING, &adapter->state);2445 2446	return 0;2447}2448 2449static int igbvf_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)2450{2451	switch (cmd) {2452	default:2453		return -EOPNOTSUPP;2454	}2455}2456 2457static int igbvf_suspend(struct device *dev_d)2458{2459	struct net_device *netdev = dev_get_drvdata(dev_d);2460	struct igbvf_adapter *adapter = netdev_priv(netdev);2461 2462	netif_device_detach(netdev);2463 2464	if (netif_running(netdev)) {2465		WARN_ON(test_bit(__IGBVF_RESETTING, &adapter->state));2466		igbvf_down(adapter);2467		igbvf_free_irq(adapter);2468	}2469 2470	return 0;2471}2472 2473static int igbvf_resume(struct device *dev_d)2474{2475	struct pci_dev *pdev = to_pci_dev(dev_d);2476	struct net_device *netdev = pci_get_drvdata(pdev);2477	struct igbvf_adapter *adapter = netdev_priv(netdev);2478	u32 err;2479 2480	pci_set_master(pdev);2481 2482	if (netif_running(netdev)) {2483		err = igbvf_request_irq(adapter);2484		if (err)2485			return err;2486	}2487 2488	igbvf_reset(adapter);2489 2490	if (netif_running(netdev))2491		igbvf_up(adapter);2492 2493	netif_device_attach(netdev);2494 2495	return 0;2496}2497 2498static void igbvf_shutdown(struct pci_dev *pdev)2499{2500	igbvf_suspend(&pdev->dev);2501}2502 2503#ifdef CONFIG_NET_POLL_CONTROLLER2504/* Polling 'interrupt' - used by things like netconsole to send skbs2505 * without having to re-enable interrupts. It's not called while2506 * the interrupt routine is executing.2507 */2508static void igbvf_netpoll(struct net_device *netdev)2509{2510	struct igbvf_adapter *adapter = netdev_priv(netdev);2511 2512	disable_irq(adapter->pdev->irq);2513 2514	igbvf_clean_tx_irq(adapter->tx_ring);2515 2516	enable_irq(adapter->pdev->irq);2517}2518#endif2519 2520/**2521 * igbvf_io_error_detected - called when PCI error is detected2522 * @pdev: Pointer to PCI device2523 * @state: The current pci connection state2524 *2525 * This function is called after a PCI bus error affecting2526 * this device has been detected.2527 */2528static pci_ers_result_t igbvf_io_error_detected(struct pci_dev *pdev,2529						pci_channel_state_t state)2530{2531	struct net_device *netdev = pci_get_drvdata(pdev);2532	struct igbvf_adapter *adapter = netdev_priv(netdev);2533 2534	netif_device_detach(netdev);2535 2536	if (state == pci_channel_io_perm_failure)2537		return PCI_ERS_RESULT_DISCONNECT;2538 2539	if (netif_running(netdev))2540		igbvf_down(adapter);2541	pci_disable_device(pdev);2542 2543	/* Request a slot reset. */2544	return PCI_ERS_RESULT_NEED_RESET;2545}2546 2547/**2548 * igbvf_io_slot_reset - called after the pci bus has been reset.2549 * @pdev: Pointer to PCI device2550 *2551 * Restart the card from scratch, as if from a cold-boot. Implementation2552 * resembles the first-half of the igbvf_resume routine.2553 */2554static pci_ers_result_t igbvf_io_slot_reset(struct pci_dev *pdev)2555{2556	struct net_device *netdev = pci_get_drvdata(pdev);2557	struct igbvf_adapter *adapter = netdev_priv(netdev);2558 2559	if (pci_enable_device_mem(pdev)) {2560		dev_err(&pdev->dev,2561			"Cannot re-enable PCI device after reset.\n");2562		return PCI_ERS_RESULT_DISCONNECT;2563	}2564	pci_set_master(pdev);2565 2566	igbvf_reset(adapter);2567 2568	return PCI_ERS_RESULT_RECOVERED;2569}2570 2571/**2572 * igbvf_io_resume - called when traffic can start flowing again.2573 * @pdev: Pointer to PCI device2574 *2575 * This callback is called when the error recovery driver tells us that2576 * its OK to resume normal operation. Implementation resembles the2577 * second-half of the igbvf_resume routine.2578 */2579static void igbvf_io_resume(struct pci_dev *pdev)2580{2581	struct net_device *netdev = pci_get_drvdata(pdev);2582	struct igbvf_adapter *adapter = netdev_priv(netdev);2583 2584	if (netif_running(netdev)) {2585		if (igbvf_up(adapter)) {2586			dev_err(&pdev->dev,2587				"can't bring device back up after reset\n");2588			return;2589		}2590	}2591 2592	netif_device_attach(netdev);2593}2594 2595/**2596 * igbvf_io_prepare - prepare device driver for PCI reset2597 * @pdev: PCI device information struct2598 */2599static void igbvf_io_prepare(struct pci_dev *pdev)2600{2601	struct net_device *netdev = pci_get_drvdata(pdev);2602	struct igbvf_adapter *adapter = netdev_priv(netdev);2603 2604	while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))2605		usleep_range(1000, 2000);2606	igbvf_down(adapter);2607}2608 2609/**2610 * igbvf_io_reset_done - PCI reset done, device driver reset can begin2611 * @pdev: PCI device information struct2612 */2613static void igbvf_io_reset_done(struct pci_dev *pdev)2614{2615	struct net_device *netdev = pci_get_drvdata(pdev);2616	struct igbvf_adapter *adapter = netdev_priv(netdev);2617 2618	igbvf_up(adapter);2619	clear_bit(__IGBVF_RESETTING, &adapter->state);2620}2621 2622static void igbvf_print_device_info(struct igbvf_adapter *adapter)2623{2624	struct e1000_hw *hw = &adapter->hw;2625	struct net_device *netdev = adapter->netdev;2626	struct pci_dev *pdev = adapter->pdev;2627 2628	if (hw->mac.type == e1000_vfadapt_i350)2629		dev_info(&pdev->dev, "Intel(R) I350 Virtual Function\n");2630	else2631		dev_info(&pdev->dev, "Intel(R) 82576 Virtual Function\n");2632	dev_info(&pdev->dev, "Address: %pM\n", netdev->dev_addr);2633}2634 2635static int igbvf_set_features(struct net_device *netdev,2636			      netdev_features_t features)2637{2638	struct igbvf_adapter *adapter = netdev_priv(netdev);2639 2640	if (features & NETIF_F_RXCSUM)2641		adapter->flags &= ~IGBVF_FLAG_RX_CSUM_DISABLED;2642	else2643		adapter->flags |= IGBVF_FLAG_RX_CSUM_DISABLED;2644 2645	return 0;2646}2647 2648#define IGBVF_MAX_MAC_HDR_LEN		1272649#define IGBVF_MAX_NETWORK_HDR_LEN	5112650 2651static netdev_features_t2652igbvf_features_check(struct sk_buff *skb, struct net_device *dev,2653		     netdev_features_t features)2654{2655	unsigned int network_hdr_len, mac_hdr_len;2656 2657	/* Make certain the headers can be described by a context descriptor */2658	mac_hdr_len = skb_network_offset(skb);2659	if (unlikely(mac_hdr_len > IGBVF_MAX_MAC_HDR_LEN))2660		return features & ~(NETIF_F_HW_CSUM |2661				    NETIF_F_SCTP_CRC |2662				    NETIF_F_HW_VLAN_CTAG_TX |2663				    NETIF_F_TSO |2664				    NETIF_F_TSO6);2665 2666	network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);2667	if (unlikely(network_hdr_len >  IGBVF_MAX_NETWORK_HDR_LEN))2668		return features & ~(NETIF_F_HW_CSUM |2669				    NETIF_F_SCTP_CRC |2670				    NETIF_F_TSO |2671				    NETIF_F_TSO6);2672 2673	/* We can only support IPV4 TSO in tunnels if we can mangle the2674	 * inner IP ID field, so strip TSO if MANGLEID is not supported.2675	 */2676	if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))2677		features &= ~NETIF_F_TSO;2678 2679	return features;2680}2681 2682static const struct net_device_ops igbvf_netdev_ops = {2683	.ndo_open		= igbvf_open,2684	.ndo_stop		= igbvf_close,2685	.ndo_start_xmit		= igbvf_xmit_frame,2686	.ndo_set_rx_mode	= igbvf_set_rx_mode,2687	.ndo_set_mac_address	= igbvf_set_mac,2688	.ndo_change_mtu		= igbvf_change_mtu,2689	.ndo_eth_ioctl		= igbvf_ioctl,2690	.ndo_tx_timeout		= igbvf_tx_timeout,2691	.ndo_vlan_rx_add_vid	= igbvf_vlan_rx_add_vid,2692	.ndo_vlan_rx_kill_vid	= igbvf_vlan_rx_kill_vid,2693#ifdef CONFIG_NET_POLL_CONTROLLER2694	.ndo_poll_controller	= igbvf_netpoll,2695#endif2696	.ndo_set_features	= igbvf_set_features,2697	.ndo_features_check	= igbvf_features_check,2698};2699 2700/**2701 * igbvf_probe - Device Initialization Routine2702 * @pdev: PCI device information struct2703 * @ent: entry in igbvf_pci_tbl2704 *2705 * Returns 0 on success, negative on failure2706 *2707 * igbvf_probe initializes an adapter identified by a pci_dev structure.2708 * The OS initialization, configuring of the adapter private structure,2709 * and a hardware reset occur.2710 **/2711static int igbvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)2712{2713	struct net_device *netdev;2714	struct igbvf_adapter *adapter;2715	struct e1000_hw *hw;2716	const struct igbvf_info *ei = igbvf_info_tbl[ent->driver_data];2717	static int cards_found;2718	int err;2719 2720	err = pci_enable_device_mem(pdev);2721	if (err)2722		return err;2723 2724	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));2725	if (err) {2726		dev_err(&pdev->dev,2727			"No usable DMA configuration, aborting\n");2728		goto err_dma;2729	}2730 2731	err = pci_request_regions(pdev, igbvf_driver_name);2732	if (err)2733		goto err_pci_reg;2734 2735	pci_set_master(pdev);2736 2737	err = -ENOMEM;2738	netdev = alloc_etherdev(sizeof(struct igbvf_adapter));2739	if (!netdev)2740		goto err_alloc_etherdev;2741 2742	SET_NETDEV_DEV(netdev, &pdev->dev);2743 2744	pci_set_drvdata(pdev, netdev);2745	adapter = netdev_priv(netdev);2746	hw = &adapter->hw;2747	adapter->netdev = netdev;2748	adapter->pdev = pdev;2749	adapter->ei = ei;2750	adapter->pba = ei->pba;2751	adapter->flags = ei->flags;2752	adapter->hw.back = adapter;2753	adapter->hw.mac.type = ei->mac;2754	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);2755 2756	/* PCI config space info */2757 2758	hw->vendor_id = pdev->vendor;2759	hw->device_id = pdev->device;2760	hw->subsystem_vendor_id = pdev->subsystem_vendor;2761	hw->subsystem_device_id = pdev->subsystem_device;2762	hw->revision_id = pdev->revision;2763 2764	err = -EIO;2765	adapter->hw.hw_addr = ioremap(pci_resource_start(pdev, 0),2766				      pci_resource_len(pdev, 0));2767 2768	if (!adapter->hw.hw_addr)2769		goto err_ioremap;2770 2771	if (ei->get_variants) {2772		err = ei->get_variants(adapter);2773		if (err)2774			goto err_get_variants;2775	}2776 2777	/* setup adapter struct */2778	err = igbvf_sw_init(adapter);2779	if (err)2780		goto err_sw_init;2781 2782	/* construct the net_device struct */2783	netdev->netdev_ops = &igbvf_netdev_ops;2784 2785	igbvf_set_ethtool_ops(netdev);2786	netdev->watchdog_timeo = 5 * HZ;2787	strscpy(netdev->name, pci_name(pdev), sizeof(netdev->name));2788 2789	adapter->bd_number = cards_found++;2790 2791	netdev->hw_features = NETIF_F_SG |2792			      NETIF_F_TSO |2793			      NETIF_F_TSO6 |2794			      NETIF_F_RXCSUM |2795			      NETIF_F_HW_CSUM |2796			      NETIF_F_SCTP_CRC;2797 2798#define IGBVF_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \2799				    NETIF_F_GSO_GRE_CSUM | \2800				    NETIF_F_GSO_IPXIP4 | \2801				    NETIF_F_GSO_IPXIP6 | \2802				    NETIF_F_GSO_UDP_TUNNEL | \2803				    NETIF_F_GSO_UDP_TUNNEL_CSUM)2804 2805	netdev->gso_partial_features = IGBVF_GSO_PARTIAL_FEATURES;2806	netdev->hw_features |= NETIF_F_GSO_PARTIAL |2807			       IGBVF_GSO_PARTIAL_FEATURES;2808 2809	netdev->features = netdev->hw_features | NETIF_F_HIGHDMA;2810 2811	netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;2812	netdev->mpls_features |= NETIF_F_HW_CSUM;2813	netdev->hw_enc_features |= netdev->vlan_features;2814 2815	/* set this bit last since it cannot be part of vlan_features */2816	netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |2817			    NETIF_F_HW_VLAN_CTAG_RX |2818			    NETIF_F_HW_VLAN_CTAG_TX;2819 2820	/* MTU range: 68 - 9216 */2821	netdev->min_mtu = ETH_MIN_MTU;2822	netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;2823 2824	spin_lock_bh(&hw->mbx_lock);2825 2826	/*reset the controller to put the device in a known good state */2827	err = hw->mac.ops.reset_hw(hw);2828	if (err) {2829		dev_info(&pdev->dev,2830			 "PF still in reset state. Is the PF interface up?\n");2831	} else {2832		err = hw->mac.ops.read_mac_addr(hw);2833		if (err)2834			dev_info(&pdev->dev, "Error reading MAC address.\n");2835		else if (is_zero_ether_addr(adapter->hw.mac.addr))2836			dev_info(&pdev->dev,2837				 "MAC address not assigned by administrator.\n");2838		eth_hw_addr_set(netdev, adapter->hw.mac.addr);2839	}2840 2841	spin_unlock_bh(&hw->mbx_lock);2842 2843	if (!is_valid_ether_addr(netdev->dev_addr)) {2844		dev_info(&pdev->dev, "Assigning random MAC address.\n");2845		eth_hw_addr_random(netdev);2846		memcpy(adapter->hw.mac.addr, netdev->dev_addr,2847		       netdev->addr_len);2848	}2849 2850	timer_setup(&adapter->watchdog_timer, igbvf_watchdog, 0);2851 2852	INIT_WORK(&adapter->reset_task, igbvf_reset_task);2853	INIT_WORK(&adapter->watchdog_task, igbvf_watchdog_task);2854 2855	/* ring size defaults */2856	adapter->rx_ring->count = 1024;2857	adapter->tx_ring->count = 1024;2858 2859	/* reset the hardware with the new settings */2860	igbvf_reset(adapter);2861 2862	/* set hardware-specific flags */2863	if (adapter->hw.mac.type == e1000_vfadapt_i350)2864		adapter->flags |= IGBVF_FLAG_RX_LB_VLAN_BSWAP;2865 2866	strcpy(netdev->name, "eth%d");2867	err = register_netdev(netdev);2868	if (err)2869		goto err_hw_init;2870 2871	/* tell the stack to leave us alone until igbvf_open() is called */2872	netif_carrier_off(netdev);2873	netif_stop_queue(netdev);2874 2875	igbvf_print_device_info(adapter);2876 2877	igbvf_initialize_last_counter_stats(adapter);2878 2879	return 0;2880 2881err_hw_init:2882	netif_napi_del(&adapter->rx_ring->napi);2883	kfree(adapter->tx_ring);2884	kfree(adapter->rx_ring);2885err_sw_init:2886	igbvf_reset_interrupt_capability(adapter);2887err_get_variants:2888	iounmap(adapter->hw.hw_addr);2889err_ioremap:2890	free_netdev(netdev);2891err_alloc_etherdev:2892	pci_release_regions(pdev);2893err_pci_reg:2894err_dma:2895	pci_disable_device(pdev);2896	return err;2897}2898 2899/**2900 * igbvf_remove - Device Removal Routine2901 * @pdev: PCI device information struct2902 *2903 * igbvf_remove is called by the PCI subsystem to alert the driver2904 * that it should release a PCI device.  The could be caused by a2905 * Hot-Plug event, or because the driver is going to be removed from2906 * memory.2907 **/2908static void igbvf_remove(struct pci_dev *pdev)2909{2910	struct net_device *netdev = pci_get_drvdata(pdev);2911	struct igbvf_adapter *adapter = netdev_priv(netdev);2912	struct e1000_hw *hw = &adapter->hw;2913 2914	/* The watchdog timer may be rescheduled, so explicitly2915	 * disable it from being rescheduled.2916	 */2917	set_bit(__IGBVF_DOWN, &adapter->state);2918	del_timer_sync(&adapter->watchdog_timer);2919 2920	cancel_work_sync(&adapter->reset_task);2921	cancel_work_sync(&adapter->watchdog_task);2922 2923	unregister_netdev(netdev);2924 2925	igbvf_reset_interrupt_capability(adapter);2926 2927	/* it is important to delete the NAPI struct prior to freeing the2928	 * Rx ring so that you do not end up with null pointer refs2929	 */2930	netif_napi_del(&adapter->rx_ring->napi);2931	kfree(adapter->tx_ring);2932	kfree(adapter->rx_ring);2933 2934	iounmap(hw->hw_addr);2935	if (hw->flash_address)2936		iounmap(hw->flash_address);2937	pci_release_regions(pdev);2938 2939	free_netdev(netdev);2940 2941	pci_disable_device(pdev);2942}2943 2944/* PCI Error Recovery (ERS) */2945static const struct pci_error_handlers igbvf_err_handler = {2946	.error_detected = igbvf_io_error_detected,2947	.slot_reset = igbvf_io_slot_reset,2948	.resume = igbvf_io_resume,2949	.reset_prepare = igbvf_io_prepare,2950	.reset_done = igbvf_io_reset_done,2951};2952 2953static const struct pci_device_id igbvf_pci_tbl[] = {2954	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_VF), board_vf },2955	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_VF), board_i350_vf },2956	{ } /* terminate list */2957};2958MODULE_DEVICE_TABLE(pci, igbvf_pci_tbl);2959 2960static DEFINE_SIMPLE_DEV_PM_OPS(igbvf_pm_ops, igbvf_suspend, igbvf_resume);2961 2962/* PCI Device API Driver */2963static struct pci_driver igbvf_driver = {2964	.name		= igbvf_driver_name,2965	.id_table	= igbvf_pci_tbl,2966	.probe		= igbvf_probe,2967	.remove		= igbvf_remove,2968	.driver.pm	= pm_sleep_ptr(&igbvf_pm_ops),2969	.shutdown	= igbvf_shutdown,2970	.err_handler	= &igbvf_err_handler2971};2972 2973/**2974 * igbvf_init_module - Driver Registration Routine2975 *2976 * igbvf_init_module is the first routine called when the driver is2977 * loaded. All it does is register with the PCI subsystem.2978 **/2979static int __init igbvf_init_module(void)2980{2981	int ret;2982 2983	pr_info("%s\n", igbvf_driver_string);2984	pr_info("%s\n", igbvf_copyright);2985 2986	ret = pci_register_driver(&igbvf_driver);2987 2988	return ret;2989}2990module_init(igbvf_init_module);2991 2992/**2993 * igbvf_exit_module - Driver Exit Cleanup Routine2994 *2995 * igbvf_exit_module is called just before the driver is removed2996 * from memory.2997 **/2998static void __exit igbvf_exit_module(void)2999{3000	pci_unregister_driver(&igbvf_driver);3001}3002module_exit(igbvf_exit_module);3003 3004MODULE_DESCRIPTION("Intel(R) Gigabit Virtual Function Network Driver");3005MODULE_LICENSE("GPL v2");3006 3007/* netdev.c */3008