2875 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) 2019 - 2022 Beijing WangXun Technology Co., Ltd. */3 4#include <linux/etherdevice.h>5#include <net/ip6_checksum.h>6#include <net/page_pool/helpers.h>7#include <net/inet_ecn.h>8#include <linux/iopoll.h>9#include <linux/sctp.h>10#include <linux/pci.h>11#include <net/tcp.h>12#include <net/ip.h>13 14#include "wx_type.h"15#include "wx_lib.h"16#include "wx_hw.h"17 18/* Lookup table mapping the HW PTYPE to the bit field for decoding */19static struct wx_dec_ptype wx_ptype_lookup[256] = {20 /* L2: mac */21 [0x11] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),22 [0x12] = WX_PTT(L2, NONE, NONE, NONE, TS, PAY2),23 [0x13] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),24 [0x14] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),25 [0x15] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),26 [0x16] = WX_PTT(L2, NONE, NONE, NONE, NONE, PAY2),27 [0x17] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),28 29 /* L2: ethertype filter */30 [0x18 ... 0x1F] = WX_PTT(L2, NONE, NONE, NONE, NONE, NONE),31 32 /* L3: ip non-tunnel */33 [0x21] = WX_PTT(IP, FGV4, NONE, NONE, NONE, PAY3),34 [0x22] = WX_PTT(IP, IPV4, NONE, NONE, NONE, PAY3),35 [0x23] = WX_PTT(IP, IPV4, NONE, NONE, UDP, PAY4),36 [0x24] = WX_PTT(IP, IPV4, NONE, NONE, TCP, PAY4),37 [0x25] = WX_PTT(IP, IPV4, NONE, NONE, SCTP, PAY4),38 [0x29] = WX_PTT(IP, FGV6, NONE, NONE, NONE, PAY3),39 [0x2A] = WX_PTT(IP, IPV6, NONE, NONE, NONE, PAY3),40 [0x2B] = WX_PTT(IP, IPV6, NONE, NONE, UDP, PAY3),41 [0x2C] = WX_PTT(IP, IPV6, NONE, NONE, TCP, PAY4),42 [0x2D] = WX_PTT(IP, IPV6, NONE, NONE, SCTP, PAY4),43 44 /* L2: fcoe */45 [0x30 ... 0x34] = WX_PTT(FCOE, NONE, NONE, NONE, NONE, PAY3),46 [0x38 ... 0x3C] = WX_PTT(FCOE, NONE, NONE, NONE, NONE, PAY3),47 48 /* IPv4 --> IPv4/IPv6 */49 [0x81] = WX_PTT(IP, IPV4, IPIP, FGV4, NONE, PAY3),50 [0x82] = WX_PTT(IP, IPV4, IPIP, IPV4, NONE, PAY3),51 [0x83] = WX_PTT(IP, IPV4, IPIP, IPV4, UDP, PAY4),52 [0x84] = WX_PTT(IP, IPV4, IPIP, IPV4, TCP, PAY4),53 [0x85] = WX_PTT(IP, IPV4, IPIP, IPV4, SCTP, PAY4),54 [0x89] = WX_PTT(IP, IPV4, IPIP, FGV6, NONE, PAY3),55 [0x8A] = WX_PTT(IP, IPV4, IPIP, IPV6, NONE, PAY3),56 [0x8B] = WX_PTT(IP, IPV4, IPIP, IPV6, UDP, PAY4),57 [0x8C] = WX_PTT(IP, IPV4, IPIP, IPV6, TCP, PAY4),58 [0x8D] = WX_PTT(IP, IPV4, IPIP, IPV6, SCTP, PAY4),59 60 /* IPv4 --> GRE/NAT --> NONE/IPv4/IPv6 */61 [0x90] = WX_PTT(IP, IPV4, IG, NONE, NONE, PAY3),62 [0x91] = WX_PTT(IP, IPV4, IG, FGV4, NONE, PAY3),63 [0x92] = WX_PTT(IP, IPV4, IG, IPV4, NONE, PAY3),64 [0x93] = WX_PTT(IP, IPV4, IG, IPV4, UDP, PAY4),65 [0x94] = WX_PTT(IP, IPV4, IG, IPV4, TCP, PAY4),66 [0x95] = WX_PTT(IP, IPV4, IG, IPV4, SCTP, PAY4),67 [0x99] = WX_PTT(IP, IPV4, IG, FGV6, NONE, PAY3),68 [0x9A] = WX_PTT(IP, IPV4, IG, IPV6, NONE, PAY3),69 [0x9B] = WX_PTT(IP, IPV4, IG, IPV6, UDP, PAY4),70 [0x9C] = WX_PTT(IP, IPV4, IG, IPV6, TCP, PAY4),71 [0x9D] = WX_PTT(IP, IPV4, IG, IPV6, SCTP, PAY4),72 73 /* IPv4 --> GRE/NAT --> MAC --> NONE/IPv4/IPv6 */74 [0xA0] = WX_PTT(IP, IPV4, IGM, NONE, NONE, PAY3),75 [0xA1] = WX_PTT(IP, IPV4, IGM, FGV4, NONE, PAY3),76 [0xA2] = WX_PTT(IP, IPV4, IGM, IPV4, NONE, PAY3),77 [0xA3] = WX_PTT(IP, IPV4, IGM, IPV4, UDP, PAY4),78 [0xA4] = WX_PTT(IP, IPV4, IGM, IPV4, TCP, PAY4),79 [0xA5] = WX_PTT(IP, IPV4, IGM, IPV4, SCTP, PAY4),80 [0xA9] = WX_PTT(IP, IPV4, IGM, FGV6, NONE, PAY3),81 [0xAA] = WX_PTT(IP, IPV4, IGM, IPV6, NONE, PAY3),82 [0xAB] = WX_PTT(IP, IPV4, IGM, IPV6, UDP, PAY4),83 [0xAC] = WX_PTT(IP, IPV4, IGM, IPV6, TCP, PAY4),84 [0xAD] = WX_PTT(IP, IPV4, IGM, IPV6, SCTP, PAY4),85 86 /* IPv4 --> GRE/NAT --> MAC+VLAN --> NONE/IPv4/IPv6 */87 [0xB0] = WX_PTT(IP, IPV4, IGMV, NONE, NONE, PAY3),88 [0xB1] = WX_PTT(IP, IPV4, IGMV, FGV4, NONE, PAY3),89 [0xB2] = WX_PTT(IP, IPV4, IGMV, IPV4, NONE, PAY3),90 [0xB3] = WX_PTT(IP, IPV4, IGMV, IPV4, UDP, PAY4),91 [0xB4] = WX_PTT(IP, IPV4, IGMV, IPV4, TCP, PAY4),92 [0xB5] = WX_PTT(IP, IPV4, IGMV, IPV4, SCTP, PAY4),93 [0xB9] = WX_PTT(IP, IPV4, IGMV, FGV6, NONE, PAY3),94 [0xBA] = WX_PTT(IP, IPV4, IGMV, IPV6, NONE, PAY3),95 [0xBB] = WX_PTT(IP, IPV4, IGMV, IPV6, UDP, PAY4),96 [0xBC] = WX_PTT(IP, IPV4, IGMV, IPV6, TCP, PAY4),97 [0xBD] = WX_PTT(IP, IPV4, IGMV, IPV6, SCTP, PAY4),98 99 /* IPv6 --> IPv4/IPv6 */100 [0xC1] = WX_PTT(IP, IPV6, IPIP, FGV4, NONE, PAY3),101 [0xC2] = WX_PTT(IP, IPV6, IPIP, IPV4, NONE, PAY3),102 [0xC3] = WX_PTT(IP, IPV6, IPIP, IPV4, UDP, PAY4),103 [0xC4] = WX_PTT(IP, IPV6, IPIP, IPV4, TCP, PAY4),104 [0xC5] = WX_PTT(IP, IPV6, IPIP, IPV4, SCTP, PAY4),105 [0xC9] = WX_PTT(IP, IPV6, IPIP, FGV6, NONE, PAY3),106 [0xCA] = WX_PTT(IP, IPV6, IPIP, IPV6, NONE, PAY3),107 [0xCB] = WX_PTT(IP, IPV6, IPIP, IPV6, UDP, PAY4),108 [0xCC] = WX_PTT(IP, IPV6, IPIP, IPV6, TCP, PAY4),109 [0xCD] = WX_PTT(IP, IPV6, IPIP, IPV6, SCTP, PAY4),110 111 /* IPv6 --> GRE/NAT -> NONE/IPv4/IPv6 */112 [0xD0] = WX_PTT(IP, IPV6, IG, NONE, NONE, PAY3),113 [0xD1] = WX_PTT(IP, IPV6, IG, FGV4, NONE, PAY3),114 [0xD2] = WX_PTT(IP, IPV6, IG, IPV4, NONE, PAY3),115 [0xD3] = WX_PTT(IP, IPV6, IG, IPV4, UDP, PAY4),116 [0xD4] = WX_PTT(IP, IPV6, IG, IPV4, TCP, PAY4),117 [0xD5] = WX_PTT(IP, IPV6, IG, IPV4, SCTP, PAY4),118 [0xD9] = WX_PTT(IP, IPV6, IG, FGV6, NONE, PAY3),119 [0xDA] = WX_PTT(IP, IPV6, IG, IPV6, NONE, PAY3),120 [0xDB] = WX_PTT(IP, IPV6, IG, IPV6, UDP, PAY4),121 [0xDC] = WX_PTT(IP, IPV6, IG, IPV6, TCP, PAY4),122 [0xDD] = WX_PTT(IP, IPV6, IG, IPV6, SCTP, PAY4),123 124 /* IPv6 --> GRE/NAT -> MAC -> NONE/IPv4/IPv6 */125 [0xE0] = WX_PTT(IP, IPV6, IGM, NONE, NONE, PAY3),126 [0xE1] = WX_PTT(IP, IPV6, IGM, FGV4, NONE, PAY3),127 [0xE2] = WX_PTT(IP, IPV6, IGM, IPV4, NONE, PAY3),128 [0xE3] = WX_PTT(IP, IPV6, IGM, IPV4, UDP, PAY4),129 [0xE4] = WX_PTT(IP, IPV6, IGM, IPV4, TCP, PAY4),130 [0xE5] = WX_PTT(IP, IPV6, IGM, IPV4, SCTP, PAY4),131 [0xE9] = WX_PTT(IP, IPV6, IGM, FGV6, NONE, PAY3),132 [0xEA] = WX_PTT(IP, IPV6, IGM, IPV6, NONE, PAY3),133 [0xEB] = WX_PTT(IP, IPV6, IGM, IPV6, UDP, PAY4),134 [0xEC] = WX_PTT(IP, IPV6, IGM, IPV6, TCP, PAY4),135 [0xED] = WX_PTT(IP, IPV6, IGM, IPV6, SCTP, PAY4),136 137 /* IPv6 --> GRE/NAT -> MAC--> NONE/IPv */138 [0xF0] = WX_PTT(IP, IPV6, IGMV, NONE, NONE, PAY3),139 [0xF1] = WX_PTT(IP, IPV6, IGMV, FGV4, NONE, PAY3),140 [0xF2] = WX_PTT(IP, IPV6, IGMV, IPV4, NONE, PAY3),141 [0xF3] = WX_PTT(IP, IPV6, IGMV, IPV4, UDP, PAY4),142 [0xF4] = WX_PTT(IP, IPV6, IGMV, IPV4, TCP, PAY4),143 [0xF5] = WX_PTT(IP, IPV6, IGMV, IPV4, SCTP, PAY4),144 [0xF9] = WX_PTT(IP, IPV6, IGMV, FGV6, NONE, PAY3),145 [0xFA] = WX_PTT(IP, IPV6, IGMV, IPV6, NONE, PAY3),146 [0xFB] = WX_PTT(IP, IPV6, IGMV, IPV6, UDP, PAY4),147 [0xFC] = WX_PTT(IP, IPV6, IGMV, IPV6, TCP, PAY4),148 [0xFD] = WX_PTT(IP, IPV6, IGMV, IPV6, SCTP, PAY4),149};150 151struct wx_dec_ptype wx_decode_ptype(const u8 ptype)152{153 return wx_ptype_lookup[ptype];154}155EXPORT_SYMBOL(wx_decode_ptype);156 157/* wx_test_staterr - tests bits in Rx descriptor status and error fields */158static __le32 wx_test_staterr(union wx_rx_desc *rx_desc,159 const u32 stat_err_bits)160{161 return rx_desc->wb.upper.status_error & cpu_to_le32(stat_err_bits);162}163 164static void wx_dma_sync_frag(struct wx_ring *rx_ring,165 struct wx_rx_buffer *rx_buffer)166{167 struct sk_buff *skb = rx_buffer->skb;168 skb_frag_t *frag = &skb_shinfo(skb)->frags[0];169 170 dma_sync_single_range_for_cpu(rx_ring->dev,171 WX_CB(skb)->dma,172 skb_frag_off(frag),173 skb_frag_size(frag),174 DMA_FROM_DEVICE);175 176 /* If the page was released, just unmap it. */177 if (unlikely(WX_CB(skb)->page_released))178 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false);179}180 181static struct wx_rx_buffer *wx_get_rx_buffer(struct wx_ring *rx_ring,182 union wx_rx_desc *rx_desc,183 struct sk_buff **skb,184 int *rx_buffer_pgcnt)185{186 struct wx_rx_buffer *rx_buffer;187 unsigned int size;188 189 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];190 size = le16_to_cpu(rx_desc->wb.upper.length);191 192#if (PAGE_SIZE < 8192)193 *rx_buffer_pgcnt = page_count(rx_buffer->page);194#else195 *rx_buffer_pgcnt = 0;196#endif197 198 prefetchw(rx_buffer->page);199 *skb = rx_buffer->skb;200 201 /* Delay unmapping of the first packet. It carries the header202 * information, HW may still access the header after the writeback.203 * Only unmap it when EOP is reached204 */205 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_EOP)) {206 if (!*skb)207 goto skip_sync;208 } else {209 if (*skb)210 wx_dma_sync_frag(rx_ring, rx_buffer);211 }212 213 /* we are reusing so sync this buffer for CPU use */214 dma_sync_single_range_for_cpu(rx_ring->dev,215 rx_buffer->dma,216 rx_buffer->page_offset,217 size,218 DMA_FROM_DEVICE);219skip_sync:220 return rx_buffer;221}222 223static void wx_put_rx_buffer(struct wx_ring *rx_ring,224 struct wx_rx_buffer *rx_buffer,225 struct sk_buff *skb,226 int rx_buffer_pgcnt)227{228 if (!IS_ERR(skb) && WX_CB(skb)->dma == rx_buffer->dma)229 /* the page has been released from the ring */230 WX_CB(skb)->page_released = true;231 232 /* clear contents of rx_buffer */233 rx_buffer->page = NULL;234 rx_buffer->skb = NULL;235}236 237static struct sk_buff *wx_build_skb(struct wx_ring *rx_ring,238 struct wx_rx_buffer *rx_buffer,239 union wx_rx_desc *rx_desc)240{241 unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);242#if (PAGE_SIZE < 8192)243 unsigned int truesize = WX_RX_BUFSZ;244#else245 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);246#endif247 struct sk_buff *skb = rx_buffer->skb;248 249 if (!skb) {250 void *page_addr = page_address(rx_buffer->page) +251 rx_buffer->page_offset;252 253 /* prefetch first cache line of first page */254 net_prefetch(page_addr);255 256 /* allocate a skb to store the frags */257 skb = napi_alloc_skb(&rx_ring->q_vector->napi, WX_RXBUFFER_256);258 if (unlikely(!skb))259 return NULL;260 261 /* we will be copying header into skb->data in262 * pskb_may_pull so it is in our interest to prefetch263 * it now to avoid a possible cache miss264 */265 prefetchw(skb->data);266 267 if (size <= WX_RXBUFFER_256) {268 memcpy(__skb_put(skb, size), page_addr,269 ALIGN(size, sizeof(long)));270 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, true);271 return skb;272 }273 274 skb_mark_for_recycle(skb);275 276 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_EOP))277 WX_CB(skb)->dma = rx_buffer->dma;278 279 skb_add_rx_frag(skb, 0, rx_buffer->page,280 rx_buffer->page_offset,281 size, truesize);282 goto out;283 284 } else {285 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,286 rx_buffer->page_offset, size, truesize);287 }288 289out:290#if (PAGE_SIZE < 8192)291 /* flip page offset to other buffer */292 rx_buffer->page_offset ^= truesize;293#else294 /* move offset up to the next cache line */295 rx_buffer->page_offset += truesize;296#endif297 298 return skb;299}300 301static bool wx_alloc_mapped_page(struct wx_ring *rx_ring,302 struct wx_rx_buffer *bi)303{304 struct page *page = bi->page;305 dma_addr_t dma;306 307 /* since we are recycling buffers we should seldom need to alloc */308 if (likely(page))309 return true;310 311 page = page_pool_dev_alloc_pages(rx_ring->page_pool);312 WARN_ON(!page);313 dma = page_pool_get_dma_addr(page);314 315 bi->page_dma = dma;316 bi->page = page;317 bi->page_offset = 0;318 319 return true;320}321 322/**323 * wx_alloc_rx_buffers - Replace used receive buffers324 * @rx_ring: ring to place buffers on325 * @cleaned_count: number of buffers to replace326 **/327void wx_alloc_rx_buffers(struct wx_ring *rx_ring, u16 cleaned_count)328{329 u16 i = rx_ring->next_to_use;330 union wx_rx_desc *rx_desc;331 struct wx_rx_buffer *bi;332 333 /* nothing to do */334 if (!cleaned_count)335 return;336 337 rx_desc = WX_RX_DESC(rx_ring, i);338 bi = &rx_ring->rx_buffer_info[i];339 i -= rx_ring->count;340 341 do {342 if (!wx_alloc_mapped_page(rx_ring, bi))343 break;344 345 /* sync the buffer for use by the device */346 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,347 bi->page_offset,348 WX_RX_BUFSZ,349 DMA_FROM_DEVICE);350 351 rx_desc->read.pkt_addr =352 cpu_to_le64(bi->page_dma + bi->page_offset);353 354 rx_desc++;355 bi++;356 i++;357 if (unlikely(!i)) {358 rx_desc = WX_RX_DESC(rx_ring, 0);359 bi = rx_ring->rx_buffer_info;360 i -= rx_ring->count;361 }362 363 /* clear the status bits for the next_to_use descriptor */364 rx_desc->wb.upper.status_error = 0;365 366 cleaned_count--;367 } while (cleaned_count);368 369 i += rx_ring->count;370 371 if (rx_ring->next_to_use != i) {372 rx_ring->next_to_use = i;373 /* update next to alloc since we have filled the ring */374 rx_ring->next_to_alloc = i;375 376 /* Force memory writes to complete before letting h/w377 * know there are new descriptors to fetch. (Only378 * applicable for weak-ordered memory model archs,379 * such as IA-64).380 */381 wmb();382 writel(i, rx_ring->tail);383 }384}385 386u16 wx_desc_unused(struct wx_ring *ring)387{388 u16 ntc = ring->next_to_clean;389 u16 ntu = ring->next_to_use;390 391 return ((ntc > ntu) ? 0 : ring->count) + ntc - ntu - 1;392}393 394/**395 * wx_is_non_eop - process handling of non-EOP buffers396 * @rx_ring: Rx ring being processed397 * @rx_desc: Rx descriptor for current buffer398 * @skb: Current socket buffer containing buffer in progress399 *400 * This function updates next to clean. If the buffer is an EOP buffer401 * this function exits returning false, otherwise it will place the402 * sk_buff in the next buffer to be chained and return true indicating403 * that this is in fact a non-EOP buffer.404 **/405static bool wx_is_non_eop(struct wx_ring *rx_ring,406 union wx_rx_desc *rx_desc,407 struct sk_buff *skb)408{409 u32 ntc = rx_ring->next_to_clean + 1;410 411 /* fetch, update, and store next to clean */412 ntc = (ntc < rx_ring->count) ? ntc : 0;413 rx_ring->next_to_clean = ntc;414 415 prefetch(WX_RX_DESC(rx_ring, ntc));416 417 /* if we are the last buffer then there is nothing else to do */418 if (likely(wx_test_staterr(rx_desc, WX_RXD_STAT_EOP)))419 return false;420 421 rx_ring->rx_buffer_info[ntc].skb = skb;422 rx_ring->rx_stats.non_eop_descs++;423 424 return true;425}426 427static void wx_pull_tail(struct sk_buff *skb)428{429 skb_frag_t *frag = &skb_shinfo(skb)->frags[0];430 unsigned int pull_len;431 unsigned char *va;432 433 /* it is valid to use page_address instead of kmap since we are434 * working with pages allocated out of the lomem pool per435 * alloc_page(GFP_ATOMIC)436 */437 va = skb_frag_address(frag);438 439 /* we need the header to contain the greater of either ETH_HLEN or440 * 60 bytes if the skb->len is less than 60 for skb_pad.441 */442 pull_len = eth_get_headlen(skb->dev, va, WX_RXBUFFER_256);443 444 /* align pull length to size of long to optimize memcpy performance */445 skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));446 447 /* update all of the pointers */448 skb_frag_size_sub(frag, pull_len);449 skb_frag_off_add(frag, pull_len);450 skb->data_len -= pull_len;451 skb->tail += pull_len;452}453 454/**455 * wx_cleanup_headers - Correct corrupted or empty headers456 * @rx_ring: rx descriptor ring packet is being transacted on457 * @rx_desc: pointer to the EOP Rx descriptor458 * @skb: pointer to current skb being fixed459 *460 * Check for corrupted packet headers caused by senders on the local L2461 * embedded NIC switch not setting up their Tx Descriptors right. These462 * should be very rare.463 *464 * Also address the case where we are pulling data in on pages only465 * and as such no data is present in the skb header.466 *467 * In addition if skb is not at least 60 bytes we need to pad it so that468 * it is large enough to qualify as a valid Ethernet frame.469 *470 * Returns true if an error was encountered and skb was freed.471 **/472static bool wx_cleanup_headers(struct wx_ring *rx_ring,473 union wx_rx_desc *rx_desc,474 struct sk_buff *skb)475{476 struct net_device *netdev = rx_ring->netdev;477 478 /* verify that the packet does not have any known errors */479 if (!netdev ||480 unlikely(wx_test_staterr(rx_desc, WX_RXD_ERR_RXE) &&481 !(netdev->features & NETIF_F_RXALL))) {482 dev_kfree_skb_any(skb);483 return true;484 }485 486 /* place header in linear portion of buffer */487 if (!skb_headlen(skb))488 wx_pull_tail(skb);489 490 /* if eth_skb_pad returns an error the skb was freed */491 if (eth_skb_pad(skb))492 return true;493 494 return false;495}496 497static void wx_rx_hash(struct wx_ring *ring,498 union wx_rx_desc *rx_desc,499 struct sk_buff *skb)500{501 u16 rss_type;502 503 if (!(ring->netdev->features & NETIF_F_RXHASH))504 return;505 506 rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &507 WX_RXD_RSSTYPE_MASK;508 509 if (!rss_type)510 return;511 512 skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),513 (WX_RSS_L4_TYPES_MASK & (1ul << rss_type)) ?514 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);515}516 517/**518 * wx_rx_checksum - indicate in skb if hw indicated a good cksum519 * @ring: structure containing ring specific data520 * @rx_desc: current Rx descriptor being processed521 * @skb: skb currently being received and modified522 **/523static void wx_rx_checksum(struct wx_ring *ring,524 union wx_rx_desc *rx_desc,525 struct sk_buff *skb)526{527 struct wx_dec_ptype dptype = wx_decode_ptype(WX_RXD_PKTTYPE(rx_desc));528 529 skb_checksum_none_assert(skb);530 /* Rx csum disabled */531 if (!(ring->netdev->features & NETIF_F_RXCSUM))532 return;533 534 /* if IPv4 header checksum error */535 if ((wx_test_staterr(rx_desc, WX_RXD_STAT_IPCS) &&536 wx_test_staterr(rx_desc, WX_RXD_ERR_IPE)) ||537 (wx_test_staterr(rx_desc, WX_RXD_STAT_OUTERIPCS) &&538 wx_test_staterr(rx_desc, WX_RXD_ERR_OUTERIPER))) {539 ring->rx_stats.csum_err++;540 return;541 }542 543 /* L4 checksum offload flag must set for the below code to work */544 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_L4CS))545 return;546 547 /* Hardware can't guarantee csum if IPv6 Dest Header found */548 if (dptype.prot != WX_DEC_PTYPE_PROT_SCTP && WX_RXD_IPV6EX(rx_desc))549 return;550 551 /* if L4 checksum error */552 if (wx_test_staterr(rx_desc, WX_RXD_ERR_TCPE)) {553 ring->rx_stats.csum_err++;554 return;555 }556 557 /* It must be a TCP or UDP or SCTP packet with a valid checksum */558 skb->ip_summed = CHECKSUM_UNNECESSARY;559 560 /* If there is an outer header present that might contain a checksum561 * we need to bump the checksum level by 1 to reflect the fact that562 * we are indicating we validated the inner checksum.563 */564 if (dptype.etype >= WX_DEC_PTYPE_ETYPE_IG)565 __skb_incr_checksum_unnecessary(skb);566 ring->rx_stats.csum_good_cnt++;567}568 569static void wx_rx_vlan(struct wx_ring *ring, union wx_rx_desc *rx_desc,570 struct sk_buff *skb)571{572 u16 ethertype;573 u8 idx = 0;574 575 if ((ring->netdev->features &576 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX)) &&577 wx_test_staterr(rx_desc, WX_RXD_STAT_VP)) {578 idx = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) &579 0x1c0) >> 6;580 ethertype = ring->q_vector->wx->tpid[idx];581 __vlan_hwaccel_put_tag(skb, htons(ethertype),582 le16_to_cpu(rx_desc->wb.upper.vlan));583 }584}585 586/**587 * wx_process_skb_fields - Populate skb header fields from Rx descriptor588 * @rx_ring: rx descriptor ring packet is being transacted on589 * @rx_desc: pointer to the EOP Rx descriptor590 * @skb: pointer to current skb being populated591 *592 * This function checks the ring, descriptor, and packet information in593 * order to populate the hash, checksum, protocol, and594 * other fields within the skb.595 **/596static void wx_process_skb_fields(struct wx_ring *rx_ring,597 union wx_rx_desc *rx_desc,598 struct sk_buff *skb)599{600 wx_rx_hash(rx_ring, rx_desc, skb);601 wx_rx_checksum(rx_ring, rx_desc, skb);602 wx_rx_vlan(rx_ring, rx_desc, skb);603 skb_record_rx_queue(skb, rx_ring->queue_index);604 skb->protocol = eth_type_trans(skb, rx_ring->netdev);605}606 607/**608 * wx_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf609 * @q_vector: structure containing interrupt and ring information610 * @rx_ring: rx descriptor ring to transact packets on611 * @budget: Total limit on number of packets to process612 *613 * This function provides a "bounce buffer" approach to Rx interrupt614 * processing. The advantage to this is that on systems that have615 * expensive overhead for IOMMU access this provides a means of avoiding616 * it by maintaining the mapping of the page to the system.617 *618 * Returns amount of work completed.619 **/620static int wx_clean_rx_irq(struct wx_q_vector *q_vector,621 struct wx_ring *rx_ring,622 int budget)623{624 unsigned int total_rx_bytes = 0, total_rx_packets = 0;625 u16 cleaned_count = wx_desc_unused(rx_ring);626 627 do {628 struct wx_rx_buffer *rx_buffer;629 union wx_rx_desc *rx_desc;630 struct sk_buff *skb;631 int rx_buffer_pgcnt;632 633 /* return some buffers to hardware, one at a time is too slow */634 if (cleaned_count >= WX_RX_BUFFER_WRITE) {635 wx_alloc_rx_buffers(rx_ring, cleaned_count);636 cleaned_count = 0;637 }638 639 rx_desc = WX_RX_DESC(rx_ring, rx_ring->next_to_clean);640 if (!wx_test_staterr(rx_desc, WX_RXD_STAT_DD))641 break;642 643 /* This memory barrier is needed to keep us from reading644 * any other fields out of the rx_desc until we know the645 * descriptor has been written back646 */647 dma_rmb();648 649 rx_buffer = wx_get_rx_buffer(rx_ring, rx_desc, &skb, &rx_buffer_pgcnt);650 651 /* retrieve a buffer from the ring */652 skb = wx_build_skb(rx_ring, rx_buffer, rx_desc);653 654 /* exit if we failed to retrieve a buffer */655 if (!skb) {656 rx_ring->rx_stats.alloc_rx_buff_failed++;657 break;658 }659 660 wx_put_rx_buffer(rx_ring, rx_buffer, skb, rx_buffer_pgcnt);661 cleaned_count++;662 663 /* place incomplete frames back on ring for completion */664 if (wx_is_non_eop(rx_ring, rx_desc, skb))665 continue;666 667 /* verify the packet layout is correct */668 if (wx_cleanup_headers(rx_ring, rx_desc, skb))669 continue;670 671 /* probably a little skewed due to removing CRC */672 total_rx_bytes += skb->len;673 674 /* populate checksum, timestamp, VLAN, and protocol */675 wx_process_skb_fields(rx_ring, rx_desc, skb);676 napi_gro_receive(&q_vector->napi, skb);677 678 /* update budget accounting */679 total_rx_packets++;680 } while (likely(total_rx_packets < budget));681 682 u64_stats_update_begin(&rx_ring->syncp);683 rx_ring->stats.packets += total_rx_packets;684 rx_ring->stats.bytes += total_rx_bytes;685 u64_stats_update_end(&rx_ring->syncp);686 q_vector->rx.total_packets += total_rx_packets;687 q_vector->rx.total_bytes += total_rx_bytes;688 689 return total_rx_packets;690}691 692static struct netdev_queue *wx_txring_txq(const struct wx_ring *ring)693{694 return netdev_get_tx_queue(ring->netdev, ring->queue_index);695}696 697/**698 * wx_clean_tx_irq - Reclaim resources after transmit completes699 * @q_vector: structure containing interrupt and ring information700 * @tx_ring: tx ring to clean701 * @napi_budget: Used to determine if we are in netpoll702 **/703static bool wx_clean_tx_irq(struct wx_q_vector *q_vector,704 struct wx_ring *tx_ring, int napi_budget)705{706 unsigned int budget = q_vector->wx->tx_work_limit;707 unsigned int total_bytes = 0, total_packets = 0;708 unsigned int i = tx_ring->next_to_clean;709 struct wx_tx_buffer *tx_buffer;710 union wx_tx_desc *tx_desc;711 712 if (!netif_carrier_ok(tx_ring->netdev))713 return true;714 715 tx_buffer = &tx_ring->tx_buffer_info[i];716 tx_desc = WX_TX_DESC(tx_ring, i);717 i -= tx_ring->count;718 719 do {720 union wx_tx_desc *eop_desc = tx_buffer->next_to_watch;721 722 /* if next_to_watch is not set then there is no work pending */723 if (!eop_desc)724 break;725 726 /* prevent any other reads prior to eop_desc */727 smp_rmb();728 729 /* if DD is not set pending work has not been completed */730 if (!(eop_desc->wb.status & cpu_to_le32(WX_TXD_STAT_DD)))731 break;732 733 /* clear next_to_watch to prevent false hangs */734 tx_buffer->next_to_watch = NULL;735 736 /* update the statistics for this packet */737 total_bytes += tx_buffer->bytecount;738 total_packets += tx_buffer->gso_segs;739 740 /* free the skb */741 napi_consume_skb(tx_buffer->skb, napi_budget);742 743 /* unmap skb header data */744 dma_unmap_single(tx_ring->dev,745 dma_unmap_addr(tx_buffer, dma),746 dma_unmap_len(tx_buffer, len),747 DMA_TO_DEVICE);748 749 /* clear tx_buffer data */750 dma_unmap_len_set(tx_buffer, len, 0);751 752 /* unmap remaining buffers */753 while (tx_desc != eop_desc) {754 tx_buffer++;755 tx_desc++;756 i++;757 if (unlikely(!i)) {758 i -= tx_ring->count;759 tx_buffer = tx_ring->tx_buffer_info;760 tx_desc = WX_TX_DESC(tx_ring, 0);761 }762 763 /* unmap any remaining paged data */764 if (dma_unmap_len(tx_buffer, len)) {765 dma_unmap_page(tx_ring->dev,766 dma_unmap_addr(tx_buffer, dma),767 dma_unmap_len(tx_buffer, len),768 DMA_TO_DEVICE);769 dma_unmap_len_set(tx_buffer, len, 0);770 }771 }772 773 /* move us one more past the eop_desc for start of next pkt */774 tx_buffer++;775 tx_desc++;776 i++;777 if (unlikely(!i)) {778 i -= tx_ring->count;779 tx_buffer = tx_ring->tx_buffer_info;780 tx_desc = WX_TX_DESC(tx_ring, 0);781 }782 783 /* issue prefetch for next Tx descriptor */784 prefetch(tx_desc);785 786 /* update budget accounting */787 budget--;788 } while (likely(budget));789 790 i += tx_ring->count;791 tx_ring->next_to_clean = i;792 u64_stats_update_begin(&tx_ring->syncp);793 tx_ring->stats.bytes += total_bytes;794 tx_ring->stats.packets += total_packets;795 u64_stats_update_end(&tx_ring->syncp);796 q_vector->tx.total_bytes += total_bytes;797 q_vector->tx.total_packets += total_packets;798 799 netdev_tx_completed_queue(wx_txring_txq(tx_ring),800 total_packets, total_bytes);801 802#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)803 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&804 (wx_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {805 /* Make sure that anybody stopping the queue after this806 * sees the new next_to_clean.807 */808 smp_mb();809 810 if (__netif_subqueue_stopped(tx_ring->netdev,811 tx_ring->queue_index) &&812 netif_running(tx_ring->netdev)) {813 netif_wake_subqueue(tx_ring->netdev,814 tx_ring->queue_index);815 ++tx_ring->tx_stats.restart_queue;816 }817 }818 819 return !!budget;820}821 822/**823 * wx_poll - NAPI polling RX/TX cleanup routine824 * @napi: napi struct with our devices info in it825 * @budget: amount of work driver is allowed to do this pass, in packets826 *827 * This function will clean all queues associated with a q_vector.828 **/829static int wx_poll(struct napi_struct *napi, int budget)830{831 struct wx_q_vector *q_vector = container_of(napi, struct wx_q_vector, napi);832 int per_ring_budget, work_done = 0;833 struct wx *wx = q_vector->wx;834 bool clean_complete = true;835 struct wx_ring *ring;836 837 wx_for_each_ring(ring, q_vector->tx) {838 if (!wx_clean_tx_irq(q_vector, ring, budget))839 clean_complete = false;840 }841 842 /* Exit if we are called by netpoll */843 if (budget <= 0)844 return budget;845 846 /* attempt to distribute budget to each queue fairly, but don't allow847 * the budget to go below 1 because we'll exit polling848 */849 if (q_vector->rx.count > 1)850 per_ring_budget = max(budget / q_vector->rx.count, 1);851 else852 per_ring_budget = budget;853 854 wx_for_each_ring(ring, q_vector->rx) {855 int cleaned = wx_clean_rx_irq(q_vector, ring, per_ring_budget);856 857 work_done += cleaned;858 if (cleaned >= per_ring_budget)859 clean_complete = false;860 }861 862 /* If all work not completed, return budget and keep polling */863 if (!clean_complete)864 return budget;865 866 /* all work done, exit the polling mode */867 if (likely(napi_complete_done(napi, work_done))) {868 if (netif_running(wx->netdev))869 wx_intr_enable(wx, WX_INTR_Q(q_vector->v_idx));870 }871 872 return min(work_done, budget - 1);873}874 875static int wx_maybe_stop_tx(struct wx_ring *tx_ring, u16 size)876{877 if (likely(wx_desc_unused(tx_ring) >= size))878 return 0;879 880 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);881 882 /* For the next check */883 smp_mb();884 885 /* We need to check again in a case another CPU has just886 * made room available.887 */888 if (likely(wx_desc_unused(tx_ring) < size))889 return -EBUSY;890 891 /* A reprieve! - use start_queue because it doesn't call schedule */892 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);893 ++tx_ring->tx_stats.restart_queue;894 895 return 0;896}897 898static u32 wx_tx_cmd_type(u32 tx_flags)899{900 /* set type for advanced descriptor with frame checksum insertion */901 u32 cmd_type = WX_TXD_DTYP_DATA | WX_TXD_IFCS;902 903 /* set HW vlan bit if vlan is present */904 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_HW_VLAN, WX_TXD_VLE);905 /* set segmentation enable bits for TSO/FSO */906 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_TSO, WX_TXD_TSE);907 /* set timestamp bit if present */908 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_TSTAMP, WX_TXD_MAC_TSTAMP);909 cmd_type |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_LINKSEC, WX_TXD_LINKSEC);910 911 return cmd_type;912}913 914static void wx_tx_olinfo_status(union wx_tx_desc *tx_desc,915 u32 tx_flags, unsigned int paylen)916{917 u32 olinfo_status = paylen << WX_TXD_PAYLEN_SHIFT;918 919 /* enable L4 checksum for TSO and TX checksum offload */920 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_CSUM, WX_TXD_L4CS);921 /* enable IPv4 checksum for TSO */922 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_IPV4, WX_TXD_IIPCS);923 /* enable outer IPv4 checksum for TSO */924 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_OUTER_IPV4,925 WX_TXD_EIPCS);926 /* Check Context must be set if Tx switch is enabled, which it927 * always is for case where virtual functions are running928 */929 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_CC, WX_TXD_CC);930 olinfo_status |= WX_SET_FLAG(tx_flags, WX_TX_FLAGS_IPSEC,931 WX_TXD_IPSEC);932 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);933}934 935static void wx_tx_map(struct wx_ring *tx_ring,936 struct wx_tx_buffer *first,937 const u8 hdr_len)938{939 struct sk_buff *skb = first->skb;940 struct wx_tx_buffer *tx_buffer;941 u32 tx_flags = first->tx_flags;942 u16 i = tx_ring->next_to_use;943 unsigned int data_len, size;944 union wx_tx_desc *tx_desc;945 skb_frag_t *frag;946 dma_addr_t dma;947 u32 cmd_type;948 949 cmd_type = wx_tx_cmd_type(tx_flags);950 tx_desc = WX_TX_DESC(tx_ring, i);951 wx_tx_olinfo_status(tx_desc, tx_flags, skb->len - hdr_len);952 953 size = skb_headlen(skb);954 data_len = skb->data_len;955 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);956 957 tx_buffer = first;958 959 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {960 if (dma_mapping_error(tx_ring->dev, dma))961 goto dma_error;962 963 /* record length, and DMA address */964 dma_unmap_len_set(tx_buffer, len, size);965 dma_unmap_addr_set(tx_buffer, dma, dma);966 967 tx_desc->read.buffer_addr = cpu_to_le64(dma);968 969 while (unlikely(size > WX_MAX_DATA_PER_TXD)) {970 tx_desc->read.cmd_type_len =971 cpu_to_le32(cmd_type ^ WX_MAX_DATA_PER_TXD);972 973 i++;974 tx_desc++;975 if (i == tx_ring->count) {976 tx_desc = WX_TX_DESC(tx_ring, 0);977 i = 0;978 }979 tx_desc->read.olinfo_status = 0;980 981 dma += WX_MAX_DATA_PER_TXD;982 size -= WX_MAX_DATA_PER_TXD;983 984 tx_desc->read.buffer_addr = cpu_to_le64(dma);985 }986 987 if (likely(!data_len))988 break;989 990 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);991 992 i++;993 tx_desc++;994 if (i == tx_ring->count) {995 tx_desc = WX_TX_DESC(tx_ring, 0);996 i = 0;997 }998 tx_desc->read.olinfo_status = 0;999 1000 size = skb_frag_size(frag);1001 1002 data_len -= size;1003 1004 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,1005 DMA_TO_DEVICE);1006 1007 tx_buffer = &tx_ring->tx_buffer_info[i];1008 }1009 1010 /* write last descriptor with RS and EOP bits */1011 cmd_type |= size | WX_TXD_EOP | WX_TXD_RS;1012 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);1013 1014 netdev_tx_sent_queue(wx_txring_txq(tx_ring), first->bytecount);1015 1016 skb_tx_timestamp(skb);1017 1018 /* Force memory writes to complete before letting h/w know there1019 * are new descriptors to fetch. (Only applicable for weak-ordered1020 * memory model archs, such as IA-64).1021 *1022 * We also need this memory barrier to make certain all of the1023 * status bits have been updated before next_to_watch is written.1024 */1025 wmb();1026 1027 /* set next_to_watch value indicating a packet is present */1028 first->next_to_watch = tx_desc;1029 1030 i++;1031 if (i == tx_ring->count)1032 i = 0;1033 1034 tx_ring->next_to_use = i;1035 1036 wx_maybe_stop_tx(tx_ring, DESC_NEEDED);1037 1038 if (netif_xmit_stopped(wx_txring_txq(tx_ring)) || !netdev_xmit_more())1039 writel(i, tx_ring->tail);1040 1041 return;1042dma_error:1043 dev_err(tx_ring->dev, "TX DMA map failed\n");1044 1045 /* clear dma mappings for failed tx_buffer_info map */1046 for (;;) {1047 tx_buffer = &tx_ring->tx_buffer_info[i];1048 if (dma_unmap_len(tx_buffer, len))1049 dma_unmap_page(tx_ring->dev,1050 dma_unmap_addr(tx_buffer, dma),1051 dma_unmap_len(tx_buffer, len),1052 DMA_TO_DEVICE);1053 dma_unmap_len_set(tx_buffer, len, 0);1054 if (tx_buffer == first)1055 break;1056 if (i == 0)1057 i += tx_ring->count;1058 i--;1059 }1060 1061 dev_kfree_skb_any(first->skb);1062 first->skb = NULL;1063 1064 tx_ring->next_to_use = i;1065}1066 1067static void wx_tx_ctxtdesc(struct wx_ring *tx_ring, u32 vlan_macip_lens,1068 u32 fcoe_sof_eof, u32 type_tucmd, u32 mss_l4len_idx)1069{1070 struct wx_tx_context_desc *context_desc;1071 u16 i = tx_ring->next_to_use;1072 1073 context_desc = WX_TX_CTXTDESC(tx_ring, i);1074 i++;1075 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;1076 1077 /* set bits to identify this as an advanced context descriptor */1078 type_tucmd |= WX_TXD_DTYP_CTXT;1079 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);1080 context_desc->seqnum_seed = cpu_to_le32(fcoe_sof_eof);1081 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);1082 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);1083}1084 1085static void wx_get_ipv6_proto(struct sk_buff *skb, int offset, u8 *nexthdr)1086{1087 struct ipv6hdr *hdr = (struct ipv6hdr *)(skb->data + offset);1088 1089 *nexthdr = hdr->nexthdr;1090 offset += sizeof(struct ipv6hdr);1091 while (ipv6_ext_hdr(*nexthdr)) {1092 struct ipv6_opt_hdr _hdr, *hp;1093 1094 if (*nexthdr == NEXTHDR_NONE)1095 return;1096 hp = skb_header_pointer(skb, offset, sizeof(_hdr), &_hdr);1097 if (!hp)1098 return;1099 if (*nexthdr == NEXTHDR_FRAGMENT)1100 break;1101 *nexthdr = hp->nexthdr;1102 }1103}1104 1105union network_header {1106 struct iphdr *ipv4;1107 struct ipv6hdr *ipv6;1108 void *raw;1109};1110 1111static u8 wx_encode_tx_desc_ptype(const struct wx_tx_buffer *first)1112{1113 u8 tun_prot = 0, l4_prot = 0, ptype = 0;1114 struct sk_buff *skb = first->skb;1115 1116 if (skb->encapsulation) {1117 union network_header hdr;1118 1119 switch (first->protocol) {1120 case htons(ETH_P_IP):1121 tun_prot = ip_hdr(skb)->protocol;1122 ptype = WX_PTYPE_TUN_IPV4;1123 break;1124 case htons(ETH_P_IPV6):1125 wx_get_ipv6_proto(skb, skb_network_offset(skb), &tun_prot);1126 ptype = WX_PTYPE_TUN_IPV6;1127 break;1128 default:1129 return ptype;1130 }1131 1132 if (tun_prot == IPPROTO_IPIP) {1133 hdr.raw = (void *)inner_ip_hdr(skb);1134 ptype |= WX_PTYPE_PKT_IPIP;1135 } else if (tun_prot == IPPROTO_UDP) {1136 hdr.raw = (void *)inner_ip_hdr(skb);1137 if (skb->inner_protocol_type != ENCAP_TYPE_ETHER ||1138 skb->inner_protocol != htons(ETH_P_TEB)) {1139 ptype |= WX_PTYPE_PKT_IG;1140 } else {1141 if (((struct ethhdr *)skb_inner_mac_header(skb))->h_proto1142 == htons(ETH_P_8021Q))1143 ptype |= WX_PTYPE_PKT_IGMV;1144 else1145 ptype |= WX_PTYPE_PKT_IGM;1146 }1147 1148 } else if (tun_prot == IPPROTO_GRE) {1149 hdr.raw = (void *)inner_ip_hdr(skb);1150 if (skb->inner_protocol == htons(ETH_P_IP) ||1151 skb->inner_protocol == htons(ETH_P_IPV6)) {1152 ptype |= WX_PTYPE_PKT_IG;1153 } else {1154 if (((struct ethhdr *)skb_inner_mac_header(skb))->h_proto1155 == htons(ETH_P_8021Q))1156 ptype |= WX_PTYPE_PKT_IGMV;1157 else1158 ptype |= WX_PTYPE_PKT_IGM;1159 }1160 } else {1161 return ptype;1162 }1163 1164 switch (hdr.ipv4->version) {1165 case IPVERSION:1166 l4_prot = hdr.ipv4->protocol;1167 break;1168 case 6:1169 wx_get_ipv6_proto(skb, skb_inner_network_offset(skb), &l4_prot);1170 ptype |= WX_PTYPE_PKT_IPV6;1171 break;1172 default:1173 return ptype;1174 }1175 } else {1176 switch (first->protocol) {1177 case htons(ETH_P_IP):1178 l4_prot = ip_hdr(skb)->protocol;1179 ptype = WX_PTYPE_PKT_IP;1180 break;1181 case htons(ETH_P_IPV6):1182 wx_get_ipv6_proto(skb, skb_network_offset(skb), &l4_prot);1183 ptype = WX_PTYPE_PKT_IP | WX_PTYPE_PKT_IPV6;1184 break;1185 default:1186 return WX_PTYPE_PKT_MAC | WX_PTYPE_TYP_MAC;1187 }1188 }1189 switch (l4_prot) {1190 case IPPROTO_TCP:1191 ptype |= WX_PTYPE_TYP_TCP;1192 break;1193 case IPPROTO_UDP:1194 ptype |= WX_PTYPE_TYP_UDP;1195 break;1196 case IPPROTO_SCTP:1197 ptype |= WX_PTYPE_TYP_SCTP;1198 break;1199 default:1200 ptype |= WX_PTYPE_TYP_IP;1201 break;1202 }1203 1204 return ptype;1205}1206 1207static int wx_tso(struct wx_ring *tx_ring, struct wx_tx_buffer *first,1208 u8 *hdr_len, u8 ptype)1209{1210 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;1211 struct net_device *netdev = tx_ring->netdev;1212 u32 l4len, tunhdr_eiplen_tunlen = 0;1213 struct sk_buff *skb = first->skb;1214 bool enc = skb->encapsulation;1215 struct ipv6hdr *ipv6h;1216 struct tcphdr *tcph;1217 struct iphdr *iph;1218 u8 tun_prot = 0;1219 int err;1220 1221 if (skb->ip_summed != CHECKSUM_PARTIAL)1222 return 0;1223 1224 if (!skb_is_gso(skb))1225 return 0;1226 1227 err = skb_cow_head(skb, 0);1228 if (err < 0)1229 return err;1230 1231 /* indicates the inner headers in the skbuff are valid. */1232 iph = enc ? inner_ip_hdr(skb) : ip_hdr(skb);1233 if (iph->version == 4) {1234 tcph = enc ? inner_tcp_hdr(skb) : tcp_hdr(skb);1235 iph->tot_len = 0;1236 iph->check = 0;1237 tcph->check = ~csum_tcpudp_magic(iph->saddr,1238 iph->daddr, 0,1239 IPPROTO_TCP, 0);1240 first->tx_flags |= WX_TX_FLAGS_TSO |1241 WX_TX_FLAGS_CSUM |1242 WX_TX_FLAGS_IPV4 |1243 WX_TX_FLAGS_CC;1244 } else if (iph->version == 6 && skb_is_gso_v6(skb)) {1245 ipv6h = enc ? inner_ipv6_hdr(skb) : ipv6_hdr(skb);1246 tcph = enc ? inner_tcp_hdr(skb) : tcp_hdr(skb);1247 ipv6h->payload_len = 0;1248 tcph->check = ~csum_ipv6_magic(&ipv6h->saddr,1249 &ipv6h->daddr, 0,1250 IPPROTO_TCP, 0);1251 first->tx_flags |= WX_TX_FLAGS_TSO |1252 WX_TX_FLAGS_CSUM |1253 WX_TX_FLAGS_CC;1254 }1255 1256 /* compute header lengths */1257 l4len = enc ? inner_tcp_hdrlen(skb) : tcp_hdrlen(skb);1258 *hdr_len = enc ? skb_inner_transport_offset(skb) :1259 skb_transport_offset(skb);1260 *hdr_len += l4len;1261 1262 /* update gso size and bytecount with header size */1263 first->gso_segs = skb_shinfo(skb)->gso_segs;1264 first->bytecount += (first->gso_segs - 1) * *hdr_len;1265 1266 /* mss_l4len_id: use 0 as index for TSO */1267 mss_l4len_idx = l4len << WX_TXD_L4LEN_SHIFT;1268 mss_l4len_idx |= skb_shinfo(skb)->gso_size << WX_TXD_MSS_SHIFT;1269 1270 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */1271 if (enc) {1272 switch (first->protocol) {1273 case htons(ETH_P_IP):1274 tun_prot = ip_hdr(skb)->protocol;1275 first->tx_flags |= WX_TX_FLAGS_OUTER_IPV4;1276 break;1277 case htons(ETH_P_IPV6):1278 tun_prot = ipv6_hdr(skb)->nexthdr;1279 break;1280 default:1281 break;1282 }1283 switch (tun_prot) {1284 case IPPROTO_UDP:1285 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_UDP;1286 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<1287 WX_TXD_OUTER_IPLEN_SHIFT) |1288 (((skb_inner_mac_header(skb) -1289 skb_transport_header(skb)) >> 1) <<1290 WX_TXD_TUNNEL_LEN_SHIFT);1291 break;1292 case IPPROTO_GRE:1293 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_GRE;1294 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<1295 WX_TXD_OUTER_IPLEN_SHIFT) |1296 (((skb_inner_mac_header(skb) -1297 skb_transport_header(skb)) >> 1) <<1298 WX_TXD_TUNNEL_LEN_SHIFT);1299 break;1300 case IPPROTO_IPIP:1301 tunhdr_eiplen_tunlen = (((char *)inner_ip_hdr(skb) -1302 (char *)ip_hdr(skb)) >> 2) <<1303 WX_TXD_OUTER_IPLEN_SHIFT;1304 break;1305 default:1306 break;1307 }1308 vlan_macip_lens = skb_inner_network_header_len(skb) >> 1;1309 } else {1310 vlan_macip_lens = skb_network_header_len(skb) >> 1;1311 }1312 1313 vlan_macip_lens |= skb_network_offset(skb) << WX_TXD_MACLEN_SHIFT;1314 vlan_macip_lens |= first->tx_flags & WX_TX_FLAGS_VLAN_MASK;1315 1316 type_tucmd = ptype << 24;1317 if (skb->vlan_proto == htons(ETH_P_8021AD) &&1318 netdev->features & NETIF_F_HW_VLAN_STAG_TX)1319 type_tucmd |= WX_SET_FLAG(first->tx_flags,1320 WX_TX_FLAGS_HW_VLAN,1321 0x1 << WX_TXD_TAG_TPID_SEL_SHIFT);1322 wx_tx_ctxtdesc(tx_ring, vlan_macip_lens, tunhdr_eiplen_tunlen,1323 type_tucmd, mss_l4len_idx);1324 1325 return 1;1326}1327 1328static void wx_tx_csum(struct wx_ring *tx_ring, struct wx_tx_buffer *first,1329 u8 ptype)1330{1331 u32 tunhdr_eiplen_tunlen = 0, vlan_macip_lens = 0;1332 struct net_device *netdev = tx_ring->netdev;1333 u32 mss_l4len_idx = 0, type_tucmd;1334 struct sk_buff *skb = first->skb;1335 u8 tun_prot = 0;1336 1337 if (skb->ip_summed != CHECKSUM_PARTIAL) {1338 if (!(first->tx_flags & WX_TX_FLAGS_HW_VLAN) &&1339 !(first->tx_flags & WX_TX_FLAGS_CC))1340 return;1341 vlan_macip_lens = skb_network_offset(skb) <<1342 WX_TXD_MACLEN_SHIFT;1343 } else {1344 u8 l4_prot = 0;1345 union {1346 struct iphdr *ipv4;1347 struct ipv6hdr *ipv6;1348 u8 *raw;1349 } network_hdr;1350 union {1351 struct tcphdr *tcphdr;1352 u8 *raw;1353 } transport_hdr;1354 1355 if (skb->encapsulation) {1356 network_hdr.raw = skb_inner_network_header(skb);1357 transport_hdr.raw = skb_inner_transport_header(skb);1358 vlan_macip_lens = skb_network_offset(skb) <<1359 WX_TXD_MACLEN_SHIFT;1360 switch (first->protocol) {1361 case htons(ETH_P_IP):1362 tun_prot = ip_hdr(skb)->protocol;1363 break;1364 case htons(ETH_P_IPV6):1365 tun_prot = ipv6_hdr(skb)->nexthdr;1366 break;1367 default:1368 return;1369 }1370 switch (tun_prot) {1371 case IPPROTO_UDP:1372 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_UDP;1373 tunhdr_eiplen_tunlen |=1374 ((skb_network_header_len(skb) >> 2) <<1375 WX_TXD_OUTER_IPLEN_SHIFT) |1376 (((skb_inner_mac_header(skb) -1377 skb_transport_header(skb)) >> 1) <<1378 WX_TXD_TUNNEL_LEN_SHIFT);1379 break;1380 case IPPROTO_GRE:1381 tunhdr_eiplen_tunlen = WX_TXD_TUNNEL_GRE;1382 tunhdr_eiplen_tunlen |= ((skb_network_header_len(skb) >> 2) <<1383 WX_TXD_OUTER_IPLEN_SHIFT) |1384 (((skb_inner_mac_header(skb) -1385 skb_transport_header(skb)) >> 1) <<1386 WX_TXD_TUNNEL_LEN_SHIFT);1387 break;1388 case IPPROTO_IPIP:1389 tunhdr_eiplen_tunlen = (((char *)inner_ip_hdr(skb) -1390 (char *)ip_hdr(skb)) >> 2) <<1391 WX_TXD_OUTER_IPLEN_SHIFT;1392 break;1393 default:1394 break;1395 }1396 1397 } else {1398 network_hdr.raw = skb_network_header(skb);1399 transport_hdr.raw = skb_transport_header(skb);1400 vlan_macip_lens = skb_network_offset(skb) <<1401 WX_TXD_MACLEN_SHIFT;1402 }1403 1404 switch (network_hdr.ipv4->version) {1405 case IPVERSION:1406 vlan_macip_lens |= (transport_hdr.raw - network_hdr.raw) >> 1;1407 l4_prot = network_hdr.ipv4->protocol;1408 break;1409 case 6:1410 vlan_macip_lens |= (transport_hdr.raw - network_hdr.raw) >> 1;1411 l4_prot = network_hdr.ipv6->nexthdr;1412 break;1413 default:1414 break;1415 }1416 1417 switch (l4_prot) {1418 case IPPROTO_TCP:1419 mss_l4len_idx = (transport_hdr.tcphdr->doff * 4) <<1420 WX_TXD_L4LEN_SHIFT;1421 break;1422 case IPPROTO_SCTP:1423 mss_l4len_idx = sizeof(struct sctphdr) <<1424 WX_TXD_L4LEN_SHIFT;1425 break;1426 case IPPROTO_UDP:1427 mss_l4len_idx = sizeof(struct udphdr) <<1428 WX_TXD_L4LEN_SHIFT;1429 break;1430 default:1431 break;1432 }1433 1434 /* update TX checksum flag */1435 first->tx_flags |= WX_TX_FLAGS_CSUM;1436 }1437 first->tx_flags |= WX_TX_FLAGS_CC;1438 /* vlan_macip_lens: MACLEN, VLAN tag */1439 vlan_macip_lens |= first->tx_flags & WX_TX_FLAGS_VLAN_MASK;1440 1441 type_tucmd = ptype << 24;1442 if (skb->vlan_proto == htons(ETH_P_8021AD) &&1443 netdev->features & NETIF_F_HW_VLAN_STAG_TX)1444 type_tucmd |= WX_SET_FLAG(first->tx_flags,1445 WX_TX_FLAGS_HW_VLAN,1446 0x1 << WX_TXD_TAG_TPID_SEL_SHIFT);1447 wx_tx_ctxtdesc(tx_ring, vlan_macip_lens, tunhdr_eiplen_tunlen,1448 type_tucmd, mss_l4len_idx);1449}1450 1451static netdev_tx_t wx_xmit_frame_ring(struct sk_buff *skb,1452 struct wx_ring *tx_ring)1453{1454 struct wx *wx = netdev_priv(tx_ring->netdev);1455 u16 count = TXD_USE_COUNT(skb_headlen(skb));1456 struct wx_tx_buffer *first;1457 u8 hdr_len = 0, ptype;1458 unsigned short f;1459 u32 tx_flags = 0;1460 int tso;1461 1462 /* need: 1 descriptor per page * PAGE_SIZE/WX_MAX_DATA_PER_TXD,1463 * + 1 desc for skb_headlen/WX_MAX_DATA_PER_TXD,1464 * + 2 desc gap to keep tail from touching head,1465 * + 1 desc for context descriptor,1466 * otherwise try next time1467 */1468 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)1469 count += TXD_USE_COUNT(skb_frag_size(&skb_shinfo(skb)->1470 frags[f]));1471 1472 if (wx_maybe_stop_tx(tx_ring, count + 3)) {1473 tx_ring->tx_stats.tx_busy++;1474 return NETDEV_TX_BUSY;1475 }1476 1477 /* record the location of the first descriptor for this packet */1478 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];1479 first->skb = skb;1480 first->bytecount = skb->len;1481 first->gso_segs = 1;1482 1483 /* if we have a HW VLAN tag being added default to the HW one */1484 if (skb_vlan_tag_present(skb)) {1485 tx_flags |= skb_vlan_tag_get(skb) << WX_TX_FLAGS_VLAN_SHIFT;1486 tx_flags |= WX_TX_FLAGS_HW_VLAN;1487 }1488 1489 /* record initial flags and protocol */1490 first->tx_flags = tx_flags;1491 first->protocol = vlan_get_protocol(skb);1492 1493 ptype = wx_encode_tx_desc_ptype(first);1494 1495 tso = wx_tso(tx_ring, first, &hdr_len, ptype);1496 if (tso < 0)1497 goto out_drop;1498 else if (!tso)1499 wx_tx_csum(tx_ring, first, ptype);1500 1501 if (test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags) && tx_ring->atr_sample_rate)1502 wx->atr(tx_ring, first, ptype);1503 1504 wx_tx_map(tx_ring, first, hdr_len);1505 1506 return NETDEV_TX_OK;1507out_drop:1508 dev_kfree_skb_any(first->skb);1509 first->skb = NULL;1510 1511 return NETDEV_TX_OK;1512}1513 1514netdev_tx_t wx_xmit_frame(struct sk_buff *skb,1515 struct net_device *netdev)1516{1517 unsigned int r_idx = skb->queue_mapping;1518 struct wx *wx = netdev_priv(netdev);1519 struct wx_ring *tx_ring;1520 1521 if (!netif_carrier_ok(netdev)) {1522 dev_kfree_skb_any(skb);1523 return NETDEV_TX_OK;1524 }1525 1526 /* The minimum packet size for olinfo paylen is 17 so pad the skb1527 * in order to meet this minimum size requirement.1528 */1529 if (skb_put_padto(skb, 17))1530 return NETDEV_TX_OK;1531 1532 if (r_idx >= wx->num_tx_queues)1533 r_idx = r_idx % wx->num_tx_queues;1534 tx_ring = wx->tx_ring[r_idx];1535 1536 return wx_xmit_frame_ring(skb, tx_ring);1537}1538EXPORT_SYMBOL(wx_xmit_frame);1539 1540void wx_napi_enable_all(struct wx *wx)1541{1542 struct wx_q_vector *q_vector;1543 int q_idx;1544 1545 for (q_idx = 0; q_idx < wx->num_q_vectors; q_idx++) {1546 q_vector = wx->q_vector[q_idx];1547 napi_enable(&q_vector->napi);1548 }1549}1550EXPORT_SYMBOL(wx_napi_enable_all);1551 1552void wx_napi_disable_all(struct wx *wx)1553{1554 struct wx_q_vector *q_vector;1555 int q_idx;1556 1557 for (q_idx = 0; q_idx < wx->num_q_vectors; q_idx++) {1558 q_vector = wx->q_vector[q_idx];1559 napi_disable(&q_vector->napi);1560 }1561}1562EXPORT_SYMBOL(wx_napi_disable_all);1563 1564/**1565 * wx_set_rss_queues: Allocate queues for RSS1566 * @wx: board private structure to initialize1567 *1568 * This is our "base" multiqueue mode. RSS (Receive Side Scaling) will try1569 * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU.1570 *1571 **/1572static void wx_set_rss_queues(struct wx *wx)1573{1574 struct wx_ring_feature *f;1575 1576 /* set mask for 16 queue limit of RSS */1577 f = &wx->ring_feature[RING_F_RSS];1578 f->indices = f->limit;1579 1580 if (!(test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags)))1581 goto out;1582 1583 clear_bit(WX_FLAG_FDIR_HASH, wx->flags);1584 1585 /* Use Flow Director in addition to RSS to ensure the best1586 * distribution of flows across cores, even when an FDIR flow1587 * isn't matched.1588 */1589 if (f->indices > 1) {1590 f = &wx->ring_feature[RING_F_FDIR];1591 1592 f->indices = f->limit;1593 1594 if (!(test_bit(WX_FLAG_FDIR_PERFECT, wx->flags)))1595 set_bit(WX_FLAG_FDIR_HASH, wx->flags);1596 }1597 1598out:1599 wx->num_rx_queues = f->indices;1600 wx->num_tx_queues = f->indices;1601}1602 1603static void wx_set_num_queues(struct wx *wx)1604{1605 /* Start with base case */1606 wx->num_rx_queues = 1;1607 wx->num_tx_queues = 1;1608 wx->queues_per_pool = 1;1609 1610 wx_set_rss_queues(wx);1611}1612 1613/**1614 * wx_acquire_msix_vectors - acquire MSI-X vectors1615 * @wx: board private structure1616 *1617 * Attempts to acquire a suitable range of MSI-X vector interrupts. Will1618 * return a negative error code if unable to acquire MSI-X vectors for any1619 * reason.1620 */1621static int wx_acquire_msix_vectors(struct wx *wx)1622{1623 struct irq_affinity affd = { .pre_vectors = 1 };1624 int nvecs, i;1625 1626 /* We start by asking for one vector per queue pair */1627 nvecs = max(wx->num_rx_queues, wx->num_tx_queues);1628 nvecs = min_t(int, nvecs, num_online_cpus());1629 nvecs = min_t(int, nvecs, wx->mac.max_msix_vectors);1630 1631 wx->msix_q_entries = kcalloc(nvecs, sizeof(struct msix_entry),1632 GFP_KERNEL);1633 if (!wx->msix_q_entries)1634 return -ENOMEM;1635 1636 /* One for non-queue interrupts */1637 nvecs += 1;1638 1639 wx->msix_entry = kcalloc(1, sizeof(struct msix_entry),1640 GFP_KERNEL);1641 if (!wx->msix_entry) {1642 kfree(wx->msix_q_entries);1643 wx->msix_q_entries = NULL;1644 return -ENOMEM;1645 }1646 1647 nvecs = pci_alloc_irq_vectors_affinity(wx->pdev, nvecs,1648 nvecs,1649 PCI_IRQ_MSIX | PCI_IRQ_AFFINITY,1650 &affd);1651 if (nvecs < 0) {1652 wx_err(wx, "Failed to allocate MSI-X interrupts. Err: %d\n", nvecs);1653 kfree(wx->msix_q_entries);1654 wx->msix_q_entries = NULL;1655 kfree(wx->msix_entry);1656 wx->msix_entry = NULL;1657 return nvecs;1658 }1659 1660 wx->msix_entry->entry = 0;1661 wx->msix_entry->vector = pci_irq_vector(wx->pdev, 0);1662 nvecs -= 1;1663 for (i = 0; i < nvecs; i++) {1664 wx->msix_q_entries[i].entry = i;1665 wx->msix_q_entries[i].vector = pci_irq_vector(wx->pdev, i + 1);1666 }1667 1668 wx->num_q_vectors = nvecs;1669 1670 return 0;1671}1672 1673/**1674 * wx_set_interrupt_capability - set MSI-X or MSI if supported1675 * @wx: board private structure to initialize1676 *1677 * Attempt to configure the interrupts using the best available1678 * capabilities of the hardware and the kernel.1679 **/1680static int wx_set_interrupt_capability(struct wx *wx)1681{1682 struct pci_dev *pdev = wx->pdev;1683 int nvecs, ret;1684 1685 /* We will try to get MSI-X interrupts first */1686 ret = wx_acquire_msix_vectors(wx);1687 if (ret == 0 || (ret == -ENOMEM))1688 return ret;1689 1690 /* Disable RSS */1691 dev_warn(&wx->pdev->dev, "Disabling RSS support\n");1692 wx->ring_feature[RING_F_RSS].limit = 1;1693 1694 wx_set_num_queues(wx);1695 1696 /* minmum one for queue, one for misc*/1697 nvecs = 1;1698 nvecs = pci_alloc_irq_vectors(pdev, nvecs,1699 nvecs, PCI_IRQ_MSI | PCI_IRQ_INTX);1700 if (nvecs == 1) {1701 if (pdev->msi_enabled)1702 wx_err(wx, "Fallback to MSI.\n");1703 else1704 wx_err(wx, "Fallback to INTx.\n");1705 } else {1706 wx_err(wx, "Failed to allocate MSI/INTx interrupts. Error: %d\n", nvecs);1707 return nvecs;1708 }1709 1710 pdev->irq = pci_irq_vector(pdev, 0);1711 wx->num_q_vectors = 1;1712 1713 return 0;1714}1715 1716/**1717 * wx_cache_ring_rss - Descriptor ring to register mapping for RSS1718 * @wx: board private structure to initialize1719 *1720 * Cache the descriptor ring offsets for RSS, ATR, FCoE, and SR-IOV.1721 *1722 **/1723static void wx_cache_ring_rss(struct wx *wx)1724{1725 u16 i;1726 1727 for (i = 0; i < wx->num_rx_queues; i++)1728 wx->rx_ring[i]->reg_idx = i;1729 1730 for (i = 0; i < wx->num_tx_queues; i++)1731 wx->tx_ring[i]->reg_idx = i;1732}1733 1734static void wx_add_ring(struct wx_ring *ring, struct wx_ring_container *head)1735{1736 ring->next = head->ring;1737 head->ring = ring;1738 head->count++;1739}1740 1741/**1742 * wx_alloc_q_vector - Allocate memory for a single interrupt vector1743 * @wx: board private structure to initialize1744 * @v_count: q_vectors allocated on wx, used for ring interleaving1745 * @v_idx: index of vector in wx struct1746 * @txr_count: total number of Tx rings to allocate1747 * @txr_idx: index of first Tx ring to allocate1748 * @rxr_count: total number of Rx rings to allocate1749 * @rxr_idx: index of first Rx ring to allocate1750 *1751 * We allocate one q_vector. If allocation fails we return -ENOMEM.1752 **/1753static int wx_alloc_q_vector(struct wx *wx,1754 unsigned int v_count, unsigned int v_idx,1755 unsigned int txr_count, unsigned int txr_idx,1756 unsigned int rxr_count, unsigned int rxr_idx)1757{1758 struct wx_q_vector *q_vector;1759 int ring_count, default_itr;1760 struct wx_ring *ring;1761 1762 /* note this will allocate space for the ring structure as well! */1763 ring_count = txr_count + rxr_count;1764 1765 q_vector = kzalloc(struct_size(q_vector, ring, ring_count),1766 GFP_KERNEL);1767 if (!q_vector)1768 return -ENOMEM;1769 1770 /* initialize NAPI */1771 netif_napi_add(wx->netdev, &q_vector->napi,1772 wx_poll);1773 1774 /* tie q_vector and wx together */1775 wx->q_vector[v_idx] = q_vector;1776 q_vector->wx = wx;1777 q_vector->v_idx = v_idx;1778 if (cpu_online(v_idx))1779 q_vector->numa_node = cpu_to_node(v_idx);1780 1781 /* initialize pointer to rings */1782 ring = q_vector->ring;1783 1784 if (wx->mac.type == wx_mac_sp)1785 default_itr = WX_12K_ITR;1786 else1787 default_itr = WX_7K_ITR;1788 /* initialize ITR */1789 if (txr_count && !rxr_count)1790 /* tx only vector */1791 q_vector->itr = wx->tx_itr_setting ?1792 default_itr : wx->tx_itr_setting;1793 else1794 /* rx or rx/tx vector */1795 q_vector->itr = wx->rx_itr_setting ?1796 default_itr : wx->rx_itr_setting;1797 1798 while (txr_count) {1799 /* assign generic ring traits */1800 ring->dev = &wx->pdev->dev;1801 ring->netdev = wx->netdev;1802 1803 /* configure backlink on ring */1804 ring->q_vector = q_vector;1805 1806 /* update q_vector Tx values */1807 wx_add_ring(ring, &q_vector->tx);1808 1809 /* apply Tx specific ring traits */1810 ring->count = wx->tx_ring_count;1811 1812 ring->queue_index = txr_idx;1813 1814 /* assign ring to wx */1815 wx->tx_ring[txr_idx] = ring;1816 1817 /* update count and index */1818 txr_count--;1819 txr_idx += v_count;1820 1821 /* push pointer to next ring */1822 ring++;1823 }1824 1825 while (rxr_count) {1826 /* assign generic ring traits */1827 ring->dev = &wx->pdev->dev;1828 ring->netdev = wx->netdev;1829 1830 /* configure backlink on ring */1831 ring->q_vector = q_vector;1832 1833 /* update q_vector Rx values */1834 wx_add_ring(ring, &q_vector->rx);1835 1836 /* apply Rx specific ring traits */1837 ring->count = wx->rx_ring_count;1838 ring->queue_index = rxr_idx;1839 1840 /* assign ring to wx */1841 wx->rx_ring[rxr_idx] = ring;1842 1843 /* update count and index */1844 rxr_count--;1845 rxr_idx += v_count;1846 1847 /* push pointer to next ring */1848 ring++;1849 }1850 1851 return 0;1852}1853 1854/**1855 * wx_free_q_vector - Free memory allocated for specific interrupt vector1856 * @wx: board private structure to initialize1857 * @v_idx: Index of vector to be freed1858 *1859 * This function frees the memory allocated to the q_vector. In addition if1860 * NAPI is enabled it will delete any references to the NAPI struct prior1861 * to freeing the q_vector.1862 **/1863static void wx_free_q_vector(struct wx *wx, int v_idx)1864{1865 struct wx_q_vector *q_vector = wx->q_vector[v_idx];1866 struct wx_ring *ring;1867 1868 wx_for_each_ring(ring, q_vector->tx)1869 wx->tx_ring[ring->queue_index] = NULL;1870 1871 wx_for_each_ring(ring, q_vector->rx)1872 wx->rx_ring[ring->queue_index] = NULL;1873 1874 wx->q_vector[v_idx] = NULL;1875 netif_napi_del(&q_vector->napi);1876 kfree_rcu(q_vector, rcu);1877}1878 1879/**1880 * wx_alloc_q_vectors - Allocate memory for interrupt vectors1881 * @wx: board private structure to initialize1882 *1883 * We allocate one q_vector per queue interrupt. If allocation fails we1884 * return -ENOMEM.1885 **/1886static int wx_alloc_q_vectors(struct wx *wx)1887{1888 unsigned int rxr_idx = 0, txr_idx = 0, v_idx = 0;1889 unsigned int rxr_remaining = wx->num_rx_queues;1890 unsigned int txr_remaining = wx->num_tx_queues;1891 unsigned int q_vectors = wx->num_q_vectors;1892 int rqpv, tqpv;1893 int err;1894 1895 for (; v_idx < q_vectors; v_idx++) {1896 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);1897 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);1898 err = wx_alloc_q_vector(wx, q_vectors, v_idx,1899 tqpv, txr_idx,1900 rqpv, rxr_idx);1901 1902 if (err)1903 goto err_out;1904 1905 /* update counts and index */1906 rxr_remaining -= rqpv;1907 txr_remaining -= tqpv;1908 rxr_idx++;1909 txr_idx++;1910 }1911 1912 return 0;1913 1914err_out:1915 wx->num_tx_queues = 0;1916 wx->num_rx_queues = 0;1917 wx->num_q_vectors = 0;1918 1919 while (v_idx--)1920 wx_free_q_vector(wx, v_idx);1921 1922 return -ENOMEM;1923}1924 1925/**1926 * wx_free_q_vectors - Free memory allocated for interrupt vectors1927 * @wx: board private structure to initialize1928 *1929 * This function frees the memory allocated to the q_vectors. In addition if1930 * NAPI is enabled it will delete any references to the NAPI struct prior1931 * to freeing the q_vector.1932 **/1933static void wx_free_q_vectors(struct wx *wx)1934{1935 int v_idx = wx->num_q_vectors;1936 1937 wx->num_tx_queues = 0;1938 wx->num_rx_queues = 0;1939 wx->num_q_vectors = 0;1940 1941 while (v_idx--)1942 wx_free_q_vector(wx, v_idx);1943}1944 1945void wx_reset_interrupt_capability(struct wx *wx)1946{1947 struct pci_dev *pdev = wx->pdev;1948 1949 if (!pdev->msi_enabled && !pdev->msix_enabled)1950 return;1951 1952 if (pdev->msix_enabled) {1953 kfree(wx->msix_q_entries);1954 wx->msix_q_entries = NULL;1955 kfree(wx->msix_entry);1956 wx->msix_entry = NULL;1957 }1958 pci_free_irq_vectors(wx->pdev);1959}1960EXPORT_SYMBOL(wx_reset_interrupt_capability);1961 1962/**1963 * wx_clear_interrupt_scheme - Clear the current interrupt scheme settings1964 * @wx: board private structure to clear interrupt scheme on1965 *1966 * We go through and clear interrupt specific resources and reset the structure1967 * to pre-load conditions1968 **/1969void wx_clear_interrupt_scheme(struct wx *wx)1970{1971 wx_free_q_vectors(wx);1972 wx_reset_interrupt_capability(wx);1973}1974EXPORT_SYMBOL(wx_clear_interrupt_scheme);1975 1976int wx_init_interrupt_scheme(struct wx *wx)1977{1978 int ret;1979 1980 /* Number of supported queues */1981 wx_set_num_queues(wx);1982 1983 /* Set interrupt mode */1984 ret = wx_set_interrupt_capability(wx);1985 if (ret) {1986 wx_err(wx, "Allocate irq vectors for failed.\n");1987 return ret;1988 }1989 1990 /* Allocate memory for queues */1991 ret = wx_alloc_q_vectors(wx);1992 if (ret) {1993 wx_err(wx, "Unable to allocate memory for queue vectors.\n");1994 wx_reset_interrupt_capability(wx);1995 return ret;1996 }1997 1998 wx_cache_ring_rss(wx);1999 2000 return 0;2001}2002EXPORT_SYMBOL(wx_init_interrupt_scheme);2003 2004irqreturn_t wx_msix_clean_rings(int __always_unused irq, void *data)2005{2006 struct wx_q_vector *q_vector = data;2007 2008 /* EIAM disabled interrupts (on this vector) for us */2009 if (q_vector->rx.ring || q_vector->tx.ring)2010 napi_schedule_irqoff(&q_vector->napi);2011 2012 return IRQ_HANDLED;2013}2014EXPORT_SYMBOL(wx_msix_clean_rings);2015 2016void wx_free_irq(struct wx *wx)2017{2018 struct pci_dev *pdev = wx->pdev;2019 int vector;2020 2021 if (!(pdev->msix_enabled)) {2022 if (!wx->misc_irq_domain)2023 free_irq(pdev->irq, wx);2024 return;2025 }2026 2027 for (vector = 0; vector < wx->num_q_vectors; vector++) {2028 struct wx_q_vector *q_vector = wx->q_vector[vector];2029 struct msix_entry *entry = &wx->msix_q_entries[vector];2030 2031 /* free only the irqs that were actually requested */2032 if (!q_vector->rx.ring && !q_vector->tx.ring)2033 continue;2034 2035 free_irq(entry->vector, q_vector);2036 }2037 2038 if (!wx->misc_irq_domain)2039 free_irq(wx->msix_entry->vector, wx);2040}2041EXPORT_SYMBOL(wx_free_irq);2042 2043/**2044 * wx_setup_isb_resources - allocate interrupt status resources2045 * @wx: board private structure2046 *2047 * Return 0 on success, negative on failure2048 **/2049int wx_setup_isb_resources(struct wx *wx)2050{2051 struct pci_dev *pdev = wx->pdev;2052 2053 if (wx->isb_mem)2054 return 0;2055 2056 wx->isb_mem = dma_alloc_coherent(&pdev->dev,2057 sizeof(u32) * 4,2058 &wx->isb_dma,2059 GFP_KERNEL);2060 if (!wx->isb_mem) {2061 wx_err(wx, "Alloc isb_mem failed\n");2062 return -ENOMEM;2063 }2064 2065 return 0;2066}2067EXPORT_SYMBOL(wx_setup_isb_resources);2068 2069/**2070 * wx_free_isb_resources - allocate all queues Rx resources2071 * @wx: board private structure2072 *2073 * Return 0 on success, negative on failure2074 **/2075void wx_free_isb_resources(struct wx *wx)2076{2077 struct pci_dev *pdev = wx->pdev;2078 2079 dma_free_coherent(&pdev->dev, sizeof(u32) * 4,2080 wx->isb_mem, wx->isb_dma);2081 wx->isb_mem = NULL;2082}2083EXPORT_SYMBOL(wx_free_isb_resources);2084 2085u32 wx_misc_isb(struct wx *wx, enum wx_isb_idx idx)2086{2087 u32 cur_tag = 0;2088 2089 cur_tag = wx->isb_mem[WX_ISB_HEADER];2090 wx->isb_tag[idx] = cur_tag;2091 2092 return (__force u32)cpu_to_le32(wx->isb_mem[idx]);2093}2094EXPORT_SYMBOL(wx_misc_isb);2095 2096/**2097 * wx_set_ivar - set the IVAR registers, mapping interrupt causes to vectors2098 * @wx: pointer to wx struct2099 * @direction: 0 for Rx, 1 for Tx, -1 for other causes2100 * @queue: queue to map the corresponding interrupt to2101 * @msix_vector: the vector to map to the corresponding queue2102 *2103 **/2104static void wx_set_ivar(struct wx *wx, s8 direction,2105 u16 queue, u16 msix_vector)2106{2107 u32 ivar, index;2108 2109 if (direction == -1) {2110 /* other causes */2111 msix_vector |= WX_PX_IVAR_ALLOC_VAL;2112 index = 0;2113 ivar = rd32(wx, WX_PX_MISC_IVAR);2114 ivar &= ~(0xFF << index);2115 ivar |= (msix_vector << index);2116 wr32(wx, WX_PX_MISC_IVAR, ivar);2117 } else {2118 /* tx or rx causes */2119 msix_vector += 1; /* offset for queue vectors */2120 msix_vector |= WX_PX_IVAR_ALLOC_VAL;2121 index = ((16 * (queue & 1)) + (8 * direction));2122 ivar = rd32(wx, WX_PX_IVAR(queue >> 1));2123 ivar &= ~(0xFF << index);2124 ivar |= (msix_vector << index);2125 wr32(wx, WX_PX_IVAR(queue >> 1), ivar);2126 }2127}2128 2129/**2130 * wx_write_eitr - write EITR register in hardware specific way2131 * @q_vector: structure containing interrupt and ring information2132 *2133 * This function is made to be called by ethtool and by the driver2134 * when it needs to update EITR registers at runtime. Hardware2135 * specific quirks/differences are taken care of here.2136 */2137void wx_write_eitr(struct wx_q_vector *q_vector)2138{2139 struct wx *wx = q_vector->wx;2140 int v_idx = q_vector->v_idx;2141 u32 itr_reg;2142 2143 if (wx->mac.type == wx_mac_sp)2144 itr_reg = q_vector->itr & WX_SP_MAX_EITR;2145 else2146 itr_reg = q_vector->itr & WX_EM_MAX_EITR;2147 2148 itr_reg |= WX_PX_ITR_CNT_WDIS;2149 2150 wr32(wx, WX_PX_ITR(v_idx + 1), itr_reg);2151}2152 2153/**2154 * wx_configure_vectors - Configure vectors for hardware2155 * @wx: board private structure2156 *2157 * wx_configure_vectors sets up the hardware to properly generate MSI-X/MSI/INTx2158 * interrupts.2159 **/2160void wx_configure_vectors(struct wx *wx)2161{2162 struct pci_dev *pdev = wx->pdev;2163 u32 eitrsel = 0;2164 u16 v_idx;2165 2166 if (pdev->msix_enabled) {2167 /* Populate MSIX to EITR Select */2168 wr32(wx, WX_PX_ITRSEL, eitrsel);2169 /* use EIAM to auto-mask when MSI-X interrupt is asserted2170 * this saves a register write for every interrupt2171 */2172 wr32(wx, WX_PX_GPIE, WX_PX_GPIE_MODEL);2173 } else {2174 /* legacy interrupts, use EIAM to auto-mask when reading EICR,2175 * specifically only auto mask tx and rx interrupts.2176 */2177 wr32(wx, WX_PX_GPIE, 0);2178 }2179 2180 /* Populate the IVAR table and set the ITR values to the2181 * corresponding register.2182 */2183 for (v_idx = 0; v_idx < wx->num_q_vectors; v_idx++) {2184 struct wx_q_vector *q_vector = wx->q_vector[v_idx];2185 struct wx_ring *ring;2186 2187 wx_for_each_ring(ring, q_vector->rx)2188 wx_set_ivar(wx, 0, ring->reg_idx, v_idx);2189 2190 wx_for_each_ring(ring, q_vector->tx)2191 wx_set_ivar(wx, 1, ring->reg_idx, v_idx);2192 2193 wx_write_eitr(q_vector);2194 }2195 2196 wx_set_ivar(wx, -1, 0, 0);2197 if (pdev->msix_enabled)2198 wr32(wx, WX_PX_ITR(0), 1950);2199}2200EXPORT_SYMBOL(wx_configure_vectors);2201 2202/**2203 * wx_clean_rx_ring - Free Rx Buffers per Queue2204 * @rx_ring: ring to free buffers from2205 **/2206static void wx_clean_rx_ring(struct wx_ring *rx_ring)2207{2208 struct wx_rx_buffer *rx_buffer;2209 u16 i = rx_ring->next_to_clean;2210 2211 rx_buffer = &rx_ring->rx_buffer_info[i];2212 2213 /* Free all the Rx ring sk_buffs */2214 while (i != rx_ring->next_to_alloc) {2215 if (rx_buffer->skb) {2216 struct sk_buff *skb = rx_buffer->skb;2217 2218 if (WX_CB(skb)->page_released)2219 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false);2220 2221 dev_kfree_skb(skb);2222 }2223 2224 /* Invalidate cache lines that may have been written to by2225 * device so that we avoid corrupting memory.2226 */2227 dma_sync_single_range_for_cpu(rx_ring->dev,2228 rx_buffer->dma,2229 rx_buffer->page_offset,2230 WX_RX_BUFSZ,2231 DMA_FROM_DEVICE);2232 2233 /* free resources associated with mapping */2234 page_pool_put_full_page(rx_ring->page_pool, rx_buffer->page, false);2235 2236 i++;2237 rx_buffer++;2238 if (i == rx_ring->count) {2239 i = 0;2240 rx_buffer = rx_ring->rx_buffer_info;2241 }2242 }2243 2244 rx_ring->next_to_alloc = 0;2245 rx_ring->next_to_clean = 0;2246 rx_ring->next_to_use = 0;2247}2248 2249/**2250 * wx_clean_all_rx_rings - Free Rx Buffers for all queues2251 * @wx: board private structure2252 **/2253void wx_clean_all_rx_rings(struct wx *wx)2254{2255 int i;2256 2257 for (i = 0; i < wx->num_rx_queues; i++)2258 wx_clean_rx_ring(wx->rx_ring[i]);2259}2260EXPORT_SYMBOL(wx_clean_all_rx_rings);2261 2262/**2263 * wx_free_rx_resources - Free Rx Resources2264 * @rx_ring: ring to clean the resources from2265 *2266 * Free all receive software resources2267 **/2268static void wx_free_rx_resources(struct wx_ring *rx_ring)2269{2270 wx_clean_rx_ring(rx_ring);2271 kvfree(rx_ring->rx_buffer_info);2272 rx_ring->rx_buffer_info = NULL;2273 2274 /* if not set, then don't free */2275 if (!rx_ring->desc)2276 return;2277 2278 dma_free_coherent(rx_ring->dev, rx_ring->size,2279 rx_ring->desc, rx_ring->dma);2280 2281 rx_ring->desc = NULL;2282 2283 if (rx_ring->page_pool) {2284 page_pool_destroy(rx_ring->page_pool);2285 rx_ring->page_pool = NULL;2286 }2287}2288 2289/**2290 * wx_free_all_rx_resources - Free Rx Resources for All Queues2291 * @wx: pointer to hardware structure2292 *2293 * Free all receive software resources2294 **/2295static void wx_free_all_rx_resources(struct wx *wx)2296{2297 int i;2298 2299 for (i = 0; i < wx->num_rx_queues; i++)2300 wx_free_rx_resources(wx->rx_ring[i]);2301}2302 2303/**2304 * wx_clean_tx_ring - Free Tx Buffers2305 * @tx_ring: ring to be cleaned2306 **/2307static void wx_clean_tx_ring(struct wx_ring *tx_ring)2308{2309 struct wx_tx_buffer *tx_buffer;2310 u16 i = tx_ring->next_to_clean;2311 2312 tx_buffer = &tx_ring->tx_buffer_info[i];2313 2314 while (i != tx_ring->next_to_use) {2315 union wx_tx_desc *eop_desc, *tx_desc;2316 2317 /* Free all the Tx ring sk_buffs */2318 dev_kfree_skb_any(tx_buffer->skb);2319 2320 /* unmap skb header data */2321 dma_unmap_single(tx_ring->dev,2322 dma_unmap_addr(tx_buffer, dma),2323 dma_unmap_len(tx_buffer, len),2324 DMA_TO_DEVICE);2325 2326 /* check for eop_desc to determine the end of the packet */2327 eop_desc = tx_buffer->next_to_watch;2328 tx_desc = WX_TX_DESC(tx_ring, i);2329 2330 /* unmap remaining buffers */2331 while (tx_desc != eop_desc) {2332 tx_buffer++;2333 tx_desc++;2334 i++;2335 if (unlikely(i == tx_ring->count)) {2336 i = 0;2337 tx_buffer = tx_ring->tx_buffer_info;2338 tx_desc = WX_TX_DESC(tx_ring, 0);2339 }2340 2341 /* unmap any remaining paged data */2342 if (dma_unmap_len(tx_buffer, len))2343 dma_unmap_page(tx_ring->dev,2344 dma_unmap_addr(tx_buffer, dma),2345 dma_unmap_len(tx_buffer, len),2346 DMA_TO_DEVICE);2347 }2348 2349 /* move us one more past the eop_desc for start of next pkt */2350 tx_buffer++;2351 i++;2352 if (unlikely(i == tx_ring->count)) {2353 i = 0;2354 tx_buffer = tx_ring->tx_buffer_info;2355 }2356 }2357 2358 netdev_tx_reset_queue(wx_txring_txq(tx_ring));2359 2360 /* reset next_to_use and next_to_clean */2361 tx_ring->next_to_use = 0;2362 tx_ring->next_to_clean = 0;2363}2364 2365/**2366 * wx_clean_all_tx_rings - Free Tx Buffers for all queues2367 * @wx: board private structure2368 **/2369void wx_clean_all_tx_rings(struct wx *wx)2370{2371 int i;2372 2373 for (i = 0; i < wx->num_tx_queues; i++)2374 wx_clean_tx_ring(wx->tx_ring[i]);2375}2376EXPORT_SYMBOL(wx_clean_all_tx_rings);2377 2378/**2379 * wx_free_tx_resources - Free Tx Resources per Queue2380 * @tx_ring: Tx descriptor ring for a specific queue2381 *2382 * Free all transmit software resources2383 **/2384static void wx_free_tx_resources(struct wx_ring *tx_ring)2385{2386 wx_clean_tx_ring(tx_ring);2387 kvfree(tx_ring->tx_buffer_info);2388 tx_ring->tx_buffer_info = NULL;2389 2390 /* if not set, then don't free */2391 if (!tx_ring->desc)2392 return;2393 2394 dma_free_coherent(tx_ring->dev, tx_ring->size,2395 tx_ring->desc, tx_ring->dma);2396 tx_ring->desc = NULL;2397}2398 2399/**2400 * wx_free_all_tx_resources - Free Tx Resources for All Queues2401 * @wx: pointer to hardware structure2402 *2403 * Free all transmit software resources2404 **/2405static void wx_free_all_tx_resources(struct wx *wx)2406{2407 int i;2408 2409 for (i = 0; i < wx->num_tx_queues; i++)2410 wx_free_tx_resources(wx->tx_ring[i]);2411}2412 2413void wx_free_resources(struct wx *wx)2414{2415 wx_free_all_rx_resources(wx);2416 wx_free_all_tx_resources(wx);2417}2418EXPORT_SYMBOL(wx_free_resources);2419 2420static int wx_alloc_page_pool(struct wx_ring *rx_ring)2421{2422 int ret = 0;2423 2424 struct page_pool_params pp_params = {2425 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,2426 .order = 0,2427 .pool_size = rx_ring->size,2428 .nid = dev_to_node(rx_ring->dev),2429 .dev = rx_ring->dev,2430 .dma_dir = DMA_FROM_DEVICE,2431 .offset = 0,2432 .max_len = PAGE_SIZE,2433 };2434 2435 rx_ring->page_pool = page_pool_create(&pp_params);2436 if (IS_ERR(rx_ring->page_pool)) {2437 ret = PTR_ERR(rx_ring->page_pool);2438 rx_ring->page_pool = NULL;2439 }2440 2441 return ret;2442}2443 2444/**2445 * wx_setup_rx_resources - allocate Rx resources (Descriptors)2446 * @rx_ring: rx descriptor ring (for a specific queue) to setup2447 *2448 * Returns 0 on success, negative on failure2449 **/2450static int wx_setup_rx_resources(struct wx_ring *rx_ring)2451{2452 struct device *dev = rx_ring->dev;2453 int orig_node = dev_to_node(dev);2454 int numa_node = NUMA_NO_NODE;2455 int size, ret;2456 2457 size = sizeof(struct wx_rx_buffer) * rx_ring->count;2458 2459 if (rx_ring->q_vector)2460 numa_node = rx_ring->q_vector->numa_node;2461 2462 rx_ring->rx_buffer_info = kvmalloc_node(size, GFP_KERNEL, numa_node);2463 if (!rx_ring->rx_buffer_info)2464 rx_ring->rx_buffer_info = kvmalloc(size, GFP_KERNEL);2465 if (!rx_ring->rx_buffer_info)2466 goto err;2467 2468 /* Round up to nearest 4K */2469 rx_ring->size = rx_ring->count * sizeof(union wx_rx_desc);2470 rx_ring->size = ALIGN(rx_ring->size, 4096);2471 2472 set_dev_node(dev, numa_node);2473 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,2474 &rx_ring->dma, GFP_KERNEL);2475 if (!rx_ring->desc) {2476 set_dev_node(dev, orig_node);2477 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,2478 &rx_ring->dma, GFP_KERNEL);2479 }2480 2481 if (!rx_ring->desc)2482 goto err;2483 2484 rx_ring->next_to_clean = 0;2485 rx_ring->next_to_use = 0;2486 2487 ret = wx_alloc_page_pool(rx_ring);2488 if (ret < 0) {2489 dev_err(rx_ring->dev, "Page pool creation failed: %d\n", ret);2490 goto err_desc;2491 }2492 2493 return 0;2494 2495err_desc:2496 dma_free_coherent(dev, rx_ring->size, rx_ring->desc, rx_ring->dma);2497err:2498 kvfree(rx_ring->rx_buffer_info);2499 rx_ring->rx_buffer_info = NULL;2500 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");2501 return -ENOMEM;2502}2503 2504/**2505 * wx_setup_all_rx_resources - allocate all queues Rx resources2506 * @wx: pointer to hardware structure2507 *2508 * If this function returns with an error, then it's possible one or2509 * more of the rings is populated (while the rest are not). It is the2510 * callers duty to clean those orphaned rings.2511 *2512 * Return 0 on success, negative on failure2513 **/2514static int wx_setup_all_rx_resources(struct wx *wx)2515{2516 int i, err = 0;2517 2518 for (i = 0; i < wx->num_rx_queues; i++) {2519 err = wx_setup_rx_resources(wx->rx_ring[i]);2520 if (!err)2521 continue;2522 2523 wx_err(wx, "Allocation for Rx Queue %u failed\n", i);2524 goto err_setup_rx;2525 }2526 2527 return 0;2528err_setup_rx:2529 /* rewind the index freeing the rings as we go */2530 while (i--)2531 wx_free_rx_resources(wx->rx_ring[i]);2532 return err;2533}2534 2535/**2536 * wx_setup_tx_resources - allocate Tx resources (Descriptors)2537 * @tx_ring: tx descriptor ring (for a specific queue) to setup2538 *2539 * Return 0 on success, negative on failure2540 **/2541static int wx_setup_tx_resources(struct wx_ring *tx_ring)2542{2543 struct device *dev = tx_ring->dev;2544 int orig_node = dev_to_node(dev);2545 int numa_node = NUMA_NO_NODE;2546 int size;2547 2548 size = sizeof(struct wx_tx_buffer) * tx_ring->count;2549 2550 if (tx_ring->q_vector)2551 numa_node = tx_ring->q_vector->numa_node;2552 2553 tx_ring->tx_buffer_info = kvmalloc_node(size, GFP_KERNEL, numa_node);2554 if (!tx_ring->tx_buffer_info)2555 tx_ring->tx_buffer_info = kvmalloc(size, GFP_KERNEL);2556 if (!tx_ring->tx_buffer_info)2557 goto err;2558 2559 /* round up to nearest 4K */2560 tx_ring->size = tx_ring->count * sizeof(union wx_tx_desc);2561 tx_ring->size = ALIGN(tx_ring->size, 4096);2562 2563 set_dev_node(dev, numa_node);2564 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,2565 &tx_ring->dma, GFP_KERNEL);2566 if (!tx_ring->desc) {2567 set_dev_node(dev, orig_node);2568 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,2569 &tx_ring->dma, GFP_KERNEL);2570 }2571 2572 if (!tx_ring->desc)2573 goto err;2574 2575 tx_ring->next_to_use = 0;2576 tx_ring->next_to_clean = 0;2577 2578 return 0;2579 2580err:2581 kvfree(tx_ring->tx_buffer_info);2582 tx_ring->tx_buffer_info = NULL;2583 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");2584 return -ENOMEM;2585}2586 2587/**2588 * wx_setup_all_tx_resources - allocate all queues Tx resources2589 * @wx: pointer to private structure2590 *2591 * If this function returns with an error, then it's possible one or2592 * more of the rings is populated (while the rest are not). It is the2593 * callers duty to clean those orphaned rings.2594 *2595 * Return 0 on success, negative on failure2596 **/2597static int wx_setup_all_tx_resources(struct wx *wx)2598{2599 int i, err = 0;2600 2601 for (i = 0; i < wx->num_tx_queues; i++) {2602 err = wx_setup_tx_resources(wx->tx_ring[i]);2603 if (!err)2604 continue;2605 2606 wx_err(wx, "Allocation for Tx Queue %u failed\n", i);2607 goto err_setup_tx;2608 }2609 2610 return 0;2611err_setup_tx:2612 /* rewind the index freeing the rings as we go */2613 while (i--)2614 wx_free_tx_resources(wx->tx_ring[i]);2615 return err;2616}2617 2618int wx_setup_resources(struct wx *wx)2619{2620 int err;2621 2622 /* allocate transmit descriptors */2623 err = wx_setup_all_tx_resources(wx);2624 if (err)2625 return err;2626 2627 /* allocate receive descriptors */2628 err = wx_setup_all_rx_resources(wx);2629 if (err)2630 goto err_free_tx;2631 2632 err = wx_setup_isb_resources(wx);2633 if (err)2634 goto err_free_rx;2635 2636 return 0;2637 2638err_free_rx:2639 wx_free_all_rx_resources(wx);2640err_free_tx:2641 wx_free_all_tx_resources(wx);2642 2643 return err;2644}2645EXPORT_SYMBOL(wx_setup_resources);2646 2647/**2648 * wx_get_stats64 - Get System Network Statistics2649 * @netdev: network interface device structure2650 * @stats: storage space for 64bit statistics2651 */2652void wx_get_stats64(struct net_device *netdev,2653 struct rtnl_link_stats64 *stats)2654{2655 struct wx *wx = netdev_priv(netdev);2656 struct wx_hw_stats *hwstats;2657 int i;2658 2659 wx_update_stats(wx);2660 2661 rcu_read_lock();2662 for (i = 0; i < wx->num_rx_queues; i++) {2663 struct wx_ring *ring = READ_ONCE(wx->rx_ring[i]);2664 u64 bytes, packets;2665 unsigned int start;2666 2667 if (ring) {2668 do {2669 start = u64_stats_fetch_begin(&ring->syncp);2670 packets = ring->stats.packets;2671 bytes = ring->stats.bytes;2672 } while (u64_stats_fetch_retry(&ring->syncp, start));2673 stats->rx_packets += packets;2674 stats->rx_bytes += bytes;2675 }2676 }2677 2678 for (i = 0; i < wx->num_tx_queues; i++) {2679 struct wx_ring *ring = READ_ONCE(wx->tx_ring[i]);2680 u64 bytes, packets;2681 unsigned int start;2682 2683 if (ring) {2684 do {2685 start = u64_stats_fetch_begin(&ring->syncp);2686 packets = ring->stats.packets;2687 bytes = ring->stats.bytes;2688 } while (u64_stats_fetch_retry(&ring->syncp,2689 start));2690 stats->tx_packets += packets;2691 stats->tx_bytes += bytes;2692 }2693 }2694 2695 rcu_read_unlock();2696 2697 hwstats = &wx->stats;2698 stats->rx_errors = hwstats->crcerrs + hwstats->rlec;2699 stats->multicast = hwstats->qmprc;2700 stats->rx_length_errors = hwstats->rlec;2701 stats->rx_crc_errors = hwstats->crcerrs;2702}2703EXPORT_SYMBOL(wx_get_stats64);2704 2705int wx_set_features(struct net_device *netdev, netdev_features_t features)2706{2707 netdev_features_t changed = netdev->features ^ features;2708 struct wx *wx = netdev_priv(netdev);2709 bool need_reset = false;2710 2711 if (features & NETIF_F_RXHASH) {2712 wr32m(wx, WX_RDB_RA_CTL, WX_RDB_RA_CTL_RSS_EN,2713 WX_RDB_RA_CTL_RSS_EN);2714 wx->rss_enabled = true;2715 } else {2716 wr32m(wx, WX_RDB_RA_CTL, WX_RDB_RA_CTL_RSS_EN, 0);2717 wx->rss_enabled = false;2718 }2719 2720 netdev->features = features;2721 2722 if (wx->mac.type == wx_mac_sp && changed & NETIF_F_HW_VLAN_CTAG_RX)2723 wx->do_reset(netdev);2724 else if (changed & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER))2725 wx_set_rx_mode(netdev);2726 2727 if (!(test_bit(WX_FLAG_FDIR_CAPABLE, wx->flags)))2728 return 0;2729 2730 /* Check if Flow Director n-tuple support was enabled or disabled. If2731 * the state changed, we need to reset.2732 */2733 switch (features & NETIF_F_NTUPLE) {2734 case NETIF_F_NTUPLE:2735 /* turn off ATR, enable perfect filters and reset */2736 if (!(test_and_set_bit(WX_FLAG_FDIR_PERFECT, wx->flags)))2737 need_reset = true;2738 2739 clear_bit(WX_FLAG_FDIR_HASH, wx->flags);2740 break;2741 default:2742 /* turn off perfect filters, enable ATR and reset */2743 if (test_and_clear_bit(WX_FLAG_FDIR_PERFECT, wx->flags))2744 need_reset = true;2745 2746 /* We cannot enable ATR if RSS is disabled */2747 if (wx->ring_feature[RING_F_RSS].limit <= 1)2748 break;2749 2750 set_bit(WX_FLAG_FDIR_HASH, wx->flags);2751 break;2752 }2753 2754 if (need_reset)2755 wx->do_reset(netdev);2756 2757 return 0;2758}2759EXPORT_SYMBOL(wx_set_features);2760 2761#define NETIF_VLAN_STRIPPING_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \2762 NETIF_F_HW_VLAN_STAG_RX)2763 2764#define NETIF_VLAN_INSERTION_FEATURES (NETIF_F_HW_VLAN_CTAG_TX | \2765 NETIF_F_HW_VLAN_STAG_TX)2766 2767#define NETIF_VLAN_FILTERING_FEATURES (NETIF_F_HW_VLAN_CTAG_FILTER | \2768 NETIF_F_HW_VLAN_STAG_FILTER)2769 2770netdev_features_t wx_fix_features(struct net_device *netdev,2771 netdev_features_t features)2772{2773 netdev_features_t changed = netdev->features ^ features;2774 struct wx *wx = netdev_priv(netdev);2775 2776 if (changed & NETIF_VLAN_STRIPPING_FEATURES) {2777 if ((features & NETIF_VLAN_STRIPPING_FEATURES) != NETIF_VLAN_STRIPPING_FEATURES &&2778 (features & NETIF_VLAN_STRIPPING_FEATURES) != 0) {2779 features &= ~NETIF_VLAN_STRIPPING_FEATURES;2780 features |= netdev->features & NETIF_VLAN_STRIPPING_FEATURES;2781 wx_err(wx, "802.1Q and 802.1ad VLAN stripping must be either both on or both off.");2782 }2783 }2784 2785 if (changed & NETIF_VLAN_INSERTION_FEATURES) {2786 if ((features & NETIF_VLAN_INSERTION_FEATURES) != NETIF_VLAN_INSERTION_FEATURES &&2787 (features & NETIF_VLAN_INSERTION_FEATURES) != 0) {2788 features &= ~NETIF_VLAN_INSERTION_FEATURES;2789 features |= netdev->features & NETIF_VLAN_INSERTION_FEATURES;2790 wx_err(wx, "802.1Q and 802.1ad VLAN insertion must be either both on or both off.");2791 }2792 }2793 2794 if (changed & NETIF_VLAN_FILTERING_FEATURES) {2795 if ((features & NETIF_VLAN_FILTERING_FEATURES) != NETIF_VLAN_FILTERING_FEATURES &&2796 (features & NETIF_VLAN_FILTERING_FEATURES) != 0) {2797 features &= ~NETIF_VLAN_FILTERING_FEATURES;2798 features |= netdev->features & NETIF_VLAN_FILTERING_FEATURES;2799 wx_err(wx, "802.1Q and 802.1ad VLAN filtering must be either both on or both off.");2800 }2801 }2802 2803 return features;2804}2805EXPORT_SYMBOL(wx_fix_features);2806 2807void wx_set_ring(struct wx *wx, u32 new_tx_count,2808 u32 new_rx_count, struct wx_ring *temp_ring)2809{2810 int i, err = 0;2811 2812 /* Setup new Tx resources and free the old Tx resources in that order.2813 * We can then assign the new resources to the rings via a memcpy.2814 * The advantage to this approach is that we are guaranteed to still2815 * have resources even in the case of an allocation failure.2816 */2817 if (new_tx_count != wx->tx_ring_count) {2818 for (i = 0; i < wx->num_tx_queues; i++) {2819 memcpy(&temp_ring[i], wx->tx_ring[i],2820 sizeof(struct wx_ring));2821 2822 temp_ring[i].count = new_tx_count;2823 err = wx_setup_tx_resources(&temp_ring[i]);2824 if (err) {2825 wx_err(wx, "setup new tx resources failed, keep using the old config\n");2826 while (i) {2827 i--;2828 wx_free_tx_resources(&temp_ring[i]);2829 }2830 return;2831 }2832 }2833 2834 for (i = 0; i < wx->num_tx_queues; i++) {2835 wx_free_tx_resources(wx->tx_ring[i]);2836 2837 memcpy(wx->tx_ring[i], &temp_ring[i],2838 sizeof(struct wx_ring));2839 }2840 2841 wx->tx_ring_count = new_tx_count;2842 }2843 2844 /* Repeat the process for the Rx rings if needed */2845 if (new_rx_count != wx->rx_ring_count) {2846 for (i = 0; i < wx->num_rx_queues; i++) {2847 memcpy(&temp_ring[i], wx->rx_ring[i],2848 sizeof(struct wx_ring));2849 2850 temp_ring[i].count = new_rx_count;2851 err = wx_setup_rx_resources(&temp_ring[i]);2852 if (err) {2853 wx_err(wx, "setup new rx resources failed, keep using the old config\n");2854 while (i) {2855 i--;2856 wx_free_rx_resources(&temp_ring[i]);2857 }2858 return;2859 }2860 }2861 2862 for (i = 0; i < wx->num_rx_queues; i++) {2863 wx_free_rx_resources(wx->rx_ring[i]);2864 memcpy(wx->rx_ring[i], &temp_ring[i],2865 sizeof(struct wx_ring));2866 }2867 2868 wx->rx_ring_count = new_rx_count;2869 }2870}2871EXPORT_SYMBOL(wx_set_ring);2872 2873MODULE_DESCRIPTION("Common library for Wangxun(R) Ethernet drivers.");2874MODULE_LICENSE("GPL");2875