2009 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright(c) 2013 - 2019 Intel Corporation. */3 4#include <linux/types.h>5#include <linux/module.h>6#include <net/ipv6.h>7#include <net/ip.h>8#include <net/tcp.h>9#include <linux/if_macvlan.h>10#include <linux/prefetch.h>11 12#include "fm10k.h"13 14#define DRV_SUMMARY "Intel(R) Ethernet Switch Host Interface Driver"15char fm10k_driver_name[] = "fm10k";16static const char fm10k_driver_string[] = DRV_SUMMARY;17static const char fm10k_copyright[] =18 "Copyright(c) 2013 - 2019 Intel Corporation.";19 20MODULE_DESCRIPTION(DRV_SUMMARY);21MODULE_LICENSE("GPL v2");22 23/* single workqueue for entire fm10k driver */24struct workqueue_struct *fm10k_workqueue;25 26/**27 * fm10k_init_module - Driver Registration Routine28 *29 * fm10k_init_module is the first routine called when the driver is30 * loaded. All it does is register with the PCI subsystem.31 **/32static int __init fm10k_init_module(void)33{34 int ret;35 36 pr_info("%s\n", fm10k_driver_string);37 pr_info("%s\n", fm10k_copyright);38 39 /* create driver workqueue */40 fm10k_workqueue = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0,41 fm10k_driver_name);42 if (!fm10k_workqueue)43 return -ENOMEM;44 45 fm10k_dbg_init();46 47 ret = fm10k_register_pci_driver();48 if (ret) {49 fm10k_dbg_exit();50 destroy_workqueue(fm10k_workqueue);51 }52 53 return ret;54}55module_init(fm10k_init_module);56 57/**58 * fm10k_exit_module - Driver Exit Cleanup Routine59 *60 * fm10k_exit_module is called just before the driver is removed61 * from memory.62 **/63static void __exit fm10k_exit_module(void)64{65 fm10k_unregister_pci_driver();66 67 fm10k_dbg_exit();68 69 /* destroy driver workqueue */70 destroy_workqueue(fm10k_workqueue);71}72module_exit(fm10k_exit_module);73 74static bool fm10k_alloc_mapped_page(struct fm10k_ring *rx_ring,75 struct fm10k_rx_buffer *bi)76{77 struct page *page = bi->page;78 dma_addr_t dma;79 80 /* Only page will be NULL if buffer was consumed */81 if (likely(page))82 return true;83 84 /* alloc new page for storage */85 page = dev_alloc_page();86 if (unlikely(!page)) {87 rx_ring->rx_stats.alloc_failed++;88 return false;89 }90 91 /* map page for use */92 dma = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);93 94 /* if mapping failed free memory back to system since95 * there isn't much point in holding memory we can't use96 */97 if (dma_mapping_error(rx_ring->dev, dma)) {98 __free_page(page);99 100 rx_ring->rx_stats.alloc_failed++;101 return false;102 }103 104 bi->dma = dma;105 bi->page = page;106 bi->page_offset = 0;107 108 return true;109}110 111/**112 * fm10k_alloc_rx_buffers - Replace used receive buffers113 * @rx_ring: ring to place buffers on114 * @cleaned_count: number of buffers to replace115 **/116void fm10k_alloc_rx_buffers(struct fm10k_ring *rx_ring, u16 cleaned_count)117{118 union fm10k_rx_desc *rx_desc;119 struct fm10k_rx_buffer *bi;120 u16 i = rx_ring->next_to_use;121 122 /* nothing to do */123 if (!cleaned_count)124 return;125 126 rx_desc = FM10K_RX_DESC(rx_ring, i);127 bi = &rx_ring->rx_buffer[i];128 i -= rx_ring->count;129 130 do {131 if (!fm10k_alloc_mapped_page(rx_ring, bi))132 break;133 134 /* Refresh the desc even if buffer_addrs didn't change135 * because each write-back erases this info.136 */137 rx_desc->q.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);138 139 rx_desc++;140 bi++;141 i++;142 if (unlikely(!i)) {143 rx_desc = FM10K_RX_DESC(rx_ring, 0);144 bi = rx_ring->rx_buffer;145 i -= rx_ring->count;146 }147 148 /* clear the status bits for the next_to_use descriptor */149 rx_desc->d.staterr = 0;150 151 cleaned_count--;152 } while (cleaned_count);153 154 i += rx_ring->count;155 156 if (rx_ring->next_to_use != i) {157 /* record the next descriptor to use */158 rx_ring->next_to_use = i;159 160 /* update next to alloc since we have filled the ring */161 rx_ring->next_to_alloc = i;162 163 /* Force memory writes to complete before letting h/w164 * know there are new descriptors to fetch. (Only165 * applicable for weak-ordered memory model archs,166 * such as IA-64).167 */168 wmb();169 170 /* notify hardware of new descriptors */171 writel(i, rx_ring->tail);172 }173}174 175/**176 * fm10k_reuse_rx_page - page flip buffer and store it back on the ring177 * @rx_ring: rx descriptor ring to store buffers on178 * @old_buff: donor buffer to have page reused179 *180 * Synchronizes page for reuse by the interface181 **/182static void fm10k_reuse_rx_page(struct fm10k_ring *rx_ring,183 struct fm10k_rx_buffer *old_buff)184{185 struct fm10k_rx_buffer *new_buff;186 u16 nta = rx_ring->next_to_alloc;187 188 new_buff = &rx_ring->rx_buffer[nta];189 190 /* update, and store next to alloc */191 nta++;192 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;193 194 /* transfer page from old buffer to new buffer */195 *new_buff = *old_buff;196 197 /* sync the buffer for use by the device */198 dma_sync_single_range_for_device(rx_ring->dev, old_buff->dma,199 old_buff->page_offset,200 FM10K_RX_BUFSZ,201 DMA_FROM_DEVICE);202}203 204static bool fm10k_can_reuse_rx_page(struct fm10k_rx_buffer *rx_buffer,205 struct page *page,206 unsigned int __maybe_unused truesize)207{208 /* avoid re-using remote and pfmemalloc pages */209 if (!dev_page_is_reusable(page))210 return false;211 212#if (PAGE_SIZE < 8192)213 /* if we are only owner of page we can reuse it */214 if (unlikely(page_count(page) != 1))215 return false;216 217 /* flip page offset to other buffer */218 rx_buffer->page_offset ^= FM10K_RX_BUFSZ;219#else220 /* move offset up to the next cache line */221 rx_buffer->page_offset += truesize;222 223 if (rx_buffer->page_offset > (PAGE_SIZE - FM10K_RX_BUFSZ))224 return false;225#endif226 227 /* Even if we own the page, we are not allowed to use atomic_set()228 * This would break get_page_unless_zero() users.229 */230 page_ref_inc(page);231 232 return true;233}234 235/**236 * fm10k_add_rx_frag - Add contents of Rx buffer to sk_buff237 * @rx_buffer: buffer containing page to add238 * @size: packet size from rx_desc239 * @rx_desc: descriptor containing length of buffer written by hardware240 * @skb: sk_buff to place the data into241 *242 * This function will add the data contained in rx_buffer->page to the skb.243 * This is done either through a direct copy if the data in the buffer is244 * less than the skb header size, otherwise it will just attach the page as245 * a frag to the skb.246 *247 * The function will then update the page offset if necessary and return248 * true if the buffer can be reused by the interface.249 **/250static bool fm10k_add_rx_frag(struct fm10k_rx_buffer *rx_buffer,251 unsigned int size,252 union fm10k_rx_desc *rx_desc,253 struct sk_buff *skb)254{255 struct page *page = rx_buffer->page;256 unsigned char *va = page_address(page) + rx_buffer->page_offset;257#if (PAGE_SIZE < 8192)258 unsigned int truesize = FM10K_RX_BUFSZ;259#else260 unsigned int truesize = ALIGN(size, 512);261#endif262 unsigned int pull_len;263 264 if (unlikely(skb_is_nonlinear(skb)))265 goto add_tail_frag;266 267 if (likely(size <= FM10K_RX_HDR_LEN)) {268 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));269 270 /* page is reusable, we can reuse buffer as-is */271 if (dev_page_is_reusable(page))272 return true;273 274 /* this page cannot be reused so discard it */275 __free_page(page);276 return false;277 }278 279 /* we need the header to contain the greater of either ETH_HLEN or280 * 60 bytes if the skb->len is less than 60 for skb_pad.281 */282 pull_len = eth_get_headlen(skb->dev, va, FM10K_RX_HDR_LEN);283 284 /* align pull length to size of long to optimize memcpy performance */285 memcpy(__skb_put(skb, pull_len), va, ALIGN(pull_len, sizeof(long)));286 287 /* update all of the pointers */288 va += pull_len;289 size -= pull_len;290 291add_tail_frag:292 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,293 (unsigned long)va & ~PAGE_MASK, size, truesize);294 295 return fm10k_can_reuse_rx_page(rx_buffer, page, truesize);296}297 298static struct sk_buff *fm10k_fetch_rx_buffer(struct fm10k_ring *rx_ring,299 union fm10k_rx_desc *rx_desc,300 struct sk_buff *skb)301{302 unsigned int size = le16_to_cpu(rx_desc->w.length);303 struct fm10k_rx_buffer *rx_buffer;304 struct page *page;305 306 rx_buffer = &rx_ring->rx_buffer[rx_ring->next_to_clean];307 page = rx_buffer->page;308 prefetchw(page);309 310 if (likely(!skb)) {311 void *page_addr = page_address(page) +312 rx_buffer->page_offset;313 314 /* prefetch first cache line of first page */315 net_prefetch(page_addr);316 317 /* allocate a skb to store the frags */318 skb = napi_alloc_skb(&rx_ring->q_vector->napi,319 FM10K_RX_HDR_LEN);320 if (unlikely(!skb)) {321 rx_ring->rx_stats.alloc_failed++;322 return NULL;323 }324 325 /* we will be copying header into skb->data in326 * pskb_may_pull so it is in our interest to prefetch327 * it now to avoid a possible cache miss328 */329 prefetchw(skb->data);330 }331 332 /* we are reusing so sync this buffer for CPU use */333 dma_sync_single_range_for_cpu(rx_ring->dev,334 rx_buffer->dma,335 rx_buffer->page_offset,336 size,337 DMA_FROM_DEVICE);338 339 /* pull page into skb */340 if (fm10k_add_rx_frag(rx_buffer, size, rx_desc, skb)) {341 /* hand second half of page back to the ring */342 fm10k_reuse_rx_page(rx_ring, rx_buffer);343 } else {344 /* we are not reusing the buffer so unmap it */345 dma_unmap_page(rx_ring->dev, rx_buffer->dma,346 PAGE_SIZE, DMA_FROM_DEVICE);347 }348 349 /* clear contents of rx_buffer */350 rx_buffer->page = NULL;351 352 return skb;353}354 355static inline void fm10k_rx_checksum(struct fm10k_ring *ring,356 union fm10k_rx_desc *rx_desc,357 struct sk_buff *skb)358{359 skb_checksum_none_assert(skb);360 361 /* Rx checksum disabled via ethtool */362 if (!(ring->netdev->features & NETIF_F_RXCSUM))363 return;364 365 /* TCP/UDP checksum error bit is set */366 if (fm10k_test_staterr(rx_desc,367 FM10K_RXD_STATUS_L4E |368 FM10K_RXD_STATUS_L4E2 |369 FM10K_RXD_STATUS_IPE |370 FM10K_RXD_STATUS_IPE2)) {371 ring->rx_stats.csum_err++;372 return;373 }374 375 /* It must be a TCP or UDP packet with a valid checksum */376 if (fm10k_test_staterr(rx_desc, FM10K_RXD_STATUS_L4CS2))377 skb->encapsulation = true;378 else if (!fm10k_test_staterr(rx_desc, FM10K_RXD_STATUS_L4CS))379 return;380 381 skb->ip_summed = CHECKSUM_UNNECESSARY;382 383 ring->rx_stats.csum_good++;384}385 386#define FM10K_RSS_L4_TYPES_MASK \387 (BIT(FM10K_RSSTYPE_IPV4_TCP) | \388 BIT(FM10K_RSSTYPE_IPV4_UDP) | \389 BIT(FM10K_RSSTYPE_IPV6_TCP) | \390 BIT(FM10K_RSSTYPE_IPV6_UDP))391 392static inline void fm10k_rx_hash(struct fm10k_ring *ring,393 union fm10k_rx_desc *rx_desc,394 struct sk_buff *skb)395{396 u16 rss_type;397 398 if (!(ring->netdev->features & NETIF_F_RXHASH))399 return;400 401 rss_type = le16_to_cpu(rx_desc->w.pkt_info) & FM10K_RXD_RSSTYPE_MASK;402 if (!rss_type)403 return;404 405 skb_set_hash(skb, le32_to_cpu(rx_desc->d.rss),406 (BIT(rss_type) & FM10K_RSS_L4_TYPES_MASK) ?407 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);408}409 410static void fm10k_type_trans(struct fm10k_ring *rx_ring,411 union fm10k_rx_desc __maybe_unused *rx_desc,412 struct sk_buff *skb)413{414 struct net_device *dev = rx_ring->netdev;415 struct fm10k_l2_accel *l2_accel = rcu_dereference_bh(rx_ring->l2_accel);416 417 /* check to see if DGLORT belongs to a MACVLAN */418 if (l2_accel) {419 u16 idx = le16_to_cpu(FM10K_CB(skb)->fi.w.dglort) - 1;420 421 idx -= l2_accel->dglort;422 if (idx < l2_accel->size && l2_accel->macvlan[idx])423 dev = l2_accel->macvlan[idx];424 else425 l2_accel = NULL;426 }427 428 /* Record Rx queue, or update macvlan statistics */429 if (!l2_accel)430 skb_record_rx_queue(skb, rx_ring->queue_index);431 else432 macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, true,433 false);434 435 skb->protocol = eth_type_trans(skb, dev);436}437 438/**439 * fm10k_process_skb_fields - Populate skb header fields from Rx descriptor440 * @rx_ring: rx descriptor ring packet is being transacted on441 * @rx_desc: pointer to the EOP Rx descriptor442 * @skb: pointer to current skb being populated443 *444 * This function checks the ring, descriptor, and packet information in445 * order to populate the hash, checksum, VLAN, timestamp, protocol, and446 * other fields within the skb.447 **/448static unsigned int fm10k_process_skb_fields(struct fm10k_ring *rx_ring,449 union fm10k_rx_desc *rx_desc,450 struct sk_buff *skb)451{452 unsigned int len = skb->len;453 454 fm10k_rx_hash(rx_ring, rx_desc, skb);455 456 fm10k_rx_checksum(rx_ring, rx_desc, skb);457 458 FM10K_CB(skb)->tstamp = rx_desc->q.timestamp;459 460 FM10K_CB(skb)->fi.w.vlan = rx_desc->w.vlan;461 462 FM10K_CB(skb)->fi.d.glort = rx_desc->d.glort;463 464 if (rx_desc->w.vlan) {465 u16 vid = le16_to_cpu(rx_desc->w.vlan);466 467 if ((vid & VLAN_VID_MASK) != rx_ring->vid)468 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);469 else if (vid & VLAN_PRIO_MASK)470 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),471 vid & VLAN_PRIO_MASK);472 }473 474 fm10k_type_trans(rx_ring, rx_desc, skb);475 476 return len;477}478 479/**480 * fm10k_is_non_eop - process handling of non-EOP buffers481 * @rx_ring: Rx ring being processed482 * @rx_desc: Rx descriptor for current buffer483 *484 * This function updates next to clean. If the buffer is an EOP buffer485 * this function exits returning false, otherwise it will place the486 * sk_buff in the next buffer to be chained and return true indicating487 * that this is in fact a non-EOP buffer.488 **/489static bool fm10k_is_non_eop(struct fm10k_ring *rx_ring,490 union fm10k_rx_desc *rx_desc)491{492 u32 ntc = rx_ring->next_to_clean + 1;493 494 /* fetch, update, and store next to clean */495 ntc = (ntc < rx_ring->count) ? ntc : 0;496 rx_ring->next_to_clean = ntc;497 498 prefetch(FM10K_RX_DESC(rx_ring, ntc));499 500 if (likely(fm10k_test_staterr(rx_desc, FM10K_RXD_STATUS_EOP)))501 return false;502 503 return true;504}505 506/**507 * fm10k_cleanup_headers - Correct corrupted or empty headers508 * @rx_ring: rx descriptor ring packet is being transacted on509 * @rx_desc: pointer to the EOP Rx descriptor510 * @skb: pointer to current skb being fixed511 *512 * Address the case where we are pulling data in on pages only513 * and as such no data is present in the skb header.514 *515 * In addition if skb is not at least 60 bytes we need to pad it so that516 * it is large enough to qualify as a valid Ethernet frame.517 *518 * Returns true if an error was encountered and skb was freed.519 **/520static bool fm10k_cleanup_headers(struct fm10k_ring *rx_ring,521 union fm10k_rx_desc *rx_desc,522 struct sk_buff *skb)523{524 if (unlikely((fm10k_test_staterr(rx_desc,525 FM10K_RXD_STATUS_RXE)))) {526#define FM10K_TEST_RXD_BIT(rxd, bit) \527 ((rxd)->w.csum_err & cpu_to_le16(bit))528 if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_SWITCH_ERROR))529 rx_ring->rx_stats.switch_errors++;530 if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_NO_DESCRIPTOR))531 rx_ring->rx_stats.drops++;532 if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_PP_ERROR))533 rx_ring->rx_stats.pp_errors++;534 if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_SWITCH_READY))535 rx_ring->rx_stats.link_errors++;536 if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_TOO_BIG))537 rx_ring->rx_stats.length_errors++;538 dev_kfree_skb_any(skb);539 rx_ring->rx_stats.errors++;540 return true;541 }542 543 /* if eth_skb_pad returns an error the skb was freed */544 if (eth_skb_pad(skb))545 return true;546 547 return false;548}549 550/**551 * fm10k_receive_skb - helper function to handle rx indications552 * @q_vector: structure containing interrupt and ring information553 * @skb: packet to send up554 **/555static void fm10k_receive_skb(struct fm10k_q_vector *q_vector,556 struct sk_buff *skb)557{558 napi_gro_receive(&q_vector->napi, skb);559}560 561static int fm10k_clean_rx_irq(struct fm10k_q_vector *q_vector,562 struct fm10k_ring *rx_ring,563 int budget)564{565 struct sk_buff *skb = rx_ring->skb;566 unsigned int total_bytes = 0, total_packets = 0;567 u16 cleaned_count = fm10k_desc_unused(rx_ring);568 569 while (likely(total_packets < budget)) {570 union fm10k_rx_desc *rx_desc;571 572 /* return some buffers to hardware, one at a time is too slow */573 if (cleaned_count >= FM10K_RX_BUFFER_WRITE) {574 fm10k_alloc_rx_buffers(rx_ring, cleaned_count);575 cleaned_count = 0;576 }577 578 rx_desc = FM10K_RX_DESC(rx_ring, rx_ring->next_to_clean);579 580 if (!rx_desc->d.staterr)581 break;582 583 /* This memory barrier is needed to keep us from reading584 * any other fields out of the rx_desc until we know the585 * descriptor has been written back586 */587 dma_rmb();588 589 /* retrieve a buffer from the ring */590 skb = fm10k_fetch_rx_buffer(rx_ring, rx_desc, skb);591 592 /* exit if we failed to retrieve a buffer */593 if (!skb)594 break;595 596 cleaned_count++;597 598 /* fetch next buffer in frame if non-eop */599 if (fm10k_is_non_eop(rx_ring, rx_desc))600 continue;601 602 /* verify the packet layout is correct */603 if (fm10k_cleanup_headers(rx_ring, rx_desc, skb)) {604 skb = NULL;605 continue;606 }607 608 /* populate checksum, timestamp, VLAN, and protocol */609 total_bytes += fm10k_process_skb_fields(rx_ring, rx_desc, skb);610 611 fm10k_receive_skb(q_vector, skb);612 613 /* reset skb pointer */614 skb = NULL;615 616 /* update budget accounting */617 total_packets++;618 }619 620 /* place incomplete frames back on ring for completion */621 rx_ring->skb = skb;622 623 u64_stats_update_begin(&rx_ring->syncp);624 rx_ring->stats.packets += total_packets;625 rx_ring->stats.bytes += total_bytes;626 u64_stats_update_end(&rx_ring->syncp);627 q_vector->rx.total_packets += total_packets;628 q_vector->rx.total_bytes += total_bytes;629 630 return total_packets;631}632 633#define VXLAN_HLEN (sizeof(struct udphdr) + 8)634static struct ethhdr *fm10k_port_is_vxlan(struct sk_buff *skb)635{636 struct fm10k_intfc *interface = netdev_priv(skb->dev);637 638 if (interface->vxlan_port != udp_hdr(skb)->dest)639 return NULL;640 641 /* return offset of udp_hdr plus 8 bytes for VXLAN header */642 return (struct ethhdr *)(skb_transport_header(skb) + VXLAN_HLEN);643}644 645#define FM10K_NVGRE_RESERVED0_FLAGS htons(0x9FFF)646#define NVGRE_TNI htons(0x2000)647struct fm10k_nvgre_hdr {648 __be16 flags;649 __be16 proto;650 __be32 tni;651};652 653static struct ethhdr *fm10k_gre_is_nvgre(struct sk_buff *skb)654{655 struct fm10k_nvgre_hdr *nvgre_hdr;656 int hlen = ip_hdrlen(skb);657 658 /* currently only IPv4 is supported due to hlen above */659 if (vlan_get_protocol(skb) != htons(ETH_P_IP))660 return NULL;661 662 /* our transport header should be NVGRE */663 nvgre_hdr = (struct fm10k_nvgre_hdr *)(skb_network_header(skb) + hlen);664 665 /* verify all reserved flags are 0 */666 if (nvgre_hdr->flags & FM10K_NVGRE_RESERVED0_FLAGS)667 return NULL;668 669 /* report start of ethernet header */670 if (nvgre_hdr->flags & NVGRE_TNI)671 return (struct ethhdr *)(nvgre_hdr + 1);672 673 return (struct ethhdr *)(&nvgre_hdr->tni);674}675 676__be16 fm10k_tx_encap_offload(struct sk_buff *skb)677{678 u8 l4_hdr = 0, inner_l4_hdr = 0, inner_l4_hlen;679 struct ethhdr *eth_hdr;680 681 if (skb->inner_protocol_type != ENCAP_TYPE_ETHER ||682 skb->inner_protocol != htons(ETH_P_TEB))683 return 0;684 685 switch (vlan_get_protocol(skb)) {686 case htons(ETH_P_IP):687 l4_hdr = ip_hdr(skb)->protocol;688 break;689 case htons(ETH_P_IPV6):690 l4_hdr = ipv6_hdr(skb)->nexthdr;691 break;692 default:693 return 0;694 }695 696 switch (l4_hdr) {697 case IPPROTO_UDP:698 eth_hdr = fm10k_port_is_vxlan(skb);699 break;700 case IPPROTO_GRE:701 eth_hdr = fm10k_gre_is_nvgre(skb);702 break;703 default:704 return 0;705 }706 707 if (!eth_hdr)708 return 0;709 710 switch (eth_hdr->h_proto) {711 case htons(ETH_P_IP):712 inner_l4_hdr = inner_ip_hdr(skb)->protocol;713 break;714 case htons(ETH_P_IPV6):715 inner_l4_hdr = inner_ipv6_hdr(skb)->nexthdr;716 break;717 default:718 return 0;719 }720 721 switch (inner_l4_hdr) {722 case IPPROTO_TCP:723 inner_l4_hlen = inner_tcp_hdrlen(skb);724 break;725 case IPPROTO_UDP:726 inner_l4_hlen = 8;727 break;728 default:729 return 0;730 }731 732 /* The hardware allows tunnel offloads only if the combined inner and733 * outer header is 184 bytes or less734 */735 if (skb_inner_transport_header(skb) + inner_l4_hlen -736 skb_mac_header(skb) > FM10K_TUNNEL_HEADER_LENGTH)737 return 0;738 739 return eth_hdr->h_proto;740}741 742static int fm10k_tso(struct fm10k_ring *tx_ring,743 struct fm10k_tx_buffer *first)744{745 struct sk_buff *skb = first->skb;746 struct fm10k_tx_desc *tx_desc;747 unsigned char *th;748 u8 hdrlen;749 750 if (skb->ip_summed != CHECKSUM_PARTIAL)751 return 0;752 753 if (!skb_is_gso(skb))754 return 0;755 756 /* compute header lengths */757 if (skb->encapsulation) {758 if (!fm10k_tx_encap_offload(skb))759 goto err_vxlan;760 th = skb_inner_transport_header(skb);761 } else {762 th = skb_transport_header(skb);763 }764 765 /* compute offset from SOF to transport header and add header len */766 hdrlen = (th - skb->data) + (((struct tcphdr *)th)->doff << 2);767 768 first->tx_flags |= FM10K_TX_FLAGS_CSUM;769 770 /* update gso size and bytecount with header size */771 first->gso_segs = skb_shinfo(skb)->gso_segs;772 first->bytecount += (first->gso_segs - 1) * hdrlen;773 774 /* populate Tx descriptor header size and mss */775 tx_desc = FM10K_TX_DESC(tx_ring, tx_ring->next_to_use);776 tx_desc->hdrlen = hdrlen;777 tx_desc->mss = cpu_to_le16(skb_shinfo(skb)->gso_size);778 779 return 1;780 781err_vxlan:782 tx_ring->netdev->features &= ~NETIF_F_GSO_UDP_TUNNEL;783 if (net_ratelimit())784 netdev_err(tx_ring->netdev,785 "TSO requested for unsupported tunnel, disabling offload\n");786 return -1;787}788 789static void fm10k_tx_csum(struct fm10k_ring *tx_ring,790 struct fm10k_tx_buffer *first)791{792 struct sk_buff *skb = first->skb;793 struct fm10k_tx_desc *tx_desc;794 union {795 struct iphdr *ipv4;796 struct ipv6hdr *ipv6;797 u8 *raw;798 } network_hdr;799 u8 *transport_hdr;800 __be16 frag_off;801 __be16 protocol;802 u8 l4_hdr = 0;803 804 if (skb->ip_summed != CHECKSUM_PARTIAL)805 goto no_csum;806 807 if (skb->encapsulation) {808 protocol = fm10k_tx_encap_offload(skb);809 if (!protocol) {810 if (skb_checksum_help(skb)) {811 dev_warn(tx_ring->dev,812 "failed to offload encap csum!\n");813 tx_ring->tx_stats.csum_err++;814 }815 goto no_csum;816 }817 network_hdr.raw = skb_inner_network_header(skb);818 transport_hdr = skb_inner_transport_header(skb);819 } else {820 protocol = vlan_get_protocol(skb);821 network_hdr.raw = skb_network_header(skb);822 transport_hdr = skb_transport_header(skb);823 }824 825 switch (protocol) {826 case htons(ETH_P_IP):827 l4_hdr = network_hdr.ipv4->protocol;828 break;829 case htons(ETH_P_IPV6):830 l4_hdr = network_hdr.ipv6->nexthdr;831 if (likely((transport_hdr - network_hdr.raw) ==832 sizeof(struct ipv6hdr)))833 break;834 ipv6_skip_exthdr(skb, network_hdr.raw - skb->data +835 sizeof(struct ipv6hdr),836 &l4_hdr, &frag_off);837 if (unlikely(frag_off))838 l4_hdr = NEXTHDR_FRAGMENT;839 break;840 default:841 break;842 }843 844 switch (l4_hdr) {845 case IPPROTO_TCP:846 case IPPROTO_UDP:847 break;848 case IPPROTO_GRE:849 if (skb->encapsulation)850 break;851 fallthrough;852 default:853 if (unlikely(net_ratelimit())) {854 dev_warn(tx_ring->dev,855 "partial checksum, version=%d l4 proto=%x\n",856 protocol, l4_hdr);857 }858 skb_checksum_help(skb);859 tx_ring->tx_stats.csum_err++;860 goto no_csum;861 }862 863 /* update TX checksum flag */864 first->tx_flags |= FM10K_TX_FLAGS_CSUM;865 tx_ring->tx_stats.csum_good++;866 867no_csum:868 /* populate Tx descriptor header size and mss */869 tx_desc = FM10K_TX_DESC(tx_ring, tx_ring->next_to_use);870 tx_desc->hdrlen = 0;871 tx_desc->mss = 0;872}873 874#define FM10K_SET_FLAG(_input, _flag, _result) \875 ((_flag <= _result) ? \876 ((u32)(_input & _flag) * (_result / _flag)) : \877 ((u32)(_input & _flag) / (_flag / _result)))878 879static u8 fm10k_tx_desc_flags(struct sk_buff *skb, u32 tx_flags)880{881 /* set type for advanced descriptor with frame checksum insertion */882 u32 desc_flags = 0;883 884 /* set checksum offload bits */885 desc_flags |= FM10K_SET_FLAG(tx_flags, FM10K_TX_FLAGS_CSUM,886 FM10K_TXD_FLAG_CSUM);887 888 return desc_flags;889}890 891static bool fm10k_tx_desc_push(struct fm10k_ring *tx_ring,892 struct fm10k_tx_desc *tx_desc, u16 i,893 dma_addr_t dma, unsigned int size, u8 desc_flags)894{895 /* set RS and INT for last frame in a cache line */896 if ((++i & (FM10K_TXD_WB_FIFO_SIZE - 1)) == 0)897 desc_flags |= FM10K_TXD_FLAG_RS | FM10K_TXD_FLAG_INT;898 899 /* record values to descriptor */900 tx_desc->buffer_addr = cpu_to_le64(dma);901 tx_desc->flags = desc_flags;902 tx_desc->buflen = cpu_to_le16(size);903 904 /* return true if we just wrapped the ring */905 return i == tx_ring->count;906}907 908static int __fm10k_maybe_stop_tx(struct fm10k_ring *tx_ring, u16 size)909{910 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);911 912 /* Memory barrier before checking head and tail */913 smp_mb();914 915 /* Check again in a case another CPU has just made room available */916 if (likely(fm10k_desc_unused(tx_ring) < size))917 return -EBUSY;918 919 /* A reprieve! - use start_queue because it doesn't call schedule */920 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);921 ++tx_ring->tx_stats.restart_queue;922 return 0;923}924 925static inline int fm10k_maybe_stop_tx(struct fm10k_ring *tx_ring, u16 size)926{927 if (likely(fm10k_desc_unused(tx_ring) >= size))928 return 0;929 return __fm10k_maybe_stop_tx(tx_ring, size);930}931 932static void fm10k_tx_map(struct fm10k_ring *tx_ring,933 struct fm10k_tx_buffer *first)934{935 struct sk_buff *skb = first->skb;936 struct fm10k_tx_buffer *tx_buffer;937 struct fm10k_tx_desc *tx_desc;938 skb_frag_t *frag;939 unsigned char *data;940 dma_addr_t dma;941 unsigned int data_len, size;942 u32 tx_flags = first->tx_flags;943 u16 i = tx_ring->next_to_use;944 u8 flags = fm10k_tx_desc_flags(skb, tx_flags);945 946 tx_desc = FM10K_TX_DESC(tx_ring, i);947 948 /* add HW VLAN tag */949 if (skb_vlan_tag_present(skb))950 tx_desc->vlan = cpu_to_le16(skb_vlan_tag_get(skb));951 else952 tx_desc->vlan = 0;953 954 size = skb_headlen(skb);955 data = skb->data;956 957 dma = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE);958 959 data_len = skb->data_len;960 tx_buffer = first;961 962 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {963 if (dma_mapping_error(tx_ring->dev, dma))964 goto dma_error;965 966 /* record length, and DMA address */967 dma_unmap_len_set(tx_buffer, len, size);968 dma_unmap_addr_set(tx_buffer, dma, dma);969 970 while (unlikely(size > FM10K_MAX_DATA_PER_TXD)) {971 if (fm10k_tx_desc_push(tx_ring, tx_desc++, i++, dma,972 FM10K_MAX_DATA_PER_TXD, flags)) {973 tx_desc = FM10K_TX_DESC(tx_ring, 0);974 i = 0;975 }976 977 dma += FM10K_MAX_DATA_PER_TXD;978 size -= FM10K_MAX_DATA_PER_TXD;979 }980 981 if (likely(!data_len))982 break;983 984 if (fm10k_tx_desc_push(tx_ring, tx_desc++, i++,985 dma, size, flags)) {986 tx_desc = FM10K_TX_DESC(tx_ring, 0);987 i = 0;988 }989 990 size = skb_frag_size(frag);991 data_len -= size;992 993 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,994 DMA_TO_DEVICE);995 996 tx_buffer = &tx_ring->tx_buffer[i];997 }998 999 /* write last descriptor with LAST bit set */1000 flags |= FM10K_TXD_FLAG_LAST;1001 1002 if (fm10k_tx_desc_push(tx_ring, tx_desc, i++, dma, size, flags))1003 i = 0;1004 1005 /* record bytecount for BQL */1006 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);1007 1008 /* record SW timestamp if HW timestamp is not available */1009 skb_tx_timestamp(first->skb);1010 1011 /* Force memory writes to complete before letting h/w know there1012 * are new descriptors to fetch. (Only applicable for weak-ordered1013 * memory model archs, such as IA-64).1014 *1015 * We also need this memory barrier to make certain all of the1016 * status bits have been updated before next_to_watch is written.1017 */1018 wmb();1019 1020 /* set next_to_watch value indicating a packet is present */1021 first->next_to_watch = tx_desc;1022 1023 tx_ring->next_to_use = i;1024 1025 /* Make sure there is space in the ring for the next send. */1026 fm10k_maybe_stop_tx(tx_ring, DESC_NEEDED);1027 1028 /* notify HW of packet */1029 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {1030 writel(i, tx_ring->tail);1031 }1032 1033 return;1034dma_error:1035 dev_err(tx_ring->dev, "TX DMA map failed\n");1036 1037 /* clear dma mappings for failed tx_buffer map */1038 for (;;) {1039 tx_buffer = &tx_ring->tx_buffer[i];1040 fm10k_unmap_and_free_tx_resource(tx_ring, tx_buffer);1041 if (tx_buffer == first)1042 break;1043 if (i == 0)1044 i = tx_ring->count;1045 i--;1046 }1047 1048 tx_ring->next_to_use = i;1049}1050 1051netdev_tx_t fm10k_xmit_frame_ring(struct sk_buff *skb,1052 struct fm10k_ring *tx_ring)1053{1054 u16 count = TXD_USE_COUNT(skb_headlen(skb));1055 struct fm10k_tx_buffer *first;1056 unsigned short f;1057 u32 tx_flags = 0;1058 int tso;1059 1060 /* need: 1 descriptor per page * PAGE_SIZE/FM10K_MAX_DATA_PER_TXD,1061 * + 1 desc for skb_headlen/FM10K_MAX_DATA_PER_TXD,1062 * + 2 desc gap to keep tail from touching head1063 * otherwise try next time1064 */1065 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {1066 skb_frag_t *frag = &skb_shinfo(skb)->frags[f];1067 1068 count += TXD_USE_COUNT(skb_frag_size(frag));1069 }1070 1071 if (fm10k_maybe_stop_tx(tx_ring, count + 3)) {1072 tx_ring->tx_stats.tx_busy++;1073 return NETDEV_TX_BUSY;1074 }1075 1076 /* record the location of the first descriptor for this packet */1077 first = &tx_ring->tx_buffer[tx_ring->next_to_use];1078 first->skb = skb;1079 first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN);1080 first->gso_segs = 1;1081 1082 /* record initial flags and protocol */1083 first->tx_flags = tx_flags;1084 1085 tso = fm10k_tso(tx_ring, first);1086 if (tso < 0)1087 goto out_drop;1088 else if (!tso)1089 fm10k_tx_csum(tx_ring, first);1090 1091 fm10k_tx_map(tx_ring, first);1092 1093 return NETDEV_TX_OK;1094 1095out_drop:1096 dev_kfree_skb_any(first->skb);1097 first->skb = NULL;1098 1099 return NETDEV_TX_OK;1100}1101 1102static u64 fm10k_get_tx_completed(struct fm10k_ring *ring)1103{1104 return ring->stats.packets;1105}1106 1107/**1108 * fm10k_get_tx_pending - how many Tx descriptors not processed1109 * @ring: the ring structure1110 * @in_sw: is tx_pending being checked in SW or in HW?1111 */1112u64 fm10k_get_tx_pending(struct fm10k_ring *ring, bool in_sw)1113{1114 struct fm10k_intfc *interface = ring->q_vector->interface;1115 struct fm10k_hw *hw = &interface->hw;1116 u32 head, tail;1117 1118 if (likely(in_sw)) {1119 head = ring->next_to_clean;1120 tail = ring->next_to_use;1121 } else {1122 head = fm10k_read_reg(hw, FM10K_TDH(ring->reg_idx));1123 tail = fm10k_read_reg(hw, FM10K_TDT(ring->reg_idx));1124 }1125 1126 return ((head <= tail) ? tail : tail + ring->count) - head;1127}1128 1129bool fm10k_check_tx_hang(struct fm10k_ring *tx_ring)1130{1131 u32 tx_done = fm10k_get_tx_completed(tx_ring);1132 u32 tx_done_old = tx_ring->tx_stats.tx_done_old;1133 u32 tx_pending = fm10k_get_tx_pending(tx_ring, true);1134 1135 clear_check_for_tx_hang(tx_ring);1136 1137 /* Check for a hung queue, but be thorough. This verifies1138 * that a transmit has been completed since the previous1139 * check AND there is at least one packet pending. By1140 * requiring this to fail twice we avoid races with1141 * clearing the ARMED bit and conditions where we1142 * run the check_tx_hang logic with a transmit completion1143 * pending but without time to complete it yet.1144 */1145 if (!tx_pending || (tx_done_old != tx_done)) {1146 /* update completed stats and continue */1147 tx_ring->tx_stats.tx_done_old = tx_done;1148 /* reset the countdown */1149 clear_bit(__FM10K_HANG_CHECK_ARMED, tx_ring->state);1150 1151 return false;1152 }1153 1154 /* make sure it is true for two checks in a row */1155 return test_and_set_bit(__FM10K_HANG_CHECK_ARMED, tx_ring->state);1156}1157 1158/**1159 * fm10k_tx_timeout_reset - initiate reset due to Tx timeout1160 * @interface: driver private struct1161 **/1162void fm10k_tx_timeout_reset(struct fm10k_intfc *interface)1163{1164 /* Do the reset outside of interrupt context */1165 if (!test_bit(__FM10K_DOWN, interface->state)) {1166 interface->tx_timeout_count++;1167 set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags);1168 fm10k_service_event_schedule(interface);1169 }1170}1171 1172/**1173 * fm10k_clean_tx_irq - Reclaim resources after transmit completes1174 * @q_vector: structure containing interrupt and ring information1175 * @tx_ring: tx ring to clean1176 * @napi_budget: Used to determine if we are in netpoll1177 **/1178static bool fm10k_clean_tx_irq(struct fm10k_q_vector *q_vector,1179 struct fm10k_ring *tx_ring, int napi_budget)1180{1181 struct fm10k_intfc *interface = q_vector->interface;1182 struct fm10k_tx_buffer *tx_buffer;1183 struct fm10k_tx_desc *tx_desc;1184 unsigned int total_bytes = 0, total_packets = 0;1185 unsigned int budget = q_vector->tx.work_limit;1186 unsigned int i = tx_ring->next_to_clean;1187 1188 if (test_bit(__FM10K_DOWN, interface->state))1189 return true;1190 1191 tx_buffer = &tx_ring->tx_buffer[i];1192 tx_desc = FM10K_TX_DESC(tx_ring, i);1193 i -= tx_ring->count;1194 1195 do {1196 struct fm10k_tx_desc *eop_desc = tx_buffer->next_to_watch;1197 1198 /* if next_to_watch is not set then there is no work pending */1199 if (!eop_desc)1200 break;1201 1202 /* prevent any other reads prior to eop_desc */1203 smp_rmb();1204 1205 /* if DD is not set pending work has not been completed */1206 if (!(eop_desc->flags & FM10K_TXD_FLAG_DONE))1207 break;1208 1209 /* clear next_to_watch to prevent false hangs */1210 tx_buffer->next_to_watch = NULL;1211 1212 /* update the statistics for this packet */1213 total_bytes += tx_buffer->bytecount;1214 total_packets += tx_buffer->gso_segs;1215 1216 /* free the skb */1217 napi_consume_skb(tx_buffer->skb, napi_budget);1218 1219 /* unmap skb header data */1220 dma_unmap_single(tx_ring->dev,1221 dma_unmap_addr(tx_buffer, dma),1222 dma_unmap_len(tx_buffer, len),1223 DMA_TO_DEVICE);1224 1225 /* clear tx_buffer data */1226 tx_buffer->skb = NULL;1227 dma_unmap_len_set(tx_buffer, len, 0);1228 1229 /* unmap remaining buffers */1230 while (tx_desc != eop_desc) {1231 tx_buffer++;1232 tx_desc++;1233 i++;1234 if (unlikely(!i)) {1235 i -= tx_ring->count;1236 tx_buffer = tx_ring->tx_buffer;1237 tx_desc = FM10K_TX_DESC(tx_ring, 0);1238 }1239 1240 /* unmap any remaining paged data */1241 if (dma_unmap_len(tx_buffer, len)) {1242 dma_unmap_page(tx_ring->dev,1243 dma_unmap_addr(tx_buffer, dma),1244 dma_unmap_len(tx_buffer, len),1245 DMA_TO_DEVICE);1246 dma_unmap_len_set(tx_buffer, len, 0);1247 }1248 }1249 1250 /* move us one more past the eop_desc for start of next pkt */1251 tx_buffer++;1252 tx_desc++;1253 i++;1254 if (unlikely(!i)) {1255 i -= tx_ring->count;1256 tx_buffer = tx_ring->tx_buffer;1257 tx_desc = FM10K_TX_DESC(tx_ring, 0);1258 }1259 1260 /* issue prefetch for next Tx descriptor */1261 prefetch(tx_desc);1262 1263 /* update budget accounting */1264 budget--;1265 } while (likely(budget));1266 1267 i += tx_ring->count;1268 tx_ring->next_to_clean = i;1269 u64_stats_update_begin(&tx_ring->syncp);1270 tx_ring->stats.bytes += total_bytes;1271 tx_ring->stats.packets += total_packets;1272 u64_stats_update_end(&tx_ring->syncp);1273 q_vector->tx.total_bytes += total_bytes;1274 q_vector->tx.total_packets += total_packets;1275 1276 if (check_for_tx_hang(tx_ring) && fm10k_check_tx_hang(tx_ring)) {1277 /* schedule immediate reset if we believe we hung */1278 struct fm10k_hw *hw = &interface->hw;1279 1280 netif_err(interface, drv, tx_ring->netdev,1281 "Detected Tx Unit Hang\n"1282 " Tx Queue <%d>\n"1283 " TDH, TDT <%x>, <%x>\n"1284 " next_to_use <%x>\n"1285 " next_to_clean <%x>\n",1286 tx_ring->queue_index,1287 fm10k_read_reg(hw, FM10K_TDH(tx_ring->reg_idx)),1288 fm10k_read_reg(hw, FM10K_TDT(tx_ring->reg_idx)),1289 tx_ring->next_to_use, i);1290 1291 netif_stop_subqueue(tx_ring->netdev,1292 tx_ring->queue_index);1293 1294 netif_info(interface, probe, tx_ring->netdev,1295 "tx hang %d detected on queue %d, resetting interface\n",1296 interface->tx_timeout_count + 1,1297 tx_ring->queue_index);1298 1299 fm10k_tx_timeout_reset(interface);1300 1301 /* the netdev is about to reset, no point in enabling stuff */1302 return true;1303 }1304 1305 /* notify netdev of completed buffers */1306 netdev_tx_completed_queue(txring_txq(tx_ring),1307 total_packets, total_bytes);1308 1309#define TX_WAKE_THRESHOLD min_t(u16, FM10K_MIN_TXD - 1, DESC_NEEDED * 2)1310 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&1311 (fm10k_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {1312 /* Make sure that anybody stopping the queue after this1313 * sees the new next_to_clean.1314 */1315 smp_mb();1316 if (__netif_subqueue_stopped(tx_ring->netdev,1317 tx_ring->queue_index) &&1318 !test_bit(__FM10K_DOWN, interface->state)) {1319 netif_wake_subqueue(tx_ring->netdev,1320 tx_ring->queue_index);1321 ++tx_ring->tx_stats.restart_queue;1322 }1323 }1324 1325 return !!budget;1326}1327 1328/**1329 * fm10k_update_itr - update the dynamic ITR value based on packet size1330 *1331 * Stores a new ITR value based on strictly on packet size. The1332 * divisors and thresholds used by this function were determined based1333 * on theoretical maximum wire speed and testing data, in order to1334 * minimize response time while increasing bulk throughput.1335 *1336 * @ring_container: Container for rings to have ITR updated1337 **/1338static void fm10k_update_itr(struct fm10k_ring_container *ring_container)1339{1340 unsigned int avg_wire_size, packets, itr_round;1341 1342 /* Only update ITR if we are using adaptive setting */1343 if (!ITR_IS_ADAPTIVE(ring_container->itr))1344 goto clear_counts;1345 1346 packets = ring_container->total_packets;1347 if (!packets)1348 goto clear_counts;1349 1350 avg_wire_size = ring_container->total_bytes / packets;1351 1352 /* The following is a crude approximation of:1353 * wmem_default / (size + overhead) = desired_pkts_per_int1354 * rate / bits_per_byte / (size + ethernet overhead) = pkt_rate1355 * (desired_pkt_rate / pkt_rate) * usecs_per_sec = ITR value1356 *1357 * Assuming wmem_default is 212992 and overhead is 640 bytes per1358 * packet, (256 skb, 64 headroom, 320 shared info), we can reduce the1359 * formula down to1360 *1361 * (34 * (size + 24)) / (size + 640) = ITR1362 *1363 * We first do some math on the packet size and then finally bitshift1364 * by 8 after rounding up. We also have to account for PCIe link speed1365 * difference as ITR scales based on this.1366 */1367 if (avg_wire_size <= 360) {1368 /* Start at 250K ints/sec and gradually drop to 77K ints/sec */1369 avg_wire_size *= 8;1370 avg_wire_size += 376;1371 } else if (avg_wire_size <= 1152) {1372 /* 77K ints/sec to 45K ints/sec */1373 avg_wire_size *= 3;1374 avg_wire_size += 2176;1375 } else if (avg_wire_size <= 1920) {1376 /* 45K ints/sec to 38K ints/sec */1377 avg_wire_size += 4480;1378 } else {1379 /* plateau at a limit of 38K ints/sec */1380 avg_wire_size = 6656;1381 }1382 1383 /* Perform final bitshift for division after rounding up to ensure1384 * that the calculation will never get below a 1. The bit shift1385 * accounts for changes in the ITR due to PCIe link speed.1386 */1387 itr_round = READ_ONCE(ring_container->itr_scale) + 8;1388 avg_wire_size += BIT(itr_round) - 1;1389 avg_wire_size >>= itr_round;1390 1391 /* write back value and retain adaptive flag */1392 ring_container->itr = avg_wire_size | FM10K_ITR_ADAPTIVE;1393 1394clear_counts:1395 ring_container->total_bytes = 0;1396 ring_container->total_packets = 0;1397}1398 1399static void fm10k_qv_enable(struct fm10k_q_vector *q_vector)1400{1401 /* Enable auto-mask and clear the current mask */1402 u32 itr = FM10K_ITR_ENABLE;1403 1404 /* Update Tx ITR */1405 fm10k_update_itr(&q_vector->tx);1406 1407 /* Update Rx ITR */1408 fm10k_update_itr(&q_vector->rx);1409 1410 /* Store Tx itr in timer slot 0 */1411 itr |= (q_vector->tx.itr & FM10K_ITR_MAX);1412 1413 /* Shift Rx itr to timer slot 1 */1414 itr |= (q_vector->rx.itr & FM10K_ITR_MAX) << FM10K_ITR_INTERVAL1_SHIFT;1415 1416 /* Write the final value to the ITR register */1417 writel(itr, q_vector->itr);1418}1419 1420static int fm10k_poll(struct napi_struct *napi, int budget)1421{1422 struct fm10k_q_vector *q_vector =1423 container_of(napi, struct fm10k_q_vector, napi);1424 struct fm10k_ring *ring;1425 int per_ring_budget, work_done = 0;1426 bool clean_complete = true;1427 1428 fm10k_for_each_ring(ring, q_vector->tx) {1429 if (!fm10k_clean_tx_irq(q_vector, ring, budget))1430 clean_complete = false;1431 }1432 1433 /* Handle case where we are called by netpoll with a budget of 0 */1434 if (budget <= 0)1435 return budget;1436 1437 /* attempt to distribute budget to each queue fairly, but don't1438 * allow the budget to go below 1 because we'll exit polling1439 */1440 if (q_vector->rx.count > 1)1441 per_ring_budget = max(budget / q_vector->rx.count, 1);1442 else1443 per_ring_budget = budget;1444 1445 fm10k_for_each_ring(ring, q_vector->rx) {1446 int work = fm10k_clean_rx_irq(q_vector, ring, per_ring_budget);1447 1448 work_done += work;1449 if (work >= per_ring_budget)1450 clean_complete = false;1451 }1452 1453 /* If all work not completed, return budget and keep polling */1454 if (!clean_complete)1455 return budget;1456 1457 /* Exit the polling mode, but don't re-enable interrupts if stack might1458 * poll us due to busy-polling1459 */1460 if (likely(napi_complete_done(napi, work_done)))1461 fm10k_qv_enable(q_vector);1462 1463 return min(work_done, budget - 1);1464}1465 1466/**1467 * fm10k_set_qos_queues: Allocate queues for a QOS-enabled device1468 * @interface: board private structure to initialize1469 *1470 * When QoS (Quality of Service) is enabled, allocate queues for1471 * each traffic class. If multiqueue isn't available,then abort QoS1472 * initialization.1473 *1474 * This function handles all combinations of Qos and RSS.1475 *1476 **/1477static bool fm10k_set_qos_queues(struct fm10k_intfc *interface)1478{1479 struct net_device *dev = interface->netdev;1480 struct fm10k_ring_feature *f;1481 int rss_i, i;1482 int pcs;1483 1484 /* Map queue offset and counts onto allocated tx queues */1485 pcs = netdev_get_num_tc(dev);1486 1487 if (pcs <= 1)1488 return false;1489 1490 /* set QoS mask and indices */1491 f = &interface->ring_feature[RING_F_QOS];1492 f->indices = pcs;1493 f->mask = BIT(fls(pcs - 1)) - 1;1494 1495 /* determine the upper limit for our current DCB mode */1496 rss_i = interface->hw.mac.max_queues / pcs;1497 rss_i = BIT(fls(rss_i) - 1);1498 1499 /* set RSS mask and indices */1500 f = &interface->ring_feature[RING_F_RSS];1501 rss_i = min_t(u16, rss_i, f->limit);1502 f->indices = rss_i;1503 f->mask = BIT(fls(rss_i - 1)) - 1;1504 1505 /* configure pause class to queue mapping */1506 for (i = 0; i < pcs; i++)1507 netdev_set_tc_queue(dev, i, rss_i, rss_i * i);1508 1509 interface->num_rx_queues = rss_i * pcs;1510 interface->num_tx_queues = rss_i * pcs;1511 1512 return true;1513}1514 1515/**1516 * fm10k_set_rss_queues: Allocate queues for RSS1517 * @interface: board private structure to initialize1518 *1519 * This is our "base" multiqueue mode. RSS (Receive Side Scaling) will try1520 * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU.1521 *1522 **/1523static bool fm10k_set_rss_queues(struct fm10k_intfc *interface)1524{1525 struct fm10k_ring_feature *f;1526 u16 rss_i;1527 1528 f = &interface->ring_feature[RING_F_RSS];1529 rss_i = min_t(u16, interface->hw.mac.max_queues, f->limit);1530 1531 /* record indices and power of 2 mask for RSS */1532 f->indices = rss_i;1533 f->mask = BIT(fls(rss_i - 1)) - 1;1534 1535 interface->num_rx_queues = rss_i;1536 interface->num_tx_queues = rss_i;1537 1538 return true;1539}1540 1541/**1542 * fm10k_set_num_queues: Allocate queues for device, feature dependent1543 * @interface: board private structure to initialize1544 *1545 * This is the top level queue allocation routine. The order here is very1546 * important, starting with the "most" number of features turned on at once,1547 * and ending with the smallest set of features. This way large combinations1548 * can be allocated if they're turned on, and smaller combinations are the1549 * fall through conditions.1550 *1551 **/1552static void fm10k_set_num_queues(struct fm10k_intfc *interface)1553{1554 /* Attempt to setup QoS and RSS first */1555 if (fm10k_set_qos_queues(interface))1556 return;1557 1558 /* If we don't have QoS, just fallback to only RSS. */1559 fm10k_set_rss_queues(interface);1560}1561 1562/**1563 * fm10k_reset_num_queues - Reset the number of queues to zero1564 * @interface: board private structure1565 *1566 * This function should be called whenever we need to reset the number of1567 * queues after an error condition.1568 */1569static void fm10k_reset_num_queues(struct fm10k_intfc *interface)1570{1571 interface->num_tx_queues = 0;1572 interface->num_rx_queues = 0;1573 interface->num_q_vectors = 0;1574}1575 1576/**1577 * fm10k_alloc_q_vector - Allocate memory for a single interrupt vector1578 * @interface: board private structure to initialize1579 * @v_count: q_vectors allocated on interface, used for ring interleaving1580 * @v_idx: index of vector in interface struct1581 * @txr_count: total number of Tx rings to allocate1582 * @txr_idx: index of first Tx ring to allocate1583 * @rxr_count: total number of Rx rings to allocate1584 * @rxr_idx: index of first Rx ring to allocate1585 *1586 * We allocate one q_vector. If allocation fails we return -ENOMEM.1587 **/1588static int fm10k_alloc_q_vector(struct fm10k_intfc *interface,1589 unsigned int v_count, unsigned int v_idx,1590 unsigned int txr_count, unsigned int txr_idx,1591 unsigned int rxr_count, unsigned int rxr_idx)1592{1593 struct fm10k_q_vector *q_vector;1594 struct fm10k_ring *ring;1595 int ring_count;1596 1597 ring_count = txr_count + rxr_count;1598 1599 /* allocate q_vector and rings */1600 q_vector = kzalloc(struct_size(q_vector, ring, ring_count), GFP_KERNEL);1601 if (!q_vector)1602 return -ENOMEM;1603 1604 /* initialize NAPI */1605 netif_napi_add(interface->netdev, &q_vector->napi, fm10k_poll);1606 1607 /* tie q_vector and interface together */1608 interface->q_vector[v_idx] = q_vector;1609 q_vector->interface = interface;1610 q_vector->v_idx = v_idx;1611 1612 /* initialize pointer to rings */1613 ring = q_vector->ring;1614 1615 /* save Tx ring container info */1616 q_vector->tx.ring = ring;1617 q_vector->tx.work_limit = FM10K_DEFAULT_TX_WORK;1618 q_vector->tx.itr = interface->tx_itr;1619 q_vector->tx.itr_scale = interface->hw.mac.itr_scale;1620 q_vector->tx.count = txr_count;1621 1622 while (txr_count) {1623 /* assign generic ring traits */1624 ring->dev = &interface->pdev->dev;1625 ring->netdev = interface->netdev;1626 1627 /* configure backlink on ring */1628 ring->q_vector = q_vector;1629 1630 /* apply Tx specific ring traits */1631 ring->count = interface->tx_ring_count;1632 ring->queue_index = txr_idx;1633 1634 /* assign ring to interface */1635 interface->tx_ring[txr_idx] = ring;1636 1637 /* update count and index */1638 txr_count--;1639 txr_idx += v_count;1640 1641 /* push pointer to next ring */1642 ring++;1643 }1644 1645 /* save Rx ring container info */1646 q_vector->rx.ring = ring;1647 q_vector->rx.itr = interface->rx_itr;1648 q_vector->rx.itr_scale = interface->hw.mac.itr_scale;1649 q_vector->rx.count = rxr_count;1650 1651 while (rxr_count) {1652 /* assign generic ring traits */1653 ring->dev = &interface->pdev->dev;1654 ring->netdev = interface->netdev;1655 rcu_assign_pointer(ring->l2_accel, interface->l2_accel);1656 1657 /* configure backlink on ring */1658 ring->q_vector = q_vector;1659 1660 /* apply Rx specific ring traits */1661 ring->count = interface->rx_ring_count;1662 ring->queue_index = rxr_idx;1663 1664 /* assign ring to interface */1665 interface->rx_ring[rxr_idx] = ring;1666 1667 /* update count and index */1668 rxr_count--;1669 rxr_idx += v_count;1670 1671 /* push pointer to next ring */1672 ring++;1673 }1674 1675 fm10k_dbg_q_vector_init(q_vector);1676 1677 return 0;1678}1679 1680/**1681 * fm10k_free_q_vector - Free memory allocated for specific interrupt vector1682 * @interface: board private structure to initialize1683 * @v_idx: Index of vector to be freed1684 *1685 * This function frees the memory allocated to the q_vector. In addition if1686 * NAPI is enabled it will delete any references to the NAPI struct prior1687 * to freeing the q_vector.1688 **/1689static void fm10k_free_q_vector(struct fm10k_intfc *interface, int v_idx)1690{1691 struct fm10k_q_vector *q_vector = interface->q_vector[v_idx];1692 struct fm10k_ring *ring;1693 1694 fm10k_dbg_q_vector_exit(q_vector);1695 1696 fm10k_for_each_ring(ring, q_vector->tx)1697 interface->tx_ring[ring->queue_index] = NULL;1698 1699 fm10k_for_each_ring(ring, q_vector->rx)1700 interface->rx_ring[ring->queue_index] = NULL;1701 1702 interface->q_vector[v_idx] = NULL;1703 netif_napi_del(&q_vector->napi);1704 kfree_rcu(q_vector, rcu);1705}1706 1707/**1708 * fm10k_alloc_q_vectors - Allocate memory for interrupt vectors1709 * @interface: board private structure to initialize1710 *1711 * We allocate one q_vector per queue interrupt. If allocation fails we1712 * return -ENOMEM.1713 **/1714static int fm10k_alloc_q_vectors(struct fm10k_intfc *interface)1715{1716 unsigned int q_vectors = interface->num_q_vectors;1717 unsigned int rxr_remaining = interface->num_rx_queues;1718 unsigned int txr_remaining = interface->num_tx_queues;1719 unsigned int rxr_idx = 0, txr_idx = 0, v_idx = 0;1720 int err;1721 1722 if (q_vectors >= (rxr_remaining + txr_remaining)) {1723 for (; rxr_remaining; v_idx++) {1724 err = fm10k_alloc_q_vector(interface, q_vectors, v_idx,1725 0, 0, 1, rxr_idx);1726 if (err)1727 goto err_out;1728 1729 /* update counts and index */1730 rxr_remaining--;1731 rxr_idx++;1732 }1733 }1734 1735 for (; v_idx < q_vectors; v_idx++) {1736 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);1737 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);1738 1739 err = fm10k_alloc_q_vector(interface, q_vectors, v_idx,1740 tqpv, txr_idx,1741 rqpv, rxr_idx);1742 1743 if (err)1744 goto err_out;1745 1746 /* update counts and index */1747 rxr_remaining -= rqpv;1748 txr_remaining -= tqpv;1749 rxr_idx++;1750 txr_idx++;1751 }1752 1753 return 0;1754 1755err_out:1756 fm10k_reset_num_queues(interface);1757 1758 while (v_idx--)1759 fm10k_free_q_vector(interface, v_idx);1760 1761 return -ENOMEM;1762}1763 1764/**1765 * fm10k_free_q_vectors - Free memory allocated for interrupt vectors1766 * @interface: board private structure to initialize1767 *1768 * This function frees the memory allocated to the q_vectors. In addition if1769 * NAPI is enabled it will delete any references to the NAPI struct prior1770 * to freeing the q_vector.1771 **/1772static void fm10k_free_q_vectors(struct fm10k_intfc *interface)1773{1774 int v_idx = interface->num_q_vectors;1775 1776 fm10k_reset_num_queues(interface);1777 1778 while (v_idx--)1779 fm10k_free_q_vector(interface, v_idx);1780}1781 1782/**1783 * fm10k_reset_msix_capability - reset MSI-X capability1784 * @interface: board private structure to initialize1785 *1786 * Reset the MSI-X capability back to its starting state1787 **/1788static void fm10k_reset_msix_capability(struct fm10k_intfc *interface)1789{1790 pci_disable_msix(interface->pdev);1791 kfree(interface->msix_entries);1792 interface->msix_entries = NULL;1793}1794 1795/**1796 * fm10k_init_msix_capability - configure MSI-X capability1797 * @interface: board private structure to initialize1798 *1799 * Attempt to configure the interrupts using the best available1800 * capabilities of the hardware and the kernel.1801 **/1802static int fm10k_init_msix_capability(struct fm10k_intfc *interface)1803{1804 struct fm10k_hw *hw = &interface->hw;1805 int v_budget, vector;1806 1807 /* It's easy to be greedy for MSI-X vectors, but it really1808 * doesn't do us much good if we have a lot more vectors1809 * than CPU's. So let's be conservative and only ask for1810 * (roughly) the same number of vectors as there are CPU's.1811 * the default is to use pairs of vectors1812 */1813 v_budget = max(interface->num_rx_queues, interface->num_tx_queues);1814 v_budget = min_t(u16, v_budget, num_online_cpus());1815 1816 /* account for vectors not related to queues */1817 v_budget += NON_Q_VECTORS;1818 1819 /* At the same time, hardware can only support a maximum of1820 * hw.mac->max_msix_vectors vectors. With features1821 * such as RSS and VMDq, we can easily surpass the number of Rx and Tx1822 * descriptor queues supported by our device. Thus, we cap it off in1823 * those rare cases where the cpu count also exceeds our vector limit.1824 */1825 v_budget = min_t(int, v_budget, hw->mac.max_msix_vectors);1826 1827 /* A failure in MSI-X entry allocation is fatal. */1828 interface->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),1829 GFP_KERNEL);1830 if (!interface->msix_entries)1831 return -ENOMEM;1832 1833 /* populate entry values */1834 for (vector = 0; vector < v_budget; vector++)1835 interface->msix_entries[vector].entry = vector;1836 1837 /* Attempt to enable MSI-X with requested value */1838 v_budget = pci_enable_msix_range(interface->pdev,1839 interface->msix_entries,1840 MIN_MSIX_COUNT(hw),1841 v_budget);1842 if (v_budget < 0) {1843 kfree(interface->msix_entries);1844 interface->msix_entries = NULL;1845 return v_budget;1846 }1847 1848 /* record the number of queues available for q_vectors */1849 interface->num_q_vectors = v_budget - NON_Q_VECTORS;1850 1851 return 0;1852}1853 1854/**1855 * fm10k_cache_ring_qos - Descriptor ring to register mapping for QoS1856 * @interface: Interface structure continaining rings and devices1857 *1858 * Cache the descriptor ring offsets for Qos1859 **/1860static bool fm10k_cache_ring_qos(struct fm10k_intfc *interface)1861{1862 struct net_device *dev = interface->netdev;1863 int pc, offset, rss_i, i;1864 u16 pc_stride = interface->ring_feature[RING_F_QOS].mask + 1;1865 u8 num_pcs = netdev_get_num_tc(dev);1866 1867 if (num_pcs <= 1)1868 return false;1869 1870 rss_i = interface->ring_feature[RING_F_RSS].indices;1871 1872 for (pc = 0, offset = 0; pc < num_pcs; pc++, offset += rss_i) {1873 int q_idx = pc;1874 1875 for (i = 0; i < rss_i; i++) {1876 interface->tx_ring[offset + i]->reg_idx = q_idx;1877 interface->tx_ring[offset + i]->qos_pc = pc;1878 interface->rx_ring[offset + i]->reg_idx = q_idx;1879 interface->rx_ring[offset + i]->qos_pc = pc;1880 q_idx += pc_stride;1881 }1882 }1883 1884 return true;1885}1886 1887/**1888 * fm10k_cache_ring_rss - Descriptor ring to register mapping for RSS1889 * @interface: Interface structure continaining rings and devices1890 *1891 * Cache the descriptor ring offsets for RSS1892 **/1893static void fm10k_cache_ring_rss(struct fm10k_intfc *interface)1894{1895 int i;1896 1897 for (i = 0; i < interface->num_rx_queues; i++)1898 interface->rx_ring[i]->reg_idx = i;1899 1900 for (i = 0; i < interface->num_tx_queues; i++)1901 interface->tx_ring[i]->reg_idx = i;1902}1903 1904/**1905 * fm10k_assign_rings - Map rings to network devices1906 * @interface: Interface structure containing rings and devices1907 *1908 * This function is meant to go though and configure both the network1909 * devices so that they contain rings, and configure the rings so that1910 * they function with their network devices.1911 **/1912static void fm10k_assign_rings(struct fm10k_intfc *interface)1913{1914 if (fm10k_cache_ring_qos(interface))1915 return;1916 1917 fm10k_cache_ring_rss(interface);1918}1919 1920static void fm10k_init_reta(struct fm10k_intfc *interface)1921{1922 u16 i, rss_i = interface->ring_feature[RING_F_RSS].indices;1923 u32 reta;1924 1925 /* If the Rx flow indirection table has been configured manually, we1926 * need to maintain it when possible.1927 */1928 if (netif_is_rxfh_configured(interface->netdev)) {1929 for (i = FM10K_RETA_SIZE; i--;) {1930 reta = interface->reta[i];1931 if ((((reta << 24) >> 24) < rss_i) &&1932 (((reta << 16) >> 24) < rss_i) &&1933 (((reta << 8) >> 24) < rss_i) &&1934 (((reta) >> 24) < rss_i))1935 continue;1936 1937 /* this should never happen */1938 dev_err(&interface->pdev->dev,1939 "RSS indirection table assigned flows out of queue bounds. Reconfiguring.\n");1940 goto repopulate_reta;1941 }1942 1943 /* do nothing if all of the elements are in bounds */1944 return;1945 }1946 1947repopulate_reta:1948 fm10k_write_reta(interface, NULL);1949}1950 1951/**1952 * fm10k_init_queueing_scheme - Determine proper queueing scheme1953 * @interface: board private structure to initialize1954 *1955 * We determine which queueing scheme to use based on...1956 * - Hardware queue count (num_*_queues)1957 * - defined by miscellaneous hardware support/features (RSS, etc.)1958 **/1959int fm10k_init_queueing_scheme(struct fm10k_intfc *interface)1960{1961 int err;1962 1963 /* Number of supported queues */1964 fm10k_set_num_queues(interface);1965 1966 /* Configure MSI-X capability */1967 err = fm10k_init_msix_capability(interface);1968 if (err) {1969 dev_err(&interface->pdev->dev,1970 "Unable to initialize MSI-X capability\n");1971 goto err_init_msix;1972 }1973 1974 /* Allocate memory for queues */1975 err = fm10k_alloc_q_vectors(interface);1976 if (err) {1977 dev_err(&interface->pdev->dev,1978 "Unable to allocate queue vectors\n");1979 goto err_alloc_q_vectors;1980 }1981 1982 /* Map rings to devices, and map devices to physical queues */1983 fm10k_assign_rings(interface);1984 1985 /* Initialize RSS redirection table */1986 fm10k_init_reta(interface);1987 1988 return 0;1989 1990err_alloc_q_vectors:1991 fm10k_reset_msix_capability(interface);1992err_init_msix:1993 fm10k_reset_num_queues(interface);1994 return err;1995}1996 1997/**1998 * fm10k_clear_queueing_scheme - Clear the current queueing scheme settings1999 * @interface: board private structure to clear queueing scheme on2000 *2001 * We go through and clear queueing specific resources and reset the structure2002 * to pre-load conditions2003 **/2004void fm10k_clear_queueing_scheme(struct fm10k_intfc *interface)2005{2006 fm10k_free_q_vectors(interface);2007 fm10k_reset_msix_capability(interface);2008}2009