2538 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) 2018, Intel Corporation. */3 4/* The driver transmit and receive code */5 6#include <linux/mm.h>7#include <linux/netdevice.h>8#include <linux/prefetch.h>9#include <linux/bpf_trace.h>10#include <net/dsfield.h>11#include <net/mpls.h>12#include <net/xdp.h>13#include "ice_txrx_lib.h"14#include "ice_lib.h"15#include "ice.h"16#include "ice_trace.h"17#include "ice_dcb_lib.h"18#include "ice_xsk.h"19#include "ice_eswitch.h"20 21#define ICE_RX_HDR_SIZE 25622 23#define FDIR_DESC_RXDID 0x4024#define ICE_FDIR_CLEAN_DELAY 1025 26/**27 * ice_prgm_fdir_fltr - Program a Flow Director filter28 * @vsi: VSI to send dummy packet29 * @fdir_desc: flow director descriptor30 * @raw_packet: allocated buffer for flow director31 */32int33ice_prgm_fdir_fltr(struct ice_vsi *vsi, struct ice_fltr_desc *fdir_desc,34 u8 *raw_packet)35{36 struct ice_tx_buf *tx_buf, *first;37 struct ice_fltr_desc *f_desc;38 struct ice_tx_desc *tx_desc;39 struct ice_tx_ring *tx_ring;40 struct device *dev;41 dma_addr_t dma;42 u32 td_cmd;43 u16 i;44 45 /* VSI and Tx ring */46 if (!vsi)47 return -ENOENT;48 tx_ring = vsi->tx_rings[0];49 if (!tx_ring || !tx_ring->desc)50 return -ENOENT;51 dev = tx_ring->dev;52 53 /* we are using two descriptors to add/del a filter and we can wait */54 for (i = ICE_FDIR_CLEAN_DELAY; ICE_DESC_UNUSED(tx_ring) < 2; i--) {55 if (!i)56 return -EAGAIN;57 msleep_interruptible(1);58 }59 60 dma = dma_map_single(dev, raw_packet, ICE_FDIR_MAX_RAW_PKT_SIZE,61 DMA_TO_DEVICE);62 63 if (dma_mapping_error(dev, dma))64 return -EINVAL;65 66 /* grab the next descriptor */67 i = tx_ring->next_to_use;68 first = &tx_ring->tx_buf[i];69 f_desc = ICE_TX_FDIRDESC(tx_ring, i);70 memcpy(f_desc, fdir_desc, sizeof(*f_desc));71 72 i++;73 i = (i < tx_ring->count) ? i : 0;74 tx_desc = ICE_TX_DESC(tx_ring, i);75 tx_buf = &tx_ring->tx_buf[i];76 77 i++;78 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;79 80 memset(tx_buf, 0, sizeof(*tx_buf));81 dma_unmap_len_set(tx_buf, len, ICE_FDIR_MAX_RAW_PKT_SIZE);82 dma_unmap_addr_set(tx_buf, dma, dma);83 84 tx_desc->buf_addr = cpu_to_le64(dma);85 td_cmd = ICE_TXD_LAST_DESC_CMD | ICE_TX_DESC_CMD_DUMMY |86 ICE_TX_DESC_CMD_RE;87 88 tx_buf->type = ICE_TX_BUF_DUMMY;89 tx_buf->raw_buf = raw_packet;90 91 tx_desc->cmd_type_offset_bsz =92 ice_build_ctob(td_cmd, 0, ICE_FDIR_MAX_RAW_PKT_SIZE, 0);93 94 /* Force memory write to complete before letting h/w know95 * there are new descriptors to fetch.96 */97 wmb();98 99 /* mark the data descriptor to be watched */100 first->next_to_watch = tx_desc;101 102 writel(tx_ring->next_to_use, tx_ring->tail);103 104 return 0;105}106 107/**108 * ice_unmap_and_free_tx_buf - Release a Tx buffer109 * @ring: the ring that owns the buffer110 * @tx_buf: the buffer to free111 */112static void113ice_unmap_and_free_tx_buf(struct ice_tx_ring *ring, struct ice_tx_buf *tx_buf)114{115 if (dma_unmap_len(tx_buf, len))116 dma_unmap_page(ring->dev,117 dma_unmap_addr(tx_buf, dma),118 dma_unmap_len(tx_buf, len),119 DMA_TO_DEVICE);120 121 switch (tx_buf->type) {122 case ICE_TX_BUF_DUMMY:123 devm_kfree(ring->dev, tx_buf->raw_buf);124 break;125 case ICE_TX_BUF_SKB:126 dev_kfree_skb_any(tx_buf->skb);127 break;128 case ICE_TX_BUF_XDP_TX:129 page_frag_free(tx_buf->raw_buf);130 break;131 case ICE_TX_BUF_XDP_XMIT:132 xdp_return_frame(tx_buf->xdpf);133 break;134 }135 136 tx_buf->next_to_watch = NULL;137 tx_buf->type = ICE_TX_BUF_EMPTY;138 dma_unmap_len_set(tx_buf, len, 0);139 /* tx_buf must be completely set up in the transmit path */140}141 142static struct netdev_queue *txring_txq(const struct ice_tx_ring *ring)143{144 return netdev_get_tx_queue(ring->netdev, ring->q_index);145}146 147/**148 * ice_clean_tx_ring - Free any empty Tx buffers149 * @tx_ring: ring to be cleaned150 */151void ice_clean_tx_ring(struct ice_tx_ring *tx_ring)152{153 u32 size;154 u16 i;155 156 if (ice_ring_is_xdp(tx_ring) && tx_ring->xsk_pool) {157 ice_xsk_clean_xdp_ring(tx_ring);158 goto tx_skip_free;159 }160 161 /* ring already cleared, nothing to do */162 if (!tx_ring->tx_buf)163 return;164 165 /* Free all the Tx ring sk_buffs */166 for (i = 0; i < tx_ring->count; i++)167 ice_unmap_and_free_tx_buf(tx_ring, &tx_ring->tx_buf[i]);168 169tx_skip_free:170 memset(tx_ring->tx_buf, 0, sizeof(*tx_ring->tx_buf) * tx_ring->count);171 172 size = ALIGN(tx_ring->count * sizeof(struct ice_tx_desc),173 PAGE_SIZE);174 /* Zero out the descriptor ring */175 memset(tx_ring->desc, 0, size);176 177 tx_ring->next_to_use = 0;178 tx_ring->next_to_clean = 0;179 180 if (!tx_ring->netdev)181 return;182 183 /* cleanup Tx queue statistics */184 netdev_tx_reset_queue(txring_txq(tx_ring));185}186 187/**188 * ice_free_tx_ring - Free Tx resources per queue189 * @tx_ring: Tx descriptor ring for a specific queue190 *191 * Free all transmit software resources192 */193void ice_free_tx_ring(struct ice_tx_ring *tx_ring)194{195 u32 size;196 197 ice_clean_tx_ring(tx_ring);198 devm_kfree(tx_ring->dev, tx_ring->tx_buf);199 tx_ring->tx_buf = NULL;200 201 if (tx_ring->desc) {202 size = ALIGN(tx_ring->count * sizeof(struct ice_tx_desc),203 PAGE_SIZE);204 dmam_free_coherent(tx_ring->dev, size,205 tx_ring->desc, tx_ring->dma);206 tx_ring->desc = NULL;207 }208}209 210/**211 * ice_clean_tx_irq - Reclaim resources after transmit completes212 * @tx_ring: Tx ring to clean213 * @napi_budget: Used to determine if we are in netpoll214 *215 * Returns true if there's any budget left (e.g. the clean is finished)216 */217static bool ice_clean_tx_irq(struct ice_tx_ring *tx_ring, int napi_budget)218{219 unsigned int total_bytes = 0, total_pkts = 0;220 unsigned int budget = ICE_DFLT_IRQ_WORK;221 struct ice_vsi *vsi = tx_ring->vsi;222 s16 i = tx_ring->next_to_clean;223 struct ice_tx_desc *tx_desc;224 struct ice_tx_buf *tx_buf;225 226 /* get the bql data ready */227 netdev_txq_bql_complete_prefetchw(txring_txq(tx_ring));228 229 tx_buf = &tx_ring->tx_buf[i];230 tx_desc = ICE_TX_DESC(tx_ring, i);231 i -= tx_ring->count;232 233 prefetch(&vsi->state);234 235 do {236 struct ice_tx_desc *eop_desc = tx_buf->next_to_watch;237 238 /* if next_to_watch is not set then there is no work pending */239 if (!eop_desc)240 break;241 242 /* follow the guidelines of other drivers */243 prefetchw(&tx_buf->skb->users);244 245 smp_rmb(); /* prevent any other reads prior to eop_desc */246 247 ice_trace(clean_tx_irq, tx_ring, tx_desc, tx_buf);248 /* if the descriptor isn't done, no work yet to do */249 if (!(eop_desc->cmd_type_offset_bsz &250 cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE)))251 break;252 253 /* clear next_to_watch to prevent false hangs */254 tx_buf->next_to_watch = NULL;255 256 /* update the statistics for this packet */257 total_bytes += tx_buf->bytecount;258 total_pkts += tx_buf->gso_segs;259 260 /* free the skb */261 napi_consume_skb(tx_buf->skb, napi_budget);262 263 /* unmap skb header data */264 dma_unmap_single(tx_ring->dev,265 dma_unmap_addr(tx_buf, dma),266 dma_unmap_len(tx_buf, len),267 DMA_TO_DEVICE);268 269 /* clear tx_buf data */270 tx_buf->type = ICE_TX_BUF_EMPTY;271 dma_unmap_len_set(tx_buf, len, 0);272 273 /* unmap remaining buffers */274 while (tx_desc != eop_desc) {275 ice_trace(clean_tx_irq_unmap, tx_ring, tx_desc, tx_buf);276 tx_buf++;277 tx_desc++;278 i++;279 if (unlikely(!i)) {280 i -= tx_ring->count;281 tx_buf = tx_ring->tx_buf;282 tx_desc = ICE_TX_DESC(tx_ring, 0);283 }284 285 /* unmap any remaining paged data */286 if (dma_unmap_len(tx_buf, len)) {287 dma_unmap_page(tx_ring->dev,288 dma_unmap_addr(tx_buf, dma),289 dma_unmap_len(tx_buf, len),290 DMA_TO_DEVICE);291 dma_unmap_len_set(tx_buf, len, 0);292 }293 }294 ice_trace(clean_tx_irq_unmap_eop, tx_ring, tx_desc, tx_buf);295 296 /* move us one more past the eop_desc for start of next pkt */297 tx_buf++;298 tx_desc++;299 i++;300 if (unlikely(!i)) {301 i -= tx_ring->count;302 tx_buf = tx_ring->tx_buf;303 tx_desc = ICE_TX_DESC(tx_ring, 0);304 }305 306 prefetch(tx_desc);307 308 /* update budget accounting */309 budget--;310 } while (likely(budget));311 312 i += tx_ring->count;313 tx_ring->next_to_clean = i;314 315 ice_update_tx_ring_stats(tx_ring, total_pkts, total_bytes);316 netdev_tx_completed_queue(txring_txq(tx_ring), total_pkts, total_bytes);317 318#define TX_WAKE_THRESHOLD ((s16)(DESC_NEEDED * 2))319 if (unlikely(total_pkts && netif_carrier_ok(tx_ring->netdev) &&320 (ICE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {321 /* Make sure that anybody stopping the queue after this322 * sees the new next_to_clean.323 */324 smp_mb();325 if (netif_tx_queue_stopped(txring_txq(tx_ring)) &&326 !test_bit(ICE_VSI_DOWN, vsi->state)) {327 netif_tx_wake_queue(txring_txq(tx_ring));328 ++tx_ring->ring_stats->tx_stats.restart_q;329 }330 }331 332 return !!budget;333}334 335/**336 * ice_setup_tx_ring - Allocate the Tx descriptors337 * @tx_ring: the Tx ring to set up338 *339 * Return 0 on success, negative on error340 */341int ice_setup_tx_ring(struct ice_tx_ring *tx_ring)342{343 struct device *dev = tx_ring->dev;344 u32 size;345 346 if (!dev)347 return -ENOMEM;348 349 /* warn if we are about to overwrite the pointer */350 WARN_ON(tx_ring->tx_buf);351 tx_ring->tx_buf =352 devm_kcalloc(dev, sizeof(*tx_ring->tx_buf), tx_ring->count,353 GFP_KERNEL);354 if (!tx_ring->tx_buf)355 return -ENOMEM;356 357 /* round up to nearest page */358 size = ALIGN(tx_ring->count * sizeof(struct ice_tx_desc),359 PAGE_SIZE);360 tx_ring->desc = dmam_alloc_coherent(dev, size, &tx_ring->dma,361 GFP_KERNEL);362 if (!tx_ring->desc) {363 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",364 size);365 goto err;366 }367 368 tx_ring->next_to_use = 0;369 tx_ring->next_to_clean = 0;370 tx_ring->ring_stats->tx_stats.prev_pkt = -1;371 return 0;372 373err:374 devm_kfree(dev, tx_ring->tx_buf);375 tx_ring->tx_buf = NULL;376 return -ENOMEM;377}378 379/**380 * ice_clean_rx_ring - Free Rx buffers381 * @rx_ring: ring to be cleaned382 */383void ice_clean_rx_ring(struct ice_rx_ring *rx_ring)384{385 struct xdp_buff *xdp = &rx_ring->xdp;386 struct device *dev = rx_ring->dev;387 u32 size;388 u16 i;389 390 /* ring already cleared, nothing to do */391 if (!rx_ring->rx_buf)392 return;393 394 if (rx_ring->xsk_pool) {395 ice_xsk_clean_rx_ring(rx_ring);396 goto rx_skip_free;397 }398 399 if (xdp->data) {400 xdp_return_buff(xdp);401 xdp->data = NULL;402 }403 404 /* Free all the Rx ring sk_buffs */405 for (i = 0; i < rx_ring->count; i++) {406 struct ice_rx_buf *rx_buf = &rx_ring->rx_buf[i];407 408 if (!rx_buf->page)409 continue;410 411 /* Invalidate cache lines that may have been written to by412 * device so that we avoid corrupting memory.413 */414 dma_sync_single_range_for_cpu(dev, rx_buf->dma,415 rx_buf->page_offset,416 rx_ring->rx_buf_len,417 DMA_FROM_DEVICE);418 419 /* free resources associated with mapping */420 dma_unmap_page_attrs(dev, rx_buf->dma, ice_rx_pg_size(rx_ring),421 DMA_FROM_DEVICE, ICE_RX_DMA_ATTR);422 __page_frag_cache_drain(rx_buf->page, rx_buf->pagecnt_bias);423 424 rx_buf->page = NULL;425 rx_buf->page_offset = 0;426 }427 428rx_skip_free:429 if (rx_ring->xsk_pool)430 memset(rx_ring->xdp_buf, 0, array_size(rx_ring->count, sizeof(*rx_ring->xdp_buf)));431 else432 memset(rx_ring->rx_buf, 0, array_size(rx_ring->count, sizeof(*rx_ring->rx_buf)));433 434 /* Zero out the descriptor ring */435 size = ALIGN(rx_ring->count * sizeof(union ice_32byte_rx_desc),436 PAGE_SIZE);437 memset(rx_ring->desc, 0, size);438 439 rx_ring->next_to_alloc = 0;440 rx_ring->next_to_clean = 0;441 rx_ring->first_desc = 0;442 rx_ring->next_to_use = 0;443}444 445/**446 * ice_free_rx_ring - Free Rx resources447 * @rx_ring: ring to clean the resources from448 *449 * Free all receive software resources450 */451void ice_free_rx_ring(struct ice_rx_ring *rx_ring)452{453 u32 size;454 455 ice_clean_rx_ring(rx_ring);456 if (rx_ring->vsi->type == ICE_VSI_PF)457 if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq))458 xdp_rxq_info_unreg(&rx_ring->xdp_rxq);459 WRITE_ONCE(rx_ring->xdp_prog, NULL);460 if (rx_ring->xsk_pool) {461 kfree(rx_ring->xdp_buf);462 rx_ring->xdp_buf = NULL;463 } else {464 kfree(rx_ring->rx_buf);465 rx_ring->rx_buf = NULL;466 }467 468 if (rx_ring->desc) {469 size = ALIGN(rx_ring->count * sizeof(union ice_32byte_rx_desc),470 PAGE_SIZE);471 dmam_free_coherent(rx_ring->dev, size,472 rx_ring->desc, rx_ring->dma);473 rx_ring->desc = NULL;474 }475}476 477/**478 * ice_setup_rx_ring - Allocate the Rx descriptors479 * @rx_ring: the Rx ring to set up480 *481 * Return 0 on success, negative on error482 */483int ice_setup_rx_ring(struct ice_rx_ring *rx_ring)484{485 struct device *dev = rx_ring->dev;486 u32 size;487 488 if (!dev)489 return -ENOMEM;490 491 /* warn if we are about to overwrite the pointer */492 WARN_ON(rx_ring->rx_buf);493 rx_ring->rx_buf =494 kcalloc(rx_ring->count, sizeof(*rx_ring->rx_buf), GFP_KERNEL);495 if (!rx_ring->rx_buf)496 return -ENOMEM;497 498 /* round up to nearest page */499 size = ALIGN(rx_ring->count * sizeof(union ice_32byte_rx_desc),500 PAGE_SIZE);501 rx_ring->desc = dmam_alloc_coherent(dev, size, &rx_ring->dma,502 GFP_KERNEL);503 if (!rx_ring->desc) {504 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",505 size);506 goto err;507 }508 509 rx_ring->next_to_use = 0;510 rx_ring->next_to_clean = 0;511 rx_ring->first_desc = 0;512 513 if (ice_is_xdp_ena_vsi(rx_ring->vsi))514 WRITE_ONCE(rx_ring->xdp_prog, rx_ring->vsi->xdp_prog);515 516 return 0;517 518err:519 kfree(rx_ring->rx_buf);520 rx_ring->rx_buf = NULL;521 return -ENOMEM;522}523 524/**525 * ice_run_xdp - Executes an XDP program on initialized xdp_buff526 * @rx_ring: Rx ring527 * @xdp: xdp_buff used as input to the XDP program528 * @xdp_prog: XDP program to run529 * @xdp_ring: ring to be used for XDP_TX action530 * @rx_buf: Rx buffer to store the XDP action531 * @eop_desc: Last descriptor in packet to read metadata from532 *533 * Returns any of ICE_XDP_{PASS, CONSUMED, TX, REDIR}534 */535static void536ice_run_xdp(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp,537 struct bpf_prog *xdp_prog, struct ice_tx_ring *xdp_ring,538 struct ice_rx_buf *rx_buf, union ice_32b_rx_flex_desc *eop_desc)539{540 unsigned int ret = ICE_XDP_PASS;541 u32 act;542 543 if (!xdp_prog)544 goto exit;545 546 ice_xdp_meta_set_desc(xdp, eop_desc);547 548 act = bpf_prog_run_xdp(xdp_prog, xdp);549 switch (act) {550 case XDP_PASS:551 break;552 case XDP_TX:553 if (static_branch_unlikely(&ice_xdp_locking_key))554 spin_lock(&xdp_ring->tx_lock);555 ret = __ice_xmit_xdp_ring(xdp, xdp_ring, false);556 if (static_branch_unlikely(&ice_xdp_locking_key))557 spin_unlock(&xdp_ring->tx_lock);558 if (ret == ICE_XDP_CONSUMED)559 goto out_failure;560 break;561 case XDP_REDIRECT:562 if (xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog))563 goto out_failure;564 ret = ICE_XDP_REDIR;565 break;566 default:567 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act);568 fallthrough;569 case XDP_ABORTED:570out_failure:571 trace_xdp_exception(rx_ring->netdev, xdp_prog, act);572 fallthrough;573 case XDP_DROP:574 ret = ICE_XDP_CONSUMED;575 }576exit:577 ice_set_rx_bufs_act(xdp, rx_ring, ret);578}579 580/**581 * ice_xmit_xdp_ring - submit frame to XDP ring for transmission582 * @xdpf: XDP frame that will be converted to XDP buff583 * @xdp_ring: XDP ring for transmission584 */585static int ice_xmit_xdp_ring(const struct xdp_frame *xdpf,586 struct ice_tx_ring *xdp_ring)587{588 struct xdp_buff xdp;589 590 xdp.data_hard_start = (void *)xdpf;591 xdp.data = xdpf->data;592 xdp.data_end = xdp.data + xdpf->len;593 xdp.frame_sz = xdpf->frame_sz;594 xdp.flags = xdpf->flags;595 596 return __ice_xmit_xdp_ring(&xdp, xdp_ring, true);597}598 599/**600 * ice_xdp_xmit - submit packets to XDP ring for transmission601 * @dev: netdev602 * @n: number of XDP frames to be transmitted603 * @frames: XDP frames to be transmitted604 * @flags: transmit flags605 *606 * Returns number of frames successfully sent. Failed frames607 * will be free'ed by XDP core.608 * For error cases, a negative errno code is returned and no-frames609 * are transmitted (caller must handle freeing frames).610 */611int612ice_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,613 u32 flags)614{615 struct ice_netdev_priv *np = netdev_priv(dev);616 unsigned int queue_index = smp_processor_id();617 struct ice_vsi *vsi = np->vsi;618 struct ice_tx_ring *xdp_ring;619 struct ice_tx_buf *tx_buf;620 int nxmit = 0, i;621 622 if (test_bit(ICE_VSI_DOWN, vsi->state))623 return -ENETDOWN;624 625 if (!ice_is_xdp_ena_vsi(vsi))626 return -ENXIO;627 628 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))629 return -EINVAL;630 631 if (static_branch_unlikely(&ice_xdp_locking_key)) {632 queue_index %= vsi->num_xdp_txq;633 xdp_ring = vsi->xdp_rings[queue_index];634 spin_lock(&xdp_ring->tx_lock);635 } else {636 /* Generally, should not happen */637 if (unlikely(queue_index >= vsi->num_xdp_txq))638 return -ENXIO;639 xdp_ring = vsi->xdp_rings[queue_index];640 }641 642 tx_buf = &xdp_ring->tx_buf[xdp_ring->next_to_use];643 for (i = 0; i < n; i++) {644 const struct xdp_frame *xdpf = frames[i];645 int err;646 647 err = ice_xmit_xdp_ring(xdpf, xdp_ring);648 if (err != ICE_XDP_TX)649 break;650 nxmit++;651 }652 653 tx_buf->rs_idx = ice_set_rs_bit(xdp_ring);654 if (unlikely(flags & XDP_XMIT_FLUSH))655 ice_xdp_ring_update_tail(xdp_ring);656 657 if (static_branch_unlikely(&ice_xdp_locking_key))658 spin_unlock(&xdp_ring->tx_lock);659 660 return nxmit;661}662 663/**664 * ice_alloc_mapped_page - recycle or make a new page665 * @rx_ring: ring to use666 * @bi: rx_buf struct to modify667 *668 * Returns true if the page was successfully allocated or669 * reused.670 */671static bool672ice_alloc_mapped_page(struct ice_rx_ring *rx_ring, struct ice_rx_buf *bi)673{674 struct page *page = bi->page;675 dma_addr_t dma;676 677 /* since we are recycling buffers we should seldom need to alloc */678 if (likely(page))679 return true;680 681 /* alloc new page for storage */682 page = dev_alloc_pages(ice_rx_pg_order(rx_ring));683 if (unlikely(!page)) {684 rx_ring->ring_stats->rx_stats.alloc_page_failed++;685 return false;686 }687 688 /* map page for use */689 dma = dma_map_page_attrs(rx_ring->dev, page, 0, ice_rx_pg_size(rx_ring),690 DMA_FROM_DEVICE, ICE_RX_DMA_ATTR);691 692 /* if mapping failed free memory back to system since693 * there isn't much point in holding memory we can't use694 */695 if (dma_mapping_error(rx_ring->dev, dma)) {696 __free_pages(page, ice_rx_pg_order(rx_ring));697 rx_ring->ring_stats->rx_stats.alloc_page_failed++;698 return false;699 }700 701 bi->dma = dma;702 bi->page = page;703 bi->page_offset = rx_ring->rx_offset;704 page_ref_add(page, USHRT_MAX - 1);705 bi->pagecnt_bias = USHRT_MAX;706 707 return true;708}709 710/**711 * ice_alloc_rx_bufs - Replace used receive buffers712 * @rx_ring: ring to place buffers on713 * @cleaned_count: number of buffers to replace714 *715 * Returns false if all allocations were successful, true if any fail. Returning716 * true signals to the caller that we didn't replace cleaned_count buffers and717 * there is more work to do.718 *719 * First, try to clean "cleaned_count" Rx buffers. Then refill the cleaned Rx720 * buffers. Then bump tail at most one time. Grouping like this lets us avoid721 * multiple tail writes per call.722 */723bool ice_alloc_rx_bufs(struct ice_rx_ring *rx_ring, unsigned int cleaned_count)724{725 union ice_32b_rx_flex_desc *rx_desc;726 u16 ntu = rx_ring->next_to_use;727 struct ice_rx_buf *bi;728 729 /* do nothing if no valid netdev defined */730 if ((!rx_ring->netdev && rx_ring->vsi->type != ICE_VSI_CTRL) ||731 !cleaned_count)732 return false;733 734 /* get the Rx descriptor and buffer based on next_to_use */735 rx_desc = ICE_RX_DESC(rx_ring, ntu);736 bi = &rx_ring->rx_buf[ntu];737 738 do {739 /* if we fail here, we have work remaining */740 if (!ice_alloc_mapped_page(rx_ring, bi))741 break;742 743 /* sync the buffer for use by the device */744 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,745 bi->page_offset,746 rx_ring->rx_buf_len,747 DMA_FROM_DEVICE);748 749 /* Refresh the desc even if buffer_addrs didn't change750 * because each write-back erases this info.751 */752 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);753 754 rx_desc++;755 bi++;756 ntu++;757 if (unlikely(ntu == rx_ring->count)) {758 rx_desc = ICE_RX_DESC(rx_ring, 0);759 bi = rx_ring->rx_buf;760 ntu = 0;761 }762 763 /* clear the status bits for the next_to_use descriptor */764 rx_desc->wb.status_error0 = 0;765 766 cleaned_count--;767 } while (cleaned_count);768 769 if (rx_ring->next_to_use != ntu)770 ice_release_rx_desc(rx_ring, ntu);771 772 return !!cleaned_count;773}774 775/**776 * ice_rx_buf_adjust_pg_offset - Prepare Rx buffer for reuse777 * @rx_buf: Rx buffer to adjust778 * @size: Size of adjustment779 *780 * Update the offset within page so that Rx buf will be ready to be reused.781 * For systems with PAGE_SIZE < 8192 this function will flip the page offset782 * so the second half of page assigned to Rx buffer will be used, otherwise783 * the offset is moved by "size" bytes784 */785static void786ice_rx_buf_adjust_pg_offset(struct ice_rx_buf *rx_buf, unsigned int size)787{788#if (PAGE_SIZE < 8192)789 /* flip page offset to other buffer */790 rx_buf->page_offset ^= size;791#else792 /* move offset up to the next cache line */793 rx_buf->page_offset += size;794#endif795}796 797/**798 * ice_can_reuse_rx_page - Determine if page can be reused for another Rx799 * @rx_buf: buffer containing the page800 *801 * If page is reusable, we have a green light for calling ice_reuse_rx_page,802 * which will assign the current buffer to the buffer that next_to_alloc is803 * pointing to; otherwise, the DMA mapping needs to be destroyed and804 * page freed805 */806static bool807ice_can_reuse_rx_page(struct ice_rx_buf *rx_buf)808{809 unsigned int pagecnt_bias = rx_buf->pagecnt_bias;810 struct page *page = rx_buf->page;811 812 /* avoid re-using remote and pfmemalloc pages */813 if (!dev_page_is_reusable(page))814 return false;815 816 /* if we are only owner of page we can reuse it */817 if (unlikely(rx_buf->pgcnt - pagecnt_bias > 1))818 return false;819#if (PAGE_SIZE >= 8192)820#define ICE_LAST_OFFSET \821 (SKB_WITH_OVERHEAD(PAGE_SIZE) - ICE_RXBUF_3072)822 if (rx_buf->page_offset > ICE_LAST_OFFSET)823 return false;824#endif /* PAGE_SIZE >= 8192) */825 826 /* If we have drained the page fragment pool we need to update827 * the pagecnt_bias and page count so that we fully restock the828 * number of references the driver holds.829 */830 if (unlikely(pagecnt_bias == 1)) {831 page_ref_add(page, USHRT_MAX - 1);832 rx_buf->pagecnt_bias = USHRT_MAX;833 }834 835 return true;836}837 838/**839 * ice_add_xdp_frag - Add contents of Rx buffer to xdp buf as a frag840 * @rx_ring: Rx descriptor ring to transact packets on841 * @xdp: xdp buff to place the data into842 * @rx_buf: buffer containing page to add843 * @size: packet length from rx_desc844 *845 * This function will add the data contained in rx_buf->page to the xdp buf.846 * It will just attach the page as a frag.847 */848static int849ice_add_xdp_frag(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp,850 struct ice_rx_buf *rx_buf, const unsigned int size)851{852 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);853 854 if (!size)855 return 0;856 857 if (!xdp_buff_has_frags(xdp)) {858 sinfo->nr_frags = 0;859 sinfo->xdp_frags_size = 0;860 xdp_buff_set_frags_flag(xdp);861 }862 863 if (unlikely(sinfo->nr_frags == MAX_SKB_FRAGS)) {864 ice_set_rx_bufs_act(xdp, rx_ring, ICE_XDP_CONSUMED);865 return -ENOMEM;866 }867 868 __skb_fill_page_desc_noacc(sinfo, sinfo->nr_frags++, rx_buf->page,869 rx_buf->page_offset, size);870 sinfo->xdp_frags_size += size;871 /* remember frag count before XDP prog execution; bpf_xdp_adjust_tail()872 * can pop off frags but driver has to handle it on its own873 */874 rx_ring->nr_frags = sinfo->nr_frags;875 876 if (page_is_pfmemalloc(rx_buf->page))877 xdp_buff_set_frag_pfmemalloc(xdp);878 879 return 0;880}881 882/**883 * ice_reuse_rx_page - page flip buffer and store it back on the ring884 * @rx_ring: Rx descriptor ring to store buffers on885 * @old_buf: donor buffer to have page reused886 *887 * Synchronizes page for reuse by the adapter888 */889static void890ice_reuse_rx_page(struct ice_rx_ring *rx_ring, struct ice_rx_buf *old_buf)891{892 u16 nta = rx_ring->next_to_alloc;893 struct ice_rx_buf *new_buf;894 895 new_buf = &rx_ring->rx_buf[nta];896 897 /* update, and store next to alloc */898 nta++;899 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;900 901 /* Transfer page from old buffer to new buffer.902 * Move each member individually to avoid possible store903 * forwarding stalls and unnecessary copy of skb.904 */905 new_buf->dma = old_buf->dma;906 new_buf->page = old_buf->page;907 new_buf->page_offset = old_buf->page_offset;908 new_buf->pagecnt_bias = old_buf->pagecnt_bias;909}910 911/**912 * ice_get_rx_buf - Fetch Rx buffer and synchronize data for use913 * @rx_ring: Rx descriptor ring to transact packets on914 * @size: size of buffer to add to skb915 * @ntc: index of next to clean element916 *917 * This function will pull an Rx buffer from the ring and synchronize it918 * for use by the CPU.919 */920static struct ice_rx_buf *921ice_get_rx_buf(struct ice_rx_ring *rx_ring, const unsigned int size,922 const unsigned int ntc)923{924 struct ice_rx_buf *rx_buf;925 926 rx_buf = &rx_ring->rx_buf[ntc];927 rx_buf->pgcnt = page_count(rx_buf->page);928 prefetchw(rx_buf->page);929 930 if (!size)931 return rx_buf;932 /* we are reusing so sync this buffer for CPU use */933 dma_sync_single_range_for_cpu(rx_ring->dev, rx_buf->dma,934 rx_buf->page_offset, size,935 DMA_FROM_DEVICE);936 937 /* We have pulled a buffer for use, so decrement pagecnt_bias */938 rx_buf->pagecnt_bias--;939 940 return rx_buf;941}942 943/**944 * ice_build_skb - Build skb around an existing buffer945 * @rx_ring: Rx descriptor ring to transact packets on946 * @xdp: xdp_buff pointing to the data947 *948 * This function builds an skb around an existing XDP buffer, taking care949 * to set up the skb correctly and avoid any memcpy overhead. Driver has950 * already combined frags (if any) to skb_shared_info.951 */952static struct sk_buff *953ice_build_skb(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp)954{955 u8 metasize = xdp->data - xdp->data_meta;956 struct skb_shared_info *sinfo = NULL;957 unsigned int nr_frags;958 struct sk_buff *skb;959 960 if (unlikely(xdp_buff_has_frags(xdp))) {961 sinfo = xdp_get_shared_info_from_buff(xdp);962 nr_frags = sinfo->nr_frags;963 }964 965 /* Prefetch first cache line of first page. If xdp->data_meta966 * is unused, this points exactly as xdp->data, otherwise we967 * likely have a consumer accessing first few bytes of meta968 * data, and then actual data.969 */970 net_prefetch(xdp->data_meta);971 /* build an skb around the page buffer */972 skb = napi_build_skb(xdp->data_hard_start, xdp->frame_sz);973 if (unlikely(!skb))974 return NULL;975 976 /* must to record Rx queue, otherwise OS features such as977 * symmetric queue won't work978 */979 skb_record_rx_queue(skb, rx_ring->q_index);980 981 /* update pointers within the skb to store the data */982 skb_reserve(skb, xdp->data - xdp->data_hard_start);983 __skb_put(skb, xdp->data_end - xdp->data);984 if (metasize)985 skb_metadata_set(skb, metasize);986 987 if (unlikely(xdp_buff_has_frags(xdp)))988 xdp_update_skb_shared_info(skb, nr_frags,989 sinfo->xdp_frags_size,990 nr_frags * xdp->frame_sz,991 xdp_buff_is_frag_pfmemalloc(xdp));992 993 return skb;994}995 996/**997 * ice_construct_skb - Allocate skb and populate it998 * @rx_ring: Rx descriptor ring to transact packets on999 * @xdp: xdp_buff pointing to the data1000 *1001 * This function allocates an skb. It then populates it with the page1002 * data from the current receive descriptor, taking care to set up the1003 * skb correctly.1004 */1005static struct sk_buff *1006ice_construct_skb(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp)1007{1008 unsigned int size = xdp->data_end - xdp->data;1009 struct skb_shared_info *sinfo = NULL;1010 struct ice_rx_buf *rx_buf;1011 unsigned int nr_frags = 0;1012 unsigned int headlen;1013 struct sk_buff *skb;1014 1015 /* prefetch first cache line of first page */1016 net_prefetch(xdp->data);1017 1018 if (unlikely(xdp_buff_has_frags(xdp))) {1019 sinfo = xdp_get_shared_info_from_buff(xdp);1020 nr_frags = sinfo->nr_frags;1021 }1022 1023 /* allocate a skb to store the frags */1024 skb = napi_alloc_skb(&rx_ring->q_vector->napi, ICE_RX_HDR_SIZE);1025 if (unlikely(!skb))1026 return NULL;1027 1028 rx_buf = &rx_ring->rx_buf[rx_ring->first_desc];1029 skb_record_rx_queue(skb, rx_ring->q_index);1030 /* Determine available headroom for copy */1031 headlen = size;1032 if (headlen > ICE_RX_HDR_SIZE)1033 headlen = eth_get_headlen(skb->dev, xdp->data, ICE_RX_HDR_SIZE);1034 1035 /* align pull length to size of long to optimize memcpy performance */1036 memcpy(__skb_put(skb, headlen), xdp->data, ALIGN(headlen,1037 sizeof(long)));1038 1039 /* if we exhaust the linear part then add what is left as a frag */1040 size -= headlen;1041 if (size) {1042 /* besides adding here a partial frag, we are going to add1043 * frags from xdp_buff, make sure there is enough space for1044 * them1045 */1046 if (unlikely(nr_frags >= MAX_SKB_FRAGS - 1)) {1047 dev_kfree_skb(skb);1048 return NULL;1049 }1050 skb_add_rx_frag(skb, 0, rx_buf->page,1051 rx_buf->page_offset + headlen, size,1052 xdp->frame_sz);1053 } else {1054 /* buffer is unused, change the act that should be taken later1055 * on; data was copied onto skb's linear part so there's no1056 * need for adjusting page offset and we can reuse this buffer1057 * as-is1058 */1059 rx_buf->act = ICE_SKB_CONSUMED;1060 }1061 1062 if (unlikely(xdp_buff_has_frags(xdp))) {1063 struct skb_shared_info *skinfo = skb_shinfo(skb);1064 1065 memcpy(&skinfo->frags[skinfo->nr_frags], &sinfo->frags[0],1066 sizeof(skb_frag_t) * nr_frags);1067 1068 xdp_update_skb_shared_info(skb, skinfo->nr_frags + nr_frags,1069 sinfo->xdp_frags_size,1070 nr_frags * xdp->frame_sz,1071 xdp_buff_is_frag_pfmemalloc(xdp));1072 }1073 1074 return skb;1075}1076 1077/**1078 * ice_put_rx_buf - Clean up used buffer and either recycle or free1079 * @rx_ring: Rx descriptor ring to transact packets on1080 * @rx_buf: Rx buffer to pull data from1081 *1082 * This function will clean up the contents of the rx_buf. It will either1083 * recycle the buffer or unmap it and free the associated resources.1084 */1085static void1086ice_put_rx_buf(struct ice_rx_ring *rx_ring, struct ice_rx_buf *rx_buf)1087{1088 if (!rx_buf)1089 return;1090 1091 if (ice_can_reuse_rx_page(rx_buf)) {1092 /* hand second half of page back to the ring */1093 ice_reuse_rx_page(rx_ring, rx_buf);1094 } else {1095 /* we are not reusing the buffer so unmap it */1096 dma_unmap_page_attrs(rx_ring->dev, rx_buf->dma,1097 ice_rx_pg_size(rx_ring), DMA_FROM_DEVICE,1098 ICE_RX_DMA_ATTR);1099 __page_frag_cache_drain(rx_buf->page, rx_buf->pagecnt_bias);1100 }1101 1102 /* clear contents of buffer_info */1103 rx_buf->page = NULL;1104}1105 1106/**1107 * ice_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf1108 * @rx_ring: Rx descriptor ring to transact packets on1109 * @budget: Total limit on number of packets to process1110 *1111 * This function provides a "bounce buffer" approach to Rx interrupt1112 * processing. The advantage to this is that on systems that have1113 * expensive overhead for IOMMU access this provides a means of avoiding1114 * it by maintaining the mapping of the page to the system.1115 *1116 * Returns amount of work completed1117 */1118int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget)1119{1120 unsigned int total_rx_bytes = 0, total_rx_pkts = 0;1121 unsigned int offset = rx_ring->rx_offset;1122 struct xdp_buff *xdp = &rx_ring->xdp;1123 u32 cached_ntc = rx_ring->first_desc;1124 struct ice_tx_ring *xdp_ring = NULL;1125 struct bpf_prog *xdp_prog = NULL;1126 u32 ntc = rx_ring->next_to_clean;1127 u32 cnt = rx_ring->count;1128 u32 xdp_xmit = 0;1129 u32 cached_ntu;1130 bool failure;1131 u32 first;1132 1133 xdp_prog = READ_ONCE(rx_ring->xdp_prog);1134 if (xdp_prog) {1135 xdp_ring = rx_ring->xdp_ring;1136 cached_ntu = xdp_ring->next_to_use;1137 }1138 1139 /* start the loop to process Rx packets bounded by 'budget' */1140 while (likely(total_rx_pkts < (unsigned int)budget)) {1141 union ice_32b_rx_flex_desc *rx_desc;1142 struct ice_rx_buf *rx_buf;1143 struct sk_buff *skb;1144 unsigned int size;1145 u16 stat_err_bits;1146 u16 vlan_tci;1147 1148 /* get the Rx desc from Rx ring based on 'next_to_clean' */1149 rx_desc = ICE_RX_DESC(rx_ring, ntc);1150 1151 /* status_error_len will always be zero for unused descriptors1152 * because it's cleared in cleanup, and overlaps with hdr_addr1153 * which is always zero because packet split isn't used, if the1154 * hardware wrote DD then it will be non-zero1155 */1156 stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_DD_S);1157 if (!ice_test_staterr(rx_desc->wb.status_error0, stat_err_bits))1158 break;1159 1160 /* This memory barrier is needed to keep us from reading1161 * any other fields out of the rx_desc until we know the1162 * DD bit is set.1163 */1164 dma_rmb();1165 1166 ice_trace(clean_rx_irq, rx_ring, rx_desc);1167 if (rx_desc->wb.rxdid == FDIR_DESC_RXDID || !rx_ring->netdev) {1168 struct ice_vsi *ctrl_vsi = rx_ring->vsi;1169 1170 if (rx_desc->wb.rxdid == FDIR_DESC_RXDID &&1171 ctrl_vsi->vf)1172 ice_vc_fdir_irq_handler(ctrl_vsi, rx_desc);1173 if (++ntc == cnt)1174 ntc = 0;1175 rx_ring->first_desc = ntc;1176 continue;1177 }1178 1179 size = le16_to_cpu(rx_desc->wb.pkt_len) &1180 ICE_RX_FLX_DESC_PKT_LEN_M;1181 1182 /* retrieve a buffer from the ring */1183 rx_buf = ice_get_rx_buf(rx_ring, size, ntc);1184 1185 if (!xdp->data) {1186 void *hard_start;1187 1188 hard_start = page_address(rx_buf->page) + rx_buf->page_offset -1189 offset;1190 xdp_prepare_buff(xdp, hard_start, offset, size, !!offset);1191 xdp_buff_clear_frags_flag(xdp);1192 } else if (ice_add_xdp_frag(rx_ring, xdp, rx_buf, size)) {1193 break;1194 }1195 if (++ntc == cnt)1196 ntc = 0;1197 1198 /* skip if it is NOP desc */1199 if (ice_is_non_eop(rx_ring, rx_desc))1200 continue;1201 1202 ice_run_xdp(rx_ring, xdp, xdp_prog, xdp_ring, rx_buf, rx_desc);1203 if (rx_buf->act == ICE_XDP_PASS)1204 goto construct_skb;1205 total_rx_bytes += xdp_get_buff_len(xdp);1206 total_rx_pkts++;1207 1208 xdp->data = NULL;1209 rx_ring->first_desc = ntc;1210 rx_ring->nr_frags = 0;1211 continue;1212construct_skb:1213 if (likely(ice_ring_uses_build_skb(rx_ring)))1214 skb = ice_build_skb(rx_ring, xdp);1215 else1216 skb = ice_construct_skb(rx_ring, xdp);1217 /* exit if we failed to retrieve a buffer */1218 if (!skb) {1219 rx_ring->ring_stats->rx_stats.alloc_page_failed++;1220 rx_buf->act = ICE_XDP_CONSUMED;1221 if (unlikely(xdp_buff_has_frags(xdp)))1222 ice_set_rx_bufs_act(xdp, rx_ring,1223 ICE_XDP_CONSUMED);1224 xdp->data = NULL;1225 rx_ring->first_desc = ntc;1226 rx_ring->nr_frags = 0;1227 break;1228 }1229 xdp->data = NULL;1230 rx_ring->first_desc = ntc;1231 rx_ring->nr_frags = 0;1232 1233 stat_err_bits = BIT(ICE_RX_FLEX_DESC_STATUS0_RXE_S);1234 if (unlikely(ice_test_staterr(rx_desc->wb.status_error0,1235 stat_err_bits))) {1236 dev_kfree_skb_any(skb);1237 continue;1238 }1239 1240 vlan_tci = ice_get_vlan_tci(rx_desc);1241 1242 /* pad the skb if needed, to make a valid ethernet frame */1243 if (eth_skb_pad(skb))1244 continue;1245 1246 /* probably a little skewed due to removing CRC */1247 total_rx_bytes += skb->len;1248 1249 /* populate checksum, VLAN, and protocol */1250 ice_process_skb_fields(rx_ring, rx_desc, skb);1251 1252 ice_trace(clean_rx_irq_indicate, rx_ring, rx_desc, skb);1253 /* send completed skb up the stack */1254 ice_receive_skb(rx_ring, skb, vlan_tci);1255 1256 /* update budget accounting */1257 total_rx_pkts++;1258 }1259 1260 first = rx_ring->first_desc;1261 while (cached_ntc != first) {1262 struct ice_rx_buf *buf = &rx_ring->rx_buf[cached_ntc];1263 1264 if (buf->act & (ICE_XDP_TX | ICE_XDP_REDIR)) {1265 ice_rx_buf_adjust_pg_offset(buf, xdp->frame_sz);1266 xdp_xmit |= buf->act;1267 } else if (buf->act & ICE_XDP_CONSUMED) {1268 buf->pagecnt_bias++;1269 } else if (buf->act == ICE_XDP_PASS) {1270 ice_rx_buf_adjust_pg_offset(buf, xdp->frame_sz);1271 }1272 1273 ice_put_rx_buf(rx_ring, buf);1274 if (++cached_ntc >= cnt)1275 cached_ntc = 0;1276 }1277 rx_ring->next_to_clean = ntc;1278 /* return up to cleaned_count buffers to hardware */1279 failure = ice_alloc_rx_bufs(rx_ring, ICE_RX_DESC_UNUSED(rx_ring));1280 1281 if (xdp_xmit)1282 ice_finalize_xdp_rx(xdp_ring, xdp_xmit, cached_ntu);1283 1284 if (rx_ring->ring_stats)1285 ice_update_rx_ring_stats(rx_ring, total_rx_pkts,1286 total_rx_bytes);1287 1288 /* guarantee a trip back through this routine if there was a failure */1289 return failure ? budget : (int)total_rx_pkts;1290}1291 1292static void __ice_update_sample(struct ice_q_vector *q_vector,1293 struct ice_ring_container *rc,1294 struct dim_sample *sample,1295 bool is_tx)1296{1297 u64 packets = 0, bytes = 0;1298 1299 if (is_tx) {1300 struct ice_tx_ring *tx_ring;1301 1302 ice_for_each_tx_ring(tx_ring, *rc) {1303 struct ice_ring_stats *ring_stats;1304 1305 ring_stats = tx_ring->ring_stats;1306 if (!ring_stats)1307 continue;1308 packets += ring_stats->stats.pkts;1309 bytes += ring_stats->stats.bytes;1310 }1311 } else {1312 struct ice_rx_ring *rx_ring;1313 1314 ice_for_each_rx_ring(rx_ring, *rc) {1315 struct ice_ring_stats *ring_stats;1316 1317 ring_stats = rx_ring->ring_stats;1318 if (!ring_stats)1319 continue;1320 packets += ring_stats->stats.pkts;1321 bytes += ring_stats->stats.bytes;1322 }1323 }1324 1325 dim_update_sample(q_vector->total_events, packets, bytes, sample);1326 sample->comp_ctr = 0;1327 1328 /* if dim settings get stale, like when not updated for 11329 * second or longer, force it to start again. This addresses the1330 * frequent case of an idle queue being switched to by the1331 * scheduler. The 1,000 here means 1,000 milliseconds.1332 */1333 if (ktime_ms_delta(sample->time, rc->dim.start_sample.time) >= 1000)1334 rc->dim.state = DIM_START_MEASURE;1335}1336 1337/**1338 * ice_net_dim - Update net DIM algorithm1339 * @q_vector: the vector associated with the interrupt1340 *1341 * Create a DIM sample and notify net_dim() so that it can possibly decide1342 * a new ITR value based on incoming packets, bytes, and interrupts.1343 *1344 * This function is a no-op if the ring is not configured to dynamic ITR.1345 */1346static void ice_net_dim(struct ice_q_vector *q_vector)1347{1348 struct ice_ring_container *tx = &q_vector->tx;1349 struct ice_ring_container *rx = &q_vector->rx;1350 1351 if (ITR_IS_DYNAMIC(tx)) {1352 struct dim_sample dim_sample;1353 1354 __ice_update_sample(q_vector, tx, &dim_sample, true);1355 net_dim(&tx->dim, dim_sample);1356 }1357 1358 if (ITR_IS_DYNAMIC(rx)) {1359 struct dim_sample dim_sample;1360 1361 __ice_update_sample(q_vector, rx, &dim_sample, false);1362 net_dim(&rx->dim, dim_sample);1363 }1364}1365 1366/**1367 * ice_buildreg_itr - build value for writing to the GLINT_DYN_CTL register1368 * @itr_idx: interrupt throttling index1369 * @itr: interrupt throttling value in usecs1370 */1371static u32 ice_buildreg_itr(u16 itr_idx, u16 itr)1372{1373 /* The ITR value is reported in microseconds, and the register value is1374 * recorded in 2 microsecond units. For this reason we only need to1375 * shift by the GLINT_DYN_CTL_INTERVAL_S - ICE_ITR_GRAN_S to apply this1376 * granularity as a shift instead of division. The mask makes sure the1377 * ITR value is never odd so we don't accidentally write into the field1378 * prior to the ITR field.1379 */1380 itr &= ICE_ITR_MASK;1381 1382 return GLINT_DYN_CTL_INTENA_M | GLINT_DYN_CTL_CLEARPBA_M |1383 (itr_idx << GLINT_DYN_CTL_ITR_INDX_S) |1384 (itr << (GLINT_DYN_CTL_INTERVAL_S - ICE_ITR_GRAN_S));1385}1386 1387/**1388 * ice_enable_interrupt - re-enable MSI-X interrupt1389 * @q_vector: the vector associated with the interrupt to enable1390 *1391 * If the VSI is down, the interrupt will not be re-enabled. Also,1392 * when enabling the interrupt always reset the wb_on_itr to false1393 * and trigger a software interrupt to clean out internal state.1394 */1395static void ice_enable_interrupt(struct ice_q_vector *q_vector)1396{1397 struct ice_vsi *vsi = q_vector->vsi;1398 bool wb_en = q_vector->wb_on_itr;1399 u32 itr_val;1400 1401 if (test_bit(ICE_DOWN, vsi->state))1402 return;1403 1404 /* trigger an ITR delayed software interrupt when exiting busy poll, to1405 * make sure to catch any pending cleanups that might have been missed1406 * due to interrupt state transition. If busy poll or poll isn't1407 * enabled, then don't update ITR, and just enable the interrupt.1408 */1409 if (!wb_en) {1410 itr_val = ice_buildreg_itr(ICE_ITR_NONE, 0);1411 } else {1412 q_vector->wb_on_itr = false;1413 1414 /* do two things here with a single write. Set up the third ITR1415 * index to be used for software interrupt moderation, and then1416 * trigger a software interrupt with a rate limit of 20K on1417 * software interrupts, this will help avoid high interrupt1418 * loads due to frequently polling and exiting polling.1419 */1420 itr_val = ice_buildreg_itr(ICE_IDX_ITR2, ICE_ITR_20K);1421 itr_val |= GLINT_DYN_CTL_SWINT_TRIG_M |1422 ICE_IDX_ITR2 << GLINT_DYN_CTL_SW_ITR_INDX_S |1423 GLINT_DYN_CTL_SW_ITR_INDX_ENA_M;1424 }1425 wr32(&vsi->back->hw, GLINT_DYN_CTL(q_vector->reg_idx), itr_val);1426}1427 1428/**1429 * ice_set_wb_on_itr - set WB_ON_ITR for this q_vector1430 * @q_vector: q_vector to set WB_ON_ITR on1431 *1432 * We need to tell hardware to write-back completed descriptors even when1433 * interrupts are disabled. Descriptors will be written back on cache line1434 * boundaries without WB_ON_ITR enabled, but if we don't enable WB_ON_ITR1435 * descriptors may not be written back if they don't fill a cache line until1436 * the next interrupt.1437 *1438 * This sets the write-back frequency to whatever was set previously for the1439 * ITR indices. Also, set the INTENA_MSK bit to make sure hardware knows we1440 * aren't meddling with the INTENA_M bit.1441 */1442static void ice_set_wb_on_itr(struct ice_q_vector *q_vector)1443{1444 struct ice_vsi *vsi = q_vector->vsi;1445 1446 /* already in wb_on_itr mode no need to change it */1447 if (q_vector->wb_on_itr)1448 return;1449 1450 /* use previously set ITR values for all of the ITR indices by1451 * specifying ICE_ITR_NONE, which will vary in adaptive (AIM) mode and1452 * be static in non-adaptive mode (user configured)1453 */1454 wr32(&vsi->back->hw, GLINT_DYN_CTL(q_vector->reg_idx),1455 FIELD_PREP(GLINT_DYN_CTL_ITR_INDX_M, ICE_ITR_NONE) |1456 FIELD_PREP(GLINT_DYN_CTL_INTENA_MSK_M, 1) |1457 FIELD_PREP(GLINT_DYN_CTL_WB_ON_ITR_M, 1));1458 1459 q_vector->wb_on_itr = true;1460}1461 1462/**1463 * ice_napi_poll - NAPI polling Rx/Tx cleanup routine1464 * @napi: napi struct with our devices info in it1465 * @budget: amount of work driver is allowed to do this pass, in packets1466 *1467 * This function will clean all queues associated with a q_vector.1468 *1469 * Returns the amount of work done1470 */1471int ice_napi_poll(struct napi_struct *napi, int budget)1472{1473 struct ice_q_vector *q_vector =1474 container_of(napi, struct ice_q_vector, napi);1475 struct ice_tx_ring *tx_ring;1476 struct ice_rx_ring *rx_ring;1477 bool clean_complete = true;1478 int budget_per_ring;1479 int work_done = 0;1480 1481 /* Since the actual Tx work is minimal, we can give the Tx a larger1482 * budget and be more aggressive about cleaning up the Tx descriptors.1483 */1484 ice_for_each_tx_ring(tx_ring, q_vector->tx) {1485 struct xsk_buff_pool *xsk_pool = READ_ONCE(tx_ring->xsk_pool);1486 bool wd;1487 1488 if (xsk_pool)1489 wd = ice_xmit_zc(tx_ring, xsk_pool);1490 else if (ice_ring_is_xdp(tx_ring))1491 wd = true;1492 else1493 wd = ice_clean_tx_irq(tx_ring, budget);1494 1495 if (!wd)1496 clean_complete = false;1497 }1498 1499 /* Handle case where we are called by netpoll with a budget of 0 */1500 if (unlikely(budget <= 0))1501 return budget;1502 1503 /* normally we have 1 Rx ring per q_vector */1504 if (unlikely(q_vector->num_ring_rx > 1))1505 /* We attempt to distribute budget to each Rx queue fairly, but1506 * don't allow the budget to go below 1 because that would exit1507 * polling early.1508 */1509 budget_per_ring = max_t(int, budget / q_vector->num_ring_rx, 1);1510 else1511 /* Max of 1 Rx ring in this q_vector so give it the budget */1512 budget_per_ring = budget;1513 1514 ice_for_each_rx_ring(rx_ring, q_vector->rx) {1515 struct xsk_buff_pool *xsk_pool = READ_ONCE(rx_ring->xsk_pool);1516 int cleaned;1517 1518 /* A dedicated path for zero-copy allows making a single1519 * comparison in the irq context instead of many inside the1520 * ice_clean_rx_irq function and makes the codebase cleaner.1521 */1522 cleaned = rx_ring->xsk_pool ?1523 ice_clean_rx_irq_zc(rx_ring, xsk_pool, budget_per_ring) :1524 ice_clean_rx_irq(rx_ring, budget_per_ring);1525 work_done += cleaned;1526 /* if we clean as many as budgeted, we must not be done */1527 if (cleaned >= budget_per_ring)1528 clean_complete = false;1529 }1530 1531 /* If work not completed, return budget and polling will return */1532 if (!clean_complete) {1533 /* Set the writeback on ITR so partial completions of1534 * cache-lines will still continue even if we're polling.1535 */1536 ice_set_wb_on_itr(q_vector);1537 return budget;1538 }1539 1540 /* Exit the polling mode, but don't re-enable interrupts if stack might1541 * poll us due to busy-polling1542 */1543 if (napi_complete_done(napi, work_done)) {1544 ice_net_dim(q_vector);1545 ice_enable_interrupt(q_vector);1546 } else {1547 ice_set_wb_on_itr(q_vector);1548 }1549 1550 return min_t(int, work_done, budget - 1);1551}1552 1553/**1554 * __ice_maybe_stop_tx - 2nd level check for Tx stop conditions1555 * @tx_ring: the ring to be checked1556 * @size: the size buffer we want to assure is available1557 *1558 * Returns -EBUSY if a stop is needed, else 01559 */1560static int __ice_maybe_stop_tx(struct ice_tx_ring *tx_ring, unsigned int size)1561{1562 netif_tx_stop_queue(txring_txq(tx_ring));1563 /* Memory barrier before checking head and tail */1564 smp_mb();1565 1566 /* Check again in a case another CPU has just made room available. */1567 if (likely(ICE_DESC_UNUSED(tx_ring) < size))1568 return -EBUSY;1569 1570 /* A reprieve! - use start_queue because it doesn't call schedule */1571 netif_tx_start_queue(txring_txq(tx_ring));1572 ++tx_ring->ring_stats->tx_stats.restart_q;1573 return 0;1574}1575 1576/**1577 * ice_maybe_stop_tx - 1st level check for Tx stop conditions1578 * @tx_ring: the ring to be checked1579 * @size: the size buffer we want to assure is available1580 *1581 * Returns 0 if stop is not needed1582 */1583static int ice_maybe_stop_tx(struct ice_tx_ring *tx_ring, unsigned int size)1584{1585 if (likely(ICE_DESC_UNUSED(tx_ring) >= size))1586 return 0;1587 1588 return __ice_maybe_stop_tx(tx_ring, size);1589}1590 1591/**1592 * ice_tx_map - Build the Tx descriptor1593 * @tx_ring: ring to send buffer on1594 * @first: first buffer info buffer to use1595 * @off: pointer to struct that holds offload parameters1596 *1597 * This function loops over the skb data pointed to by *first1598 * and gets a physical address for each memory location and programs1599 * it and the length into the transmit descriptor.1600 */1601static void1602ice_tx_map(struct ice_tx_ring *tx_ring, struct ice_tx_buf *first,1603 struct ice_tx_offload_params *off)1604{1605 u64 td_offset, td_tag, td_cmd;1606 u16 i = tx_ring->next_to_use;1607 unsigned int data_len, size;1608 struct ice_tx_desc *tx_desc;1609 struct ice_tx_buf *tx_buf;1610 struct sk_buff *skb;1611 skb_frag_t *frag;1612 dma_addr_t dma;1613 bool kick;1614 1615 td_tag = off->td_l2tag1;1616 td_cmd = off->td_cmd;1617 td_offset = off->td_offset;1618 skb = first->skb;1619 1620 data_len = skb->data_len;1621 size = skb_headlen(skb);1622 1623 tx_desc = ICE_TX_DESC(tx_ring, i);1624 1625 if (first->tx_flags & ICE_TX_FLAGS_HW_VLAN) {1626 td_cmd |= (u64)ICE_TX_DESC_CMD_IL2TAG1;1627 td_tag = first->vid;1628 }1629 1630 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);1631 1632 tx_buf = first;1633 1634 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {1635 unsigned int max_data = ICE_MAX_DATA_PER_TXD_ALIGNED;1636 1637 if (dma_mapping_error(tx_ring->dev, dma))1638 goto dma_error;1639 1640 /* record length, and DMA address */1641 dma_unmap_len_set(tx_buf, len, size);1642 dma_unmap_addr_set(tx_buf, dma, dma);1643 1644 /* align size to end of page */1645 max_data += -dma & (ICE_MAX_READ_REQ_SIZE - 1);1646 tx_desc->buf_addr = cpu_to_le64(dma);1647 1648 /* account for data chunks larger than the hardware1649 * can handle1650 */1651 while (unlikely(size > ICE_MAX_DATA_PER_TXD)) {1652 tx_desc->cmd_type_offset_bsz =1653 ice_build_ctob(td_cmd, td_offset, max_data,1654 td_tag);1655 1656 tx_desc++;1657 i++;1658 1659 if (i == tx_ring->count) {1660 tx_desc = ICE_TX_DESC(tx_ring, 0);1661 i = 0;1662 }1663 1664 dma += max_data;1665 size -= max_data;1666 1667 max_data = ICE_MAX_DATA_PER_TXD_ALIGNED;1668 tx_desc->buf_addr = cpu_to_le64(dma);1669 }1670 1671 if (likely(!data_len))1672 break;1673 1674 tx_desc->cmd_type_offset_bsz = ice_build_ctob(td_cmd, td_offset,1675 size, td_tag);1676 1677 tx_desc++;1678 i++;1679 1680 if (i == tx_ring->count) {1681 tx_desc = ICE_TX_DESC(tx_ring, 0);1682 i = 0;1683 }1684 1685 size = skb_frag_size(frag);1686 data_len -= size;1687 1688 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,1689 DMA_TO_DEVICE);1690 1691 tx_buf = &tx_ring->tx_buf[i];1692 tx_buf->type = ICE_TX_BUF_FRAG;1693 }1694 1695 /* record SW timestamp if HW timestamp is not available */1696 skb_tx_timestamp(first->skb);1697 1698 i++;1699 if (i == tx_ring->count)1700 i = 0;1701 1702 /* write last descriptor with RS and EOP bits */1703 td_cmd |= (u64)ICE_TXD_LAST_DESC_CMD;1704 tx_desc->cmd_type_offset_bsz =1705 ice_build_ctob(td_cmd, td_offset, size, td_tag);1706 1707 /* Force memory writes to complete before letting h/w know there1708 * are new descriptors to fetch.1709 *1710 * We also use this memory barrier to make certain all of the1711 * status bits have been updated before next_to_watch is written.1712 */1713 wmb();1714 1715 /* set next_to_watch value indicating a packet is present */1716 first->next_to_watch = tx_desc;1717 1718 tx_ring->next_to_use = i;1719 1720 ice_maybe_stop_tx(tx_ring, DESC_NEEDED);1721 1722 /* notify HW of packet */1723 kick = __netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount,1724 netdev_xmit_more());1725 if (kick)1726 /* notify HW of packet */1727 writel(i, tx_ring->tail);1728 1729 return;1730 1731dma_error:1732 /* clear DMA mappings for failed tx_buf map */1733 for (;;) {1734 tx_buf = &tx_ring->tx_buf[i];1735 ice_unmap_and_free_tx_buf(tx_ring, tx_buf);1736 if (tx_buf == first)1737 break;1738 if (i == 0)1739 i = tx_ring->count;1740 i--;1741 }1742 1743 tx_ring->next_to_use = i;1744}1745 1746/**1747 * ice_tx_csum - Enable Tx checksum offloads1748 * @first: pointer to the first descriptor1749 * @off: pointer to struct that holds offload parameters1750 *1751 * Returns 0 or error (negative) if checksum offload can't happen, 1 otherwise.1752 */1753static1754int ice_tx_csum(struct ice_tx_buf *first, struct ice_tx_offload_params *off)1755{1756 u32 l4_len = 0, l3_len = 0, l2_len = 0;1757 struct sk_buff *skb = first->skb;1758 union {1759 struct iphdr *v4;1760 struct ipv6hdr *v6;1761 unsigned char *hdr;1762 } ip;1763 union {1764 struct tcphdr *tcp;1765 unsigned char *hdr;1766 } l4;1767 __be16 frag_off, protocol;1768 unsigned char *exthdr;1769 u32 offset, cmd = 0;1770 u8 l4_proto = 0;1771 1772 if (skb->ip_summed != CHECKSUM_PARTIAL)1773 return 0;1774 1775 protocol = vlan_get_protocol(skb);1776 1777 if (eth_p_mpls(protocol)) {1778 ip.hdr = skb_inner_network_header(skb);1779 l4.hdr = skb_checksum_start(skb);1780 } else {1781 ip.hdr = skb_network_header(skb);1782 l4.hdr = skb_transport_header(skb);1783 }1784 1785 /* compute outer L2 header size */1786 l2_len = ip.hdr - skb->data;1787 offset = (l2_len / 2) << ICE_TX_DESC_LEN_MACLEN_S;1788 1789 /* set the tx_flags to indicate the IP protocol type. this is1790 * required so that checksum header computation below is accurate.1791 */1792 if (ip.v4->version == 4)1793 first->tx_flags |= ICE_TX_FLAGS_IPV4;1794 else if (ip.v6->version == 6)1795 first->tx_flags |= ICE_TX_FLAGS_IPV6;1796 1797 if (skb->encapsulation) {1798 bool gso_ena = false;1799 u32 tunnel = 0;1800 1801 /* define outer network header type */1802 if (first->tx_flags & ICE_TX_FLAGS_IPV4) {1803 tunnel |= (first->tx_flags & ICE_TX_FLAGS_TSO) ?1804 ICE_TX_CTX_EIPT_IPV4 :1805 ICE_TX_CTX_EIPT_IPV4_NO_CSUM;1806 l4_proto = ip.v4->protocol;1807 } else if (first->tx_flags & ICE_TX_FLAGS_IPV6) {1808 int ret;1809 1810 tunnel |= ICE_TX_CTX_EIPT_IPV6;1811 exthdr = ip.hdr + sizeof(*ip.v6);1812 l4_proto = ip.v6->nexthdr;1813 ret = ipv6_skip_exthdr(skb, exthdr - skb->data,1814 &l4_proto, &frag_off);1815 if (ret < 0)1816 return -1;1817 }1818 1819 /* define outer transport */1820 switch (l4_proto) {1821 case IPPROTO_UDP:1822 tunnel |= ICE_TXD_CTX_UDP_TUNNELING;1823 first->tx_flags |= ICE_TX_FLAGS_TUNNEL;1824 break;1825 case IPPROTO_GRE:1826 tunnel |= ICE_TXD_CTX_GRE_TUNNELING;1827 first->tx_flags |= ICE_TX_FLAGS_TUNNEL;1828 break;1829 case IPPROTO_IPIP:1830 case IPPROTO_IPV6:1831 first->tx_flags |= ICE_TX_FLAGS_TUNNEL;1832 l4.hdr = skb_inner_network_header(skb);1833 break;1834 default:1835 if (first->tx_flags & ICE_TX_FLAGS_TSO)1836 return -1;1837 1838 skb_checksum_help(skb);1839 return 0;1840 }1841 1842 /* compute outer L3 header size */1843 tunnel |= ((l4.hdr - ip.hdr) / 4) <<1844 ICE_TXD_CTX_QW0_EIPLEN_S;1845 1846 /* switch IP header pointer from outer to inner header */1847 ip.hdr = skb_inner_network_header(skb);1848 1849 /* compute tunnel header size */1850 tunnel |= ((ip.hdr - l4.hdr) / 2) <<1851 ICE_TXD_CTX_QW0_NATLEN_S;1852 1853 gso_ena = skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL;1854 /* indicate if we need to offload outer UDP header */1855 if ((first->tx_flags & ICE_TX_FLAGS_TSO) && !gso_ena &&1856 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM))1857 tunnel |= ICE_TXD_CTX_QW0_L4T_CS_M;1858 1859 /* record tunnel offload values */1860 off->cd_tunnel_params |= tunnel;1861 1862 /* set DTYP=1 to indicate that it's an Tx context descriptor1863 * in IPsec tunnel mode with Tx offloads in Quad word 11864 */1865 off->cd_qw1 |= (u64)ICE_TX_DESC_DTYPE_CTX;1866 1867 /* switch L4 header pointer from outer to inner */1868 l4.hdr = skb_inner_transport_header(skb);1869 l4_proto = 0;1870 1871 /* reset type as we transition from outer to inner headers */1872 first->tx_flags &= ~(ICE_TX_FLAGS_IPV4 | ICE_TX_FLAGS_IPV6);1873 if (ip.v4->version == 4)1874 first->tx_flags |= ICE_TX_FLAGS_IPV4;1875 if (ip.v6->version == 6)1876 first->tx_flags |= ICE_TX_FLAGS_IPV6;1877 }1878 1879 /* Enable IP checksum offloads */1880 if (first->tx_flags & ICE_TX_FLAGS_IPV4) {1881 l4_proto = ip.v4->protocol;1882 /* the stack computes the IP header already, the only time we1883 * need the hardware to recompute it is in the case of TSO.1884 */1885 if (first->tx_flags & ICE_TX_FLAGS_TSO)1886 cmd |= ICE_TX_DESC_CMD_IIPT_IPV4_CSUM;1887 else1888 cmd |= ICE_TX_DESC_CMD_IIPT_IPV4;1889 1890 } else if (first->tx_flags & ICE_TX_FLAGS_IPV6) {1891 cmd |= ICE_TX_DESC_CMD_IIPT_IPV6;1892 exthdr = ip.hdr + sizeof(*ip.v6);1893 l4_proto = ip.v6->nexthdr;1894 if (l4.hdr != exthdr)1895 ipv6_skip_exthdr(skb, exthdr - skb->data, &l4_proto,1896 &frag_off);1897 } else {1898 return -1;1899 }1900 1901 /* compute inner L3 header size */1902 l3_len = l4.hdr - ip.hdr;1903 offset |= (l3_len / 4) << ICE_TX_DESC_LEN_IPLEN_S;1904 1905 /* Enable L4 checksum offloads */1906 switch (l4_proto) {1907 case IPPROTO_TCP:1908 /* enable checksum offloads */1909 cmd |= ICE_TX_DESC_CMD_L4T_EOFT_TCP;1910 l4_len = l4.tcp->doff;1911 offset |= l4_len << ICE_TX_DESC_LEN_L4_LEN_S;1912 break;1913 case IPPROTO_UDP:1914 /* enable UDP checksum offload */1915 cmd |= ICE_TX_DESC_CMD_L4T_EOFT_UDP;1916 l4_len = (sizeof(struct udphdr) >> 2);1917 offset |= l4_len << ICE_TX_DESC_LEN_L4_LEN_S;1918 break;1919 case IPPROTO_SCTP:1920 /* enable SCTP checksum offload */1921 cmd |= ICE_TX_DESC_CMD_L4T_EOFT_SCTP;1922 l4_len = sizeof(struct sctphdr) >> 2;1923 offset |= l4_len << ICE_TX_DESC_LEN_L4_LEN_S;1924 break;1925 1926 default:1927 if (first->tx_flags & ICE_TX_FLAGS_TSO)1928 return -1;1929 skb_checksum_help(skb);1930 return 0;1931 }1932 1933 off->td_cmd |= cmd;1934 off->td_offset |= offset;1935 return 1;1936}1937 1938/**1939 * ice_tx_prepare_vlan_flags - prepare generic Tx VLAN tagging flags for HW1940 * @tx_ring: ring to send buffer on1941 * @first: pointer to struct ice_tx_buf1942 *1943 * Checks the skb and set up correspondingly several generic transmit flags1944 * related to VLAN tagging for the HW, such as VLAN, DCB, etc.1945 */1946static void1947ice_tx_prepare_vlan_flags(struct ice_tx_ring *tx_ring, struct ice_tx_buf *first)1948{1949 struct sk_buff *skb = first->skb;1950 1951 /* nothing left to do, software offloaded VLAN */1952 if (!skb_vlan_tag_present(skb) && eth_type_vlan(skb->protocol))1953 return;1954 1955 /* the VLAN ethertype/tpid is determined by VSI configuration and netdev1956 * feature flags, which the driver only allows either 802.1Q or 802.1ad1957 * VLAN offloads exclusively so we only care about the VLAN ID here1958 */1959 if (skb_vlan_tag_present(skb)) {1960 first->vid = skb_vlan_tag_get(skb);1961 if (tx_ring->flags & ICE_TX_FLAGS_RING_VLAN_L2TAG2)1962 first->tx_flags |= ICE_TX_FLAGS_HW_OUTER_SINGLE_VLAN;1963 else1964 first->tx_flags |= ICE_TX_FLAGS_HW_VLAN;1965 }1966 1967 ice_tx_prepare_vlan_flags_dcb(tx_ring, first);1968}1969 1970/**1971 * ice_tso - computes mss and TSO length to prepare for TSO1972 * @first: pointer to struct ice_tx_buf1973 * @off: pointer to struct that holds offload parameters1974 *1975 * Returns 0 or error (negative) if TSO can't happen, 1 otherwise.1976 */1977static1978int ice_tso(struct ice_tx_buf *first, struct ice_tx_offload_params *off)1979{1980 struct sk_buff *skb = first->skb;1981 union {1982 struct iphdr *v4;1983 struct ipv6hdr *v6;1984 unsigned char *hdr;1985 } ip;1986 union {1987 struct tcphdr *tcp;1988 struct udphdr *udp;1989 unsigned char *hdr;1990 } l4;1991 u64 cd_mss, cd_tso_len;1992 __be16 protocol;1993 u32 paylen;1994 u8 l4_start;1995 int err;1996 1997 if (skb->ip_summed != CHECKSUM_PARTIAL)1998 return 0;1999 2000 if (!skb_is_gso(skb))2001 return 0;2002 2003 err = skb_cow_head(skb, 0);2004 if (err < 0)2005 return err;2006 2007 protocol = vlan_get_protocol(skb);2008 2009 if (eth_p_mpls(protocol))2010 ip.hdr = skb_inner_network_header(skb);2011 else2012 ip.hdr = skb_network_header(skb);2013 l4.hdr = skb_checksum_start(skb);2014 2015 /* initialize outer IP header fields */2016 if (ip.v4->version == 4) {2017 ip.v4->tot_len = 0;2018 ip.v4->check = 0;2019 } else {2020 ip.v6->payload_len = 0;2021 }2022 2023 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |2024 SKB_GSO_GRE_CSUM |2025 SKB_GSO_IPXIP4 |2026 SKB_GSO_IPXIP6 |2027 SKB_GSO_UDP_TUNNEL |2028 SKB_GSO_UDP_TUNNEL_CSUM)) {2029 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) &&2030 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) {2031 l4.udp->len = 0;2032 2033 /* determine offset of outer transport header */2034 l4_start = (u8)(l4.hdr - skb->data);2035 2036 /* remove payload length from outer checksum */2037 paylen = skb->len - l4_start;2038 csum_replace_by_diff(&l4.udp->check,2039 (__force __wsum)htonl(paylen));2040 }2041 2042 /* reset pointers to inner headers */2043 ip.hdr = skb_inner_network_header(skb);2044 l4.hdr = skb_inner_transport_header(skb);2045 2046 /* initialize inner IP header fields */2047 if (ip.v4->version == 4) {2048 ip.v4->tot_len = 0;2049 ip.v4->check = 0;2050 } else {2051 ip.v6->payload_len = 0;2052 }2053 }2054 2055 /* determine offset of transport header */2056 l4_start = (u8)(l4.hdr - skb->data);2057 2058 /* remove payload length from checksum */2059 paylen = skb->len - l4_start;2060 2061 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {2062 csum_replace_by_diff(&l4.udp->check,2063 (__force __wsum)htonl(paylen));2064 /* compute length of UDP segmentation header */2065 off->header_len = (u8)sizeof(l4.udp) + l4_start;2066 } else {2067 csum_replace_by_diff(&l4.tcp->check,2068 (__force __wsum)htonl(paylen));2069 /* compute length of TCP segmentation header */2070 off->header_len = (u8)((l4.tcp->doff * 4) + l4_start);2071 }2072 2073 /* update gso_segs and bytecount */2074 first->gso_segs = skb_shinfo(skb)->gso_segs;2075 first->bytecount += (first->gso_segs - 1) * off->header_len;2076 2077 cd_tso_len = skb->len - off->header_len;2078 cd_mss = skb_shinfo(skb)->gso_size;2079 2080 /* record cdesc_qw1 with TSO parameters */2081 off->cd_qw1 |= (u64)(ICE_TX_DESC_DTYPE_CTX |2082 (ICE_TX_CTX_DESC_TSO << ICE_TXD_CTX_QW1_CMD_S) |2083 (cd_tso_len << ICE_TXD_CTX_QW1_TSO_LEN_S) |2084 (cd_mss << ICE_TXD_CTX_QW1_MSS_S));2085 first->tx_flags |= ICE_TX_FLAGS_TSO;2086 return 1;2087}2088 2089/**2090 * ice_txd_use_count - estimate the number of descriptors needed for Tx2091 * @size: transmit request size in bytes2092 *2093 * Due to hardware alignment restrictions (4K alignment), we need to2094 * assume that we can have no more than 12K of data per descriptor, even2095 * though each descriptor can take up to 16K - 1 bytes of aligned memory.2096 * Thus, we need to divide by 12K. But division is slow! Instead,2097 * we decompose the operation into shifts and one relatively cheap2098 * multiply operation.2099 *2100 * To divide by 12K, we first divide by 4K, then divide by 3:2101 * To divide by 4K, shift right by 12 bits2102 * To divide by 3, multiply by 85, then divide by 2562103 * (Divide by 256 is done by shifting right by 8 bits)2104 * Finally, we add one to round up. Because 256 isn't an exact multiple of2105 * 3, we'll underestimate near each multiple of 12K. This is actually more2106 * accurate as we have 4K - 1 of wiggle room that we can fit into the last2107 * segment. For our purposes this is accurate out to 1M which is orders of2108 * magnitude greater than our largest possible GSO size.2109 *2110 * This would then be implemented as:2111 * return (((size >> 12) * 85) >> 8) + ICE_DESCS_FOR_SKB_DATA_PTR;2112 *2113 * Since multiplication and division are commutative, we can reorder2114 * operations into:2115 * return ((size * 85) >> 20) + ICE_DESCS_FOR_SKB_DATA_PTR;2116 */2117static unsigned int ice_txd_use_count(unsigned int size)2118{2119 return ((size * 85) >> 20) + ICE_DESCS_FOR_SKB_DATA_PTR;2120}2121 2122/**2123 * ice_xmit_desc_count - calculate number of Tx descriptors needed2124 * @skb: send buffer2125 *2126 * Returns number of data descriptors needed for this skb.2127 */2128static unsigned int ice_xmit_desc_count(struct sk_buff *skb)2129{2130 const skb_frag_t *frag = &skb_shinfo(skb)->frags[0];2131 unsigned int nr_frags = skb_shinfo(skb)->nr_frags;2132 unsigned int count = 0, size = skb_headlen(skb);2133 2134 for (;;) {2135 count += ice_txd_use_count(size);2136 2137 if (!nr_frags--)2138 break;2139 2140 size = skb_frag_size(frag++);2141 }2142 2143 return count;2144}2145 2146/**2147 * __ice_chk_linearize - Check if there are more than 8 buffers per packet2148 * @skb: send buffer2149 *2150 * Note: This HW can't DMA more than 8 buffers to build a packet on the wire2151 * and so we need to figure out the cases where we need to linearize the skb.2152 *2153 * For TSO we need to count the TSO header and segment payload separately.2154 * As such we need to check cases where we have 7 fragments or more as we2155 * can potentially require 9 DMA transactions, 1 for the TSO header, 1 for2156 * the segment payload in the first descriptor, and another 7 for the2157 * fragments.2158 */2159static bool __ice_chk_linearize(struct sk_buff *skb)2160{2161 const skb_frag_t *frag, *stale;2162 int nr_frags, sum;2163 2164 /* no need to check if number of frags is less than 7 */2165 nr_frags = skb_shinfo(skb)->nr_frags;2166 if (nr_frags < (ICE_MAX_BUF_TXD - 1))2167 return false;2168 2169 /* We need to walk through the list and validate that each group2170 * of 6 fragments totals at least gso_size.2171 */2172 nr_frags -= ICE_MAX_BUF_TXD - 2;2173 frag = &skb_shinfo(skb)->frags[0];2174 2175 /* Initialize size to the negative value of gso_size minus 1. We2176 * use this as the worst case scenario in which the frag ahead2177 * of us only provides one byte which is why we are limited to 62178 * descriptors for a single transmit as the header and previous2179 * fragment are already consuming 2 descriptors.2180 */2181 sum = 1 - skb_shinfo(skb)->gso_size;2182 2183 /* Add size of frags 0 through 4 to create our initial sum */2184 sum += skb_frag_size(frag++);2185 sum += skb_frag_size(frag++);2186 sum += skb_frag_size(frag++);2187 sum += skb_frag_size(frag++);2188 sum += skb_frag_size(frag++);2189 2190 /* Walk through fragments adding latest fragment, testing it, and2191 * then removing stale fragments from the sum.2192 */2193 for (stale = &skb_shinfo(skb)->frags[0];; stale++) {2194 int stale_size = skb_frag_size(stale);2195 2196 sum += skb_frag_size(frag++);2197 2198 /* The stale fragment may present us with a smaller2199 * descriptor than the actual fragment size. To account2200 * for that we need to remove all the data on the front and2201 * figure out what the remainder would be in the last2202 * descriptor associated with the fragment.2203 */2204 if (stale_size > ICE_MAX_DATA_PER_TXD) {2205 int align_pad = -(skb_frag_off(stale)) &2206 (ICE_MAX_READ_REQ_SIZE - 1);2207 2208 sum -= align_pad;2209 stale_size -= align_pad;2210 2211 do {2212 sum -= ICE_MAX_DATA_PER_TXD_ALIGNED;2213 stale_size -= ICE_MAX_DATA_PER_TXD_ALIGNED;2214 } while (stale_size > ICE_MAX_DATA_PER_TXD);2215 }2216 2217 /* if sum is negative we failed to make sufficient progress */2218 if (sum < 0)2219 return true;2220 2221 if (!nr_frags--)2222 break;2223 2224 sum -= stale_size;2225 }2226 2227 return false;2228}2229 2230/**2231 * ice_chk_linearize - Check if there are more than 8 fragments per packet2232 * @skb: send buffer2233 * @count: number of buffers used2234 *2235 * Note: Our HW can't scatter-gather more than 8 fragments to build2236 * a packet on the wire and so we need to figure out the cases where we2237 * need to linearize the skb.2238 */2239static bool ice_chk_linearize(struct sk_buff *skb, unsigned int count)2240{2241 /* Both TSO and single send will work if count is less than 8 */2242 if (likely(count < ICE_MAX_BUF_TXD))2243 return false;2244 2245 if (skb_is_gso(skb))2246 return __ice_chk_linearize(skb);2247 2248 /* we can support up to 8 data buffers for a single send */2249 return count != ICE_MAX_BUF_TXD;2250}2251 2252/**2253 * ice_tstamp - set up context descriptor for hardware timestamp2254 * @tx_ring: pointer to the Tx ring to send buffer on2255 * @skb: pointer to the SKB we're sending2256 * @first: Tx buffer2257 * @off: Tx offload parameters2258 */2259static void2260ice_tstamp(struct ice_tx_ring *tx_ring, struct sk_buff *skb,2261 struct ice_tx_buf *first, struct ice_tx_offload_params *off)2262{2263 s8 idx;2264 2265 /* only timestamp the outbound packet if the user has requested it */2266 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))2267 return;2268 2269 /* Tx timestamps cannot be sampled when doing TSO */2270 if (first->tx_flags & ICE_TX_FLAGS_TSO)2271 return;2272 2273 /* Grab an open timestamp slot */2274 idx = ice_ptp_request_ts(tx_ring->tx_tstamps, skb);2275 if (idx < 0) {2276 tx_ring->vsi->back->ptp.tx_hwtstamp_skipped++;2277 return;2278 }2279 2280 off->cd_qw1 |= (u64)(ICE_TX_DESC_DTYPE_CTX |2281 (ICE_TX_CTX_DESC_TSYN << ICE_TXD_CTX_QW1_CMD_S) |2282 ((u64)idx << ICE_TXD_CTX_QW1_TSO_LEN_S));2283 first->tx_flags |= ICE_TX_FLAGS_TSYN;2284}2285 2286/**2287 * ice_xmit_frame_ring - Sends buffer on Tx ring2288 * @skb: send buffer2289 * @tx_ring: ring to send buffer on2290 *2291 * Returns NETDEV_TX_OK if sent, else an error code2292 */2293static netdev_tx_t2294ice_xmit_frame_ring(struct sk_buff *skb, struct ice_tx_ring *tx_ring)2295{2296 struct ice_tx_offload_params offload = { 0 };2297 struct ice_vsi *vsi = tx_ring->vsi;2298 struct ice_tx_buf *first;2299 struct ethhdr *eth;2300 unsigned int count;2301 int tso, csum;2302 2303 ice_trace(xmit_frame_ring, tx_ring, skb);2304 2305 if (unlikely(ipv6_hopopt_jumbo_remove(skb)))2306 goto out_drop;2307 2308 count = ice_xmit_desc_count(skb);2309 if (ice_chk_linearize(skb, count)) {2310 if (__skb_linearize(skb))2311 goto out_drop;2312 count = ice_txd_use_count(skb->len);2313 tx_ring->ring_stats->tx_stats.tx_linearize++;2314 }2315 2316 /* need: 1 descriptor per page * PAGE_SIZE/ICE_MAX_DATA_PER_TXD,2317 * + 1 desc for skb_head_len/ICE_MAX_DATA_PER_TXD,2318 * + 4 desc gap to avoid the cache line where head is,2319 * + 1 desc for context descriptor,2320 * otherwise try next time2321 */2322 if (ice_maybe_stop_tx(tx_ring, count + ICE_DESCS_PER_CACHE_LINE +2323 ICE_DESCS_FOR_CTX_DESC)) {2324 tx_ring->ring_stats->tx_stats.tx_busy++;2325 return NETDEV_TX_BUSY;2326 }2327 2328 /* prefetch for bql data which is infrequently used */2329 netdev_txq_bql_enqueue_prefetchw(txring_txq(tx_ring));2330 2331 offload.tx_ring = tx_ring;2332 2333 /* record the location of the first descriptor for this packet */2334 first = &tx_ring->tx_buf[tx_ring->next_to_use];2335 first->skb = skb;2336 first->type = ICE_TX_BUF_SKB;2337 first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN);2338 first->gso_segs = 1;2339 first->tx_flags = 0;2340 2341 /* prepare the VLAN tagging flags for Tx */2342 ice_tx_prepare_vlan_flags(tx_ring, first);2343 if (first->tx_flags & ICE_TX_FLAGS_HW_OUTER_SINGLE_VLAN) {2344 offload.cd_qw1 |= (u64)(ICE_TX_DESC_DTYPE_CTX |2345 (ICE_TX_CTX_DESC_IL2TAG2 <<2346 ICE_TXD_CTX_QW1_CMD_S));2347 offload.cd_l2tag2 = first->vid;2348 }2349 2350 /* set up TSO offload */2351 tso = ice_tso(first, &offload);2352 if (tso < 0)2353 goto out_drop;2354 2355 /* always set up Tx checksum offload */2356 csum = ice_tx_csum(first, &offload);2357 if (csum < 0)2358 goto out_drop;2359 2360 /* allow CONTROL frames egress from main VSI if FW LLDP disabled */2361 eth = (struct ethhdr *)skb_mac_header(skb);2362 if (unlikely((skb->priority == TC_PRIO_CONTROL ||2363 eth->h_proto == htons(ETH_P_LLDP)) &&2364 vsi->type == ICE_VSI_PF &&2365 vsi->port_info->qos_cfg.is_sw_lldp))2366 offload.cd_qw1 |= (u64)(ICE_TX_DESC_DTYPE_CTX |2367 ICE_TX_CTX_DESC_SWTCH_UPLINK <<2368 ICE_TXD_CTX_QW1_CMD_S);2369 2370 ice_tstamp(tx_ring, skb, first, &offload);2371 if (ice_is_switchdev_running(vsi->back) && vsi->type != ICE_VSI_SF)2372 ice_eswitch_set_target_vsi(skb, &offload);2373 2374 if (offload.cd_qw1 & ICE_TX_DESC_DTYPE_CTX) {2375 struct ice_tx_ctx_desc *cdesc;2376 u16 i = tx_ring->next_to_use;2377 2378 /* grab the next descriptor */2379 cdesc = ICE_TX_CTX_DESC(tx_ring, i);2380 i++;2381 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;2382 2383 /* setup context descriptor */2384 cdesc->tunneling_params = cpu_to_le32(offload.cd_tunnel_params);2385 cdesc->l2tag2 = cpu_to_le16(offload.cd_l2tag2);2386 cdesc->rsvd = cpu_to_le16(0);2387 cdesc->qw1 = cpu_to_le64(offload.cd_qw1);2388 }2389 2390 ice_tx_map(tx_ring, first, &offload);2391 return NETDEV_TX_OK;2392 2393out_drop:2394 ice_trace(xmit_frame_ring_drop, tx_ring, skb);2395 dev_kfree_skb_any(skb);2396 return NETDEV_TX_OK;2397}2398 2399/**2400 * ice_start_xmit - Selects the correct VSI and Tx queue to send buffer2401 * @skb: send buffer2402 * @netdev: network interface device structure2403 *2404 * Returns NETDEV_TX_OK if sent, else an error code2405 */2406netdev_tx_t ice_start_xmit(struct sk_buff *skb, struct net_device *netdev)2407{2408 struct ice_netdev_priv *np = netdev_priv(netdev);2409 struct ice_vsi *vsi = np->vsi;2410 struct ice_tx_ring *tx_ring;2411 2412 tx_ring = vsi->tx_rings[skb->queue_mapping];2413 2414 /* hardware can't handle really short frames, hardware padding works2415 * beyond this point2416 */2417 if (skb_put_padto(skb, ICE_MIN_TX_LEN))2418 return NETDEV_TX_OK;2419 2420 return ice_xmit_frame_ring(skb, tx_ring);2421}2422 2423/**2424 * ice_get_dscp_up - return the UP/TC value for a SKB2425 * @dcbcfg: DCB config that contains DSCP to UP/TC mapping2426 * @skb: SKB to query for info to determine UP/TC2427 *2428 * This function is to only be called when the PF is in L3 DSCP PFC mode2429 */2430static u8 ice_get_dscp_up(struct ice_dcbx_cfg *dcbcfg, struct sk_buff *skb)2431{2432 u8 dscp = 0;2433 2434 if (skb->protocol == htons(ETH_P_IP))2435 dscp = ipv4_get_dsfield(ip_hdr(skb)) >> 2;2436 else if (skb->protocol == htons(ETH_P_IPV6))2437 dscp = ipv6_get_dsfield(ipv6_hdr(skb)) >> 2;2438 2439 return dcbcfg->dscp_map[dscp];2440}2441 2442u162443ice_select_queue(struct net_device *netdev, struct sk_buff *skb,2444 struct net_device *sb_dev)2445{2446 struct ice_pf *pf = ice_netdev_to_pf(netdev);2447 struct ice_dcbx_cfg *dcbcfg;2448 2449 dcbcfg = &pf->hw.port_info->qos_cfg.local_dcbx_cfg;2450 if (dcbcfg->pfc_mode == ICE_QOS_MODE_DSCP)2451 skb->priority = ice_get_dscp_up(dcbcfg, skb);2452 2453 return netdev_pick_tx(netdev, skb, sb_dev);2454}2455 2456/**2457 * ice_clean_ctrl_tx_irq - interrupt handler for flow director Tx queue2458 * @tx_ring: tx_ring to clean2459 */2460void ice_clean_ctrl_tx_irq(struct ice_tx_ring *tx_ring)2461{2462 struct ice_vsi *vsi = tx_ring->vsi;2463 s16 i = tx_ring->next_to_clean;2464 int budget = ICE_DFLT_IRQ_WORK;2465 struct ice_tx_desc *tx_desc;2466 struct ice_tx_buf *tx_buf;2467 2468 tx_buf = &tx_ring->tx_buf[i];2469 tx_desc = ICE_TX_DESC(tx_ring, i);2470 i -= tx_ring->count;2471 2472 do {2473 struct ice_tx_desc *eop_desc = tx_buf->next_to_watch;2474 2475 /* if next_to_watch is not set then there is no pending work */2476 if (!eop_desc)2477 break;2478 2479 /* prevent any other reads prior to eop_desc */2480 smp_rmb();2481 2482 /* if the descriptor isn't done, no work to do */2483 if (!(eop_desc->cmd_type_offset_bsz &2484 cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE)))2485 break;2486 2487 /* clear next_to_watch to prevent false hangs */2488 tx_buf->next_to_watch = NULL;2489 tx_desc->buf_addr = 0;2490 tx_desc->cmd_type_offset_bsz = 0;2491 2492 /* move past filter desc */2493 tx_buf++;2494 tx_desc++;2495 i++;2496 if (unlikely(!i)) {2497 i -= tx_ring->count;2498 tx_buf = tx_ring->tx_buf;2499 tx_desc = ICE_TX_DESC(tx_ring, 0);2500 }2501 2502 /* unmap the data header */2503 if (dma_unmap_len(tx_buf, len))2504 dma_unmap_single(tx_ring->dev,2505 dma_unmap_addr(tx_buf, dma),2506 dma_unmap_len(tx_buf, len),2507 DMA_TO_DEVICE);2508 if (tx_buf->type == ICE_TX_BUF_DUMMY)2509 devm_kfree(tx_ring->dev, tx_buf->raw_buf);2510 2511 /* clear next_to_watch to prevent false hangs */2512 tx_buf->type = ICE_TX_BUF_EMPTY;2513 tx_buf->tx_flags = 0;2514 tx_buf->next_to_watch = NULL;2515 dma_unmap_len_set(tx_buf, len, 0);2516 tx_desc->buf_addr = 0;2517 tx_desc->cmd_type_offset_bsz = 0;2518 2519 /* move past eop_desc for start of next FD desc */2520 tx_buf++;2521 tx_desc++;2522 i++;2523 if (unlikely(!i)) {2524 i -= tx_ring->count;2525 tx_buf = tx_ring->tx_buf;2526 tx_desc = ICE_TX_DESC(tx_ring, 0);2527 }2528 2529 budget--;2530 } while (likely(budget));2531 2532 i += tx_ring->count;2533 tx_ring->next_to_clean = i;2534 2535 /* re-enable interrupt if needed */2536 ice_irq_dynamic_ena(&vsi->back->hw, vsi, vsi->q_vectors[0]);2537}2538