486 lines · c
1/* Broadcom NetXtreme-C/E network driver.2 *3 * Copyright (c) 2016-2017 Broadcom Limited4 *5 * This program is free software; you can redistribute it and/or modify6 * it under the terms of the GNU General Public License as published by7 * the Free Software Foundation.8 */9#include <linux/kernel.h>10#include <linux/errno.h>11#include <linux/pci.h>12#include <linux/netdevice.h>13#include <linux/etherdevice.h>14#include <linux/if_vlan.h>15#include <linux/bpf.h>16#include <linux/bpf_trace.h>17#include <linux/filter.h>18#include <net/page_pool/helpers.h>19#include "bnxt_hsi.h"20#include "bnxt.h"21#include "bnxt_xdp.h"22 23DEFINE_STATIC_KEY_FALSE(bnxt_xdp_locking_key);24 25struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp,26 struct bnxt_tx_ring_info *txr,27 dma_addr_t mapping, u32 len,28 struct xdp_buff *xdp)29{30 struct skb_shared_info *sinfo;31 struct bnxt_sw_tx_bd *tx_buf;32 struct tx_bd *txbd;33 int num_frags = 0;34 u32 flags;35 u16 prod;36 int i;37 38 if (xdp && xdp_buff_has_frags(xdp)) {39 sinfo = xdp_get_shared_info_from_buff(xdp);40 num_frags = sinfo->nr_frags;41 }42 43 /* fill up the first buffer */44 prod = txr->tx_prod;45 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];46 tx_buf->nr_frags = num_frags;47 if (xdp)48 tx_buf->page = virt_to_head_page(xdp->data);49 50 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];51 flags = (len << TX_BD_LEN_SHIFT) |52 ((num_frags + 1) << TX_BD_FLAGS_BD_CNT_SHIFT) |53 bnxt_lhint_arr[len >> 9];54 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);55 txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 1 + num_frags);56 txbd->tx_bd_haddr = cpu_to_le64(mapping);57 58 /* now let us fill up the frags into the next buffers */59 for (i = 0; i < num_frags ; i++) {60 skb_frag_t *frag = &sinfo->frags[i];61 struct bnxt_sw_tx_bd *frag_tx_buf;62 dma_addr_t frag_mapping;63 int frag_len;64 65 prod = NEXT_TX(prod);66 WRITE_ONCE(txr->tx_prod, prod);67 68 /* first fill up the first buffer */69 frag_tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)];70 frag_tx_buf->page = skb_frag_page(frag);71 72 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)];73 74 frag_len = skb_frag_size(frag);75 flags = frag_len << TX_BD_LEN_SHIFT;76 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);77 frag_mapping = page_pool_get_dma_addr(skb_frag_page(frag)) +78 skb_frag_off(frag);79 txbd->tx_bd_haddr = cpu_to_le64(frag_mapping);80 81 len = frag_len;82 }83 84 flags &= ~TX_BD_LEN;85 txbd->tx_bd_len_flags_type = cpu_to_le32(((len) << TX_BD_LEN_SHIFT) | flags |86 TX_BD_FLAGS_PACKET_END);87 /* Sync TX BD */88 wmb();89 prod = NEXT_TX(prod);90 WRITE_ONCE(txr->tx_prod, prod);91 92 return tx_buf;93}94 95static void __bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr,96 dma_addr_t mapping, u32 len, u16 rx_prod,97 struct xdp_buff *xdp)98{99 struct bnxt_sw_tx_bd *tx_buf;100 101 tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, xdp);102 tx_buf->rx_prod = rx_prod;103 tx_buf->action = XDP_TX;104 105}106 107static void __bnxt_xmit_xdp_redirect(struct bnxt *bp,108 struct bnxt_tx_ring_info *txr,109 dma_addr_t mapping, u32 len,110 struct xdp_frame *xdpf)111{112 struct bnxt_sw_tx_bd *tx_buf;113 114 tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, NULL);115 tx_buf->action = XDP_REDIRECT;116 tx_buf->xdpf = xdpf;117 dma_unmap_addr_set(tx_buf, mapping, mapping);118 dma_unmap_len_set(tx_buf, len, 0);119}120 121void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int budget)122{123 struct bnxt_tx_ring_info *txr = bnapi->tx_ring[0];124 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;125 u16 tx_hw_cons = txr->tx_hw_cons;126 bool rx_doorbell_needed = false;127 struct bnxt_sw_tx_bd *tx_buf;128 u16 tx_cons = txr->tx_cons;129 u16 last_tx_cons = tx_cons;130 int j, frags;131 132 if (!budget)133 return;134 135 while (RING_TX(bp, tx_cons) != tx_hw_cons) {136 tx_buf = &txr->tx_buf_ring[RING_TX(bp, tx_cons)];137 138 if (tx_buf->action == XDP_REDIRECT) {139 struct pci_dev *pdev = bp->pdev;140 141 dma_unmap_single(&pdev->dev,142 dma_unmap_addr(tx_buf, mapping),143 dma_unmap_len(tx_buf, len),144 DMA_TO_DEVICE);145 xdp_return_frame(tx_buf->xdpf);146 tx_buf->action = 0;147 tx_buf->xdpf = NULL;148 } else if (tx_buf->action == XDP_TX) {149 tx_buf->action = 0;150 rx_doorbell_needed = true;151 last_tx_cons = tx_cons;152 153 frags = tx_buf->nr_frags;154 for (j = 0; j < frags; j++) {155 tx_cons = NEXT_TX(tx_cons);156 tx_buf = &txr->tx_buf_ring[RING_TX(bp, tx_cons)];157 page_pool_recycle_direct(rxr->page_pool, tx_buf->page);158 }159 } else {160 bnxt_sched_reset_txr(bp, txr, tx_cons);161 return;162 }163 tx_cons = NEXT_TX(tx_cons);164 }165 166 bnapi->events &= ~BNXT_TX_CMP_EVENT;167 WRITE_ONCE(txr->tx_cons, tx_cons);168 if (rx_doorbell_needed) {169 tx_buf = &txr->tx_buf_ring[RING_TX(bp, last_tx_cons)];170 bnxt_db_write(bp, &rxr->rx_db, tx_buf->rx_prod);171 172 }173}174 175bool bnxt_xdp_attached(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)176{177 struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog);178 179 return !!xdp_prog;180}181 182void bnxt_xdp_buff_init(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,183 u16 cons, u8 *data_ptr, unsigned int len,184 struct xdp_buff *xdp)185{186 u32 buflen = BNXT_RX_PAGE_SIZE;187 struct bnxt_sw_rx_bd *rx_buf;188 struct pci_dev *pdev;189 dma_addr_t mapping;190 u32 offset;191 192 pdev = bp->pdev;193 rx_buf = &rxr->rx_buf_ring[cons];194 offset = bp->rx_offset;195 196 mapping = rx_buf->mapping - bp->rx_dma_offset;197 dma_sync_single_for_cpu(&pdev->dev, mapping + offset, len, bp->rx_dir);198 199 xdp_init_buff(xdp, buflen, &rxr->xdp_rxq);200 xdp_prepare_buff(xdp, data_ptr - offset, offset, len, true);201}202 203void bnxt_xdp_buff_frags_free(struct bnxt_rx_ring_info *rxr,204 struct xdp_buff *xdp)205{206 struct skb_shared_info *shinfo;207 int i;208 209 if (!xdp || !xdp_buff_has_frags(xdp))210 return;211 shinfo = xdp_get_shared_info_from_buff(xdp);212 for (i = 0; i < shinfo->nr_frags; i++) {213 struct page *page = skb_frag_page(&shinfo->frags[i]);214 215 page_pool_recycle_direct(rxr->page_pool, page);216 }217 shinfo->nr_frags = 0;218}219 220/* returns the following:221 * true - packet consumed by XDP and new buffer is allocated.222 * false - packet should be passed to the stack.223 */224bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons,225 struct xdp_buff *xdp, struct page *page, u8 **data_ptr,226 unsigned int *len, u8 *event)227{228 struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog);229 struct bnxt_tx_ring_info *txr;230 struct bnxt_sw_rx_bd *rx_buf;231 struct pci_dev *pdev;232 dma_addr_t mapping;233 u32 tx_needed = 1;234 void *orig_data;235 u32 tx_avail;236 u32 offset;237 u32 act;238 239 if (!xdp_prog)240 return false;241 242 pdev = bp->pdev;243 offset = bp->rx_offset;244 245 txr = rxr->bnapi->tx_ring[0];246 /* BNXT_RX_PAGE_MODE(bp) when XDP enabled */247 orig_data = xdp->data;248 249 act = bpf_prog_run_xdp(xdp_prog, xdp);250 251 tx_avail = bnxt_tx_avail(bp, txr);252 /* If the tx ring is not full, we must not update the rx producer yet253 * because we may still be transmitting on some BDs.254 */255 if (tx_avail != bp->tx_ring_size)256 *event &= ~BNXT_RX_EVENT;257 258 *len = xdp->data_end - xdp->data;259 if (orig_data != xdp->data) {260 offset = xdp->data - xdp->data_hard_start;261 *data_ptr = xdp->data_hard_start + offset;262 }263 264 switch (act) {265 case XDP_PASS:266 return false;267 268 case XDP_TX:269 rx_buf = &rxr->rx_buf_ring[cons];270 mapping = rx_buf->mapping - bp->rx_dma_offset;271 *event &= BNXT_TX_CMP_EVENT;272 273 if (unlikely(xdp_buff_has_frags(xdp))) {274 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);275 276 tx_needed += sinfo->nr_frags;277 *event = BNXT_AGG_EVENT;278 }279 280 if (tx_avail < tx_needed) {281 trace_xdp_exception(bp->dev, xdp_prog, act);282 bnxt_xdp_buff_frags_free(rxr, xdp);283 bnxt_reuse_rx_data(rxr, cons, page);284 return true;285 }286 287 dma_sync_single_for_device(&pdev->dev, mapping + offset, *len,288 bp->rx_dir);289 290 *event |= BNXT_TX_EVENT;291 __bnxt_xmit_xdp(bp, txr, mapping + offset, *len,292 NEXT_RX(rxr->rx_prod), xdp);293 bnxt_reuse_rx_data(rxr, cons, page);294 return true;295 case XDP_REDIRECT:296 /* if we are calling this here then we know that the297 * redirect is coming from a frame received by the298 * bnxt_en driver.299 */300 301 /* if we are unable to allocate a new buffer, abort and reuse */302 if (bnxt_alloc_rx_data(bp, rxr, rxr->rx_prod, GFP_ATOMIC)) {303 trace_xdp_exception(bp->dev, xdp_prog, act);304 bnxt_xdp_buff_frags_free(rxr, xdp);305 bnxt_reuse_rx_data(rxr, cons, page);306 return true;307 }308 309 if (xdp_do_redirect(bp->dev, xdp, xdp_prog)) {310 trace_xdp_exception(bp->dev, xdp_prog, act);311 page_pool_recycle_direct(rxr->page_pool, page);312 return true;313 }314 315 *event |= BNXT_REDIRECT_EVENT;316 break;317 default:318 bpf_warn_invalid_xdp_action(bp->dev, xdp_prog, act);319 fallthrough;320 case XDP_ABORTED:321 trace_xdp_exception(bp->dev, xdp_prog, act);322 fallthrough;323 case XDP_DROP:324 bnxt_xdp_buff_frags_free(rxr, xdp);325 bnxt_reuse_rx_data(rxr, cons, page);326 break;327 }328 return true;329}330 331int bnxt_xdp_xmit(struct net_device *dev, int num_frames,332 struct xdp_frame **frames, u32 flags)333{334 struct bnxt *bp = netdev_priv(dev);335 struct bpf_prog *xdp_prog = READ_ONCE(bp->xdp_prog);336 struct pci_dev *pdev = bp->pdev;337 struct bnxt_tx_ring_info *txr;338 dma_addr_t mapping;339 int nxmit = 0;340 int ring;341 int i;342 343 if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||344 !bp->tx_nr_rings_xdp ||345 !xdp_prog)346 return -EINVAL;347 348 ring = smp_processor_id() % bp->tx_nr_rings_xdp;349 txr = &bp->tx_ring[ring];350 351 if (READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING)352 return -EINVAL;353 354 if (static_branch_unlikely(&bnxt_xdp_locking_key))355 spin_lock(&txr->xdp_tx_lock);356 357 for (i = 0; i < num_frames; i++) {358 struct xdp_frame *xdp = frames[i];359 360 if (!bnxt_tx_avail(bp, txr))361 break;362 363 mapping = dma_map_single(&pdev->dev, xdp->data, xdp->len,364 DMA_TO_DEVICE);365 366 if (dma_mapping_error(&pdev->dev, mapping))367 break;368 369 __bnxt_xmit_xdp_redirect(bp, txr, mapping, xdp->len, xdp);370 nxmit++;371 }372 373 if (flags & XDP_XMIT_FLUSH) {374 /* Sync BD data before updating doorbell */375 wmb();376 bnxt_db_write(bp, &txr->tx_db, txr->tx_prod);377 }378 379 if (static_branch_unlikely(&bnxt_xdp_locking_key))380 spin_unlock(&txr->xdp_tx_lock);381 382 return nxmit;383}384 385/* Under rtnl_lock */386static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog)387{388 struct net_device *dev = bp->dev;389 int tx_xdp = 0, tx_cp, rc, tc;390 struct bpf_prog *old;391 392 if (prog && !prog->aux->xdp_has_frags &&393 bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) {394 netdev_warn(dev, "MTU %d larger than %d without XDP frag support.\n",395 bp->dev->mtu, BNXT_MAX_PAGE_MODE_MTU);396 return -EOPNOTSUPP;397 }398 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) {399 netdev_warn(dev, "ethtool rx/tx channels must be combined to support XDP.\n");400 return -EOPNOTSUPP;401 }402 if (prog)403 tx_xdp = bp->rx_nr_rings;404 405 tc = bp->num_tc;406 if (!tc)407 tc = 1;408 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,409 true, tc, tx_xdp);410 if (rc) {411 netdev_warn(dev, "Unable to reserve enough TX rings to support XDP.\n");412 return rc;413 }414 if (netif_running(dev))415 bnxt_close_nic(bp, true, false);416 417 old = xchg(&bp->xdp_prog, prog);418 if (old)419 bpf_prog_put(old);420 421 if (prog) {422 bnxt_set_rx_skb_mode(bp, true);423 xdp_features_set_redirect_target(dev, true);424 } else {425 int rx, tx;426 427 xdp_features_clear_redirect_target(dev);428 bnxt_set_rx_skb_mode(bp, false);429 bnxt_get_max_rings(bp, &rx, &tx, true);430 if (rx > 1) {431 bp->flags &= ~BNXT_FLAG_NO_AGG_RINGS;432 bp->dev->hw_features |= NETIF_F_LRO;433 }434 }435 bp->tx_nr_rings_xdp = tx_xdp;436 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc + tx_xdp;437 tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings);438 bp->cp_nr_rings = max_t(int, tx_cp, bp->rx_nr_rings);439 bnxt_set_tpa_flags(bp);440 bnxt_set_ring_params(bp);441 442 if (netif_running(dev))443 return bnxt_open_nic(bp, true, false);444 445 return 0;446}447 448int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp)449{450 struct bnxt *bp = netdev_priv(dev);451 int rc;452 453 switch (xdp->command) {454 case XDP_SETUP_PROG:455 rc = bnxt_xdp_set(bp, xdp->prog);456 break;457 default:458 rc = -EINVAL;459 break;460 }461 return rc;462}463 464struct sk_buff *465bnxt_xdp_build_skb(struct bnxt *bp, struct sk_buff *skb, u8 num_frags,466 struct page_pool *pool, struct xdp_buff *xdp,467 struct rx_cmp_ext *rxcmp1)468{469 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);470 471 if (!skb)472 return NULL;473 skb_checksum_none_assert(skb);474 if (RX_CMP_L4_CS_OK(rxcmp1)) {475 if (bp->dev->features & NETIF_F_RXCSUM) {476 skb->ip_summed = CHECKSUM_UNNECESSARY;477 skb->csum_level = RX_CMP_ENCAP(rxcmp1);478 }479 }480 xdp_update_skb_shared_info(skb, num_frags,481 sinfo->xdp_frags_size,482 BNXT_RX_PAGE_SIZE * sinfo->nr_frags,483 xdp_buff_is_frag_pfmemalloc(xdp));484 return skb;485}486