175 lines · c
1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)2/* Copyright (C) 2018 Netronome Systems, Inc */3/* Copyright (C) 2021 Corigine, Inc */4 5#include <linux/dma-direction.h>6#include <linux/dma-mapping.h>7#include <linux/slab.h>8#include <net/xdp_sock_drv.h>9#include <trace/events/xdp.h>10 11#include "nfp_app.h"12#include "nfp_net.h"13#include "nfp_net_dp.h"14#include "nfp_net_xsk.h"15 16static void17nfp_net_xsk_rx_bufs_stash(struct nfp_net_rx_ring *rx_ring, unsigned int idx,18 struct xdp_buff *xdp)19{20 unsigned int headroom;21 22 headroom = xsk_pool_get_headroom(rx_ring->r_vec->xsk_pool);23 24 rx_ring->rxds[idx].fld.reserved = 0;25 rx_ring->rxds[idx].fld.meta_len_dd = 0;26 27 rx_ring->xsk_rxbufs[idx].xdp = xdp;28 rx_ring->xsk_rxbufs[idx].dma_addr =29 xsk_buff_xdp_get_frame_dma(xdp) + headroom;30}31 32void nfp_net_xsk_rx_unstash(struct nfp_net_xsk_rx_buf *rxbuf)33{34 rxbuf->dma_addr = 0;35 rxbuf->xdp = NULL;36}37 38void nfp_net_xsk_rx_free(struct nfp_net_xsk_rx_buf *rxbuf)39{40 if (rxbuf->xdp)41 xsk_buff_free(rxbuf->xdp);42 43 nfp_net_xsk_rx_unstash(rxbuf);44}45 46void nfp_net_xsk_rx_bufs_free(struct nfp_net_rx_ring *rx_ring)47{48 unsigned int i;49 50 if (!rx_ring->cnt)51 return;52 53 for (i = 0; i < rx_ring->cnt - 1; i++)54 nfp_net_xsk_rx_free(&rx_ring->xsk_rxbufs[i]);55}56 57void nfp_net_xsk_rx_ring_fill_freelist(struct nfp_net_rx_ring *rx_ring)58{59 struct nfp_net_r_vector *r_vec = rx_ring->r_vec;60 struct xsk_buff_pool *pool = r_vec->xsk_pool;61 unsigned int wr_idx, wr_ptr_add = 0;62 struct xdp_buff *xdp;63 64 while (nfp_net_rx_space(rx_ring)) {65 wr_idx = D_IDX(rx_ring, rx_ring->wr_p);66 67 xdp = xsk_buff_alloc(pool);68 if (!xdp)69 break;70 71 nfp_net_xsk_rx_bufs_stash(rx_ring, wr_idx, xdp);72 73 /* DMA address is expanded to 48-bit width in freelist for NFP3800,74 * so the *_48b macro is used accordingly, it's also OK to fill75 * a 40-bit address since the top 8 bits are get set to 0.76 */77 nfp_desc_set_dma_addr_48b(&rx_ring->rxds[wr_idx].fld,78 rx_ring->xsk_rxbufs[wr_idx].dma_addr);79 80 rx_ring->wr_p++;81 wr_ptr_add++;82 }83 84 /* Ensure all records are visible before incrementing write counter. */85 wmb();86 nfp_qcp_wr_ptr_add(rx_ring->qcp_fl, wr_ptr_add);87}88 89void nfp_net_xsk_rx_drop(struct nfp_net_r_vector *r_vec,90 struct nfp_net_xsk_rx_buf *xrxbuf)91{92 u64_stats_update_begin(&r_vec->rx_sync);93 r_vec->rx_drops++;94 u64_stats_update_end(&r_vec->rx_sync);95 96 nfp_net_xsk_rx_free(xrxbuf);97}98 99static void nfp_net_xsk_pool_unmap(struct device *dev,100 struct xsk_buff_pool *pool)101{102 return xsk_pool_dma_unmap(pool, 0);103}104 105static int nfp_net_xsk_pool_map(struct device *dev, struct xsk_buff_pool *pool)106{107 return xsk_pool_dma_map(pool, dev, 0);108}109 110int nfp_net_xsk_setup_pool(struct net_device *netdev,111 struct xsk_buff_pool *pool, u16 queue_id)112{113 struct nfp_net *nn = netdev_priv(netdev);114 115 struct xsk_buff_pool *prev_pool;116 struct nfp_net_dp *dp;117 int err;118 119 /* NFDK doesn't implement xsk yet. */120 if (nn->dp.ops->version == NFP_NFD_VER_NFDK)121 return -EOPNOTSUPP;122 123 /* Reject on old FWs so we can drop some checks on datapath. */124 if (nn->dp.rx_offset != NFP_NET_CFG_RX_OFFSET_DYNAMIC)125 return -EOPNOTSUPP;126 if (!nn->dp.chained_metadata_format)127 return -EOPNOTSUPP;128 129 /* Install */130 if (pool) {131 err = nfp_net_xsk_pool_map(nn->dp.dev, pool);132 if (err)133 return err;134 }135 136 /* Reconfig/swap */137 dp = nfp_net_clone_dp(nn);138 if (!dp) {139 err = -ENOMEM;140 goto err_unmap;141 }142 143 prev_pool = dp->xsk_pools[queue_id];144 dp->xsk_pools[queue_id] = pool;145 146 err = nfp_net_ring_reconfig(nn, dp, NULL);147 if (err)148 goto err_unmap;149 150 /* Uninstall */151 if (prev_pool)152 nfp_net_xsk_pool_unmap(nn->dp.dev, prev_pool);153 154 return 0;155err_unmap:156 if (pool)157 nfp_net_xsk_pool_unmap(nn->dp.dev, pool);158 159 return err;160}161 162int nfp_net_xsk_wakeup(struct net_device *netdev, u32 queue_id, u32 flags)163{164 struct nfp_net *nn = netdev_priv(netdev);165 166 /* queue_id comes from a zero-copy socket, installed with XDP_SETUP_XSK_POOL,167 * so it must be within our vector range. Moreover, our napi structs168 * are statically allocated, so we can always kick them without worrying169 * if reconfig is in progress or interface down.170 */171 napi_schedule(&nn->r_vecs[queue_id].napi);172 173 return 0;174}175