280 lines · c
1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)2/* Copyright (C) 2015-2019 Netronome Systems, Inc. */3 4#include <linux/seq_file.h>5 6#include "../nfp_net.h"7#include "../nfp_net_dp.h"8#include "../nfp_net_xsk.h"9#include "nfd3.h"10 11static void nfp_nfd3_xsk_tx_bufs_free(struct nfp_net_tx_ring *tx_ring)12{13 struct nfp_nfd3_tx_buf *txbuf;14 unsigned int idx;15 16 while (tx_ring->rd_p != tx_ring->wr_p) {17 idx = D_IDX(tx_ring, tx_ring->rd_p);18 txbuf = &tx_ring->txbufs[idx];19 20 txbuf->real_len = 0;21 22 tx_ring->qcp_rd_p++;23 tx_ring->rd_p++;24 25 if (tx_ring->r_vec->xsk_pool) {26 if (txbuf->is_xsk_tx)27 nfp_nfd3_xsk_tx_free(txbuf);28 29 xsk_tx_completed(tx_ring->r_vec->xsk_pool, 1);30 }31 }32}33 34/**35 * nfp_nfd3_tx_ring_reset() - Free any untransmitted buffers and reset pointers36 * @dp: NFP Net data path struct37 * @tx_ring: TX ring structure38 *39 * Assumes that the device is stopped, must be idempotent.40 */41static void42nfp_nfd3_tx_ring_reset(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring)43{44 struct netdev_queue *nd_q;45 const skb_frag_t *frag;46 47 while (!tx_ring->is_xdp && tx_ring->rd_p != tx_ring->wr_p) {48 struct nfp_nfd3_tx_buf *tx_buf;49 struct sk_buff *skb;50 int idx, nr_frags;51 52 idx = D_IDX(tx_ring, tx_ring->rd_p);53 tx_buf = &tx_ring->txbufs[idx];54 55 skb = tx_ring->txbufs[idx].skb;56 nr_frags = skb_shinfo(skb)->nr_frags;57 58 if (tx_buf->fidx == -1) {59 /* unmap head */60 dma_unmap_single(dp->dev, tx_buf->dma_addr,61 skb_headlen(skb), DMA_TO_DEVICE);62 } else {63 /* unmap fragment */64 frag = &skb_shinfo(skb)->frags[tx_buf->fidx];65 dma_unmap_page(dp->dev, tx_buf->dma_addr,66 skb_frag_size(frag), DMA_TO_DEVICE);67 }68 69 /* check for last gather fragment */70 if (tx_buf->fidx == nr_frags - 1)71 dev_kfree_skb_any(skb);72 73 tx_buf->dma_addr = 0;74 tx_buf->skb = NULL;75 tx_buf->fidx = -2;76 77 tx_ring->qcp_rd_p++;78 tx_ring->rd_p++;79 }80 81 if (tx_ring->is_xdp)82 nfp_nfd3_xsk_tx_bufs_free(tx_ring);83 84 memset(tx_ring->txds, 0, tx_ring->size);85 tx_ring->wr_p = 0;86 tx_ring->rd_p = 0;87 tx_ring->qcp_rd_p = 0;88 tx_ring->wr_ptr_add = 0;89 90 if (tx_ring->is_xdp || !dp->netdev)91 return;92 93 nd_q = netdev_get_tx_queue(dp->netdev, tx_ring->idx);94 netdev_tx_reset_queue(nd_q);95}96 97/**98 * nfp_nfd3_tx_ring_free() - Free resources allocated to a TX ring99 * @tx_ring: TX ring to free100 */101static void nfp_nfd3_tx_ring_free(struct nfp_net_tx_ring *tx_ring)102{103 struct nfp_net_r_vector *r_vec = tx_ring->r_vec;104 struct nfp_net_dp *dp = &r_vec->nfp_net->dp;105 106 kvfree(tx_ring->txbufs);107 108 if (tx_ring->txds)109 dma_free_coherent(dp->dev, tx_ring->size,110 tx_ring->txds, tx_ring->dma);111 112 tx_ring->cnt = 0;113 tx_ring->txbufs = NULL;114 tx_ring->txds = NULL;115 tx_ring->dma = 0;116 tx_ring->size = 0;117}118 119/**120 * nfp_nfd3_tx_ring_alloc() - Allocate resource for a TX ring121 * @dp: NFP Net data path struct122 * @tx_ring: TX Ring structure to allocate123 *124 * Return: 0 on success, negative errno otherwise.125 */126static int127nfp_nfd3_tx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring)128{129 struct nfp_net_r_vector *r_vec = tx_ring->r_vec;130 131 tx_ring->cnt = dp->txd_cnt;132 133 tx_ring->size = array_size(tx_ring->cnt, sizeof(*tx_ring->txds));134 tx_ring->txds = dma_alloc_coherent(dp->dev, tx_ring->size,135 &tx_ring->dma,136 GFP_KERNEL | __GFP_NOWARN);137 if (!tx_ring->txds) {138 netdev_warn(dp->netdev, "failed to allocate TX descriptor ring memory, requested descriptor count: %d, consider lowering descriptor count\n",139 tx_ring->cnt);140 goto err_alloc;141 }142 143 tx_ring->txbufs = kvcalloc(tx_ring->cnt, sizeof(*tx_ring->txbufs),144 GFP_KERNEL);145 if (!tx_ring->txbufs)146 goto err_alloc;147 148 if (!tx_ring->is_xdp && dp->netdev)149 netif_set_xps_queue(dp->netdev, &r_vec->affinity_mask,150 tx_ring->idx);151 152 return 0;153 154err_alloc:155 nfp_nfd3_tx_ring_free(tx_ring);156 return -ENOMEM;157}158 159static void160nfp_nfd3_tx_ring_bufs_free(struct nfp_net_dp *dp,161 struct nfp_net_tx_ring *tx_ring)162{163 unsigned int i;164 165 if (!tx_ring->is_xdp)166 return;167 168 for (i = 0; i < tx_ring->cnt; i++) {169 if (!tx_ring->txbufs[i].frag)170 return;171 172 nfp_net_dma_unmap_rx(dp, tx_ring->txbufs[i].dma_addr);173 __free_page(virt_to_page(tx_ring->txbufs[i].frag));174 }175}176 177static int178nfp_nfd3_tx_ring_bufs_alloc(struct nfp_net_dp *dp,179 struct nfp_net_tx_ring *tx_ring)180{181 struct nfp_nfd3_tx_buf *txbufs = tx_ring->txbufs;182 unsigned int i;183 184 if (!tx_ring->is_xdp)185 return 0;186 187 for (i = 0; i < tx_ring->cnt; i++) {188 txbufs[i].frag = nfp_net_rx_alloc_one(dp, &txbufs[i].dma_addr);189 if (!txbufs[i].frag) {190 nfp_nfd3_tx_ring_bufs_free(dp, tx_ring);191 return -ENOMEM;192 }193 }194 195 return 0;196}197 198static void199nfp_nfd3_print_tx_descs(struct seq_file *file,200 struct nfp_net_r_vector *r_vec,201 struct nfp_net_tx_ring *tx_ring,202 u32 d_rd_p, u32 d_wr_p)203{204 struct nfp_nfd3_tx_desc *txd;205 u32 txd_cnt = tx_ring->cnt;206 int i;207 208 for (i = 0; i < txd_cnt; i++) {209 struct xdp_buff *xdp;210 struct sk_buff *skb;211 212 txd = &tx_ring->txds[i];213 seq_printf(file, "%04d: 0x%08x 0x%08x 0x%08x 0x%08x", i,214 txd->vals[0], txd->vals[1],215 txd->vals[2], txd->vals[3]);216 217 if (!tx_ring->is_xdp) {218 skb = READ_ONCE(tx_ring->txbufs[i].skb);219 if (skb)220 seq_printf(file, " skb->head=%p skb->data=%p",221 skb->head, skb->data);222 } else {223 xdp = READ_ONCE(tx_ring->txbufs[i].xdp);224 if (xdp)225 seq_printf(file, " xdp->data=%p", xdp->data);226 }227 228 if (tx_ring->txbufs[i].dma_addr)229 seq_printf(file, " dma_addr=%pad",230 &tx_ring->txbufs[i].dma_addr);231 232 if (i == tx_ring->rd_p % txd_cnt)233 seq_puts(file, " H_RD");234 if (i == tx_ring->wr_p % txd_cnt)235 seq_puts(file, " H_WR");236 if (i == d_rd_p % txd_cnt)237 seq_puts(file, " D_RD");238 if (i == d_wr_p % txd_cnt)239 seq_puts(file, " D_WR");240 241 seq_putc(file, '\n');242 }243}244 245#define NFP_NFD3_CFG_CTRL_SUPPORTED \246 (NFP_NET_CFG_CTRL_ENABLE | NFP_NET_CFG_CTRL_PROMISC | \247 NFP_NET_CFG_CTRL_L2BC | NFP_NET_CFG_CTRL_L2MC | \248 NFP_NET_CFG_CTRL_RXCSUM | NFP_NET_CFG_CTRL_TXCSUM | \249 NFP_NET_CFG_CTRL_RXVLAN | NFP_NET_CFG_CTRL_TXVLAN | \250 NFP_NET_CFG_CTRL_RXVLAN_V2 | NFP_NET_CFG_CTRL_RXQINQ | \251 NFP_NET_CFG_CTRL_TXVLAN_V2 | \252 NFP_NET_CFG_CTRL_GATHER | NFP_NET_CFG_CTRL_LSO | \253 NFP_NET_CFG_CTRL_CTAG_FILTER | NFP_NET_CFG_CTRL_CMSG_DATA | \254 NFP_NET_CFG_CTRL_RINGCFG | NFP_NET_CFG_CTRL_RSS | \255 NFP_NET_CFG_CTRL_IRQMOD | NFP_NET_CFG_CTRL_TXRWB | \256 NFP_NET_CFG_CTRL_VEPA | \257 NFP_NET_CFG_CTRL_VXLAN | NFP_NET_CFG_CTRL_NVGRE | \258 NFP_NET_CFG_CTRL_BPF | NFP_NET_CFG_CTRL_LSO2 | \259 NFP_NET_CFG_CTRL_RSS2 | NFP_NET_CFG_CTRL_CSUM_COMPLETE | \260 NFP_NET_CFG_CTRL_LIVE_ADDR)261 262const struct nfp_dp_ops nfp_nfd3_ops = {263 .version = NFP_NFD_VER_NFD3,264 .tx_min_desc_per_pkt = 1,265 .cap_mask = NFP_NFD3_CFG_CTRL_SUPPORTED,266 .dma_mask = DMA_BIT_MASK(40),267 .poll = nfp_nfd3_poll,268 .xsk_poll = nfp_nfd3_xsk_poll,269 .ctrl_poll = nfp_nfd3_ctrl_poll,270 .xmit = nfp_nfd3_tx,271 .ctrl_tx_one = nfp_nfd3_ctrl_tx_one,272 .rx_ring_fill_freelist = nfp_nfd3_rx_ring_fill_freelist,273 .tx_ring_alloc = nfp_nfd3_tx_ring_alloc,274 .tx_ring_reset = nfp_nfd3_tx_ring_reset,275 .tx_ring_free = nfp_nfd3_tx_ring_free,276 .tx_ring_bufs_alloc = nfp_nfd3_tx_ring_bufs_alloc,277 .tx_ring_bufs_free = nfp_nfd3_tx_ring_bufs_free,278 .print_tx_descs = nfp_nfd3_print_tx_descs279};280