467 lines · c
1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)2/* Copyright (C) 2015-2019 Netronome Systems, Inc. */3 4#include "nfp_app.h"5#include "nfp_net_dp.h"6#include "nfp_net_xsk.h"7 8/**9 * nfp_net_rx_alloc_one() - Allocate and map page frag for RX10 * @dp: NFP Net data path struct11 * @dma_addr: Pointer to storage for DMA address (output param)12 *13 * This function will allcate a new page frag, map it for DMA.14 *15 * Return: allocated page frag or NULL on failure.16 */17void *nfp_net_rx_alloc_one(struct nfp_net_dp *dp, dma_addr_t *dma_addr)18{19 void *frag;20 21 if (!dp->xdp_prog) {22 frag = netdev_alloc_frag(dp->fl_bufsz);23 } else {24 struct page *page;25 26 page = alloc_page(GFP_KERNEL);27 frag = page ? page_address(page) : NULL;28 }29 if (!frag) {30 nn_dp_warn(dp, "Failed to alloc receive page frag\n");31 return NULL;32 }33 34 *dma_addr = nfp_net_dma_map_rx(dp, frag);35 if (dma_mapping_error(dp->dev, *dma_addr)) {36 nfp_net_free_frag(frag, dp->xdp_prog);37 nn_dp_warn(dp, "Failed to map DMA RX buffer\n");38 return NULL;39 }40 41 return frag;42}43 44/**45 * nfp_net_tx_ring_init() - Fill in the boilerplate for a TX ring46 * @tx_ring: TX ring structure47 * @dp: NFP Net data path struct48 * @r_vec: IRQ vector servicing this ring49 * @idx: Ring index50 * @is_xdp: Is this an XDP TX ring?51 */52static void53nfp_net_tx_ring_init(struct nfp_net_tx_ring *tx_ring, struct nfp_net_dp *dp,54 struct nfp_net_r_vector *r_vec, unsigned int idx,55 bool is_xdp)56{57 struct nfp_net *nn = r_vec->nfp_net;58 59 tx_ring->idx = idx;60 tx_ring->r_vec = r_vec;61 tx_ring->is_xdp = is_xdp;62 u64_stats_init(&tx_ring->r_vec->tx_sync);63 64 tx_ring->qcidx = tx_ring->idx * nn->stride_tx;65 tx_ring->txrwb = dp->txrwb ? &dp->txrwb[idx] : NULL;66 tx_ring->qcp_q = nn->tx_bar + NFP_QCP_QUEUE_OFF(tx_ring->qcidx);67}68 69/**70 * nfp_net_rx_ring_init() - Fill in the boilerplate for a RX ring71 * @rx_ring: RX ring structure72 * @r_vec: IRQ vector servicing this ring73 * @idx: Ring index74 */75static void76nfp_net_rx_ring_init(struct nfp_net_rx_ring *rx_ring,77 struct nfp_net_r_vector *r_vec, unsigned int idx)78{79 struct nfp_net *nn = r_vec->nfp_net;80 81 rx_ring->idx = idx;82 rx_ring->r_vec = r_vec;83 u64_stats_init(&rx_ring->r_vec->rx_sync);84 85 rx_ring->fl_qcidx = rx_ring->idx * nn->stride_rx;86 rx_ring->qcp_fl = nn->rx_bar + NFP_QCP_QUEUE_OFF(rx_ring->fl_qcidx);87}88 89/**90 * nfp_net_rx_ring_reset() - Reflect in SW state of freelist after disable91 * @rx_ring: RX ring structure92 *93 * Assumes that the device is stopped, must be idempotent.94 */95void nfp_net_rx_ring_reset(struct nfp_net_rx_ring *rx_ring)96{97 unsigned int wr_idx, last_idx;98 99 /* wr_p == rd_p means ring was never fed FL bufs. RX rings are always100 * kept at cnt - 1 FL bufs.101 */102 if (rx_ring->wr_p == 0 && rx_ring->rd_p == 0)103 return;104 105 /* Move the empty entry to the end of the list */106 wr_idx = D_IDX(rx_ring, rx_ring->wr_p);107 last_idx = rx_ring->cnt - 1;108 if (rx_ring->r_vec->xsk_pool) {109 rx_ring->xsk_rxbufs[wr_idx] = rx_ring->xsk_rxbufs[last_idx];110 memset(&rx_ring->xsk_rxbufs[last_idx], 0,111 sizeof(*rx_ring->xsk_rxbufs));112 } else {113 rx_ring->rxbufs[wr_idx] = rx_ring->rxbufs[last_idx];114 memset(&rx_ring->rxbufs[last_idx], 0, sizeof(*rx_ring->rxbufs));115 }116 117 memset(rx_ring->rxds, 0, rx_ring->size);118 rx_ring->wr_p = 0;119 rx_ring->rd_p = 0;120}121 122/**123 * nfp_net_rx_ring_bufs_free() - Free any buffers currently on the RX ring124 * @dp: NFP Net data path struct125 * @rx_ring: RX ring to remove buffers from126 *127 * Assumes that the device is stopped and buffers are in [0, ring->cnt - 1)128 * entries. After device is disabled nfp_net_rx_ring_reset() must be called129 * to restore required ring geometry.130 */131static void132nfp_net_rx_ring_bufs_free(struct nfp_net_dp *dp,133 struct nfp_net_rx_ring *rx_ring)134{135 unsigned int i;136 137 if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx))138 return;139 140 for (i = 0; i < rx_ring->cnt - 1; i++) {141 /* NULL skb can only happen when initial filling of the ring142 * fails to allocate enough buffers and calls here to free143 * already allocated ones.144 */145 if (!rx_ring->rxbufs[i].frag)146 continue;147 148 nfp_net_dma_unmap_rx(dp, rx_ring->rxbufs[i].dma_addr);149 nfp_net_free_frag(rx_ring->rxbufs[i].frag, dp->xdp_prog);150 rx_ring->rxbufs[i].dma_addr = 0;151 rx_ring->rxbufs[i].frag = NULL;152 }153}154 155/**156 * nfp_net_rx_ring_bufs_alloc() - Fill RX ring with buffers (don't give to FW)157 * @dp: NFP Net data path struct158 * @rx_ring: RX ring to remove buffers from159 */160static int161nfp_net_rx_ring_bufs_alloc(struct nfp_net_dp *dp,162 struct nfp_net_rx_ring *rx_ring)163{164 struct nfp_net_rx_buf *rxbufs;165 unsigned int i;166 167 if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx))168 return 0;169 170 rxbufs = rx_ring->rxbufs;171 172 for (i = 0; i < rx_ring->cnt - 1; i++) {173 rxbufs[i].frag = nfp_net_rx_alloc_one(dp, &rxbufs[i].dma_addr);174 if (!rxbufs[i].frag) {175 nfp_net_rx_ring_bufs_free(dp, rx_ring);176 return -ENOMEM;177 }178 }179 180 return 0;181}182 183int nfp_net_tx_rings_prepare(struct nfp_net *nn, struct nfp_net_dp *dp)184{185 unsigned int r;186 187 dp->tx_rings = kcalloc(dp->num_tx_rings, sizeof(*dp->tx_rings),188 GFP_KERNEL);189 if (!dp->tx_rings)190 return -ENOMEM;191 192 if (dp->ctrl & NFP_NET_CFG_CTRL_TXRWB) {193 dp->txrwb = dma_alloc_coherent(dp->dev,194 dp->num_tx_rings * sizeof(u64),195 &dp->txrwb_dma, GFP_KERNEL);196 if (!dp->txrwb)197 goto err_free_rings;198 }199 200 for (r = 0; r < dp->num_tx_rings; r++) {201 int bias = 0;202 203 if (r >= dp->num_stack_tx_rings)204 bias = dp->num_stack_tx_rings;205 206 nfp_net_tx_ring_init(&dp->tx_rings[r], dp,207 &nn->r_vecs[r - bias], r, bias);208 209 if (nfp_net_tx_ring_alloc(dp, &dp->tx_rings[r]))210 goto err_free_prev;211 212 if (nfp_net_tx_ring_bufs_alloc(dp, &dp->tx_rings[r]))213 goto err_free_ring;214 }215 216 return 0;217 218err_free_prev:219 while (r--) {220 nfp_net_tx_ring_bufs_free(dp, &dp->tx_rings[r]);221err_free_ring:222 nfp_net_tx_ring_free(dp, &dp->tx_rings[r]);223 }224 if (dp->txrwb)225 dma_free_coherent(dp->dev, dp->num_tx_rings * sizeof(u64),226 dp->txrwb, dp->txrwb_dma);227err_free_rings:228 kfree(dp->tx_rings);229 return -ENOMEM;230}231 232void nfp_net_tx_rings_free(struct nfp_net_dp *dp)233{234 unsigned int r;235 236 for (r = 0; r < dp->num_tx_rings; r++) {237 nfp_net_tx_ring_bufs_free(dp, &dp->tx_rings[r]);238 nfp_net_tx_ring_free(dp, &dp->tx_rings[r]);239 }240 241 if (dp->txrwb)242 dma_free_coherent(dp->dev, dp->num_tx_rings * sizeof(u64),243 dp->txrwb, dp->txrwb_dma);244 kfree(dp->tx_rings);245}246 247/**248 * nfp_net_rx_ring_free() - Free resources allocated to a RX ring249 * @rx_ring: RX ring to free250 */251static void nfp_net_rx_ring_free(struct nfp_net_rx_ring *rx_ring)252{253 struct nfp_net_r_vector *r_vec = rx_ring->r_vec;254 struct nfp_net_dp *dp = &r_vec->nfp_net->dp;255 256 if (dp->netdev)257 xdp_rxq_info_unreg(&rx_ring->xdp_rxq);258 259 if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx))260 kvfree(rx_ring->xsk_rxbufs);261 else262 kvfree(rx_ring->rxbufs);263 264 if (rx_ring->rxds)265 dma_free_coherent(dp->dev, rx_ring->size,266 rx_ring->rxds, rx_ring->dma);267 268 rx_ring->cnt = 0;269 rx_ring->rxbufs = NULL;270 rx_ring->xsk_rxbufs = NULL;271 rx_ring->rxds = NULL;272 rx_ring->dma = 0;273 rx_ring->size = 0;274}275 276/**277 * nfp_net_rx_ring_alloc() - Allocate resource for a RX ring278 * @dp: NFP Net data path struct279 * @rx_ring: RX ring to allocate280 *281 * Return: 0 on success, negative errno otherwise.282 */283static int284nfp_net_rx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_rx_ring *rx_ring)285{286 enum xdp_mem_type mem_type;287 size_t rxbuf_sw_desc_sz;288 int err;289 290 if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx)) {291 mem_type = MEM_TYPE_XSK_BUFF_POOL;292 rxbuf_sw_desc_sz = sizeof(*rx_ring->xsk_rxbufs);293 } else {294 mem_type = MEM_TYPE_PAGE_ORDER0;295 rxbuf_sw_desc_sz = sizeof(*rx_ring->rxbufs);296 }297 298 if (dp->netdev) {299 err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, dp->netdev,300 rx_ring->idx, rx_ring->r_vec->napi.napi_id);301 if (err < 0)302 return err;303 304 err = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, mem_type, NULL);305 if (err)306 goto err_alloc;307 }308 309 rx_ring->cnt = dp->rxd_cnt;310 rx_ring->size = array_size(rx_ring->cnt, sizeof(*rx_ring->rxds));311 rx_ring->rxds = dma_alloc_coherent(dp->dev, rx_ring->size,312 &rx_ring->dma,313 GFP_KERNEL | __GFP_NOWARN);314 if (!rx_ring->rxds) {315 netdev_warn(dp->netdev, "failed to allocate RX descriptor ring memory, requested descriptor count: %d, consider lowering descriptor count\n",316 rx_ring->cnt);317 goto err_alloc;318 }319 320 if (nfp_net_has_xsk_pool_slow(dp, rx_ring->idx)) {321 rx_ring->xsk_rxbufs = kvcalloc(rx_ring->cnt, rxbuf_sw_desc_sz,322 GFP_KERNEL);323 if (!rx_ring->xsk_rxbufs)324 goto err_alloc;325 } else {326 rx_ring->rxbufs = kvcalloc(rx_ring->cnt, rxbuf_sw_desc_sz,327 GFP_KERNEL);328 if (!rx_ring->rxbufs)329 goto err_alloc;330 }331 332 return 0;333 334err_alloc:335 nfp_net_rx_ring_free(rx_ring);336 return -ENOMEM;337}338 339int nfp_net_rx_rings_prepare(struct nfp_net *nn, struct nfp_net_dp *dp)340{341 unsigned int r;342 343 dp->rx_rings = kcalloc(dp->num_rx_rings, sizeof(*dp->rx_rings),344 GFP_KERNEL);345 if (!dp->rx_rings)346 return -ENOMEM;347 348 for (r = 0; r < dp->num_rx_rings; r++) {349 nfp_net_rx_ring_init(&dp->rx_rings[r], &nn->r_vecs[r], r);350 351 if (nfp_net_rx_ring_alloc(dp, &dp->rx_rings[r]))352 goto err_free_prev;353 354 if (nfp_net_rx_ring_bufs_alloc(dp, &dp->rx_rings[r]))355 goto err_free_ring;356 }357 358 return 0;359 360err_free_prev:361 while (r--) {362 nfp_net_rx_ring_bufs_free(dp, &dp->rx_rings[r]);363err_free_ring:364 nfp_net_rx_ring_free(&dp->rx_rings[r]);365 }366 kfree(dp->rx_rings);367 return -ENOMEM;368}369 370void nfp_net_rx_rings_free(struct nfp_net_dp *dp)371{372 unsigned int r;373 374 for (r = 0; r < dp->num_rx_rings; r++) {375 nfp_net_rx_ring_bufs_free(dp, &dp->rx_rings[r]);376 nfp_net_rx_ring_free(&dp->rx_rings[r]);377 }378 379 kfree(dp->rx_rings);380}381 382void383nfp_net_rx_ring_hw_cfg_write(struct nfp_net *nn,384 struct nfp_net_rx_ring *rx_ring, unsigned int idx)385{386 /* Write the DMA address, size and MSI-X info to the device */387 nn_writeq(nn, NFP_NET_CFG_RXR_ADDR(idx), rx_ring->dma);388 nn_writeb(nn, NFP_NET_CFG_RXR_SZ(idx), ilog2(rx_ring->cnt));389 nn_writeb(nn, NFP_NET_CFG_RXR_VEC(idx), rx_ring->r_vec->irq_entry);390}391 392void393nfp_net_tx_ring_hw_cfg_write(struct nfp_net *nn,394 struct nfp_net_tx_ring *tx_ring, unsigned int idx)395{396 nn_writeq(nn, NFP_NET_CFG_TXR_ADDR(idx), tx_ring->dma);397 if (tx_ring->txrwb) {398 *tx_ring->txrwb = 0;399 nn_writeq(nn, NFP_NET_CFG_TXR_WB_ADDR(idx),400 nn->dp.txrwb_dma + idx * sizeof(u64));401 }402 nn_writeb(nn, NFP_NET_CFG_TXR_SZ(idx), ilog2(tx_ring->cnt));403 nn_writeb(nn, NFP_NET_CFG_TXR_VEC(idx), tx_ring->r_vec->irq_entry);404}405 406void nfp_net_vec_clear_ring_data(struct nfp_net *nn, unsigned int idx)407{408 nn_writeq(nn, NFP_NET_CFG_RXR_ADDR(idx), 0);409 nn_writeb(nn, NFP_NET_CFG_RXR_SZ(idx), 0);410 nn_writeb(nn, NFP_NET_CFG_RXR_VEC(idx), 0);411 412 nn_writeq(nn, NFP_NET_CFG_TXR_ADDR(idx), 0);413 nn_writeq(nn, NFP_NET_CFG_TXR_WB_ADDR(idx), 0);414 nn_writeb(nn, NFP_NET_CFG_TXR_SZ(idx), 0);415 nn_writeb(nn, NFP_NET_CFG_TXR_VEC(idx), 0);416}417 418netdev_tx_t nfp_net_tx(struct sk_buff *skb, struct net_device *netdev)419{420 struct nfp_net *nn = netdev_priv(netdev);421 422 return nn->dp.ops->xmit(skb, netdev);423}424 425bool __nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb)426{427 struct nfp_net_r_vector *r_vec = &nn->r_vecs[0];428 429 return nn->dp.ops->ctrl_tx_one(nn, r_vec, skb, false);430}431 432bool nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb)433{434 struct nfp_net_r_vector *r_vec = &nn->r_vecs[0];435 bool ret;436 437 spin_lock_bh(&r_vec->lock);438 ret = nn->dp.ops->ctrl_tx_one(nn, r_vec, skb, false);439 spin_unlock_bh(&r_vec->lock);440 441 return ret;442}443 444bool nfp_net_vlan_strip(struct sk_buff *skb, const struct nfp_net_rx_desc *rxd,445 const struct nfp_meta_parsed *meta)446{447 u16 tpid = 0, tci = 0;448 449 if (rxd->rxd.flags & PCIE_DESC_RX_VLAN) {450 tpid = ETH_P_8021Q;451 tci = le16_to_cpu(rxd->rxd.vlan);452 } else if (meta->vlan.stripped) {453 if (meta->vlan.tpid == NFP_NET_VLAN_CTAG)454 tpid = ETH_P_8021Q;455 else if (meta->vlan.tpid == NFP_NET_VLAN_STAG)456 tpid = ETH_P_8021AD;457 else458 return false;459 460 tci = meta->vlan.tci;461 }462 if (tpid)463 __vlan_hwaccel_put_tag(skb, htons(tpid), tci);464 465 return true;466}467