455 lines · c
1// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)2/* Copyright 2022 NXP3 */4#include <linux/filter.h>5#include <linux/compiler.h>6#include <linux/bpf_trace.h>7#include <net/xdp.h>8#include <net/xdp_sock_drv.h>9 10#include "dpaa2-eth.h"11 12static void dpaa2_eth_setup_consume_func(struct dpaa2_eth_priv *priv,13 struct dpaa2_eth_channel *ch,14 enum dpaa2_eth_fq_type type,15 dpaa2_eth_consume_cb_t *consume)16{17 struct dpaa2_eth_fq *fq;18 int i;19 20 for (i = 0; i < priv->num_fqs; i++) {21 fq = &priv->fq[i];22 23 if (fq->type != type)24 continue;25 if (fq->channel != ch)26 continue;27 28 fq->consume = consume;29 }30}31 32static u32 dpaa2_xsk_run_xdp(struct dpaa2_eth_priv *priv,33 struct dpaa2_eth_channel *ch,34 struct dpaa2_eth_fq *rx_fq,35 struct dpaa2_fd *fd, void *vaddr)36{37 dma_addr_t addr = dpaa2_fd_get_addr(fd);38 struct bpf_prog *xdp_prog;39 struct xdp_buff *xdp_buff;40 struct dpaa2_eth_swa *swa;41 u32 xdp_act = XDP_PASS;42 int err;43 44 xdp_prog = READ_ONCE(ch->xdp.prog);45 if (!xdp_prog)46 goto out;47 48 swa = (struct dpaa2_eth_swa *)(vaddr + DPAA2_ETH_RX_HWA_SIZE +49 ch->xsk_pool->umem->headroom);50 xdp_buff = swa->xsk.xdp_buff;51 52 xdp_buff->data_hard_start = vaddr;53 xdp_buff->data = vaddr + dpaa2_fd_get_offset(fd);54 xdp_buff->data_end = xdp_buff->data + dpaa2_fd_get_len(fd);55 xdp_set_data_meta_invalid(xdp_buff);56 xdp_buff->rxq = &ch->xdp_rxq;57 58 xsk_buff_dma_sync_for_cpu(xdp_buff);59 xdp_act = bpf_prog_run_xdp(xdp_prog, xdp_buff);60 61 /* xdp.data pointer may have changed */62 dpaa2_fd_set_offset(fd, xdp_buff->data - vaddr);63 dpaa2_fd_set_len(fd, xdp_buff->data_end - xdp_buff->data);64 65 if (likely(xdp_act == XDP_REDIRECT)) {66 err = xdp_do_redirect(priv->net_dev, xdp_buff, xdp_prog);67 if (unlikely(err)) {68 ch->stats.xdp_drop++;69 dpaa2_eth_recycle_buf(priv, ch, addr);70 } else {71 ch->buf_count--;72 ch->stats.xdp_redirect++;73 }74 75 goto xdp_redir;76 }77 78 switch (xdp_act) {79 case XDP_PASS:80 break;81 case XDP_TX:82 dpaa2_eth_xdp_enqueue(priv, ch, fd, vaddr, rx_fq->flowid);83 break;84 default:85 bpf_warn_invalid_xdp_action(priv->net_dev, xdp_prog, xdp_act);86 fallthrough;87 case XDP_ABORTED:88 trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act);89 fallthrough;90 case XDP_DROP:91 dpaa2_eth_recycle_buf(priv, ch, addr);92 ch->stats.xdp_drop++;93 break;94 }95 96xdp_redir:97 ch->xdp.res |= xdp_act;98out:99 return xdp_act;100}101 102/* Rx frame processing routine for the AF_XDP fast path */103static void dpaa2_xsk_rx(struct dpaa2_eth_priv *priv,104 struct dpaa2_eth_channel *ch,105 const struct dpaa2_fd *fd,106 struct dpaa2_eth_fq *fq)107{108 dma_addr_t addr = dpaa2_fd_get_addr(fd);109 u8 fd_format = dpaa2_fd_get_format(fd);110 struct rtnl_link_stats64 *percpu_stats;111 u32 fd_length = dpaa2_fd_get_len(fd);112 struct sk_buff *skb;113 void *vaddr;114 u32 xdp_act;115 116 trace_dpaa2_rx_xsk_fd(priv->net_dev, fd);117 118 vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr);119 percpu_stats = this_cpu_ptr(priv->percpu_stats);120 121 if (fd_format != dpaa2_fd_single) {122 WARN_ON(priv->xdp_prog);123 /* AF_XDP doesn't support any other formats */124 goto err_frame_format;125 }126 127 xdp_act = dpaa2_xsk_run_xdp(priv, ch, fq, (struct dpaa2_fd *)fd, vaddr);128 if (xdp_act != XDP_PASS) {129 percpu_stats->rx_packets++;130 percpu_stats->rx_bytes += dpaa2_fd_get_len(fd);131 return;132 }133 134 /* Build skb */135 skb = dpaa2_eth_alloc_skb(priv, ch, fd, fd_length, vaddr);136 if (!skb)137 /* Nothing else we can do, recycle the buffer and138 * drop the frame.139 */140 goto err_alloc_skb;141 142 /* Send the skb to the Linux networking stack */143 dpaa2_eth_receive_skb(priv, ch, fd, vaddr, fq, percpu_stats, skb);144 145 return;146 147err_alloc_skb:148 dpaa2_eth_recycle_buf(priv, ch, addr);149err_frame_format:150 percpu_stats->rx_dropped++;151}152 153static void dpaa2_xsk_set_bp_per_qdbin(struct dpaa2_eth_priv *priv,154 struct dpni_pools_cfg *pools_params)155{156 int curr_bp = 0, i, j;157 158 pools_params->pool_options = DPNI_POOL_ASSOC_QDBIN;159 for (i = 0; i < priv->num_bps; i++) {160 for (j = 0; j < priv->num_channels; j++)161 if (priv->bp[i] == priv->channel[j]->bp)162 pools_params->pools[curr_bp].priority_mask |= (1 << j);163 if (!pools_params->pools[curr_bp].priority_mask)164 continue;165 166 pools_params->pools[curr_bp].dpbp_id = priv->bp[i]->bpid;167 pools_params->pools[curr_bp].buffer_size = priv->rx_buf_size;168 pools_params->pools[curr_bp++].backup_pool = 0;169 }170 pools_params->num_dpbp = curr_bp;171}172 173static int dpaa2_xsk_disable_pool(struct net_device *dev, u16 qid)174{175 struct xsk_buff_pool *pool = xsk_get_pool_from_qid(dev, qid);176 struct dpaa2_eth_priv *priv = netdev_priv(dev);177 struct dpni_pools_cfg pools_params = { 0 };178 struct dpaa2_eth_channel *ch;179 int err;180 bool up;181 182 ch = priv->channel[qid];183 if (!ch->xsk_pool)184 return -EINVAL;185 186 up = netif_running(dev);187 if (up)188 dev_close(dev);189 190 xsk_pool_dma_unmap(pool, 0);191 err = xdp_rxq_info_reg_mem_model(&ch->xdp_rxq,192 MEM_TYPE_PAGE_ORDER0, NULL);193 if (err)194 netdev_err(dev, "xsk_rxq_info_reg_mem_model() failed (err = %d)\n",195 err);196 197 dpaa2_eth_free_dpbp(priv, ch->bp);198 199 ch->xsk_zc = false;200 ch->xsk_pool = NULL;201 ch->xsk_tx_pkts_sent = 0;202 ch->bp = priv->bp[DPAA2_ETH_DEFAULT_BP_IDX];203 204 dpaa2_eth_setup_consume_func(priv, ch, DPAA2_RX_FQ, dpaa2_eth_rx);205 206 dpaa2_xsk_set_bp_per_qdbin(priv, &pools_params);207 err = dpni_set_pools(priv->mc_io, 0, priv->mc_token, &pools_params);208 if (err)209 netdev_err(dev, "dpni_set_pools() failed\n");210 211 if (up) {212 err = dev_open(dev, NULL);213 if (err)214 return err;215 }216 217 return 0;218}219 220static int dpaa2_xsk_enable_pool(struct net_device *dev,221 struct xsk_buff_pool *pool,222 u16 qid)223{224 struct dpaa2_eth_priv *priv = netdev_priv(dev);225 struct dpni_pools_cfg pools_params = { 0 };226 struct dpaa2_eth_channel *ch;227 int err, err2;228 bool up;229 230 if (priv->dpni_attrs.wriop_version < DPAA2_WRIOP_VERSION(3, 0, 0)) {231 netdev_err(dev, "AF_XDP zero-copy not supported on devices <= WRIOP(3, 0, 0)\n");232 return -EOPNOTSUPP;233 }234 235 if (priv->dpni_attrs.num_queues > 8) {236 netdev_err(dev, "AF_XDP zero-copy not supported on DPNI with more then 8 queues\n");237 return -EOPNOTSUPP;238 }239 240 up = netif_running(dev);241 if (up)242 dev_close(dev);243 244 err = xsk_pool_dma_map(pool, priv->net_dev->dev.parent, 0);245 if (err) {246 netdev_err(dev, "xsk_pool_dma_map() failed (err = %d)\n",247 err);248 goto err_dma_unmap;249 }250 251 ch = priv->channel[qid];252 err = xdp_rxq_info_reg_mem_model(&ch->xdp_rxq, MEM_TYPE_XSK_BUFF_POOL, NULL);253 if (err) {254 netdev_err(dev, "xdp_rxq_info_reg_mem_model() failed (err = %d)\n", err);255 goto err_mem_model;256 }257 xsk_pool_set_rxq_info(pool, &ch->xdp_rxq);258 259 priv->bp[priv->num_bps] = dpaa2_eth_allocate_dpbp(priv);260 if (IS_ERR(priv->bp[priv->num_bps])) {261 err = PTR_ERR(priv->bp[priv->num_bps]);262 goto err_bp_alloc;263 }264 ch->xsk_zc = true;265 ch->xsk_pool = pool;266 ch->bp = priv->bp[priv->num_bps++];267 268 dpaa2_eth_setup_consume_func(priv, ch, DPAA2_RX_FQ, dpaa2_xsk_rx);269 270 dpaa2_xsk_set_bp_per_qdbin(priv, &pools_params);271 err = dpni_set_pools(priv->mc_io, 0, priv->mc_token, &pools_params);272 if (err) {273 netdev_err(dev, "dpni_set_pools() failed\n");274 goto err_set_pools;275 }276 277 if (up) {278 err = dev_open(dev, NULL);279 if (err)280 return err;281 }282 283 return 0;284 285err_set_pools:286 err2 = dpaa2_xsk_disable_pool(dev, qid);287 if (err2)288 netdev_err(dev, "dpaa2_xsk_disable_pool() failed %d\n", err2);289err_bp_alloc:290 err2 = xdp_rxq_info_reg_mem_model(&priv->channel[qid]->xdp_rxq,291 MEM_TYPE_PAGE_ORDER0, NULL);292 if (err2)293 netdev_err(dev, "xsk_rxq_info_reg_mem_model() failed with %d)\n", err2);294err_mem_model:295 xsk_pool_dma_unmap(pool, 0);296err_dma_unmap:297 if (up)298 dev_open(dev, NULL);299 300 return err;301}302 303int dpaa2_xsk_setup_pool(struct net_device *dev, struct xsk_buff_pool *pool, u16 qid)304{305 return pool ? dpaa2_xsk_enable_pool(dev, pool, qid) :306 dpaa2_xsk_disable_pool(dev, qid);307}308 309int dpaa2_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags)310{311 struct dpaa2_eth_priv *priv = netdev_priv(dev);312 struct dpaa2_eth_channel *ch = priv->channel[qid];313 314 if (!priv->link_state.up)315 return -ENETDOWN;316 317 if (!priv->xdp_prog)318 return -EINVAL;319 320 if (!ch->xsk_zc)321 return -EINVAL;322 323 /* We do not have access to a per channel SW interrupt, so instead we324 * schedule a NAPI instance.325 */326 if (!napi_if_scheduled_mark_missed(&ch->napi))327 napi_schedule(&ch->napi);328 329 return 0;330}331 332static int dpaa2_xsk_tx_build_fd(struct dpaa2_eth_priv *priv,333 struct dpaa2_eth_channel *ch,334 struct dpaa2_fd *fd,335 struct xdp_desc *xdp_desc)336{337 struct device *dev = priv->net_dev->dev.parent;338 struct dpaa2_sg_entry *sgt;339 struct dpaa2_eth_swa *swa;340 void *sgt_buf = NULL;341 dma_addr_t sgt_addr;342 int sgt_buf_size;343 dma_addr_t addr;344 int err = 0;345 346 /* Prepare the HW SGT structure */347 sgt_buf_size = priv->tx_data_offset + sizeof(struct dpaa2_sg_entry);348 sgt_buf = dpaa2_eth_sgt_get(priv);349 if (unlikely(!sgt_buf))350 return -ENOMEM;351 sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset);352 353 /* Get the address of the XSK Tx buffer */354 addr = xsk_buff_raw_get_dma(ch->xsk_pool, xdp_desc->addr);355 xsk_buff_raw_dma_sync_for_device(ch->xsk_pool, addr, xdp_desc->len);356 357 /* Fill in the HW SGT structure */358 dpaa2_sg_set_addr(sgt, addr);359 dpaa2_sg_set_len(sgt, xdp_desc->len);360 dpaa2_sg_set_final(sgt, true);361 362 /* Store the necessary info in the SGT buffer */363 swa = (struct dpaa2_eth_swa *)sgt_buf;364 swa->type = DPAA2_ETH_SWA_XSK;365 swa->xsk.sgt_size = sgt_buf_size;366 367 /* Separately map the SGT buffer */368 sgt_addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL);369 if (unlikely(dma_mapping_error(dev, sgt_addr))) {370 err = -ENOMEM;371 goto sgt_map_failed;372 }373 374 /* Initialize FD fields */375 memset(fd, 0, sizeof(struct dpaa2_fd));376 dpaa2_fd_set_offset(fd, priv->tx_data_offset);377 dpaa2_fd_set_format(fd, dpaa2_fd_sg);378 dpaa2_fd_set_addr(fd, sgt_addr);379 dpaa2_fd_set_len(fd, xdp_desc->len);380 dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA);381 382 return 0;383 384sgt_map_failed:385 dpaa2_eth_sgt_recycle(priv, sgt_buf);386 387 return err;388}389 390bool dpaa2_xsk_tx(struct dpaa2_eth_priv *priv,391 struct dpaa2_eth_channel *ch)392{393 struct xdp_desc *xdp_descs = ch->xsk_pool->tx_descs;394 struct dpaa2_eth_drv_stats *percpu_extras;395 struct rtnl_link_stats64 *percpu_stats;396 int budget = DPAA2_ETH_TX_ZC_PER_NAPI;397 int total_enqueued, enqueued;398 int retries, max_retries;399 struct dpaa2_eth_fq *fq;400 struct dpaa2_fd *fds;401 int batch, i, err;402 403 percpu_stats = this_cpu_ptr(priv->percpu_stats);404 percpu_extras = this_cpu_ptr(priv->percpu_extras);405 fds = (this_cpu_ptr(priv->fd))->array;406 407 /* Use the FQ with the same idx as the affine CPU */408 fq = &priv->fq[ch->nctx.desired_cpu];409 410 batch = xsk_tx_peek_release_desc_batch(ch->xsk_pool, budget);411 if (!batch)412 return false;413 414 /* Create a FD for each XSK frame to be sent */415 for (i = 0; i < batch; i++) {416 err = dpaa2_xsk_tx_build_fd(priv, ch, &fds[i], &xdp_descs[i]);417 if (err) {418 batch = i;419 break;420 }421 422 trace_dpaa2_tx_xsk_fd(priv->net_dev, &fds[i]);423 }424 425 /* Enqueue all the created FDs */426 max_retries = batch * DPAA2_ETH_ENQUEUE_RETRIES;427 total_enqueued = 0;428 enqueued = 0;429 retries = 0;430 while (total_enqueued < batch && retries < max_retries) {431 err = priv->enqueue(priv, fq, &fds[total_enqueued], 0,432 batch - total_enqueued, &enqueued);433 if (err == -EBUSY) {434 retries++;435 continue;436 }437 438 total_enqueued += enqueued;439 }440 percpu_extras->tx_portal_busy += retries;441 442 /* Update statistics */443 percpu_stats->tx_packets += total_enqueued;444 for (i = 0; i < total_enqueued; i++)445 percpu_stats->tx_bytes += dpaa2_fd_get_len(&fds[i]);446 for (i = total_enqueued; i < batch; i++) {447 dpaa2_eth_free_tx_fd(priv, ch, fq, &fds[i], false);448 percpu_stats->tx_errors++;449 }450 451 xsk_tx_release(ch->xsk_pool);452 453 return total_enqueued == budget;454}455