227 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause2/* Copyright (c) 2021, Microsoft Corporation. */3 4#include <linux/inetdevice.h>5#include <linux/etherdevice.h>6#include <linux/mm.h>7#include <linux/bpf.h>8#include <linux/bpf_trace.h>9#include <net/xdp.h>10 11#include <net/mana/mana.h>12 13void mana_xdp_tx(struct sk_buff *skb, struct net_device *ndev)14{15 u16 txq_idx = skb_get_queue_mapping(skb);16 struct netdev_queue *ndevtxq;17 int rc;18 19 __skb_push(skb, ETH_HLEN);20 21 ndevtxq = netdev_get_tx_queue(ndev, txq_idx);22 __netif_tx_lock(ndevtxq, smp_processor_id());23 24 rc = mana_start_xmit(skb, ndev);25 26 __netif_tx_unlock(ndevtxq);27 28 if (dev_xmit_complete(rc))29 return;30 31 dev_kfree_skb_any(skb);32 ndev->stats.tx_dropped++;33}34 35static int mana_xdp_xmit_fm(struct net_device *ndev, struct xdp_frame *frame,36 u16 q_idx)37{38 struct sk_buff *skb;39 40 skb = xdp_build_skb_from_frame(frame, ndev);41 if (unlikely(!skb))42 return -ENOMEM;43 44 skb_set_queue_mapping(skb, q_idx);45 46 mana_xdp_tx(skb, ndev);47 48 return 0;49}50 51int mana_xdp_xmit(struct net_device *ndev, int n, struct xdp_frame **frames,52 u32 flags)53{54 struct mana_port_context *apc = netdev_priv(ndev);55 struct mana_stats_tx *tx_stats;56 int i, count = 0;57 u16 q_idx;58 59 if (unlikely(!apc->port_is_up))60 return 0;61 62 q_idx = smp_processor_id() % ndev->real_num_tx_queues;63 64 for (i = 0; i < n; i++) {65 if (mana_xdp_xmit_fm(ndev, frames[i], q_idx))66 break;67 68 count++;69 }70 71 tx_stats = &apc->tx_qp[q_idx].txq.stats;72 73 u64_stats_update_begin(&tx_stats->syncp);74 tx_stats->xdp_xmit += count;75 u64_stats_update_end(&tx_stats->syncp);76 77 return count;78}79 80u32 mana_run_xdp(struct net_device *ndev, struct mana_rxq *rxq,81 struct xdp_buff *xdp, void *buf_va, uint pkt_len)82{83 struct mana_stats_rx *rx_stats;84 struct bpf_prog *prog;85 u32 act = XDP_PASS;86 87 rcu_read_lock();88 prog = rcu_dereference(rxq->bpf_prog);89 90 if (!prog)91 goto out;92 93 xdp_init_buff(xdp, PAGE_SIZE, &rxq->xdp_rxq);94 xdp_prepare_buff(xdp, buf_va, XDP_PACKET_HEADROOM, pkt_len, false);95 96 act = bpf_prog_run_xdp(prog, xdp);97 98 rx_stats = &rxq->stats;99 100 switch (act) {101 case XDP_PASS:102 case XDP_TX:103 case XDP_DROP:104 break;105 106 case XDP_REDIRECT:107 rxq->xdp_rc = xdp_do_redirect(ndev, xdp, prog);108 if (!rxq->xdp_rc) {109 rxq->xdp_flush = true;110 111 u64_stats_update_begin(&rx_stats->syncp);112 rx_stats->packets++;113 rx_stats->bytes += pkt_len;114 rx_stats->xdp_redirect++;115 u64_stats_update_end(&rx_stats->syncp);116 117 break;118 }119 120 fallthrough;121 122 case XDP_ABORTED:123 trace_xdp_exception(ndev, prog, act);124 break;125 126 default:127 bpf_warn_invalid_xdp_action(ndev, prog, act);128 }129 130out:131 rcu_read_unlock();132 133 return act;134}135 136struct bpf_prog *mana_xdp_get(struct mana_port_context *apc)137{138 ASSERT_RTNL();139 140 return apc->bpf_prog;141}142 143static struct bpf_prog *mana_chn_xdp_get(struct mana_port_context *apc)144{145 return rtnl_dereference(apc->rxqs[0]->bpf_prog);146}147 148/* Set xdp program on channels */149void mana_chn_setxdp(struct mana_port_context *apc, struct bpf_prog *prog)150{151 struct bpf_prog *old_prog = mana_chn_xdp_get(apc);152 unsigned int num_queues = apc->num_queues;153 int i;154 155 ASSERT_RTNL();156 157 if (old_prog == prog)158 return;159 160 if (prog)161 bpf_prog_add(prog, num_queues);162 163 for (i = 0; i < num_queues; i++)164 rcu_assign_pointer(apc->rxqs[i]->bpf_prog, prog);165 166 if (old_prog)167 for (i = 0; i < num_queues; i++)168 bpf_prog_put(old_prog);169}170 171static int mana_xdp_set(struct net_device *ndev, struct bpf_prog *prog,172 struct netlink_ext_ack *extack)173{174 struct mana_port_context *apc = netdev_priv(ndev);175 struct bpf_prog *old_prog;176 struct gdma_context *gc;177 178 gc = apc->ac->gdma_dev->gdma_context;179 180 old_prog = mana_xdp_get(apc);181 182 if (!old_prog && !prog)183 return 0;184 185 if (prog && ndev->mtu > MANA_XDP_MTU_MAX) {186 netdev_err(ndev, "XDP: mtu:%u too large, mtu_max:%lu\n",187 ndev->mtu, MANA_XDP_MTU_MAX);188 NL_SET_ERR_MSG_MOD(extack, "XDP: mtu too large");189 190 return -EOPNOTSUPP;191 }192 193 /* One refcnt of the prog is hold by the caller already, so194 * don't increase refcnt for this one.195 */196 apc->bpf_prog = prog;197 198 if (old_prog)199 bpf_prog_put(old_prog);200 201 if (apc->port_is_up)202 mana_chn_setxdp(apc, prog);203 204 if (prog)205 ndev->max_mtu = MANA_XDP_MTU_MAX;206 else207 ndev->max_mtu = gc->adapter_mtu - ETH_HLEN;208 209 return 0;210}211 212int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf)213{214 struct netlink_ext_ack *extack = bpf->extack;215 int ret;216 217 switch (bpf->command) {218 case XDP_SETUP_PROG:219 return mana_xdp_set(ndev, bpf->prog, extack);220 221 default:222 return -EOPNOTSUPP;223 }224 225 return ret;226}227