152 lines · c
1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */2/*3 * Copyright 2015-2021 Amazon.com, Inc. or its affiliates. All rights reserved.4 */5 6#ifndef ENA_XDP_H7#define ENA_XDP_H8 9#include "ena_netdev.h"10#include <linux/bpf_trace.h>11 12/* The max MTU size is configured to be the ethernet frame size without13 * the overhead of the ethernet header, which can have a VLAN header, and14 * a frame check sequence (FCS).15 * The buffer size we share with the device is defined to be ENA_PAGE_SIZE16 */17#define ENA_XDP_MAX_MTU (ENA_PAGE_SIZE - ETH_HLEN - ETH_FCS_LEN - \18 VLAN_HLEN - XDP_PACKET_HEADROOM - \19 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))20 21#define ENA_IS_XDP_INDEX(adapter, index) (((index) >= (adapter)->xdp_first_ring) && \22 ((index) < (adapter)->xdp_first_ring + (adapter)->xdp_num_queues))23 24enum ENA_XDP_ACTIONS {25 ENA_XDP_PASS = 0,26 ENA_XDP_TX = BIT(0),27 ENA_XDP_REDIRECT = BIT(1),28 ENA_XDP_DROP = BIT(2)29};30 31#define ENA_XDP_FORWARDED (ENA_XDP_TX | ENA_XDP_REDIRECT)32 33int ena_setup_and_create_all_xdp_queues(struct ena_adapter *adapter);34void ena_xdp_exchange_program_rx_in_range(struct ena_adapter *adapter,35 struct bpf_prog *prog,36 int first, int count);37int ena_xdp_io_poll(struct napi_struct *napi, int budget);38int ena_xdp_xmit_frame(struct ena_ring *tx_ring,39 struct ena_adapter *adapter,40 struct xdp_frame *xdpf,41 int flags);42int ena_xdp_xmit(struct net_device *dev, int n,43 struct xdp_frame **frames, u32 flags);44int ena_xdp(struct net_device *netdev, struct netdev_bpf *bpf);45int ena_xdp_register_rxq_info(struct ena_ring *rx_ring);46void ena_xdp_unregister_rxq_info(struct ena_ring *rx_ring);47 48enum ena_xdp_errors_t {49 ENA_XDP_ALLOWED = 0,50 ENA_XDP_CURRENT_MTU_TOO_LARGE,51 ENA_XDP_NO_ENOUGH_QUEUES,52};53 54static inline bool ena_xdp_present(struct ena_adapter *adapter)55{56 return !!adapter->xdp_bpf_prog;57}58 59static inline bool ena_xdp_present_ring(struct ena_ring *ring)60{61 return !!ring->xdp_bpf_prog;62}63 64static inline bool ena_xdp_legal_queue_count(struct ena_adapter *adapter,65 u32 queues)66{67 return 2 * queues <= adapter->max_num_io_queues;68}69 70static inline enum ena_xdp_errors_t ena_xdp_allowed(struct ena_adapter *adapter)71{72 enum ena_xdp_errors_t rc = ENA_XDP_ALLOWED;73 74 if (adapter->netdev->mtu > ENA_XDP_MAX_MTU)75 rc = ENA_XDP_CURRENT_MTU_TOO_LARGE;76 else if (!ena_xdp_legal_queue_count(adapter, adapter->num_io_queues))77 rc = ENA_XDP_NO_ENOUGH_QUEUES;78 79 return rc;80}81 82static inline int ena_xdp_execute(struct ena_ring *rx_ring, struct xdp_buff *xdp)83{84 u32 verdict = ENA_XDP_PASS;85 struct bpf_prog *xdp_prog;86 struct ena_ring *xdp_ring;87 struct xdp_frame *xdpf;88 u64 *xdp_stat;89 90 xdp_prog = READ_ONCE(rx_ring->xdp_bpf_prog);91 92 verdict = bpf_prog_run_xdp(xdp_prog, xdp);93 94 switch (verdict) {95 case XDP_TX:96 xdpf = xdp_convert_buff_to_frame(xdp);97 if (unlikely(!xdpf)) {98 trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict);99 xdp_stat = &rx_ring->rx_stats.xdp_aborted;100 verdict = ENA_XDP_DROP;101 break;102 }103 104 /* Find xmit queue */105 xdp_ring = rx_ring->xdp_ring;106 107 /* The XDP queues are shared between XDP_TX and XDP_REDIRECT */108 spin_lock(&xdp_ring->xdp_tx_lock);109 110 if (ena_xdp_xmit_frame(xdp_ring, rx_ring->adapter, xdpf,111 XDP_XMIT_FLUSH))112 xdp_return_frame(xdpf);113 114 spin_unlock(&xdp_ring->xdp_tx_lock);115 xdp_stat = &rx_ring->rx_stats.xdp_tx;116 verdict = ENA_XDP_TX;117 break;118 case XDP_REDIRECT:119 if (likely(!xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog))) {120 xdp_stat = &rx_ring->rx_stats.xdp_redirect;121 verdict = ENA_XDP_REDIRECT;122 break;123 }124 trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict);125 xdp_stat = &rx_ring->rx_stats.xdp_aborted;126 verdict = ENA_XDP_DROP;127 break;128 case XDP_ABORTED:129 trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict);130 xdp_stat = &rx_ring->rx_stats.xdp_aborted;131 verdict = ENA_XDP_DROP;132 break;133 case XDP_DROP:134 xdp_stat = &rx_ring->rx_stats.xdp_drop;135 verdict = ENA_XDP_DROP;136 break;137 case XDP_PASS:138 xdp_stat = &rx_ring->rx_stats.xdp_pass;139 verdict = ENA_XDP_PASS;140 break;141 default:142 bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, verdict);143 xdp_stat = &rx_ring->rx_stats.xdp_invalid;144 verdict = ENA_XDP_DROP;145 }146 147 ena_increase_stat(xdp_stat, 1, &rx_ring->syncp);148 149 return verdict;150}151#endif /* ENA_XDP_H */152