267 lines · c
1// SPDX-License-Identifier: GPL-2.02/* GPLv2, Copyright(c) 2017 Jesper Dangaard Brouer, Red Hat, Inc. */3#include "xdp_sample.bpf.h"4 5#include <bpf/bpf_tracing.h>6#include <bpf/bpf_core_read.h>7#include <bpf/bpf_helpers.h>8 9array_map rx_cnt SEC(".maps");10array_map redir_err_cnt SEC(".maps");11array_map cpumap_enqueue_cnt SEC(".maps");12array_map cpumap_kthread_cnt SEC(".maps");13array_map exception_cnt SEC(".maps");14array_map devmap_xmit_cnt SEC(".maps");15 16struct {17 __uint(type, BPF_MAP_TYPE_PERCPU_HASH);18 __uint(max_entries, 32 * 32);19 __type(key, u64);20 __type(value, struct datarec);21} devmap_xmit_cnt_multi SEC(".maps");22 23const volatile int nr_cpus = 0;24 25/* These can be set before loading so that redundant comparisons can be DCE'd by26 * the verifier, and only actual matches are tried after loading tp_btf program.27 * This allows sample to filter tracepoint stats based on net_device.28 */29const volatile int from_match[32] = {};30const volatile int to_match[32] = {};31 32int cpumap_map_id = 0;33 34/* Find if b is part of set a, but if a is empty set then evaluate to true */35#define IN_SET(a, b) \36 ({ \37 bool __res = !(a)[0]; \38 for (int i = 0; i < ARRAY_SIZE(a) && (a)[i]; i++) { \39 __res = (a)[i] == (b); \40 if (__res) \41 break; \42 } \43 __res; \44 })45 46static __always_inline __u32 xdp_get_err_key(int err)47{48 switch (err) {49 case 0:50 return 0;51 case -EINVAL:52 return 2;53 case -ENETDOWN:54 return 3;55 case -EMSGSIZE:56 return 4;57 case -EOPNOTSUPP:58 return 5;59 case -ENOSPC:60 return 6;61 default:62 return 1;63 }64}65 66static __always_inline int xdp_redirect_collect_stat(int from, int err)67{68 u32 cpu = bpf_get_smp_processor_id();69 u32 key = XDP_REDIRECT_ERROR;70 struct datarec *rec;71 u32 idx;72 73 if (!IN_SET(from_match, from))74 return 0;75 76 key = xdp_get_err_key(err);77 78 idx = key * nr_cpus + cpu;79 rec = bpf_map_lookup_elem(&redir_err_cnt, &idx);80 if (!rec)81 return 0;82 if (key)83 NO_TEAR_INC(rec->dropped);84 else85 NO_TEAR_INC(rec->processed);86 return 0; /* Indicate event was filtered (no further processing)*/87 /*88 * Returning 1 here would allow e.g. a perf-record tracepoint89 * to see and record these events, but it doesn't work well90 * in-practice as stopping perf-record also unload this91 * bpf_prog. Plus, there is additional overhead of doing so.92 */93}94 95SEC("tp_btf/xdp_redirect_err")96int BPF_PROG(tp_xdp_redirect_err, const struct net_device *dev,97 const struct bpf_prog *xdp, const void *tgt, int err,98 const struct bpf_map *map, u32 index)99{100 return xdp_redirect_collect_stat(dev->ifindex, err);101}102 103SEC("tp_btf/xdp_redirect_map_err")104int BPF_PROG(tp_xdp_redirect_map_err, const struct net_device *dev,105 const struct bpf_prog *xdp, const void *tgt, int err,106 const struct bpf_map *map, u32 index)107{108 return xdp_redirect_collect_stat(dev->ifindex, err);109}110 111SEC("tp_btf/xdp_redirect")112int BPF_PROG(tp_xdp_redirect, const struct net_device *dev,113 const struct bpf_prog *xdp, const void *tgt, int err,114 const struct bpf_map *map, u32 index)115{116 return xdp_redirect_collect_stat(dev->ifindex, err);117}118 119SEC("tp_btf/xdp_redirect_map")120int BPF_PROG(tp_xdp_redirect_map, const struct net_device *dev,121 const struct bpf_prog *xdp, const void *tgt, int err,122 const struct bpf_map *map, u32 index)123{124 return xdp_redirect_collect_stat(dev->ifindex, err);125}126 127SEC("tp_btf/xdp_cpumap_enqueue")128int BPF_PROG(tp_xdp_cpumap_enqueue, int map_id, unsigned int processed,129 unsigned int drops, int to_cpu)130{131 u32 cpu = bpf_get_smp_processor_id();132 struct datarec *rec;133 u32 idx;134 135 if (cpumap_map_id && cpumap_map_id != map_id)136 return 0;137 138 idx = to_cpu * nr_cpus + cpu;139 rec = bpf_map_lookup_elem(&cpumap_enqueue_cnt, &idx);140 if (!rec)141 return 0;142 NO_TEAR_ADD(rec->processed, processed);143 NO_TEAR_ADD(rec->dropped, drops);144 /* Record bulk events, then userspace can calc average bulk size */145 if (processed > 0)146 NO_TEAR_INC(rec->issue);147 /* Inception: It's possible to detect overload situations, via148 * this tracepoint. This can be used for creating a feedback149 * loop to XDP, which can take appropriate actions to mitigate150 * this overload situation.151 */152 return 0;153}154 155SEC("tp_btf/xdp_cpumap_kthread")156int BPF_PROG(tp_xdp_cpumap_kthread, int map_id, unsigned int processed,157 unsigned int drops, int sched, struct xdp_cpumap_stats *xdp_stats)158{159 struct datarec *rec;160 u32 cpu;161 162 if (cpumap_map_id && cpumap_map_id != map_id)163 return 0;164 165 cpu = bpf_get_smp_processor_id();166 rec = bpf_map_lookup_elem(&cpumap_kthread_cnt, &cpu);167 if (!rec)168 return 0;169 NO_TEAR_ADD(rec->processed, processed);170 NO_TEAR_ADD(rec->dropped, drops);171 NO_TEAR_ADD(rec->xdp_pass, xdp_stats->pass);172 NO_TEAR_ADD(rec->xdp_drop, xdp_stats->drop);173 NO_TEAR_ADD(rec->xdp_redirect, xdp_stats->redirect);174 /* Count times kthread yielded CPU via schedule call */175 if (sched)176 NO_TEAR_INC(rec->issue);177 return 0;178}179 180SEC("tp_btf/xdp_exception")181int BPF_PROG(tp_xdp_exception, const struct net_device *dev,182 const struct bpf_prog *xdp, u32 act)183{184 u32 cpu = bpf_get_smp_processor_id();185 struct datarec *rec;186 u32 key = act, idx;187 188 if (!IN_SET(from_match, dev->ifindex))189 return 0;190 if (!IN_SET(to_match, dev->ifindex))191 return 0;192 193 if (key > XDP_REDIRECT)194 key = XDP_REDIRECT + 1;195 196 idx = key * nr_cpus + cpu;197 rec = bpf_map_lookup_elem(&exception_cnt, &idx);198 if (!rec)199 return 0;200 NO_TEAR_INC(rec->dropped);201 202 return 0;203}204 205SEC("tp_btf/xdp_devmap_xmit")206int BPF_PROG(tp_xdp_devmap_xmit, const struct net_device *from_dev,207 const struct net_device *to_dev, int sent, int drops, int err)208{209 struct datarec *rec;210 int idx_in, idx_out;211 u32 cpu;212 213 idx_in = from_dev->ifindex;214 idx_out = to_dev->ifindex;215 216 if (!IN_SET(from_match, idx_in))217 return 0;218 if (!IN_SET(to_match, idx_out))219 return 0;220 221 cpu = bpf_get_smp_processor_id();222 rec = bpf_map_lookup_elem(&devmap_xmit_cnt, &cpu);223 if (!rec)224 return 0;225 NO_TEAR_ADD(rec->processed, sent);226 NO_TEAR_ADD(rec->dropped, drops);227 /* Record bulk events, then userspace can calc average bulk size */228 NO_TEAR_INC(rec->info);229 /* Record error cases, where no frame were sent */230 /* Catch API error of drv ndo_xdp_xmit sent more than count */231 if (err || drops < 0)232 NO_TEAR_INC(rec->issue);233 return 0;234}235 236SEC("tp_btf/xdp_devmap_xmit")237int BPF_PROG(tp_xdp_devmap_xmit_multi, const struct net_device *from_dev,238 const struct net_device *to_dev, int sent, int drops, int err)239{240 struct datarec empty = {};241 struct datarec *rec;242 int idx_in, idx_out;243 u64 idx;244 245 idx_in = from_dev->ifindex;246 idx_out = to_dev->ifindex;247 idx = idx_in;248 idx = idx << 32 | idx_out;249 250 if (!IN_SET(from_match, idx_in))251 return 0;252 if (!IN_SET(to_match, idx_out))253 return 0;254 255 bpf_map_update_elem(&devmap_xmit_cnt_multi, &idx, &empty, BPF_NOEXIST);256 rec = bpf_map_lookup_elem(&devmap_xmit_cnt_multi, &idx);257 if (!rec)258 return 0;259 260 NO_TEAR_ADD(rec->processed, sent);261 NO_TEAR_ADD(rec->dropped, drops);262 NO_TEAR_INC(rec->info);263 if (err || drops < 0)264 NO_TEAR_INC(rec->issue);265 return 0;266}267