1674 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2#define _GNU_SOURCE3 4#include <arpa/inet.h>5#include <bpf/bpf.h>6#include <bpf/libbpf.h>7#include <errno.h>8#include <fcntl.h>9#include <getopt.h>10#include <linux/ethtool.h>11#include <linux/hashtable.h>12#include <linux/if_link.h>13#include <linux/jhash.h>14#include <linux/limits.h>15#include <linux/list.h>16#include <linux/sockios.h>17#include <locale.h>18#include <math.h>19#include <net/if.h>20#include <poll.h>21#include <signal.h>22#include <stdbool.h>23#include <stdio.h>24#include <stdlib.h>25#include <string.h>26#include <sys/ioctl.h>27#include <sys/mman.h>28#include <sys/signalfd.h>29#include <sys/sysinfo.h>30#include <sys/timerfd.h>31#include <sys/utsname.h>32#include <time.h>33#include <unistd.h>34 35#include "bpf_util.h"36#include "xdp_sample_user.h"37 38#define __sample_print(fmt, cond, ...) \39 ({ \40 if (cond) \41 printf(fmt, ##__VA_ARGS__); \42 })43 44#define print_always(fmt, ...) __sample_print(fmt, 1, ##__VA_ARGS__)45#define print_default(fmt, ...) \46 __sample_print(fmt, sample_log_level & LL_DEFAULT, ##__VA_ARGS__)47#define __print_err(err, fmt, ...) \48 ({ \49 __sample_print(fmt, err > 0 || sample_log_level & LL_DEFAULT, \50 ##__VA_ARGS__); \51 sample_err_exp = sample_err_exp ? true : err > 0; \52 })53#define print_err(err, fmt, ...) __print_err(err, fmt, ##__VA_ARGS__)54 55#define __COLUMN(x) "%'10" x " %-13s"56#define FMT_COLUMNf __COLUMN(".0f")57#define FMT_COLUMNd __COLUMN("d")58#define FMT_COLUMNl __COLUMN("llu")59#define RX(rx) rx, "rx/s"60#define PPS(pps) pps, "pkt/s"61#define DROP(drop) drop, "drop/s"62#define ERR(err) err, "error/s"63#define HITS(hits) hits, "hit/s"64#define XMIT(xmit) xmit, "xmit/s"65#define PASS(pass) pass, "pass/s"66#define REDIR(redir) redir, "redir/s"67#define NANOSEC_PER_SEC 1000000000 /* 10^9 */68 69#define XDP_UNKNOWN (XDP_REDIRECT + 1)70#define XDP_ACTION_MAX (XDP_UNKNOWN + 1)71#define XDP_REDIRECT_ERR_MAX 772 73enum map_type {74 MAP_RX,75 MAP_REDIRECT_ERR,76 MAP_CPUMAP_ENQUEUE,77 MAP_CPUMAP_KTHREAD,78 MAP_EXCEPTION,79 MAP_DEVMAP_XMIT,80 MAP_DEVMAP_XMIT_MULTI,81 NUM_MAP,82};83 84enum log_level {85 LL_DEFAULT = 1U << 0,86 LL_SIMPLE = 1U << 1,87 LL_DEBUG = 1U << 2,88};89 90struct record {91 __u64 timestamp;92 struct datarec total;93 struct datarec *cpu;94};95 96struct map_entry {97 struct hlist_node node;98 __u64 pair;99 struct record val;100};101 102struct stats_record {103 struct record rx_cnt;104 struct record redir_err[XDP_REDIRECT_ERR_MAX];105 struct record kthread;106 struct record exception[XDP_ACTION_MAX];107 struct record devmap_xmit;108 DECLARE_HASHTABLE(xmit_map, 5);109 struct record enq[];110};111 112struct sample_output {113 struct {114 __u64 rx;115 __u64 redir;116 __u64 drop;117 __u64 drop_xmit;118 __u64 err;119 __u64 xmit;120 } totals;121 struct {122 union {123 __u64 pps;124 __u64 num;125 };126 __u64 drop;127 __u64 err;128 } rx_cnt;129 struct {130 __u64 suc;131 __u64 err;132 } redir_cnt;133 struct {134 __u64 hits;135 } except_cnt;136 struct {137 __u64 pps;138 __u64 drop;139 __u64 err;140 double bavg;141 } xmit_cnt;142};143 144struct xdp_desc {145 int ifindex;146 __u32 prog_id;147 int flags;148} sample_xdp_progs[32];149 150struct datarec *sample_mmap[NUM_MAP];151struct bpf_map *sample_map[NUM_MAP];152size_t sample_map_count[NUM_MAP];153enum log_level sample_log_level;154struct sample_output sample_out;155unsigned long sample_interval;156bool sample_err_exp;157int sample_xdp_cnt;158int sample_n_cpus;159int sample_sig_fd;160int sample_mask;161 162static const char *xdp_redirect_err_names[XDP_REDIRECT_ERR_MAX] = {163 /* Key=1 keeps unknown errors */164 "Success",165 "Unknown",166 "EINVAL",167 "ENETDOWN",168 "EMSGSIZE",169 "EOPNOTSUPP",170 "ENOSPC",171};172 173/* Keyed from Unknown */174static const char *xdp_redirect_err_help[XDP_REDIRECT_ERR_MAX - 1] = {175 "Unknown error",176 "Invalid redirection",177 "Device being redirected to is down",178 "Packet length too large for device",179 "Operation not supported",180 "No space in ptr_ring of cpumap kthread",181};182 183static const char *xdp_action_names[XDP_ACTION_MAX] = {184 [XDP_ABORTED] = "XDP_ABORTED",185 [XDP_DROP] = "XDP_DROP",186 [XDP_PASS] = "XDP_PASS",187 [XDP_TX] = "XDP_TX",188 [XDP_REDIRECT] = "XDP_REDIRECT",189 [XDP_UNKNOWN] = "XDP_UNKNOWN",190};191 192static __u64 gettime(void)193{194 struct timespec t;195 int res;196 197 res = clock_gettime(CLOCK_MONOTONIC, &t);198 if (res < 0) {199 fprintf(stderr, "Error with gettimeofday! (%i)\n", res);200 return UINT64_MAX;201 }202 return (__u64)t.tv_sec * NANOSEC_PER_SEC + t.tv_nsec;203}204 205static const char *action2str(int action)206{207 if (action < XDP_ACTION_MAX)208 return xdp_action_names[action];209 return NULL;210}211 212static void sample_print_help(int mask)213{214 printf("Output format description\n\n"215 "By default, redirect success statistics are disabled, use -s to enable.\n"216 "The terse output mode is default, verbose mode can be activated using -v\n"217 "Use SIGQUIT (Ctrl + \\) to switch the mode dynamically at runtime\n\n"218 "Terse mode displays at most the following fields:\n"219 " rx/s Number of packets received per second\n"220 " redir/s Number of packets successfully redirected per second\n"221 " err,drop/s Aggregated count of errors per second (including dropped packets)\n"222 " xmit/s Number of packets transmitted on the output device per second\n\n"223 "Output description for verbose mode:\n"224 " FIELD DESCRIPTION\n");225 226 if (mask & SAMPLE_RX_CNT) {227 printf(" receive\t\tDisplays the number of packets received & errors encountered\n"228 " \t\t\tWhenever an error or packet drop occurs, details of per CPU error\n"229 " \t\t\tand drop statistics will be expanded inline in terse mode.\n"230 " \t\t\t\tpkt/s - Packets received per second\n"231 " \t\t\t\tdrop/s - Packets dropped per second\n"232 " \t\t\t\terror/s - Errors encountered per second\n\n");233 }234 if (mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) {235 printf(" redirect\t\tDisplays the number of packets successfully redirected\n"236 " \t\t\tErrors encountered are expanded under redirect_err field\n"237 " \t\t\tNote that passing -s to enable it has a per packet overhead\n"238 " \t\t\t\tredir/s - Packets redirected successfully per second\n\n"239 " redirect_err\t\tDisplays the number of packets that failed redirection\n"240 " \t\t\tThe errno is expanded under this field with per CPU count\n"241 " \t\t\tThe recognized errors are:\n");242 243 for (int i = 2; i < XDP_REDIRECT_ERR_MAX; i++)244 printf("\t\t\t %s: %s\n", xdp_redirect_err_names[i],245 xdp_redirect_err_help[i - 1]);246 247 printf(" \n\t\t\t\terror/s - Packets that failed redirection per second\n\n");248 }249 250 if (mask & SAMPLE_CPUMAP_ENQUEUE_CNT) {251 printf(" enqueue to cpu N\tDisplays the number of packets enqueued to bulk queue of CPU N\n"252 " \t\t\tExpands to cpu:FROM->N to display enqueue stats for each CPU enqueuing to CPU N\n"253 " \t\t\tReceived packets can be associated with the CPU redirect program is enqueuing \n"254 " \t\t\tpackets to.\n"255 " \t\t\t\tpkt/s - Packets enqueued per second from other CPU to CPU N\n"256 " \t\t\t\tdrop/s - Packets dropped when trying to enqueue to CPU N\n"257 " \t\t\t\tbulk-avg - Average number of packets processed for each event\n\n");258 }259 260 if (mask & SAMPLE_CPUMAP_KTHREAD_CNT) {261 printf(" kthread\t\tDisplays the number of packets processed in CPUMAP kthread for each CPU\n"262 " \t\t\tPackets consumed from ptr_ring in kthread, and its xdp_stats (after calling \n"263 " \t\t\tCPUMAP bpf prog) are expanded below this. xdp_stats are expanded as a total and\n"264 " \t\t\tthen per-CPU to associate it to each CPU's pinned CPUMAP kthread.\n"265 " \t\t\t\tpkt/s - Packets consumed per second from ptr_ring\n"266 " \t\t\t\tdrop/s - Packets dropped per second in kthread\n"267 " \t\t\t\tsched - Number of times kthread called schedule()\n\n"268 " \t\t\txdp_stats (also expands to per-CPU counts)\n"269 " \t\t\t\tpass/s - XDP_PASS count for CPUMAP program execution\n"270 " \t\t\t\tdrop/s - XDP_DROP count for CPUMAP program execution\n"271 " \t\t\t\tredir/s - XDP_REDIRECT count for CPUMAP program execution\n\n");272 }273 274 if (mask & SAMPLE_EXCEPTION_CNT) {275 printf(" xdp_exception\t\tDisplays xdp_exception tracepoint events\n"276 " \t\t\tThis can occur due to internal driver errors, unrecognized\n"277 " \t\t\tXDP actions and due to explicit user trigger by use of XDP_ABORTED\n"278 " \t\t\tEach action is expanded below this field with its count\n"279 " \t\t\t\thit/s - Number of times the tracepoint was hit per second\n\n");280 }281 282 if (mask & SAMPLE_DEVMAP_XMIT_CNT) {283 printf(" devmap_xmit\t\tDisplays devmap_xmit tracepoint events\n"284 " \t\t\tThis tracepoint is invoked for successful transmissions on output\n"285 " \t\t\tdevice but these statistics are not available for generic XDP mode,\n"286 " \t\t\thence they will be omitted from the output when using SKB mode\n"287 " \t\t\t\txmit/s - Number of packets that were transmitted per second\n"288 " \t\t\t\tdrop/s - Number of packets that failed transmissions per second\n"289 " \t\t\t\tdrv_err/s - Number of internal driver errors per second\n"290 " \t\t\t\tbulk-avg - Average number of packets processed for each event\n\n");291 }292}293 294void sample_usage(char *argv[], const struct option *long_options,295 const char *doc, int mask, bool error)296{297 int i;298 299 if (!error)300 sample_print_help(mask);301 302 printf("\n%s\nOption for %s:\n", doc, argv[0]);303 for (i = 0; long_options[i].name != 0; i++) {304 printf(" --%-15s", long_options[i].name);305 if (long_options[i].flag != NULL)306 printf(" flag (internal value: %d)",307 *long_options[i].flag);308 else309 printf("\t short-option: -%c", long_options[i].val);310 printf("\n");311 }312 printf("\n");313}314 315static struct datarec *alloc_record_per_cpu(void)316{317 unsigned int nr_cpus = libbpf_num_possible_cpus();318 struct datarec *array;319 320 array = calloc(nr_cpus, sizeof(*array));321 if (!array) {322 fprintf(stderr, "Failed to allocate memory (nr_cpus: %u)\n",323 nr_cpus);324 return NULL;325 }326 return array;327}328 329static int map_entry_init(struct map_entry *e, __u64 pair)330{331 e->pair = pair;332 INIT_HLIST_NODE(&e->node);333 e->val.timestamp = gettime();334 e->val.cpu = alloc_record_per_cpu();335 if (!e->val.cpu)336 return -ENOMEM;337 return 0;338}339 340static void map_collect_percpu(struct datarec *values, struct record *rec)341{342 /* For percpu maps, userspace gets a value per possible CPU */343 unsigned int nr_cpus = libbpf_num_possible_cpus();344 __u64 sum_xdp_redirect = 0;345 __u64 sum_processed = 0;346 __u64 sum_xdp_pass = 0;347 __u64 sum_xdp_drop = 0;348 __u64 sum_dropped = 0;349 __u64 sum_issue = 0;350 int i;351 352 /* Get time as close as possible to reading map contents */353 rec->timestamp = gettime();354 355 /* Record and sum values from each CPU */356 for (i = 0; i < nr_cpus; i++) {357 rec->cpu[i].processed = READ_ONCE(values[i].processed);358 rec->cpu[i].dropped = READ_ONCE(values[i].dropped);359 rec->cpu[i].issue = READ_ONCE(values[i].issue);360 rec->cpu[i].xdp_pass = READ_ONCE(values[i].xdp_pass);361 rec->cpu[i].xdp_drop = READ_ONCE(values[i].xdp_drop);362 rec->cpu[i].xdp_redirect = READ_ONCE(values[i].xdp_redirect);363 364 sum_processed += rec->cpu[i].processed;365 sum_dropped += rec->cpu[i].dropped;366 sum_issue += rec->cpu[i].issue;367 sum_xdp_pass += rec->cpu[i].xdp_pass;368 sum_xdp_drop += rec->cpu[i].xdp_drop;369 sum_xdp_redirect += rec->cpu[i].xdp_redirect;370 }371 372 rec->total.processed = sum_processed;373 rec->total.dropped = sum_dropped;374 rec->total.issue = sum_issue;375 rec->total.xdp_pass = sum_xdp_pass;376 rec->total.xdp_drop = sum_xdp_drop;377 rec->total.xdp_redirect = sum_xdp_redirect;378}379 380static int map_collect_percpu_devmap(int map_fd, struct stats_record *rec)381{382 unsigned int nr_cpus = bpf_num_possible_cpus();383 __u32 batch, count = 32;384 struct datarec *values;385 bool init = false;386 __u64 *keys;387 int i, ret;388 389 keys = calloc(count, sizeof(__u64));390 if (!keys)391 return -ENOMEM;392 values = calloc(count * nr_cpus, sizeof(struct datarec));393 if (!values) {394 free(keys);395 return -ENOMEM;396 }397 398 for (;;) {399 bool exit = false;400 401 ret = bpf_map_lookup_batch(map_fd, init ? &batch : NULL, &batch,402 keys, values, &count, NULL);403 if (ret < 0 && errno != ENOENT)404 break;405 if (errno == ENOENT)406 exit = true;407 408 init = true;409 for (i = 0; i < count; i++) {410 struct map_entry *e, *x = NULL;411 __u64 pair = keys[i];412 struct datarec *arr;413 414 arr = &values[i * nr_cpus];415 hash_for_each_possible(rec->xmit_map, e, node, pair) {416 if (e->pair == pair) {417 x = e;418 break;419 }420 }421 if (!x) {422 x = calloc(1, sizeof(*x));423 if (!x)424 goto cleanup;425 if (map_entry_init(x, pair) < 0) {426 free(x);427 goto cleanup;428 }429 hash_add(rec->xmit_map, &x->node, pair);430 }431 map_collect_percpu(arr, &x->val);432 }433 434 if (exit)435 break;436 count = 32;437 }438 439 free(values);440 free(keys);441 return 0;442cleanup:443 free(values);444 free(keys);445 return -ENOMEM;446}447 448static struct stats_record *alloc_stats_record(void)449{450 struct stats_record *rec;451 int i;452 453 rec = calloc(1, sizeof(*rec) + sample_n_cpus * sizeof(struct record));454 if (!rec) {455 fprintf(stderr, "Failed to allocate memory\n");456 return NULL;457 }458 459 if (sample_mask & SAMPLE_RX_CNT) {460 rec->rx_cnt.cpu = alloc_record_per_cpu();461 if (!rec->rx_cnt.cpu) {462 fprintf(stderr,463 "Failed to allocate rx_cnt per-CPU array\n");464 goto end_rec;465 }466 }467 if (sample_mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) {468 for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++) {469 rec->redir_err[i].cpu = alloc_record_per_cpu();470 if (!rec->redir_err[i].cpu) {471 fprintf(stderr,472 "Failed to allocate redir_err per-CPU array for "473 "\"%s\" case\n",474 xdp_redirect_err_names[i]);475 while (i--)476 free(rec->redir_err[i].cpu);477 goto end_rx_cnt;478 }479 }480 }481 if (sample_mask & SAMPLE_CPUMAP_KTHREAD_CNT) {482 rec->kthread.cpu = alloc_record_per_cpu();483 if (!rec->kthread.cpu) {484 fprintf(stderr,485 "Failed to allocate kthread per-CPU array\n");486 goto end_redir;487 }488 }489 if (sample_mask & SAMPLE_EXCEPTION_CNT) {490 for (i = 0; i < XDP_ACTION_MAX; i++) {491 rec->exception[i].cpu = alloc_record_per_cpu();492 if (!rec->exception[i].cpu) {493 fprintf(stderr,494 "Failed to allocate exception per-CPU array for "495 "\"%s\" case\n",496 action2str(i));497 while (i--)498 free(rec->exception[i].cpu);499 goto end_kthread;500 }501 }502 }503 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT) {504 rec->devmap_xmit.cpu = alloc_record_per_cpu();505 if (!rec->devmap_xmit.cpu) {506 fprintf(stderr,507 "Failed to allocate devmap_xmit per-CPU array\n");508 goto end_exception;509 }510 }511 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)512 hash_init(rec->xmit_map);513 if (sample_mask & SAMPLE_CPUMAP_ENQUEUE_CNT) {514 for (i = 0; i < sample_n_cpus; i++) {515 rec->enq[i].cpu = alloc_record_per_cpu();516 if (!rec->enq[i].cpu) {517 fprintf(stderr,518 "Failed to allocate enqueue per-CPU array for "519 "CPU %d\n",520 i);521 while (i--)522 free(rec->enq[i].cpu);523 goto end_devmap_xmit;524 }525 }526 }527 528 return rec;529 530end_devmap_xmit:531 free(rec->devmap_xmit.cpu);532end_exception:533 for (i = 0; i < XDP_ACTION_MAX; i++)534 free(rec->exception[i].cpu);535end_kthread:536 free(rec->kthread.cpu);537end_redir:538 for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++)539 free(rec->redir_err[i].cpu);540end_rx_cnt:541 free(rec->rx_cnt.cpu);542end_rec:543 free(rec);544 return NULL;545}546 547static void free_stats_record(struct stats_record *r)548{549 struct hlist_node *tmp;550 struct map_entry *e;551 int i;552 553 for (i = 0; i < sample_n_cpus; i++)554 free(r->enq[i].cpu);555 hash_for_each_safe(r->xmit_map, i, tmp, e, node) {556 hash_del(&e->node);557 free(e->val.cpu);558 free(e);559 }560 free(r->devmap_xmit.cpu);561 for (i = 0; i < XDP_ACTION_MAX; i++)562 free(r->exception[i].cpu);563 free(r->kthread.cpu);564 for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++)565 free(r->redir_err[i].cpu);566 free(r->rx_cnt.cpu);567 free(r);568}569 570static double calc_period(struct record *r, struct record *p)571{572 double period_ = 0;573 __u64 period = 0;574 575 period = r->timestamp - p->timestamp;576 if (period > 0)577 period_ = ((double)period / NANOSEC_PER_SEC);578 579 return period_;580}581 582static double sample_round(double val)583{584 if (val - floor(val) < 0.5)585 return floor(val);586 return ceil(val);587}588 589static __u64 calc_pps(struct datarec *r, struct datarec *p, double period_)590{591 __u64 packets = 0;592 __u64 pps = 0;593 594 if (period_ > 0) {595 packets = r->processed - p->processed;596 pps = sample_round(packets / period_);597 }598 return pps;599}600 601static __u64 calc_drop_pps(struct datarec *r, struct datarec *p, double period_)602{603 __u64 packets = 0;604 __u64 pps = 0;605 606 if (period_ > 0) {607 packets = r->dropped - p->dropped;608 pps = sample_round(packets / period_);609 }610 return pps;611}612 613static __u64 calc_errs_pps(struct datarec *r, struct datarec *p, double period_)614{615 __u64 packets = 0;616 __u64 pps = 0;617 618 if (period_ > 0) {619 packets = r->issue - p->issue;620 pps = sample_round(packets / period_);621 }622 return pps;623}624 625static __u64 calc_info_pps(struct datarec *r, struct datarec *p, double period_)626{627 __u64 packets = 0;628 __u64 pps = 0;629 630 if (period_ > 0) {631 packets = r->info - p->info;632 pps = sample_round(packets / period_);633 }634 return pps;635}636 637static void calc_xdp_pps(struct datarec *r, struct datarec *p, double *xdp_pass,638 double *xdp_drop, double *xdp_redirect, double period_)639{640 *xdp_pass = 0, *xdp_drop = 0, *xdp_redirect = 0;641 if (period_ > 0) {642 *xdp_redirect = (r->xdp_redirect - p->xdp_redirect) / period_;643 *xdp_pass = (r->xdp_pass - p->xdp_pass) / period_;644 *xdp_drop = (r->xdp_drop - p->xdp_drop) / period_;645 }646}647 648static void stats_get_rx_cnt(struct stats_record *stats_rec,649 struct stats_record *stats_prev,650 unsigned int nr_cpus, struct sample_output *out)651{652 struct record *rec, *prev;653 double t, pps, drop, err;654 int i;655 656 rec = &stats_rec->rx_cnt;657 prev = &stats_prev->rx_cnt;658 t = calc_period(rec, prev);659 660 for (i = 0; i < nr_cpus; i++) {661 struct datarec *r = &rec->cpu[i];662 struct datarec *p = &prev->cpu[i];663 char str[64];664 665 pps = calc_pps(r, p, t);666 drop = calc_drop_pps(r, p, t);667 err = calc_errs_pps(r, p, t);668 if (!pps && !drop && !err)669 continue;670 671 snprintf(str, sizeof(str), "cpu:%d", i);672 print_default(" %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf673 "\n",674 str, PPS(pps), DROP(drop), ERR(err));675 }676 677 if (out) {678 pps = calc_pps(&rec->total, &prev->total, t);679 drop = calc_drop_pps(&rec->total, &prev->total, t);680 err = calc_errs_pps(&rec->total, &prev->total, t);681 682 out->rx_cnt.pps = pps;683 out->rx_cnt.drop = drop;684 out->rx_cnt.err = err;685 out->totals.rx += pps;686 out->totals.drop += drop;687 out->totals.err += err;688 }689}690 691static void stats_get_cpumap_enqueue(struct stats_record *stats_rec,692 struct stats_record *stats_prev,693 unsigned int nr_cpus)694{695 struct record *rec, *prev;696 double t, pps, drop, err;697 int i, to_cpu;698 699 /* cpumap enqueue stats */700 for (to_cpu = 0; to_cpu < sample_n_cpus; to_cpu++) {701 rec = &stats_rec->enq[to_cpu];702 prev = &stats_prev->enq[to_cpu];703 t = calc_period(rec, prev);704 705 pps = calc_pps(&rec->total, &prev->total, t);706 drop = calc_drop_pps(&rec->total, &prev->total, t);707 err = calc_errs_pps(&rec->total, &prev->total, t);708 709 if (pps > 0 || drop > 0) {710 char str[64];711 712 snprintf(str, sizeof(str), "enqueue to cpu %d", to_cpu);713 714 if (err > 0)715 err = pps / err; /* calc average bulk size */716 717 print_err(drop,718 " %-20s " FMT_COLUMNf FMT_COLUMNf __COLUMN(719 ".2f") "\n",720 str, PPS(pps), DROP(drop), err, "bulk-avg");721 }722 723 for (i = 0; i < nr_cpus; i++) {724 struct datarec *r = &rec->cpu[i];725 struct datarec *p = &prev->cpu[i];726 char str[64];727 728 pps = calc_pps(r, p, t);729 drop = calc_drop_pps(r, p, t);730 err = calc_errs_pps(r, p, t);731 if (!pps && !drop && !err)732 continue;733 734 snprintf(str, sizeof(str), "cpu:%d->%d", i, to_cpu);735 if (err > 0)736 err = pps / err; /* calc average bulk size */737 print_default(738 " %-18s " FMT_COLUMNf FMT_COLUMNf __COLUMN(739 ".2f") "\n",740 str, PPS(pps), DROP(drop), err, "bulk-avg");741 }742 }743}744 745static void stats_get_cpumap_remote(struct stats_record *stats_rec,746 struct stats_record *stats_prev,747 unsigned int nr_cpus)748{749 double xdp_pass, xdp_drop, xdp_redirect;750 struct record *rec, *prev;751 double t;752 int i;753 754 rec = &stats_rec->kthread;755 prev = &stats_prev->kthread;756 t = calc_period(rec, prev);757 758 calc_xdp_pps(&rec->total, &prev->total, &xdp_pass, &xdp_drop,759 &xdp_redirect, t);760 if (xdp_pass || xdp_drop || xdp_redirect) {761 print_err(xdp_drop,762 " %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf "\n",763 "xdp_stats", PASS(xdp_pass), DROP(xdp_drop),764 REDIR(xdp_redirect));765 }766 767 for (i = 0; i < nr_cpus; i++) {768 struct datarec *r = &rec->cpu[i];769 struct datarec *p = &prev->cpu[i];770 char str[64];771 772 calc_xdp_pps(r, p, &xdp_pass, &xdp_drop, &xdp_redirect, t);773 if (!xdp_pass && !xdp_drop && !xdp_redirect)774 continue;775 776 snprintf(str, sizeof(str), "cpu:%d", i);777 print_default(" %-16s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf778 "\n",779 str, PASS(xdp_pass), DROP(xdp_drop),780 REDIR(xdp_redirect));781 }782}783 784static void stats_get_cpumap_kthread(struct stats_record *stats_rec,785 struct stats_record *stats_prev,786 unsigned int nr_cpus)787{788 struct record *rec, *prev;789 double t, pps, drop, err;790 int i;791 792 rec = &stats_rec->kthread;793 prev = &stats_prev->kthread;794 t = calc_period(rec, prev);795 796 pps = calc_pps(&rec->total, &prev->total, t);797 drop = calc_drop_pps(&rec->total, &prev->total, t);798 err = calc_errs_pps(&rec->total, &prev->total, t);799 800 print_err(drop, " %-20s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf "\n",801 pps ? "kthread total" : "kthread", PPS(pps), DROP(drop), err,802 "sched");803 804 for (i = 0; i < nr_cpus; i++) {805 struct datarec *r = &rec->cpu[i];806 struct datarec *p = &prev->cpu[i];807 char str[64];808 809 pps = calc_pps(r, p, t);810 drop = calc_drop_pps(r, p, t);811 err = calc_errs_pps(r, p, t);812 if (!pps && !drop && !err)813 continue;814 815 snprintf(str, sizeof(str), "cpu:%d", i);816 print_default(" %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf817 "\n",818 str, PPS(pps), DROP(drop), err, "sched");819 }820}821 822static void stats_get_redirect_cnt(struct stats_record *stats_rec,823 struct stats_record *stats_prev,824 unsigned int nr_cpus,825 struct sample_output *out)826{827 struct record *rec, *prev;828 double t, pps;829 int i;830 831 rec = &stats_rec->redir_err[0];832 prev = &stats_prev->redir_err[0];833 t = calc_period(rec, prev);834 for (i = 0; i < nr_cpus; i++) {835 struct datarec *r = &rec->cpu[i];836 struct datarec *p = &prev->cpu[i];837 char str[64];838 839 pps = calc_pps(r, p, t);840 if (!pps)841 continue;842 843 snprintf(str, sizeof(str), "cpu:%d", i);844 print_default(" %-18s " FMT_COLUMNf "\n", str, REDIR(pps));845 }846 847 if (out) {848 pps = calc_pps(&rec->total, &prev->total, t);849 out->redir_cnt.suc = pps;850 out->totals.redir += pps;851 }852}853 854static void stats_get_redirect_err_cnt(struct stats_record *stats_rec,855 struct stats_record *stats_prev,856 unsigned int nr_cpus,857 struct sample_output *out)858{859 struct record *rec, *prev;860 double t, drop, sum = 0;861 int rec_i, i;862 863 for (rec_i = 1; rec_i < XDP_REDIRECT_ERR_MAX; rec_i++) {864 char str[64];865 866 rec = &stats_rec->redir_err[rec_i];867 prev = &stats_prev->redir_err[rec_i];868 t = calc_period(rec, prev);869 870 drop = calc_drop_pps(&rec->total, &prev->total, t);871 if (drop > 0 && !out) {872 snprintf(str, sizeof(str),873 sample_log_level & LL_DEFAULT ? "%s total" :874 "%s",875 xdp_redirect_err_names[rec_i]);876 print_err(drop, " %-18s " FMT_COLUMNf "\n", str,877 ERR(drop));878 }879 880 for (i = 0; i < nr_cpus; i++) {881 struct datarec *r = &rec->cpu[i];882 struct datarec *p = &prev->cpu[i];883 double drop;884 885 drop = calc_drop_pps(r, p, t);886 if (!drop)887 continue;888 889 snprintf(str, sizeof(str), "cpu:%d", i);890 print_default(" %-16s" FMT_COLUMNf "\n", str,891 ERR(drop));892 }893 894 sum += drop;895 }896 897 if (out) {898 out->redir_cnt.err = sum;899 out->totals.err += sum;900 }901}902 903static void stats_get_exception_cnt(struct stats_record *stats_rec,904 struct stats_record *stats_prev,905 unsigned int nr_cpus,906 struct sample_output *out)907{908 double t, drop, sum = 0;909 struct record *rec, *prev;910 int rec_i, i;911 912 for (rec_i = 0; rec_i < XDP_ACTION_MAX; rec_i++) {913 rec = &stats_rec->exception[rec_i];914 prev = &stats_prev->exception[rec_i];915 t = calc_period(rec, prev);916 917 drop = calc_drop_pps(&rec->total, &prev->total, t);918 /* Fold out errors after heading */919 sum += drop;920 921 if (drop > 0 && !out) {922 print_always(" %-18s " FMT_COLUMNf "\n",923 action2str(rec_i), ERR(drop));924 925 for (i = 0; i < nr_cpus; i++) {926 struct datarec *r = &rec->cpu[i];927 struct datarec *p = &prev->cpu[i];928 char str[64];929 double drop;930 931 drop = calc_drop_pps(r, p, t);932 if (!drop)933 continue;934 935 snprintf(str, sizeof(str), "cpu:%d", i);936 print_default(" %-16s" FMT_COLUMNf "\n",937 str, ERR(drop));938 }939 }940 }941 942 if (out) {943 out->except_cnt.hits = sum;944 out->totals.err += sum;945 }946}947 948static void stats_get_devmap_xmit(struct stats_record *stats_rec,949 struct stats_record *stats_prev,950 unsigned int nr_cpus,951 struct sample_output *out)952{953 double pps, drop, info, err;954 struct record *rec, *prev;955 double t;956 int i;957 958 rec = &stats_rec->devmap_xmit;959 prev = &stats_prev->devmap_xmit;960 t = calc_period(rec, prev);961 for (i = 0; i < nr_cpus; i++) {962 struct datarec *r = &rec->cpu[i];963 struct datarec *p = &prev->cpu[i];964 char str[64];965 966 pps = calc_pps(r, p, t);967 drop = calc_drop_pps(r, p, t);968 err = calc_errs_pps(r, p, t);969 970 if (!pps && !drop && !err)971 continue;972 973 snprintf(str, sizeof(str), "cpu:%d", i);974 info = calc_info_pps(r, p, t);975 if (info > 0)976 info = (pps + drop) / info; /* calc avg bulk */977 print_default(" %-18s" FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf978 __COLUMN(".2f") "\n",979 str, XMIT(pps), DROP(drop), err, "drv_err/s",980 info, "bulk-avg");981 }982 if (out) {983 pps = calc_pps(&rec->total, &prev->total, t);984 drop = calc_drop_pps(&rec->total, &prev->total, t);985 info = calc_info_pps(&rec->total, &prev->total, t);986 if (info > 0)987 info = (pps + drop) / info; /* calc avg bulk */988 err = calc_errs_pps(&rec->total, &prev->total, t);989 990 out->xmit_cnt.pps = pps;991 out->xmit_cnt.drop = drop;992 out->xmit_cnt.bavg = info;993 out->xmit_cnt.err = err;994 out->totals.xmit += pps;995 out->totals.drop_xmit += drop;996 out->totals.err += err;997 }998}999 1000static void stats_get_devmap_xmit_multi(struct stats_record *stats_rec,1001 struct stats_record *stats_prev,1002 unsigned int nr_cpus,1003 struct sample_output *out,1004 bool xmit_total)1005{1006 double pps, drop, info, err;1007 struct map_entry *entry;1008 struct record *r, *p;1009 double t;1010 int bkt;1011 1012 hash_for_each(stats_rec->xmit_map, bkt, entry, node) {1013 struct map_entry *e, *x = NULL;1014 char ifname_from[IFNAMSIZ];1015 char ifname_to[IFNAMSIZ];1016 const char *fstr, *tstr;1017 unsigned long prev_time;1018 struct record beg = {};1019 __u32 from_idx, to_idx;1020 char str[128];1021 __u64 pair;1022 int i;1023 1024 prev_time = sample_interval * NANOSEC_PER_SEC;1025 1026 pair = entry->pair;1027 from_idx = pair >> 32;1028 to_idx = pair & 0xFFFFFFFF;1029 1030 r = &entry->val;1031 beg.timestamp = r->timestamp - prev_time;1032 1033 /* Find matching entry from stats_prev map */1034 hash_for_each_possible(stats_prev->xmit_map, e, node, pair) {1035 if (e->pair == pair) {1036 x = e;1037 break;1038 }1039 }1040 if (x)1041 p = &x->val;1042 else1043 p = &beg;1044 t = calc_period(r, p);1045 pps = calc_pps(&r->total, &p->total, t);1046 drop = calc_drop_pps(&r->total, &p->total, t);1047 info = calc_info_pps(&r->total, &p->total, t);1048 if (info > 0)1049 info = (pps + drop) / info; /* calc avg bulk */1050 err = calc_errs_pps(&r->total, &p->total, t);1051 1052 if (out) {1053 /* We are responsible for filling out totals */1054 out->totals.xmit += pps;1055 out->totals.drop_xmit += drop;1056 out->totals.err += err;1057 continue;1058 }1059 1060 fstr = tstr = NULL;1061 if (if_indextoname(from_idx, ifname_from))1062 fstr = ifname_from;1063 if (if_indextoname(to_idx, ifname_to))1064 tstr = ifname_to;1065 1066 snprintf(str, sizeof(str), "xmit %s->%s", fstr ?: "?",1067 tstr ?: "?");1068 /* Skip idle streams of redirection */1069 if (pps || drop || err) {1070 print_err(drop,1071 " %-20s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf1072 __COLUMN(".2f") "\n", str, XMIT(pps), DROP(drop),1073 err, "drv_err/s", info, "bulk-avg");1074 }1075 1076 for (i = 0; i < nr_cpus; i++) {1077 struct datarec *rc = &r->cpu[i];1078 struct datarec *pc, p_beg = {};1079 char str[64];1080 1081 pc = p == &beg ? &p_beg : &p->cpu[i];1082 1083 pps = calc_pps(rc, pc, t);1084 drop = calc_drop_pps(rc, pc, t);1085 err = calc_errs_pps(rc, pc, t);1086 1087 if (!pps && !drop && !err)1088 continue;1089 1090 snprintf(str, sizeof(str), "cpu:%d", i);1091 info = calc_info_pps(rc, pc, t);1092 if (info > 0)1093 info = (pps + drop) / info; /* calc avg bulk */1094 1095 print_default(" %-18s" FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf1096 __COLUMN(".2f") "\n", str, XMIT(pps),1097 DROP(drop), err, "drv_err/s", info, "bulk-avg");1098 }1099 }1100}1101 1102static void stats_print(const char *prefix, int mask, struct stats_record *r,1103 struct stats_record *p, struct sample_output *out)1104{1105 int nr_cpus = libbpf_num_possible_cpus();1106 const char *str;1107 1108 print_always("%-23s", prefix ?: "Summary");1109 if (mask & SAMPLE_RX_CNT)1110 print_always(FMT_COLUMNl, RX(out->totals.rx));1111 if (mask & SAMPLE_REDIRECT_CNT)1112 print_always(FMT_COLUMNl, REDIR(out->totals.redir));1113 printf(FMT_COLUMNl,1114 out->totals.err + out->totals.drop + out->totals.drop_xmit,1115 "err,drop/s");1116 if (mask & SAMPLE_DEVMAP_XMIT_CNT ||1117 mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)1118 printf(FMT_COLUMNl, XMIT(out->totals.xmit));1119 printf("\n");1120 1121 if (mask & SAMPLE_RX_CNT) {1122 str = (sample_log_level & LL_DEFAULT) && out->rx_cnt.pps ?1123 "receive total" :1124 "receive";1125 print_err((out->rx_cnt.err || out->rx_cnt.drop),1126 " %-20s " FMT_COLUMNl FMT_COLUMNl FMT_COLUMNl "\n",1127 str, PPS(out->rx_cnt.pps), DROP(out->rx_cnt.drop),1128 ERR(out->rx_cnt.err));1129 1130 stats_get_rx_cnt(r, p, nr_cpus, NULL);1131 }1132 1133 if (mask & SAMPLE_CPUMAP_ENQUEUE_CNT)1134 stats_get_cpumap_enqueue(r, p, nr_cpus);1135 1136 if (mask & SAMPLE_CPUMAP_KTHREAD_CNT) {1137 stats_get_cpumap_kthread(r, p, nr_cpus);1138 stats_get_cpumap_remote(r, p, nr_cpus);1139 }1140 1141 if (mask & SAMPLE_REDIRECT_CNT) {1142 str = out->redir_cnt.suc ? "redirect total" : "redirect";1143 print_default(" %-20s " FMT_COLUMNl "\n", str,1144 REDIR(out->redir_cnt.suc));1145 1146 stats_get_redirect_cnt(r, p, nr_cpus, NULL);1147 }1148 1149 if (mask & SAMPLE_REDIRECT_ERR_CNT) {1150 str = (sample_log_level & LL_DEFAULT) && out->redir_cnt.err ?1151 "redirect_err total" :1152 "redirect_err";1153 print_err(out->redir_cnt.err, " %-20s " FMT_COLUMNl "\n", str,1154 ERR(out->redir_cnt.err));1155 1156 stats_get_redirect_err_cnt(r, p, nr_cpus, NULL);1157 }1158 1159 if (mask & SAMPLE_EXCEPTION_CNT) {1160 str = out->except_cnt.hits ? "xdp_exception total" :1161 "xdp_exception";1162 1163 print_err(out->except_cnt.hits, " %-20s " FMT_COLUMNl "\n", str,1164 HITS(out->except_cnt.hits));1165 1166 stats_get_exception_cnt(r, p, nr_cpus, NULL);1167 }1168 1169 if (mask & SAMPLE_DEVMAP_XMIT_CNT) {1170 str = (sample_log_level & LL_DEFAULT) && out->xmit_cnt.pps ?1171 "devmap_xmit total" :1172 "devmap_xmit";1173 1174 print_err(out->xmit_cnt.err || out->xmit_cnt.drop,1175 " %-20s " FMT_COLUMNl FMT_COLUMNl FMT_COLUMNl1176 __COLUMN(".2f") "\n",1177 str, XMIT(out->xmit_cnt.pps),1178 DROP(out->xmit_cnt.drop), out->xmit_cnt.err,1179 "drv_err/s", out->xmit_cnt.bavg, "bulk-avg");1180 1181 stats_get_devmap_xmit(r, p, nr_cpus, NULL);1182 }1183 1184 if (mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)1185 stats_get_devmap_xmit_multi(r, p, nr_cpus, NULL,1186 mask & SAMPLE_DEVMAP_XMIT_CNT);1187 1188 if (sample_log_level & LL_DEFAULT ||1189 ((sample_log_level & LL_SIMPLE) && sample_err_exp)) {1190 sample_err_exp = false;1191 printf("\n");1192 }1193}1194 1195int sample_setup_maps(struct bpf_map **maps)1196{1197 sample_n_cpus = libbpf_num_possible_cpus();1198 1199 for (int i = 0; i < MAP_DEVMAP_XMIT_MULTI; i++) {1200 sample_map[i] = maps[i];1201 1202 switch (i) {1203 case MAP_RX:1204 case MAP_CPUMAP_KTHREAD:1205 case MAP_DEVMAP_XMIT:1206 sample_map_count[i] = sample_n_cpus;1207 break;1208 case MAP_REDIRECT_ERR:1209 sample_map_count[i] =1210 XDP_REDIRECT_ERR_MAX * sample_n_cpus;1211 break;1212 case MAP_EXCEPTION:1213 sample_map_count[i] = XDP_ACTION_MAX * sample_n_cpus;1214 case MAP_CPUMAP_ENQUEUE:1215 sample_map_count[i] = sample_n_cpus * sample_n_cpus;1216 break;1217 default:1218 return -EINVAL;1219 }1220 if (bpf_map__set_max_entries(sample_map[i], sample_map_count[i]) < 0)1221 return -errno;1222 }1223 sample_map[MAP_DEVMAP_XMIT_MULTI] = maps[MAP_DEVMAP_XMIT_MULTI];1224 return 0;1225}1226 1227static int sample_setup_maps_mappings(void)1228{1229 for (int i = 0; i < MAP_DEVMAP_XMIT_MULTI; i++) {1230 size_t size = sample_map_count[i] * sizeof(struct datarec);1231 1232 sample_mmap[i] = mmap(NULL, size, PROT_READ | PROT_WRITE,1233 MAP_SHARED, bpf_map__fd(sample_map[i]), 0);1234 if (sample_mmap[i] == MAP_FAILED)1235 return -errno;1236 }1237 return 0;1238}1239 1240int __sample_init(int mask)1241{1242 sigset_t st;1243 1244 sigemptyset(&st);1245 sigaddset(&st, SIGQUIT);1246 sigaddset(&st, SIGINT);1247 sigaddset(&st, SIGTERM);1248 1249 if (sigprocmask(SIG_BLOCK, &st, NULL) < 0)1250 return -errno;1251 1252 sample_sig_fd = signalfd(-1, &st, SFD_CLOEXEC | SFD_NONBLOCK);1253 if (sample_sig_fd < 0)1254 return -errno;1255 1256 sample_mask = mask;1257 1258 return sample_setup_maps_mappings();1259}1260 1261static int __sample_remove_xdp(int ifindex, __u32 prog_id, int xdp_flags)1262{1263 __u32 cur_prog_id = 0;1264 int ret;1265 1266 if (prog_id) {1267 ret = bpf_xdp_query_id(ifindex, xdp_flags, &cur_prog_id);1268 if (ret < 0)1269 return -errno;1270 1271 if (prog_id != cur_prog_id) {1272 print_always(1273 "Program on ifindex %d does not match installed "1274 "program, skipping unload\n",1275 ifindex);1276 return -ENOENT;1277 }1278 }1279 1280 return bpf_xdp_detach(ifindex, xdp_flags, NULL);1281}1282 1283int sample_install_xdp(struct bpf_program *xdp_prog, int ifindex, bool generic,1284 bool force)1285{1286 int ret, xdp_flags = 0;1287 __u32 prog_id = 0;1288 1289 if (sample_xdp_cnt == 32) {1290 fprintf(stderr,1291 "Total limit for installed XDP programs in a sample reached\n");1292 return -ENOTSUP;1293 }1294 1295 xdp_flags |= !force ? XDP_FLAGS_UPDATE_IF_NOEXIST : 0;1296 xdp_flags |= generic ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE;1297 ret = bpf_xdp_attach(ifindex, bpf_program__fd(xdp_prog), xdp_flags, NULL);1298 if (ret < 0) {1299 ret = -errno;1300 fprintf(stderr,1301 "Failed to install program \"%s\" on ifindex %d, mode = %s, "1302 "force = %s: %s\n",1303 bpf_program__name(xdp_prog), ifindex,1304 generic ? "skb" : "native", force ? "true" : "false",1305 strerror(-ret));1306 return ret;1307 }1308 1309 ret = bpf_xdp_query_id(ifindex, xdp_flags, &prog_id);1310 if (ret < 0) {1311 ret = -errno;1312 fprintf(stderr,1313 "Failed to get XDP program id for ifindex %d, removing program: %s\n",1314 ifindex, strerror(errno));1315 __sample_remove_xdp(ifindex, 0, xdp_flags);1316 return ret;1317 }1318 sample_xdp_progs[sample_xdp_cnt++] =1319 (struct xdp_desc){ ifindex, prog_id, xdp_flags };1320 1321 return 0;1322}1323 1324static void sample_summary_print(void)1325{1326 double num = sample_out.rx_cnt.num;1327 1328 if (sample_out.totals.rx) {1329 double pkts = sample_out.totals.rx;1330 1331 print_always(" Packets received : %'-10llu\n",1332 sample_out.totals.rx);1333 print_always(" Average packets/s : %'-10.0f\n",1334 sample_round(pkts / num));1335 }1336 if (sample_out.totals.redir) {1337 double pkts = sample_out.totals.redir;1338 1339 print_always(" Packets redirected : %'-10llu\n",1340 sample_out.totals.redir);1341 print_always(" Average redir/s : %'-10.0f\n",1342 sample_round(pkts / num));1343 }1344 if (sample_out.totals.drop)1345 print_always(" Rx dropped : %'-10llu\n",1346 sample_out.totals.drop);1347 if (sample_out.totals.drop_xmit)1348 print_always(" Tx dropped : %'-10llu\n",1349 sample_out.totals.drop_xmit);1350 if (sample_out.totals.err)1351 print_always(" Errors recorded : %'-10llu\n",1352 sample_out.totals.err);1353 if (sample_out.totals.xmit) {1354 double pkts = sample_out.totals.xmit;1355 1356 print_always(" Packets transmitted : %'-10llu\n",1357 sample_out.totals.xmit);1358 print_always(" Average transmit/s : %'-10.0f\n",1359 sample_round(pkts / num));1360 }1361}1362 1363void sample_exit(int status)1364{1365 size_t size;1366 1367 for (int i = 0; i < NUM_MAP; i++) {1368 size = sample_map_count[i] * sizeof(**sample_mmap);1369 munmap(sample_mmap[i], size);1370 }1371 while (sample_xdp_cnt--) {1372 int i = sample_xdp_cnt, ifindex, xdp_flags;1373 __u32 prog_id;1374 1375 prog_id = sample_xdp_progs[i].prog_id;1376 ifindex = sample_xdp_progs[i].ifindex;1377 xdp_flags = sample_xdp_progs[i].flags;1378 1379 __sample_remove_xdp(ifindex, prog_id, xdp_flags);1380 }1381 sample_summary_print();1382 close(sample_sig_fd);1383 exit(status);1384}1385 1386static int sample_stats_collect(struct stats_record *rec)1387{1388 int i;1389 1390 if (sample_mask & SAMPLE_RX_CNT)1391 map_collect_percpu(sample_mmap[MAP_RX], &rec->rx_cnt);1392 1393 if (sample_mask & SAMPLE_REDIRECT_CNT)1394 map_collect_percpu(sample_mmap[MAP_REDIRECT_ERR], &rec->redir_err[0]);1395 1396 if (sample_mask & SAMPLE_REDIRECT_ERR_CNT) {1397 for (i = 1; i < XDP_REDIRECT_ERR_MAX; i++)1398 map_collect_percpu(&sample_mmap[MAP_REDIRECT_ERR][i * sample_n_cpus],1399 &rec->redir_err[i]);1400 }1401 1402 if (sample_mask & SAMPLE_CPUMAP_ENQUEUE_CNT)1403 for (i = 0; i < sample_n_cpus; i++)1404 map_collect_percpu(&sample_mmap[MAP_CPUMAP_ENQUEUE][i * sample_n_cpus],1405 &rec->enq[i]);1406 1407 if (sample_mask & SAMPLE_CPUMAP_KTHREAD_CNT)1408 map_collect_percpu(sample_mmap[MAP_CPUMAP_KTHREAD],1409 &rec->kthread);1410 1411 if (sample_mask & SAMPLE_EXCEPTION_CNT)1412 for (i = 0; i < XDP_ACTION_MAX; i++)1413 map_collect_percpu(&sample_mmap[MAP_EXCEPTION][i * sample_n_cpus],1414 &rec->exception[i]);1415 1416 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT)1417 map_collect_percpu(sample_mmap[MAP_DEVMAP_XMIT], &rec->devmap_xmit);1418 1419 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI) {1420 if (map_collect_percpu_devmap(bpf_map__fd(sample_map[MAP_DEVMAP_XMIT_MULTI]), rec) < 0)1421 return -EINVAL;1422 }1423 return 0;1424}1425 1426static void sample_summary_update(struct sample_output *out)1427{1428 sample_out.totals.rx += out->totals.rx;1429 sample_out.totals.redir += out->totals.redir;1430 sample_out.totals.drop += out->totals.drop;1431 sample_out.totals.drop_xmit += out->totals.drop_xmit;1432 sample_out.totals.err += out->totals.err;1433 sample_out.totals.xmit += out->totals.xmit;1434 sample_out.rx_cnt.num++;1435}1436 1437static void sample_stats_print(int mask, struct stats_record *cur,1438 struct stats_record *prev, char *prog_name)1439{1440 struct sample_output out = {};1441 1442 if (mask & SAMPLE_RX_CNT)1443 stats_get_rx_cnt(cur, prev, 0, &out);1444 if (mask & SAMPLE_REDIRECT_CNT)1445 stats_get_redirect_cnt(cur, prev, 0, &out);1446 if (mask & SAMPLE_REDIRECT_ERR_CNT)1447 stats_get_redirect_err_cnt(cur, prev, 0, &out);1448 if (mask & SAMPLE_EXCEPTION_CNT)1449 stats_get_exception_cnt(cur, prev, 0, &out);1450 if (mask & SAMPLE_DEVMAP_XMIT_CNT)1451 stats_get_devmap_xmit(cur, prev, 0, &out);1452 else if (mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)1453 stats_get_devmap_xmit_multi(cur, prev, 0, &out,1454 mask & SAMPLE_DEVMAP_XMIT_CNT);1455 sample_summary_update(&out);1456 1457 stats_print(prog_name, mask, cur, prev, &out);1458}1459 1460void sample_switch_mode(void)1461{1462 sample_log_level ^= LL_DEBUG - 1;1463}1464 1465static int sample_signal_cb(void)1466{1467 struct signalfd_siginfo si;1468 int r;1469 1470 r = read(sample_sig_fd, &si, sizeof(si));1471 if (r < 0)1472 return -errno;1473 1474 switch (si.ssi_signo) {1475 case SIGQUIT:1476 sample_switch_mode();1477 printf("\n");1478 break;1479 default:1480 printf("\n");1481 return 1;1482 }1483 1484 return 0;1485}1486 1487/* Pointer swap trick */1488static void swap(struct stats_record **a, struct stats_record **b)1489{1490 struct stats_record *tmp;1491 1492 tmp = *a;1493 *a = *b;1494 *b = tmp;1495}1496 1497static int sample_timer_cb(int timerfd, struct stats_record **rec,1498 struct stats_record **prev)1499{1500 char line[64] = "Summary";1501 int ret;1502 __u64 t;1503 1504 ret = read(timerfd, &t, sizeof(t));1505 if (ret < 0)1506 return -errno;1507 1508 swap(prev, rec);1509 ret = sample_stats_collect(*rec);1510 if (ret < 0)1511 return ret;1512 1513 if (sample_xdp_cnt == 2 && !(sample_mask & SAMPLE_SKIP_HEADING)) {1514 char fi[IFNAMSIZ];1515 char to[IFNAMSIZ];1516 const char *f, *t;1517 1518 f = t = NULL;1519 if (if_indextoname(sample_xdp_progs[0].ifindex, fi))1520 f = fi;1521 if (if_indextoname(sample_xdp_progs[1].ifindex, to))1522 t = to;1523 1524 snprintf(line, sizeof(line), "%s->%s", f ?: "?", t ?: "?");1525 }1526 1527 sample_stats_print(sample_mask, *rec, *prev, line);1528 return 0;1529}1530 1531int sample_run(int interval, void (*post_cb)(void *), void *ctx)1532{1533 struct timespec ts = { interval, 0 };1534 struct itimerspec its = { ts, ts };1535 struct stats_record *rec, *prev;1536 struct pollfd pfd[2] = {};1537 int timerfd, ret;1538 1539 if (!interval) {1540 fprintf(stderr, "Incorrect interval 0\n");1541 return -EINVAL;1542 }1543 sample_interval = interval;1544 /* Pretty print numbers */1545 setlocale(LC_NUMERIC, "en_US.UTF-8");1546 1547 timerfd = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC | TFD_NONBLOCK);1548 if (timerfd < 0)1549 return -errno;1550 timerfd_settime(timerfd, 0, &its, NULL);1551 1552 pfd[0].fd = sample_sig_fd;1553 pfd[0].events = POLLIN;1554 1555 pfd[1].fd = timerfd;1556 pfd[1].events = POLLIN;1557 1558 ret = -ENOMEM;1559 rec = alloc_stats_record();1560 if (!rec)1561 goto end;1562 prev = alloc_stats_record();1563 if (!prev)1564 goto end_rec;1565 1566 ret = sample_stats_collect(rec);1567 if (ret < 0)1568 goto end_rec_prev;1569 1570 for (;;) {1571 ret = poll(pfd, 2, -1);1572 if (ret < 0) {1573 if (errno == EINTR)1574 continue;1575 else1576 break;1577 }1578 1579 if (pfd[0].revents & POLLIN)1580 ret = sample_signal_cb();1581 else if (pfd[1].revents & POLLIN)1582 ret = sample_timer_cb(timerfd, &rec, &prev);1583 1584 if (ret)1585 break;1586 1587 if (post_cb)1588 post_cb(ctx);1589 }1590 1591end_rec_prev:1592 free_stats_record(prev);1593end_rec:1594 free_stats_record(rec);1595end:1596 close(timerfd);1597 1598 return ret;1599}1600 1601const char *get_driver_name(int ifindex)1602{1603 struct ethtool_drvinfo drv = {};1604 char ifname[IF_NAMESIZE];1605 static char drvname[32];1606 struct ifreq ifr = {};1607 int fd, r = 0;1608 1609 fd = socket(AF_INET, SOCK_DGRAM, 0);1610 if (fd < 0)1611 return "[error]";1612 1613 if (!if_indextoname(ifindex, ifname))1614 goto end;1615 1616 drv.cmd = ETHTOOL_GDRVINFO;1617 safe_strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name));1618 ifr.ifr_data = (void *)&drv;1619 1620 r = ioctl(fd, SIOCETHTOOL, &ifr);1621 if (r)1622 goto end;1623 1624 safe_strncpy(drvname, drv.driver, sizeof(drvname));1625 1626 close(fd);1627 return drvname;1628 1629end:1630 r = errno;1631 close(fd);1632 return r == EOPNOTSUPP ? "loopback" : "[error]";1633}1634 1635int get_mac_addr(int ifindex, void *mac_addr)1636{1637 char ifname[IF_NAMESIZE];1638 struct ifreq ifr = {};1639 int fd, r;1640 1641 fd = socket(AF_INET, SOCK_DGRAM, 0);1642 if (fd < 0)1643 return -errno;1644 1645 if (!if_indextoname(ifindex, ifname)) {1646 r = -errno;1647 goto end;1648 }1649 1650 safe_strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name));1651 1652 r = ioctl(fd, SIOCGIFHWADDR, &ifr);1653 if (r) {1654 r = -errno;1655 goto end;1656 }1657 1658 memcpy(mac_addr, ifr.ifr_hwaddr.sa_data, 6 * sizeof(char));1659 1660end:1661 close(fd);1662 return r;1663}1664 1665__attribute__((constructor)) static void sample_ctor(void)1666{1667 if (libbpf_set_strict_mode(LIBBPF_STRICT_ALL) < 0) {1668 fprintf(stderr, "Failed to set libbpf strict mode: %s\n",1669 strerror(errno));1670 /* Just exit, nothing to cleanup right now */1671 exit(EXIT_FAIL_BPF);1672 }1673}1674