1731 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) 2020 Facebook */3#include <test_progs.h>4#include <unistd.h>5#include <sys/syscall.h>6#include <task_local_storage_helpers.h>7#include "bpf_iter_ipv6_route.skel.h"8#include "bpf_iter_netlink.skel.h"9#include "bpf_iter_bpf_map.skel.h"10#include "bpf_iter_tasks.skel.h"11#include "bpf_iter_task_stack.skel.h"12#include "bpf_iter_task_file.skel.h"13#include "bpf_iter_task_vmas.skel.h"14#include "bpf_iter_task_btf.skel.h"15#include "bpf_iter_tcp4.skel.h"16#include "bpf_iter_tcp6.skel.h"17#include "bpf_iter_udp4.skel.h"18#include "bpf_iter_udp6.skel.h"19#include "bpf_iter_unix.skel.h"20#include "bpf_iter_vma_offset.skel.h"21#include "bpf_iter_test_kern1.skel.h"22#include "bpf_iter_test_kern2.skel.h"23#include "bpf_iter_test_kern3.skel.h"24#include "bpf_iter_test_kern4.skel.h"25#include "bpf_iter_bpf_hash_map.skel.h"26#include "bpf_iter_bpf_percpu_hash_map.skel.h"27#include "bpf_iter_bpf_array_map.skel.h"28#include "bpf_iter_bpf_percpu_array_map.skel.h"29#include "bpf_iter_bpf_sk_storage_helpers.skel.h"30#include "bpf_iter_bpf_sk_storage_map.skel.h"31#include "bpf_iter_test_kern5.skel.h"32#include "bpf_iter_test_kern6.skel.h"33#include "bpf_iter_bpf_link.skel.h"34#include "bpf_iter_ksym.skel.h"35#include "bpf_iter_sockmap.skel.h"36 37static void test_btf_id_or_null(void)38{39 struct bpf_iter_test_kern3 *skel;40 41 skel = bpf_iter_test_kern3__open_and_load();42 if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern3__open_and_load")) {43 bpf_iter_test_kern3__destroy(skel);44 return;45 }46}47 48static void do_dummy_read_opts(struct bpf_program *prog, struct bpf_iter_attach_opts *opts)49{50 struct bpf_link *link;51 char buf[16] = {};52 int iter_fd, len;53 54 link = bpf_program__attach_iter(prog, opts);55 if (!ASSERT_OK_PTR(link, "attach_iter"))56 return;57 58 iter_fd = bpf_iter_create(bpf_link__fd(link));59 if (!ASSERT_GE(iter_fd, 0, "create_iter"))60 goto free_link;61 62 /* not check contents, but ensure read() ends without error */63 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)64 ;65 ASSERT_GE(len, 0, "read");66 67 close(iter_fd);68 69free_link:70 bpf_link__destroy(link);71}72 73static void do_dummy_read(struct bpf_program *prog)74{75 do_dummy_read_opts(prog, NULL);76}77 78static void do_read_map_iter_fd(struct bpf_object_skeleton **skel, struct bpf_program *prog,79 struct bpf_map *map)80{81 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);82 union bpf_iter_link_info linfo;83 struct bpf_link *link;84 char buf[16] = {};85 int iter_fd, len;86 87 memset(&linfo, 0, sizeof(linfo));88 linfo.map.map_fd = bpf_map__fd(map);89 opts.link_info = &linfo;90 opts.link_info_len = sizeof(linfo);91 link = bpf_program__attach_iter(prog, &opts);92 if (!ASSERT_OK_PTR(link, "attach_map_iter"))93 return;94 95 iter_fd = bpf_iter_create(bpf_link__fd(link));96 if (!ASSERT_GE(iter_fd, 0, "create_map_iter")) {97 bpf_link__destroy(link);98 return;99 }100 101 /* Close link and map fd prematurely */102 bpf_link__destroy(link);103 bpf_object__destroy_skeleton(*skel);104 *skel = NULL;105 106 /* Try to let map free work to run first if map is freed */107 usleep(100);108 /* Memory used by both sock map and sock local storage map are109 * freed after two synchronize_rcu() calls, so wait for it110 */111 kern_sync_rcu();112 kern_sync_rcu();113 114 /* Read after both map fd and link fd are closed */115 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)116 ;117 ASSERT_GE(len, 0, "read_iterator");118 119 close(iter_fd);120}121 122static int read_fd_into_buffer(int fd, char *buf, int size)123{124 int bufleft = size;125 int len;126 127 do {128 len = read(fd, buf, bufleft);129 if (len > 0) {130 buf += len;131 bufleft -= len;132 }133 } while (len > 0);134 135 return len < 0 ? len : size - bufleft;136}137 138static void test_ipv6_route(void)139{140 struct bpf_iter_ipv6_route *skel;141 142 skel = bpf_iter_ipv6_route__open_and_load();143 if (!ASSERT_OK_PTR(skel, "bpf_iter_ipv6_route__open_and_load"))144 return;145 146 do_dummy_read(skel->progs.dump_ipv6_route);147 148 bpf_iter_ipv6_route__destroy(skel);149}150 151static void test_netlink(void)152{153 struct bpf_iter_netlink *skel;154 155 skel = bpf_iter_netlink__open_and_load();156 if (!ASSERT_OK_PTR(skel, "bpf_iter_netlink__open_and_load"))157 return;158 159 do_dummy_read(skel->progs.dump_netlink);160 161 bpf_iter_netlink__destroy(skel);162}163 164static void test_bpf_map(void)165{166 struct bpf_iter_bpf_map *skel;167 168 skel = bpf_iter_bpf_map__open_and_load();169 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_map__open_and_load"))170 return;171 172 do_dummy_read(skel->progs.dump_bpf_map);173 174 bpf_iter_bpf_map__destroy(skel);175}176 177static void check_bpf_link_info(const struct bpf_program *prog)178{179 LIBBPF_OPTS(bpf_iter_attach_opts, opts);180 union bpf_iter_link_info linfo;181 struct bpf_link_info info = {};182 struct bpf_link *link;183 __u32 info_len;184 int err;185 186 memset(&linfo, 0, sizeof(linfo));187 linfo.task.tid = getpid();188 opts.link_info = &linfo;189 opts.link_info_len = sizeof(linfo);190 191 link = bpf_program__attach_iter(prog, &opts);192 if (!ASSERT_OK_PTR(link, "attach_iter"))193 return;194 195 info_len = sizeof(info);196 err = bpf_link_get_info_by_fd(bpf_link__fd(link), &info, &info_len);197 ASSERT_OK(err, "bpf_link_get_info_by_fd");198 ASSERT_EQ(info.iter.task.tid, getpid(), "check_task_tid");199 200 bpf_link__destroy(link);201}202 203static pthread_mutex_t do_nothing_mutex;204 205static void *do_nothing_wait(void *arg)206{207 pthread_mutex_lock(&do_nothing_mutex);208 pthread_mutex_unlock(&do_nothing_mutex);209 210 pthread_exit(arg);211}212 213static void test_task_common_nocheck(struct bpf_iter_attach_opts *opts,214 int *num_unknown, int *num_known)215{216 struct bpf_iter_tasks *skel;217 pthread_t thread_id;218 void *ret;219 220 skel = bpf_iter_tasks__open_and_load();221 if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))222 return;223 224 ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");225 226 ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),227 "pthread_create");228 229 skel->bss->tid = gettid();230 231 do_dummy_read_opts(skel->progs.dump_task, opts);232 233 *num_unknown = skel->bss->num_unknown_tid;234 *num_known = skel->bss->num_known_tid;235 236 ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");237 ASSERT_FALSE(pthread_join(thread_id, &ret) || ret != NULL,238 "pthread_join");239 240 bpf_iter_tasks__destroy(skel);241}242 243static void test_task_common(struct bpf_iter_attach_opts *opts, int num_unknown, int num_known)244{245 int num_unknown_tid, num_known_tid;246 247 test_task_common_nocheck(opts, &num_unknown_tid, &num_known_tid);248 ASSERT_EQ(num_unknown_tid, num_unknown, "check_num_unknown_tid");249 ASSERT_EQ(num_known_tid, num_known, "check_num_known_tid");250}251 252static void *run_test_task_tid(void *arg)253{254 LIBBPF_OPTS(bpf_iter_attach_opts, opts);255 union bpf_iter_link_info linfo;256 int num_unknown_tid, num_known_tid;257 258 ASSERT_NEQ(getpid(), gettid(), "check_new_thread_id");259 260 memset(&linfo, 0, sizeof(linfo));261 linfo.task.tid = gettid();262 opts.link_info = &linfo;263 opts.link_info_len = sizeof(linfo);264 test_task_common(&opts, 0, 1);265 266 linfo.task.tid = 0;267 linfo.task.pid = getpid();268 /* This includes the parent thread, this thread,269 * and the do_nothing_wait thread270 */271 test_task_common(&opts, 2, 1);272 273 test_task_common_nocheck(NULL, &num_unknown_tid, &num_known_tid);274 ASSERT_GT(num_unknown_tid, 2, "check_num_unknown_tid");275 ASSERT_EQ(num_known_tid, 1, "check_num_known_tid");276 277 return NULL;278}279 280static void test_task_tid(void)281{282 pthread_t thread_id;283 284 /* Create a new thread so pid and tid aren't the same */285 ASSERT_OK(pthread_create(&thread_id, NULL, &run_test_task_tid, NULL),286 "pthread_create");287 ASSERT_FALSE(pthread_join(thread_id, NULL), "pthread_join");288}289 290static void test_task_pid(void)291{292 LIBBPF_OPTS(bpf_iter_attach_opts, opts);293 union bpf_iter_link_info linfo;294 295 memset(&linfo, 0, sizeof(linfo));296 linfo.task.pid = getpid();297 opts.link_info = &linfo;298 opts.link_info_len = sizeof(linfo);299 300 test_task_common(&opts, 1, 1);301}302 303static void test_task_pidfd(void)304{305 LIBBPF_OPTS(bpf_iter_attach_opts, opts);306 union bpf_iter_link_info linfo;307 int pidfd;308 309 pidfd = sys_pidfd_open(getpid(), 0);310 if (!ASSERT_GT(pidfd, 0, "sys_pidfd_open"))311 return;312 313 memset(&linfo, 0, sizeof(linfo));314 linfo.task.pid_fd = pidfd;315 opts.link_info = &linfo;316 opts.link_info_len = sizeof(linfo);317 318 test_task_common(&opts, 1, 1);319 320 close(pidfd);321}322 323static void test_task_sleepable(void)324{325 struct bpf_iter_tasks *skel;326 327 skel = bpf_iter_tasks__open_and_load();328 if (!ASSERT_OK_PTR(skel, "bpf_iter_tasks__open_and_load"))329 return;330 331 do_dummy_read(skel->progs.dump_task_sleepable);332 333 ASSERT_GT(skel->bss->num_expected_failure_copy_from_user_task, 0,334 "num_expected_failure_copy_from_user_task");335 ASSERT_GT(skel->bss->num_success_copy_from_user_task, 0,336 "num_success_copy_from_user_task");337 338 bpf_iter_tasks__destroy(skel);339}340 341static void test_task_stack(void)342{343 struct bpf_iter_task_stack *skel;344 345 skel = bpf_iter_task_stack__open_and_load();346 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_stack__open_and_load"))347 return;348 349 do_dummy_read(skel->progs.dump_task_stack);350 do_dummy_read(skel->progs.get_task_user_stacks);351 352 ASSERT_EQ(skel->bss->num_user_stacks, 1, "num_user_stacks");353 354 bpf_iter_task_stack__destroy(skel);355}356 357static void test_task_file(void)358{359 LIBBPF_OPTS(bpf_iter_attach_opts, opts);360 struct bpf_iter_task_file *skel;361 union bpf_iter_link_info linfo;362 pthread_t thread_id;363 void *ret;364 365 skel = bpf_iter_task_file__open_and_load();366 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_file__open_and_load"))367 return;368 369 skel->bss->tgid = getpid();370 371 ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");372 373 ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),374 "pthread_create");375 376 memset(&linfo, 0, sizeof(linfo));377 linfo.task.tid = getpid();378 opts.link_info = &linfo;379 opts.link_info_len = sizeof(linfo);380 381 do_dummy_read_opts(skel->progs.dump_task_file, &opts);382 383 ASSERT_EQ(skel->bss->count, 0, "check_count");384 ASSERT_EQ(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");385 386 skel->bss->last_tgid = 0;387 skel->bss->count = 0;388 skel->bss->unique_tgid_count = 0;389 390 do_dummy_read(skel->progs.dump_task_file);391 392 ASSERT_EQ(skel->bss->count, 0, "check_count");393 ASSERT_GT(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");394 395 check_bpf_link_info(skel->progs.dump_task_file);396 397 ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");398 ASSERT_OK(pthread_join(thread_id, &ret), "pthread_join");399 ASSERT_NULL(ret, "pthread_join");400 401 bpf_iter_task_file__destroy(skel);402}403 404#define TASKBUFSZ 32768405 406static char taskbuf[TASKBUFSZ];407 408static int do_btf_read(struct bpf_iter_task_btf *skel)409{410 struct bpf_program *prog = skel->progs.dump_task_struct;411 struct bpf_iter_task_btf__bss *bss = skel->bss;412 int iter_fd = -1, err;413 struct bpf_link *link;414 char *buf = taskbuf;415 int ret = 0;416 417 link = bpf_program__attach_iter(prog, NULL);418 if (!ASSERT_OK_PTR(link, "attach_iter"))419 return ret;420 421 iter_fd = bpf_iter_create(bpf_link__fd(link));422 if (!ASSERT_GE(iter_fd, 0, "create_iter"))423 goto free_link;424 425 err = read_fd_into_buffer(iter_fd, buf, TASKBUFSZ);426 if (bss->skip) {427 printf("%s:SKIP:no __builtin_btf_type_id\n", __func__);428 ret = 1;429 test__skip();430 goto free_link;431 }432 433 if (!ASSERT_GE(err, 0, "read"))434 goto free_link;435 436 ASSERT_HAS_SUBSTR(taskbuf, "(struct task_struct)",437 "check for btf representation of task_struct in iter data");438free_link:439 if (iter_fd > 0)440 close(iter_fd);441 bpf_link__destroy(link);442 return ret;443}444 445static void test_task_btf(void)446{447 struct bpf_iter_task_btf__bss *bss;448 struct bpf_iter_task_btf *skel;449 int ret;450 451 skel = bpf_iter_task_btf__open_and_load();452 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_btf__open_and_load"))453 return;454 455 bss = skel->bss;456 457 ret = do_btf_read(skel);458 if (ret)459 goto cleanup;460 461 if (!ASSERT_NEQ(bss->tasks, 0, "no task iteration, did BPF program run?"))462 goto cleanup;463 464 ASSERT_EQ(bss->seq_err, 0, "check for unexpected err");465 466cleanup:467 bpf_iter_task_btf__destroy(skel);468}469 470static void test_tcp4(void)471{472 struct bpf_iter_tcp4 *skel;473 474 skel = bpf_iter_tcp4__open_and_load();475 if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp4__open_and_load"))476 return;477 478 do_dummy_read(skel->progs.dump_tcp4);479 480 bpf_iter_tcp4__destroy(skel);481}482 483static void test_tcp6(void)484{485 struct bpf_iter_tcp6 *skel;486 487 skel = bpf_iter_tcp6__open_and_load();488 if (!ASSERT_OK_PTR(skel, "bpf_iter_tcp6__open_and_load"))489 return;490 491 do_dummy_read(skel->progs.dump_tcp6);492 493 bpf_iter_tcp6__destroy(skel);494}495 496static void test_udp4(void)497{498 struct bpf_iter_udp4 *skel;499 500 skel = bpf_iter_udp4__open_and_load();501 if (!ASSERT_OK_PTR(skel, "bpf_iter_udp4__open_and_load"))502 return;503 504 do_dummy_read(skel->progs.dump_udp4);505 506 bpf_iter_udp4__destroy(skel);507}508 509static void test_udp6(void)510{511 struct bpf_iter_udp6 *skel;512 513 skel = bpf_iter_udp6__open_and_load();514 if (!ASSERT_OK_PTR(skel, "bpf_iter_udp6__open_and_load"))515 return;516 517 do_dummy_read(skel->progs.dump_udp6);518 519 bpf_iter_udp6__destroy(skel);520}521 522static void test_unix(void)523{524 struct bpf_iter_unix *skel;525 526 skel = bpf_iter_unix__open_and_load();527 if (!ASSERT_OK_PTR(skel, "bpf_iter_unix__open_and_load"))528 return;529 530 do_dummy_read(skel->progs.dump_unix);531 532 bpf_iter_unix__destroy(skel);533}534 535/* The expected string is less than 16 bytes */536static int do_read_with_fd(int iter_fd, const char *expected,537 bool read_one_char)538{539 int len, read_buf_len, start;540 char buf[16] = {};541 542 read_buf_len = read_one_char ? 1 : 16;543 start = 0;544 while ((len = read(iter_fd, buf + start, read_buf_len)) > 0) {545 start += len;546 if (!ASSERT_LT(start, 16, "read"))547 return -1;548 read_buf_len = read_one_char ? 1 : 16 - start;549 }550 if (!ASSERT_GE(len, 0, "read"))551 return -1;552 553 if (!ASSERT_STREQ(buf, expected, "read"))554 return -1;555 556 return 0;557}558 559static void test_anon_iter(bool read_one_char)560{561 struct bpf_iter_test_kern1 *skel;562 struct bpf_link *link;563 int iter_fd, err;564 565 skel = bpf_iter_test_kern1__open_and_load();566 if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern1__open_and_load"))567 return;568 569 err = bpf_iter_test_kern1__attach(skel);570 if (!ASSERT_OK(err, "bpf_iter_test_kern1__attach")) {571 goto out;572 }573 574 link = skel->links.dump_task;575 iter_fd = bpf_iter_create(bpf_link__fd(link));576 if (!ASSERT_GE(iter_fd, 0, "create_iter"))577 goto out;578 579 do_read_with_fd(iter_fd, "abcd", read_one_char);580 close(iter_fd);581 582out:583 bpf_iter_test_kern1__destroy(skel);584}585 586static int do_read(const char *path, const char *expected)587{588 int err, iter_fd;589 590 iter_fd = open(path, O_RDONLY);591 if (!ASSERT_GE(iter_fd, 0, "open"))592 return -1;593 594 err = do_read_with_fd(iter_fd, expected, false);595 close(iter_fd);596 return err;597}598 599static void test_file_iter(void)600{601 const char *path = "/sys/fs/bpf/bpf_iter_test1";602 struct bpf_iter_test_kern1 *skel1;603 struct bpf_iter_test_kern2 *skel2;604 struct bpf_link *link;605 int err;606 607 skel1 = bpf_iter_test_kern1__open_and_load();608 if (!ASSERT_OK_PTR(skel1, "bpf_iter_test_kern1__open_and_load"))609 return;610 611 link = bpf_program__attach_iter(skel1->progs.dump_task, NULL);612 if (!ASSERT_OK_PTR(link, "attach_iter"))613 goto out;614 615 /* unlink this path if it exists. */616 unlink(path);617 618 err = bpf_link__pin(link, path);619 if (!ASSERT_OK(err, "pin_iter"))620 goto free_link;621 622 err = do_read(path, "abcd");623 if (err)624 goto unlink_path;625 626 /* file based iterator seems working fine. Let us a link update627 * of the underlying link and `cat` the iterator again, its content628 * should change.629 */630 skel2 = bpf_iter_test_kern2__open_and_load();631 if (!ASSERT_OK_PTR(skel2, "bpf_iter_test_kern2__open_and_load"))632 goto unlink_path;633 634 err = bpf_link__update_program(link, skel2->progs.dump_task);635 if (!ASSERT_OK(err, "update_prog"))636 goto destroy_skel2;637 638 do_read(path, "ABCD");639 640destroy_skel2:641 bpf_iter_test_kern2__destroy(skel2);642unlink_path:643 unlink(path);644free_link:645 bpf_link__destroy(link);646out:647 bpf_iter_test_kern1__destroy(skel1);648}649 650static void test_overflow(bool test_e2big_overflow, bool ret1)651{652 __u32 map_info_len, total_read_len, expected_read_len;653 int err, iter_fd, map1_fd, map2_fd, len;654 struct bpf_map_info map_info = {};655 struct bpf_iter_test_kern4 *skel;656 struct bpf_link *link;657 __u32 iter_size;658 char *buf;659 660 skel = bpf_iter_test_kern4__open();661 if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern4__open"))662 return;663 664 /* create two maps: bpf program will only do bpf_seq_write665 * for these two maps. The goal is one map output almost666 * fills seq_file buffer and then the other will trigger667 * overflow and needs restart.668 */669 map1_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);670 if (!ASSERT_GE(map1_fd, 0, "bpf_map_create"))671 goto out;672 map2_fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, 4, 8, 1, NULL);673 if (!ASSERT_GE(map2_fd, 0, "bpf_map_create"))674 goto free_map1;675 676 /* bpf_seq_printf kernel buffer is 8 pages, so one map677 * bpf_seq_write will mostly fill it, and the other map678 * will partially fill and then trigger overflow and need679 * bpf_seq_read restart.680 */681 iter_size = sysconf(_SC_PAGE_SIZE) << 3;682 683 if (test_e2big_overflow) {684 skel->rodata->print_len = (iter_size + 8) / 8;685 expected_read_len = 2 * (iter_size + 8);686 } else if (!ret1) {687 skel->rodata->print_len = (iter_size - 8) / 8;688 expected_read_len = 2 * (iter_size - 8);689 } else {690 skel->rodata->print_len = 1;691 expected_read_len = 2 * 8;692 }693 skel->rodata->ret1 = ret1;694 695 if (!ASSERT_OK(bpf_iter_test_kern4__load(skel),696 "bpf_iter_test_kern4__load"))697 goto free_map2;698 699 /* setup filtering map_id in bpf program */700 map_info_len = sizeof(map_info);701 err = bpf_map_get_info_by_fd(map1_fd, &map_info, &map_info_len);702 if (!ASSERT_OK(err, "get_map_info"))703 goto free_map2;704 skel->bss->map1_id = map_info.id;705 706 err = bpf_map_get_info_by_fd(map2_fd, &map_info, &map_info_len);707 if (!ASSERT_OK(err, "get_map_info"))708 goto free_map2;709 skel->bss->map2_id = map_info.id;710 711 link = bpf_program__attach_iter(skel->progs.dump_bpf_map, NULL);712 if (!ASSERT_OK_PTR(link, "attach_iter"))713 goto free_map2;714 715 iter_fd = bpf_iter_create(bpf_link__fd(link));716 if (!ASSERT_GE(iter_fd, 0, "create_iter"))717 goto free_link;718 719 buf = malloc(expected_read_len);720 if (!ASSERT_OK_PTR(buf, "malloc"))721 goto close_iter;722 723 /* do read */724 total_read_len = 0;725 if (test_e2big_overflow) {726 while ((len = read(iter_fd, buf, expected_read_len)) > 0)727 total_read_len += len;728 729 ASSERT_EQ(len, -1, "read");730 ASSERT_EQ(errno, E2BIG, "read");731 goto free_buf;732 } else if (!ret1) {733 while ((len = read(iter_fd, buf, expected_read_len)) > 0)734 total_read_len += len;735 736 if (!ASSERT_GE(len, 0, "read"))737 goto free_buf;738 } else {739 do {740 len = read(iter_fd, buf, expected_read_len);741 if (len > 0)742 total_read_len += len;743 } while (len > 0 || len == -EAGAIN);744 745 if (!ASSERT_GE(len, 0, "read"))746 goto free_buf;747 }748 749 if (!ASSERT_EQ(total_read_len, expected_read_len, "read"))750 goto free_buf;751 752 if (!ASSERT_EQ(skel->bss->map1_accessed, 1, "map1_accessed"))753 goto free_buf;754 755 if (!ASSERT_EQ(skel->bss->map2_accessed, 2, "map2_accessed"))756 goto free_buf;757 758 ASSERT_EQ(skel->bss->map2_seqnum1, skel->bss->map2_seqnum2, "map2_seqnum");759 760free_buf:761 free(buf);762close_iter:763 close(iter_fd);764free_link:765 bpf_link__destroy(link);766free_map2:767 close(map2_fd);768free_map1:769 close(map1_fd);770out:771 bpf_iter_test_kern4__destroy(skel);772}773 774static void test_bpf_hash_map(void)775{776 __u32 expected_key_a = 0, expected_key_b = 0;777 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);778 struct bpf_iter_bpf_hash_map *skel;779 int err, i, len, map_fd, iter_fd;780 union bpf_iter_link_info linfo;781 __u64 val, expected_val = 0;782 struct bpf_link *link;783 struct key_t {784 int a;785 int b;786 int c;787 } key;788 char buf[64];789 790 skel = bpf_iter_bpf_hash_map__open();791 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_hash_map__open"))792 return;793 794 skel->bss->in_test_mode = true;795 796 err = bpf_iter_bpf_hash_map__load(skel);797 if (!ASSERT_OK(err, "bpf_iter_bpf_hash_map__load"))798 goto out;799 800 /* iterator with hashmap2 and hashmap3 should fail */801 memset(&linfo, 0, sizeof(linfo));802 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap2);803 opts.link_info = &linfo;804 opts.link_info_len = sizeof(linfo);805 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);806 if (!ASSERT_ERR_PTR(link, "attach_iter"))807 goto out;808 809 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap3);810 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);811 if (!ASSERT_ERR_PTR(link, "attach_iter"))812 goto out;813 814 /* hashmap1 should be good, update map values here */815 map_fd = bpf_map__fd(skel->maps.hashmap1);816 for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) {817 key.a = i + 1;818 key.b = i + 2;819 key.c = i + 3;820 val = i + 4;821 expected_key_a += key.a;822 expected_key_b += key.b;823 expected_val += val;824 825 err = bpf_map_update_elem(map_fd, &key, &val, BPF_ANY);826 if (!ASSERT_OK(err, "map_update"))827 goto out;828 }829 830 /* Sleepable program is prohibited for hash map iterator */831 linfo.map.map_fd = map_fd;832 link = bpf_program__attach_iter(skel->progs.sleepable_dummy_dump, &opts);833 if (!ASSERT_ERR_PTR(link, "attach_sleepable_prog_to_iter"))834 goto out;835 836 linfo.map.map_fd = map_fd;837 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);838 if (!ASSERT_OK_PTR(link, "attach_iter"))839 goto out;840 841 iter_fd = bpf_iter_create(bpf_link__fd(link));842 if (!ASSERT_GE(iter_fd, 0, "create_iter"))843 goto free_link;844 845 /* do some tests */846 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)847 ;848 if (!ASSERT_GE(len, 0, "read"))849 goto close_iter;850 851 /* test results */852 if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a"))853 goto close_iter;854 if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b"))855 goto close_iter;856 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))857 goto close_iter;858 859close_iter:860 close(iter_fd);861free_link:862 bpf_link__destroy(link);863out:864 bpf_iter_bpf_hash_map__destroy(skel);865}866 867static void test_bpf_percpu_hash_map(void)868{869 __u32 expected_key_a = 0, expected_key_b = 0;870 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);871 struct bpf_iter_bpf_percpu_hash_map *skel;872 int err, i, j, len, map_fd, iter_fd;873 union bpf_iter_link_info linfo;874 __u32 expected_val = 0;875 struct bpf_link *link;876 struct key_t {877 int a;878 int b;879 int c;880 } key;881 char buf[64];882 void *val;883 884 skel = bpf_iter_bpf_percpu_hash_map__open();885 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__open"))886 return;887 888 skel->rodata->num_cpus = bpf_num_possible_cpus();889 val = malloc(8 * bpf_num_possible_cpus());890 if (!ASSERT_OK_PTR(val, "malloc"))891 goto out;892 893 err = bpf_iter_bpf_percpu_hash_map__load(skel);894 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_hash_map__load"))895 goto out;896 897 /* update map values here */898 map_fd = bpf_map__fd(skel->maps.hashmap1);899 for (i = 0; i < bpf_map__max_entries(skel->maps.hashmap1); i++) {900 key.a = i + 1;901 key.b = i + 2;902 key.c = i + 3;903 expected_key_a += key.a;904 expected_key_b += key.b;905 906 for (j = 0; j < bpf_num_possible_cpus(); j++) {907 *(__u32 *)(val + j * 8) = i + j;908 expected_val += i + j;909 }910 911 err = bpf_map_update_elem(map_fd, &key, val, BPF_ANY);912 if (!ASSERT_OK(err, "map_update"))913 goto out;914 }915 916 memset(&linfo, 0, sizeof(linfo));917 linfo.map.map_fd = map_fd;918 opts.link_info = &linfo;919 opts.link_info_len = sizeof(linfo);920 link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_hash_map, &opts);921 if (!ASSERT_OK_PTR(link, "attach_iter"))922 goto out;923 924 iter_fd = bpf_iter_create(bpf_link__fd(link));925 if (!ASSERT_GE(iter_fd, 0, "create_iter"))926 goto free_link;927 928 /* do some tests */929 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)930 ;931 if (!ASSERT_GE(len, 0, "read"))932 goto close_iter;933 934 /* test results */935 if (!ASSERT_EQ(skel->bss->key_sum_a, expected_key_a, "key_sum_a"))936 goto close_iter;937 if (!ASSERT_EQ(skel->bss->key_sum_b, expected_key_b, "key_sum_b"))938 goto close_iter;939 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))940 goto close_iter;941 942close_iter:943 close(iter_fd);944free_link:945 bpf_link__destroy(link);946out:947 bpf_iter_bpf_percpu_hash_map__destroy(skel);948 free(val);949}950 951static void test_bpf_array_map(void)952{953 __u64 val, expected_val = 0, res_first_val, first_val = 0;954 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);955 __u32 key, expected_key = 0, res_first_key;956 int err, i, map_fd, hash_fd, iter_fd;957 struct bpf_iter_bpf_array_map *skel;958 union bpf_iter_link_info linfo;959 struct bpf_link *link;960 char buf[64] = {};961 int len, start;962 963 skel = bpf_iter_bpf_array_map__open_and_load();964 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load"))965 return;966 967 map_fd = bpf_map__fd(skel->maps.arraymap1);968 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {969 val = i + 4;970 expected_key += i;971 expected_val += val;972 973 if (i == 0)974 first_val = val;975 976 err = bpf_map_update_elem(map_fd, &i, &val, BPF_ANY);977 if (!ASSERT_OK(err, "map_update"))978 goto out;979 }980 981 memset(&linfo, 0, sizeof(linfo));982 linfo.map.map_fd = map_fd;983 opts.link_info = &linfo;984 opts.link_info_len = sizeof(linfo);985 link = bpf_program__attach_iter(skel->progs.dump_bpf_array_map, &opts);986 if (!ASSERT_OK_PTR(link, "attach_iter"))987 goto out;988 989 iter_fd = bpf_iter_create(bpf_link__fd(link));990 if (!ASSERT_GE(iter_fd, 0, "create_iter"))991 goto free_link;992 993 /* do some tests */994 start = 0;995 while ((len = read(iter_fd, buf + start, sizeof(buf) - start)) > 0)996 start += len;997 if (!ASSERT_GE(len, 0, "read"))998 goto close_iter;999 1000 /* test results */1001 res_first_key = *(__u32 *)buf;1002 res_first_val = *(__u64 *)(buf + sizeof(__u32));1003 if (!ASSERT_EQ(res_first_key, 0, "bpf_seq_write") ||1004 !ASSERT_EQ(res_first_val, first_val, "bpf_seq_write"))1005 goto close_iter;1006 1007 if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum"))1008 goto close_iter;1009 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))1010 goto close_iter;1011 1012 hash_fd = bpf_map__fd(skel->maps.hashmap1);1013 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {1014 err = bpf_map_lookup_elem(map_fd, &i, &val);1015 if (!ASSERT_OK(err, "map_lookup arraymap1"))1016 goto close_iter;1017 if (!ASSERT_EQ(i, val, "invalid_val arraymap1"))1018 goto close_iter;1019 1020 val = i + 4;1021 err = bpf_map_lookup_elem(hash_fd, &val, &key);1022 if (!ASSERT_OK(err, "map_lookup hashmap1"))1023 goto close_iter;1024 if (!ASSERT_EQ(key, val - 4, "invalid_val hashmap1"))1025 goto close_iter;1026 }1027 1028close_iter:1029 close(iter_fd);1030free_link:1031 bpf_link__destroy(link);1032out:1033 bpf_iter_bpf_array_map__destroy(skel);1034}1035 1036static void test_bpf_array_map_iter_fd(void)1037{1038 struct bpf_iter_bpf_array_map *skel;1039 1040 skel = bpf_iter_bpf_array_map__open_and_load();1041 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_array_map__open_and_load"))1042 return;1043 1044 do_read_map_iter_fd(&skel->skeleton, skel->progs.dump_bpf_array_map,1045 skel->maps.arraymap1);1046 1047 bpf_iter_bpf_array_map__destroy(skel);1048}1049 1050static void test_bpf_percpu_array_map(void)1051{1052 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);1053 struct bpf_iter_bpf_percpu_array_map *skel;1054 __u32 expected_key = 0, expected_val = 0;1055 union bpf_iter_link_info linfo;1056 int err, i, j, map_fd, iter_fd;1057 struct bpf_link *link;1058 char buf[64];1059 void *val;1060 int len;1061 1062 skel = bpf_iter_bpf_percpu_array_map__open();1063 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__open"))1064 return;1065 1066 skel->rodata->num_cpus = bpf_num_possible_cpus();1067 val = malloc(8 * bpf_num_possible_cpus());1068 if (!ASSERT_OK_PTR(val, "malloc"))1069 goto out;1070 1071 err = bpf_iter_bpf_percpu_array_map__load(skel);1072 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_percpu_array_map__load"))1073 goto out;1074 1075 /* update map values here */1076 map_fd = bpf_map__fd(skel->maps.arraymap1);1077 for (i = 0; i < bpf_map__max_entries(skel->maps.arraymap1); i++) {1078 expected_key += i;1079 1080 for (j = 0; j < bpf_num_possible_cpus(); j++) {1081 *(__u32 *)(val + j * 8) = i + j;1082 expected_val += i + j;1083 }1084 1085 err = bpf_map_update_elem(map_fd, &i, val, BPF_ANY);1086 if (!ASSERT_OK(err, "map_update"))1087 goto out;1088 }1089 1090 memset(&linfo, 0, sizeof(linfo));1091 linfo.map.map_fd = map_fd;1092 opts.link_info = &linfo;1093 opts.link_info_len = sizeof(linfo);1094 link = bpf_program__attach_iter(skel->progs.dump_bpf_percpu_array_map, &opts);1095 if (!ASSERT_OK_PTR(link, "attach_iter"))1096 goto out;1097 1098 iter_fd = bpf_iter_create(bpf_link__fd(link));1099 if (!ASSERT_GE(iter_fd, 0, "create_iter"))1100 goto free_link;1101 1102 /* do some tests */1103 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)1104 ;1105 if (!ASSERT_GE(len, 0, "read"))1106 goto close_iter;1107 1108 /* test results */1109 if (!ASSERT_EQ(skel->bss->key_sum, expected_key, "key_sum"))1110 goto close_iter;1111 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))1112 goto close_iter;1113 1114close_iter:1115 close(iter_fd);1116free_link:1117 bpf_link__destroy(link);1118out:1119 bpf_iter_bpf_percpu_array_map__destroy(skel);1120 free(val);1121}1122 1123/* An iterator program deletes all local storage in a map. */1124static void test_bpf_sk_storage_delete(void)1125{1126 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);1127 struct bpf_iter_bpf_sk_storage_helpers *skel;1128 union bpf_iter_link_info linfo;1129 int err, len, map_fd, iter_fd;1130 struct bpf_link *link;1131 int sock_fd = -1;1132 __u32 val = 42;1133 char buf[64];1134 1135 skel = bpf_iter_bpf_sk_storage_helpers__open_and_load();1136 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load"))1137 return;1138 1139 map_fd = bpf_map__fd(skel->maps.sk_stg_map);1140 1141 sock_fd = socket(AF_INET6, SOCK_STREAM, 0);1142 if (!ASSERT_GE(sock_fd, 0, "socket"))1143 goto out;1144 1145 err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST);1146 if (!ASSERT_OK(err, "map_update"))1147 goto out;1148 1149 memset(&linfo, 0, sizeof(linfo));1150 linfo.map.map_fd = map_fd;1151 opts.link_info = &linfo;1152 opts.link_info_len = sizeof(linfo);1153 link = bpf_program__attach_iter(skel->progs.delete_bpf_sk_storage_map,1154 &opts);1155 if (!ASSERT_OK_PTR(link, "attach_iter"))1156 goto out;1157 1158 iter_fd = bpf_iter_create(bpf_link__fd(link));1159 if (!ASSERT_GE(iter_fd, 0, "create_iter"))1160 goto free_link;1161 1162 /* do some tests */1163 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)1164 ;1165 if (!ASSERT_GE(len, 0, "read"))1166 goto close_iter;1167 1168 /* test results */1169 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);1170 1171 /* Note: The following assertions serve to ensure1172 * the value was deleted. It does so by asserting1173 * that bpf_map_lookup_elem has failed. This might1174 * seem counterintuitive at first.1175 */1176 ASSERT_ERR(err, "bpf_map_lookup_elem");1177 ASSERT_EQ(errno, ENOENT, "bpf_map_lookup_elem");1178 1179close_iter:1180 close(iter_fd);1181free_link:1182 bpf_link__destroy(link);1183out:1184 if (sock_fd >= 0)1185 close(sock_fd);1186 bpf_iter_bpf_sk_storage_helpers__destroy(skel);1187}1188 1189/* This creates a socket and its local storage. It then runs a task_iter BPF1190 * program that replaces the existing socket local storage with the tgid of the1191 * only task owning a file descriptor to this socket, this process, prog_tests.1192 * It then runs a tcp socket iterator that negates the value in the existing1193 * socket local storage, the test verifies that the resulting value is -pid.1194 */1195static void test_bpf_sk_storage_get(void)1196{1197 struct bpf_iter_bpf_sk_storage_helpers *skel;1198 int err, map_fd, val = -1;1199 int sock_fd = -1;1200 1201 skel = bpf_iter_bpf_sk_storage_helpers__open_and_load();1202 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_helpers__open_and_load"))1203 return;1204 1205 sock_fd = socket(AF_INET6, SOCK_STREAM, 0);1206 if (!ASSERT_GE(sock_fd, 0, "socket"))1207 goto out;1208 1209 err = listen(sock_fd, 1);1210 if (!ASSERT_OK(err, "listen"))1211 goto close_socket;1212 1213 map_fd = bpf_map__fd(skel->maps.sk_stg_map);1214 1215 err = bpf_map_update_elem(map_fd, &sock_fd, &val, BPF_NOEXIST);1216 if (!ASSERT_OK(err, "bpf_map_update_elem"))1217 goto close_socket;1218 1219 do_dummy_read(skel->progs.fill_socket_owner);1220 1221 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);1222 if (!ASSERT_OK(err, "bpf_map_lookup_elem") ||1223 !ASSERT_EQ(val, getpid(), "bpf_map_lookup_elem"))1224 goto close_socket;1225 1226 do_dummy_read(skel->progs.negate_socket_local_storage);1227 1228 err = bpf_map_lookup_elem(map_fd, &sock_fd, &val);1229 ASSERT_OK(err, "bpf_map_lookup_elem");1230 ASSERT_EQ(val, -getpid(), "bpf_map_lookup_elem");1231 1232close_socket:1233 close(sock_fd);1234out:1235 bpf_iter_bpf_sk_storage_helpers__destroy(skel);1236}1237 1238static void test_bpf_sk_storage_map_iter_fd(void)1239{1240 struct bpf_iter_bpf_sk_storage_map *skel;1241 1242 skel = bpf_iter_bpf_sk_storage_map__open_and_load();1243 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load"))1244 return;1245 1246 do_read_map_iter_fd(&skel->skeleton, skel->progs.rw_bpf_sk_storage_map,1247 skel->maps.sk_stg_map);1248 1249 bpf_iter_bpf_sk_storage_map__destroy(skel);1250}1251 1252static void test_bpf_sk_storage_map(void)1253{1254 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);1255 int err, i, len, map_fd, iter_fd, num_sockets;1256 struct bpf_iter_bpf_sk_storage_map *skel;1257 union bpf_iter_link_info linfo;1258 int sock_fd[3] = {-1, -1, -1};1259 __u32 val, expected_val = 0;1260 struct bpf_link *link;1261 char buf[64];1262 1263 skel = bpf_iter_bpf_sk_storage_map__open_and_load();1264 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_sk_storage_map__open_and_load"))1265 return;1266 1267 map_fd = bpf_map__fd(skel->maps.sk_stg_map);1268 num_sockets = ARRAY_SIZE(sock_fd);1269 for (i = 0; i < num_sockets; i++) {1270 sock_fd[i] = socket(AF_INET6, SOCK_STREAM, 0);1271 if (!ASSERT_GE(sock_fd[i], 0, "socket"))1272 goto out;1273 1274 val = i + 1;1275 expected_val += val;1276 1277 err = bpf_map_update_elem(map_fd, &sock_fd[i], &val,1278 BPF_NOEXIST);1279 if (!ASSERT_OK(err, "map_update"))1280 goto out;1281 }1282 1283 memset(&linfo, 0, sizeof(linfo));1284 linfo.map.map_fd = map_fd;1285 opts.link_info = &linfo;1286 opts.link_info_len = sizeof(linfo);1287 link = bpf_program__attach_iter(skel->progs.oob_write_bpf_sk_storage_map, &opts);1288 err = libbpf_get_error(link);1289 if (!ASSERT_EQ(err, -EACCES, "attach_oob_write_iter")) {1290 if (!err)1291 bpf_link__destroy(link);1292 goto out;1293 }1294 1295 link = bpf_program__attach_iter(skel->progs.rw_bpf_sk_storage_map, &opts);1296 if (!ASSERT_OK_PTR(link, "attach_iter"))1297 goto out;1298 1299 iter_fd = bpf_iter_create(bpf_link__fd(link));1300 if (!ASSERT_GE(iter_fd, 0, "create_iter"))1301 goto free_link;1302 1303 skel->bss->to_add_val = time(NULL);1304 /* do some tests */1305 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)1306 ;1307 if (!ASSERT_GE(len, 0, "read"))1308 goto close_iter;1309 1310 /* test results */1311 if (!ASSERT_EQ(skel->bss->ipv6_sk_count, num_sockets, "ipv6_sk_count"))1312 goto close_iter;1313 1314 if (!ASSERT_EQ(skel->bss->val_sum, expected_val, "val_sum"))1315 goto close_iter;1316 1317 for (i = 0; i < num_sockets; i++) {1318 err = bpf_map_lookup_elem(map_fd, &sock_fd[i], &val);1319 if (!ASSERT_OK(err, "map_lookup") ||1320 !ASSERT_EQ(val, i + 1 + skel->bss->to_add_val, "check_map_value"))1321 break;1322 }1323 1324close_iter:1325 close(iter_fd);1326free_link:1327 bpf_link__destroy(link);1328out:1329 for (i = 0; i < num_sockets; i++) {1330 if (sock_fd[i] >= 0)1331 close(sock_fd[i]);1332 }1333 bpf_iter_bpf_sk_storage_map__destroy(skel);1334}1335 1336static void test_rdonly_buf_out_of_bound(void)1337{1338 DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);1339 struct bpf_iter_test_kern5 *skel;1340 union bpf_iter_link_info linfo;1341 struct bpf_link *link;1342 1343 skel = bpf_iter_test_kern5__open_and_load();1344 if (!ASSERT_OK_PTR(skel, "bpf_iter_test_kern5__open_and_load"))1345 return;1346 1347 memset(&linfo, 0, sizeof(linfo));1348 linfo.map.map_fd = bpf_map__fd(skel->maps.hashmap1);1349 opts.link_info = &linfo;1350 opts.link_info_len = sizeof(linfo);1351 link = bpf_program__attach_iter(skel->progs.dump_bpf_hash_map, &opts);1352 if (!ASSERT_ERR_PTR(link, "attach_iter"))1353 bpf_link__destroy(link);1354 1355 bpf_iter_test_kern5__destroy(skel);1356}1357 1358static void test_buf_neg_offset(void)1359{1360 struct bpf_iter_test_kern6 *skel;1361 1362 skel = bpf_iter_test_kern6__open_and_load();1363 if (!ASSERT_ERR_PTR(skel, "bpf_iter_test_kern6__open_and_load"))1364 bpf_iter_test_kern6__destroy(skel);1365}1366 1367static void test_link_iter(void)1368{1369 struct bpf_iter_bpf_link *skel;1370 1371 skel = bpf_iter_bpf_link__open_and_load();1372 if (!ASSERT_OK_PTR(skel, "bpf_iter_bpf_link__open_and_load"))1373 return;1374 1375 do_dummy_read(skel->progs.dump_bpf_link);1376 1377 bpf_iter_bpf_link__destroy(skel);1378}1379 1380static void test_ksym_iter(void)1381{1382 struct bpf_iter_ksym *skel;1383 1384 skel = bpf_iter_ksym__open_and_load();1385 if (!ASSERT_OK_PTR(skel, "bpf_iter_ksym__open_and_load"))1386 return;1387 1388 do_dummy_read(skel->progs.dump_ksym);1389 1390 bpf_iter_ksym__destroy(skel);1391}1392 1393#define CMP_BUFFER_SIZE 10241394static char task_vma_output[CMP_BUFFER_SIZE];1395static char proc_maps_output[CMP_BUFFER_SIZE];1396 1397/* remove \0 and \t from str, and only keep the first line */1398static void str_strip_first_line(char *str)1399{1400 char *dst = str, *src = str;1401 1402 do {1403 if (*src == ' ' || *src == '\t')1404 src++;1405 else1406 *(dst++) = *(src++);1407 1408 } while (*src != '\0' && *src != '\n');1409 1410 *dst = '\0';1411}1412 1413static void test_task_vma_common(struct bpf_iter_attach_opts *opts)1414{1415 int err, iter_fd = -1, proc_maps_fd = -1;1416 struct bpf_iter_task_vmas *skel;1417 int len, read_size = 4;1418 char maps_path[64];1419 1420 skel = bpf_iter_task_vmas__open();1421 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vmas__open"))1422 return;1423 1424 skel->bss->pid = getpid();1425 skel->bss->one_task = opts ? 1 : 0;1426 1427 err = bpf_iter_task_vmas__load(skel);1428 if (!ASSERT_OK(err, "bpf_iter_task_vmas__load"))1429 goto out;1430 1431 skel->links.proc_maps = bpf_program__attach_iter(1432 skel->progs.proc_maps, opts);1433 1434 if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) {1435 skel->links.proc_maps = NULL;1436 goto out;1437 }1438 1439 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));1440 if (!ASSERT_GE(iter_fd, 0, "create_iter"))1441 goto out;1442 1443 /* Read CMP_BUFFER_SIZE (1kB) from bpf_iter. Read in small chunks1444 * to trigger seq_file corner cases.1445 */1446 len = 0;1447 while (len < CMP_BUFFER_SIZE) {1448 err = read_fd_into_buffer(iter_fd, task_vma_output + len,1449 MIN(read_size, CMP_BUFFER_SIZE - len));1450 if (!err)1451 break;1452 if (!ASSERT_GE(err, 0, "read_iter_fd"))1453 goto out;1454 len += err;1455 }1456 if (opts)1457 ASSERT_EQ(skel->bss->one_task_error, 0, "unexpected task");1458 1459 /* read CMP_BUFFER_SIZE (1kB) from /proc/pid/maps */1460 snprintf(maps_path, 64, "/proc/%u/maps", skel->bss->pid);1461 proc_maps_fd = open(maps_path, O_RDONLY);1462 if (!ASSERT_GE(proc_maps_fd, 0, "open_proc_maps"))1463 goto out;1464 err = read_fd_into_buffer(proc_maps_fd, proc_maps_output, CMP_BUFFER_SIZE);1465 if (!ASSERT_GE(err, 0, "read_prog_maps_fd"))1466 goto out;1467 1468 /* strip and compare the first line of the two files */1469 str_strip_first_line(task_vma_output);1470 str_strip_first_line(proc_maps_output);1471 1472 ASSERT_STREQ(task_vma_output, proc_maps_output, "compare_output");1473 1474 check_bpf_link_info(skel->progs.proc_maps);1475 1476out:1477 close(proc_maps_fd);1478 close(iter_fd);1479 bpf_iter_task_vmas__destroy(skel);1480}1481 1482static void test_task_vma_dead_task(void)1483{1484 struct bpf_iter_task_vmas *skel;1485 int wstatus, child_pid = -1;1486 time_t start_tm, cur_tm;1487 int err, iter_fd = -1;1488 int wait_sec = 3;1489 1490 skel = bpf_iter_task_vmas__open();1491 if (!ASSERT_OK_PTR(skel, "bpf_iter_task_vmas__open"))1492 return;1493 1494 skel->bss->pid = getpid();1495 1496 err = bpf_iter_task_vmas__load(skel);1497 if (!ASSERT_OK(err, "bpf_iter_task_vmas__load"))1498 goto out;1499 1500 skel->links.proc_maps = bpf_program__attach_iter(1501 skel->progs.proc_maps, NULL);1502 1503 if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) {1504 skel->links.proc_maps = NULL;1505 goto out;1506 }1507 1508 start_tm = time(NULL);1509 cur_tm = start_tm;1510 1511 child_pid = fork();1512 if (child_pid == 0) {1513 /* Fork short-lived processes in the background. */1514 while (cur_tm < start_tm + wait_sec) {1515 system("echo > /dev/null");1516 cur_tm = time(NULL);1517 }1518 exit(0);1519 }1520 1521 if (!ASSERT_GE(child_pid, 0, "fork_child"))1522 goto out;1523 1524 while (cur_tm < start_tm + wait_sec) {1525 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.proc_maps));1526 if (!ASSERT_GE(iter_fd, 0, "create_iter"))1527 goto out;1528 1529 /* Drain all data from iter_fd. */1530 while (cur_tm < start_tm + wait_sec) {1531 err = read_fd_into_buffer(iter_fd, task_vma_output, CMP_BUFFER_SIZE);1532 if (!ASSERT_GE(err, 0, "read_iter_fd"))1533 goto out;1534 1535 cur_tm = time(NULL);1536 1537 if (err == 0)1538 break;1539 }1540 1541 close(iter_fd);1542 iter_fd = -1;1543 }1544 1545 check_bpf_link_info(skel->progs.proc_maps);1546 1547out:1548 waitpid(child_pid, &wstatus, 0);1549 close(iter_fd);1550 bpf_iter_task_vmas__destroy(skel);1551}1552 1553void test_bpf_sockmap_map_iter_fd(void)1554{1555 struct bpf_iter_sockmap *skel;1556 1557 skel = bpf_iter_sockmap__open_and_load();1558 if (!ASSERT_OK_PTR(skel, "bpf_iter_sockmap__open_and_load"))1559 return;1560 1561 do_read_map_iter_fd(&skel->skeleton, skel->progs.copy, skel->maps.sockmap);1562 1563 bpf_iter_sockmap__destroy(skel);1564}1565 1566static void test_task_vma(void)1567{1568 LIBBPF_OPTS(bpf_iter_attach_opts, opts);1569 union bpf_iter_link_info linfo;1570 1571 memset(&linfo, 0, sizeof(linfo));1572 linfo.task.tid = getpid();1573 opts.link_info = &linfo;1574 opts.link_info_len = sizeof(linfo);1575 1576 test_task_vma_common(&opts);1577 test_task_vma_common(NULL);1578}1579 1580/* uprobe attach point */1581static noinline int trigger_func(int arg)1582{1583 asm volatile ("");1584 return arg + 1;1585}1586 1587static void test_task_vma_offset_common(struct bpf_iter_attach_opts *opts, bool one_proc)1588{1589 struct bpf_iter_vma_offset *skel;1590 char buf[16] = {};1591 int iter_fd, len;1592 int pgsz, shift;1593 1594 skel = bpf_iter_vma_offset__open_and_load();1595 if (!ASSERT_OK_PTR(skel, "bpf_iter_vma_offset__open_and_load"))1596 return;1597 1598 skel->bss->pid = getpid();1599 skel->bss->address = (uintptr_t)trigger_func;1600 for (pgsz = getpagesize(), shift = 0; pgsz > 1; pgsz >>= 1, shift++)1601 ;1602 skel->bss->page_shift = shift;1603 1604 skel->links.get_vma_offset = bpf_program__attach_iter(skel->progs.get_vma_offset, opts);1605 if (!ASSERT_OK_PTR(skel->links.get_vma_offset, "attach_iter"))1606 goto exit;1607 1608 iter_fd = bpf_iter_create(bpf_link__fd(skel->links.get_vma_offset));1609 if (!ASSERT_GT(iter_fd, 0, "create_iter"))1610 goto exit;1611 1612 while ((len = read(iter_fd, buf, sizeof(buf))) > 0)1613 ;1614 buf[15] = 0;1615 ASSERT_EQ(strcmp(buf, "OK\n"), 0, "strcmp");1616 1617 ASSERT_EQ(skel->bss->offset, get_uprobe_offset(trigger_func), "offset");1618 if (one_proc)1619 ASSERT_EQ(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");1620 else1621 ASSERT_GT(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");1622 1623 close(iter_fd);1624 1625exit:1626 bpf_iter_vma_offset__destroy(skel);1627}1628 1629static void test_task_vma_offset(void)1630{1631 LIBBPF_OPTS(bpf_iter_attach_opts, opts);1632 union bpf_iter_link_info linfo;1633 1634 memset(&linfo, 0, sizeof(linfo));1635 linfo.task.pid = getpid();1636 opts.link_info = &linfo;1637 opts.link_info_len = sizeof(linfo);1638 1639 test_task_vma_offset_common(&opts, true);1640 1641 linfo.task.pid = 0;1642 linfo.task.tid = getpid();1643 test_task_vma_offset_common(&opts, true);1644 1645 test_task_vma_offset_common(NULL, false);1646}1647 1648void test_bpf_iter(void)1649{1650 ASSERT_OK(pthread_mutex_init(&do_nothing_mutex, NULL), "pthread_mutex_init");1651 1652 if (test__start_subtest("btf_id_or_null"))1653 test_btf_id_or_null();1654 if (test__start_subtest("ipv6_route"))1655 test_ipv6_route();1656 if (test__start_subtest("netlink"))1657 test_netlink();1658 if (test__start_subtest("bpf_map"))1659 test_bpf_map();1660 if (test__start_subtest("task_tid"))1661 test_task_tid();1662 if (test__start_subtest("task_pid"))1663 test_task_pid();1664 if (test__start_subtest("task_pidfd"))1665 test_task_pidfd();1666 if (test__start_subtest("task_sleepable"))1667 test_task_sleepable();1668 if (test__start_subtest("task_stack"))1669 test_task_stack();1670 if (test__start_subtest("task_file"))1671 test_task_file();1672 if (test__start_subtest("task_vma"))1673 test_task_vma();1674 if (test__start_subtest("task_vma_dead_task"))1675 test_task_vma_dead_task();1676 if (test__start_subtest("task_btf"))1677 test_task_btf();1678 if (test__start_subtest("tcp4"))1679 test_tcp4();1680 if (test__start_subtest("tcp6"))1681 test_tcp6();1682 if (test__start_subtest("udp4"))1683 test_udp4();1684 if (test__start_subtest("udp6"))1685 test_udp6();1686 if (test__start_subtest("unix"))1687 test_unix();1688 if (test__start_subtest("anon"))1689 test_anon_iter(false);1690 if (test__start_subtest("anon-read-one-char"))1691 test_anon_iter(true);1692 if (test__start_subtest("file"))1693 test_file_iter();1694 if (test__start_subtest("overflow"))1695 test_overflow(false, false);1696 if (test__start_subtest("overflow-e2big"))1697 test_overflow(true, false);1698 if (test__start_subtest("prog-ret-1"))1699 test_overflow(false, true);1700 if (test__start_subtest("bpf_hash_map"))1701 test_bpf_hash_map();1702 if (test__start_subtest("bpf_percpu_hash_map"))1703 test_bpf_percpu_hash_map();1704 if (test__start_subtest("bpf_array_map"))1705 test_bpf_array_map();1706 if (test__start_subtest("bpf_array_map_iter_fd"))1707 test_bpf_array_map_iter_fd();1708 if (test__start_subtest("bpf_percpu_array_map"))1709 test_bpf_percpu_array_map();1710 if (test__start_subtest("bpf_sk_storage_map"))1711 test_bpf_sk_storage_map();1712 if (test__start_subtest("bpf_sk_storage_map_iter_fd"))1713 test_bpf_sk_storage_map_iter_fd();1714 if (test__start_subtest("bpf_sk_storage_delete"))1715 test_bpf_sk_storage_delete();1716 if (test__start_subtest("bpf_sk_storage_get"))1717 test_bpf_sk_storage_get();1718 if (test__start_subtest("rdonly-buf-out-of-bound"))1719 test_rdonly_buf_out_of_bound();1720 if (test__start_subtest("buf-neg-offset"))1721 test_buf_neg_offset();1722 if (test__start_subtest("link-iter"))1723 test_link_iter();1724 if (test__start_subtest("ksym"))1725 test_ksym_iter();1726 if (test__start_subtest("bpf_sockmap_map_iter_fd"))1727 test_bpf_sockmap_map_iter_fd();1728 if (test__start_subtest("vma_offset"))1729 test_task_vma_offset();1730}1731