698 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */3 4#define _GNU_SOURCE5#include <linux/compiler.h>6#include <linux/ring_buffer.h>7#include <linux/build_bug.h>8#include <pthread.h>9#include <stdio.h>10#include <stdlib.h>11#include <sys/mman.h>12#include <sys/syscall.h>13#include <sys/sysinfo.h>14#include <test_progs.h>15#include <uapi/linux/bpf.h>16#include <unistd.h>17 18#include "user_ringbuf_fail.skel.h"19#include "user_ringbuf_success.skel.h"20 21#include "../progs/test_user_ringbuf.h"22 23static const long c_sample_size = sizeof(struct sample) + BPF_RINGBUF_HDR_SZ;24static const long c_ringbuf_size = 1 << 12; /* 1 small page */25static const long c_max_entries = c_ringbuf_size / c_sample_size;26 27static void drain_current_samples(void)28{29 syscall(__NR_getpgid);30}31 32static int write_samples(struct user_ring_buffer *ringbuf, uint32_t num_samples)33{34 int i, err = 0;35 36 /* Write some number of samples to the ring buffer. */37 for (i = 0; i < num_samples; i++) {38 struct sample *entry;39 int read;40 41 entry = user_ring_buffer__reserve(ringbuf, sizeof(*entry));42 if (!entry) {43 err = -errno;44 goto done;45 }46 47 entry->pid = getpid();48 entry->seq = i;49 entry->value = i * i;50 51 read = snprintf(entry->comm, sizeof(entry->comm), "%u", i);52 if (read <= 0) {53 /* Assert on the error path to avoid spamming logs with54 * mostly success messages.55 */56 ASSERT_GT(read, 0, "snprintf_comm");57 err = read;58 user_ring_buffer__discard(ringbuf, entry);59 goto done;60 }61 62 user_ring_buffer__submit(ringbuf, entry);63 }64 65done:66 drain_current_samples();67 68 return err;69}70 71static struct user_ringbuf_success *open_load_ringbuf_skel(void)72{73 struct user_ringbuf_success *skel;74 int err;75 76 skel = user_ringbuf_success__open();77 if (!ASSERT_OK_PTR(skel, "skel_open"))78 return NULL;79 80 err = bpf_map__set_max_entries(skel->maps.user_ringbuf, c_ringbuf_size);81 if (!ASSERT_OK(err, "set_max_entries"))82 goto cleanup;83 84 err = bpf_map__set_max_entries(skel->maps.kernel_ringbuf, c_ringbuf_size);85 if (!ASSERT_OK(err, "set_max_entries"))86 goto cleanup;87 88 err = user_ringbuf_success__load(skel);89 if (!ASSERT_OK(err, "skel_load"))90 goto cleanup;91 92 return skel;93 94cleanup:95 user_ringbuf_success__destroy(skel);96 return NULL;97}98 99static void test_user_ringbuf_mappings(void)100{101 int err, rb_fd;102 int page_size = getpagesize();103 void *mmap_ptr;104 struct user_ringbuf_success *skel;105 106 skel = open_load_ringbuf_skel();107 if (!skel)108 return;109 110 rb_fd = bpf_map__fd(skel->maps.user_ringbuf);111 /* cons_pos can be mapped R/O, can't add +X with mprotect. */112 mmap_ptr = mmap(NULL, page_size, PROT_READ, MAP_SHARED, rb_fd, 0);113 ASSERT_OK_PTR(mmap_ptr, "ro_cons_pos");114 ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_WRITE), "write_cons_pos_protect");115 ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_EXEC), "exec_cons_pos_protect");116 ASSERT_ERR_PTR(mremap(mmap_ptr, 0, 4 * page_size, MREMAP_MAYMOVE), "wr_prod_pos");117 err = -errno;118 ASSERT_ERR(err, "wr_prod_pos_err");119 ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_ro_cons");120 121 /* prod_pos can be mapped RW, can't add +X with mprotect. */122 mmap_ptr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED,123 rb_fd, page_size);124 ASSERT_OK_PTR(mmap_ptr, "rw_prod_pos");125 ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_EXEC), "exec_prod_pos_protect");126 err = -errno;127 ASSERT_ERR(err, "wr_prod_pos_err");128 ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_rw_prod");129 130 /* data pages can be mapped RW, can't add +X with mprotect. */131 mmap_ptr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, rb_fd,132 2 * page_size);133 ASSERT_OK_PTR(mmap_ptr, "rw_data");134 ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_EXEC), "exec_data_protect");135 err = -errno;136 ASSERT_ERR(err, "exec_data_err");137 ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_rw_data");138 139 user_ringbuf_success__destroy(skel);140}141 142static int load_skel_create_ringbufs(struct user_ringbuf_success **skel_out,143 struct ring_buffer **kern_ringbuf_out,144 ring_buffer_sample_fn callback,145 struct user_ring_buffer **user_ringbuf_out)146{147 struct user_ringbuf_success *skel;148 struct ring_buffer *kern_ringbuf = NULL;149 struct user_ring_buffer *user_ringbuf = NULL;150 int err = -ENOMEM, rb_fd;151 152 skel = open_load_ringbuf_skel();153 if (!skel)154 return err;155 156 /* only trigger BPF program for current process */157 skel->bss->pid = getpid();158 159 if (kern_ringbuf_out) {160 rb_fd = bpf_map__fd(skel->maps.kernel_ringbuf);161 kern_ringbuf = ring_buffer__new(rb_fd, callback, skel, NULL);162 if (!ASSERT_OK_PTR(kern_ringbuf, "kern_ringbuf_create"))163 goto cleanup;164 165 *kern_ringbuf_out = kern_ringbuf;166 }167 168 if (user_ringbuf_out) {169 rb_fd = bpf_map__fd(skel->maps.user_ringbuf);170 user_ringbuf = user_ring_buffer__new(rb_fd, NULL);171 if (!ASSERT_OK_PTR(user_ringbuf, "user_ringbuf_create"))172 goto cleanup;173 174 *user_ringbuf_out = user_ringbuf;175 ASSERT_EQ(skel->bss->read, 0, "no_reads_after_load");176 }177 178 err = user_ringbuf_success__attach(skel);179 if (!ASSERT_OK(err, "skel_attach"))180 goto cleanup;181 182 *skel_out = skel;183 return 0;184 185cleanup:186 if (kern_ringbuf_out)187 *kern_ringbuf_out = NULL;188 if (user_ringbuf_out)189 *user_ringbuf_out = NULL;190 ring_buffer__free(kern_ringbuf);191 user_ring_buffer__free(user_ringbuf);192 user_ringbuf_success__destroy(skel);193 return err;194}195 196static int load_skel_create_user_ringbuf(struct user_ringbuf_success **skel_out,197 struct user_ring_buffer **ringbuf_out)198{199 return load_skel_create_ringbufs(skel_out, NULL, NULL, ringbuf_out);200}201 202static void manually_write_test_invalid_sample(struct user_ringbuf_success *skel,203 __u32 size, __u64 producer_pos, int err)204{205 void *data_ptr;206 __u64 *producer_pos_ptr;207 int rb_fd, page_size = getpagesize();208 209 rb_fd = bpf_map__fd(skel->maps.user_ringbuf);210 211 ASSERT_EQ(skel->bss->read, 0, "num_samples_before_bad_sample");212 213 /* Map the producer_pos as RW. */214 producer_pos_ptr = mmap(NULL, page_size, PROT_READ | PROT_WRITE,215 MAP_SHARED, rb_fd, page_size);216 ASSERT_OK_PTR(producer_pos_ptr, "producer_pos_ptr");217 218 /* Map the data pages as RW. */219 data_ptr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, rb_fd, 2 * page_size);220 ASSERT_OK_PTR(data_ptr, "rw_data");221 222 memset(data_ptr, 0, BPF_RINGBUF_HDR_SZ);223 *(__u32 *)data_ptr = size;224 225 /* Synchronizes with smp_load_acquire() in __bpf_user_ringbuf_peek() in the kernel. */226 smp_store_release(producer_pos_ptr, producer_pos + BPF_RINGBUF_HDR_SZ);227 228 drain_current_samples();229 ASSERT_EQ(skel->bss->read, 0, "num_samples_after_bad_sample");230 ASSERT_EQ(skel->bss->err, err, "err_after_bad_sample");231 232 ASSERT_OK(munmap(producer_pos_ptr, page_size), "unmap_producer_pos");233 ASSERT_OK(munmap(data_ptr, page_size), "unmap_data_ptr");234}235 236static void test_user_ringbuf_post_misaligned(void)237{238 struct user_ringbuf_success *skel;239 struct user_ring_buffer *ringbuf;240 int err;241 __u32 size = (1 << 5) + 7;242 243 err = load_skel_create_user_ringbuf(&skel, &ringbuf);244 if (!ASSERT_OK(err, "misaligned_skel"))245 return;246 247 manually_write_test_invalid_sample(skel, size, size, -EINVAL);248 user_ring_buffer__free(ringbuf);249 user_ringbuf_success__destroy(skel);250}251 252static void test_user_ringbuf_post_producer_wrong_offset(void)253{254 struct user_ringbuf_success *skel;255 struct user_ring_buffer *ringbuf;256 int err;257 __u32 size = (1 << 5);258 259 err = load_skel_create_user_ringbuf(&skel, &ringbuf);260 if (!ASSERT_OK(err, "wrong_offset_skel"))261 return;262 263 manually_write_test_invalid_sample(skel, size, size - 8, -EINVAL);264 user_ring_buffer__free(ringbuf);265 user_ringbuf_success__destroy(skel);266}267 268static void test_user_ringbuf_post_larger_than_ringbuf_sz(void)269{270 struct user_ringbuf_success *skel;271 struct user_ring_buffer *ringbuf;272 int err;273 __u32 size = c_ringbuf_size;274 275 err = load_skel_create_user_ringbuf(&skel, &ringbuf);276 if (!ASSERT_OK(err, "huge_sample_skel"))277 return;278 279 manually_write_test_invalid_sample(skel, size, size, -E2BIG);280 user_ring_buffer__free(ringbuf);281 user_ringbuf_success__destroy(skel);282}283 284static void test_user_ringbuf_basic(void)285{286 struct user_ringbuf_success *skel;287 struct user_ring_buffer *ringbuf;288 int err;289 290 err = load_skel_create_user_ringbuf(&skel, &ringbuf);291 if (!ASSERT_OK(err, "ringbuf_basic_skel"))292 return;293 294 ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before");295 296 err = write_samples(ringbuf, 2);297 if (!ASSERT_OK(err, "write_samples"))298 goto cleanup;299 300 ASSERT_EQ(skel->bss->read, 2, "num_samples_read_after");301 302cleanup:303 user_ring_buffer__free(ringbuf);304 user_ringbuf_success__destroy(skel);305}306 307static void test_user_ringbuf_sample_full_ring_buffer(void)308{309 struct user_ringbuf_success *skel;310 struct user_ring_buffer *ringbuf;311 int err;312 void *sample;313 314 err = load_skel_create_user_ringbuf(&skel, &ringbuf);315 if (!ASSERT_OK(err, "ringbuf_full_sample_skel"))316 return;317 318 sample = user_ring_buffer__reserve(ringbuf, c_ringbuf_size - BPF_RINGBUF_HDR_SZ);319 if (!ASSERT_OK_PTR(sample, "full_sample"))320 goto cleanup;321 322 user_ring_buffer__submit(ringbuf, sample);323 ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before");324 drain_current_samples();325 ASSERT_EQ(skel->bss->read, 1, "num_samples_read_after");326 327cleanup:328 user_ring_buffer__free(ringbuf);329 user_ringbuf_success__destroy(skel);330}331 332static void test_user_ringbuf_post_alignment_autoadjust(void)333{334 struct user_ringbuf_success *skel;335 struct user_ring_buffer *ringbuf;336 struct sample *sample;337 int err;338 339 err = load_skel_create_user_ringbuf(&skel, &ringbuf);340 if (!ASSERT_OK(err, "ringbuf_align_autoadjust_skel"))341 return;342 343 /* libbpf should automatically round any sample up to an 8-byte alignment. */344 sample = user_ring_buffer__reserve(ringbuf, sizeof(*sample) + 1);345 ASSERT_OK_PTR(sample, "reserve_autoaligned");346 user_ring_buffer__submit(ringbuf, sample);347 348 ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before");349 drain_current_samples();350 ASSERT_EQ(skel->bss->read, 1, "num_samples_read_after");351 352 user_ring_buffer__free(ringbuf);353 user_ringbuf_success__destroy(skel);354}355 356static void test_user_ringbuf_overfill(void)357{358 struct user_ringbuf_success *skel;359 struct user_ring_buffer *ringbuf;360 int err;361 362 err = load_skel_create_user_ringbuf(&skel, &ringbuf);363 if (err)364 return;365 366 err = write_samples(ringbuf, c_max_entries * 5);367 ASSERT_ERR(err, "write_samples");368 ASSERT_EQ(skel->bss->read, c_max_entries, "max_entries");369 370 user_ring_buffer__free(ringbuf);371 user_ringbuf_success__destroy(skel);372}373 374static void test_user_ringbuf_discards_properly_ignored(void)375{376 struct user_ringbuf_success *skel;377 struct user_ring_buffer *ringbuf;378 int err, num_discarded = 0;379 __u64 *token;380 381 err = load_skel_create_user_ringbuf(&skel, &ringbuf);382 if (err)383 return;384 385 ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before");386 387 while (1) {388 /* Write samples until the buffer is full. */389 token = user_ring_buffer__reserve(ringbuf, sizeof(*token));390 if (!token)391 break;392 393 user_ring_buffer__discard(ringbuf, token);394 num_discarded++;395 }396 397 if (!ASSERT_GE(num_discarded, 0, "num_discarded"))398 goto cleanup;399 400 /* Should not read any samples, as they are all discarded. */401 ASSERT_EQ(skel->bss->read, 0, "num_pre_kick");402 drain_current_samples();403 ASSERT_EQ(skel->bss->read, 0, "num_post_kick");404 405 /* Now that the ring buffer has been drained, we should be able to406 * reserve another token.407 */408 token = user_ring_buffer__reserve(ringbuf, sizeof(*token));409 410 if (!ASSERT_OK_PTR(token, "new_token"))411 goto cleanup;412 413 user_ring_buffer__discard(ringbuf, token);414cleanup:415 user_ring_buffer__free(ringbuf);416 user_ringbuf_success__destroy(skel);417}418 419static void test_user_ringbuf_loop(void)420{421 struct user_ringbuf_success *skel;422 struct user_ring_buffer *ringbuf;423 uint32_t total_samples = 8192;424 uint32_t remaining_samples = total_samples;425 int err;426 427 BUILD_BUG_ON(total_samples <= c_max_entries);428 err = load_skel_create_user_ringbuf(&skel, &ringbuf);429 if (err)430 return;431 432 do {433 uint32_t curr_samples;434 435 curr_samples = remaining_samples > c_max_entries436 ? c_max_entries : remaining_samples;437 err = write_samples(ringbuf, curr_samples);438 if (err != 0) {439 /* Assert inside of if statement to avoid flooding logs440 * on the success path.441 */442 ASSERT_OK(err, "write_samples");443 goto cleanup;444 }445 446 remaining_samples -= curr_samples;447 ASSERT_EQ(skel->bss->read, total_samples - remaining_samples,448 "current_batched_entries");449 } while (remaining_samples > 0);450 ASSERT_EQ(skel->bss->read, total_samples, "total_batched_entries");451 452cleanup:453 user_ring_buffer__free(ringbuf);454 user_ringbuf_success__destroy(skel);455}456 457static int send_test_message(struct user_ring_buffer *ringbuf,458 enum test_msg_op op, s64 operand_64,459 s32 operand_32)460{461 struct test_msg *msg;462 463 msg = user_ring_buffer__reserve(ringbuf, sizeof(*msg));464 if (!msg) {465 /* Assert on the error path to avoid spamming logs with mostly466 * success messages.467 */468 ASSERT_OK_PTR(msg, "reserve_msg");469 return -ENOMEM;470 }471 472 msg->msg_op = op;473 474 switch (op) {475 case TEST_MSG_OP_INC64:476 case TEST_MSG_OP_MUL64:477 msg->operand_64 = operand_64;478 break;479 case TEST_MSG_OP_INC32:480 case TEST_MSG_OP_MUL32:481 msg->operand_32 = operand_32;482 break;483 default:484 PRINT_FAIL("Invalid operand %d\n", op);485 user_ring_buffer__discard(ringbuf, msg);486 return -EINVAL;487 }488 489 user_ring_buffer__submit(ringbuf, msg);490 491 return 0;492}493 494static void kick_kernel_read_messages(void)495{496 syscall(__NR_prctl);497}498 499static int handle_kernel_msg(void *ctx, void *data, size_t len)500{501 struct user_ringbuf_success *skel = ctx;502 struct test_msg *msg = data;503 504 switch (msg->msg_op) {505 case TEST_MSG_OP_INC64:506 skel->bss->user_mutated += msg->operand_64;507 return 0;508 case TEST_MSG_OP_INC32:509 skel->bss->user_mutated += msg->operand_32;510 return 0;511 case TEST_MSG_OP_MUL64:512 skel->bss->user_mutated *= msg->operand_64;513 return 0;514 case TEST_MSG_OP_MUL32:515 skel->bss->user_mutated *= msg->operand_32;516 return 0;517 default:518 fprintf(stderr, "Invalid operand %d\n", msg->msg_op);519 return -EINVAL;520 }521}522 523static void drain_kernel_messages_buffer(struct ring_buffer *kern_ringbuf,524 struct user_ringbuf_success *skel)525{526 int cnt;527 528 cnt = ring_buffer__consume(kern_ringbuf);529 ASSERT_EQ(cnt, 8, "consume_kern_ringbuf");530 ASSERT_OK(skel->bss->err, "consume_kern_ringbuf_err");531}532 533static void test_user_ringbuf_msg_protocol(void)534{535 struct user_ringbuf_success *skel;536 struct user_ring_buffer *user_ringbuf;537 struct ring_buffer *kern_ringbuf;538 int err, i;539 __u64 expected_kern = 0;540 541 err = load_skel_create_ringbufs(&skel, &kern_ringbuf, handle_kernel_msg, &user_ringbuf);542 if (!ASSERT_OK(err, "create_ringbufs"))543 return;544 545 for (i = 0; i < 64; i++) {546 enum test_msg_op op = i % TEST_MSG_OP_NUM_OPS;547 __u64 operand_64 = TEST_OP_64;548 __u32 operand_32 = TEST_OP_32;549 550 err = send_test_message(user_ringbuf, op, operand_64, operand_32);551 if (err) {552 /* Only assert on a failure to avoid spamming success logs. */553 ASSERT_OK(err, "send_test_message");554 goto cleanup;555 }556 557 switch (op) {558 case TEST_MSG_OP_INC64:559 expected_kern += operand_64;560 break;561 case TEST_MSG_OP_INC32:562 expected_kern += operand_32;563 break;564 case TEST_MSG_OP_MUL64:565 expected_kern *= operand_64;566 break;567 case TEST_MSG_OP_MUL32:568 expected_kern *= operand_32;569 break;570 default:571 PRINT_FAIL("Unexpected op %d\n", op);572 goto cleanup;573 }574 575 if (i % 8 == 0) {576 kick_kernel_read_messages();577 ASSERT_EQ(skel->bss->kern_mutated, expected_kern, "expected_kern");578 ASSERT_EQ(skel->bss->err, 0, "bpf_prog_err");579 drain_kernel_messages_buffer(kern_ringbuf, skel);580 }581 }582 583cleanup:584 ring_buffer__free(kern_ringbuf);585 user_ring_buffer__free(user_ringbuf);586 user_ringbuf_success__destroy(skel);587}588 589static void *kick_kernel_cb(void *arg)590{591 /* Kick the kernel, causing it to drain the ring buffer and then wake592 * up the test thread waiting on epoll.593 */594 syscall(__NR_prlimit64);595 596 return NULL;597}598 599static int spawn_kick_thread_for_poll(void)600{601 pthread_t thread;602 603 return pthread_create(&thread, NULL, kick_kernel_cb, NULL);604}605 606static void test_user_ringbuf_blocking_reserve(void)607{608 struct user_ringbuf_success *skel;609 struct user_ring_buffer *ringbuf;610 int err, num_written = 0;611 __u64 *token;612 613 err = load_skel_create_user_ringbuf(&skel, &ringbuf);614 if (err)615 return;616 617 ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before");618 619 while (1) {620 /* Write samples until the buffer is full. */621 token = user_ring_buffer__reserve(ringbuf, sizeof(*token));622 if (!token)623 break;624 625 *token = 0xdeadbeef;626 627 user_ring_buffer__submit(ringbuf, token);628 num_written++;629 }630 631 if (!ASSERT_GE(num_written, 0, "num_written"))632 goto cleanup;633 634 /* Should not have read any samples until the kernel is kicked. */635 ASSERT_EQ(skel->bss->read, 0, "num_pre_kick");636 637 /* We correctly time out after 1 second, without a sample. */638 token = user_ring_buffer__reserve_blocking(ringbuf, sizeof(*token), 1000);639 if (!ASSERT_EQ(token, NULL, "pre_kick_timeout_token"))640 goto cleanup;641 642 err = spawn_kick_thread_for_poll();643 if (!ASSERT_EQ(err, 0, "deferred_kick_thread\n"))644 goto cleanup;645 646 /* After spawning another thread that asynchronously kicks the kernel to647 * drain the messages, we're able to block and successfully get a648 * sample once we receive an event notification.649 */650 token = user_ring_buffer__reserve_blocking(ringbuf, sizeof(*token), 10000);651 652 if (!ASSERT_OK_PTR(token, "block_token"))653 goto cleanup;654 655 ASSERT_GT(skel->bss->read, 0, "num_post_kill");656 ASSERT_LE(skel->bss->read, num_written, "num_post_kill");657 ASSERT_EQ(skel->bss->err, 0, "err_post_poll");658 user_ring_buffer__discard(ringbuf, token);659 660cleanup:661 user_ring_buffer__free(ringbuf);662 user_ringbuf_success__destroy(skel);663}664 665#define SUCCESS_TEST(_func) { _func, #_func }666 667static struct {668 void (*test_callback)(void);669 const char *test_name;670} success_tests[] = {671 SUCCESS_TEST(test_user_ringbuf_mappings),672 SUCCESS_TEST(test_user_ringbuf_post_misaligned),673 SUCCESS_TEST(test_user_ringbuf_post_producer_wrong_offset),674 SUCCESS_TEST(test_user_ringbuf_post_larger_than_ringbuf_sz),675 SUCCESS_TEST(test_user_ringbuf_basic),676 SUCCESS_TEST(test_user_ringbuf_sample_full_ring_buffer),677 SUCCESS_TEST(test_user_ringbuf_post_alignment_autoadjust),678 SUCCESS_TEST(test_user_ringbuf_overfill),679 SUCCESS_TEST(test_user_ringbuf_discards_properly_ignored),680 SUCCESS_TEST(test_user_ringbuf_loop),681 SUCCESS_TEST(test_user_ringbuf_msg_protocol),682 SUCCESS_TEST(test_user_ringbuf_blocking_reserve),683};684 685void test_user_ringbuf(void)686{687 int i;688 689 for (i = 0; i < ARRAY_SIZE(success_tests); i++) {690 if (!test__start_subtest(success_tests[i].test_name))691 continue;692 693 success_tests[i].test_callback();694 }695 696 RUN_TESTS(user_ringbuf_fail);697}698