511 lines · c
1// SPDX-License-Identifier: GPL-2.02#define _GNU_SOURCE3#include <linux/compiler.h>4#include <asm/barrier.h>5#include <test_progs.h>6#include <sys/mman.h>7#include <sys/epoll.h>8#include <time.h>9#include <sched.h>10#include <signal.h>11#include <pthread.h>12#include <sys/sysinfo.h>13#include <linux/perf_event.h>14#include <linux/ring_buffer.h>15 16#include "test_ringbuf.lskel.h"17#include "test_ringbuf_n.lskel.h"18#include "test_ringbuf_map_key.lskel.h"19#include "test_ringbuf_write.lskel.h"20 21#define EDONE 777722 23static int duration = 0;24 25struct sample {26 int pid;27 int seq;28 long value;29 char comm[16];30};31 32static int sample_cnt;33 34static void atomic_inc(int *cnt)35{36 __atomic_add_fetch(cnt, 1, __ATOMIC_SEQ_CST);37}38 39static int atomic_xchg(int *cnt, int val)40{41 return __atomic_exchange_n(cnt, val, __ATOMIC_SEQ_CST);42}43 44static int process_sample(void *ctx, void *data, size_t len)45{46 struct sample *s = data;47 48 atomic_inc(&sample_cnt);49 50 switch (s->seq) {51 case 0:52 CHECK(s->value != 333, "sample1_value", "exp %ld, got %ld\n",53 333L, s->value);54 return 0;55 case 1:56 CHECK(s->value != 777, "sample2_value", "exp %ld, got %ld\n",57 777L, s->value);58 return -EDONE;59 default:60 /* we don't care about the rest */61 return 0;62 }63}64 65static struct test_ringbuf_map_key_lskel *skel_map_key;66static struct test_ringbuf_lskel *skel;67static struct ring_buffer *ringbuf;68 69static void trigger_samples()70{71 skel->bss->dropped = 0;72 skel->bss->total = 0;73 skel->bss->discarded = 0;74 75 /* trigger exactly two samples */76 skel->bss->value = 333;77 syscall(__NR_getpgid);78 skel->bss->value = 777;79 syscall(__NR_getpgid);80}81 82static void *poll_thread(void *input)83{84 long timeout = (long)input;85 86 return (void *)(long)ring_buffer__poll(ringbuf, timeout);87}88 89static void ringbuf_write_subtest(void)90{91 struct test_ringbuf_write_lskel *skel;92 int page_size = getpagesize();93 size_t *mmap_ptr;94 int err, rb_fd;95 96 skel = test_ringbuf_write_lskel__open();97 if (!ASSERT_OK_PTR(skel, "skel_open"))98 return;99 100 skel->maps.ringbuf.max_entries = 0x4000;101 102 err = test_ringbuf_write_lskel__load(skel);103 if (!ASSERT_OK(err, "skel_load"))104 goto cleanup;105 106 rb_fd = skel->maps.ringbuf.map_fd;107 108 mmap_ptr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, rb_fd, 0);109 if (!ASSERT_OK_PTR(mmap_ptr, "rw_cons_pos"))110 goto cleanup;111 *mmap_ptr = 0x3000;112 ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_rw");113 114 skel->bss->pid = getpid();115 116 ringbuf = ring_buffer__new(rb_fd, process_sample, NULL, NULL);117 if (!ASSERT_OK_PTR(ringbuf, "ringbuf_new"))118 goto cleanup;119 120 err = test_ringbuf_write_lskel__attach(skel);121 if (!ASSERT_OK(err, "skel_attach"))122 goto cleanup_ringbuf;123 124 skel->bss->discarded = 0;125 skel->bss->passed = 0;126 127 /* trigger exactly two samples */128 syscall(__NR_getpgid);129 syscall(__NR_getpgid);130 131 ASSERT_EQ(skel->bss->discarded, 2, "discarded");132 ASSERT_EQ(skel->bss->passed, 0, "passed");133 134 test_ringbuf_write_lskel__detach(skel);135cleanup_ringbuf:136 ring_buffer__free(ringbuf);137cleanup:138 test_ringbuf_write_lskel__destroy(skel);139}140 141static void ringbuf_subtest(void)142{143 const size_t rec_sz = BPF_RINGBUF_HDR_SZ + sizeof(struct sample);144 pthread_t thread;145 long bg_ret = -1;146 int err, cnt, rb_fd;147 int page_size = getpagesize();148 void *mmap_ptr, *tmp_ptr;149 struct ring *ring;150 int map_fd;151 unsigned long avail_data, ring_size, cons_pos, prod_pos;152 153 skel = test_ringbuf_lskel__open();154 if (CHECK(!skel, "skel_open", "skeleton open failed\n"))155 return;156 157 skel->maps.ringbuf.max_entries = page_size;158 159 err = test_ringbuf_lskel__load(skel);160 if (CHECK(err != 0, "skel_load", "skeleton load failed\n"))161 goto cleanup;162 163 rb_fd = skel->maps.ringbuf.map_fd;164 /* good read/write cons_pos */165 mmap_ptr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, rb_fd, 0);166 ASSERT_OK_PTR(mmap_ptr, "rw_cons_pos");167 tmp_ptr = mremap(mmap_ptr, page_size, 2 * page_size, MREMAP_MAYMOVE);168 if (!ASSERT_ERR_PTR(tmp_ptr, "rw_extend"))169 goto cleanup;170 ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_EXEC), "exec_cons_pos_protect");171 ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_rw");172 173 /* bad writeable prod_pos */174 mmap_ptr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, rb_fd, page_size);175 err = -errno;176 ASSERT_ERR_PTR(mmap_ptr, "wr_prod_pos");177 ASSERT_EQ(err, -EPERM, "wr_prod_pos_err");178 179 /* bad writeable data pages */180 mmap_ptr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, rb_fd, 2 * page_size);181 err = -errno;182 ASSERT_ERR_PTR(mmap_ptr, "wr_data_page_one");183 ASSERT_EQ(err, -EPERM, "wr_data_page_one_err");184 mmap_ptr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, rb_fd, 3 * page_size);185 ASSERT_ERR_PTR(mmap_ptr, "wr_data_page_two");186 mmap_ptr = mmap(NULL, 2 * page_size, PROT_WRITE, MAP_SHARED, rb_fd, 2 * page_size);187 ASSERT_ERR_PTR(mmap_ptr, "wr_data_page_all");188 189 /* good read-only pages */190 mmap_ptr = mmap(NULL, 4 * page_size, PROT_READ, MAP_SHARED, rb_fd, 0);191 if (!ASSERT_OK_PTR(mmap_ptr, "ro_prod_pos"))192 goto cleanup;193 194 ASSERT_ERR(mprotect(mmap_ptr, 4 * page_size, PROT_WRITE), "write_protect");195 ASSERT_ERR(mprotect(mmap_ptr, 4 * page_size, PROT_EXEC), "exec_protect");196 ASSERT_ERR_PTR(mremap(mmap_ptr, 0, 4 * page_size, MREMAP_MAYMOVE), "ro_remap");197 ASSERT_OK(munmap(mmap_ptr, 4 * page_size), "unmap_ro");198 199 /* good read-only pages with initial offset */200 mmap_ptr = mmap(NULL, page_size, PROT_READ, MAP_SHARED, rb_fd, page_size);201 if (!ASSERT_OK_PTR(mmap_ptr, "ro_prod_pos"))202 goto cleanup;203 204 ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_WRITE), "write_protect");205 ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_EXEC), "exec_protect");206 ASSERT_ERR_PTR(mremap(mmap_ptr, 0, 3 * page_size, MREMAP_MAYMOVE), "ro_remap");207 ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_ro");208 209 /* only trigger BPF program for current process */210 skel->bss->pid = getpid();211 212 ringbuf = ring_buffer__new(skel->maps.ringbuf.map_fd,213 process_sample, NULL, NULL);214 if (CHECK(!ringbuf, "ringbuf_create", "failed to create ringbuf\n"))215 goto cleanup;216 217 err = test_ringbuf_lskel__attach(skel);218 if (CHECK(err, "skel_attach", "skeleton attachment failed: %d\n", err))219 goto cleanup;220 221 trigger_samples();222 223 ring = ring_buffer__ring(ringbuf, 0);224 if (!ASSERT_OK_PTR(ring, "ring_buffer__ring_idx_0"))225 goto cleanup;226 227 map_fd = ring__map_fd(ring);228 ASSERT_EQ(map_fd, skel->maps.ringbuf.map_fd, "ring_map_fd");229 230 /* 2 submitted + 1 discarded records */231 CHECK(skel->bss->avail_data != 3 * rec_sz,232 "err_avail_size", "exp %ld, got %ld\n",233 3L * rec_sz, skel->bss->avail_data);234 CHECK(skel->bss->ring_size != page_size,235 "err_ring_size", "exp %ld, got %ld\n",236 (long)page_size, skel->bss->ring_size);237 CHECK(skel->bss->cons_pos != 0,238 "err_cons_pos", "exp %ld, got %ld\n",239 0L, skel->bss->cons_pos);240 CHECK(skel->bss->prod_pos != 3 * rec_sz,241 "err_prod_pos", "exp %ld, got %ld\n",242 3L * rec_sz, skel->bss->prod_pos);243 244 /* verify getting this data directly via the ring object yields the same245 * results246 */247 avail_data = ring__avail_data_size(ring);248 ASSERT_EQ(avail_data, 3 * rec_sz, "ring_avail_size");249 ring_size = ring__size(ring);250 ASSERT_EQ(ring_size, page_size, "ring_ring_size");251 cons_pos = ring__consumer_pos(ring);252 ASSERT_EQ(cons_pos, 0, "ring_cons_pos");253 prod_pos = ring__producer_pos(ring);254 ASSERT_EQ(prod_pos, 3 * rec_sz, "ring_prod_pos");255 256 /* poll for samples */257 err = ring_buffer__poll(ringbuf, -1);258 259 /* -EDONE is used as an indicator that we are done */260 if (CHECK(err != -EDONE, "err_done", "done err: %d\n", err))261 goto cleanup;262 cnt = atomic_xchg(&sample_cnt, 0);263 CHECK(cnt != 2, "cnt", "exp %d samples, got %d\n", 2, cnt);264 265 /* we expect extra polling to return nothing */266 err = ring_buffer__poll(ringbuf, 0);267 if (CHECK(err != 0, "extra_samples", "poll result: %d\n", err))268 goto cleanup;269 cnt = atomic_xchg(&sample_cnt, 0);270 CHECK(cnt != 0, "cnt", "exp %d samples, got %d\n", 0, cnt);271 272 CHECK(skel->bss->dropped != 0, "err_dropped", "exp %ld, got %ld\n",273 0L, skel->bss->dropped);274 CHECK(skel->bss->total != 2, "err_total", "exp %ld, got %ld\n",275 2L, skel->bss->total);276 CHECK(skel->bss->discarded != 1, "err_discarded", "exp %ld, got %ld\n",277 1L, skel->bss->discarded);278 279 /* now validate consumer position is updated and returned */280 trigger_samples();281 CHECK(skel->bss->cons_pos != 3 * rec_sz,282 "err_cons_pos", "exp %ld, got %ld\n",283 3L * rec_sz, skel->bss->cons_pos);284 err = ring_buffer__poll(ringbuf, -1);285 CHECK(err <= 0, "poll_err", "err %d\n", err);286 cnt = atomic_xchg(&sample_cnt, 0);287 CHECK(cnt != 2, "cnt", "exp %d samples, got %d\n", 2, cnt);288 289 /* start poll in background w/ long timeout */290 err = pthread_create(&thread, NULL, poll_thread, (void *)(long)10000);291 if (CHECK(err, "bg_poll", "pthread_create failed: %d\n", err))292 goto cleanup;293 294 /* turn off notifications now */295 skel->bss->flags = BPF_RB_NO_WAKEUP;296 297 /* give background thread a bit of a time */298 usleep(50000);299 trigger_samples();300 /* sleeping arbitrarily is bad, but no better way to know that301 * epoll_wait() **DID NOT** unblock in background thread302 */303 usleep(50000);304 /* background poll should still be blocked */305 err = pthread_tryjoin_np(thread, (void **)&bg_ret);306 if (CHECK(err != EBUSY, "try_join", "err %d\n", err))307 goto cleanup;308 309 /* BPF side did everything right */310 CHECK(skel->bss->dropped != 0, "err_dropped", "exp %ld, got %ld\n",311 0L, skel->bss->dropped);312 CHECK(skel->bss->total != 2, "err_total", "exp %ld, got %ld\n",313 2L, skel->bss->total);314 CHECK(skel->bss->discarded != 1, "err_discarded", "exp %ld, got %ld\n",315 1L, skel->bss->discarded);316 cnt = atomic_xchg(&sample_cnt, 0);317 CHECK(cnt != 0, "cnt", "exp %d samples, got %d\n", 0, cnt);318 319 /* clear flags to return to "adaptive" notification mode */320 skel->bss->flags = 0;321 322 /* produce new samples, no notification should be triggered, because323 * consumer is now behind324 */325 trigger_samples();326 327 /* background poll should still be blocked */328 err = pthread_tryjoin_np(thread, (void **)&bg_ret);329 if (CHECK(err != EBUSY, "try_join", "err %d\n", err))330 goto cleanup;331 332 /* still no samples, because consumer is behind */333 cnt = atomic_xchg(&sample_cnt, 0);334 CHECK(cnt != 0, "cnt", "exp %d samples, got %d\n", 0, cnt);335 336 skel->bss->dropped = 0;337 skel->bss->total = 0;338 skel->bss->discarded = 0;339 340 skel->bss->value = 333;341 syscall(__NR_getpgid);342 /* now force notifications */343 skel->bss->flags = BPF_RB_FORCE_WAKEUP;344 skel->bss->value = 777;345 syscall(__NR_getpgid);346 347 /* now we should get a pending notification */348 usleep(50000);349 err = pthread_tryjoin_np(thread, (void **)&bg_ret);350 if (CHECK(err, "join_bg", "err %d\n", err))351 goto cleanup;352 353 if (CHECK(bg_ret <= 0, "bg_ret", "epoll_wait result: %ld", bg_ret))354 goto cleanup;355 356 /* due to timing variations, there could still be non-notified357 * samples, so consume them here to collect all the samples358 */359 err = ring_buffer__consume(ringbuf);360 CHECK(err < 0, "rb_consume", "failed: %d\b", err);361 362 /* also consume using ring__consume to make sure it works the same */363 err = ring__consume(ring);364 ASSERT_GE(err, 0, "ring_consume");365 366 /* 3 rounds, 2 samples each */367 cnt = atomic_xchg(&sample_cnt, 0);368 CHECK(cnt != 6, "cnt", "exp %d samples, got %d\n", 6, cnt);369 370 /* BPF side did everything right */371 CHECK(skel->bss->dropped != 0, "err_dropped", "exp %ld, got %ld\n",372 0L, skel->bss->dropped);373 CHECK(skel->bss->total != 2, "err_total", "exp %ld, got %ld\n",374 2L, skel->bss->total);375 CHECK(skel->bss->discarded != 1, "err_discarded", "exp %ld, got %ld\n",376 1L, skel->bss->discarded);377 378 test_ringbuf_lskel__detach(skel);379cleanup:380 ring_buffer__free(ringbuf);381 test_ringbuf_lskel__destroy(skel);382}383 384/*385 * Test ring_buffer__consume_n() by producing N_TOT_SAMPLES samples in the ring386 * buffer, via getpid(), and consuming them in chunks of N_SAMPLES.387 */388#define N_TOT_SAMPLES 32389#define N_SAMPLES 4390 391/* Sample value to verify the callback validity */392#define SAMPLE_VALUE 42L393 394static int process_n_sample(void *ctx, void *data, size_t len)395{396 struct sample *s = data;397 398 ASSERT_EQ(s->value, SAMPLE_VALUE, "sample_value");399 400 return 0;401}402 403static void ringbuf_n_subtest(void)404{405 struct test_ringbuf_n_lskel *skel_n;406 int err, i;407 408 skel_n = test_ringbuf_n_lskel__open();409 if (!ASSERT_OK_PTR(skel_n, "test_ringbuf_n_lskel__open"))410 return;411 412 skel_n->maps.ringbuf.max_entries = getpagesize();413 skel_n->bss->pid = getpid();414 415 err = test_ringbuf_n_lskel__load(skel_n);416 if (!ASSERT_OK(err, "test_ringbuf_n_lskel__load"))417 goto cleanup;418 419 ringbuf = ring_buffer__new(skel_n->maps.ringbuf.map_fd,420 process_n_sample, NULL, NULL);421 if (!ASSERT_OK_PTR(ringbuf, "ring_buffer__new"))422 goto cleanup;423 424 err = test_ringbuf_n_lskel__attach(skel_n);425 if (!ASSERT_OK(err, "test_ringbuf_n_lskel__attach"))426 goto cleanup_ringbuf;427 428 /* Produce N_TOT_SAMPLES samples in the ring buffer by calling getpid() */429 skel_n->bss->value = SAMPLE_VALUE;430 for (i = 0; i < N_TOT_SAMPLES; i++)431 syscall(__NR_getpgid);432 433 /* Consume all samples from the ring buffer in batches of N_SAMPLES */434 for (i = 0; i < N_TOT_SAMPLES; i += err) {435 err = ring_buffer__consume_n(ringbuf, N_SAMPLES);436 if (!ASSERT_EQ(err, N_SAMPLES, "rb_consume"))437 goto cleanup_ringbuf;438 }439 440cleanup_ringbuf:441 ring_buffer__free(ringbuf);442cleanup:443 test_ringbuf_n_lskel__destroy(skel_n);444}445 446static int process_map_key_sample(void *ctx, void *data, size_t len)447{448 struct sample *s;449 int err, val;450 451 s = data;452 switch (s->seq) {453 case 1:454 ASSERT_EQ(s->value, 42, "sample_value");455 err = bpf_map_lookup_elem(skel_map_key->maps.hash_map.map_fd,456 s, &val);457 ASSERT_OK(err, "hash_map bpf_map_lookup_elem");458 ASSERT_EQ(val, 1, "hash_map val");459 return -EDONE;460 default:461 return 0;462 }463}464 465static void ringbuf_map_key_subtest(void)466{467 int err;468 469 skel_map_key = test_ringbuf_map_key_lskel__open();470 if (!ASSERT_OK_PTR(skel_map_key, "test_ringbuf_map_key_lskel__open"))471 return;472 473 skel_map_key->maps.ringbuf.max_entries = getpagesize();474 skel_map_key->bss->pid = getpid();475 476 err = test_ringbuf_map_key_lskel__load(skel_map_key);477 if (!ASSERT_OK(err, "test_ringbuf_map_key_lskel__load"))478 goto cleanup;479 480 ringbuf = ring_buffer__new(skel_map_key->maps.ringbuf.map_fd,481 process_map_key_sample, NULL, NULL);482 if (!ASSERT_OK_PTR(ringbuf, "ring_buffer__new"))483 goto cleanup;484 485 err = test_ringbuf_map_key_lskel__attach(skel_map_key);486 if (!ASSERT_OK(err, "test_ringbuf_map_key_lskel__attach"))487 goto cleanup_ringbuf;488 489 syscall(__NR_getpgid);490 ASSERT_EQ(skel_map_key->bss->seq, 1, "skel_map_key->bss->seq");491 err = ring_buffer__poll(ringbuf, -1);492 ASSERT_EQ(err, -EDONE, "ring_buffer__poll");493 494cleanup_ringbuf:495 ring_buffer__free(ringbuf);496cleanup:497 test_ringbuf_map_key_lskel__destroy(skel_map_key);498}499 500void test_ringbuf(void)501{502 if (test__start_subtest("ringbuf"))503 ringbuf_subtest();504 if (test__start_subtest("ringbuf_n"))505 ringbuf_n_subtest();506 if (test__start_subtest("ringbuf_map_key"))507 ringbuf_map_key_subtest();508 if (test__start_subtest("ringbuf_write"))509 ringbuf_write_subtest();510}511