628 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) 2019 Facebook */3#include <linux/compiler.h>4#include <linux/err.h>5 6#include <sys/resource.h>7#include <sys/socket.h>8#include <sys/types.h>9#include <linux/btf.h>10#include <unistd.h>11#include <signal.h>12#include <errno.h>13#include <string.h>14#include <pthread.h>15 16#include <bpf/bpf.h>17#include <bpf/libbpf.h>18 19#include <test_btf.h>20#include <test_maps.h>21 22static struct bpf_map_create_opts map_opts = {23 .sz = sizeof(map_opts),24 .btf_key_type_id = 1,25 .btf_value_type_id = 3,26 .btf_fd = -1,27 .map_flags = BPF_F_NO_PREALLOC,28};29 30static unsigned int nr_sk_threads_done;31static unsigned int nr_sk_threads_err;32static unsigned int nr_sk_per_thread = 4096;33static unsigned int nr_sk_threads = 4;34static int sk_storage_map = -1;35static unsigned int stop;36static int runtime_s = 5;37 38static bool is_stopped(void)39{40 return READ_ONCE(stop);41}42 43static unsigned int threads_err(void)44{45 return READ_ONCE(nr_sk_threads_err);46}47 48static void notify_thread_err(void)49{50 __sync_add_and_fetch(&nr_sk_threads_err, 1);51}52 53static bool wait_for_threads_err(void)54{55 while (!is_stopped() && !threads_err())56 usleep(500);57 58 return !is_stopped();59}60 61static unsigned int threads_done(void)62{63 return READ_ONCE(nr_sk_threads_done);64}65 66static void notify_thread_done(void)67{68 __sync_add_and_fetch(&nr_sk_threads_done, 1);69}70 71static void notify_thread_redo(void)72{73 __sync_sub_and_fetch(&nr_sk_threads_done, 1);74}75 76static bool wait_for_threads_done(void)77{78 while (threads_done() != nr_sk_threads && !is_stopped() &&79 !threads_err())80 usleep(50);81 82 return !is_stopped() && !threads_err();83}84 85static bool wait_for_threads_redo(void)86{87 while (threads_done() && !is_stopped() && !threads_err())88 usleep(50);89 90 return !is_stopped() && !threads_err();91}92 93static bool wait_for_map(void)94{95 while (READ_ONCE(sk_storage_map) == -1 && !is_stopped())96 usleep(50);97 98 return !is_stopped();99}100 101static bool wait_for_map_close(void)102{103 while (READ_ONCE(sk_storage_map) != -1 && !is_stopped())104 ;105 106 return !is_stopped();107}108 109static int load_btf(void)110{111 const char btf_str_sec[] = "\0bpf_spin_lock\0val\0cnt\0l";112 __u32 btf_raw_types[] = {113 /* int */114 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */115 /* struct bpf_spin_lock */ /* [2] */116 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 1), 4),117 BTF_MEMBER_ENC(15, 1, 0), /* int val; */118 /* struct val */ /* [3] */119 BTF_TYPE_ENC(15, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 2), 8),120 BTF_MEMBER_ENC(19, 1, 0), /* int cnt; */121 BTF_MEMBER_ENC(23, 2, 32),/* struct bpf_spin_lock l; */122 };123 struct btf_header btf_hdr = {124 .magic = BTF_MAGIC,125 .version = BTF_VERSION,126 .hdr_len = sizeof(struct btf_header),127 .type_len = sizeof(btf_raw_types),128 .str_off = sizeof(btf_raw_types),129 .str_len = sizeof(btf_str_sec),130 };131 __u8 raw_btf[sizeof(struct btf_header) + sizeof(btf_raw_types) +132 sizeof(btf_str_sec)];133 134 memcpy(raw_btf, &btf_hdr, sizeof(btf_hdr));135 memcpy(raw_btf + sizeof(btf_hdr), btf_raw_types, sizeof(btf_raw_types));136 memcpy(raw_btf + sizeof(btf_hdr) + sizeof(btf_raw_types),137 btf_str_sec, sizeof(btf_str_sec));138 139 return bpf_btf_load(raw_btf, sizeof(raw_btf), NULL);140}141 142static int create_sk_storage_map(void)143{144 int btf_fd, map_fd;145 146 btf_fd = load_btf();147 CHECK(btf_fd == -1, "bpf_load_btf", "btf_fd:%d errno:%d\n",148 btf_fd, errno);149 map_opts.btf_fd = btf_fd;150 151 map_fd = bpf_map_create(BPF_MAP_TYPE_SK_STORAGE, "sk_storage_map", 4, 8, 0, &map_opts);152 map_opts.btf_fd = -1;153 close(btf_fd);154 CHECK(map_fd == -1,155 "bpf_map_create()", "errno:%d\n", errno);156 157 return map_fd;158}159 160static void *insert_close_thread(void *arg)161{162 struct {163 int cnt;164 int lock;165 } value = { .cnt = 0xeB9F, .lock = 0, };166 int i, map_fd, err, *sk_fds;167 168 sk_fds = malloc(sizeof(*sk_fds) * nr_sk_per_thread);169 if (!sk_fds) {170 notify_thread_err();171 return ERR_PTR(-ENOMEM);172 }173 174 for (i = 0; i < nr_sk_per_thread; i++)175 sk_fds[i] = -1;176 177 while (!is_stopped()) {178 if (!wait_for_map())179 goto close_all;180 181 map_fd = READ_ONCE(sk_storage_map);182 for (i = 0; i < nr_sk_per_thread && !is_stopped(); i++) {183 sk_fds[i] = socket(AF_INET6, SOCK_STREAM, 0);184 if (sk_fds[i] == -1) {185 err = -errno;186 fprintf(stderr, "socket(): errno:%d\n", errno);187 goto errout;188 }189 err = bpf_map_update_elem(map_fd, &sk_fds[i], &value,190 BPF_NOEXIST);191 if (err) {192 err = -errno;193 fprintf(stderr,194 "bpf_map_update_elem(): errno:%d\n",195 errno);196 goto errout;197 }198 }199 200 notify_thread_done();201 wait_for_map_close();202 203close_all:204 for (i = 0; i < nr_sk_per_thread; i++) {205 close(sk_fds[i]);206 sk_fds[i] = -1;207 }208 209 notify_thread_redo();210 }211 212 free(sk_fds);213 return NULL;214 215errout:216 for (i = 0; i < nr_sk_per_thread && sk_fds[i] != -1; i++)217 close(sk_fds[i]);218 free(sk_fds);219 notify_thread_err();220 return ERR_PTR(err);221}222 223static int do_sk_storage_map_stress_free(void)224{225 int i, map_fd = -1, err = 0, nr_threads_created = 0;226 pthread_t *sk_thread_ids;227 void *thread_ret;228 229 sk_thread_ids = malloc(sizeof(pthread_t) * nr_sk_threads);230 if (!sk_thread_ids) {231 fprintf(stderr, "malloc(sk_threads): NULL\n");232 return -ENOMEM;233 }234 235 for (i = 0; i < nr_sk_threads; i++) {236 err = pthread_create(&sk_thread_ids[i], NULL,237 insert_close_thread, NULL);238 if (err) {239 err = -errno;240 goto done;241 }242 nr_threads_created++;243 }244 245 while (!is_stopped()) {246 map_fd = create_sk_storage_map();247 WRITE_ONCE(sk_storage_map, map_fd);248 249 if (!wait_for_threads_done())250 break;251 252 WRITE_ONCE(sk_storage_map, -1);253 close(map_fd);254 map_fd = -1;255 256 if (!wait_for_threads_redo())257 break;258 }259 260done:261 WRITE_ONCE(stop, 1);262 for (i = 0; i < nr_threads_created; i++) {263 pthread_join(sk_thread_ids[i], &thread_ret);264 if (IS_ERR(thread_ret) && !err) {265 err = PTR_ERR(thread_ret);266 fprintf(stderr, "threads#%u: err:%d\n", i, err);267 }268 }269 free(sk_thread_ids);270 271 if (map_fd != -1)272 close(map_fd);273 274 return err;275}276 277static void *update_thread(void *arg)278{279 struct {280 int cnt;281 int lock;282 } value = { .cnt = 0xeB9F, .lock = 0, };283 int map_fd = READ_ONCE(sk_storage_map);284 int sk_fd = *(int *)arg;285 int err = 0; /* Suppress compiler false alarm */286 287 while (!is_stopped()) {288 err = bpf_map_update_elem(map_fd, &sk_fd, &value, 0);289 if (err && errno != EAGAIN) {290 err = -errno;291 fprintf(stderr, "bpf_map_update_elem: %d %d\n",292 err, errno);293 break;294 }295 }296 297 if (!is_stopped()) {298 notify_thread_err();299 return ERR_PTR(err);300 }301 302 return NULL;303}304 305static void *delete_thread(void *arg)306{307 int map_fd = READ_ONCE(sk_storage_map);308 int sk_fd = *(int *)arg;309 int err = 0; /* Suppress compiler false alarm */310 311 while (!is_stopped()) {312 err = bpf_map_delete_elem(map_fd, &sk_fd);313 if (err && errno != ENOENT) {314 err = -errno;315 fprintf(stderr, "bpf_map_delete_elem: %d %d\n",316 err, errno);317 break;318 }319 }320 321 if (!is_stopped()) {322 notify_thread_err();323 return ERR_PTR(err);324 }325 326 return NULL;327}328 329static int do_sk_storage_map_stress_change(void)330{331 int i, sk_fd, map_fd = -1, err = 0, nr_threads_created = 0;332 pthread_t *sk_thread_ids;333 void *thread_ret;334 335 sk_thread_ids = malloc(sizeof(pthread_t) * nr_sk_threads);336 if (!sk_thread_ids) {337 fprintf(stderr, "malloc(sk_threads): NULL\n");338 return -ENOMEM;339 }340 341 sk_fd = socket(AF_INET6, SOCK_STREAM, 0);342 if (sk_fd == -1) {343 err = -errno;344 goto done;345 }346 347 map_fd = create_sk_storage_map();348 WRITE_ONCE(sk_storage_map, map_fd);349 350 for (i = 0; i < nr_sk_threads; i++) {351 if (i & 0x1)352 err = pthread_create(&sk_thread_ids[i], NULL,353 update_thread, &sk_fd);354 else355 err = pthread_create(&sk_thread_ids[i], NULL,356 delete_thread, &sk_fd);357 if (err) {358 err = -errno;359 goto done;360 }361 nr_threads_created++;362 }363 364 wait_for_threads_err();365 366done:367 WRITE_ONCE(stop, 1);368 for (i = 0; i < nr_threads_created; i++) {369 pthread_join(sk_thread_ids[i], &thread_ret);370 if (IS_ERR(thread_ret) && !err) {371 err = PTR_ERR(thread_ret);372 fprintf(stderr, "threads#%u: err:%d\n", i, err);373 }374 }375 free(sk_thread_ids);376 377 if (sk_fd != -1)378 close(sk_fd);379 close(map_fd);380 381 return err;382}383 384static void stop_handler(int signum)385{386 if (signum != SIGALRM)387 printf("stopping...\n");388 WRITE_ONCE(stop, 1);389}390 391#define BPF_SK_STORAGE_MAP_TEST_NR_THREADS "BPF_SK_STORAGE_MAP_TEST_NR_THREADS"392#define BPF_SK_STORAGE_MAP_TEST_SK_PER_THREAD "BPF_SK_STORAGE_MAP_TEST_SK_PER_THREAD"393#define BPF_SK_STORAGE_MAP_TEST_RUNTIME_S "BPF_SK_STORAGE_MAP_TEST_RUNTIME_S"394#define BPF_SK_STORAGE_MAP_TEST_NAME "BPF_SK_STORAGE_MAP_TEST_NAME"395 396static void test_sk_storage_map_stress_free(void)397{398 struct rlimit rlim_old, rlim_new = {};399 int err;400 401 getrlimit(RLIMIT_NOFILE, &rlim_old);402 403 signal(SIGTERM, stop_handler);404 signal(SIGINT, stop_handler);405 if (runtime_s > 0) {406 signal(SIGALRM, stop_handler);407 alarm(runtime_s);408 }409 410 if (rlim_old.rlim_cur < nr_sk_threads * nr_sk_per_thread) {411 rlim_new.rlim_cur = nr_sk_threads * nr_sk_per_thread + 128;412 rlim_new.rlim_max = rlim_new.rlim_cur + 128;413 err = setrlimit(RLIMIT_NOFILE, &rlim_new);414 CHECK(err, "setrlimit(RLIMIT_NOFILE)", "rlim_new:%lu errno:%d",415 (unsigned long) rlim_new.rlim_cur, errno);416 }417 418 err = do_sk_storage_map_stress_free();419 420 signal(SIGTERM, SIG_DFL);421 signal(SIGINT, SIG_DFL);422 if (runtime_s > 0) {423 signal(SIGALRM, SIG_DFL);424 alarm(0);425 }426 427 if (rlim_new.rlim_cur)428 setrlimit(RLIMIT_NOFILE, &rlim_old);429 430 CHECK(err, "test_sk_storage_map_stress_free", "err:%d\n", err);431}432 433static void test_sk_storage_map_stress_change(void)434{435 int err;436 437 signal(SIGTERM, stop_handler);438 signal(SIGINT, stop_handler);439 if (runtime_s > 0) {440 signal(SIGALRM, stop_handler);441 alarm(runtime_s);442 }443 444 err = do_sk_storage_map_stress_change();445 446 signal(SIGTERM, SIG_DFL);447 signal(SIGINT, SIG_DFL);448 if (runtime_s > 0) {449 signal(SIGALRM, SIG_DFL);450 alarm(0);451 }452 453 CHECK(err, "test_sk_storage_map_stress_change", "err:%d\n", err);454}455 456static void test_sk_storage_map_basic(void)457{458 struct {459 int cnt;460 int lock;461 } value = { .cnt = 0xeB9f, .lock = 1, }, lookup_value;462 struct bpf_map_create_opts bad_xattr;463 int btf_fd, map_fd, sk_fd, err;464 465 btf_fd = load_btf();466 CHECK(btf_fd == -1, "bpf_load_btf", "btf_fd:%d errno:%d\n",467 btf_fd, errno);468 map_opts.btf_fd = btf_fd;469 470 sk_fd = socket(AF_INET6, SOCK_STREAM, 0);471 CHECK(sk_fd == -1, "socket()", "sk_fd:%d errno:%d\n",472 sk_fd, errno);473 474 map_fd = bpf_map_create(BPF_MAP_TYPE_SK_STORAGE, "sk_storage_map", 4, 8, 0, &map_opts);475 CHECK(map_fd == -1, "bpf_map_create(good_xattr)",476 "map_fd:%d errno:%d\n", map_fd, errno);477 478 /* Add new elem */479 memcpy(&lookup_value, &value, sizeof(value));480 err = bpf_map_update_elem(map_fd, &sk_fd, &value,481 BPF_NOEXIST | BPF_F_LOCK);482 CHECK(err, "bpf_map_update_elem(BPF_NOEXIST|BPF_F_LOCK)",483 "err:%d errno:%d\n", err, errno);484 err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,485 BPF_F_LOCK);486 CHECK(err || lookup_value.lock || lookup_value.cnt != value.cnt,487 "bpf_map_lookup_elem_flags(BPF_F_LOCK)",488 "err:%d errno:%d lock:%x cnt:%x(%x)\n",489 err, errno, lookup_value.lock, lookup_value.cnt, value.cnt);490 491 /* Bump the cnt and update with BPF_EXIST | BPF_F_LOCK */492 value.cnt += 1;493 value.lock = 2;494 err = bpf_map_update_elem(map_fd, &sk_fd, &value,495 BPF_EXIST | BPF_F_LOCK);496 CHECK(err, "bpf_map_update_elem(BPF_EXIST|BPF_F_LOCK)",497 "err:%d errno:%d\n", err, errno);498 err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,499 BPF_F_LOCK);500 CHECK(err || lookup_value.lock || lookup_value.cnt != value.cnt,501 "bpf_map_lookup_elem_flags(BPF_F_LOCK)",502 "err:%d errno:%d lock:%x cnt:%x(%x)\n",503 err, errno, lookup_value.lock, lookup_value.cnt, value.cnt);504 505 /* Bump the cnt and update with BPF_EXIST */506 value.cnt += 1;507 value.lock = 2;508 err = bpf_map_update_elem(map_fd, &sk_fd, &value, BPF_EXIST);509 CHECK(err, "bpf_map_update_elem(BPF_EXIST)",510 "err:%d errno:%d\n", err, errno);511 err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,512 BPF_F_LOCK);513 CHECK(err || lookup_value.lock || lookup_value.cnt != value.cnt,514 "bpf_map_lookup_elem_flags(BPF_F_LOCK)",515 "err:%d errno:%d lock:%x cnt:%x(%x)\n",516 err, errno, lookup_value.lock, lookup_value.cnt, value.cnt);517 518 /* Update with BPF_NOEXIST */519 value.cnt += 1;520 value.lock = 2;521 err = bpf_map_update_elem(map_fd, &sk_fd, &value,522 BPF_NOEXIST | BPF_F_LOCK);523 CHECK(!err || errno != EEXIST,524 "bpf_map_update_elem(BPF_NOEXIST|BPF_F_LOCK)",525 "err:%d errno:%d\n", err, errno);526 err = bpf_map_update_elem(map_fd, &sk_fd, &value, BPF_NOEXIST);527 CHECK(!err || errno != EEXIST, "bpf_map_update_elem(BPF_NOEXIST)",528 "err:%d errno:%d\n", err, errno);529 value.cnt -= 1;530 err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,531 BPF_F_LOCK);532 CHECK(err || lookup_value.lock || lookup_value.cnt != value.cnt,533 "bpf_map_lookup_elem_flags(BPF_F_LOCK)",534 "err:%d errno:%d lock:%x cnt:%x(%x)\n",535 err, errno, lookup_value.lock, lookup_value.cnt, value.cnt);536 537 /* Bump the cnt again and update with map_flags == 0 */538 value.cnt += 1;539 value.lock = 2;540 err = bpf_map_update_elem(map_fd, &sk_fd, &value, 0);541 CHECK(err, "bpf_map_update_elem()", "err:%d errno:%d\n",542 err, errno);543 err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,544 BPF_F_LOCK);545 CHECK(err || lookup_value.lock || lookup_value.cnt != value.cnt,546 "bpf_map_lookup_elem_flags(BPF_F_LOCK)",547 "err:%d errno:%d lock:%x cnt:%x(%x)\n",548 err, errno, lookup_value.lock, lookup_value.cnt, value.cnt);549 550 /* Test delete elem */551 err = bpf_map_delete_elem(map_fd, &sk_fd);552 CHECK(err, "bpf_map_delete_elem()", "err:%d errno:%d\n",553 err, errno);554 err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,555 BPF_F_LOCK);556 CHECK(!err || errno != ENOENT,557 "bpf_map_lookup_elem_flags(BPF_F_LOCK)",558 "err:%d errno:%d\n", err, errno);559 err = bpf_map_delete_elem(map_fd, &sk_fd);560 CHECK(!err || errno != ENOENT, "bpf_map_delete_elem()",561 "err:%d errno:%d\n", err, errno);562 563 memcpy(&bad_xattr, &map_opts, sizeof(map_opts));564 bad_xattr.btf_key_type_id = 0;565 err = bpf_map_create(BPF_MAP_TYPE_SK_STORAGE, "sk_storage_map", 4, 8, 0, &bad_xattr);566 CHECK(!err || errno != EINVAL, "bpf_map_create(bad_xattr)",567 "err:%d errno:%d\n", err, errno);568 569 memcpy(&bad_xattr, &map_opts, sizeof(map_opts));570 bad_xattr.btf_key_type_id = 3;571 err = bpf_map_create(BPF_MAP_TYPE_SK_STORAGE, "sk_storage_map", 4, 8, 0, &bad_xattr);572 CHECK(!err || errno != EINVAL, "bpf_map_create(bad_xattr)",573 "err:%d errno:%d\n", err, errno);574 575 err = bpf_map_create(BPF_MAP_TYPE_SK_STORAGE, "sk_storage_map", 4, 8, 1, &map_opts);576 CHECK(!err || errno != EINVAL, "bpf_map_create(bad_xattr)",577 "err:%d errno:%d\n", err, errno);578 579 memcpy(&bad_xattr, &map_opts, sizeof(map_opts));580 bad_xattr.map_flags = 0;581 err = bpf_map_create(BPF_MAP_TYPE_SK_STORAGE, "sk_storage_map", 4, 8, 0, &bad_xattr);582 CHECK(!err || errno != EINVAL, "bap_create_map_xattr(bad_xattr)",583 "err:%d errno:%d\n", err, errno);584 585 map_opts.btf_fd = -1;586 close(btf_fd);587 close(map_fd);588 close(sk_fd);589}590 591void test_sk_storage_map(void)592{593 const char *test_name, *env_opt;594 bool test_ran = false;595 596 test_name = getenv(BPF_SK_STORAGE_MAP_TEST_NAME);597 598 env_opt = getenv(BPF_SK_STORAGE_MAP_TEST_NR_THREADS);599 if (env_opt)600 nr_sk_threads = atoi(env_opt);601 602 env_opt = getenv(BPF_SK_STORAGE_MAP_TEST_SK_PER_THREAD);603 if (env_opt)604 nr_sk_per_thread = atoi(env_opt);605 606 env_opt = getenv(BPF_SK_STORAGE_MAP_TEST_RUNTIME_S);607 if (env_opt)608 runtime_s = atoi(env_opt);609 610 if (!test_name || !strcmp(test_name, "basic")) {611 test_sk_storage_map_basic();612 test_ran = true;613 }614 if (!test_name || !strcmp(test_name, "stress_free")) {615 test_sk_storage_map_stress_free();616 test_ran = true;617 }618 if (!test_name || !strcmp(test_name, "stress_change")) {619 test_sk_storage_map_stress_change();620 test_ran = true;621 }622 623 if (test_ran)624 printf("%s:PASS\n", __func__);625 else626 CHECK(1, "Invalid test_name", "%s\n", test_name);627}628