489 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) 2023 Isovalent */3 4#include <errno.h>5#include <unistd.h>6#include <pthread.h>7 8#include <bpf/bpf.h>9#include <bpf/libbpf.h>10 11#include <bpf_util.h>12#include <test_maps.h>13 14#include "map_percpu_stats.skel.h"15 16#define MAX_ENTRIES 1638417#define MAX_ENTRIES_HASH_OF_MAPS 6418#define N_THREADS 819#define MAX_MAP_KEY_SIZE 420#define PCPU_MIN_UNIT_SIZE 3276821 22static void map_info(int map_fd, struct bpf_map_info *info)23{24 __u32 len = sizeof(*info);25 int ret;26 27 memset(info, 0, sizeof(*info));28 29 ret = bpf_obj_get_info_by_fd(map_fd, info, &len);30 CHECK(ret < 0, "bpf_obj_get_info_by_fd", "error: %s\n", strerror(errno));31}32 33static const char *map_type_to_s(__u32 type)34{35 switch (type) {36 case BPF_MAP_TYPE_HASH:37 return "HASH";38 case BPF_MAP_TYPE_PERCPU_HASH:39 return "PERCPU_HASH";40 case BPF_MAP_TYPE_LRU_HASH:41 return "LRU_HASH";42 case BPF_MAP_TYPE_LRU_PERCPU_HASH:43 return "LRU_PERCPU_HASH";44 case BPF_MAP_TYPE_HASH_OF_MAPS:45 return "BPF_MAP_TYPE_HASH_OF_MAPS";46 default:47 return "<define-me>";48 }49}50 51static __u32 map_count_elements(__u32 type, int map_fd)52{53 __u32 key = -1;54 int n = 0;55 56 while (!bpf_map_get_next_key(map_fd, &key, &key))57 n++;58 return n;59}60 61#define BATCH true62 63static void delete_and_lookup_batch(int map_fd, void *keys, __u32 count)64{65 static __u8 values[(8 << 10) * MAX_ENTRIES];66 void *in_batch = NULL, *out_batch;67 __u32 save_count = count;68 int ret;69 70 ret = bpf_map_lookup_and_delete_batch(map_fd,71 &in_batch, &out_batch,72 keys, values, &count,73 NULL);74 75 /*76 * Despite what uapi header says, lookup_and_delete_batch will return77 * -ENOENT in case we successfully have deleted all elements, so check78 * this separately79 */80 CHECK(ret < 0 && (errno != ENOENT || !count), "bpf_map_lookup_and_delete_batch",81 "error: %s\n", strerror(errno));82 83 CHECK(count != save_count,84 "bpf_map_lookup_and_delete_batch",85 "deleted not all elements: removed=%u expected=%u\n",86 count, save_count);87}88 89static void delete_all_elements(__u32 type, int map_fd, bool batch)90{91 static __u8 val[8 << 10]; /* enough for 1024 CPUs */92 __u32 key = -1;93 void *keys;94 __u32 i, n;95 int ret;96 97 keys = calloc(MAX_MAP_KEY_SIZE, MAX_ENTRIES);98 CHECK(!keys, "calloc", "error: %s\n", strerror(errno));99 100 for (n = 0; !bpf_map_get_next_key(map_fd, &key, &key); n++)101 memcpy(keys + n*MAX_MAP_KEY_SIZE, &key, MAX_MAP_KEY_SIZE);102 103 if (batch) {104 /* Can't mix delete_batch and delete_and_lookup_batch because105 * they have different semantics in relation to the keys106 * argument. However, delete_batch utilize map_delete_elem,107 * so we actually test it in non-batch scenario */108 delete_and_lookup_batch(map_fd, keys, n);109 } else {110 /* Intentionally mix delete and lookup_and_delete so we can test both */111 for (i = 0; i < n; i++) {112 void *keyp = keys + i*MAX_MAP_KEY_SIZE;113 114 if (i % 2 || type == BPF_MAP_TYPE_HASH_OF_MAPS) {115 ret = bpf_map_delete_elem(map_fd, keyp);116 CHECK(ret < 0, "bpf_map_delete_elem",117 "error: key %u: %s\n", i, strerror(errno));118 } else {119 ret = bpf_map_lookup_and_delete_elem(map_fd, keyp, val);120 CHECK(ret < 0, "bpf_map_lookup_and_delete_elem",121 "error: key %u: %s\n", i, strerror(errno));122 }123 }124 }125 126 free(keys);127}128 129static bool is_lru(__u32 map_type)130{131 return map_type == BPF_MAP_TYPE_LRU_HASH ||132 map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH;133}134 135static bool is_percpu(__u32 map_type)136{137 return map_type == BPF_MAP_TYPE_PERCPU_HASH ||138 map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH;139}140 141struct upsert_opts {142 __u32 map_type;143 int map_fd;144 __u32 n;145 bool retry_for_nomem;146};147 148static int create_small_hash(void)149{150 int map_fd;151 152 map_fd = bpf_map_create(BPF_MAP_TYPE_HASH, "small", 4, 4, 4, NULL);153 CHECK(map_fd < 0, "bpf_map_create()", "error:%s (name=%s)\n",154 strerror(errno), "small");155 156 return map_fd;157}158 159static bool retry_for_nomem_fn(int err)160{161 return err == ENOMEM;162}163 164static void *patch_map_thread(void *arg)165{166 /* 8KB is enough for 1024 CPUs. And it is shared between N_THREADS. */167 static __u8 blob[8 << 10];168 struct upsert_opts *opts = arg;169 void *val_ptr;170 int val;171 int ret;172 int i;173 174 for (i = 0; i < opts->n; i++) {175 if (opts->map_type == BPF_MAP_TYPE_HASH_OF_MAPS) {176 val = create_small_hash();177 val_ptr = &val;178 } else if (is_percpu(opts->map_type)) {179 val_ptr = blob;180 } else {181 val = rand();182 val_ptr = &val;183 }184 185 /* 2 seconds may be enough ? */186 if (opts->retry_for_nomem)187 ret = map_update_retriable(opts->map_fd, &i, val_ptr, 0,188 40, retry_for_nomem_fn);189 else190 ret = bpf_map_update_elem(opts->map_fd, &i, val_ptr, 0);191 CHECK(ret < 0, "bpf_map_update_elem", "key=%d error: %s\n", i, strerror(errno));192 193 if (opts->map_type == BPF_MAP_TYPE_HASH_OF_MAPS)194 close(val);195 }196 return NULL;197}198 199static void upsert_elements(struct upsert_opts *opts)200{201 pthread_t threads[N_THREADS];202 int ret;203 int i;204 205 for (i = 0; i < ARRAY_SIZE(threads); i++) {206 ret = pthread_create(&i[threads], NULL, patch_map_thread, opts);207 CHECK(ret != 0, "pthread_create", "error: %s\n", strerror(ret));208 }209 210 for (i = 0; i < ARRAY_SIZE(threads); i++) {211 ret = pthread_join(i[threads], NULL);212 CHECK(ret != 0, "pthread_join", "error: %s\n", strerror(ret));213 }214}215 216static __u32 read_cur_elements(int iter_fd)217{218 char buf[64];219 ssize_t n;220 __u32 ret;221 222 n = read(iter_fd, buf, sizeof(buf)-1);223 CHECK(n <= 0, "read", "error: %s\n", strerror(errno));224 buf[n] = '\0';225 226 errno = 0;227 ret = (__u32)strtol(buf, NULL, 10);228 CHECK(errno != 0, "strtol", "error: %s\n", strerror(errno));229 230 return ret;231}232 233static __u32 get_cur_elements(int map_id)234{235 struct map_percpu_stats *skel;236 struct bpf_link *link;237 __u32 n_elements;238 int iter_fd;239 int ret;240 241 skel = map_percpu_stats__open();242 CHECK(skel == NULL, "map_percpu_stats__open", "error: %s", strerror(errno));243 244 skel->bss->target_id = map_id;245 246 ret = map_percpu_stats__load(skel);247 CHECK(ret != 0, "map_percpu_stats__load", "error: %s", strerror(errno));248 249 link = bpf_program__attach_iter(skel->progs.dump_bpf_map, NULL);250 CHECK(!link, "bpf_program__attach_iter", "error: %s\n", strerror(errno));251 252 iter_fd = bpf_iter_create(bpf_link__fd(link));253 CHECK(iter_fd < 0, "bpf_iter_create", "error: %s\n", strerror(errno));254 255 n_elements = read_cur_elements(iter_fd);256 257 close(iter_fd);258 bpf_link__destroy(link);259 map_percpu_stats__destroy(skel);260 261 return n_elements;262}263 264static void check_expected_number_elements(__u32 n_inserted, int map_fd,265 struct bpf_map_info *info)266{267 __u32 n_real;268 __u32 n_iter;269 270 /* Count the current number of elements in the map by iterating through271 * all the map keys via bpf_get_next_key272 */273 n_real = map_count_elements(info->type, map_fd);274 275 /* The "real" number of elements should be the same as the inserted276 * number of elements in all cases except LRU maps, where some elements277 * may have been evicted278 */279 if (n_inserted == 0 || !is_lru(info->type))280 CHECK(n_inserted != n_real, "map_count_elements",281 "n_real(%u) != n_inserted(%u)\n", n_real, n_inserted);282 283 /* Count the current number of elements in the map using an iterator */284 n_iter = get_cur_elements(info->id);285 286 /* Both counts should be the same, as all updates are over */287 CHECK(n_iter != n_real, "get_cur_elements",288 "n_iter=%u, expected %u (map_type=%s,map_flags=%08x)\n",289 n_iter, n_real, map_type_to_s(info->type), info->map_flags);290}291 292static void __test(int map_fd)293{294 struct upsert_opts opts = {295 .map_fd = map_fd,296 };297 struct bpf_map_info info;298 299 map_info(map_fd, &info);300 opts.map_type = info.type;301 opts.n = info.max_entries;302 303 /* Reduce the number of elements we are updating such that we don't304 * bump into -E2BIG from non-preallocated hash maps, but still will305 * have some evictions for LRU maps */306 if (opts.map_type != BPF_MAP_TYPE_HASH_OF_MAPS)307 opts.n -= 512;308 else309 opts.n /= 2;310 311 /* per-cpu bpf memory allocator may not be able to allocate per-cpu312 * pointer successfully and it can not refill free llist timely, and313 * bpf_map_update_elem() will return -ENOMEM. so just retry to mitigate314 * the problem temporarily.315 */316 opts.retry_for_nomem = is_percpu(opts.map_type) && (info.map_flags & BPF_F_NO_PREALLOC);317 318 /*319 * Upsert keys [0, n) under some competition: with random values from320 * N_THREADS threads. Check values, then delete all elements and check321 * values again.322 */323 upsert_elements(&opts);324 check_expected_number_elements(opts.n, map_fd, &info);325 delete_all_elements(info.type, map_fd, !BATCH);326 check_expected_number_elements(0, map_fd, &info);327 328 /* Now do the same, but using batch delete operations */329 upsert_elements(&opts);330 check_expected_number_elements(opts.n, map_fd, &info);331 delete_all_elements(info.type, map_fd, BATCH);332 check_expected_number_elements(0, map_fd, &info);333 334 close(map_fd);335}336 337static int map_create_opts(__u32 type, const char *name,338 struct bpf_map_create_opts *map_opts,339 __u32 key_size, __u32 val_size)340{341 int max_entries;342 int map_fd;343 344 if (type == BPF_MAP_TYPE_HASH_OF_MAPS)345 max_entries = MAX_ENTRIES_HASH_OF_MAPS;346 else347 max_entries = MAX_ENTRIES;348 349 map_fd = bpf_map_create(type, name, key_size, val_size, max_entries, map_opts);350 CHECK(map_fd < 0, "bpf_map_create()", "error:%s (name=%s)\n",351 strerror(errno), name);352 353 return map_fd;354}355 356static int map_create(__u32 type, const char *name, struct bpf_map_create_opts *map_opts)357{358 return map_create_opts(type, name, map_opts, sizeof(int), sizeof(int));359}360 361static int create_hash(void)362{363 LIBBPF_OPTS(bpf_map_create_opts, map_opts, .map_flags = BPF_F_NO_PREALLOC);364 365 return map_create(BPF_MAP_TYPE_HASH, "hash", &map_opts);366}367 368static int create_percpu_hash(void)369{370 LIBBPF_OPTS(bpf_map_create_opts, map_opts, .map_flags = BPF_F_NO_PREALLOC);371 372 return map_create(BPF_MAP_TYPE_PERCPU_HASH, "percpu_hash", &map_opts);373}374 375static int create_hash_prealloc(void)376{377 return map_create(BPF_MAP_TYPE_HASH, "hash", NULL);378}379 380static int create_percpu_hash_prealloc(void)381{382 return map_create(BPF_MAP_TYPE_PERCPU_HASH, "percpu_hash_prealloc", NULL);383}384 385static int create_lru_hash(__u32 type, __u32 map_flags)386{387 LIBBPF_OPTS(bpf_map_create_opts, map_opts, .map_flags = map_flags);388 389 return map_create(type, "lru_hash", &map_opts);390}391 392static int create_hash_of_maps(void)393{394 LIBBPF_OPTS(bpf_map_create_opts, map_opts,395 .map_flags = BPF_F_NO_PREALLOC,396 .inner_map_fd = create_small_hash(),397 );398 int ret;399 400 ret = map_create_opts(BPF_MAP_TYPE_HASH_OF_MAPS, "hash_of_maps",401 &map_opts, sizeof(int), sizeof(int));402 close(map_opts.inner_map_fd);403 return ret;404}405 406static void map_percpu_stats_hash(void)407{408 __test(create_hash());409 printf("test_%s:PASS\n", __func__);410}411 412static void map_percpu_stats_percpu_hash(void)413{414 __test(create_percpu_hash());415 printf("test_%s:PASS\n", __func__);416}417 418static void map_percpu_stats_hash_prealloc(void)419{420 __test(create_hash_prealloc());421 printf("test_%s:PASS\n", __func__);422}423 424static void map_percpu_stats_percpu_hash_prealloc(void)425{426 __test(create_percpu_hash_prealloc());427 printf("test_%s:PASS\n", __func__);428}429 430static void map_percpu_stats_lru_hash(void)431{432 __test(create_lru_hash(BPF_MAP_TYPE_LRU_HASH, 0));433 printf("test_%s:PASS\n", __func__);434}435 436static void map_percpu_stats_lru_hash_no_common(void)437{438 __test(create_lru_hash(BPF_MAP_TYPE_LRU_HASH, BPF_F_NO_COMMON_LRU));439 printf("test_%s:PASS\n", __func__);440}441 442static void map_percpu_stats_percpu_lru_hash(void)443{444 __test(create_lru_hash(BPF_MAP_TYPE_LRU_PERCPU_HASH, 0));445 printf("test_%s:PASS\n", __func__);446}447 448static void map_percpu_stats_percpu_lru_hash_no_common(void)449{450 __test(create_lru_hash(BPF_MAP_TYPE_LRU_PERCPU_HASH, BPF_F_NO_COMMON_LRU));451 printf("test_%s:PASS\n", __func__);452}453 454static void map_percpu_stats_hash_of_maps(void)455{456 __test(create_hash_of_maps());457 printf("test_%s:PASS\n", __func__);458}459 460static void map_percpu_stats_map_value_size(void)461{462 int fd;463 int value_sz = PCPU_MIN_UNIT_SIZE + 1;464 struct bpf_map_create_opts opts = { .sz = sizeof(opts) };465 enum bpf_map_type map_types[] = { BPF_MAP_TYPE_PERCPU_ARRAY,466 BPF_MAP_TYPE_PERCPU_HASH,467 BPF_MAP_TYPE_LRU_PERCPU_HASH };468 for (int i = 0; i < ARRAY_SIZE(map_types); i++) {469 fd = bpf_map_create(map_types[i], NULL, sizeof(__u32), value_sz, 1, &opts);470 CHECK(fd < 0 && errno != E2BIG, "percpu map value size",471 "error: %s\n", strerror(errno));472 }473 printf("test_%s:PASS\n", __func__);474}475 476void test_map_percpu_stats(void)477{478 map_percpu_stats_hash();479 map_percpu_stats_percpu_hash();480 map_percpu_stats_hash_prealloc();481 map_percpu_stats_percpu_hash_prealloc();482 map_percpu_stats_lru_hash();483 map_percpu_stats_lru_hash_no_common();484 map_percpu_stats_percpu_lru_hash();485 map_percpu_stats_percpu_lru_hash_no_common();486 map_percpu_stats_hash_of_maps();487 map_percpu_stats_map_value_size();488}489