1940 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Testsuite for eBPF maps4 *5 * Copyright (c) 2014 PLUMgrid, http://plumgrid.com6 * Copyright (c) 2016 Facebook7 */8 9#include <stdio.h>10#include <unistd.h>11#include <errno.h>12#include <string.h>13#include <assert.h>14#include <stdlib.h>15#include <time.h>16 17#include <sys/wait.h>18#include <sys/socket.h>19#include <netinet/in.h>20#include <linux/bpf.h>21 22#include <bpf/bpf.h>23#include <bpf/libbpf.h>24 25#include "bpf_util.h"26#include "test_maps.h"27#include "testing_helpers.h"28 29#ifndef ENOTSUPP30#define ENOTSUPP 52431#endif32 33int skips;34 35static struct bpf_map_create_opts map_opts = { .sz = sizeof(map_opts) };36 37static void test_hashmap(unsigned int task, void *data)38{39 long long key, next_key, first_key, value;40 int fd;41 42 fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, sizeof(key), sizeof(value), 2, &map_opts);43 if (fd < 0) {44 printf("Failed to create hashmap '%s'!\n", strerror(errno));45 exit(1);46 }47 48 key = 1;49 value = 1234;50 /* Insert key=1 element. */51 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0);52 53 value = 0;54 /* BPF_NOEXIST means add new element if it doesn't exist. */55 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) < 0 &&56 /* key=1 already exists. */57 errno == EEXIST);58 59 /* -1 is an invalid flag. */60 assert(bpf_map_update_elem(fd, &key, &value, -1) < 0 &&61 errno == EINVAL);62 63 /* Check that key=1 can be found. */64 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 1234);65 66 key = 2;67 value = 1234;68 /* Insert key=2 element. */69 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0);70 71 /* Check that key=2 matches the value and delete it */72 assert(bpf_map_lookup_and_delete_elem(fd, &key, &value) == 0 && value == 1234);73 74 /* Check that key=2 is not found. */75 assert(bpf_map_lookup_elem(fd, &key, &value) < 0 && errno == ENOENT);76 77 /* BPF_EXIST means update existing element. */78 assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) < 0 &&79 /* key=2 is not there. */80 errno == ENOENT);81 82 /* Insert key=2 element. */83 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == 0);84 85 /* key=1 and key=2 were inserted, check that key=0 cannot be86 * inserted due to max_entries limit.87 */88 key = 0;89 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) < 0 &&90 errno == E2BIG);91 92 /* Update existing element, though the map is full. */93 key = 1;94 assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) == 0);95 key = 2;96 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0);97 key = 3;98 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) < 0 &&99 errno == E2BIG);100 101 /* Check that key = 0 doesn't exist. */102 key = 0;103 assert(bpf_map_delete_elem(fd, &key) < 0 && errno == ENOENT);104 105 /* Iterate over two elements. */106 assert(bpf_map_get_next_key(fd, NULL, &first_key) == 0 &&107 (first_key == 1 || first_key == 2));108 assert(bpf_map_get_next_key(fd, &key, &next_key) == 0 &&109 (next_key == first_key));110 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == 0 &&111 (next_key == 1 || next_key == 2) &&112 (next_key != first_key));113 assert(bpf_map_get_next_key(fd, &next_key, &next_key) < 0 &&114 errno == ENOENT);115 116 /* Delete both elements. */117 key = 1;118 assert(bpf_map_delete_elem(fd, &key) == 0);119 key = 2;120 assert(bpf_map_delete_elem(fd, &key) == 0);121 assert(bpf_map_delete_elem(fd, &key) < 0 && errno == ENOENT);122 123 key = 0;124 /* Check that map is empty. */125 assert(bpf_map_get_next_key(fd, NULL, &next_key) < 0 &&126 errno == ENOENT);127 assert(bpf_map_get_next_key(fd, &key, &next_key) < 0 &&128 errno == ENOENT);129 130 close(fd);131}132 133static void test_hashmap_sizes(unsigned int task, void *data)134{135 int fd, i, j;136 137 for (i = 1; i <= 512; i <<= 1)138 for (j = 1; j <= 1 << 18; j <<= 1) {139 fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, i, j, 2, &map_opts);140 if (fd < 0) {141 if (errno == ENOMEM)142 return;143 printf("Failed to create hashmap key=%d value=%d '%s'\n",144 i, j, strerror(errno));145 exit(1);146 }147 close(fd);148 usleep(10); /* give kernel time to destroy */149 }150}151 152static void test_hashmap_percpu(unsigned int task, void *data)153{154 unsigned int nr_cpus = bpf_num_possible_cpus();155 BPF_DECLARE_PERCPU(long, value);156 long long key, next_key, first_key;157 int expected_key_mask = 0;158 int fd, i;159 160 fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_HASH, NULL, sizeof(key),161 sizeof(bpf_percpu(value, 0)), 2, &map_opts);162 if (fd < 0) {163 printf("Failed to create hashmap '%s'!\n", strerror(errno));164 exit(1);165 }166 167 for (i = 0; i < nr_cpus; i++)168 bpf_percpu(value, i) = i + 100;169 170 key = 1;171 /* Insert key=1 element. */172 assert(!(expected_key_mask & key));173 assert(bpf_map_update_elem(fd, &key, value, BPF_ANY) == 0);174 175 /* Lookup and delete elem key=1 and check value. */176 assert(bpf_map_lookup_and_delete_elem(fd, &key, value) == 0 &&177 bpf_percpu(value,0) == 100);178 179 for (i = 0; i < nr_cpus; i++)180 bpf_percpu(value,i) = i + 100;181 182 /* Insert key=1 element which should not exist. */183 assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) == 0);184 expected_key_mask |= key;185 186 /* BPF_NOEXIST means add new element if it doesn't exist. */187 assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) < 0 &&188 /* key=1 already exists. */189 errno == EEXIST);190 191 /* -1 is an invalid flag. */192 assert(bpf_map_update_elem(fd, &key, value, -1) < 0 &&193 errno == EINVAL);194 195 /* Check that key=1 can be found. Value could be 0 if the lookup196 * was run from a different CPU.197 */198 bpf_percpu(value, 0) = 1;199 assert(bpf_map_lookup_elem(fd, &key, value) == 0 &&200 bpf_percpu(value, 0) == 100);201 202 key = 2;203 /* Check that key=2 is not found. */204 assert(bpf_map_lookup_elem(fd, &key, value) < 0 && errno == ENOENT);205 206 /* BPF_EXIST means update existing element. */207 assert(bpf_map_update_elem(fd, &key, value, BPF_EXIST) < 0 &&208 /* key=2 is not there. */209 errno == ENOENT);210 211 /* Insert key=2 element. */212 assert(!(expected_key_mask & key));213 assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) == 0);214 expected_key_mask |= key;215 216 /* key=1 and key=2 were inserted, check that key=0 cannot be217 * inserted due to max_entries limit.218 */219 key = 0;220 assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) < 0 &&221 errno == E2BIG);222 223 /* Check that key = 0 doesn't exist. */224 assert(bpf_map_delete_elem(fd, &key) < 0 && errno == ENOENT);225 226 /* Iterate over two elements. */227 assert(bpf_map_get_next_key(fd, NULL, &first_key) == 0 &&228 ((expected_key_mask & first_key) == first_key));229 while (!bpf_map_get_next_key(fd, &key, &next_key)) {230 if (first_key) {231 assert(next_key == first_key);232 first_key = 0;233 }234 assert((expected_key_mask & next_key) == next_key);235 expected_key_mask &= ~next_key;236 237 assert(bpf_map_lookup_elem(fd, &next_key, value) == 0);238 239 for (i = 0; i < nr_cpus; i++)240 assert(bpf_percpu(value, i) == i + 100);241 242 key = next_key;243 }244 assert(errno == ENOENT);245 246 /* Update with BPF_EXIST. */247 key = 1;248 assert(bpf_map_update_elem(fd, &key, value, BPF_EXIST) == 0);249 250 /* Delete both elements. */251 key = 1;252 assert(bpf_map_delete_elem(fd, &key) == 0);253 key = 2;254 assert(bpf_map_delete_elem(fd, &key) == 0);255 assert(bpf_map_delete_elem(fd, &key) < 0 && errno == ENOENT);256 257 key = 0;258 /* Check that map is empty. */259 assert(bpf_map_get_next_key(fd, NULL, &next_key) < 0 &&260 errno == ENOENT);261 assert(bpf_map_get_next_key(fd, &key, &next_key) < 0 &&262 errno == ENOENT);263 264 close(fd);265}266 267#define VALUE_SIZE 3268static int helper_fill_hashmap(int max_entries)269{270 int i, fd, ret;271 long long key, value[VALUE_SIZE] = {};272 273 fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, sizeof(key), sizeof(value),274 max_entries, &map_opts);275 CHECK(fd < 0,276 "failed to create hashmap",277 "err: %s, flags: 0x%x\n", strerror(errno), map_opts.map_flags);278 279 for (i = 0; i < max_entries; i++) {280 key = i; value[0] = key;281 ret = bpf_map_update_elem(fd, &key, value, BPF_NOEXIST);282 CHECK(ret != 0,283 "can't update hashmap",284 "err: %s\n", strerror(ret));285 }286 287 return fd;288}289 290static void test_hashmap_walk(unsigned int task, void *data)291{292 int fd, i, max_entries = 10000;293 long long key, value[VALUE_SIZE], next_key;294 bool next_key_valid = true;295 296 fd = helper_fill_hashmap(max_entries);297 298 for (i = 0; bpf_map_get_next_key(fd, !i ? NULL : &key,299 &next_key) == 0; i++) {300 key = next_key;301 assert(bpf_map_lookup_elem(fd, &key, value) == 0);302 }303 304 assert(i == max_entries);305 306 assert(bpf_map_get_next_key(fd, NULL, &key) == 0);307 for (i = 0; next_key_valid; i++) {308 next_key_valid = bpf_map_get_next_key(fd, &key, &next_key) == 0;309 assert(bpf_map_lookup_elem(fd, &key, value) == 0);310 value[0]++;311 assert(bpf_map_update_elem(fd, &key, value, BPF_EXIST) == 0);312 key = next_key;313 }314 315 assert(i == max_entries);316 317 for (i = 0; bpf_map_get_next_key(fd, !i ? NULL : &key,318 &next_key) == 0; i++) {319 key = next_key;320 assert(bpf_map_lookup_elem(fd, &key, value) == 0);321 assert(value[0] - 1 == key);322 }323 324 assert(i == max_entries);325 close(fd);326}327 328static void test_hashmap_zero_seed(void)329{330 int i, first, second, old_flags;331 long long key, next_first, next_second;332 333 old_flags = map_opts.map_flags;334 map_opts.map_flags |= BPF_F_ZERO_SEED;335 336 first = helper_fill_hashmap(3);337 second = helper_fill_hashmap(3);338 339 for (i = 0; ; i++) {340 void *key_ptr = !i ? NULL : &key;341 342 if (bpf_map_get_next_key(first, key_ptr, &next_first) != 0)343 break;344 345 CHECK(bpf_map_get_next_key(second, key_ptr, &next_second) != 0,346 "next_key for second map must succeed",347 "key_ptr: %p", key_ptr);348 CHECK(next_first != next_second,349 "keys must match",350 "i: %d first: %lld second: %lld\n", i,351 next_first, next_second);352 353 key = next_first;354 }355 356 map_opts.map_flags = old_flags;357 close(first);358 close(second);359}360 361static void test_arraymap(unsigned int task, void *data)362{363 int key, next_key, fd;364 long long value;365 366 fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, sizeof(key), sizeof(value), 2, NULL);367 if (fd < 0) {368 printf("Failed to create arraymap '%s'!\n", strerror(errno));369 exit(1);370 }371 372 key = 1;373 value = 1234;374 /* Insert key=1 element. */375 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0);376 377 value = 0;378 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) < 0 &&379 errno == EEXIST);380 381 /* Check that key=1 can be found. */382 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 1234);383 384 key = 0;385 /* Check that key=0 is also found and zero initialized. */386 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 0);387 388 /* key=0 and key=1 were inserted, check that key=2 cannot be inserted389 * due to max_entries limit.390 */391 key = 2;392 assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) < 0 &&393 errno == E2BIG);394 395 /* Check that key = 2 doesn't exist. */396 assert(bpf_map_lookup_elem(fd, &key, &value) < 0 && errno == ENOENT);397 398 /* Iterate over two elements. */399 assert(bpf_map_get_next_key(fd, NULL, &next_key) == 0 &&400 next_key == 0);401 assert(bpf_map_get_next_key(fd, &key, &next_key) == 0 &&402 next_key == 0);403 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == 0 &&404 next_key == 1);405 assert(bpf_map_get_next_key(fd, &next_key, &next_key) < 0 &&406 errno == ENOENT);407 408 /* Delete shouldn't succeed. */409 key = 1;410 assert(bpf_map_delete_elem(fd, &key) < 0 && errno == EINVAL);411 412 close(fd);413}414 415static void test_arraymap_percpu(unsigned int task, void *data)416{417 unsigned int nr_cpus = bpf_num_possible_cpus();418 BPF_DECLARE_PERCPU(long, values);419 int key, next_key, fd, i;420 421 fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, NULL, sizeof(key),422 sizeof(bpf_percpu(values, 0)), 2, NULL);423 if (fd < 0) {424 printf("Failed to create arraymap '%s'!\n", strerror(errno));425 exit(1);426 }427 428 for (i = 0; i < nr_cpus; i++)429 bpf_percpu(values, i) = i + 100;430 431 key = 1;432 /* Insert key=1 element. */433 assert(bpf_map_update_elem(fd, &key, values, BPF_ANY) == 0);434 435 bpf_percpu(values, 0) = 0;436 assert(bpf_map_update_elem(fd, &key, values, BPF_NOEXIST) < 0 &&437 errno == EEXIST);438 439 /* Check that key=1 can be found. */440 assert(bpf_map_lookup_elem(fd, &key, values) == 0 &&441 bpf_percpu(values, 0) == 100);442 443 key = 0;444 /* Check that key=0 is also found and zero initialized. */445 assert(bpf_map_lookup_elem(fd, &key, values) == 0 &&446 bpf_percpu(values, 0) == 0 &&447 bpf_percpu(values, nr_cpus - 1) == 0);448 449 /* Check that key=2 cannot be inserted due to max_entries limit. */450 key = 2;451 assert(bpf_map_update_elem(fd, &key, values, BPF_EXIST) < 0 &&452 errno == E2BIG);453 454 /* Check that key = 2 doesn't exist. */455 assert(bpf_map_lookup_elem(fd, &key, values) < 0 && errno == ENOENT);456 457 /* Iterate over two elements. */458 assert(bpf_map_get_next_key(fd, NULL, &next_key) == 0 &&459 next_key == 0);460 assert(bpf_map_get_next_key(fd, &key, &next_key) == 0 &&461 next_key == 0);462 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == 0 &&463 next_key == 1);464 assert(bpf_map_get_next_key(fd, &next_key, &next_key) < 0 &&465 errno == ENOENT);466 467 /* Delete shouldn't succeed. */468 key = 1;469 assert(bpf_map_delete_elem(fd, &key) < 0 && errno == EINVAL);470 471 close(fd);472}473 474static void test_arraymap_percpu_many_keys(void)475{476 unsigned int nr_cpus = bpf_num_possible_cpus();477 BPF_DECLARE_PERCPU(long, values);478 /* nr_keys is not too large otherwise the test stresses percpu479 * allocator more than anything else480 */481 unsigned int nr_keys = 2000;482 int key, fd, i;483 484 fd = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, NULL, sizeof(key),485 sizeof(bpf_percpu(values, 0)), nr_keys, NULL);486 if (fd < 0) {487 printf("Failed to create per-cpu arraymap '%s'!\n",488 strerror(errno));489 exit(1);490 }491 492 for (i = 0; i < nr_cpus; i++)493 bpf_percpu(values, i) = i + 10;494 495 for (key = 0; key < nr_keys; key++)496 assert(bpf_map_update_elem(fd, &key, values, BPF_ANY) == 0);497 498 for (key = 0; key < nr_keys; key++) {499 for (i = 0; i < nr_cpus; i++)500 bpf_percpu(values, i) = 0;501 502 assert(bpf_map_lookup_elem(fd, &key, values) == 0);503 504 for (i = 0; i < nr_cpus; i++)505 assert(bpf_percpu(values, i) == i + 10);506 }507 508 close(fd);509}510 511static void test_devmap(unsigned int task, void *data)512{513 int fd;514 __u32 key, value;515 516 fd = bpf_map_create(BPF_MAP_TYPE_DEVMAP, NULL, sizeof(key), sizeof(value), 2, NULL);517 if (fd < 0) {518 printf("Failed to create devmap '%s'!\n", strerror(errno));519 exit(1);520 }521 522 close(fd);523}524 525static void test_devmap_hash(unsigned int task, void *data)526{527 int fd;528 __u32 key, value;529 530 fd = bpf_map_create(BPF_MAP_TYPE_DEVMAP_HASH, NULL, sizeof(key), sizeof(value), 2, NULL);531 if (fd < 0) {532 printf("Failed to create devmap_hash '%s'!\n", strerror(errno));533 exit(1);534 }535 536 close(fd);537}538 539static void test_queuemap(unsigned int task, void *data)540{541 const int MAP_SIZE = 32;542 __u32 vals[MAP_SIZE + MAP_SIZE/2], val;543 int fd, i;544 545 /* Fill test values to be used */546 for (i = 0; i < MAP_SIZE + MAP_SIZE/2; i++)547 vals[i] = rand();548 549 /* Invalid key size */550 fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 4, sizeof(val), MAP_SIZE, &map_opts);551 assert(fd < 0 && errno == EINVAL);552 553 fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0, sizeof(val), MAP_SIZE, &map_opts);554 /* Queue map does not support BPF_F_NO_PREALLOC */555 if (map_opts.map_flags & BPF_F_NO_PREALLOC) {556 assert(fd < 0 && errno == EINVAL);557 return;558 }559 if (fd < 0) {560 printf("Failed to create queuemap '%s'!\n", strerror(errno));561 exit(1);562 }563 564 /* Push MAP_SIZE elements */565 for (i = 0; i < MAP_SIZE; i++)566 assert(bpf_map_update_elem(fd, NULL, &vals[i], 0) == 0);567 568 /* Check that element cannot be pushed due to max_entries limit */569 assert(bpf_map_update_elem(fd, NULL, &val, 0) < 0 &&570 errno == E2BIG);571 572 /* Peek element */573 assert(bpf_map_lookup_elem(fd, NULL, &val) == 0 && val == vals[0]);574 575 /* Replace half elements */576 for (i = MAP_SIZE; i < MAP_SIZE + MAP_SIZE/2; i++)577 assert(bpf_map_update_elem(fd, NULL, &vals[i], BPF_EXIST) == 0);578 579 /* Pop all elements */580 for (i = MAP_SIZE/2; i < MAP_SIZE + MAP_SIZE/2; i++)581 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) == 0 &&582 val == vals[i]);583 584 /* Check that there are not elements left */585 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) < 0 &&586 errno == ENOENT);587 588 /* Check that non supported functions set errno to EINVAL */589 assert(bpf_map_delete_elem(fd, NULL) < 0 && errno == EINVAL);590 assert(bpf_map_get_next_key(fd, NULL, NULL) < 0 && errno == EINVAL);591 592 close(fd);593}594 595static void test_stackmap(unsigned int task, void *data)596{597 const int MAP_SIZE = 32;598 __u32 vals[MAP_SIZE + MAP_SIZE/2], val;599 int fd, i;600 601 /* Fill test values to be used */602 for (i = 0; i < MAP_SIZE + MAP_SIZE/2; i++)603 vals[i] = rand();604 605 /* Invalid key size */606 fd = bpf_map_create(BPF_MAP_TYPE_STACK, NULL, 4, sizeof(val), MAP_SIZE, &map_opts);607 assert(fd < 0 && errno == EINVAL);608 609 fd = bpf_map_create(BPF_MAP_TYPE_STACK, NULL, 0, sizeof(val), MAP_SIZE, &map_opts);610 /* Stack map does not support BPF_F_NO_PREALLOC */611 if (map_opts.map_flags & BPF_F_NO_PREALLOC) {612 assert(fd < 0 && errno == EINVAL);613 return;614 }615 if (fd < 0) {616 printf("Failed to create stackmap '%s'!\n", strerror(errno));617 exit(1);618 }619 620 /* Push MAP_SIZE elements */621 for (i = 0; i < MAP_SIZE; i++)622 assert(bpf_map_update_elem(fd, NULL, &vals[i], 0) == 0);623 624 /* Check that element cannot be pushed due to max_entries limit */625 assert(bpf_map_update_elem(fd, NULL, &val, 0) < 0 &&626 errno == E2BIG);627 628 /* Peek element */629 assert(bpf_map_lookup_elem(fd, NULL, &val) == 0 && val == vals[i - 1]);630 631 /* Replace half elements */632 for (i = MAP_SIZE; i < MAP_SIZE + MAP_SIZE/2; i++)633 assert(bpf_map_update_elem(fd, NULL, &vals[i], BPF_EXIST) == 0);634 635 /* Pop all elements */636 for (i = MAP_SIZE + MAP_SIZE/2 - 1; i >= MAP_SIZE/2; i--)637 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) == 0 &&638 val == vals[i]);639 640 /* Check that there are not elements left */641 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) < 0 &&642 errno == ENOENT);643 644 /* Check that non supported functions set errno to EINVAL */645 assert(bpf_map_delete_elem(fd, NULL) < 0 && errno == EINVAL);646 assert(bpf_map_get_next_key(fd, NULL, NULL) < 0 && errno == EINVAL);647 648 close(fd);649}650 651#include <sys/ioctl.h>652#include <arpa/inet.h>653#include <sys/select.h>654#include <linux/err.h>655#define SOCKMAP_PARSE_PROG "./sockmap_parse_prog.bpf.o"656#define SOCKMAP_VERDICT_PROG "./sockmap_verdict_prog.bpf.o"657#define SOCKMAP_TCP_MSG_PROG "./sockmap_tcp_msg_prog.bpf.o"658static void test_sockmap(unsigned int tasks, void *data)659{660 struct bpf_map *bpf_map_rx, *bpf_map_tx, *bpf_map_msg, *bpf_map_break;661 int map_fd_msg = 0, map_fd_rx = 0, map_fd_tx = 0, map_fd_break;662 struct bpf_object *parse_obj, *verdict_obj, *msg_obj;663 int ports[] = {50200, 50201, 50202, 50204};664 int err, i, fd, udp, sfd[6] = {0xdeadbeef};665 u8 buf[20] = {0x0, 0x5, 0x3, 0x2, 0x1, 0x0};666 int parse_prog, verdict_prog, msg_prog;667 struct sockaddr_in addr;668 int one = 1, s, sc, rc;669 struct timeval to;670 __u32 key, value;671 pid_t pid[tasks];672 fd_set w;673 674 /* Create some sockets to use with sockmap */675 for (i = 0; i < 2; i++) {676 sfd[i] = socket(AF_INET, SOCK_STREAM, 0);677 if (sfd[i] < 0)678 goto out;679 err = setsockopt(sfd[i], SOL_SOCKET, SO_REUSEADDR,680 (char *)&one, sizeof(one));681 if (err) {682 printf("failed to setsockopt\n");683 goto out;684 }685 err = ioctl(sfd[i], FIONBIO, (char *)&one);686 if (err < 0) {687 printf("failed to ioctl\n");688 goto out;689 }690 memset(&addr, 0, sizeof(struct sockaddr_in));691 addr.sin_family = AF_INET;692 addr.sin_addr.s_addr = inet_addr("127.0.0.1");693 addr.sin_port = htons(ports[i]);694 err = bind(sfd[i], (struct sockaddr *)&addr, sizeof(addr));695 if (err < 0) {696 printf("failed to bind: err %i: %i:%i\n",697 err, i, sfd[i]);698 goto out;699 }700 err = listen(sfd[i], 32);701 if (err < 0) {702 printf("failed to listen\n");703 goto out;704 }705 }706 707 for (i = 2; i < 4; i++) {708 sfd[i] = socket(AF_INET, SOCK_STREAM, 0);709 if (sfd[i] < 0)710 goto out;711 err = setsockopt(sfd[i], SOL_SOCKET, SO_REUSEADDR,712 (char *)&one, sizeof(one));713 if (err) {714 printf("set sock opt\n");715 goto out;716 }717 memset(&addr, 0, sizeof(struct sockaddr_in));718 addr.sin_family = AF_INET;719 addr.sin_addr.s_addr = inet_addr("127.0.0.1");720 addr.sin_port = htons(ports[i - 2]);721 err = connect(sfd[i], (struct sockaddr *)&addr, sizeof(addr));722 if (err) {723 printf("failed to connect\n");724 goto out;725 }726 }727 728 729 for (i = 4; i < 6; i++) {730 sfd[i] = accept(sfd[i - 4], NULL, NULL);731 if (sfd[i] < 0) {732 printf("accept failed\n");733 goto out;734 }735 }736 737 /* Test sockmap with connected sockets */738 fd = bpf_map_create(BPF_MAP_TYPE_SOCKMAP, NULL,739 sizeof(key), sizeof(value),740 6, NULL);741 if (fd < 0) {742 if (!libbpf_probe_bpf_map_type(BPF_MAP_TYPE_SOCKMAP, NULL)) {743 printf("%s SKIP (unsupported map type BPF_MAP_TYPE_SOCKMAP)\n",744 __func__);745 skips++;746 for (i = 0; i < 6; i++)747 close(sfd[i]);748 return;749 }750 751 printf("Failed to create sockmap %i\n", fd);752 goto out_sockmap;753 }754 755 /* Test update with unsupported UDP socket */756 udp = socket(AF_INET, SOCK_DGRAM, 0);757 i = 0;758 err = bpf_map_update_elem(fd, &i, &udp, BPF_ANY);759 if (err) {760 printf("Failed socket update SOCK_DGRAM '%i:%i'\n",761 i, udp);762 goto out_sockmap;763 }764 close(udp);765 766 /* Test update without programs */767 for (i = 0; i < 6; i++) {768 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_ANY);769 if (err) {770 printf("Failed noprog update sockmap '%i:%i'\n",771 i, sfd[i]);772 goto out_sockmap;773 }774 }775 776 /* Test attaching/detaching bad fds */777 err = bpf_prog_attach(-1, fd, BPF_SK_SKB_STREAM_PARSER, 0);778 if (!err) {779 printf("Failed invalid parser prog attach\n");780 goto out_sockmap;781 }782 783 err = bpf_prog_attach(-1, fd, BPF_SK_SKB_STREAM_VERDICT, 0);784 if (!err) {785 printf("Failed invalid verdict prog attach\n");786 goto out_sockmap;787 }788 789 err = bpf_prog_attach(-1, fd, BPF_SK_MSG_VERDICT, 0);790 if (!err) {791 printf("Failed invalid msg verdict prog attach\n");792 goto out_sockmap;793 }794 795 err = bpf_prog_attach(-1, fd, __MAX_BPF_ATTACH_TYPE, 0);796 if (!err) {797 printf("Failed unknown prog attach\n");798 goto out_sockmap;799 }800 801 err = bpf_prog_detach(fd, BPF_SK_SKB_STREAM_PARSER);802 if (!err) {803 printf("Failed empty parser prog detach\n");804 goto out_sockmap;805 }806 807 err = bpf_prog_detach(fd, BPF_SK_SKB_STREAM_VERDICT);808 if (!err) {809 printf("Failed empty verdict prog detach\n");810 goto out_sockmap;811 }812 813 err = bpf_prog_detach(fd, BPF_SK_MSG_VERDICT);814 if (!err) {815 printf("Failed empty msg verdict prog detach\n");816 goto out_sockmap;817 }818 819 err = bpf_prog_detach(fd, __MAX_BPF_ATTACH_TYPE);820 if (!err) {821 printf("Detach invalid prog successful\n");822 goto out_sockmap;823 }824 825 /* Load SK_SKB program and Attach */826 err = bpf_prog_test_load(SOCKMAP_PARSE_PROG,827 BPF_PROG_TYPE_SK_SKB, &parse_obj, &parse_prog);828 if (err) {829 printf("Failed to load SK_SKB parse prog\n");830 goto out_sockmap;831 }832 833 err = bpf_prog_test_load(SOCKMAP_TCP_MSG_PROG,834 BPF_PROG_TYPE_SK_MSG, &msg_obj, &msg_prog);835 if (err) {836 printf("Failed to load SK_SKB msg prog\n");837 goto out_sockmap;838 }839 840 err = bpf_prog_test_load(SOCKMAP_VERDICT_PROG,841 BPF_PROG_TYPE_SK_SKB, &verdict_obj, &verdict_prog);842 if (err) {843 printf("Failed to load SK_SKB verdict prog\n");844 goto out_sockmap;845 }846 847 bpf_map_rx = bpf_object__find_map_by_name(verdict_obj, "sock_map_rx");848 if (!bpf_map_rx) {849 printf("Failed to load map rx from verdict prog\n");850 goto out_sockmap;851 }852 853 map_fd_rx = bpf_map__fd(bpf_map_rx);854 if (map_fd_rx < 0) {855 printf("Failed to get map rx fd\n");856 goto out_sockmap;857 }858 859 bpf_map_tx = bpf_object__find_map_by_name(verdict_obj, "sock_map_tx");860 if (!bpf_map_tx) {861 printf("Failed to load map tx from verdict prog\n");862 goto out_sockmap;863 }864 865 map_fd_tx = bpf_map__fd(bpf_map_tx);866 if (map_fd_tx < 0) {867 printf("Failed to get map tx fd\n");868 goto out_sockmap;869 }870 871 bpf_map_msg = bpf_object__find_map_by_name(verdict_obj, "sock_map_msg");872 if (!bpf_map_msg) {873 printf("Failed to load map msg from msg_verdict prog\n");874 goto out_sockmap;875 }876 877 map_fd_msg = bpf_map__fd(bpf_map_msg);878 if (map_fd_msg < 0) {879 printf("Failed to get map msg fd\n");880 goto out_sockmap;881 }882 883 bpf_map_break = bpf_object__find_map_by_name(verdict_obj, "sock_map_break");884 if (!bpf_map_break) {885 printf("Failed to load map tx from verdict prog\n");886 goto out_sockmap;887 }888 889 map_fd_break = bpf_map__fd(bpf_map_break);890 if (map_fd_break < 0) {891 printf("Failed to get map tx fd\n");892 goto out_sockmap;893 }894 895 err = bpf_prog_attach(parse_prog, map_fd_break,896 BPF_SK_SKB_STREAM_PARSER, 0);897 if (!err) {898 printf("Allowed attaching SK_SKB program to invalid map\n");899 goto out_sockmap;900 }901 902 err = bpf_prog_attach(parse_prog, map_fd_rx,903 BPF_SK_SKB_STREAM_PARSER, 0);904 if (err) {905 printf("Failed stream parser bpf prog attach\n");906 goto out_sockmap;907 }908 909 err = bpf_prog_attach(verdict_prog, map_fd_rx,910 BPF_SK_SKB_STREAM_VERDICT, 0);911 if (err) {912 printf("Failed stream verdict bpf prog attach\n");913 goto out_sockmap;914 }915 916 err = bpf_prog_attach(msg_prog, map_fd_msg, BPF_SK_MSG_VERDICT, 0);917 if (err) {918 printf("Failed msg verdict bpf prog attach\n");919 goto out_sockmap;920 }921 922 err = bpf_prog_attach(verdict_prog, map_fd_rx,923 __MAX_BPF_ATTACH_TYPE, 0);924 if (!err) {925 printf("Attached unknown bpf prog\n");926 goto out_sockmap;927 }928 929 /* Test map update elem afterwards fd lives in fd and map_fd */930 for (i = 2; i < 6; i++) {931 err = bpf_map_update_elem(map_fd_rx, &i, &sfd[i], BPF_ANY);932 if (err) {933 printf("Failed map_fd_rx update sockmap %i '%i:%i'\n",934 err, i, sfd[i]);935 goto out_sockmap;936 }937 err = bpf_map_update_elem(map_fd_tx, &i, &sfd[i], BPF_ANY);938 if (err) {939 printf("Failed map_fd_tx update sockmap %i '%i:%i'\n",940 err, i, sfd[i]);941 goto out_sockmap;942 }943 }944 945 /* Test map delete elem and remove send/recv sockets */946 for (i = 2; i < 4; i++) {947 err = bpf_map_delete_elem(map_fd_rx, &i);948 if (err) {949 printf("Failed delete sockmap rx %i '%i:%i'\n",950 err, i, sfd[i]);951 goto out_sockmap;952 }953 err = bpf_map_delete_elem(map_fd_tx, &i);954 if (err) {955 printf("Failed delete sockmap tx %i '%i:%i'\n",956 err, i, sfd[i]);957 goto out_sockmap;958 }959 }960 961 /* Put sfd[2] (sending fd below) into msg map to test sendmsg bpf */962 i = 0;963 err = bpf_map_update_elem(map_fd_msg, &i, &sfd[2], BPF_ANY);964 if (err) {965 printf("Failed map_fd_msg update sockmap %i\n", err);966 goto out_sockmap;967 }968 969 /* Test map send/recv */970 for (i = 0; i < 2; i++) {971 buf[0] = i;972 buf[1] = 0x5;973 sc = send(sfd[2], buf, 20, 0);974 if (sc < 0) {975 printf("Failed sockmap send\n");976 goto out_sockmap;977 }978 979 FD_ZERO(&w);980 FD_SET(sfd[3], &w);981 to.tv_sec = 30;982 to.tv_usec = 0;983 s = select(sfd[3] + 1, &w, NULL, NULL, &to);984 if (s == -1) {985 perror("Failed sockmap select()");986 goto out_sockmap;987 } else if (!s) {988 printf("Failed sockmap unexpected timeout\n");989 goto out_sockmap;990 }991 992 if (!FD_ISSET(sfd[3], &w)) {993 printf("Failed sockmap select/recv\n");994 goto out_sockmap;995 }996 997 rc = recv(sfd[3], buf, sizeof(buf), 0);998 if (rc < 0) {999 printf("Failed sockmap recv\n");1000 goto out_sockmap;1001 }1002 }1003 1004 /* Negative null entry lookup from datapath should be dropped */1005 buf[0] = 1;1006 buf[1] = 12;1007 sc = send(sfd[2], buf, 20, 0);1008 if (sc < 0) {1009 printf("Failed sockmap send\n");1010 goto out_sockmap;1011 }1012 1013 /* Push fd into same slot */1014 i = 2;1015 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_NOEXIST);1016 if (!err) {1017 printf("Failed allowed sockmap dup slot BPF_NOEXIST\n");1018 goto out_sockmap;1019 }1020 1021 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_ANY);1022 if (err) {1023 printf("Failed sockmap update new slot BPF_ANY\n");1024 goto out_sockmap;1025 }1026 1027 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_EXIST);1028 if (err) {1029 printf("Failed sockmap update new slot BPF_EXIST\n");1030 goto out_sockmap;1031 }1032 1033 /* Delete the elems without programs */1034 for (i = 2; i < 6; i++) {1035 err = bpf_map_delete_elem(fd, &i);1036 if (err) {1037 printf("Failed delete sockmap %i '%i:%i'\n",1038 err, i, sfd[i]);1039 }1040 }1041 1042 /* Test having multiple maps open and set with programs on same fds */1043 err = bpf_prog_attach(parse_prog, fd,1044 BPF_SK_SKB_STREAM_PARSER, 0);1045 if (err) {1046 printf("Failed fd bpf parse prog attach\n");1047 goto out_sockmap;1048 }1049 err = bpf_prog_attach(verdict_prog, fd,1050 BPF_SK_SKB_STREAM_VERDICT, 0);1051 if (err) {1052 printf("Failed fd bpf verdict prog attach\n");1053 goto out_sockmap;1054 }1055 1056 for (i = 4; i < 6; i++) {1057 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_ANY);1058 if (!err) {1059 printf("Failed allowed duplicate programs in update ANY sockmap %i '%i:%i'\n",1060 err, i, sfd[i]);1061 goto out_sockmap;1062 }1063 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_NOEXIST);1064 if (!err) {1065 printf("Failed allowed duplicate program in update NOEXIST sockmap %i '%i:%i'\n",1066 err, i, sfd[i]);1067 goto out_sockmap;1068 }1069 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_EXIST);1070 if (!err) {1071 printf("Failed allowed duplicate program in update EXIST sockmap %i '%i:%i'\n",1072 err, i, sfd[i]);1073 goto out_sockmap;1074 }1075 }1076 1077 /* Test tasks number of forked operations */1078 for (i = 0; i < tasks; i++) {1079 pid[i] = fork();1080 if (pid[i] == 0) {1081 for (i = 0; i < 6; i++) {1082 bpf_map_delete_elem(map_fd_tx, &i);1083 bpf_map_delete_elem(map_fd_rx, &i);1084 bpf_map_update_elem(map_fd_tx, &i,1085 &sfd[i], BPF_ANY);1086 bpf_map_update_elem(map_fd_rx, &i,1087 &sfd[i], BPF_ANY);1088 }1089 exit(0);1090 } else if (pid[i] == -1) {1091 printf("Couldn't spawn #%d process!\n", i);1092 exit(1);1093 }1094 }1095 1096 for (i = 0; i < tasks; i++) {1097 int status;1098 1099 assert(waitpid(pid[i], &status, 0) == pid[i]);1100 assert(status == 0);1101 }1102 1103 err = bpf_prog_detach2(parse_prog, map_fd_rx, __MAX_BPF_ATTACH_TYPE);1104 if (!err) {1105 printf("Detached an invalid prog type.\n");1106 goto out_sockmap;1107 }1108 1109 err = bpf_prog_detach2(parse_prog, map_fd_rx, BPF_SK_SKB_STREAM_PARSER);1110 if (err) {1111 printf("Failed parser prog detach\n");1112 goto out_sockmap;1113 }1114 1115 err = bpf_prog_detach2(verdict_prog, map_fd_rx, BPF_SK_SKB_STREAM_VERDICT);1116 if (err) {1117 printf("Failed parser prog detach\n");1118 goto out_sockmap;1119 }1120 1121 /* Test map close sockets and empty maps */1122 for (i = 0; i < 6; i++) {1123 bpf_map_delete_elem(map_fd_tx, &i);1124 bpf_map_delete_elem(map_fd_rx, &i);1125 close(sfd[i]);1126 }1127 close(fd);1128 close(map_fd_rx);1129 bpf_object__close(parse_obj);1130 bpf_object__close(msg_obj);1131 bpf_object__close(verdict_obj);1132 return;1133out:1134 for (i = 0; i < 6; i++)1135 close(sfd[i]);1136 printf("Failed to create sockmap '%i:%s'!\n", i, strerror(errno));1137 exit(1);1138out_sockmap:1139 for (i = 0; i < 6; i++) {1140 if (map_fd_tx)1141 bpf_map_delete_elem(map_fd_tx, &i);1142 if (map_fd_rx)1143 bpf_map_delete_elem(map_fd_rx, &i);1144 close(sfd[i]);1145 }1146 close(fd);1147 exit(1);1148}1149 1150#define MAPINMAP_PROG "./test_map_in_map.bpf.o"1151#define MAPINMAP_INVALID_PROG "./test_map_in_map_invalid.bpf.o"1152static void test_map_in_map(void)1153{1154 struct bpf_object *obj;1155 struct bpf_map *map;1156 int mim_fd, fd, err;1157 int pos = 0;1158 struct bpf_map_info info = {};1159 __u32 len = sizeof(info);1160 __u32 id = 0;1161 libbpf_print_fn_t old_print_fn;1162 1163 obj = bpf_object__open(MAPINMAP_PROG);1164 1165 fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, sizeof(int), sizeof(int), 2, NULL);1166 if (fd < 0) {1167 printf("Failed to create hashmap '%s'!\n", strerror(errno));1168 exit(1);1169 }1170 1171 map = bpf_object__find_map_by_name(obj, "mim_array");1172 if (!map) {1173 printf("Failed to load array of maps from test prog\n");1174 goto out_map_in_map;1175 }1176 err = bpf_map__set_inner_map_fd(map, fd);1177 if (err) {1178 printf("Failed to set inner_map_fd for array of maps\n");1179 goto out_map_in_map;1180 }1181 1182 map = bpf_object__find_map_by_name(obj, "mim_hash");1183 if (!map) {1184 printf("Failed to load hash of maps from test prog\n");1185 goto out_map_in_map;1186 }1187 err = bpf_map__set_inner_map_fd(map, fd);1188 if (err) {1189 printf("Failed to set inner_map_fd for hash of maps\n");1190 goto out_map_in_map;1191 }1192 1193 err = bpf_object__load(obj);1194 if (err) {1195 printf("Failed to load test prog\n");1196 goto out_map_in_map;1197 }1198 1199 map = bpf_object__find_map_by_name(obj, "mim_array");1200 if (!map) {1201 printf("Failed to load array of maps from test prog\n");1202 goto out_map_in_map;1203 }1204 mim_fd = bpf_map__fd(map);1205 if (mim_fd < 0) {1206 printf("Failed to get descriptor for array of maps\n");1207 goto out_map_in_map;1208 }1209 1210 err = bpf_map_update_elem(mim_fd, &pos, &fd, 0);1211 if (err) {1212 printf("Failed to update array of maps\n");1213 goto out_map_in_map;1214 }1215 1216 map = bpf_object__find_map_by_name(obj, "mim_hash");1217 if (!map) {1218 printf("Failed to load hash of maps from test prog\n");1219 goto out_map_in_map;1220 }1221 mim_fd = bpf_map__fd(map);1222 if (mim_fd < 0) {1223 printf("Failed to get descriptor for hash of maps\n");1224 goto out_map_in_map;1225 }1226 1227 err = bpf_map_update_elem(mim_fd, &pos, &fd, 0);1228 if (err) {1229 printf("Failed to update hash of maps\n");1230 goto out_map_in_map;1231 }1232 1233 close(fd);1234 fd = -1;1235 bpf_object__close(obj);1236 1237 /* Test that failing bpf_object__create_map() destroys the inner map */1238 obj = bpf_object__open(MAPINMAP_INVALID_PROG);1239 err = libbpf_get_error(obj);1240 if (err) {1241 printf("Failed to load %s program: %d %d",1242 MAPINMAP_INVALID_PROG, err, errno);1243 goto out_map_in_map;1244 }1245 1246 map = bpf_object__find_map_by_name(obj, "mim");1247 if (!map) {1248 printf("Failed to load array of maps from test prog\n");1249 goto out_map_in_map;1250 }1251 1252 old_print_fn = libbpf_set_print(NULL);1253 1254 err = bpf_object__load(obj);1255 if (!err) {1256 printf("Loading obj supposed to fail\n");1257 goto out_map_in_map;1258 }1259 1260 libbpf_set_print(old_print_fn);1261 1262 /* Iterate over all maps to check whether the internal map1263 * ("mim.internal") has been destroyed.1264 */1265 while (true) {1266 err = bpf_map_get_next_id(id, &id);1267 if (err) {1268 if (errno == ENOENT)1269 break;1270 printf("Failed to get next map: %d", errno);1271 goto out_map_in_map;1272 }1273 1274 fd = bpf_map_get_fd_by_id(id);1275 if (fd < 0) {1276 if (errno == ENOENT)1277 continue;1278 printf("Failed to get map by id %u: %d", id, errno);1279 goto out_map_in_map;1280 }1281 1282 err = bpf_map_get_info_by_fd(fd, &info, &len);1283 if (err) {1284 printf("Failed to get map info by fd %d: %d", fd,1285 errno);1286 goto out_map_in_map;1287 }1288 1289 if (!strcmp(info.name, "mim.inner")) {1290 printf("Inner map mim.inner was not destroyed\n");1291 goto out_map_in_map;1292 }1293 1294 close(fd);1295 }1296 1297 bpf_object__close(obj);1298 return;1299 1300out_map_in_map:1301 if (fd >= 0)1302 close(fd);1303 exit(1);1304}1305 1306#define MAP_SIZE (32 * 1024)1307 1308static void test_map_large(void)1309{1310 1311 struct bigkey {1312 int a;1313 char b[4096];1314 long long c;1315 } key;1316 int fd, i, value;1317 1318 fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, sizeof(key), sizeof(value),1319 MAP_SIZE, &map_opts);1320 if (fd < 0) {1321 printf("Failed to create large map '%s'!\n", strerror(errno));1322 exit(1);1323 }1324 1325 for (i = 0; i < MAP_SIZE; i++) {1326 key = (struct bigkey) { .c = i };1327 value = i;1328 1329 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == 0);1330 }1331 1332 key.c = -1;1333 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) < 0 &&1334 errno == E2BIG);1335 1336 /* Iterate through all elements. */1337 assert(bpf_map_get_next_key(fd, NULL, &key) == 0);1338 key.c = -1;1339 for (i = 0; i < MAP_SIZE; i++)1340 assert(bpf_map_get_next_key(fd, &key, &key) == 0);1341 assert(bpf_map_get_next_key(fd, &key, &key) < 0 && errno == ENOENT);1342 1343 key.c = 0;1344 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 0);1345 key.a = 1;1346 assert(bpf_map_lookup_elem(fd, &key, &value) < 0 && errno == ENOENT);1347 1348 close(fd);1349}1350 1351#define run_parallel(N, FN, DATA) \1352 printf("Fork %u tasks to '" #FN "'\n", N); \1353 __run_parallel(N, FN, DATA)1354 1355static void __run_parallel(unsigned int tasks,1356 void (*fn)(unsigned int task, void *data),1357 void *data)1358{1359 pid_t pid[tasks];1360 int i;1361 1362 fflush(stdout);1363 1364 for (i = 0; i < tasks; i++) {1365 pid[i] = fork();1366 if (pid[i] == 0) {1367 fn(i, data);1368 exit(0);1369 } else if (pid[i] == -1) {1370 printf("Couldn't spawn #%d process!\n", i);1371 exit(1);1372 }1373 }1374 1375 for (i = 0; i < tasks; i++) {1376 int status;1377 1378 assert(waitpid(pid[i], &status, 0) == pid[i]);1379 assert(status == 0);1380 }1381}1382 1383static void test_map_stress(void)1384{1385 run_parallel(100, test_hashmap_walk, NULL);1386 run_parallel(100, test_hashmap, NULL);1387 run_parallel(100, test_hashmap_percpu, NULL);1388 run_parallel(100, test_hashmap_sizes, NULL);1389 1390 run_parallel(100, test_arraymap, NULL);1391 run_parallel(100, test_arraymap_percpu, NULL);1392}1393 1394#define TASKS 1001395 1396#define DO_UPDATE 11397#define DO_DELETE 01398 1399#define MAP_RETRIES 201400#define MAX_DELAY_US 500001401#define MIN_DELAY_RANGE_US 50001402 1403static bool retry_for_again_or_busy(int err)1404{1405 return (err == EAGAIN || err == EBUSY);1406}1407 1408int map_update_retriable(int map_fd, const void *key, const void *value, int flags, int attempts,1409 retry_for_error_fn need_retry)1410{1411 int delay = rand() % MIN_DELAY_RANGE_US;1412 1413 while (bpf_map_update_elem(map_fd, key, value, flags)) {1414 if (!attempts || !need_retry(errno))1415 return -errno;1416 1417 if (delay <= MAX_DELAY_US / 2)1418 delay *= 2;1419 1420 usleep(delay);1421 attempts--;1422 }1423 1424 return 0;1425}1426 1427static int map_delete_retriable(int map_fd, const void *key, int attempts)1428{1429 int delay = rand() % MIN_DELAY_RANGE_US;1430 1431 while (bpf_map_delete_elem(map_fd, key)) {1432 if (!attempts || (errno != EAGAIN && errno != EBUSY))1433 return -errno;1434 1435 if (delay <= MAX_DELAY_US / 2)1436 delay *= 2;1437 1438 usleep(delay);1439 attempts--;1440 }1441 1442 return 0;1443}1444 1445static void test_update_delete(unsigned int fn, void *data)1446{1447 int do_update = ((int *)data)[1];1448 int fd = ((int *)data)[0];1449 int i, key, value, err;1450 1451 if (fn & 1)1452 test_hashmap_walk(fn, NULL);1453 for (i = fn; i < MAP_SIZE; i += TASKS) {1454 key = value = i;1455 1456 if (do_update) {1457 err = map_update_retriable(fd, &key, &value, BPF_NOEXIST, MAP_RETRIES,1458 retry_for_again_or_busy);1459 if (err)1460 printf("error %d %d\n", err, errno);1461 assert(err == 0);1462 err = map_update_retriable(fd, &key, &value, BPF_EXIST, MAP_RETRIES,1463 retry_for_again_or_busy);1464 if (err)1465 printf("error %d %d\n", err, errno);1466 assert(err == 0);1467 } else {1468 err = map_delete_retriable(fd, &key, MAP_RETRIES);1469 if (err)1470 printf("error %d %d\n", err, errno);1471 assert(err == 0);1472 }1473 }1474}1475 1476static void test_map_parallel(void)1477{1478 int i, fd, key = 0, value = 0, j = 0;1479 int data[2];1480 1481 fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, sizeof(key), sizeof(value),1482 MAP_SIZE, &map_opts);1483 if (fd < 0) {1484 printf("Failed to create map for parallel test '%s'!\n",1485 strerror(errno));1486 exit(1);1487 }1488 1489again:1490 /* Use the same fd in children to add elements to this map:1491 * child_0 adds key=0, key=1024, key=2048, ...1492 * child_1 adds key=1, key=1025, key=2049, ...1493 * child_1023 adds key=1023, ...1494 */1495 data[0] = fd;1496 data[1] = DO_UPDATE;1497 run_parallel(TASKS, test_update_delete, data);1498 1499 /* Check that key=0 is already there. */1500 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) < 0 &&1501 errno == EEXIST);1502 1503 /* Check that all elements were inserted. */1504 assert(bpf_map_get_next_key(fd, NULL, &key) == 0);1505 key = -1;1506 for (i = 0; i < MAP_SIZE; i++)1507 assert(bpf_map_get_next_key(fd, &key, &key) == 0);1508 assert(bpf_map_get_next_key(fd, &key, &key) < 0 && errno == ENOENT);1509 1510 /* Another check for all elements */1511 for (i = 0; i < MAP_SIZE; i++) {1512 key = MAP_SIZE - i - 1;1513 1514 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 &&1515 value == key);1516 }1517 1518 /* Now let's delete all elements in parallel. */1519 data[1] = DO_DELETE;1520 run_parallel(TASKS, test_update_delete, data);1521 1522 /* Nothing should be left. */1523 key = -1;1524 assert(bpf_map_get_next_key(fd, NULL, &key) < 0 && errno == ENOENT);1525 assert(bpf_map_get_next_key(fd, &key, &key) < 0 && errno == ENOENT);1526 1527 key = 0;1528 bpf_map_delete_elem(fd, &key);1529 if (j++ < 5)1530 goto again;1531 close(fd);1532}1533 1534static void test_map_rdonly(void)1535{1536 int fd, key = 0, value = 0;1537 __u32 old_flags;1538 1539 old_flags = map_opts.map_flags;1540 map_opts.map_flags |= BPF_F_RDONLY;1541 fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, sizeof(key), sizeof(value),1542 MAP_SIZE, &map_opts);1543 map_opts.map_flags = old_flags;1544 if (fd < 0) {1545 printf("Failed to create map for read only test '%s'!\n",1546 strerror(errno));1547 exit(1);1548 }1549 1550 key = 1;1551 value = 1234;1552 /* Try to insert key=1 element. */1553 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) < 0 &&1554 errno == EPERM);1555 1556 /* Check that key=1 is not found. */1557 assert(bpf_map_lookup_elem(fd, &key, &value) < 0 && errno == ENOENT);1558 assert(bpf_map_get_next_key(fd, &key, &value) < 0 && errno == ENOENT);1559 1560 close(fd);1561}1562 1563static void test_map_wronly_hash(void)1564{1565 int fd, key = 0, value = 0;1566 __u32 old_flags;1567 1568 old_flags = map_opts.map_flags;1569 map_opts.map_flags |= BPF_F_WRONLY;1570 fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, sizeof(key), sizeof(value),1571 MAP_SIZE, &map_opts);1572 map_opts.map_flags = old_flags;1573 if (fd < 0) {1574 printf("Failed to create map for write only test '%s'!\n",1575 strerror(errno));1576 exit(1);1577 }1578 1579 key = 1;1580 value = 1234;1581 /* Insert key=1 element. */1582 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0);1583 1584 /* Check that reading elements and keys from the map is not allowed. */1585 assert(bpf_map_lookup_elem(fd, &key, &value) < 0 && errno == EPERM);1586 assert(bpf_map_get_next_key(fd, &key, &value) < 0 && errno == EPERM);1587 1588 close(fd);1589}1590 1591static void test_map_wronly_stack_or_queue(enum bpf_map_type map_type)1592{1593 int fd, value = 0;1594 __u32 old_flags;1595 1596 1597 assert(map_type == BPF_MAP_TYPE_QUEUE ||1598 map_type == BPF_MAP_TYPE_STACK);1599 old_flags = map_opts.map_flags;1600 map_opts.map_flags |= BPF_F_WRONLY;1601 fd = bpf_map_create(map_type, NULL, 0, sizeof(value), MAP_SIZE, &map_opts);1602 map_opts.map_flags = old_flags;1603 /* Stack/Queue maps do not support BPF_F_NO_PREALLOC */1604 if (map_opts.map_flags & BPF_F_NO_PREALLOC) {1605 assert(fd < 0 && errno == EINVAL);1606 return;1607 }1608 if (fd < 0) {1609 printf("Failed to create map '%s'!\n", strerror(errno));1610 exit(1);1611 }1612 1613 value = 1234;1614 assert(bpf_map_update_elem(fd, NULL, &value, BPF_ANY) == 0);1615 1616 /* Peek element should fail */1617 assert(bpf_map_lookup_elem(fd, NULL, &value) < 0 && errno == EPERM);1618 1619 /* Pop element should fail */1620 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &value) < 0 &&1621 errno == EPERM);1622 1623 close(fd);1624}1625 1626static void test_map_wronly(void)1627{1628 test_map_wronly_hash();1629 test_map_wronly_stack_or_queue(BPF_MAP_TYPE_STACK);1630 test_map_wronly_stack_or_queue(BPF_MAP_TYPE_QUEUE);1631}1632 1633static void prepare_reuseport_grp(int type, int map_fd, size_t map_elem_size,1634 __s64 *fds64, __u64 *sk_cookies,1635 unsigned int n)1636{1637 socklen_t optlen, addrlen;1638 struct sockaddr_in6 s6;1639 const __u32 index0 = 0;1640 const int optval = 1;1641 unsigned int i;1642 u64 sk_cookie;1643 void *value;1644 __s32 fd32;1645 __s64 fd64;1646 int err;1647 1648 s6.sin6_family = AF_INET6;1649 s6.sin6_addr = in6addr_any;1650 s6.sin6_port = 0;1651 addrlen = sizeof(s6);1652 optlen = sizeof(sk_cookie);1653 1654 for (i = 0; i < n; i++) {1655 fd64 = socket(AF_INET6, type, 0);1656 CHECK(fd64 == -1, "socket()",1657 "sock_type:%d fd64:%lld errno:%d\n",1658 type, fd64, errno);1659 1660 err = setsockopt(fd64, SOL_SOCKET, SO_REUSEPORT,1661 &optval, sizeof(optval));1662 CHECK(err == -1, "setsockopt(SO_REUSEPORT)",1663 "err:%d errno:%d\n", err, errno);1664 1665 /* reuseport_array does not allow unbound sk */1666 if (map_elem_size == sizeof(__u64))1667 value = &fd64;1668 else {1669 assert(map_elem_size == sizeof(__u32));1670 fd32 = (__s32)fd64;1671 value = &fd32;1672 }1673 err = bpf_map_update_elem(map_fd, &index0, value, BPF_ANY);1674 CHECK(err >= 0 || errno != EINVAL,1675 "reuseport array update unbound sk",1676 "sock_type:%d err:%d errno:%d\n",1677 type, err, errno);1678 1679 err = bind(fd64, (struct sockaddr *)&s6, sizeof(s6));1680 CHECK(err == -1, "bind()",1681 "sock_type:%d err:%d errno:%d\n", type, err, errno);1682 1683 if (i == 0) {1684 err = getsockname(fd64, (struct sockaddr *)&s6,1685 &addrlen);1686 CHECK(err == -1, "getsockname()",1687 "sock_type:%d err:%d errno:%d\n",1688 type, err, errno);1689 }1690 1691 err = getsockopt(fd64, SOL_SOCKET, SO_COOKIE, &sk_cookie,1692 &optlen);1693 CHECK(err == -1, "getsockopt(SO_COOKIE)",1694 "sock_type:%d err:%d errno:%d\n", type, err, errno);1695 1696 if (type == SOCK_STREAM) {1697 /*1698 * reuseport_array does not allow1699 * non-listening tcp sk.1700 */1701 err = bpf_map_update_elem(map_fd, &index0, value,1702 BPF_ANY);1703 CHECK(err >= 0 || errno != EINVAL,1704 "reuseport array update non-listening sk",1705 "sock_type:%d err:%d errno:%d\n",1706 type, err, errno);1707 err = listen(fd64, 0);1708 CHECK(err == -1, "listen()",1709 "sock_type:%d, err:%d errno:%d\n",1710 type, err, errno);1711 }1712 1713 fds64[i] = fd64;1714 sk_cookies[i] = sk_cookie;1715 }1716}1717 1718static void test_reuseport_array(void)1719{1720#define REUSEPORT_FD_IDX(err, last) ({ (err) ? last : !last; })1721 1722 const __u32 array_size = 4, index0 = 0, index3 = 3;1723 int types[2] = { SOCK_STREAM, SOCK_DGRAM }, type;1724 __u64 grpa_cookies[2], sk_cookie, map_cookie;1725 __s64 grpa_fds64[2] = { -1, -1 }, fd64 = -1;1726 const __u32 bad_index = array_size;1727 int map_fd, err, t, f;1728 __u32 fds_idx = 0;1729 int fd;1730 1731 map_fd = bpf_map_create(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, NULL,1732 sizeof(__u32), sizeof(__u64), array_size, NULL);1733 CHECK(map_fd < 0, "reuseport array create",1734 "map_fd:%d, errno:%d\n", map_fd, errno);1735 1736 /* Test lookup/update/delete with invalid index */1737 err = bpf_map_delete_elem(map_fd, &bad_index);1738 CHECK(err >= 0 || errno != E2BIG, "reuseport array del >=max_entries",1739 "err:%d errno:%d\n", err, errno);1740 1741 err = bpf_map_update_elem(map_fd, &bad_index, &fd64, BPF_ANY);1742 CHECK(err >= 0 || errno != E2BIG,1743 "reuseport array update >=max_entries",1744 "err:%d errno:%d\n", err, errno);1745 1746 err = bpf_map_lookup_elem(map_fd, &bad_index, &map_cookie);1747 CHECK(err >= 0 || errno != ENOENT,1748 "reuseport array update >=max_entries",1749 "err:%d errno:%d\n", err, errno);1750 1751 /* Test lookup/delete non existence elem */1752 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie);1753 CHECK(err >= 0 || errno != ENOENT,1754 "reuseport array lookup not-exist elem",1755 "err:%d errno:%d\n", err, errno);1756 err = bpf_map_delete_elem(map_fd, &index3);1757 CHECK(err >= 0 || errno != ENOENT,1758 "reuseport array del not-exist elem",1759 "err:%d errno:%d\n", err, errno);1760 1761 for (t = 0; t < ARRAY_SIZE(types); t++) {1762 type = types[t];1763 1764 prepare_reuseport_grp(type, map_fd, sizeof(__u64), grpa_fds64,1765 grpa_cookies, ARRAY_SIZE(grpa_fds64));1766 1767 /* Test BPF_* update flags */1768 /* BPF_EXIST failure case */1769 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx],1770 BPF_EXIST);1771 CHECK(err >= 0 || errno != ENOENT,1772 "reuseport array update empty elem BPF_EXIST",1773 "sock_type:%d err:%d errno:%d\n",1774 type, err, errno);1775 fds_idx = REUSEPORT_FD_IDX(err, fds_idx);1776 1777 /* BPF_NOEXIST success case */1778 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx],1779 BPF_NOEXIST);1780 CHECK(err < 0,1781 "reuseport array update empty elem BPF_NOEXIST",1782 "sock_type:%d err:%d errno:%d\n",1783 type, err, errno);1784 fds_idx = REUSEPORT_FD_IDX(err, fds_idx);1785 1786 /* BPF_EXIST success case. */1787 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx],1788 BPF_EXIST);1789 CHECK(err < 0,1790 "reuseport array update same elem BPF_EXIST",1791 "sock_type:%d err:%d errno:%d\n", type, err, errno);1792 fds_idx = REUSEPORT_FD_IDX(err, fds_idx);1793 1794 /* BPF_NOEXIST failure case */1795 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx],1796 BPF_NOEXIST);1797 CHECK(err >= 0 || errno != EEXIST,1798 "reuseport array update non-empty elem BPF_NOEXIST",1799 "sock_type:%d err:%d errno:%d\n",1800 type, err, errno);1801 fds_idx = REUSEPORT_FD_IDX(err, fds_idx);1802 1803 /* BPF_ANY case (always succeed) */1804 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx],1805 BPF_ANY);1806 CHECK(err < 0,1807 "reuseport array update same sk with BPF_ANY",1808 "sock_type:%d err:%d errno:%d\n", type, err, errno);1809 1810 fd64 = grpa_fds64[fds_idx];1811 sk_cookie = grpa_cookies[fds_idx];1812 1813 /* The same sk cannot be added to reuseport_array twice */1814 err = bpf_map_update_elem(map_fd, &index3, &fd64, BPF_ANY);1815 CHECK(err >= 0 || errno != EBUSY,1816 "reuseport array update same sk with same index",1817 "sock_type:%d err:%d errno:%d\n",1818 type, err, errno);1819 1820 err = bpf_map_update_elem(map_fd, &index0, &fd64, BPF_ANY);1821 CHECK(err >= 0 || errno != EBUSY,1822 "reuseport array update same sk with different index",1823 "sock_type:%d err:%d errno:%d\n",1824 type, err, errno);1825 1826 /* Test delete elem */1827 err = bpf_map_delete_elem(map_fd, &index3);1828 CHECK(err < 0, "reuseport array delete sk",1829 "sock_type:%d err:%d errno:%d\n",1830 type, err, errno);1831 1832 /* Add it back with BPF_NOEXIST */1833 err = bpf_map_update_elem(map_fd, &index3, &fd64, BPF_NOEXIST);1834 CHECK(err < 0,1835 "reuseport array re-add with BPF_NOEXIST after del",1836 "sock_type:%d err:%d errno:%d\n", type, err, errno);1837 1838 /* Test cookie */1839 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie);1840 CHECK(err < 0 || sk_cookie != map_cookie,1841 "reuseport array lookup re-added sk",1842 "sock_type:%d err:%d errno:%d sk_cookie:0x%llx map_cookie:0x%llxn",1843 type, err, errno, sk_cookie, map_cookie);1844 1845 /* Test elem removed by close() */1846 for (f = 0; f < ARRAY_SIZE(grpa_fds64); f++)1847 close(grpa_fds64[f]);1848 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie);1849 CHECK(err >= 0 || errno != ENOENT,1850 "reuseport array lookup after close()",1851 "sock_type:%d err:%d errno:%d\n",1852 type, err, errno);1853 }1854 1855 /* Test SOCK_RAW */1856 fd64 = socket(AF_INET6, SOCK_RAW, IPPROTO_UDP);1857 CHECK(fd64 == -1, "socket(SOCK_RAW)", "err:%d errno:%d\n",1858 err, errno);1859 err = bpf_map_update_elem(map_fd, &index3, &fd64, BPF_NOEXIST);1860 CHECK(err >= 0 || errno != ENOTSUPP, "reuseport array update SOCK_RAW",1861 "err:%d errno:%d\n", err, errno);1862 close(fd64);1863 1864 /* Close the 64 bit value map */1865 close(map_fd);1866 1867 /* Test 32 bit fd */1868 map_fd = bpf_map_create(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY, NULL,1869 sizeof(__u32), sizeof(__u32), array_size, NULL);1870 CHECK(map_fd < 0, "reuseport array create",1871 "map_fd:%d, errno:%d\n", map_fd, errno);1872 prepare_reuseport_grp(SOCK_STREAM, map_fd, sizeof(__u32), &fd64,1873 &sk_cookie, 1);1874 fd = fd64;1875 err = bpf_map_update_elem(map_fd, &index3, &fd, BPF_NOEXIST);1876 CHECK(err < 0, "reuseport array update 32 bit fd",1877 "err:%d errno:%d\n", err, errno);1878 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie);1879 CHECK(err >= 0 || errno != ENOSPC,1880 "reuseport array lookup 32 bit fd",1881 "err:%d errno:%d\n", err, errno);1882 close(fd);1883 close(map_fd);1884}1885 1886static void run_all_tests(void)1887{1888 test_hashmap(0, NULL);1889 test_hashmap_percpu(0, NULL);1890 test_hashmap_walk(0, NULL);1891 test_hashmap_zero_seed();1892 1893 test_arraymap(0, NULL);1894 test_arraymap_percpu(0, NULL);1895 1896 test_arraymap_percpu_many_keys();1897 1898 test_devmap(0, NULL);1899 test_devmap_hash(0, NULL);1900 test_sockmap(0, NULL);1901 1902 test_map_large();1903 test_map_parallel();1904 test_map_stress();1905 1906 test_map_rdonly();1907 test_map_wronly();1908 1909 test_reuseport_array();1910 1911 test_queuemap(0, NULL);1912 test_stackmap(0, NULL);1913 1914 test_map_in_map();1915}1916 1917#define DEFINE_TEST(name) extern void test_##name(void);1918#include <map_tests/tests.h>1919#undef DEFINE_TEST1920 1921int main(void)1922{1923 srand(time(NULL));1924 1925 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);1926 1927 map_opts.map_flags = 0;1928 run_all_tests();1929 1930 map_opts.map_flags = BPF_F_NO_PREALLOC;1931 run_all_tests();1932 1933#define DEFINE_TEST(name) test_##name();1934#include <map_tests/tests.h>1935#undef DEFINE_TEST1936 1937 printf("test_maps: OK, %d SKIPPED\n", skips);1938 return 0;1939}1940