286 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (C) 2023. Huawei Technologies Co., Ltd */3#include <vmlinux.h>4#include <bpf/bpf_tracing.h>5#include <bpf/bpf_helpers.h>6 7#include "bpf_experimental.h"8#include "bpf_misc.h"9 10struct generic_map_value {11 void *data;12};13 14char _license[] SEC("license") = "GPL";15 16const unsigned int data_sizes[] = {16, 32, 64, 96, 128, 192, 256, 512, 1024, 2048, 4096};17const volatile unsigned int data_btf_ids[ARRAY_SIZE(data_sizes)] = {};18 19const unsigned int percpu_data_sizes[] = {8, 16, 32, 64, 96, 128, 192, 256, 512};20const volatile unsigned int percpu_data_btf_ids[ARRAY_SIZE(data_sizes)] = {};21 22int err = 0;23u32 pid = 0;24 25#define DEFINE_ARRAY_WITH_KPTR(_size) \26 struct bin_data_##_size { \27 char data[_size - sizeof(void *)]; \28 }; \29 /* See Commit 5d8d6634ccc, force btf generation for type bin_data_##_size */ \30 struct bin_data_##_size *__bin_data_##_size; \31 struct map_value_##_size { \32 struct bin_data_##_size __kptr * data; \33 }; \34 struct { \35 __uint(type, BPF_MAP_TYPE_ARRAY); \36 __type(key, int); \37 __type(value, struct map_value_##_size); \38 __uint(max_entries, 128); \39 } array_##_size SEC(".maps")40 41#define DEFINE_ARRAY_WITH_PERCPU_KPTR(_size) \42 struct percpu_bin_data_##_size { \43 char data[_size]; \44 }; \45 struct percpu_bin_data_##_size *__percpu_bin_data_##_size; \46 struct map_value_percpu_##_size { \47 struct percpu_bin_data_##_size __percpu_kptr * data; \48 }; \49 struct { \50 __uint(type, BPF_MAP_TYPE_ARRAY); \51 __type(key, int); \52 __type(value, struct map_value_percpu_##_size); \53 __uint(max_entries, 128); \54 } array_percpu_##_size SEC(".maps")55 56static __always_inline void batch_alloc(struct bpf_map *map, unsigned int batch, unsigned int idx)57{58 struct generic_map_value *value;59 unsigned int i, key;60 void *old, *new;61 62 for (i = 0; i < batch; i++) {63 key = i;64 value = bpf_map_lookup_elem(map, &key);65 if (!value) {66 err = 1;67 return;68 }69 new = bpf_obj_new_impl(data_btf_ids[idx], NULL);70 if (!new) {71 err = 2;72 return;73 }74 old = bpf_kptr_xchg(&value->data, new);75 if (old) {76 bpf_obj_drop(old);77 err = 3;78 return;79 }80 }81}82 83static __always_inline void batch_free(struct bpf_map *map, unsigned int batch, unsigned int idx)84{85 struct generic_map_value *value;86 unsigned int i, key;87 void *old;88 89 for (i = 0; i < batch; i++) {90 key = i;91 value = bpf_map_lookup_elem(map, &key);92 if (!value) {93 err = 4;94 return;95 }96 old = bpf_kptr_xchg(&value->data, NULL);97 if (!old) {98 err = 5;99 return;100 }101 bpf_obj_drop(old);102 }103}104 105static __always_inline void batch_percpu_alloc(struct bpf_map *map, unsigned int batch,106 unsigned int idx)107{108 struct generic_map_value *value;109 unsigned int i, key;110 void *old, *new;111 112 for (i = 0; i < batch; i++) {113 key = i;114 value = bpf_map_lookup_elem(map, &key);115 if (!value) {116 err = 1;117 return;118 }119 /* per-cpu allocator may not be able to refill in time */120 new = bpf_percpu_obj_new_impl(percpu_data_btf_ids[idx], NULL);121 if (!new)122 continue;123 124 old = bpf_kptr_xchg(&value->data, new);125 if (old) {126 bpf_percpu_obj_drop(old);127 err = 2;128 return;129 }130 }131}132 133static __always_inline void batch_percpu_free(struct bpf_map *map, unsigned int batch,134 unsigned int idx)135{136 struct generic_map_value *value;137 unsigned int i, key;138 void *old;139 140 for (i = 0; i < batch; i++) {141 key = i;142 value = bpf_map_lookup_elem(map, &key);143 if (!value) {144 err = 3;145 return;146 }147 old = bpf_kptr_xchg(&value->data, NULL);148 if (!old)149 continue;150 bpf_percpu_obj_drop(old);151 }152}153 154#define CALL_BATCH_ALLOC(size, batch, idx) \155 batch_alloc((struct bpf_map *)(&array_##size), batch, idx)156 157#define CALL_BATCH_ALLOC_FREE(size, batch, idx) \158 do { \159 batch_alloc((struct bpf_map *)(&array_##size), batch, idx); \160 batch_free((struct bpf_map *)(&array_##size), batch, idx); \161 } while (0)162 163#define CALL_BATCH_PERCPU_ALLOC(size, batch, idx) \164 batch_percpu_alloc((struct bpf_map *)(&array_percpu_##size), batch, idx)165 166#define CALL_BATCH_PERCPU_ALLOC_FREE(size, batch, idx) \167 do { \168 batch_percpu_alloc((struct bpf_map *)(&array_percpu_##size), batch, idx); \169 batch_percpu_free((struct bpf_map *)(&array_percpu_##size), batch, idx); \170 } while (0)171 172/* kptr doesn't support bin_data_8 which is a zero-sized array */173DEFINE_ARRAY_WITH_KPTR(16);174DEFINE_ARRAY_WITH_KPTR(32);175DEFINE_ARRAY_WITH_KPTR(64);176DEFINE_ARRAY_WITH_KPTR(96);177DEFINE_ARRAY_WITH_KPTR(128);178DEFINE_ARRAY_WITH_KPTR(192);179DEFINE_ARRAY_WITH_KPTR(256);180DEFINE_ARRAY_WITH_KPTR(512);181DEFINE_ARRAY_WITH_KPTR(1024);182DEFINE_ARRAY_WITH_KPTR(2048);183DEFINE_ARRAY_WITH_KPTR(4096);184 185DEFINE_ARRAY_WITH_PERCPU_KPTR(8);186DEFINE_ARRAY_WITH_PERCPU_KPTR(16);187DEFINE_ARRAY_WITH_PERCPU_KPTR(32);188DEFINE_ARRAY_WITH_PERCPU_KPTR(64);189DEFINE_ARRAY_WITH_PERCPU_KPTR(96);190DEFINE_ARRAY_WITH_PERCPU_KPTR(128);191DEFINE_ARRAY_WITH_PERCPU_KPTR(192);192DEFINE_ARRAY_WITH_PERCPU_KPTR(256);193DEFINE_ARRAY_WITH_PERCPU_KPTR(512);194 195SEC("?fentry/" SYS_PREFIX "sys_nanosleep")196int test_batch_alloc_free(void *ctx)197{198 if ((u32)bpf_get_current_pid_tgid() != pid)199 return 0;200 201 /* Alloc 128 16-bytes objects in batch to trigger refilling,202 * then free 128 16-bytes objects in batch to trigger freeing.203 */204 CALL_BATCH_ALLOC_FREE(16, 128, 0);205 CALL_BATCH_ALLOC_FREE(32, 128, 1);206 CALL_BATCH_ALLOC_FREE(64, 128, 2);207 CALL_BATCH_ALLOC_FREE(96, 128, 3);208 CALL_BATCH_ALLOC_FREE(128, 128, 4);209 CALL_BATCH_ALLOC_FREE(192, 128, 5);210 CALL_BATCH_ALLOC_FREE(256, 128, 6);211 CALL_BATCH_ALLOC_FREE(512, 64, 7);212 CALL_BATCH_ALLOC_FREE(1024, 32, 8);213 CALL_BATCH_ALLOC_FREE(2048, 16, 9);214 CALL_BATCH_ALLOC_FREE(4096, 8, 10);215 216 return 0;217}218 219SEC("?fentry/" SYS_PREFIX "sys_nanosleep")220int test_free_through_map_free(void *ctx)221{222 if ((u32)bpf_get_current_pid_tgid() != pid)223 return 0;224 225 /* Alloc 128 16-bytes objects in batch to trigger refilling,226 * then free these objects through map free.227 */228 CALL_BATCH_ALLOC(16, 128, 0);229 CALL_BATCH_ALLOC(32, 128, 1);230 CALL_BATCH_ALLOC(64, 128, 2);231 CALL_BATCH_ALLOC(96, 128, 3);232 CALL_BATCH_ALLOC(128, 128, 4);233 CALL_BATCH_ALLOC(192, 128, 5);234 CALL_BATCH_ALLOC(256, 128, 6);235 CALL_BATCH_ALLOC(512, 64, 7);236 CALL_BATCH_ALLOC(1024, 32, 8);237 CALL_BATCH_ALLOC(2048, 16, 9);238 CALL_BATCH_ALLOC(4096, 8, 10);239 240 return 0;241}242 243SEC("?fentry/" SYS_PREFIX "sys_nanosleep")244int test_batch_percpu_alloc_free(void *ctx)245{246 if ((u32)bpf_get_current_pid_tgid() != pid)247 return 0;248 249 /* Alloc 128 8-bytes per-cpu objects in batch to trigger refilling,250 * then free 128 8-bytes per-cpu objects in batch to trigger freeing.251 */252 CALL_BATCH_PERCPU_ALLOC_FREE(8, 128, 0);253 CALL_BATCH_PERCPU_ALLOC_FREE(16, 128, 1);254 CALL_BATCH_PERCPU_ALLOC_FREE(32, 128, 2);255 CALL_BATCH_PERCPU_ALLOC_FREE(64, 128, 3);256 CALL_BATCH_PERCPU_ALLOC_FREE(96, 128, 4);257 CALL_BATCH_PERCPU_ALLOC_FREE(128, 128, 5);258 CALL_BATCH_PERCPU_ALLOC_FREE(192, 128, 6);259 CALL_BATCH_PERCPU_ALLOC_FREE(256, 128, 7);260 CALL_BATCH_PERCPU_ALLOC_FREE(512, 64, 8);261 262 return 0;263}264 265SEC("?fentry/" SYS_PREFIX "sys_nanosleep")266int test_percpu_free_through_map_free(void *ctx)267{268 if ((u32)bpf_get_current_pid_tgid() != pid)269 return 0;270 271 /* Alloc 128 8-bytes per-cpu objects in batch to trigger refilling,272 * then free these object through map free.273 */274 CALL_BATCH_PERCPU_ALLOC(8, 128, 0);275 CALL_BATCH_PERCPU_ALLOC(16, 128, 1);276 CALL_BATCH_PERCPU_ALLOC(32, 128, 2);277 CALL_BATCH_PERCPU_ALLOC(64, 128, 3);278 CALL_BATCH_PERCPU_ALLOC(96, 128, 4);279 CALL_BATCH_PERCPU_ALLOC(128, 128, 5);280 CALL_BATCH_PERCPU_ALLOC(192, 128, 6);281 CALL_BATCH_PERCPU_ALLOC(256, 128, 7);282 CALL_BATCH_PERCPU_ALLOC(512, 64, 8);283 284 return 0;285}286