773 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) 2023 Meta Platforms, Inc. and affiliates. */3 4#include <vmlinux.h>5#include <bpf/bpf_tracing.h>6#include <bpf/bpf_helpers.h>7 8#include "bpf_misc.h"9#include "cpumask_common.h"10 11char _license[] SEC("license") = "GPL";12 13int pid, nr_cpus;14 15struct kptr_nested {16 struct bpf_cpumask __kptr * mask;17};18 19struct kptr_nested_pair {20 struct bpf_cpumask __kptr * mask_1;21 struct bpf_cpumask __kptr * mask_2;22};23 24struct kptr_nested_mid {25 int dummy;26 struct kptr_nested m;27};28 29struct kptr_nested_deep {30 struct kptr_nested_mid ptrs[2];31 struct kptr_nested_pair ptr_pairs[3];32};33 34struct kptr_nested_deep_array_1_2 {35 int dummy;36 struct bpf_cpumask __kptr * mask[CPUMASK_KPTR_FIELDS_MAX];37};38 39struct kptr_nested_deep_array_1_1 {40 int dummy;41 struct kptr_nested_deep_array_1_2 d_2;42};43 44struct kptr_nested_deep_array_1 {45 long dummy;46 struct kptr_nested_deep_array_1_1 d_1;47};48 49struct kptr_nested_deep_array_2_2 {50 long dummy[2];51 struct bpf_cpumask __kptr * mask;52};53 54struct kptr_nested_deep_array_2_1 {55 int dummy;56 struct kptr_nested_deep_array_2_2 d_2[CPUMASK_KPTR_FIELDS_MAX];57};58 59struct kptr_nested_deep_array_2 {60 long dummy;61 struct kptr_nested_deep_array_2_1 d_1;62};63 64struct kptr_nested_deep_array_3_2 {65 long dummy[2];66 struct bpf_cpumask __kptr * mask;67};68 69struct kptr_nested_deep_array_3_1 {70 int dummy;71 struct kptr_nested_deep_array_3_2 d_2;72};73 74struct kptr_nested_deep_array_3 {75 long dummy;76 struct kptr_nested_deep_array_3_1 d_1[CPUMASK_KPTR_FIELDS_MAX];77};78 79private(MASK) static struct bpf_cpumask __kptr * global_mask_array[2];80private(MASK) static struct bpf_cpumask __kptr * global_mask_array_l2[2][1];81private(MASK) static struct bpf_cpumask __kptr * global_mask_array_one[1];82private(MASK) static struct kptr_nested global_mask_nested[2];83private(MASK_DEEP) static struct kptr_nested_deep global_mask_nested_deep;84private(MASK_1) static struct kptr_nested_deep_array_1 global_mask_nested_deep_array_1;85private(MASK_2) static struct kptr_nested_deep_array_2 global_mask_nested_deep_array_2;86private(MASK_3) static struct kptr_nested_deep_array_3 global_mask_nested_deep_array_3;87 88static bool is_test_task(void)89{90 int cur_pid = bpf_get_current_pid_tgid() >> 32;91 92 return pid == cur_pid;93}94 95static bool create_cpumask_set(struct bpf_cpumask **out1,96 struct bpf_cpumask **out2,97 struct bpf_cpumask **out3,98 struct bpf_cpumask **out4)99{100 struct bpf_cpumask *mask1, *mask2, *mask3, *mask4;101 102 mask1 = create_cpumask();103 if (!mask1)104 return false;105 106 mask2 = create_cpumask();107 if (!mask2) {108 bpf_cpumask_release(mask1);109 err = 3;110 return false;111 }112 113 mask3 = create_cpumask();114 if (!mask3) {115 bpf_cpumask_release(mask1);116 bpf_cpumask_release(mask2);117 err = 4;118 return false;119 }120 121 mask4 = create_cpumask();122 if (!mask4) {123 bpf_cpumask_release(mask1);124 bpf_cpumask_release(mask2);125 bpf_cpumask_release(mask3);126 err = 5;127 return false;128 }129 130 *out1 = mask1;131 *out2 = mask2;132 *out3 = mask3;133 *out4 = mask4;134 135 return true;136}137 138SEC("tp_btf/task_newtask")139int BPF_PROG(test_alloc_free_cpumask, struct task_struct *task, u64 clone_flags)140{141 struct bpf_cpumask *cpumask;142 143 if (!is_test_task())144 return 0;145 146 cpumask = create_cpumask();147 if (!cpumask)148 return 0;149 150 bpf_cpumask_release(cpumask);151 return 0;152}153 154SEC("tp_btf/task_newtask")155int BPF_PROG(test_set_clear_cpu, struct task_struct *task, u64 clone_flags)156{157 struct bpf_cpumask *cpumask;158 159 if (!is_test_task())160 return 0;161 162 cpumask = create_cpumask();163 if (!cpumask)164 return 0;165 166 bpf_cpumask_set_cpu(0, cpumask);167 if (!bpf_cpumask_test_cpu(0, cast(cpumask))) {168 err = 3;169 goto release_exit;170 }171 172 bpf_cpumask_clear_cpu(0, cpumask);173 if (bpf_cpumask_test_cpu(0, cast(cpumask))) {174 err = 4;175 goto release_exit;176 }177 178release_exit:179 bpf_cpumask_release(cpumask);180 return 0;181}182 183SEC("tp_btf/task_newtask")184int BPF_PROG(test_setall_clear_cpu, struct task_struct *task, u64 clone_flags)185{186 struct bpf_cpumask *cpumask;187 188 if (!is_test_task())189 return 0;190 191 cpumask = create_cpumask();192 if (!cpumask)193 return 0;194 195 bpf_cpumask_setall(cpumask);196 if (!bpf_cpumask_full(cast(cpumask))) {197 err = 3;198 goto release_exit;199 }200 201 bpf_cpumask_clear(cpumask);202 if (!bpf_cpumask_empty(cast(cpumask))) {203 err = 4;204 goto release_exit;205 }206 207release_exit:208 bpf_cpumask_release(cpumask);209 return 0;210}211 212SEC("tp_btf/task_newtask")213int BPF_PROG(test_first_firstzero_cpu, struct task_struct *task, u64 clone_flags)214{215 struct bpf_cpumask *cpumask;216 217 if (!is_test_task())218 return 0;219 220 cpumask = create_cpumask();221 if (!cpumask)222 return 0;223 224 if (bpf_cpumask_first(cast(cpumask)) < nr_cpus) {225 err = 3;226 goto release_exit;227 }228 229 if (bpf_cpumask_first_zero(cast(cpumask)) != 0) {230 bpf_printk("first zero: %d", bpf_cpumask_first_zero(cast(cpumask)));231 err = 4;232 goto release_exit;233 }234 235 bpf_cpumask_set_cpu(0, cpumask);236 if (bpf_cpumask_first(cast(cpumask)) != 0) {237 err = 5;238 goto release_exit;239 }240 241 if (bpf_cpumask_first_zero(cast(cpumask)) != 1) {242 err = 6;243 goto release_exit;244 }245 246release_exit:247 bpf_cpumask_release(cpumask);248 return 0;249}250 251SEC("tp_btf/task_newtask")252int BPF_PROG(test_firstand_nocpu, struct task_struct *task, u64 clone_flags)253{254 struct bpf_cpumask *mask1, *mask2;255 u32 first;256 257 if (!is_test_task())258 return 0;259 260 mask1 = create_cpumask();261 if (!mask1)262 return 0;263 264 mask2 = create_cpumask();265 if (!mask2)266 goto release_exit;267 268 bpf_cpumask_set_cpu(0, mask1);269 bpf_cpumask_set_cpu(1, mask2);270 271 first = bpf_cpumask_first_and(cast(mask1), cast(mask2));272 if (first <= 1)273 err = 3;274 275release_exit:276 if (mask1)277 bpf_cpumask_release(mask1);278 if (mask2)279 bpf_cpumask_release(mask2);280 return 0;281}282 283SEC("tp_btf/task_newtask")284int BPF_PROG(test_test_and_set_clear, struct task_struct *task, u64 clone_flags)285{286 struct bpf_cpumask *cpumask;287 288 if (!is_test_task())289 return 0;290 291 cpumask = create_cpumask();292 if (!cpumask)293 return 0;294 295 if (bpf_cpumask_test_and_set_cpu(0, cpumask)) {296 err = 3;297 goto release_exit;298 }299 300 if (!bpf_cpumask_test_and_set_cpu(0, cpumask)) {301 err = 4;302 goto release_exit;303 }304 305 if (!bpf_cpumask_test_and_clear_cpu(0, cpumask)) {306 err = 5;307 goto release_exit;308 }309 310release_exit:311 bpf_cpumask_release(cpumask);312 return 0;313}314 315SEC("tp_btf/task_newtask")316int BPF_PROG(test_and_or_xor, struct task_struct *task, u64 clone_flags)317{318 struct bpf_cpumask *mask1, *mask2, *dst1, *dst2;319 320 if (!is_test_task())321 return 0;322 323 if (!create_cpumask_set(&mask1, &mask2, &dst1, &dst2))324 return 0;325 326 bpf_cpumask_set_cpu(0, mask1);327 bpf_cpumask_set_cpu(1, mask2);328 329 if (bpf_cpumask_and(dst1, cast(mask1), cast(mask2))) {330 err = 6;331 goto release_exit;332 }333 if (!bpf_cpumask_empty(cast(dst1))) {334 err = 7;335 goto release_exit;336 }337 338 bpf_cpumask_or(dst1, cast(mask1), cast(mask2));339 if (!bpf_cpumask_test_cpu(0, cast(dst1))) {340 err = 8;341 goto release_exit;342 }343 if (!bpf_cpumask_test_cpu(1, cast(dst1))) {344 err = 9;345 goto release_exit;346 }347 348 bpf_cpumask_xor(dst2, cast(mask1), cast(mask2));349 if (!bpf_cpumask_equal(cast(dst1), cast(dst2))) {350 err = 10;351 goto release_exit;352 }353 354release_exit:355 bpf_cpumask_release(mask1);356 bpf_cpumask_release(mask2);357 bpf_cpumask_release(dst1);358 bpf_cpumask_release(dst2);359 return 0;360}361 362SEC("tp_btf/task_newtask")363int BPF_PROG(test_intersects_subset, struct task_struct *task, u64 clone_flags)364{365 struct bpf_cpumask *mask1, *mask2, *dst1, *dst2;366 367 if (!is_test_task())368 return 0;369 370 if (!create_cpumask_set(&mask1, &mask2, &dst1, &dst2))371 return 0;372 373 bpf_cpumask_set_cpu(0, mask1);374 bpf_cpumask_set_cpu(1, mask2);375 if (bpf_cpumask_intersects(cast(mask1), cast(mask2))) {376 err = 6;377 goto release_exit;378 }379 380 bpf_cpumask_or(dst1, cast(mask1), cast(mask2));381 if (!bpf_cpumask_subset(cast(mask1), cast(dst1))) {382 err = 7;383 goto release_exit;384 }385 386 if (!bpf_cpumask_subset(cast(mask2), cast(dst1))) {387 err = 8;388 goto release_exit;389 }390 391 if (bpf_cpumask_subset(cast(dst1), cast(mask1))) {392 err = 9;393 goto release_exit;394 }395 396release_exit:397 bpf_cpumask_release(mask1);398 bpf_cpumask_release(mask2);399 bpf_cpumask_release(dst1);400 bpf_cpumask_release(dst2);401 return 0;402}403 404SEC("tp_btf/task_newtask")405int BPF_PROG(test_copy_any_anyand, struct task_struct *task, u64 clone_flags)406{407 struct bpf_cpumask *mask1, *mask2, *dst1, *dst2;408 int cpu;409 410 if (!is_test_task())411 return 0;412 413 if (!create_cpumask_set(&mask1, &mask2, &dst1, &dst2))414 return 0;415 416 bpf_cpumask_set_cpu(0, mask1);417 bpf_cpumask_set_cpu(1, mask2);418 bpf_cpumask_or(dst1, cast(mask1), cast(mask2));419 420 cpu = bpf_cpumask_any_distribute(cast(mask1));421 if (cpu != 0) {422 err = 6;423 goto release_exit;424 }425 426 cpu = bpf_cpumask_any_distribute(cast(dst2));427 if (cpu < nr_cpus) {428 err = 7;429 goto release_exit;430 }431 432 bpf_cpumask_copy(dst2, cast(dst1));433 if (!bpf_cpumask_equal(cast(dst1), cast(dst2))) {434 err = 8;435 goto release_exit;436 }437 438 cpu = bpf_cpumask_any_distribute(cast(dst2));439 if (cpu > 1) {440 err = 9;441 goto release_exit;442 }443 444 cpu = bpf_cpumask_any_and_distribute(cast(mask1), cast(mask2));445 if (cpu < nr_cpus) {446 err = 10;447 goto release_exit;448 }449 450release_exit:451 bpf_cpumask_release(mask1);452 bpf_cpumask_release(mask2);453 bpf_cpumask_release(dst1);454 bpf_cpumask_release(dst2);455 return 0;456}457 458SEC("tp_btf/task_newtask")459int BPF_PROG(test_insert_leave, struct task_struct *task, u64 clone_flags)460{461 struct bpf_cpumask *cpumask;462 463 cpumask = create_cpumask();464 if (!cpumask)465 return 0;466 467 if (cpumask_map_insert(cpumask))468 err = 3;469 470 return 0;471}472 473SEC("tp_btf/task_newtask")474int BPF_PROG(test_insert_remove_release, struct task_struct *task, u64 clone_flags)475{476 struct bpf_cpumask *cpumask;477 struct __cpumask_map_value *v;478 479 cpumask = create_cpumask();480 if (!cpumask)481 return 0;482 483 if (cpumask_map_insert(cpumask)) {484 err = 3;485 return 0;486 }487 488 v = cpumask_map_value_lookup();489 if (!v) {490 err = 4;491 return 0;492 }493 494 cpumask = bpf_kptr_xchg(&v->cpumask, NULL);495 if (cpumask)496 bpf_cpumask_release(cpumask);497 else498 err = 5;499 500 return 0;501}502 503SEC("tp_btf/task_newtask")504int BPF_PROG(test_global_mask_rcu, struct task_struct *task, u64 clone_flags)505{506 struct bpf_cpumask *local, *prev;507 508 if (!is_test_task())509 return 0;510 511 local = create_cpumask();512 if (!local)513 return 0;514 515 prev = bpf_kptr_xchg(&global_mask, local);516 if (prev) {517 bpf_cpumask_release(prev);518 err = 3;519 return 0;520 }521 522 bpf_rcu_read_lock();523 local = global_mask;524 if (!local) {525 err = 4;526 bpf_rcu_read_unlock();527 return 0;528 }529 530 bpf_cpumask_test_cpu(0, (const struct cpumask *)local);531 bpf_rcu_read_unlock();532 533 return 0;534}535 536SEC("tp_btf/task_newtask")537int BPF_PROG(test_global_mask_array_one_rcu, struct task_struct *task, u64 clone_flags)538{539 struct bpf_cpumask *local, *prev;540 541 if (!is_test_task())542 return 0;543 544 /* Kptr arrays with one element are special cased, being treated545 * just like a single pointer.546 */547 548 local = create_cpumask();549 if (!local)550 return 0;551 552 prev = bpf_kptr_xchg(&global_mask_array_one[0], local);553 if (prev) {554 bpf_cpumask_release(prev);555 err = 3;556 return 0;557 }558 559 bpf_rcu_read_lock();560 local = global_mask_array_one[0];561 if (!local) {562 err = 4;563 bpf_rcu_read_unlock();564 return 0;565 }566 567 bpf_rcu_read_unlock();568 569 return 0;570}571 572static int _global_mask_array_rcu(struct bpf_cpumask **mask0,573 struct bpf_cpumask **mask1)574{575 struct bpf_cpumask *local;576 577 if (!is_test_task())578 return 0;579 580 /* Check if two kptrs in the array work and independently */581 582 local = create_cpumask();583 if (!local)584 return 0;585 586 bpf_rcu_read_lock();587 588 local = bpf_kptr_xchg(mask0, local);589 if (local) {590 err = 1;591 goto err_exit;592 }593 594 /* [<mask 0>, *] */595 if (!*mask0) {596 err = 2;597 goto err_exit;598 }599 600 if (!mask1)601 goto err_exit;602 603 /* [*, NULL] */604 if (*mask1) {605 err = 3;606 goto err_exit;607 }608 609 local = create_cpumask();610 if (!local) {611 err = 9;612 goto err_exit;613 }614 615 local = bpf_kptr_xchg(mask1, local);616 if (local) {617 err = 10;618 goto err_exit;619 }620 621 /* [<mask 0>, <mask 1>] */622 if (!*mask0 || !*mask1 || *mask0 == *mask1) {623 err = 11;624 goto err_exit;625 }626 627err_exit:628 if (local)629 bpf_cpumask_release(local);630 bpf_rcu_read_unlock();631 return 0;632}633 634SEC("tp_btf/task_newtask")635int BPF_PROG(test_global_mask_array_rcu, struct task_struct *task, u64 clone_flags)636{637 return _global_mask_array_rcu(&global_mask_array[0], &global_mask_array[1]);638}639 640SEC("tp_btf/task_newtask")641int BPF_PROG(test_global_mask_array_l2_rcu, struct task_struct *task, u64 clone_flags)642{643 return _global_mask_array_rcu(&global_mask_array_l2[0][0], &global_mask_array_l2[1][0]);644}645 646SEC("tp_btf/task_newtask")647int BPF_PROG(test_global_mask_nested_rcu, struct task_struct *task, u64 clone_flags)648{649 return _global_mask_array_rcu(&global_mask_nested[0].mask, &global_mask_nested[1].mask);650}651 652/* Ensure that the field->offset has been correctly advanced from one653 * nested struct or array sub-tree to another. In the case of654 * kptr_nested_deep, it comprises two sub-trees: ktpr_1 and kptr_2. By655 * calling bpf_kptr_xchg() on every single kptr in both nested sub-trees,656 * the verifier should reject the program if the field->offset of any kptr657 * is incorrect.658 *659 * For instance, if we have 10 kptrs in a nested struct and a program that660 * accesses each kptr individually with bpf_kptr_xchg(), the compiler661 * should emit instructions to access 10 different offsets if it works662 * correctly. If the field->offset values of any pair of them are663 * incorrectly the same, the number of unique offsets in btf_record for664 * this nested struct should be less than 10. The verifier should fail to665 * discover some of the offsets emitted by the compiler.666 *667 * Even if the field->offset values of kptrs are not duplicated, the668 * verifier should fail to find a btf_field for the instruction accessing a669 * kptr if the corresponding field->offset is pointing to a random670 * incorrect offset.671 */672SEC("tp_btf/task_newtask")673int BPF_PROG(test_global_mask_nested_deep_rcu, struct task_struct *task, u64 clone_flags)674{675 int r, i;676 677 r = _global_mask_array_rcu(&global_mask_nested_deep.ptrs[0].m.mask,678 &global_mask_nested_deep.ptrs[1].m.mask);679 if (r)680 return r;681 682 for (i = 0; i < 3; i++) {683 r = _global_mask_array_rcu(&global_mask_nested_deep.ptr_pairs[i].mask_1,684 &global_mask_nested_deep.ptr_pairs[i].mask_2);685 if (r)686 return r;687 }688 return 0;689}690 691SEC("tp_btf/task_newtask")692int BPF_PROG(test_global_mask_nested_deep_array_rcu, struct task_struct *task, u64 clone_flags)693{694 int i;695 696 for (i = 0; i < CPUMASK_KPTR_FIELDS_MAX; i++)697 _global_mask_array_rcu(&global_mask_nested_deep_array_1.d_1.d_2.mask[i], NULL);698 699 for (i = 0; i < CPUMASK_KPTR_FIELDS_MAX; i++)700 _global_mask_array_rcu(&global_mask_nested_deep_array_2.d_1.d_2[i].mask, NULL);701 702 for (i = 0; i < CPUMASK_KPTR_FIELDS_MAX; i++)703 _global_mask_array_rcu(&global_mask_nested_deep_array_3.d_1[i].d_2.mask, NULL);704 705 return 0;706}707 708SEC("tp_btf/task_newtask")709int BPF_PROG(test_cpumask_weight, struct task_struct *task, u64 clone_flags)710{711 struct bpf_cpumask *local;712 713 if (!is_test_task())714 return 0;715 716 local = create_cpumask();717 if (!local)718 return 0;719 720 if (bpf_cpumask_weight(cast(local)) != 0) {721 err = 3;722 goto out;723 }724 725 bpf_cpumask_set_cpu(0, local);726 if (bpf_cpumask_weight(cast(local)) != 1) {727 err = 4;728 goto out;729 }730 731 /*732 * Make sure that adding additional CPUs changes the weight. Test to733 * see whether the CPU was set to account for running on UP machines.734 */735 bpf_cpumask_set_cpu(1, local);736 if (bpf_cpumask_test_cpu(1, cast(local)) && bpf_cpumask_weight(cast(local)) != 2) {737 err = 5;738 goto out;739 }740 741 bpf_cpumask_clear(local);742 if (bpf_cpumask_weight(cast(local)) != 0) {743 err = 6;744 goto out;745 }746out:747 bpf_cpumask_release(local);748 return 0;749}750 751SEC("tp_btf/task_newtask")752__success753int BPF_PROG(test_refcount_null_tracking, struct task_struct *task, u64 clone_flags)754{755 struct bpf_cpumask *mask1, *mask2;756 757 mask1 = bpf_cpumask_create();758 mask2 = bpf_cpumask_create();759 760 if (!mask1 || !mask2)761 goto free_masks_return;762 763 bpf_cpumask_test_cpu(0, (const struct cpumask *)mask1);764 bpf_cpumask_test_cpu(0, (const struct cpumask *)mask2);765 766free_masks_return:767 if (mask1)768 bpf_cpumask_release(mask1);769 if (mask2)770 bpf_cpumask_release(mask2);771 return 0;772}773