749 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) 2020 Facebook */3#define _GNU_SOURCE4#include <argp.h>5#include <linux/compiler.h>6#include <sys/time.h>7#include <sched.h>8#include <fcntl.h>9#include <pthread.h>10#include <sys/sysinfo.h>11#include <signal.h>12#include "bench.h"13#include "bpf_util.h"14#include "testing_helpers.h"15 16struct env env = {17 .warmup_sec = 1,18 .duration_sec = 5,19 .affinity = false,20 .quiet = false,21 .consumer_cnt = 0,22 .producer_cnt = 1,23};24 25static int libbpf_print_fn(enum libbpf_print_level level,26 const char *format, va_list args)27{28 if (level == LIBBPF_DEBUG && !env.verbose)29 return 0;30 return vfprintf(stderr, format, args);31}32 33void setup_libbpf(void)34{35 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);36 libbpf_set_print(libbpf_print_fn);37}38 39void false_hits_report_progress(int iter, struct bench_res *res, long delta_ns)40{41 long total = res->false_hits + res->hits + res->drops;42 43 printf("Iter %3d (%7.3lfus): ",44 iter, (delta_ns - 1000000000) / 1000.0);45 46 printf("%ld false hits of %ld total operations. Percentage = %2.2f %%\n",47 res->false_hits, total, ((float)res->false_hits / total) * 100);48}49 50void false_hits_report_final(struct bench_res res[], int res_cnt)51{52 long total_hits = 0, total_drops = 0, total_false_hits = 0, total_ops = 0;53 int i;54 55 for (i = 0; i < res_cnt; i++) {56 total_hits += res[i].hits;57 total_false_hits += res[i].false_hits;58 total_drops += res[i].drops;59 }60 total_ops = total_hits + total_false_hits + total_drops;61 62 printf("Summary: %ld false hits of %ld total operations. ",63 total_false_hits, total_ops);64 printf("Percentage = %2.2f %%\n",65 ((float)total_false_hits / total_ops) * 100);66}67 68void hits_drops_report_progress(int iter, struct bench_res *res, long delta_ns)69{70 double hits_per_sec, drops_per_sec;71 double hits_per_prod;72 73 hits_per_sec = res->hits / 1000000.0 / (delta_ns / 1000000000.0);74 hits_per_prod = hits_per_sec / env.producer_cnt;75 drops_per_sec = res->drops / 1000000.0 / (delta_ns / 1000000000.0);76 77 printf("Iter %3d (%7.3lfus): ",78 iter, (delta_ns - 1000000000) / 1000.0);79 80 printf("hits %8.3lfM/s (%7.3lfM/prod), drops %8.3lfM/s, total operations %8.3lfM/s\n",81 hits_per_sec, hits_per_prod, drops_per_sec, hits_per_sec + drops_per_sec);82}83 84void85grace_period_latency_basic_stats(struct bench_res res[], int res_cnt, struct basic_stats *gp_stat)86{87 int i;88 89 memset(gp_stat, 0, sizeof(struct basic_stats));90 91 for (i = 0; i < res_cnt; i++)92 gp_stat->mean += res[i].gp_ns / 1000.0 / (double)res[i].gp_ct / (0.0 + res_cnt);93 94#define IT_MEAN_DIFF (res[i].gp_ns / 1000.0 / (double)res[i].gp_ct - gp_stat->mean)95 if (res_cnt > 1) {96 for (i = 0; i < res_cnt; i++)97 gp_stat->stddev += (IT_MEAN_DIFF * IT_MEAN_DIFF) / (res_cnt - 1.0);98 }99 gp_stat->stddev = sqrt(gp_stat->stddev);100#undef IT_MEAN_DIFF101}102 103void104grace_period_ticks_basic_stats(struct bench_res res[], int res_cnt, struct basic_stats *gp_stat)105{106 int i;107 108 memset(gp_stat, 0, sizeof(struct basic_stats));109 for (i = 0; i < res_cnt; i++)110 gp_stat->mean += res[i].stime / (double)res[i].gp_ct / (0.0 + res_cnt);111 112#define IT_MEAN_DIFF (res[i].stime / (double)res[i].gp_ct - gp_stat->mean)113 if (res_cnt > 1) {114 for (i = 0; i < res_cnt; i++)115 gp_stat->stddev += (IT_MEAN_DIFF * IT_MEAN_DIFF) / (res_cnt - 1.0);116 }117 gp_stat->stddev = sqrt(gp_stat->stddev);118#undef IT_MEAN_DIFF119}120 121void hits_drops_report_final(struct bench_res res[], int res_cnt)122{123 int i;124 double hits_mean = 0.0, drops_mean = 0.0, total_ops_mean = 0.0;125 double hits_stddev = 0.0, drops_stddev = 0.0, total_ops_stddev = 0.0;126 double total_ops;127 128 for (i = 0; i < res_cnt; i++) {129 hits_mean += res[i].hits / 1000000.0 / (0.0 + res_cnt);130 drops_mean += res[i].drops / 1000000.0 / (0.0 + res_cnt);131 }132 total_ops_mean = hits_mean + drops_mean;133 134 if (res_cnt > 1) {135 for (i = 0; i < res_cnt; i++) {136 hits_stddev += (hits_mean - res[i].hits / 1000000.0) *137 (hits_mean - res[i].hits / 1000000.0) /138 (res_cnt - 1.0);139 drops_stddev += (drops_mean - res[i].drops / 1000000.0) *140 (drops_mean - res[i].drops / 1000000.0) /141 (res_cnt - 1.0);142 total_ops = res[i].hits + res[i].drops;143 total_ops_stddev += (total_ops_mean - total_ops / 1000000.0) *144 (total_ops_mean - total_ops / 1000000.0) /145 (res_cnt - 1.0);146 }147 hits_stddev = sqrt(hits_stddev);148 drops_stddev = sqrt(drops_stddev);149 total_ops_stddev = sqrt(total_ops_stddev);150 }151 printf("Summary: hits %8.3lf \u00B1 %5.3lfM/s (%7.3lfM/prod), ",152 hits_mean, hits_stddev, hits_mean / env.producer_cnt);153 printf("drops %8.3lf \u00B1 %5.3lfM/s, ",154 drops_mean, drops_stddev);155 printf("total operations %8.3lf \u00B1 %5.3lfM/s\n",156 total_ops_mean, total_ops_stddev);157}158 159void ops_report_progress(int iter, struct bench_res *res, long delta_ns)160{161 double hits_per_sec, hits_per_prod;162 163 hits_per_sec = res->hits / 1000000.0 / (delta_ns / 1000000000.0);164 hits_per_prod = hits_per_sec / env.producer_cnt;165 166 printf("Iter %3d (%7.3lfus): ", iter, (delta_ns - 1000000000) / 1000.0);167 168 printf("hits %8.3lfM/s (%7.3lfM/prod)\n", hits_per_sec, hits_per_prod);169}170 171void ops_report_final(struct bench_res res[], int res_cnt)172{173 double hits_mean = 0.0, hits_stddev = 0.0;174 int i;175 176 for (i = 0; i < res_cnt; i++)177 hits_mean += res[i].hits / 1000000.0 / (0.0 + res_cnt);178 179 if (res_cnt > 1) {180 for (i = 0; i < res_cnt; i++)181 hits_stddev += (hits_mean - res[i].hits / 1000000.0) *182 (hits_mean - res[i].hits / 1000000.0) /183 (res_cnt - 1.0);184 185 hits_stddev = sqrt(hits_stddev);186 }187 printf("Summary: throughput %8.3lf \u00B1 %5.3lf M ops/s (%7.3lfM ops/prod), ",188 hits_mean, hits_stddev, hits_mean / env.producer_cnt);189 printf("latency %8.3lf ns/op\n", 1000.0 / hits_mean * env.producer_cnt);190}191 192void local_storage_report_progress(int iter, struct bench_res *res,193 long delta_ns)194{195 double important_hits_per_sec, hits_per_sec;196 double delta_sec = delta_ns / 1000000000.0;197 198 hits_per_sec = res->hits / 1000000.0 / delta_sec;199 important_hits_per_sec = res->important_hits / 1000000.0 / delta_sec;200 201 printf("Iter %3d (%7.3lfus): ", iter, (delta_ns - 1000000000) / 1000.0);202 203 printf("hits %8.3lfM/s ", hits_per_sec);204 printf("important_hits %8.3lfM/s\n", important_hits_per_sec);205}206 207void local_storage_report_final(struct bench_res res[], int res_cnt)208{209 double important_hits_mean = 0.0, important_hits_stddev = 0.0;210 double hits_mean = 0.0, hits_stddev = 0.0;211 int i;212 213 for (i = 0; i < res_cnt; i++) {214 hits_mean += res[i].hits / 1000000.0 / (0.0 + res_cnt);215 important_hits_mean += res[i].important_hits / 1000000.0 / (0.0 + res_cnt);216 }217 218 if (res_cnt > 1) {219 for (i = 0; i < res_cnt; i++) {220 hits_stddev += (hits_mean - res[i].hits / 1000000.0) *221 (hits_mean - res[i].hits / 1000000.0) /222 (res_cnt - 1.0);223 important_hits_stddev +=224 (important_hits_mean - res[i].important_hits / 1000000.0) *225 (important_hits_mean - res[i].important_hits / 1000000.0) /226 (res_cnt - 1.0);227 }228 229 hits_stddev = sqrt(hits_stddev);230 important_hits_stddev = sqrt(important_hits_stddev);231 }232 printf("Summary: hits throughput %8.3lf \u00B1 %5.3lf M ops/s, ",233 hits_mean, hits_stddev);234 printf("hits latency %8.3lf ns/op, ", 1000.0 / hits_mean);235 printf("important_hits throughput %8.3lf \u00B1 %5.3lf M ops/s\n",236 important_hits_mean, important_hits_stddev);237}238 239const char *argp_program_version = "benchmark";240const char *argp_program_bug_address = "<bpf@vger.kernel.org>";241const char argp_program_doc[] =242"benchmark Generic benchmarking framework.\n"243"\n"244"This tool runs benchmarks.\n"245"\n"246"USAGE: benchmark <bench-name>\n"247"\n"248"EXAMPLES:\n"249" # run 'count-local' benchmark with 1 producer and 1 consumer\n"250" benchmark count-local\n"251" # run 'count-local' with 16 producer and 8 consumer thread, pinned to CPUs\n"252" benchmark -p16 -c8 -a count-local\n";253 254enum {255 ARG_PROD_AFFINITY_SET = 1000,256 ARG_CONS_AFFINITY_SET = 1001,257};258 259static const struct argp_option opts[] = {260 { "list", 'l', NULL, 0, "List available benchmarks"},261 { "duration", 'd', "SEC", 0, "Duration of benchmark, seconds"},262 { "warmup", 'w', "SEC", 0, "Warm-up period, seconds"},263 { "producers", 'p', "NUM", 0, "Number of producer threads"},264 { "consumers", 'c', "NUM", 0, "Number of consumer threads"},265 { "verbose", 'v', NULL, 0, "Verbose debug output"},266 { "affinity", 'a', NULL, 0, "Set consumer/producer thread affinity"},267 { "quiet", 'q', NULL, 0, "Be more quiet"},268 { "prod-affinity", ARG_PROD_AFFINITY_SET, "CPUSET", 0,269 "Set of CPUs for producer threads; implies --affinity"},270 { "cons-affinity", ARG_CONS_AFFINITY_SET, "CPUSET", 0,271 "Set of CPUs for consumer threads; implies --affinity"},272 {},273};274 275extern struct argp bench_ringbufs_argp;276extern struct argp bench_bloom_map_argp;277extern struct argp bench_bpf_loop_argp;278extern struct argp bench_local_storage_argp;279extern struct argp bench_local_storage_rcu_tasks_trace_argp;280extern struct argp bench_strncmp_argp;281extern struct argp bench_hashmap_lookup_argp;282extern struct argp bench_local_storage_create_argp;283extern struct argp bench_htab_mem_argp;284extern struct argp bench_trigger_batch_argp;285extern struct argp bench_crypto_argp;286 287static const struct argp_child bench_parsers[] = {288 { &bench_ringbufs_argp, 0, "Ring buffers benchmark", 0 },289 { &bench_bloom_map_argp, 0, "Bloom filter map benchmark", 0 },290 { &bench_bpf_loop_argp, 0, "bpf_loop helper benchmark", 0 },291 { &bench_local_storage_argp, 0, "local_storage benchmark", 0 },292 { &bench_strncmp_argp, 0, "bpf_strncmp helper benchmark", 0 },293 { &bench_local_storage_rcu_tasks_trace_argp, 0,294 "local_storage RCU Tasks Trace slowdown benchmark", 0 },295 { &bench_hashmap_lookup_argp, 0, "Hashmap lookup benchmark", 0 },296 { &bench_local_storage_create_argp, 0, "local-storage-create benchmark", 0 },297 { &bench_htab_mem_argp, 0, "hash map memory benchmark", 0 },298 { &bench_trigger_batch_argp, 0, "BPF triggering benchmark", 0 },299 { &bench_crypto_argp, 0, "bpf crypto benchmark", 0 },300 {},301};302 303/* Make pos_args global, so that we can run argp_parse twice, if necessary */304static int pos_args;305 306static error_t parse_arg(int key, char *arg, struct argp_state *state)307{308 switch (key) {309 case 'v':310 env.verbose = true;311 break;312 case 'l':313 env.list = true;314 break;315 case 'd':316 env.duration_sec = strtol(arg, NULL, 10);317 if (env.duration_sec <= 0) {318 fprintf(stderr, "Invalid duration: %s\n", arg);319 argp_usage(state);320 }321 break;322 case 'w':323 env.warmup_sec = strtol(arg, NULL, 10);324 if (env.warmup_sec <= 0) {325 fprintf(stderr, "Invalid warm-up duration: %s\n", arg);326 argp_usage(state);327 }328 break;329 case 'p':330 env.producer_cnt = strtol(arg, NULL, 10);331 if (env.producer_cnt < 0) {332 fprintf(stderr, "Invalid producer count: %s\n", arg);333 argp_usage(state);334 }335 break;336 case 'c':337 env.consumer_cnt = strtol(arg, NULL, 10);338 if (env.consumer_cnt < 0) {339 fprintf(stderr, "Invalid consumer count: %s\n", arg);340 argp_usage(state);341 }342 break;343 case 'a':344 env.affinity = true;345 break;346 case 'q':347 env.quiet = true;348 break;349 case ARG_PROD_AFFINITY_SET:350 env.affinity = true;351 if (parse_num_list(arg, &env.prod_cpus.cpus,352 &env.prod_cpus.cpus_len)) {353 fprintf(stderr, "Invalid format of CPU set for producers.");354 argp_usage(state);355 }356 break;357 case ARG_CONS_AFFINITY_SET:358 env.affinity = true;359 if (parse_num_list(arg, &env.cons_cpus.cpus,360 &env.cons_cpus.cpus_len)) {361 fprintf(stderr, "Invalid format of CPU set for consumers.");362 argp_usage(state);363 }364 break;365 case ARGP_KEY_ARG:366 if (pos_args++) {367 fprintf(stderr,368 "Unrecognized positional argument: %s\n", arg);369 argp_usage(state);370 }371 env.bench_name = strdup(arg);372 break;373 default:374 return ARGP_ERR_UNKNOWN;375 }376 return 0;377}378 379static void parse_cmdline_args_init(int argc, char **argv)380{381 static const struct argp argp = {382 .options = opts,383 .parser = parse_arg,384 .doc = argp_program_doc,385 .children = bench_parsers,386 };387 if (argp_parse(&argp, argc, argv, 0, NULL, NULL))388 exit(1);389}390 391static void parse_cmdline_args_final(int argc, char **argv)392{393 struct argp_child bench_parsers[2] = {};394 const struct argp argp = {395 .options = opts,396 .parser = parse_arg,397 .doc = argp_program_doc,398 .children = bench_parsers,399 };400 401 /* Parse arguments the second time with the correct set of parsers */402 if (bench->argp) {403 bench_parsers[0].argp = bench->argp;404 bench_parsers[0].header = bench->name;405 pos_args = 0;406 if (argp_parse(&argp, argc, argv, 0, NULL, NULL))407 exit(1);408 }409}410 411static void collect_measurements(long delta_ns);412 413static __u64 last_time_ns;414static void sigalarm_handler(int signo)415{416 long new_time_ns = get_time_ns();417 long delta_ns = new_time_ns - last_time_ns;418 419 collect_measurements(delta_ns);420 421 last_time_ns = new_time_ns;422}423 424/* set up periodic 1-second timer */425static void setup_timer()426{427 static struct sigaction sigalarm_action = {428 .sa_handler = sigalarm_handler,429 };430 struct itimerval timer_settings = {};431 int err;432 433 last_time_ns = get_time_ns();434 err = sigaction(SIGALRM, &sigalarm_action, NULL);435 if (err < 0) {436 fprintf(stderr, "failed to install SIGALRM handler: %d\n", -errno);437 exit(1);438 }439 timer_settings.it_interval.tv_sec = 1;440 timer_settings.it_value.tv_sec = 1;441 err = setitimer(ITIMER_REAL, &timer_settings, NULL);442 if (err < 0) {443 fprintf(stderr, "failed to arm interval timer: %d\n", -errno);444 exit(1);445 }446}447 448static void set_thread_affinity(pthread_t thread, int cpu)449{450 cpu_set_t cpuset;451 int err;452 453 CPU_ZERO(&cpuset);454 CPU_SET(cpu, &cpuset);455 err = pthread_setaffinity_np(thread, sizeof(cpuset), &cpuset);456 if (err) {457 fprintf(stderr, "setting affinity to CPU #%d failed: %d\n",458 cpu, -err);459 exit(1);460 }461}462 463static int next_cpu(struct cpu_set *cpu_set)464{465 if (cpu_set->cpus) {466 int i;467 468 /* find next available CPU */469 for (i = cpu_set->next_cpu; i < cpu_set->cpus_len; i++) {470 if (cpu_set->cpus[i]) {471 cpu_set->next_cpu = i + 1;472 return i;473 }474 }475 fprintf(stderr, "Not enough CPUs specified, need CPU #%d or higher.\n", i);476 exit(1);477 }478 479 return cpu_set->next_cpu++ % env.nr_cpus;480}481 482static struct bench_state {483 int res_cnt;484 struct bench_res *results;485 pthread_t *consumers;486 pthread_t *producers;487} state;488 489const struct bench *bench = NULL;490 491extern const struct bench bench_count_global;492extern const struct bench bench_count_local;493extern const struct bench bench_rename_base;494extern const struct bench bench_rename_kprobe;495extern const struct bench bench_rename_kretprobe;496extern const struct bench bench_rename_rawtp;497extern const struct bench bench_rename_fentry;498extern const struct bench bench_rename_fexit;499 500/* pure counting benchmarks to establish theoretical lmits */501extern const struct bench bench_trig_usermode_count;502extern const struct bench bench_trig_syscall_count;503extern const struct bench bench_trig_kernel_count;504 505/* batched, staying mostly in-kernel benchmarks */506extern const struct bench bench_trig_kprobe;507extern const struct bench bench_trig_kretprobe;508extern const struct bench bench_trig_kprobe_multi;509extern const struct bench bench_trig_kretprobe_multi;510extern const struct bench bench_trig_fentry;511extern const struct bench bench_trig_fexit;512extern const struct bench bench_trig_fmodret;513extern const struct bench bench_trig_tp;514extern const struct bench bench_trig_rawtp;515 516/* uprobe/uretprobe benchmarks */517extern const struct bench bench_trig_uprobe_nop;518extern const struct bench bench_trig_uretprobe_nop;519extern const struct bench bench_trig_uprobe_push;520extern const struct bench bench_trig_uretprobe_push;521extern const struct bench bench_trig_uprobe_ret;522extern const struct bench bench_trig_uretprobe_ret;523extern const struct bench bench_trig_uprobe_multi_nop;524extern const struct bench bench_trig_uretprobe_multi_nop;525extern const struct bench bench_trig_uprobe_multi_push;526extern const struct bench bench_trig_uretprobe_multi_push;527extern const struct bench bench_trig_uprobe_multi_ret;528extern const struct bench bench_trig_uretprobe_multi_ret;529 530extern const struct bench bench_rb_libbpf;531extern const struct bench bench_rb_custom;532extern const struct bench bench_pb_libbpf;533extern const struct bench bench_pb_custom;534extern const struct bench bench_bloom_lookup;535extern const struct bench bench_bloom_update;536extern const struct bench bench_bloom_false_positive;537extern const struct bench bench_hashmap_without_bloom;538extern const struct bench bench_hashmap_with_bloom;539extern const struct bench bench_bpf_loop;540extern const struct bench bench_strncmp_no_helper;541extern const struct bench bench_strncmp_helper;542extern const struct bench bench_bpf_hashmap_full_update;543extern const struct bench bench_local_storage_cache_seq_get;544extern const struct bench bench_local_storage_cache_interleaved_get;545extern const struct bench bench_local_storage_cache_hashmap_control;546extern const struct bench bench_local_storage_tasks_trace;547extern const struct bench bench_bpf_hashmap_lookup;548extern const struct bench bench_local_storage_create;549extern const struct bench bench_htab_mem;550extern const struct bench bench_crypto_encrypt;551extern const struct bench bench_crypto_decrypt;552 553static const struct bench *benchs[] = {554 &bench_count_global,555 &bench_count_local,556 &bench_rename_base,557 &bench_rename_kprobe,558 &bench_rename_kretprobe,559 &bench_rename_rawtp,560 &bench_rename_fentry,561 &bench_rename_fexit,562 /* pure counting benchmarks for establishing theoretical limits */563 &bench_trig_usermode_count,564 &bench_trig_kernel_count,565 &bench_trig_syscall_count,566 /* batched, staying mostly in-kernel triggers */567 &bench_trig_kprobe,568 &bench_trig_kretprobe,569 &bench_trig_kprobe_multi,570 &bench_trig_kretprobe_multi,571 &bench_trig_fentry,572 &bench_trig_fexit,573 &bench_trig_fmodret,574 &bench_trig_tp,575 &bench_trig_rawtp,576 /* uprobes */577 &bench_trig_uprobe_nop,578 &bench_trig_uretprobe_nop,579 &bench_trig_uprobe_push,580 &bench_trig_uretprobe_push,581 &bench_trig_uprobe_ret,582 &bench_trig_uretprobe_ret,583 &bench_trig_uprobe_multi_nop,584 &bench_trig_uretprobe_multi_nop,585 &bench_trig_uprobe_multi_push,586 &bench_trig_uretprobe_multi_push,587 &bench_trig_uprobe_multi_ret,588 &bench_trig_uretprobe_multi_ret,589 /* ringbuf/perfbuf benchmarks */590 &bench_rb_libbpf,591 &bench_rb_custom,592 &bench_pb_libbpf,593 &bench_pb_custom,594 &bench_bloom_lookup,595 &bench_bloom_update,596 &bench_bloom_false_positive,597 &bench_hashmap_without_bloom,598 &bench_hashmap_with_bloom,599 &bench_bpf_loop,600 &bench_strncmp_no_helper,601 &bench_strncmp_helper,602 &bench_bpf_hashmap_full_update,603 &bench_local_storage_cache_seq_get,604 &bench_local_storage_cache_interleaved_get,605 &bench_local_storage_cache_hashmap_control,606 &bench_local_storage_tasks_trace,607 &bench_bpf_hashmap_lookup,608 &bench_local_storage_create,609 &bench_htab_mem,610 &bench_crypto_encrypt,611 &bench_crypto_decrypt,612};613 614static void find_benchmark(void)615{616 int i;617 618 if (!env.bench_name) {619 fprintf(stderr, "benchmark name is not specified\n");620 exit(1);621 }622 for (i = 0; i < ARRAY_SIZE(benchs); i++) {623 if (strcmp(benchs[i]->name, env.bench_name) == 0) {624 bench = benchs[i];625 break;626 }627 }628 if (!bench) {629 fprintf(stderr, "benchmark '%s' not found\n", env.bench_name);630 exit(1);631 }632}633 634static void setup_benchmark(void)635{636 int i, err;637 638 if (!env.quiet)639 printf("Setting up benchmark '%s'...\n", bench->name);640 641 state.producers = calloc(env.producer_cnt, sizeof(*state.producers));642 state.consumers = calloc(env.consumer_cnt, sizeof(*state.consumers));643 state.results = calloc(env.duration_sec + env.warmup_sec + 2,644 sizeof(*state.results));645 if (!state.producers || !state.consumers || !state.results)646 exit(1);647 648 if (bench->validate)649 bench->validate();650 if (bench->setup)651 bench->setup();652 653 for (i = 0; i < env.consumer_cnt; i++) {654 if (!bench->consumer_thread) {655 fprintf(stderr, "benchmark doesn't support consumers!\n");656 exit(1);657 }658 err = pthread_create(&state.consumers[i], NULL,659 bench->consumer_thread, (void *)(long)i);660 if (err) {661 fprintf(stderr, "failed to create consumer thread #%d: %d\n",662 i, -err);663 exit(1);664 }665 if (env.affinity)666 set_thread_affinity(state.consumers[i],667 next_cpu(&env.cons_cpus));668 }669 670 /* unless explicit producer CPU list is specified, continue after671 * last consumer CPU672 */673 if (!env.prod_cpus.cpus)674 env.prod_cpus.next_cpu = env.cons_cpus.next_cpu;675 676 for (i = 0; i < env.producer_cnt; i++) {677 if (!bench->producer_thread) {678 fprintf(stderr, "benchmark doesn't support producers!\n");679 exit(1);680 }681 err = pthread_create(&state.producers[i], NULL,682 bench->producer_thread, (void *)(long)i);683 if (err) {684 fprintf(stderr, "failed to create producer thread #%d: %d\n",685 i, -err);686 exit(1);687 }688 if (env.affinity)689 set_thread_affinity(state.producers[i],690 next_cpu(&env.prod_cpus));691 }692 693 if (!env.quiet)694 printf("Benchmark '%s' started.\n", bench->name);695}696 697static pthread_mutex_t bench_done_mtx = PTHREAD_MUTEX_INITIALIZER;698static pthread_cond_t bench_done = PTHREAD_COND_INITIALIZER;699 700static void collect_measurements(long delta_ns) {701 int iter = state.res_cnt++;702 struct bench_res *res = &state.results[iter];703 704 bench->measure(res);705 706 if (bench->report_progress)707 bench->report_progress(iter, res, delta_ns);708 709 if (iter == env.duration_sec + env.warmup_sec) {710 pthread_mutex_lock(&bench_done_mtx);711 pthread_cond_signal(&bench_done);712 pthread_mutex_unlock(&bench_done_mtx);713 }714}715 716int main(int argc, char **argv)717{718 env.nr_cpus = get_nprocs();719 parse_cmdline_args_init(argc, argv);720 721 if (env.list) {722 int i;723 724 printf("Available benchmarks:\n");725 for (i = 0; i < ARRAY_SIZE(benchs); i++) {726 printf("- %s\n", benchs[i]->name);727 }728 return 0;729 }730 731 find_benchmark();732 parse_cmdline_args_final(argc, argv);733 734 setup_benchmark();735 736 setup_timer();737 738 pthread_mutex_lock(&bench_done_mtx);739 pthread_cond_wait(&bench_done, &bench_done_mtx);740 pthread_mutex_unlock(&bench_done_mtx);741 742 if (bench->report_final)743 /* skip first sample */744 bench->report_final(state.results + env.warmup_sec,745 state.res_cnt - env.warmup_sec);746 747 return 0;748}749