562 lines · c
1// SPDX-License-Identifier: GPL-2.02#ifdef HAVE_EVENTFD_SUPPORT3/*4 * Copyright (C) 2018 Davidlohr Bueso.5 *6 * This program benchmarks concurrent epoll_wait(2) monitoring multiple7 * file descriptors under one or two load balancing models. The first,8 * and default, is the single/combined queueing (which refers to a single9 * epoll instance for N worker threads):10 *11 * |---> [worker A]12 * |---> [worker B]13 * [combined queue] .---> [worker C]14 * |---> [worker D]15 * |---> [worker E]16 *17 * While the second model, enabled via --multiq option, uses multiple18 * queueing (which refers to one epoll instance per worker). For example,19 * short lived tcp connections in a high throughput httpd server will20 * distribute the accept()'ing connections across CPUs. In this case each21 * worker does a limited amount of processing.22 *23 * [queue A] ---> [worker]24 * [queue B] ---> [worker]25 * [queue C] ---> [worker]26 * [queue D] ---> [worker]27 * [queue E] ---> [worker]28 *29 * Naturally, the single queue will enforce more concurrency on the epoll30 * instance, and can therefore scale poorly compared to multiple queues.31 * However, this is a benchmark raw data and must be taken with a grain of32 * salt when choosing how to make use of sys_epoll.33 34 * Each thread has a number of private, nonblocking file descriptors,35 * referred to as fdmap. A writer thread will constantly be writing to36 * the fdmaps of all threads, minimizing each threads's chances of37 * epoll_wait not finding any ready read events and blocking as this38 * is not what we want to stress. The size of the fdmap can be adjusted39 * by the user; enlarging the value will increase the chances of40 * epoll_wait(2) blocking as the lineal writer thread will take "longer",41 * at least at a high level.42 *43 * Note that because fds are private to each thread, this workload does44 * not stress scenarios where multiple tasks are awoken per ready IO; ie:45 * EPOLLEXCLUSIVE semantics.46 *47 * The end result/metric is throughput: number of ops/second where an48 * operation consists of:49 *50 * epoll_wait(2) + [others]51 *52 * ... where [others] is the cost of re-adding the fd (EPOLLET),53 * or rearming it (EPOLLONESHOT).54 *55 *56 * The purpose of this is program is that it be useful for measuring57 * kernel related changes to the sys_epoll, and not comparing different58 * IO polling methods, for example. Hence everything is very adhoc and59 * outputs raw microbenchmark numbers. Also this uses eventfd, similar60 * tools tend to use pipes or sockets, but the result is the same.61 */62 63/* For the CLR_() macros */64#include <string.h>65#include <pthread.h>66#include <unistd.h>67 68#include <errno.h>69#include <inttypes.h>70#include <signal.h>71#include <stdlib.h>72#include <linux/compiler.h>73#include <linux/kernel.h>74#include <sys/time.h>75#include <sys/resource.h>76#include <sys/epoll.h>77#include <sys/eventfd.h>78#include <sys/types.h>79#include <perf/cpumap.h>80 81#include "../util/stat.h"82#include "../util/mutex.h"83#include <subcmd/parse-options.h>84#include "bench.h"85 86#include <err.h>87 88#define printinfo(fmt, arg...) \89 do { if (__verbose) { printf(fmt, ## arg); fflush(stdout); } } while (0)90 91static unsigned int nthreads = 0;92static unsigned int nsecs = 8;93static bool wdone, done, __verbose, randomize, nonblocking;94 95/*96 * epoll related shared variables.97 */98 99/* Maximum number of nesting allowed inside epoll sets */100#define EPOLL_MAXNESTS 4101 102static int epollfd;103static int *epollfdp;104static bool noaffinity;105static unsigned int nested = 0;106static bool et; /* edge-trigger */107static bool oneshot;108static bool multiq; /* use an epoll instance per thread */109 110/* amount of fds to monitor, per thread */111static unsigned int nfds = 64;112 113static struct mutex thread_lock;114static unsigned int threads_starting;115static struct stats throughput_stats;116static struct cond thread_parent, thread_worker;117 118struct worker {119 int tid;120 int epollfd; /* for --multiq */121 pthread_t thread;122 unsigned long ops;123 int *fdmap;124};125 126static const struct option options[] = {127 /* general benchmark options */128 OPT_UINTEGER('t', "threads", &nthreads, "Specify amount of threads"),129 OPT_UINTEGER('r', "runtime", &nsecs, "Specify runtime (in seconds)"),130 OPT_UINTEGER('f', "nfds", &nfds, "Specify amount of file descriptors to monitor for each thread"),131 OPT_BOOLEAN( 'n', "noaffinity", &noaffinity, "Disables CPU affinity"),132 OPT_BOOLEAN('R', "randomize", &randomize, "Enable random write behaviour (default is lineal)"),133 OPT_BOOLEAN( 'v', "verbose", &__verbose, "Verbose mode"),134 135 /* epoll specific options */136 OPT_BOOLEAN( 'm', "multiq", &multiq, "Use multiple epoll instances (one per thread)"),137 OPT_BOOLEAN( 'B', "nonblocking", &nonblocking, "Nonblocking epoll_wait(2) behaviour"),138 OPT_UINTEGER( 'N', "nested", &nested, "Nesting level epoll hierarchy (default is 0, no nesting)"),139 OPT_BOOLEAN( 'S', "oneshot", &oneshot, "Use EPOLLONESHOT semantics"),140 OPT_BOOLEAN( 'E', "edge", &et, "Use Edge-triggered interface (default is LT)"),141 142 OPT_END()143};144 145static const char * const bench_epoll_wait_usage[] = {146 "perf bench epoll wait <options>",147 NULL148};149 150 151/*152 * Arrange the N elements of ARRAY in random order.153 * Only effective if N is much smaller than RAND_MAX;154 * if this may not be the case, use a better random155 * number generator. -- Ben Pfaff.156 */157static void shuffle(void *array, size_t n, size_t size)158{159 char *carray = array;160 void *aux;161 size_t i;162 163 if (n <= 1)164 return;165 166 aux = calloc(1, size);167 if (!aux)168 err(EXIT_FAILURE, "calloc");169 170 for (i = 1; i < n; ++i) {171 size_t j = i + rand() / (RAND_MAX / (n - i) + 1);172 j *= size;173 174 memcpy(aux, &carray[j], size);175 memcpy(&carray[j], &carray[i*size], size);176 memcpy(&carray[i*size], aux, size);177 }178 179 free(aux);180}181 182 183static void *workerfn(void *arg)184{185 int fd, ret, r;186 struct worker *w = (struct worker *) arg;187 unsigned long ops = w->ops;188 struct epoll_event ev;189 uint64_t val;190 int to = nonblocking? 0 : -1;191 int efd = multiq ? w->epollfd : epollfd;192 193 mutex_lock(&thread_lock);194 threads_starting--;195 if (!threads_starting)196 cond_signal(&thread_parent);197 cond_wait(&thread_worker, &thread_lock);198 mutex_unlock(&thread_lock);199 200 do {201 /*202 * Block indefinitely waiting for the IN event.203 * In order to stress the epoll_wait(2) syscall,204 * call it event per event, instead of a larger205 * batch (max)limit.206 */207 do {208 ret = epoll_wait(efd, &ev, 1, to);209 } while (ret < 0 && errno == EINTR);210 if (ret < 0)211 err(EXIT_FAILURE, "epoll_wait");212 213 fd = ev.data.fd;214 215 do {216 r = read(fd, &val, sizeof(val));217 } while (!done && (r < 0 && errno == EAGAIN));218 219 if (et) {220 ev.events = EPOLLIN | EPOLLET;221 ret = epoll_ctl(efd, EPOLL_CTL_ADD, fd, &ev);222 }223 224 if (oneshot) {225 /* rearm the file descriptor with a new event mask */226 ev.events |= EPOLLIN | EPOLLONESHOT;227 ret = epoll_ctl(efd, EPOLL_CTL_MOD, fd, &ev);228 }229 230 ops++;231 } while (!done);232 233 if (multiq)234 close(w->epollfd);235 236 w->ops = ops;237 return NULL;238}239 240static void nest_epollfd(struct worker *w)241{242 unsigned int i;243 struct epoll_event ev;244 int efd = multiq ? w->epollfd : epollfd;245 246 if (nested > EPOLL_MAXNESTS)247 nested = EPOLL_MAXNESTS;248 249 epollfdp = calloc(nested, sizeof(*epollfdp));250 if (!epollfdp)251 err(EXIT_FAILURE, "calloc");252 253 for (i = 0; i < nested; i++) {254 epollfdp[i] = epoll_create(1);255 if (epollfdp[i] < 0)256 err(EXIT_FAILURE, "epoll_create");257 }258 259 ev.events = EPOLLHUP; /* anything */260 ev.data.u64 = i; /* any number */261 262 for (i = nested - 1; i; i--) {263 if (epoll_ctl(epollfdp[i - 1], EPOLL_CTL_ADD,264 epollfdp[i], &ev) < 0)265 err(EXIT_FAILURE, "epoll_ctl");266 }267 268 if (epoll_ctl(efd, EPOLL_CTL_ADD, *epollfdp, &ev) < 0)269 err(EXIT_FAILURE, "epoll_ctl");270}271 272static void toggle_done(int sig __maybe_unused,273 siginfo_t *info __maybe_unused,274 void *uc __maybe_unused)275{276 /* inform all threads that we're done for the day */277 done = true;278 gettimeofday(&bench__end, NULL);279 timersub(&bench__end, &bench__start, &bench__runtime);280}281 282static void print_summary(void)283{284 unsigned long avg = avg_stats(&throughput_stats);285 double stddev = stddev_stats(&throughput_stats);286 287 printf("\nAveraged %ld operations/sec (+- %.2f%%), total secs = %d\n",288 avg, rel_stddev_stats(stddev, avg),289 (int)bench__runtime.tv_sec);290}291 292static int do_threads(struct worker *worker, struct perf_cpu_map *cpu)293{294 pthread_attr_t thread_attr, *attrp = NULL;295 cpu_set_t *cpuset;296 unsigned int i, j;297 int ret = 0, events = EPOLLIN;298 int nrcpus;299 size_t size;300 301 if (oneshot)302 events |= EPOLLONESHOT;303 if (et)304 events |= EPOLLET;305 306 printinfo("starting worker/consumer %sthreads%s\n",307 noaffinity ? "":"CPU affinity ",308 nonblocking ? " (nonblocking)":"");309 if (!noaffinity)310 pthread_attr_init(&thread_attr);311 312 nrcpus = cpu__max_cpu().cpu;313 cpuset = CPU_ALLOC(nrcpus);314 BUG_ON(!cpuset);315 size = CPU_ALLOC_SIZE(nrcpus);316 317 for (i = 0; i < nthreads; i++) {318 struct worker *w = &worker[i];319 320 if (multiq) {321 w->epollfd = epoll_create(1);322 if (w->epollfd < 0)323 err(EXIT_FAILURE, "epoll_create");324 325 if (nested)326 nest_epollfd(w);327 }328 329 w->tid = i;330 w->fdmap = calloc(nfds, sizeof(int));331 if (!w->fdmap)332 return 1;333 334 for (j = 0; j < nfds; j++) {335 int efd = multiq ? w->epollfd : epollfd;336 struct epoll_event ev;337 338 w->fdmap[j] = eventfd(0, EFD_NONBLOCK);339 if (w->fdmap[j] < 0)340 err(EXIT_FAILURE, "eventfd");341 342 ev.data.fd = w->fdmap[j];343 ev.events = events;344 345 ret = epoll_ctl(efd, EPOLL_CTL_ADD,346 w->fdmap[j], &ev);347 if (ret < 0)348 err(EXIT_FAILURE, "epoll_ctl");349 }350 351 if (!noaffinity) {352 CPU_ZERO_S(size, cpuset);353 CPU_SET_S(perf_cpu_map__cpu(cpu, i % perf_cpu_map__nr(cpu)).cpu,354 size, cpuset);355 356 ret = pthread_attr_setaffinity_np(&thread_attr, size, cpuset);357 if (ret) {358 CPU_FREE(cpuset);359 err(EXIT_FAILURE, "pthread_attr_setaffinity_np");360 }361 362 attrp = &thread_attr;363 }364 365 ret = pthread_create(&w->thread, attrp, workerfn,366 (void *)(struct worker *) w);367 if (ret) {368 CPU_FREE(cpuset);369 err(EXIT_FAILURE, "pthread_create");370 }371 }372 373 CPU_FREE(cpuset);374 if (!noaffinity)375 pthread_attr_destroy(&thread_attr);376 377 return ret;378}379 380static void *writerfn(void *p)381{382 struct worker *worker = p;383 size_t i, j, iter;384 const uint64_t val = 1;385 ssize_t sz;386 struct timespec ts = { .tv_sec = 0,387 .tv_nsec = 500 };388 389 printinfo("starting writer-thread: doing %s writes ...\n",390 randomize? "random":"lineal");391 392 for (iter = 0; !wdone; iter++) {393 if (randomize) {394 shuffle((void *)worker, nthreads, sizeof(*worker));395 }396 397 for (i = 0; i < nthreads; i++) {398 struct worker *w = &worker[i];399 400 if (randomize) {401 shuffle((void *)w->fdmap, nfds, sizeof(int));402 }403 404 for (j = 0; j < nfds; j++) {405 do {406 sz = write(w->fdmap[j], &val, sizeof(val));407 } while (!wdone && (sz < 0 && errno == EAGAIN));408 }409 }410 411 nanosleep(&ts, NULL);412 }413 414 printinfo("exiting writer-thread (total full-loops: %zd)\n", iter);415 return NULL;416}417 418static int cmpworker(const void *p1, const void *p2)419{420 421 struct worker *w1 = (struct worker *) p1;422 struct worker *w2 = (struct worker *) p2;423 return w1->tid > w2->tid;424}425 426int bench_epoll_wait(int argc, const char **argv)427{428 int ret = 0;429 struct sigaction act;430 unsigned int i;431 struct worker *worker = NULL;432 struct perf_cpu_map *cpu;433 pthread_t wthread;434 struct rlimit rl, prevrl;435 436 argc = parse_options(argc, argv, options, bench_epoll_wait_usage, 0);437 if (argc) {438 usage_with_options(bench_epoll_wait_usage, options);439 exit(EXIT_FAILURE);440 }441 442 memset(&act, 0, sizeof(act));443 sigfillset(&act.sa_mask);444 act.sa_sigaction = toggle_done;445 sigaction(SIGINT, &act, NULL);446 447 cpu = perf_cpu_map__new_online_cpus();448 if (!cpu)449 goto errmem;450 451 /* a single, main epoll instance */452 if (!multiq) {453 epollfd = epoll_create(1);454 if (epollfd < 0)455 err(EXIT_FAILURE, "epoll_create");456 457 /*458 * Deal with nested epolls, if any.459 */460 if (nested)461 nest_epollfd(NULL);462 }463 464 printinfo("Using %s queue model\n", multiq ? "multi" : "single");465 printinfo("Nesting level(s): %d\n", nested);466 467 /* default to the number of CPUs and leave one for the writer pthread */468 if (!nthreads)469 nthreads = perf_cpu_map__nr(cpu) - 1;470 471 worker = calloc(nthreads, sizeof(*worker));472 if (!worker) {473 goto errmem;474 }475 476 if (getrlimit(RLIMIT_NOFILE, &prevrl))477 err(EXIT_FAILURE, "getrlimit");478 rl.rlim_cur = rl.rlim_max = nfds * nthreads * 2 + 50;479 printinfo("Setting RLIMIT_NOFILE rlimit from %" PRIu64 " to: %" PRIu64 "\n",480 (uint64_t)prevrl.rlim_max, (uint64_t)rl.rlim_max);481 if (setrlimit(RLIMIT_NOFILE, &rl) < 0)482 err(EXIT_FAILURE, "setrlimit");483 484 printf("Run summary [PID %d]: %d threads monitoring%s on "485 "%d file-descriptors for %d secs.\n\n",486 getpid(), nthreads, oneshot ? " (EPOLLONESHOT semantics)": "", nfds, nsecs);487 488 init_stats(&throughput_stats);489 mutex_init(&thread_lock);490 cond_init(&thread_parent);491 cond_init(&thread_worker);492 493 threads_starting = nthreads;494 495 gettimeofday(&bench__start, NULL);496 497 do_threads(worker, cpu);498 499 mutex_lock(&thread_lock);500 while (threads_starting)501 cond_wait(&thread_parent, &thread_lock);502 cond_broadcast(&thread_worker);503 mutex_unlock(&thread_lock);504 505 /*506 * At this point the workers should be blocked waiting for read events507 * to become ready. Launch the writer which will constantly be writing508 * to each thread's fdmap.509 */510 ret = pthread_create(&wthread, NULL, writerfn,511 (void *)(struct worker *) worker);512 if (ret)513 err(EXIT_FAILURE, "pthread_create");514 515 sleep(nsecs);516 toggle_done(0, NULL, NULL);517 printinfo("main thread: toggling done\n");518 519 sleep(1); /* meh */520 wdone = true;521 ret = pthread_join(wthread, NULL);522 if (ret)523 err(EXIT_FAILURE, "pthread_join");524 525 /* cleanup & report results */526 cond_destroy(&thread_parent);527 cond_destroy(&thread_worker);528 mutex_destroy(&thread_lock);529 530 /* sort the array back before reporting */531 if (randomize)532 qsort(worker, nthreads, sizeof(struct worker), cmpworker);533 534 for (i = 0; i < nthreads; i++) {535 unsigned long t = bench__runtime.tv_sec > 0 ?536 worker[i].ops / bench__runtime.tv_sec : 0;537 538 update_stats(&throughput_stats, t);539 540 if (nfds == 1)541 printf("[thread %2d] fdmap: %p [ %04ld ops/sec ]\n",542 worker[i].tid, &worker[i].fdmap[0], t);543 else544 printf("[thread %2d] fdmap: %p ... %p [ %04ld ops/sec ]\n",545 worker[i].tid, &worker[i].fdmap[0],546 &worker[i].fdmap[nfds-1], t);547 }548 549 print_summary();550 551 close(epollfd);552 perf_cpu_map__put(cpu);553 for (i = 0; i < nthreads; i++)554 free(worker[i].fdmap);555 556 free(worker);557 return ret;558errmem:559 err(EXIT_FAILURE, "calloc");560}561#endif // HAVE_EVENTFD_SUPPORT562