321 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright (C) 2022-2024 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.4 */5 6#include <assert.h>7#include <pthread.h>8#include <stdint.h>9#include <stdio.h>10#include <stdlib.h>11#include <string.h>12#include <time.h>13#include <unistd.h>14#include <sched.h>15#include <signal.h>16#include <sys/auxv.h>17#include <sys/mman.h>18#include <sys/random.h>19#include <sys/syscall.h>20#include <sys/ptrace.h>21#include <sys/wait.h>22#include <sys/types.h>23#include <linux/random.h>24#include <linux/compiler.h>25#include <linux/ptrace.h>26 27#include "../kselftest.h"28#include "parse_vdso.h"29#include "vdso_config.h"30#include "vdso_call.h"31 32#ifndef timespecsub33#define timespecsub(tsp, usp, vsp) \34 do { \35 (vsp)->tv_sec = (tsp)->tv_sec - (usp)->tv_sec; \36 (vsp)->tv_nsec = (tsp)->tv_nsec - (usp)->tv_nsec; \37 if ((vsp)->tv_nsec < 0) { \38 (vsp)->tv_sec--; \39 (vsp)->tv_nsec += 1000000000L; \40 } \41 } while (0)42#endif43 44#define ksft_assert(condition) \45 do { if (!(condition)) ksft_exit_fail_msg("Assertion failed: %s\n", #condition); } while (0)46 47static struct {48 pthread_mutex_t lock;49 void **states;50 size_t len, cap;51 ssize_t(*fn)(void *, size_t, unsigned long, void *, size_t);52 struct vgetrandom_opaque_params params;53} vgrnd = {54 .lock = PTHREAD_MUTEX_INITIALIZER55};56 57static void *vgetrandom_get_state(void)58{59 void *state = NULL;60 61 pthread_mutex_lock(&vgrnd.lock);62 if (!vgrnd.len) {63 size_t page_size = getpagesize();64 size_t new_cap;65 size_t alloc_size, num = sysconf(_SC_NPROCESSORS_ONLN); /* Just a decent heuristic. */66 size_t state_size_aligned, cache_line_size = sysconf(_SC_LEVEL1_DCACHE_LINESIZE) ?: 1;67 void *new_block, *new_states;68 69 state_size_aligned = (vgrnd.params.size_of_opaque_state + cache_line_size - 1) & (~(cache_line_size - 1));70 alloc_size = (num * state_size_aligned + page_size - 1) & (~(page_size - 1));71 num = (page_size / state_size_aligned) * (alloc_size / page_size);72 new_block = mmap(0, alloc_size, vgrnd.params.mmap_prot, vgrnd.params.mmap_flags, -1, 0);73 if (new_block == MAP_FAILED)74 goto out;75 76 new_cap = vgrnd.cap + num;77 new_states = reallocarray(vgrnd.states, new_cap, sizeof(*vgrnd.states));78 if (!new_states)79 goto unmap;80 vgrnd.cap = new_cap;81 vgrnd.states = new_states;82 83 for (size_t i = 0; i < num; ++i) {84 if (((uintptr_t)new_block & (page_size - 1)) + vgrnd.params.size_of_opaque_state > page_size)85 new_block = (void *)(((uintptr_t)new_block + page_size - 1) & (~(page_size - 1)));86 vgrnd.states[i] = new_block;87 new_block += state_size_aligned;88 }89 vgrnd.len = num;90 goto success;91 92 unmap:93 munmap(new_block, alloc_size);94 goto out;95 }96success:97 state = vgrnd.states[--vgrnd.len];98 99out:100 pthread_mutex_unlock(&vgrnd.lock);101 return state;102}103 104static void vgetrandom_put_state(void *state)105{106 if (!state)107 return;108 pthread_mutex_lock(&vgrnd.lock);109 vgrnd.states[vgrnd.len++] = state;110 pthread_mutex_unlock(&vgrnd.lock);111}112 113static void vgetrandom_init(void)114{115 const char *version = versions[VDSO_VERSION];116 const char *name = names[VDSO_NAMES][6];117 unsigned long sysinfo_ehdr = getauxval(AT_SYSINFO_EHDR);118 ssize_t ret;119 120 if (!sysinfo_ehdr)121 ksft_exit_skip("AT_SYSINFO_EHDR is not present\n");122 vdso_init_from_sysinfo_ehdr(sysinfo_ehdr);123 vgrnd.fn = (__typeof__(vgrnd.fn))vdso_sym(version, name);124 if (!vgrnd.fn)125 ksft_exit_skip("%s@%s symbol is missing from vDSO\n", name, version);126 ret = VDSO_CALL(vgrnd.fn, 5, NULL, 0, 0, &vgrnd.params, ~0UL);127 if (ret == -ENOSYS)128 ksft_exit_skip("CPU does not have runtime support\n");129 else if (ret)130 ksft_exit_fail_msg("Failed to fetch vgetrandom params: %zd\n", ret);131}132 133static ssize_t vgetrandom(void *buf, size_t len, unsigned long flags)134{135 static __thread void *state;136 137 if (!state) {138 state = vgetrandom_get_state();139 ksft_assert(state);140 }141 return VDSO_CALL(vgrnd.fn, 5, buf, len, flags, state, vgrnd.params.size_of_opaque_state);142}143 144enum { TRIALS = 25000000, THREADS = 256 };145 146static void *test_vdso_getrandom(void *ctx)147{148 for (size_t i = 0; i < TRIALS; ++i) {149 unsigned int val;150 ssize_t ret = vgetrandom(&val, sizeof(val), 0);151 ksft_assert(ret == sizeof(val));152 }153 return NULL;154}155 156static void *test_libc_getrandom(void *ctx)157{158 for (size_t i = 0; i < TRIALS; ++i) {159 unsigned int val;160 ssize_t ret = getrandom(&val, sizeof(val), 0);161 ksft_assert(ret == sizeof(val));162 }163 return NULL;164}165 166static void *test_syscall_getrandom(void *ctx)167{168 for (size_t i = 0; i < TRIALS; ++i) {169 unsigned int val;170 ssize_t ret = syscall(__NR_getrandom, &val, sizeof(val), 0);171 ksft_assert(ret == sizeof(val));172 }173 return NULL;174}175 176static void bench_single(void)177{178 struct timespec start, end, diff;179 180 clock_gettime(CLOCK_MONOTONIC, &start);181 test_vdso_getrandom(NULL);182 clock_gettime(CLOCK_MONOTONIC, &end);183 timespecsub(&end, &start, &diff);184 printf(" vdso: %u times in %lu.%09lu seconds\n", TRIALS, diff.tv_sec, diff.tv_nsec);185 186 clock_gettime(CLOCK_MONOTONIC, &start);187 test_libc_getrandom(NULL);188 clock_gettime(CLOCK_MONOTONIC, &end);189 timespecsub(&end, &start, &diff);190 printf(" libc: %u times in %lu.%09lu seconds\n", TRIALS, diff.tv_sec, diff.tv_nsec);191 192 clock_gettime(CLOCK_MONOTONIC, &start);193 test_syscall_getrandom(NULL);194 clock_gettime(CLOCK_MONOTONIC, &end);195 timespecsub(&end, &start, &diff);196 printf("syscall: %u times in %lu.%09lu seconds\n", TRIALS, diff.tv_sec, diff.tv_nsec);197}198 199static void bench_multi(void)200{201 struct timespec start, end, diff;202 pthread_t threads[THREADS];203 204 clock_gettime(CLOCK_MONOTONIC, &start);205 for (size_t i = 0; i < THREADS; ++i)206 ksft_assert(pthread_create(&threads[i], NULL, test_vdso_getrandom, NULL) == 0);207 for (size_t i = 0; i < THREADS; ++i)208 pthread_join(threads[i], NULL);209 clock_gettime(CLOCK_MONOTONIC, &end);210 timespecsub(&end, &start, &diff);211 printf(" vdso: %u x %u times in %lu.%09lu seconds\n", TRIALS, THREADS, diff.tv_sec, diff.tv_nsec);212 213 clock_gettime(CLOCK_MONOTONIC, &start);214 for (size_t i = 0; i < THREADS; ++i)215 ksft_assert(pthread_create(&threads[i], NULL, test_libc_getrandom, NULL) == 0);216 for (size_t i = 0; i < THREADS; ++i)217 pthread_join(threads[i], NULL);218 clock_gettime(CLOCK_MONOTONIC, &end);219 timespecsub(&end, &start, &diff);220 printf(" libc: %u x %u times in %lu.%09lu seconds\n", TRIALS, THREADS, diff.tv_sec, diff.tv_nsec);221 222 clock_gettime(CLOCK_MONOTONIC, &start);223 for (size_t i = 0; i < THREADS; ++i)224 ksft_assert(pthread_create(&threads[i], NULL, test_syscall_getrandom, NULL) == 0);225 for (size_t i = 0; i < THREADS; ++i)226 pthread_join(threads[i], NULL);227 clock_gettime(CLOCK_MONOTONIC, &end);228 timespecsub(&end, &start, &diff);229 printf(" syscall: %u x %u times in %lu.%09lu seconds\n", TRIALS, THREADS, diff.tv_sec, diff.tv_nsec);230}231 232static void fill(void)233{234 uint8_t weird_size[323929];235 for (;;)236 vgetrandom(weird_size, sizeof(weird_size), 0);237}238 239static void kselftest(void)240{241 uint8_t weird_size[1263];242 pid_t child;243 244 ksft_print_header();245 ksft_set_plan(2);246 247 for (size_t i = 0; i < 1000; ++i) {248 ssize_t ret = vgetrandom(weird_size, sizeof(weird_size), 0);249 ksft_assert(ret == sizeof(weird_size));250 }251 252 ksft_test_result_pass("getrandom: PASS\n");253 254 unshare(CLONE_NEWUSER);255 ksft_assert(unshare(CLONE_NEWTIME) == 0);256 child = fork();257 ksft_assert(child >= 0);258 if (!child) {259 vgetrandom_init();260 child = getpid();261 ksft_assert(ptrace(PTRACE_TRACEME, 0, NULL, NULL) == 0);262 ksft_assert(kill(child, SIGSTOP) == 0);263 ksft_assert(vgetrandom(weird_size, sizeof(weird_size), 0) == sizeof(weird_size));264 _exit(0);265 }266 for (;;) {267 struct ptrace_syscall_info info = { 0 };268 int status, ret;269 ksft_assert(waitpid(child, &status, 0) >= 0);270 if (WIFEXITED(status)) {271 ksft_assert(WEXITSTATUS(status) == 0);272 break;273 }274 ksft_assert(WIFSTOPPED(status));275 if (WSTOPSIG(status) == SIGSTOP)276 ksft_assert(ptrace(PTRACE_SETOPTIONS, child, 0, PTRACE_O_TRACESYSGOOD) == 0);277 else if (WSTOPSIG(status) == (SIGTRAP | 0x80)) {278 ksft_assert(ptrace(PTRACE_GET_SYSCALL_INFO, child, sizeof(info), &info) > 0);279 if (info.op == PTRACE_SYSCALL_INFO_ENTRY && info.entry.nr == __NR_getrandom &&280 info.entry.args[0] == (uintptr_t)weird_size && info.entry.args[1] == sizeof(weird_size))281 ksft_exit_fail_msg("vgetrandom passed buffer to syscall getrandom unexpectedly\n");282 }283 ksft_assert(ptrace(PTRACE_SYSCALL, child, 0, 0) == 0);284 }285 286 ksft_test_result_pass("getrandom timens: PASS\n");287 288 ksft_exit_pass();289}290 291static void usage(const char *argv0)292{293 fprintf(stderr, "Usage: %s [bench-single|bench-multi|fill]\n", argv0);294}295 296int main(int argc, char *argv[])297{298 vgetrandom_init();299 300 if (argc == 1) {301 kselftest();302 return 0;303 }304 305 if (argc != 2) {306 usage(argv[0]);307 return 1;308 }309 if (!strcmp(argv[1], "bench-single"))310 bench_single();311 else if (!strcmp(argv[1], "bench-multi"))312 bench_multi();313 else if (!strcmp(argv[1], "fill"))314 fill();315 else {316 usage(argv[0]);317 return 1;318 }319 return 0;320}321