334 lines · c
1#include <stdbool.h>2#include <stddef.h>3#include <stdint.h>4 5#include "blake3_impl.h"6 7#if defined(_MSC_VER)8#include <Windows.h>9#endif10 11#if defined(IS_X86)12#if defined(_MSC_VER)13#include <intrin.h>14#elif defined(__GNUC__)15#include <immintrin.h>16#else17#undef IS_X86 /* Unimplemented! */18#endif19#endif20 21#if !defined(BLAKE3_ATOMICS)22#if defined(__has_include)23#if __has_include(<stdatomic.h>) && !defined(_MSC_VER)24#define BLAKE3_ATOMICS 125#else26#define BLAKE3_ATOMICS 027#endif /* __has_include(<stdatomic.h>) && !defined(_MSC_VER) */28#else29#define BLAKE3_ATOMICS 030#endif /* defined(__has_include) */31#endif /* BLAKE3_ATOMICS */32 33#if BLAKE3_ATOMICS34#define ATOMIC_INT _Atomic int35#define ATOMIC_LOAD(x) x36#define ATOMIC_STORE(x, y) x = y37#elif defined(_MSC_VER)38#define ATOMIC_INT LONG39#define ATOMIC_LOAD(x) InterlockedOr(&x, 0)40#define ATOMIC_STORE(x, y) InterlockedExchange(&x, y)41#else42#define ATOMIC_INT int43#define ATOMIC_LOAD(x) x44#define ATOMIC_STORE(x, y) x = y45#endif46 47#define MAYBE_UNUSED(x) (void)((x))48 49#if defined(IS_X86)50static uint64_t xgetbv(void) {51#if defined(_MSC_VER)52 return _xgetbv(0);53#else54 uint32_t eax = 0, edx = 0;55 __asm__ __volatile__("xgetbv\n" : "=a"(eax), "=d"(edx) : "c"(0));56 return ((uint64_t)edx << 32) | eax;57#endif58}59 60static void cpuid(uint32_t out[4], uint32_t id) {61#if defined(_MSC_VER)62 __cpuid((int *)out, id);63#elif defined(__i386__) || defined(_M_IX86)64 __asm__ __volatile__("movl %%ebx, %1\n"65 "cpuid\n"66 "xchgl %1, %%ebx\n"67 : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3])68 : "a"(id));69#else70 __asm__ __volatile__("cpuid\n"71 : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3])72 : "a"(id));73#endif74}75 76static void cpuidex(uint32_t out[4], uint32_t id, uint32_t sid) {77#if defined(_MSC_VER)78 __cpuidex((int *)out, id, sid);79#elif defined(__i386__) || defined(_M_IX86)80 __asm__ __volatile__("movl %%ebx, %1\n"81 "cpuid\n"82 "xchgl %1, %%ebx\n"83 : "=a"(out[0]), "=r"(out[1]), "=c"(out[2]), "=d"(out[3])84 : "a"(id), "c"(sid));85#else86 __asm__ __volatile__("cpuid\n"87 : "=a"(out[0]), "=b"(out[1]), "=c"(out[2]), "=d"(out[3])88 : "a"(id), "c"(sid));89#endif90}91 92#endif93 94enum cpu_feature {95 SSE2 = 1 << 0,96 SSSE3 = 1 << 1,97 SSE41 = 1 << 2,98 AVX = 1 << 3,99 AVX2 = 1 << 4,100 AVX512F = 1 << 5,101 AVX512VL = 1 << 6,102 /* ... */103 UNDEFINED = 1 << 30104};105 106#if !defined(BLAKE3_TESTING)107static /* Allow the variable to be controlled manually for testing */108#endif109 ATOMIC_INT g_cpu_features = UNDEFINED;110 111LLVM_ATTRIBUTE_USED112#if !defined(BLAKE3_TESTING)113static114#endif115 enum cpu_feature116 get_cpu_features(void) {117 118 /* If TSAN detects a data race here, try compiling with -DBLAKE3_ATOMICS=1 */119 enum cpu_feature features = ATOMIC_LOAD(g_cpu_features);120 if (features != UNDEFINED) {121 return features;122 } else {123#if defined(IS_X86)124 uint32_t regs[4] = {0};125 uint32_t *eax = ®s[0], *ebx = ®s[1], *ecx = ®s[2], *edx = ®s[3];126 (void)edx;127 features = 0;128 cpuid(regs, 0);129 const int max_id = *eax;130 cpuid(regs, 1);131#if defined(__amd64__) || defined(_M_X64)132 features |= SSE2;133#else134 if (*edx & (1UL << 26))135 features |= SSE2;136#endif137 if (*ecx & (1UL << 9))138 features |= SSSE3;139 if (*ecx & (1UL << 19))140 features |= SSE41;141 142 if (*ecx & (1UL << 27)) { // OSXSAVE143 const uint64_t mask = xgetbv();144 if ((mask & 6) == 6) { // SSE and AVX states145 if (*ecx & (1UL << 28))146 features |= AVX;147 if (max_id >= 7) {148 cpuidex(regs, 7, 0);149 if (*ebx & (1UL << 5))150 features |= AVX2;151 if ((mask & 224) == 224) { // Opmask, ZMM_Hi256, Hi16_Zmm152 if (*ebx & (1UL << 31))153 features |= AVX512VL;154 if (*ebx & (1UL << 16))155 features |= AVX512F;156 }157 }158 }159 }160 ATOMIC_STORE(g_cpu_features, features);161 return features;162#else163 /* How to detect NEON? */164 return 0;165#endif166 }167}168 169void blake3_compress_in_place(uint32_t cv[8],170 const uint8_t block[BLAKE3_BLOCK_LEN],171 uint8_t block_len, uint64_t counter,172 uint8_t flags) {173#if defined(IS_X86)174 const enum cpu_feature features = get_cpu_features();175 MAYBE_UNUSED(features);176#if !defined(BLAKE3_NO_AVX512)177 if (features & AVX512VL) {178 blake3_compress_in_place_avx512(cv, block, block_len, counter, flags);179 return;180 }181#endif182#if !defined(BLAKE3_NO_SSE41)183 if (features & SSE41) {184 blake3_compress_in_place_sse41(cv, block, block_len, counter, flags);185 return;186 }187#endif188#if !defined(BLAKE3_NO_SSE2)189 if (features & SSE2) {190 blake3_compress_in_place_sse2(cv, block, block_len, counter, flags);191 return;192 }193#endif194#endif195 blake3_compress_in_place_portable(cv, block, block_len, counter, flags);196}197 198void blake3_compress_xof(const uint32_t cv[8],199 const uint8_t block[BLAKE3_BLOCK_LEN],200 uint8_t block_len, uint64_t counter, uint8_t flags,201 uint8_t out[64]) {202#if defined(IS_X86)203 const enum cpu_feature features = get_cpu_features();204 MAYBE_UNUSED(features);205#if !defined(BLAKE3_NO_AVX512)206 if (features & AVX512VL) {207 blake3_compress_xof_avx512(cv, block, block_len, counter, flags, out);208 return;209 }210#endif211#if !defined(BLAKE3_NO_SSE41)212 if (features & SSE41) {213 blake3_compress_xof_sse41(cv, block, block_len, counter, flags, out);214 return;215 }216#endif217#if !defined(BLAKE3_NO_SSE2)218 if (features & SSE2) {219 blake3_compress_xof_sse2(cv, block, block_len, counter, flags, out);220 return;221 }222#endif223#endif224 blake3_compress_xof_portable(cv, block, block_len, counter, flags, out);225}226 227 228void blake3_xof_many(const uint32_t cv[8],229 const uint8_t block[BLAKE3_BLOCK_LEN],230 uint8_t block_len, uint64_t counter, uint8_t flags,231 uint8_t out[64], size_t outblocks) {232 if (outblocks == 0) {233 // The current assembly implementation always outputs at least 1 block.234 return;235 }236#if defined(IS_X86)237 const enum cpu_feature features = get_cpu_features();238 MAYBE_UNUSED(features);239#if !defined(_WIN32) && !defined(__CYGWIN__) && !defined(BLAKE3_NO_AVX512)240 if (features & AVX512VL) {241 blake3_xof_many_avx512(cv, block, block_len, counter, flags, out, outblocks);242 return;243 }244#endif245#endif246 for(size_t i = 0; i < outblocks; ++i) {247 blake3_compress_xof(cv, block, block_len, counter + i, flags, out + 64*i);248 }249}250 251void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs,252 size_t blocks, const uint32_t key[8], uint64_t counter,253 bool increment_counter, uint8_t flags,254 uint8_t flags_start, uint8_t flags_end, uint8_t *out) {255#if defined(IS_X86)256 const enum cpu_feature features = get_cpu_features();257 MAYBE_UNUSED(features);258#if !defined(BLAKE3_NO_AVX512)259 if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) {260 blake3_hash_many_avx512(inputs, num_inputs, blocks, key, counter,261 increment_counter, flags, flags_start, flags_end,262 out);263 return;264 }265#endif266#if !defined(BLAKE3_NO_AVX2)267 if (features & AVX2) {268 blake3_hash_many_avx2(inputs, num_inputs, blocks, key, counter,269 increment_counter, flags, flags_start, flags_end,270 out);271 return;272 }273#endif274#if !defined(BLAKE3_NO_SSE41)275 if (features & SSE41) {276 blake3_hash_many_sse41(inputs, num_inputs, blocks, key, counter,277 increment_counter, flags, flags_start, flags_end,278 out);279 return;280 }281#endif282#if !defined(BLAKE3_NO_SSE2)283 if (features & SSE2) {284 blake3_hash_many_sse2(inputs, num_inputs, blocks, key, counter,285 increment_counter, flags, flags_start, flags_end,286 out);287 return;288 }289#endif290#endif291 292#if BLAKE3_USE_NEON == 1293 blake3_hash_many_neon(inputs, num_inputs, blocks, key, counter,294 increment_counter, flags, flags_start, flags_end, out);295 return;296#endif297 298 blake3_hash_many_portable(inputs, num_inputs, blocks, key, counter,299 increment_counter, flags, flags_start, flags_end,300 out);301}302 303// The dynamically detected SIMD degree of the current platform.304size_t blake3_simd_degree(void) {305#if defined(IS_X86)306 const enum cpu_feature features = get_cpu_features();307 MAYBE_UNUSED(features);308#if !defined(BLAKE3_NO_AVX512)309 if ((features & (AVX512F|AVX512VL)) == (AVX512F|AVX512VL)) {310 return 16;311 }312#endif313#if !defined(BLAKE3_NO_AVX2)314 if (features & AVX2) {315 return 8;316 }317#endif318#if !defined(BLAKE3_NO_SSE41)319 if (features & SSE41) {320 return 4;321 }322#endif323#if !defined(BLAKE3_NO_SSE2)324 if (features & SSE2) {325 return 4;326 }327#endif328#endif329#if BLAKE3_USE_NEON == 1330 return 4;331#endif332 return 1;333}334