348 lines · c
1#ifndef BLAKE3_IMPL_H2#define BLAKE3_IMPL_H3 4#include <assert.h>5#include <stdbool.h>6#include <stddef.h>7#include <stdint.h>8#include <string.h>9 10#include "llvm-c/blake3.h"11// For \p LLVM_LIBRARY_VISIBILITY12#include "llvm/Support/Compiler.h"13 14#include "llvm_blake3_prefix.h"15 16#define BLAKE3_PRIVATE17 18// internal flags19enum blake3_flags {20 CHUNK_START = 1 << 0,21 CHUNK_END = 1 << 1,22 PARENT = 1 << 2,23 ROOT = 1 << 3,24 KEYED_HASH = 1 << 4,25 DERIVE_KEY_CONTEXT = 1 << 5,26 DERIVE_KEY_MATERIAL = 1 << 6,27};28 29// This C implementation tries to support recent versions of GCC, Clang, and30// MSVC.31#if defined(_MSC_VER)32#define INLINE static __forceinline33#else34#define INLINE static inline __attribute__((always_inline))35#endif36 37#if (defined(__x86_64__) || defined(_M_X64)) && !defined(_M_ARM64EC)38#define IS_X8639#define IS_X86_6440#endif41 42#if defined(__i386__) || defined(_M_IX86)43#define IS_X8644#define IS_X86_3245#endif46 47#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)48#define IS_AARCH6449#endif50 51#if defined(IS_X86)52#if defined(_MSC_VER)53#include <intrin.h>54#endif55#include <immintrin.h>56#endif57 58#if !defined(BLAKE3_USE_NEON) 59 // If BLAKE3_USE_NEON not manually set, autodetect based on AArch64ness60 #if defined(IS_AARCH64)61 #if defined(__ARM_BIG_ENDIAN)62 #define BLAKE3_USE_NEON 063 #else64 #define BLAKE3_USE_NEON 165 #endif66 #else67 #define BLAKE3_USE_NEON 068 #endif69#endif70 71#if defined(IS_X86)72#define MAX_SIMD_DEGREE 1673#elif BLAKE3_USE_NEON == 174#define MAX_SIMD_DEGREE 475#else76#define MAX_SIMD_DEGREE 177#endif78 79// There are some places where we want a static size that's equal to the80// MAX_SIMD_DEGREE, but also at least 2.81#define MAX_SIMD_DEGREE_OR_2 (MAX_SIMD_DEGREE > 2 ? MAX_SIMD_DEGREE : 2)82 83static const uint32_t IV[8] = {0x6A09E667UL, 0xBB67AE85UL, 0x3C6EF372UL,84 0xA54FF53AUL, 0x510E527FUL, 0x9B05688CUL,85 0x1F83D9ABUL, 0x5BE0CD19UL};86 87static const uint8_t MSG_SCHEDULE[7][16] = {88 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},89 {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8},90 {3, 4, 10, 12, 13, 2, 7, 14, 6, 5, 9, 0, 11, 15, 8, 1},91 {10, 7, 12, 9, 14, 3, 13, 15, 4, 0, 11, 2, 5, 8, 1, 6},92 {12, 13, 9, 11, 15, 10, 14, 8, 7, 2, 5, 3, 0, 1, 6, 4},93 {9, 14, 11, 5, 8, 12, 15, 1, 13, 3, 0, 10, 2, 6, 4, 7},94 {11, 15, 5, 0, 1, 9, 8, 6, 14, 10, 2, 12, 3, 4, 7, 13},95};96 97/* Find index of the highest set bit */98/* x is assumed to be nonzero. */99static unsigned int highest_one(uint64_t x) {100#if defined(__GNUC__) || defined(__clang__)101 return 63 ^ (unsigned int)__builtin_clzll(x);102#elif defined(_MSC_VER) && defined(IS_X86_64)103 unsigned long index;104 _BitScanReverse64(&index, x);105 return index;106#elif defined(_MSC_VER) && defined(IS_X86_32)107 if(x >> 32) {108 unsigned long index;109 _BitScanReverse(&index, (unsigned long)(x >> 32));110 return 32 + index;111 } else {112 unsigned long index;113 _BitScanReverse(&index, (unsigned long)x);114 return index;115 }116#else117 unsigned int c = 0;118 if(x & 0xffffffff00000000ULL) { x >>= 32; c += 32; }119 if(x & 0x00000000ffff0000ULL) { x >>= 16; c += 16; }120 if(x & 0x000000000000ff00ULL) { x >>= 8; c += 8; }121 if(x & 0x00000000000000f0ULL) { x >>= 4; c += 4; }122 if(x & 0x000000000000000cULL) { x >>= 2; c += 2; }123 if(x & 0x0000000000000002ULL) { c += 1; }124 return c;125#endif126}127 128// Count the number of 1 bits.129INLINE unsigned int popcnt(uint64_t x) {130#if defined(__GNUC__) || defined(__clang__)131 return (unsigned int)__builtin_popcountll(x);132#else133 unsigned int count = 0;134 while (x != 0) {135 count += 1;136 x &= x - 1;137 }138 return count;139#endif140}141 142// Largest power of two less than or equal to x. As a special case, returns 1143// when x is 0. 144INLINE uint64_t round_down_to_power_of_2(uint64_t x) {145 return 1ULL << highest_one(x | 1);146}147 148INLINE uint32_t counter_low(uint64_t counter) { return (uint32_t)counter; }149 150INLINE uint32_t counter_high(uint64_t counter) {151 return (uint32_t)(counter >> 32);152}153 154INLINE uint32_t load32(const void *src) {155 const uint8_t *p = (const uint8_t *)src;156 return ((uint32_t)(p[0]) << 0) | ((uint32_t)(p[1]) << 8) |157 ((uint32_t)(p[2]) << 16) | ((uint32_t)(p[3]) << 24);158}159 160INLINE void load_key_words(const uint8_t key[BLAKE3_KEY_LEN],161 uint32_t key_words[8]) {162 key_words[0] = load32(&key[0 * 4]);163 key_words[1] = load32(&key[1 * 4]);164 key_words[2] = load32(&key[2 * 4]);165 key_words[3] = load32(&key[3 * 4]);166 key_words[4] = load32(&key[4 * 4]);167 key_words[5] = load32(&key[5 * 4]);168 key_words[6] = load32(&key[6 * 4]);169 key_words[7] = load32(&key[7 * 4]);170}171 172INLINE void load_block_words(const uint8_t block[BLAKE3_BLOCK_LEN],173 uint32_t block_words[16]) {174 for (size_t i = 0; i < 16; i++) {175 block_words[i] = load32(&block[i * 4]);176 }177}178 179INLINE void store32(void *dst, uint32_t w) {180 uint8_t *p = (uint8_t *)dst;181 p[0] = (uint8_t)(w >> 0);182 p[1] = (uint8_t)(w >> 8);183 p[2] = (uint8_t)(w >> 16);184 p[3] = (uint8_t)(w >> 24);185}186 187INLINE void store_cv_words(uint8_t bytes_out[32], uint32_t cv_words[8]) {188 store32(&bytes_out[0 * 4], cv_words[0]);189 store32(&bytes_out[1 * 4], cv_words[1]);190 store32(&bytes_out[2 * 4], cv_words[2]);191 store32(&bytes_out[3 * 4], cv_words[3]);192 store32(&bytes_out[4 * 4], cv_words[4]);193 store32(&bytes_out[5 * 4], cv_words[5]);194 store32(&bytes_out[6 * 4], cv_words[6]);195 store32(&bytes_out[7 * 4], cv_words[7]);196}197 198LLVM_LIBRARY_VISIBILITY199void blake3_compress_in_place(uint32_t cv[8],200 const uint8_t block[BLAKE3_BLOCK_LEN],201 uint8_t block_len, uint64_t counter,202 uint8_t flags);203 204LLVM_LIBRARY_VISIBILITY205void blake3_compress_xof(const uint32_t cv[8],206 const uint8_t block[BLAKE3_BLOCK_LEN],207 uint8_t block_len, uint64_t counter, uint8_t flags,208 uint8_t out[64]);209 210LLVM_LIBRARY_VISIBILITY211void blake3_xof_many(const uint32_t cv[8],212 const uint8_t block[BLAKE3_BLOCK_LEN],213 uint8_t block_len, uint64_t counter, uint8_t flags,214 uint8_t out[64], size_t outblocks);215 216LLVM_LIBRARY_VISIBILITY217void blake3_hash_many(const uint8_t *const *inputs, size_t num_inputs,218 size_t blocks, const uint32_t key[8], uint64_t counter,219 bool increment_counter, uint8_t flags,220 uint8_t flags_start, uint8_t flags_end, uint8_t *out);221 222LLVM_LIBRARY_VISIBILITY223size_t blake3_simd_degree(void);224 225BLAKE3_PRIVATE size_t blake3_compress_subtree_wide(const uint8_t *input, size_t input_len,226 const uint32_t key[8],227 uint64_t chunk_counter, uint8_t flags,228 uint8_t *out, bool use_tbb);229 230#if defined(BLAKE3_USE_TBB)231BLAKE3_PRIVATE void blake3_compress_subtree_wide_join_tbb(232 // shared params233 const uint32_t key[8], uint8_t flags, bool use_tbb,234 // left-hand side params235 const uint8_t *l_input, size_t l_input_len, uint64_t l_chunk_counter,236 uint8_t *l_cvs, size_t *l_n,237 // right-hand side params238 const uint8_t *r_input, size_t r_input_len, uint64_t r_chunk_counter,239 uint8_t *r_cvs, size_t *r_n) NOEXCEPT;240#endif241 242// Declarations for implementation-specific functions.243LLVM_LIBRARY_VISIBILITY244void blake3_compress_in_place_portable(uint32_t cv[8],245 const uint8_t block[BLAKE3_BLOCK_LEN],246 uint8_t block_len, uint64_t counter,247 uint8_t flags);248 249LLVM_LIBRARY_VISIBILITY250void blake3_compress_xof_portable(const uint32_t cv[8],251 const uint8_t block[BLAKE3_BLOCK_LEN],252 uint8_t block_len, uint64_t counter,253 uint8_t flags, uint8_t out[64]);254 255LLVM_LIBRARY_VISIBILITY256void blake3_hash_many_portable(const uint8_t *const *inputs, size_t num_inputs,257 size_t blocks, const uint32_t key[8],258 uint64_t counter, bool increment_counter,259 uint8_t flags, uint8_t flags_start,260 uint8_t flags_end, uint8_t *out);261 262#if defined(IS_X86)263#if !defined(BLAKE3_NO_SSE2)264LLVM_LIBRARY_VISIBILITY265void blake3_compress_in_place_sse2(uint32_t cv[8],266 const uint8_t block[BLAKE3_BLOCK_LEN],267 uint8_t block_len, uint64_t counter,268 uint8_t flags);269LLVM_LIBRARY_VISIBILITY270void blake3_compress_xof_sse2(const uint32_t cv[8],271 const uint8_t block[BLAKE3_BLOCK_LEN],272 uint8_t block_len, uint64_t counter,273 uint8_t flags, uint8_t out[64]);274LLVM_LIBRARY_VISIBILITY275void blake3_hash_many_sse2(const uint8_t *const *inputs, size_t num_inputs,276 size_t blocks, const uint32_t key[8],277 uint64_t counter, bool increment_counter,278 uint8_t flags, uint8_t flags_start,279 uint8_t flags_end, uint8_t *out);280#endif281#if !defined(BLAKE3_NO_SSE41)282LLVM_LIBRARY_VISIBILITY283void blake3_compress_in_place_sse41(uint32_t cv[8],284 const uint8_t block[BLAKE3_BLOCK_LEN],285 uint8_t block_len, uint64_t counter,286 uint8_t flags);287LLVM_LIBRARY_VISIBILITY288void blake3_compress_xof_sse41(const uint32_t cv[8],289 const uint8_t block[BLAKE3_BLOCK_LEN],290 uint8_t block_len, uint64_t counter,291 uint8_t flags, uint8_t out[64]);292LLVM_LIBRARY_VISIBILITY293void blake3_hash_many_sse41(const uint8_t *const *inputs, size_t num_inputs,294 size_t blocks, const uint32_t key[8],295 uint64_t counter, bool increment_counter,296 uint8_t flags, uint8_t flags_start,297 uint8_t flags_end, uint8_t *out);298#endif299#if !defined(BLAKE3_NO_AVX2)300LLVM_LIBRARY_VISIBILITY301void blake3_hash_many_avx2(const uint8_t *const *inputs, size_t num_inputs,302 size_t blocks, const uint32_t key[8],303 uint64_t counter, bool increment_counter,304 uint8_t flags, uint8_t flags_start,305 uint8_t flags_end, uint8_t *out);306#endif307#if !defined(BLAKE3_NO_AVX512)308LLVM_LIBRARY_VISIBILITY309void blake3_compress_in_place_avx512(uint32_t cv[8],310 const uint8_t block[BLAKE3_BLOCK_LEN],311 uint8_t block_len, uint64_t counter,312 uint8_t flags);313 314LLVM_LIBRARY_VISIBILITY315void blake3_compress_xof_avx512(const uint32_t cv[8],316 const uint8_t block[BLAKE3_BLOCK_LEN],317 uint8_t block_len, uint64_t counter,318 uint8_t flags, uint8_t out[64]);319 320LLVM_LIBRARY_VISIBILITY321void blake3_hash_many_avx512(const uint8_t *const *inputs, size_t num_inputs,322 size_t blocks, const uint32_t key[8],323 uint64_t counter, bool increment_counter,324 uint8_t flags, uint8_t flags_start,325 uint8_t flags_end, uint8_t *out);326 327#if !defined(_WIN32) && !defined(__CYGWIN__)328LLVM_LIBRARY_VISIBILITY329void blake3_xof_many_avx512(const uint32_t cv[8],330 const uint8_t block[BLAKE3_BLOCK_LEN],331 uint8_t block_len, uint64_t counter, uint8_t flags,332 uint8_t* out, size_t outblocks);333#endif334#endif335#endif336 337#if BLAKE3_USE_NEON == 1338LLVM_LIBRARY_VISIBILITY339void blake3_hash_many_neon(const uint8_t *const *inputs, size_t num_inputs,340 size_t blocks, const uint32_t key[8],341 uint64_t counter, bool increment_counter,342 uint8_t flags, uint8_t flags_start,343 uint8_t flags_end, uint8_t *out);344#endif345 346 347#endif /* BLAKE3_IMPL_H */348