539 lines · c
1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)2/* Copyright (C) 2016-2022 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.3 *4 * SipHash: a fast short-input PRF5 * https://131002.net/siphash/6 *7 * This implementation is specifically for SipHash2-4 for a secure PRF8 * and HalfSipHash1-3/SipHash1-3 for an insecure PRF only suitable for9 * hashtables.10 */11 12#include <linux/siphash.h>13#include <linux/unaligned.h>14 15#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 6416#include <linux/dcache.h>17#include <asm/word-at-a-time.h>18#endif19 20#define SIPROUND SIPHASH_PERMUTATION(v0, v1, v2, v3)21 22#define PREAMBLE(len) \23 u64 v0 = SIPHASH_CONST_0; \24 u64 v1 = SIPHASH_CONST_1; \25 u64 v2 = SIPHASH_CONST_2; \26 u64 v3 = SIPHASH_CONST_3; \27 u64 b = ((u64)(len)) << 56; \28 v3 ^= key->key[1]; \29 v2 ^= key->key[0]; \30 v1 ^= key->key[1]; \31 v0 ^= key->key[0];32 33#define POSTAMBLE \34 v3 ^= b; \35 SIPROUND; \36 SIPROUND; \37 v0 ^= b; \38 v2 ^= 0xff; \39 SIPROUND; \40 SIPROUND; \41 SIPROUND; \42 SIPROUND; \43 return (v0 ^ v1) ^ (v2 ^ v3);44 45#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS46u64 __siphash_aligned(const void *data, size_t len, const siphash_key_t *key)47{48 const u8 *end = data + len - (len % sizeof(u64));49 const u8 left = len & (sizeof(u64) - 1);50 u64 m;51 PREAMBLE(len)52 for (; data != end; data += sizeof(u64)) {53 m = le64_to_cpup(data);54 v3 ^= m;55 SIPROUND;56 SIPROUND;57 v0 ^= m;58 }59#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 6460 if (left)61 b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &62 bytemask_from_count(left)));63#else64 switch (left) {65 case 7: b |= ((u64)end[6]) << 48; fallthrough;66 case 6: b |= ((u64)end[5]) << 40; fallthrough;67 case 5: b |= ((u64)end[4]) << 32; fallthrough;68 case 4: b |= le32_to_cpup(data); break;69 case 3: b |= ((u64)end[2]) << 16; fallthrough;70 case 2: b |= le16_to_cpup(data); break;71 case 1: b |= end[0];72 }73#endif74 POSTAMBLE75}76EXPORT_SYMBOL(__siphash_aligned);77#endif78 79u64 __siphash_unaligned(const void *data, size_t len, const siphash_key_t *key)80{81 const u8 *end = data + len - (len % sizeof(u64));82 const u8 left = len & (sizeof(u64) - 1);83 u64 m;84 PREAMBLE(len)85 for (; data != end; data += sizeof(u64)) {86 m = get_unaligned_le64(data);87 v3 ^= m;88 SIPROUND;89 SIPROUND;90 v0 ^= m;91 }92#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 6493 if (left)94 b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &95 bytemask_from_count(left)));96#else97 switch (left) {98 case 7: b |= ((u64)end[6]) << 48; fallthrough;99 case 6: b |= ((u64)end[5]) << 40; fallthrough;100 case 5: b |= ((u64)end[4]) << 32; fallthrough;101 case 4: b |= get_unaligned_le32(end); break;102 case 3: b |= ((u64)end[2]) << 16; fallthrough;103 case 2: b |= get_unaligned_le16(end); break;104 case 1: b |= end[0];105 }106#endif107 POSTAMBLE108}109EXPORT_SYMBOL(__siphash_unaligned);110 111/**112 * siphash_1u64 - compute 64-bit siphash PRF value of a u64113 * @first: first u64114 * @key: the siphash key115 */116u64 siphash_1u64(const u64 first, const siphash_key_t *key)117{118 PREAMBLE(8)119 v3 ^= first;120 SIPROUND;121 SIPROUND;122 v0 ^= first;123 POSTAMBLE124}125EXPORT_SYMBOL(siphash_1u64);126 127/**128 * siphash_2u64 - compute 64-bit siphash PRF value of 2 u64129 * @first: first u64130 * @second: second u64131 * @key: the siphash key132 */133u64 siphash_2u64(const u64 first, const u64 second, const siphash_key_t *key)134{135 PREAMBLE(16)136 v3 ^= first;137 SIPROUND;138 SIPROUND;139 v0 ^= first;140 v3 ^= second;141 SIPROUND;142 SIPROUND;143 v0 ^= second;144 POSTAMBLE145}146EXPORT_SYMBOL(siphash_2u64);147 148/**149 * siphash_3u64 - compute 64-bit siphash PRF value of 3 u64150 * @first: first u64151 * @second: second u64152 * @third: third u64153 * @key: the siphash key154 */155u64 siphash_3u64(const u64 first, const u64 second, const u64 third,156 const siphash_key_t *key)157{158 PREAMBLE(24)159 v3 ^= first;160 SIPROUND;161 SIPROUND;162 v0 ^= first;163 v3 ^= second;164 SIPROUND;165 SIPROUND;166 v0 ^= second;167 v3 ^= third;168 SIPROUND;169 SIPROUND;170 v0 ^= third;171 POSTAMBLE172}173EXPORT_SYMBOL(siphash_3u64);174 175/**176 * siphash_4u64 - compute 64-bit siphash PRF value of 4 u64177 * @first: first u64178 * @second: second u64179 * @third: third u64180 * @forth: forth u64181 * @key: the siphash key182 */183u64 siphash_4u64(const u64 first, const u64 second, const u64 third,184 const u64 forth, const siphash_key_t *key)185{186 PREAMBLE(32)187 v3 ^= first;188 SIPROUND;189 SIPROUND;190 v0 ^= first;191 v3 ^= second;192 SIPROUND;193 SIPROUND;194 v0 ^= second;195 v3 ^= third;196 SIPROUND;197 SIPROUND;198 v0 ^= third;199 v3 ^= forth;200 SIPROUND;201 SIPROUND;202 v0 ^= forth;203 POSTAMBLE204}205EXPORT_SYMBOL(siphash_4u64);206 207u64 siphash_1u32(const u32 first, const siphash_key_t *key)208{209 PREAMBLE(4)210 b |= first;211 POSTAMBLE212}213EXPORT_SYMBOL(siphash_1u32);214 215u64 siphash_3u32(const u32 first, const u32 second, const u32 third,216 const siphash_key_t *key)217{218 u64 combined = (u64)second << 32 | first;219 PREAMBLE(12)220 v3 ^= combined;221 SIPROUND;222 SIPROUND;223 v0 ^= combined;224 b |= third;225 POSTAMBLE226}227EXPORT_SYMBOL(siphash_3u32);228 229#if BITS_PER_LONG == 64230/* Note that on 64-bit, we make HalfSipHash1-3 actually be SipHash1-3, for231 * performance reasons. On 32-bit, below, we actually implement HalfSipHash1-3.232 */233 234#define HSIPROUND SIPROUND235#define HPREAMBLE(len) PREAMBLE(len)236#define HPOSTAMBLE \237 v3 ^= b; \238 HSIPROUND; \239 v0 ^= b; \240 v2 ^= 0xff; \241 HSIPROUND; \242 HSIPROUND; \243 HSIPROUND; \244 return (v0 ^ v1) ^ (v2 ^ v3);245 246#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS247u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key)248{249 const u8 *end = data + len - (len % sizeof(u64));250 const u8 left = len & (sizeof(u64) - 1);251 u64 m;252 HPREAMBLE(len)253 for (; data != end; data += sizeof(u64)) {254 m = le64_to_cpup(data);255 v3 ^= m;256 HSIPROUND;257 v0 ^= m;258 }259#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64260 if (left)261 b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &262 bytemask_from_count(left)));263#else264 switch (left) {265 case 7: b |= ((u64)end[6]) << 48; fallthrough;266 case 6: b |= ((u64)end[5]) << 40; fallthrough;267 case 5: b |= ((u64)end[4]) << 32; fallthrough;268 case 4: b |= le32_to_cpup(data); break;269 case 3: b |= ((u64)end[2]) << 16; fallthrough;270 case 2: b |= le16_to_cpup(data); break;271 case 1: b |= end[0];272 }273#endif274 HPOSTAMBLE275}276EXPORT_SYMBOL(__hsiphash_aligned);277#endif278 279u32 __hsiphash_unaligned(const void *data, size_t len,280 const hsiphash_key_t *key)281{282 const u8 *end = data + len - (len % sizeof(u64));283 const u8 left = len & (sizeof(u64) - 1);284 u64 m;285 HPREAMBLE(len)286 for (; data != end; data += sizeof(u64)) {287 m = get_unaligned_le64(data);288 v3 ^= m;289 HSIPROUND;290 v0 ^= m;291 }292#if defined(CONFIG_DCACHE_WORD_ACCESS) && BITS_PER_LONG == 64293 if (left)294 b |= le64_to_cpu((__force __le64)(load_unaligned_zeropad(data) &295 bytemask_from_count(left)));296#else297 switch (left) {298 case 7: b |= ((u64)end[6]) << 48; fallthrough;299 case 6: b |= ((u64)end[5]) << 40; fallthrough;300 case 5: b |= ((u64)end[4]) << 32; fallthrough;301 case 4: b |= get_unaligned_le32(end); break;302 case 3: b |= ((u64)end[2]) << 16; fallthrough;303 case 2: b |= get_unaligned_le16(end); break;304 case 1: b |= end[0];305 }306#endif307 HPOSTAMBLE308}309EXPORT_SYMBOL(__hsiphash_unaligned);310 311/**312 * hsiphash_1u32 - compute 64-bit hsiphash PRF value of a u32313 * @first: first u32314 * @key: the hsiphash key315 */316u32 hsiphash_1u32(const u32 first, const hsiphash_key_t *key)317{318 HPREAMBLE(4)319 b |= first;320 HPOSTAMBLE321}322EXPORT_SYMBOL(hsiphash_1u32);323 324/**325 * hsiphash_2u32 - compute 32-bit hsiphash PRF value of 2 u32326 * @first: first u32327 * @second: second u32328 * @key: the hsiphash key329 */330u32 hsiphash_2u32(const u32 first, const u32 second, const hsiphash_key_t *key)331{332 u64 combined = (u64)second << 32 | first;333 HPREAMBLE(8)334 v3 ^= combined;335 HSIPROUND;336 v0 ^= combined;337 HPOSTAMBLE338}339EXPORT_SYMBOL(hsiphash_2u32);340 341/**342 * hsiphash_3u32 - compute 32-bit hsiphash PRF value of 3 u32343 * @first: first u32344 * @second: second u32345 * @third: third u32346 * @key: the hsiphash key347 */348u32 hsiphash_3u32(const u32 first, const u32 second, const u32 third,349 const hsiphash_key_t *key)350{351 u64 combined = (u64)second << 32 | first;352 HPREAMBLE(12)353 v3 ^= combined;354 HSIPROUND;355 v0 ^= combined;356 b |= third;357 HPOSTAMBLE358}359EXPORT_SYMBOL(hsiphash_3u32);360 361/**362 * hsiphash_4u32 - compute 32-bit hsiphash PRF value of 4 u32363 * @first: first u32364 * @second: second u32365 * @third: third u32366 * @forth: forth u32367 * @key: the hsiphash key368 */369u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third,370 const u32 forth, const hsiphash_key_t *key)371{372 u64 combined = (u64)second << 32 | first;373 HPREAMBLE(16)374 v3 ^= combined;375 HSIPROUND;376 v0 ^= combined;377 combined = (u64)forth << 32 | third;378 v3 ^= combined;379 HSIPROUND;380 v0 ^= combined;381 HPOSTAMBLE382}383EXPORT_SYMBOL(hsiphash_4u32);384#else385#define HSIPROUND HSIPHASH_PERMUTATION(v0, v1, v2, v3)386 387#define HPREAMBLE(len) \388 u32 v0 = HSIPHASH_CONST_0; \389 u32 v1 = HSIPHASH_CONST_1; \390 u32 v2 = HSIPHASH_CONST_2; \391 u32 v3 = HSIPHASH_CONST_3; \392 u32 b = ((u32)(len)) << 24; \393 v3 ^= key->key[1]; \394 v2 ^= key->key[0]; \395 v1 ^= key->key[1]; \396 v0 ^= key->key[0];397 398#define HPOSTAMBLE \399 v3 ^= b; \400 HSIPROUND; \401 v0 ^= b; \402 v2 ^= 0xff; \403 HSIPROUND; \404 HSIPROUND; \405 HSIPROUND; \406 return v1 ^ v3;407 408#ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS409u32 __hsiphash_aligned(const void *data, size_t len, const hsiphash_key_t *key)410{411 const u8 *end = data + len - (len % sizeof(u32));412 const u8 left = len & (sizeof(u32) - 1);413 u32 m;414 HPREAMBLE(len)415 for (; data != end; data += sizeof(u32)) {416 m = le32_to_cpup(data);417 v3 ^= m;418 HSIPROUND;419 v0 ^= m;420 }421 switch (left) {422 case 3: b |= ((u32)end[2]) << 16; fallthrough;423 case 2: b |= le16_to_cpup(data); break;424 case 1: b |= end[0];425 }426 HPOSTAMBLE427}428EXPORT_SYMBOL(__hsiphash_aligned);429#endif430 431u32 __hsiphash_unaligned(const void *data, size_t len,432 const hsiphash_key_t *key)433{434 const u8 *end = data + len - (len % sizeof(u32));435 const u8 left = len & (sizeof(u32) - 1);436 u32 m;437 HPREAMBLE(len)438 for (; data != end; data += sizeof(u32)) {439 m = get_unaligned_le32(data);440 v3 ^= m;441 HSIPROUND;442 v0 ^= m;443 }444 switch (left) {445 case 3: b |= ((u32)end[2]) << 16; fallthrough;446 case 2: b |= get_unaligned_le16(end); break;447 case 1: b |= end[0];448 }449 HPOSTAMBLE450}451EXPORT_SYMBOL(__hsiphash_unaligned);452 453/**454 * hsiphash_1u32 - compute 32-bit hsiphash PRF value of a u32455 * @first: first u32456 * @key: the hsiphash key457 */458u32 hsiphash_1u32(const u32 first, const hsiphash_key_t *key)459{460 HPREAMBLE(4)461 v3 ^= first;462 HSIPROUND;463 v0 ^= first;464 HPOSTAMBLE465}466EXPORT_SYMBOL(hsiphash_1u32);467 468/**469 * hsiphash_2u32 - compute 32-bit hsiphash PRF value of 2 u32470 * @first: first u32471 * @second: second u32472 * @key: the hsiphash key473 */474u32 hsiphash_2u32(const u32 first, const u32 second, const hsiphash_key_t *key)475{476 HPREAMBLE(8)477 v3 ^= first;478 HSIPROUND;479 v0 ^= first;480 v3 ^= second;481 HSIPROUND;482 v0 ^= second;483 HPOSTAMBLE484}485EXPORT_SYMBOL(hsiphash_2u32);486 487/**488 * hsiphash_3u32 - compute 32-bit hsiphash PRF value of 3 u32489 * @first: first u32490 * @second: second u32491 * @third: third u32492 * @key: the hsiphash key493 */494u32 hsiphash_3u32(const u32 first, const u32 second, const u32 third,495 const hsiphash_key_t *key)496{497 HPREAMBLE(12)498 v3 ^= first;499 HSIPROUND;500 v0 ^= first;501 v3 ^= second;502 HSIPROUND;503 v0 ^= second;504 v3 ^= third;505 HSIPROUND;506 v0 ^= third;507 HPOSTAMBLE508}509EXPORT_SYMBOL(hsiphash_3u32);510 511/**512 * hsiphash_4u32 - compute 32-bit hsiphash PRF value of 4 u32513 * @first: first u32514 * @second: second u32515 * @third: third u32516 * @forth: forth u32517 * @key: the hsiphash key518 */519u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third,520 const u32 forth, const hsiphash_key_t *key)521{522 HPREAMBLE(16)523 v3 ^= first;524 HSIPROUND;525 v0 ^= first;526 v3 ^= second;527 HSIPROUND;528 v0 ^= second;529 v3 ^= third;530 HSIPROUND;531 v0 ^= third;532 v3 ^= forth;533 HSIPROUND;534 v0 ^= forth;535 HPOSTAMBLE536}537EXPORT_SYMBOL(hsiphash_4u32);538#endif539