185 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR MIT2/*3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.4 *5 * This is based in part on Andrew Moon's poly1305-donna, which is in the6 * public domain.7 */8 9#include <linux/kernel.h>10#include <linux/unaligned.h>11#include <crypto/internal/poly1305.h>12 13void poly1305_core_setkey(struct poly1305_core_key *key,14 const u8 raw_key[POLY1305_BLOCK_SIZE])15{16 u64 t0, t1;17 18 /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */19 t0 = get_unaligned_le64(&raw_key[0]);20 t1 = get_unaligned_le64(&raw_key[8]);21 22 key->key.r64[0] = t0 & 0xffc0fffffffULL;23 key->key.r64[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL;24 key->key.r64[2] = ((t1 >> 24)) & 0x00ffffffc0fULL;25 26 /* s = 20*r */27 key->precomputed_s.r64[0] = key->key.r64[1] * 20;28 key->precomputed_s.r64[1] = key->key.r64[2] * 20;29}30EXPORT_SYMBOL(poly1305_core_setkey);31 32void poly1305_core_blocks(struct poly1305_state *state,33 const struct poly1305_core_key *key, const void *src,34 unsigned int nblocks, u32 hibit)35{36 const u8 *input = src;37 u64 hibit64;38 u64 r0, r1, r2;39 u64 s1, s2;40 u64 h0, h1, h2;41 u64 c;42 u128 d0, d1, d2, d;43 44 if (!nblocks)45 return;46 47 hibit64 = ((u64)hibit) << 40;48 49 r0 = key->key.r64[0];50 r1 = key->key.r64[1];51 r2 = key->key.r64[2];52 53 h0 = state->h64[0];54 h1 = state->h64[1];55 h2 = state->h64[2];56 57 s1 = key->precomputed_s.r64[0];58 s2 = key->precomputed_s.r64[1];59 60 do {61 u64 t0, t1;62 63 /* h += m[i] */64 t0 = get_unaligned_le64(&input[0]);65 t1 = get_unaligned_le64(&input[8]);66 67 h0 += t0 & 0xfffffffffffULL;68 h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL;69 h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit64;70 71 /* h *= r */72 d0 = (u128)h0 * r0;73 d = (u128)h1 * s2;74 d0 += d;75 d = (u128)h2 * s1;76 d0 += d;77 d1 = (u128)h0 * r1;78 d = (u128)h1 * r0;79 d1 += d;80 d = (u128)h2 * s2;81 d1 += d;82 d2 = (u128)h0 * r2;83 d = (u128)h1 * r1;84 d2 += d;85 d = (u128)h2 * r0;86 d2 += d;87 88 /* (partial) h %= p */89 c = (u64)(d0 >> 44);90 h0 = (u64)d0 & 0xfffffffffffULL;91 d1 += c;92 c = (u64)(d1 >> 44);93 h1 = (u64)d1 & 0xfffffffffffULL;94 d2 += c;95 c = (u64)(d2 >> 42);96 h2 = (u64)d2 & 0x3ffffffffffULL;97 h0 += c * 5;98 c = h0 >> 44;99 h0 = h0 & 0xfffffffffffULL;100 h1 += c;101 102 input += POLY1305_BLOCK_SIZE;103 } while (--nblocks);104 105 state->h64[0] = h0;106 state->h64[1] = h1;107 state->h64[2] = h2;108}109EXPORT_SYMBOL(poly1305_core_blocks);110 111void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4],112 void *dst)113{114 u8 *mac = dst;115 u64 h0, h1, h2, c;116 u64 g0, g1, g2;117 u64 t0, t1;118 119 /* fully carry h */120 h0 = state->h64[0];121 h1 = state->h64[1];122 h2 = state->h64[2];123 124 c = h1 >> 44;125 h1 &= 0xfffffffffffULL;126 h2 += c;127 c = h2 >> 42;128 h2 &= 0x3ffffffffffULL;129 h0 += c * 5;130 c = h0 >> 44;131 h0 &= 0xfffffffffffULL;132 h1 += c;133 c = h1 >> 44;134 h1 &= 0xfffffffffffULL;135 h2 += c;136 c = h2 >> 42;137 h2 &= 0x3ffffffffffULL;138 h0 += c * 5;139 c = h0 >> 44;140 h0 &= 0xfffffffffffULL;141 h1 += c;142 143 /* compute h + -p */144 g0 = h0 + 5;145 c = g0 >> 44;146 g0 &= 0xfffffffffffULL;147 g1 = h1 + c;148 c = g1 >> 44;149 g1 &= 0xfffffffffffULL;150 g2 = h2 + c - (1ULL << 42);151 152 /* select h if h < p, or h + -p if h >= p */153 c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1;154 g0 &= c;155 g1 &= c;156 g2 &= c;157 c = ~c;158 h0 = (h0 & c) | g0;159 h1 = (h1 & c) | g1;160 h2 = (h2 & c) | g2;161 162 if (likely(nonce)) {163 /* h = (h + nonce) */164 t0 = ((u64)nonce[1] << 32) | nonce[0];165 t1 = ((u64)nonce[3] << 32) | nonce[2];166 167 h0 += t0 & 0xfffffffffffULL;168 c = h0 >> 44;169 h0 &= 0xfffffffffffULL;170 h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c;171 c = h1 >> 44;172 h1 &= 0xfffffffffffULL;173 h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c;174 h2 &= 0x3ffffffffffULL;175 }176 177 /* mac = h % (2^128) */178 h0 = h0 | (h1 << 44);179 h1 = (h1 >> 20) | (h2 << 24);180 181 put_unaligned_le64(h0, &mac[0]);182 put_unaligned_le64(h1, &mac[8]);183}184EXPORT_SYMBOL(poly1305_core_emit);185