374 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR MIT2/*3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.4 *5 * This is an implementation of the ChaCha20Poly1305 AEAD construction.6 *7 * Information: https://tools.ietf.org/html/rfc84398 */9 10#include <crypto/algapi.h>11#include <crypto/chacha20poly1305.h>12#include <crypto/chacha.h>13#include <crypto/poly1305.h>14#include <crypto/scatterwalk.h>15 16#include <linux/unaligned.h>17#include <linux/kernel.h>18#include <linux/init.h>19#include <linux/mm.h>20#include <linux/module.h>21 22#define CHACHA_KEY_WORDS (CHACHA_KEY_SIZE / sizeof(u32))23 24static void chacha_load_key(u32 *k, const u8 *in)25{26 k[0] = get_unaligned_le32(in);27 k[1] = get_unaligned_le32(in + 4);28 k[2] = get_unaligned_le32(in + 8);29 k[3] = get_unaligned_le32(in + 12);30 k[4] = get_unaligned_le32(in + 16);31 k[5] = get_unaligned_le32(in + 20);32 k[6] = get_unaligned_le32(in + 24);33 k[7] = get_unaligned_le32(in + 28);34}35 36static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce)37{38 u32 k[CHACHA_KEY_WORDS];39 u8 iv[CHACHA_IV_SIZE];40 41 memset(iv, 0, 8);42 memcpy(iv + 8, nonce + 16, 8);43 44 chacha_load_key(k, key);45 46 /* Compute the subkey given the original key and first 128 nonce bits */47 chacha_init(chacha_state, k, nonce);48 hchacha_block(chacha_state, k, 20);49 50 chacha_init(chacha_state, k, iv);51 52 memzero_explicit(k, sizeof(k));53 memzero_explicit(iv, sizeof(iv));54}55 56static void57__chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,58 const u8 *ad, const size_t ad_len, u32 *chacha_state)59{60 const u8 *pad0 = page_address(ZERO_PAGE(0));61 struct poly1305_desc_ctx poly1305_state;62 union {63 u8 block0[POLY1305_KEY_SIZE];64 __le64 lens[2];65 } b;66 67 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));68 poly1305_init(&poly1305_state, b.block0);69 70 poly1305_update(&poly1305_state, ad, ad_len);71 if (ad_len & 0xf)72 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));73 74 chacha20_crypt(chacha_state, dst, src, src_len);75 76 poly1305_update(&poly1305_state, dst, src_len);77 if (src_len & 0xf)78 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));79 80 b.lens[0] = cpu_to_le64(ad_len);81 b.lens[1] = cpu_to_le64(src_len);82 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));83 84 poly1305_final(&poly1305_state, dst + src_len);85 86 memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32));87 memzero_explicit(&b, sizeof(b));88}89 90void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,91 const u8 *ad, const size_t ad_len,92 const u64 nonce,93 const u8 key[CHACHA20POLY1305_KEY_SIZE])94{95 u32 chacha_state[CHACHA_STATE_WORDS];96 u32 k[CHACHA_KEY_WORDS];97 __le64 iv[2];98 99 chacha_load_key(k, key);100 101 iv[0] = 0;102 iv[1] = cpu_to_le64(nonce);103 104 chacha_init(chacha_state, k, (u8 *)iv);105 __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);106 107 memzero_explicit(iv, sizeof(iv));108 memzero_explicit(k, sizeof(k));109}110EXPORT_SYMBOL(chacha20poly1305_encrypt);111 112void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,113 const u8 *ad, const size_t ad_len,114 const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],115 const u8 key[CHACHA20POLY1305_KEY_SIZE])116{117 u32 chacha_state[CHACHA_STATE_WORDS];118 119 xchacha_init(chacha_state, key, nonce);120 __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);121}122EXPORT_SYMBOL(xchacha20poly1305_encrypt);123 124static bool125__chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,126 const u8 *ad, const size_t ad_len, u32 *chacha_state)127{128 const u8 *pad0 = page_address(ZERO_PAGE(0));129 struct poly1305_desc_ctx poly1305_state;130 size_t dst_len;131 int ret;132 union {133 u8 block0[POLY1305_KEY_SIZE];134 u8 mac[POLY1305_DIGEST_SIZE];135 __le64 lens[2];136 } b;137 138 if (unlikely(src_len < POLY1305_DIGEST_SIZE))139 return false;140 141 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));142 poly1305_init(&poly1305_state, b.block0);143 144 poly1305_update(&poly1305_state, ad, ad_len);145 if (ad_len & 0xf)146 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));147 148 dst_len = src_len - POLY1305_DIGEST_SIZE;149 poly1305_update(&poly1305_state, src, dst_len);150 if (dst_len & 0xf)151 poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf));152 153 b.lens[0] = cpu_to_le64(ad_len);154 b.lens[1] = cpu_to_le64(dst_len);155 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));156 157 poly1305_final(&poly1305_state, b.mac);158 159 ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE);160 if (likely(!ret))161 chacha20_crypt(chacha_state, dst, src, dst_len);162 163 memzero_explicit(&b, sizeof(b));164 165 return !ret;166}167 168bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,169 const u8 *ad, const size_t ad_len,170 const u64 nonce,171 const u8 key[CHACHA20POLY1305_KEY_SIZE])172{173 u32 chacha_state[CHACHA_STATE_WORDS];174 u32 k[CHACHA_KEY_WORDS];175 __le64 iv[2];176 bool ret;177 178 chacha_load_key(k, key);179 180 iv[0] = 0;181 iv[1] = cpu_to_le64(nonce);182 183 chacha_init(chacha_state, k, (u8 *)iv);184 ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,185 chacha_state);186 187 memzero_explicit(chacha_state, sizeof(chacha_state));188 memzero_explicit(iv, sizeof(iv));189 memzero_explicit(k, sizeof(k));190 return ret;191}192EXPORT_SYMBOL(chacha20poly1305_decrypt);193 194bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,195 const u8 *ad, const size_t ad_len,196 const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],197 const u8 key[CHACHA20POLY1305_KEY_SIZE])198{199 u32 chacha_state[CHACHA_STATE_WORDS];200 201 xchacha_init(chacha_state, key, nonce);202 return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,203 chacha_state);204}205EXPORT_SYMBOL(xchacha20poly1305_decrypt);206 207static208bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,209 const size_t src_len,210 const u8 *ad, const size_t ad_len,211 const u64 nonce,212 const u8 key[CHACHA20POLY1305_KEY_SIZE],213 int encrypt)214{215 const u8 *pad0 = page_address(ZERO_PAGE(0));216 struct poly1305_desc_ctx poly1305_state;217 u32 chacha_state[CHACHA_STATE_WORDS];218 struct sg_mapping_iter miter;219 size_t partial = 0;220 unsigned int flags;221 bool ret = true;222 int sl;223 union {224 struct {225 u32 k[CHACHA_KEY_WORDS];226 __le64 iv[2];227 };228 u8 block0[POLY1305_KEY_SIZE];229 u8 chacha_stream[CHACHA_BLOCK_SIZE];230 struct {231 u8 mac[2][POLY1305_DIGEST_SIZE];232 };233 __le64 lens[2];234 } b __aligned(16);235 236 if (WARN_ON(src_len > INT_MAX))237 return false;238 239 chacha_load_key(b.k, key);240 241 b.iv[0] = 0;242 b.iv[1] = cpu_to_le64(nonce);243 244 chacha_init(chacha_state, b.k, (u8 *)b.iv);245 chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));246 poly1305_init(&poly1305_state, b.block0);247 248 if (unlikely(ad_len)) {249 poly1305_update(&poly1305_state, ad, ad_len);250 if (ad_len & 0xf)251 poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));252 }253 254 flags = SG_MITER_TO_SG | SG_MITER_ATOMIC;255 256 sg_miter_start(&miter, src, sg_nents(src), flags);257 258 for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {259 u8 *addr = miter.addr;260 size_t length = min_t(size_t, sl, miter.length);261 262 if (!encrypt)263 poly1305_update(&poly1305_state, addr, length);264 265 if (unlikely(partial)) {266 size_t l = min(length, CHACHA_BLOCK_SIZE - partial);267 268 crypto_xor(addr, b.chacha_stream + partial, l);269 partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1);270 271 addr += l;272 length -= l;273 }274 275 if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) {276 size_t l = length;277 278 if (unlikely(length < sl))279 l &= ~(CHACHA_BLOCK_SIZE - 1);280 chacha20_crypt(chacha_state, addr, addr, l);281 addr += l;282 length -= l;283 }284 285 if (unlikely(length > 0)) {286 chacha20_crypt(chacha_state, b.chacha_stream, pad0,287 CHACHA_BLOCK_SIZE);288 crypto_xor(addr, b.chacha_stream, length);289 partial = length;290 }291 292 if (encrypt)293 poly1305_update(&poly1305_state, miter.addr,294 min_t(size_t, sl, miter.length));295 }296 297 if (src_len & 0xf)298 poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));299 300 b.lens[0] = cpu_to_le64(ad_len);301 b.lens[1] = cpu_to_le64(src_len);302 poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));303 304 if (likely(sl <= -POLY1305_DIGEST_SIZE)) {305 if (encrypt) {306 poly1305_final(&poly1305_state,307 miter.addr + miter.length + sl);308 ret = true;309 } else {310 poly1305_final(&poly1305_state, b.mac[0]);311 ret = !crypto_memneq(b.mac[0],312 miter.addr + miter.length + sl,313 POLY1305_DIGEST_SIZE);314 }315 }316 317 sg_miter_stop(&miter);318 319 if (unlikely(sl > -POLY1305_DIGEST_SIZE)) {320 poly1305_final(&poly1305_state, b.mac[1]);321 scatterwalk_map_and_copy(b.mac[encrypt], src, src_len,322 sizeof(b.mac[1]), encrypt);323 ret = encrypt ||324 !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE);325 }326 327 memzero_explicit(chacha_state, sizeof(chacha_state));328 memzero_explicit(&b, sizeof(b));329 330 return ret;331}332 333bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len,334 const u8 *ad, const size_t ad_len,335 const u64 nonce,336 const u8 key[CHACHA20POLY1305_KEY_SIZE])337{338 return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len,339 nonce, key, 1);340}341EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);342 343bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len,344 const u8 *ad, const size_t ad_len,345 const u64 nonce,346 const u8 key[CHACHA20POLY1305_KEY_SIZE])347{348 if (unlikely(src_len < POLY1305_DIGEST_SIZE))349 return false;350 351 return chacha20poly1305_crypt_sg_inplace(src,352 src_len - POLY1305_DIGEST_SIZE,353 ad, ad_len, nonce, key, 0);354}355EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);356 357static int __init chacha20poly1305_init(void)358{359 if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&360 WARN_ON(!chacha20poly1305_selftest()))361 return -ENODEV;362 return 0;363}364 365static void __exit chacha20poly1305_exit(void)366{367}368 369module_init(chacha20poly1305_init);370module_exit(chacha20poly1305_exit);371MODULE_LICENSE("GPL v2");372MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");373MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");374