xref: /linux/lib/crypto/chacha20poly1305.c (revision b83deaa741558babf4b8d51d34f6637ccfff1b26) 
1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
4  *
5  * This is an implementation of the ChaCha20Poly1305 AEAD construction.
6  *
7  * Information: https://tools.ietf.org/html/rfc8439
8  */
9 
10 #include <crypto/algapi.h>
11 #include <crypto/chacha20poly1305.h>
12 #include <crypto/chacha.h>
13 #include <crypto/poly1305.h>
14 #include <crypto/scatterwalk.h>
15 
16 #include <asm/unaligned.h>
17 #include <linux/kernel.h>
18 #include <linux/init.h>
19 #include <linux/mm.h>
20 #include <linux/module.h>
21 
22 #define CHACHA_KEY_WORDS	(CHACHA_KEY_SIZE / sizeof(u32))
23 
24 static void chacha_load_key(u32 *k, const u8 *in)
25 {
26 	k[0] = get_unaligned_le32(in);
27 	k[1] = get_unaligned_le32(in + 4);
28 	k[2] = get_unaligned_le32(in + 8);
29 	k[3] = get_unaligned_le32(in + 12);
30 	k[4] = get_unaligned_le32(in + 16);
31 	k[5] = get_unaligned_le32(in + 20);
32 	k[6] = get_unaligned_le32(in + 24);
33 	k[7] = get_unaligned_le32(in + 28);
34 }
35 
36 static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce)
37 {
38 	u32 k[CHACHA_KEY_WORDS];
39 	u8 iv[CHACHA_IV_SIZE];
40 
41 	memset(iv, 0, 8);
42 	memcpy(iv + 8, nonce + 16, 8);
43 
44 	chacha_load_key(k, key);
45 
46 	/* Compute the subkey given the original key and first 128 nonce bits */
47 	chacha_init(chacha_state, k, nonce);
48 	hchacha_block(chacha_state, k, 20);
49 
50 	chacha_init(chacha_state, k, iv);
51 
52 	memzero_explicit(k, sizeof(k));
53 	memzero_explicit(iv, sizeof(iv));
54 }
55 
56 static void
57 __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
58 			   const u8 *ad, const size_t ad_len, u32 *chacha_state)
59 {
60 	const u8 *pad0 = page_address(ZERO_PAGE(0));
61 	struct poly1305_desc_ctx poly1305_state;
62 	union {
63 		u8 block0[POLY1305_KEY_SIZE];
64 		__le64 lens[2];
65 	} b;
66 
67 	chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
68 	poly1305_init(&poly1305_state, b.block0);
69 
70 	poly1305_update(&poly1305_state, ad, ad_len);
71 	if (ad_len & 0xf)
72 		poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
73 
74 	chacha20_crypt(chacha_state, dst, src, src_len);
75 
76 	poly1305_update(&poly1305_state, dst, src_len);
77 	if (src_len & 0xf)
78 		poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
79 
80 	b.lens[0] = cpu_to_le64(ad_len);
81 	b.lens[1] = cpu_to_le64(src_len);
82 	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
83 
84 	poly1305_final(&poly1305_state, dst + src_len);
85 
86 	memzero_explicit(chacha_state, CHACHA_STATE_WORDS * sizeof(u32));
87 	memzero_explicit(&b, sizeof(b));
88 }
89 
90 void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
91 			      const u8 *ad, const size_t ad_len,
92 			      const u64 nonce,
93 			      const u8 key[CHACHA20POLY1305_KEY_SIZE])
94 {
95 	u32 chacha_state[CHACHA_STATE_WORDS];
96 	u32 k[CHACHA_KEY_WORDS];
97 	__le64 iv[2];
98 
99 	chacha_load_key(k, key);
100 
101 	iv[0] = 0;
102 	iv[1] = cpu_to_le64(nonce);
103 
104 	chacha_init(chacha_state, k, (u8 *)iv);
105 	__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
106 
107 	memzero_explicit(iv, sizeof(iv));
108 	memzero_explicit(k, sizeof(k));
109 }
110 EXPORT_SYMBOL(chacha20poly1305_encrypt);
111 
112 void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len,
113 			       const u8 *ad, const size_t ad_len,
114 			       const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
115 			       const u8 key[CHACHA20POLY1305_KEY_SIZE])
116 {
117 	u32 chacha_state[CHACHA_STATE_WORDS];
118 
119 	xchacha_init(chacha_state, key, nonce);
120 	__chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state);
121 }
122 EXPORT_SYMBOL(xchacha20poly1305_encrypt);
123 
124 static bool
125 __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
126 			   const u8 *ad, const size_t ad_len, u32 *chacha_state)
127 {
128 	const u8 *pad0 = page_address(ZERO_PAGE(0));
129 	struct poly1305_desc_ctx poly1305_state;
130 	size_t dst_len;
131 	int ret;
132 	union {
133 		u8 block0[POLY1305_KEY_SIZE];
134 		u8 mac[POLY1305_DIGEST_SIZE];
135 		__le64 lens[2];
136 	} b;
137 
138 	if (unlikely(src_len < POLY1305_DIGEST_SIZE))
139 		return false;
140 
141 	chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
142 	poly1305_init(&poly1305_state, b.block0);
143 
144 	poly1305_update(&poly1305_state, ad, ad_len);
145 	if (ad_len & 0xf)
146 		poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
147 
148 	dst_len = src_len - POLY1305_DIGEST_SIZE;
149 	poly1305_update(&poly1305_state, src, dst_len);
150 	if (dst_len & 0xf)
151 		poly1305_update(&poly1305_state, pad0, 0x10 - (dst_len & 0xf));
152 
153 	b.lens[0] = cpu_to_le64(ad_len);
154 	b.lens[1] = cpu_to_le64(dst_len);
155 	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
156 
157 	poly1305_final(&poly1305_state, b.mac);
158 
159 	ret = crypto_memneq(b.mac, src + dst_len, POLY1305_DIGEST_SIZE);
160 	if (likely(!ret))
161 		chacha20_crypt(chacha_state, dst, src, dst_len);
162 
163 	memzero_explicit(&b, sizeof(b));
164 
165 	return !ret;
166 }
167 
168 bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
169 			      const u8 *ad, const size_t ad_len,
170 			      const u64 nonce,
171 			      const u8 key[CHACHA20POLY1305_KEY_SIZE])
172 {
173 	u32 chacha_state[CHACHA_STATE_WORDS];
174 	u32 k[CHACHA_KEY_WORDS];
175 	__le64 iv[2];
176 	bool ret;
177 
178 	chacha_load_key(k, key);
179 
180 	iv[0] = 0;
181 	iv[1] = cpu_to_le64(nonce);
182 
183 	chacha_init(chacha_state, k, (u8 *)iv);
184 	ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
185 					 chacha_state);
186 
187 	memzero_explicit(chacha_state, sizeof(chacha_state));
188 	memzero_explicit(iv, sizeof(iv));
189 	memzero_explicit(k, sizeof(k));
190 	return ret;
191 }
192 EXPORT_SYMBOL(chacha20poly1305_decrypt);
193 
194 bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len,
195 			       const u8 *ad, const size_t ad_len,
196 			       const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE],
197 			       const u8 key[CHACHA20POLY1305_KEY_SIZE])
198 {
199 	u32 chacha_state[CHACHA_STATE_WORDS];
200 
201 	xchacha_init(chacha_state, key, nonce);
202 	return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len,
203 					  chacha_state);
204 }
205 EXPORT_SYMBOL(xchacha20poly1305_decrypt);
206 
207 static
208 bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src,
209 				       const size_t src_len,
210 				       const u8 *ad, const size_t ad_len,
211 				       const u64 nonce,
212 				       const u8 key[CHACHA20POLY1305_KEY_SIZE],
213 				       int encrypt)
214 {
215 	const u8 *pad0 = page_address(ZERO_PAGE(0));
216 	struct poly1305_desc_ctx poly1305_state;
217 	u32 chacha_state[CHACHA_STATE_WORDS];
218 	struct sg_mapping_iter miter;
219 	size_t partial = 0;
220 	unsigned int flags;
221 	bool ret = true;
222 	int sl;
223 	union {
224 		struct {
225 			u32 k[CHACHA_KEY_WORDS];
226 			__le64 iv[2];
227 		};
228 		u8 block0[POLY1305_KEY_SIZE];
229 		u8 chacha_stream[CHACHA_BLOCK_SIZE];
230 		struct {
231 			u8 mac[2][POLY1305_DIGEST_SIZE];
232 		};
233 		__le64 lens[2];
234 	} b __aligned(16);
235 
236 	if (WARN_ON(src_len > INT_MAX))
237 		return false;
238 
239 	chacha_load_key(b.k, key);
240 
241 	b.iv[0] = 0;
242 	b.iv[1] = cpu_to_le64(nonce);
243 
244 	chacha_init(chacha_state, b.k, (u8 *)b.iv);
245 	chacha20_crypt(chacha_state, b.block0, pad0, sizeof(b.block0));
246 	poly1305_init(&poly1305_state, b.block0);
247 
248 	if (unlikely(ad_len)) {
249 		poly1305_update(&poly1305_state, ad, ad_len);
250 		if (ad_len & 0xf)
251 			poly1305_update(&poly1305_state, pad0, 0x10 - (ad_len & 0xf));
252 	}
253 
254 	flags = SG_MITER_TO_SG | SG_MITER_ATOMIC;
255 
256 	sg_miter_start(&miter, src, sg_nents(src), flags);
257 
258 	for (sl = src_len; sl > 0 && sg_miter_next(&miter); sl -= miter.length) {
259 		u8 *addr = miter.addr;
260 		size_t length = min_t(size_t, sl, miter.length);
261 
262 		if (!encrypt)
263 			poly1305_update(&poly1305_state, addr, length);
264 
265 		if (unlikely(partial)) {
266 			size_t l = min(length, CHACHA_BLOCK_SIZE - partial);
267 
268 			crypto_xor(addr, b.chacha_stream + partial, l);
269 			partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1);
270 
271 			addr += l;
272 			length -= l;
273 		}
274 
275 		if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) {
276 			size_t l = length;
277 
278 			if (unlikely(length < sl))
279 				l &= ~(CHACHA_BLOCK_SIZE - 1);
280 			chacha20_crypt(chacha_state, addr, addr, l);
281 			addr += l;
282 			length -= l;
283 		}
284 
285 		if (unlikely(length > 0)) {
286 			chacha20_crypt(chacha_state, b.chacha_stream, pad0,
287 				       CHACHA_BLOCK_SIZE);
288 			crypto_xor(addr, b.chacha_stream, length);
289 			partial = length;
290 		}
291 
292 		if (encrypt)
293 			poly1305_update(&poly1305_state, miter.addr,
294 					min_t(size_t, sl, miter.length));
295 	}
296 
297 	if (src_len & 0xf)
298 		poly1305_update(&poly1305_state, pad0, 0x10 - (src_len & 0xf));
299 
300 	b.lens[0] = cpu_to_le64(ad_len);
301 	b.lens[1] = cpu_to_le64(src_len);
302 	poly1305_update(&poly1305_state, (u8 *)b.lens, sizeof(b.lens));
303 
304 	if (likely(sl <= -POLY1305_DIGEST_SIZE)) {
305 		if (encrypt) {
306 			poly1305_final(&poly1305_state,
307 				       miter.addr + miter.length + sl);
308 			ret = true;
309 		} else {
310 			poly1305_final(&poly1305_state, b.mac[0]);
311 			ret = !crypto_memneq(b.mac[0],
312 					     miter.addr + miter.length + sl,
313 					     POLY1305_DIGEST_SIZE);
314 		}
315 	}
316 
317 	sg_miter_stop(&miter);
318 
319 	if (unlikely(sl > -POLY1305_DIGEST_SIZE)) {
320 		poly1305_final(&poly1305_state, b.mac[1]);
321 		scatterwalk_map_and_copy(b.mac[encrypt], src, src_len,
322 					 sizeof(b.mac[1]), encrypt);
323 		ret = encrypt ||
324 		      !crypto_memneq(b.mac[0], b.mac[1], POLY1305_DIGEST_SIZE);
325 	}
326 
327 	memzero_explicit(chacha_state, sizeof(chacha_state));
328 	memzero_explicit(&b, sizeof(b));
329 
330 	return ret;
331 }
332 
333 bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len,
334 					 const u8 *ad, const size_t ad_len,
335 					 const u64 nonce,
336 					 const u8 key[CHACHA20POLY1305_KEY_SIZE])
337 {
338 	return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len,
339 						 nonce, key, 1);
340 }
341 EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace);
342 
343 bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len,
344 					 const u8 *ad, const size_t ad_len,
345 					 const u64 nonce,
346 					 const u8 key[CHACHA20POLY1305_KEY_SIZE])
347 {
348 	if (unlikely(src_len < POLY1305_DIGEST_SIZE))
349 		return false;
350 
351 	return chacha20poly1305_crypt_sg_inplace(src,
352 						 src_len - POLY1305_DIGEST_SIZE,
353 						 ad, ad_len, nonce, key, 0);
354 }
355 EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace);
356 
357 static int __init chacha20poly1305_init(void)
358 {
359 	if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
360 	    WARN_ON(!chacha20poly1305_selftest()))
361 		return -ENODEV;
362 	return 0;
363 }
364 
365 static void __exit chacha20poly1305_exit(void)
366 {
367 }
368 
369 module_init(chacha20poly1305_init);
370 module_exit(chacha20poly1305_exit);
371 MODULE_LICENSE("GPL v2");
372 MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction");
373 MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>");
374