brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * Algorithm testing framework and tests.4 *5 * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>6 * Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org>7 * Copyright (c) 2007 Nokia Siemens Networks8 * Copyright (c) 2008 Herbert Xu <herbert@gondor.apana.org.au>9 * Copyright (c) 2019 Google LLC10 *11 * Updated RFC4106 AES-GCM testing.12 *    Authors: Aidan O'Mahony (aidan.o.mahony@intel.com)13 *             Adrian Hoban <adrian.hoban@intel.com>14 *             Gabriele Paoloni <gabriele.paoloni@intel.com>15 *             Tadeusz Struk (tadeusz.struk@intel.com)16 *    Copyright (c) 2010, Intel Corporation.17 */18 19#include <crypto/aead.h>20#include <crypto/hash.h>21#include <crypto/skcipher.h>22#include <linux/err.h>23#include <linux/fips.h>24#include <linux/module.h>25#include <linux/once.h>26#include <linux/random.h>27#include <linux/scatterlist.h>28#include <linux/slab.h>29#include <linux/string.h>30#include <linux/uio.h>31#include <crypto/rng.h>32#include <crypto/drbg.h>33#include <crypto/akcipher.h>34#include <crypto/kpp.h>35#include <crypto/acompress.h>36#include <crypto/internal/cipher.h>37#include <crypto/internal/simd.h>38 39#include "internal.h"40 41MODULE_IMPORT_NS(CRYPTO_INTERNAL);42 43static bool notests;44module_param(notests, bool, 0644);45MODULE_PARM_DESC(notests, "disable crypto self-tests");46 47static bool panic_on_fail;48module_param(panic_on_fail, bool, 0444);49 50#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS51static bool noextratests;52module_param(noextratests, bool, 0644);53MODULE_PARM_DESC(noextratests, "disable expensive crypto self-tests");54 55static unsigned int fuzz_iterations = 100;56module_param(fuzz_iterations, uint, 0644);57MODULE_PARM_DESC(fuzz_iterations, "number of fuzz test iterations");58#endif59 60#ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS61 62/* a perfect nop */63int alg_test(const char *driver, const char *alg, u32 type, u32 mask)64{65	return 0;66}67 68#else69 70#include "testmgr.h"71 72/*73 * Need slab memory for testing (size in number of pages).74 */75#define XBUFSIZE	876 77/*78* Used by test_cipher()79*/80#define ENCRYPT 181#define DECRYPT 082 83struct aead_test_suite {84	const struct aead_testvec *vecs;85	unsigned int count;86 87	/*88	 * Set if trying to decrypt an inauthentic ciphertext with this89	 * algorithm might result in EINVAL rather than EBADMSG, due to other90	 * validation the algorithm does on the inputs such as length checks.91	 */92	unsigned int einval_allowed : 1;93 94	/*95	 * Set if this algorithm requires that the IV be located at the end of96	 * the AAD buffer, in addition to being given in the normal way.  The97	 * behavior when the two IV copies differ is implementation-defined.98	 */99	unsigned int aad_iv : 1;100};101 102struct cipher_test_suite {103	const struct cipher_testvec *vecs;104	unsigned int count;105};106 107struct comp_test_suite {108	struct {109		const struct comp_testvec *vecs;110		unsigned int count;111	} comp, decomp;112};113 114struct hash_test_suite {115	const struct hash_testvec *vecs;116	unsigned int count;117};118 119struct cprng_test_suite {120	const struct cprng_testvec *vecs;121	unsigned int count;122};123 124struct drbg_test_suite {125	const struct drbg_testvec *vecs;126	unsigned int count;127};128 129struct akcipher_test_suite {130	const struct akcipher_testvec *vecs;131	unsigned int count;132};133 134struct kpp_test_suite {135	const struct kpp_testvec *vecs;136	unsigned int count;137};138 139struct alg_test_desc {140	const char *alg;141	const char *generic_driver;142	int (*test)(const struct alg_test_desc *desc, const char *driver,143		    u32 type, u32 mask);144	int fips_allowed;	/* set if alg is allowed in fips mode */145 146	union {147		struct aead_test_suite aead;148		struct cipher_test_suite cipher;149		struct comp_test_suite comp;150		struct hash_test_suite hash;151		struct cprng_test_suite cprng;152		struct drbg_test_suite drbg;153		struct akcipher_test_suite akcipher;154		struct kpp_test_suite kpp;155	} suite;156};157 158static void hexdump(unsigned char *buf, unsigned int len)159{160	print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,161			16, 1,162			buf, len, false);163}164 165static int __testmgr_alloc_buf(char *buf[XBUFSIZE], int order)166{167	int i;168 169	for (i = 0; i < XBUFSIZE; i++) {170		buf[i] = (char *)__get_free_pages(GFP_KERNEL, order);171		if (!buf[i])172			goto err_free_buf;173	}174 175	return 0;176 177err_free_buf:178	while (i-- > 0)179		free_pages((unsigned long)buf[i], order);180 181	return -ENOMEM;182}183 184static int testmgr_alloc_buf(char *buf[XBUFSIZE])185{186	return __testmgr_alloc_buf(buf, 0);187}188 189static void __testmgr_free_buf(char *buf[XBUFSIZE], int order)190{191	int i;192 193	for (i = 0; i < XBUFSIZE; i++)194		free_pages((unsigned long)buf[i], order);195}196 197static void testmgr_free_buf(char *buf[XBUFSIZE])198{199	__testmgr_free_buf(buf, 0);200}201 202#define TESTMGR_POISON_BYTE	0xfe203#define TESTMGR_POISON_LEN	16204 205static inline void testmgr_poison(void *addr, size_t len)206{207	memset(addr, TESTMGR_POISON_BYTE, len);208}209 210/* Is the memory region still fully poisoned? */211static inline bool testmgr_is_poison(const void *addr, size_t len)212{213	return memchr_inv(addr, TESTMGR_POISON_BYTE, len) == NULL;214}215 216/* flush type for hash algorithms */217enum flush_type {218	/* merge with update of previous buffer(s) */219	FLUSH_TYPE_NONE = 0,220 221	/* update with previous buffer(s) before doing this one */222	FLUSH_TYPE_FLUSH,223 224	/* likewise, but also export and re-import the intermediate state */225	FLUSH_TYPE_REIMPORT,226};227 228/* finalization function for hash algorithms */229enum finalization_type {230	FINALIZATION_TYPE_FINAL,	/* use final() */231	FINALIZATION_TYPE_FINUP,	/* use finup() */232	FINALIZATION_TYPE_DIGEST,	/* use digest() */233};234 235/*236 * Whether the crypto operation will occur in-place, and if so whether the237 * source and destination scatterlist pointers will coincide (req->src ==238 * req->dst), or whether they'll merely point to two separate scatterlists239 * (req->src != req->dst) that reference the same underlying memory.240 *241 * This is only relevant for algorithm types that support in-place operation.242 */243enum inplace_mode {244	OUT_OF_PLACE,245	INPLACE_ONE_SGLIST,246	INPLACE_TWO_SGLISTS,247};248 249#define TEST_SG_TOTAL	10000250 251/**252 * struct test_sg_division - description of a scatterlist entry253 *254 * This struct describes one entry of a scatterlist being constructed to check a255 * crypto test vector.256 *257 * @proportion_of_total: length of this chunk relative to the total length,258 *			 given as a proportion out of TEST_SG_TOTAL so that it259 *			 scales to fit any test vector260 * @offset: byte offset into a 2-page buffer at which this chunk will start261 * @offset_relative_to_alignmask: if true, add the algorithm's alignmask to the262 *				  @offset263 * @flush_type: for hashes, whether an update() should be done now vs.264 *		continuing to accumulate data265 * @nosimd: if doing the pending update(), do it with SIMD disabled?266 */267struct test_sg_division {268	unsigned int proportion_of_total;269	unsigned int offset;270	bool offset_relative_to_alignmask;271	enum flush_type flush_type;272	bool nosimd;273};274 275/**276 * struct testvec_config - configuration for testing a crypto test vector277 *278 * This struct describes the data layout and other parameters with which each279 * crypto test vector can be tested.280 *281 * @name: name of this config, logged for debugging purposes if a test fails282 * @inplace_mode: whether and how to operate on the data in-place, if applicable283 * @req_flags: extra request_flags, e.g. CRYPTO_TFM_REQ_MAY_SLEEP284 * @src_divs: description of how to arrange the source scatterlist285 * @dst_divs: description of how to arrange the dst scatterlist, if applicable286 *	      for the algorithm type.  Defaults to @src_divs if unset.287 * @iv_offset: misalignment of the IV in the range [0..MAX_ALGAPI_ALIGNMASK+1],288 *	       where 0 is aligned to a 2*(MAX_ALGAPI_ALIGNMASK+1) byte boundary289 * @iv_offset_relative_to_alignmask: if true, add the algorithm's alignmask to290 *				     the @iv_offset291 * @key_offset: misalignment of the key, where 0 is default alignment292 * @key_offset_relative_to_alignmask: if true, add the algorithm's alignmask to293 *				      the @key_offset294 * @finalization_type: what finalization function to use for hashes295 * @nosimd: execute with SIMD disabled?  Requires !CRYPTO_TFM_REQ_MAY_SLEEP.296 *	    This applies to the parts of the operation that aren't controlled297 *	    individually by @nosimd_setkey or @src_divs[].nosimd.298 * @nosimd_setkey: set the key (if applicable) with SIMD disabled?  Requires299 *		   !CRYPTO_TFM_REQ_MAY_SLEEP.300 */301struct testvec_config {302	const char *name;303	enum inplace_mode inplace_mode;304	u32 req_flags;305	struct test_sg_division src_divs[XBUFSIZE];306	struct test_sg_division dst_divs[XBUFSIZE];307	unsigned int iv_offset;308	unsigned int key_offset;309	bool iv_offset_relative_to_alignmask;310	bool key_offset_relative_to_alignmask;311	enum finalization_type finalization_type;312	bool nosimd;313	bool nosimd_setkey;314};315 316#define TESTVEC_CONFIG_NAMELEN	192317 318/*319 * The following are the lists of testvec_configs to test for each algorithm320 * type when the basic crypto self-tests are enabled, i.e. when321 * CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is unset.  They aim to provide good test322 * coverage, while keeping the test time much shorter than the full fuzz tests323 * so that the basic tests can be enabled in a wider range of circumstances.324 */325 326/* Configs for skciphers and aeads */327static const struct testvec_config default_cipher_testvec_configs[] = {328	{329		.name = "in-place (one sglist)",330		.inplace_mode = INPLACE_ONE_SGLIST,331		.src_divs = { { .proportion_of_total = 10000 } },332	}, {333		.name = "in-place (two sglists)",334		.inplace_mode = INPLACE_TWO_SGLISTS,335		.src_divs = { { .proportion_of_total = 10000 } },336	}, {337		.name = "out-of-place",338		.inplace_mode = OUT_OF_PLACE,339		.src_divs = { { .proportion_of_total = 10000 } },340	}, {341		.name = "unaligned buffer, offset=1",342		.src_divs = { { .proportion_of_total = 10000, .offset = 1 } },343		.iv_offset = 1,344		.key_offset = 1,345	}, {346		.name = "buffer aligned only to alignmask",347		.src_divs = {348			{349				.proportion_of_total = 10000,350				.offset = 1,351				.offset_relative_to_alignmask = true,352			},353		},354		.iv_offset = 1,355		.iv_offset_relative_to_alignmask = true,356		.key_offset = 1,357		.key_offset_relative_to_alignmask = true,358	}, {359		.name = "two even aligned splits",360		.src_divs = {361			{ .proportion_of_total = 5000 },362			{ .proportion_of_total = 5000 },363		},364	}, {365		.name = "one src, two even splits dst",366		.inplace_mode = OUT_OF_PLACE,367		.src_divs = { { .proportion_of_total = 10000 } },368		.dst_divs = {369			{ .proportion_of_total = 5000 },370			{ .proportion_of_total = 5000 },371		 },372	}, {373		.name = "uneven misaligned splits, may sleep",374		.req_flags = CRYPTO_TFM_REQ_MAY_SLEEP,375		.src_divs = {376			{ .proportion_of_total = 1900, .offset = 33 },377			{ .proportion_of_total = 3300, .offset = 7  },378			{ .proportion_of_total = 4800, .offset = 18 },379		},380		.iv_offset = 3,381		.key_offset = 3,382	}, {383		.name = "misaligned splits crossing pages, inplace",384		.inplace_mode = INPLACE_ONE_SGLIST,385		.src_divs = {386			{387				.proportion_of_total = 7500,388				.offset = PAGE_SIZE - 32389			}, {390				.proportion_of_total = 2500,391				.offset = PAGE_SIZE - 7392			},393		},394	}395};396 397static const struct testvec_config default_hash_testvec_configs[] = {398	{399		.name = "init+update+final aligned buffer",400		.src_divs = { { .proportion_of_total = 10000 } },401		.finalization_type = FINALIZATION_TYPE_FINAL,402	}, {403		.name = "init+finup aligned buffer",404		.src_divs = { { .proportion_of_total = 10000 } },405		.finalization_type = FINALIZATION_TYPE_FINUP,406	}, {407		.name = "digest aligned buffer",408		.src_divs = { { .proportion_of_total = 10000 } },409		.finalization_type = FINALIZATION_TYPE_DIGEST,410	}, {411		.name = "init+update+final misaligned buffer",412		.src_divs = { { .proportion_of_total = 10000, .offset = 1 } },413		.finalization_type = FINALIZATION_TYPE_FINAL,414		.key_offset = 1,415	}, {416		.name = "digest misaligned buffer",417		.src_divs = {418			{419				.proportion_of_total = 10000,420				.offset = 1,421			},422		},423		.finalization_type = FINALIZATION_TYPE_DIGEST,424		.key_offset = 1,425	}, {426		.name = "init+update+update+final two even splits",427		.src_divs = {428			{ .proportion_of_total = 5000 },429			{430				.proportion_of_total = 5000,431				.flush_type = FLUSH_TYPE_FLUSH,432			},433		},434		.finalization_type = FINALIZATION_TYPE_FINAL,435	}, {436		.name = "digest uneven misaligned splits, may sleep",437		.req_flags = CRYPTO_TFM_REQ_MAY_SLEEP,438		.src_divs = {439			{ .proportion_of_total = 1900, .offset = 33 },440			{ .proportion_of_total = 3300, .offset = 7  },441			{ .proportion_of_total = 4800, .offset = 18 },442		},443		.finalization_type = FINALIZATION_TYPE_DIGEST,444	}, {445		.name = "digest misaligned splits crossing pages",446		.src_divs = {447			{448				.proportion_of_total = 7500,449				.offset = PAGE_SIZE - 32,450			}, {451				.proportion_of_total = 2500,452				.offset = PAGE_SIZE - 7,453			},454		},455		.finalization_type = FINALIZATION_TYPE_DIGEST,456	}, {457		.name = "import/export",458		.src_divs = {459			{460				.proportion_of_total = 6500,461				.flush_type = FLUSH_TYPE_REIMPORT,462			}, {463				.proportion_of_total = 3500,464				.flush_type = FLUSH_TYPE_REIMPORT,465			},466		},467		.finalization_type = FINALIZATION_TYPE_FINAL,468	}469};470 471static unsigned int count_test_sg_divisions(const struct test_sg_division *divs)472{473	unsigned int remaining = TEST_SG_TOTAL;474	unsigned int ndivs = 0;475 476	do {477		remaining -= divs[ndivs++].proportion_of_total;478	} while (remaining);479 480	return ndivs;481}482 483#define SGDIVS_HAVE_FLUSHES	BIT(0)484#define SGDIVS_HAVE_NOSIMD	BIT(1)485 486static bool valid_sg_divisions(const struct test_sg_division *divs,487			       unsigned int count, int *flags_ret)488{489	unsigned int total = 0;490	unsigned int i;491 492	for (i = 0; i < count && total != TEST_SG_TOTAL; i++) {493		if (divs[i].proportion_of_total <= 0 ||494		    divs[i].proportion_of_total > TEST_SG_TOTAL - total)495			return false;496		total += divs[i].proportion_of_total;497		if (divs[i].flush_type != FLUSH_TYPE_NONE)498			*flags_ret |= SGDIVS_HAVE_FLUSHES;499		if (divs[i].nosimd)500			*flags_ret |= SGDIVS_HAVE_NOSIMD;501	}502	return total == TEST_SG_TOTAL &&503		memchr_inv(&divs[i], 0, (count - i) * sizeof(divs[0])) == NULL;504}505 506/*507 * Check whether the given testvec_config is valid.  This isn't strictly needed508 * since every testvec_config should be valid, but check anyway so that people509 * don't unknowingly add broken configs that don't do what they wanted.510 */511static bool valid_testvec_config(const struct testvec_config *cfg)512{513	int flags = 0;514 515	if (cfg->name == NULL)516		return false;517 518	if (!valid_sg_divisions(cfg->src_divs, ARRAY_SIZE(cfg->src_divs),519				&flags))520		return false;521 522	if (cfg->dst_divs[0].proportion_of_total) {523		if (!valid_sg_divisions(cfg->dst_divs,524					ARRAY_SIZE(cfg->dst_divs), &flags))525			return false;526	} else {527		if (memchr_inv(cfg->dst_divs, 0, sizeof(cfg->dst_divs)))528			return false;529		/* defaults to dst_divs=src_divs */530	}531 532	if (cfg->iv_offset +533	    (cfg->iv_offset_relative_to_alignmask ? MAX_ALGAPI_ALIGNMASK : 0) >534	    MAX_ALGAPI_ALIGNMASK + 1)535		return false;536 537	if ((flags & (SGDIVS_HAVE_FLUSHES | SGDIVS_HAVE_NOSIMD)) &&538	    cfg->finalization_type == FINALIZATION_TYPE_DIGEST)539		return false;540 541	if ((cfg->nosimd || cfg->nosimd_setkey ||542	     (flags & SGDIVS_HAVE_NOSIMD)) &&543	    (cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP))544		return false;545 546	return true;547}548 549struct test_sglist {550	char *bufs[XBUFSIZE];551	struct scatterlist sgl[XBUFSIZE];552	struct scatterlist sgl_saved[XBUFSIZE];553	struct scatterlist *sgl_ptr;554	unsigned int nents;555};556 557static int init_test_sglist(struct test_sglist *tsgl)558{559	return __testmgr_alloc_buf(tsgl->bufs, 1 /* two pages per buffer */);560}561 562static void destroy_test_sglist(struct test_sglist *tsgl)563{564	return __testmgr_free_buf(tsgl->bufs, 1 /* two pages per buffer */);565}566 567/**568 * build_test_sglist() - build a scatterlist for a crypto test569 *570 * @tsgl: the scatterlist to build.  @tsgl->bufs[] contains an array of 2-page571 *	  buffers which the scatterlist @tsgl->sgl[] will be made to point into.572 * @divs: the layout specification on which the scatterlist will be based573 * @alignmask: the algorithm's alignmask574 * @total_len: the total length of the scatterlist to build in bytes575 * @data: if non-NULL, the buffers will be filled with this data until it ends.576 *	  Otherwise the buffers will be poisoned.  In both cases, some bytes577 *	  past the end of each buffer will be poisoned to help detect overruns.578 * @out_divs: if non-NULL, the test_sg_division to which each scatterlist entry579 *	      corresponds will be returned here.  This will match @divs except580 *	      that divisions resolving to a length of 0 are omitted as they are581 *	      not included in the scatterlist.582 *583 * Return: 0 or a -errno value584 */585static int build_test_sglist(struct test_sglist *tsgl,586			     const struct test_sg_division *divs,587			     const unsigned int alignmask,588			     const unsigned int total_len,589			     struct iov_iter *data,590			     const struct test_sg_division *out_divs[XBUFSIZE])591{592	struct {593		const struct test_sg_division *div;594		size_t length;595	} partitions[XBUFSIZE];596	const unsigned int ndivs = count_test_sg_divisions(divs);597	unsigned int len_remaining = total_len;598	unsigned int i;599 600	BUILD_BUG_ON(ARRAY_SIZE(partitions) != ARRAY_SIZE(tsgl->sgl));601	if (WARN_ON(ndivs > ARRAY_SIZE(partitions)))602		return -EINVAL;603 604	/* Calculate the (div, length) pairs */605	tsgl->nents = 0;606	for (i = 0; i < ndivs; i++) {607		unsigned int len_this_sg =608			min(len_remaining,609			    (total_len * divs[i].proportion_of_total +610			     TEST_SG_TOTAL / 2) / TEST_SG_TOTAL);611 612		if (len_this_sg != 0) {613			partitions[tsgl->nents].div = &divs[i];614			partitions[tsgl->nents].length = len_this_sg;615			tsgl->nents++;616			len_remaining -= len_this_sg;617		}618	}619	if (tsgl->nents == 0) {620		partitions[tsgl->nents].div = &divs[0];621		partitions[tsgl->nents].length = 0;622		tsgl->nents++;623	}624	partitions[tsgl->nents - 1].length += len_remaining;625 626	/* Set up the sgl entries and fill the data or poison */627	sg_init_table(tsgl->sgl, tsgl->nents);628	for (i = 0; i < tsgl->nents; i++) {629		unsigned int offset = partitions[i].div->offset;630		void *addr;631 632		if (partitions[i].div->offset_relative_to_alignmask)633			offset += alignmask;634 635		while (offset + partitions[i].length + TESTMGR_POISON_LEN >636		       2 * PAGE_SIZE) {637			if (WARN_ON(offset <= 0))638				return -EINVAL;639			offset /= 2;640		}641 642		addr = &tsgl->bufs[i][offset];643		sg_set_buf(&tsgl->sgl[i], addr, partitions[i].length);644 645		if (out_divs)646			out_divs[i] = partitions[i].div;647 648		if (data) {649			size_t copy_len, copied;650 651			copy_len = min(partitions[i].length, data->count);652			copied = copy_from_iter(addr, copy_len, data);653			if (WARN_ON(copied != copy_len))654				return -EINVAL;655			testmgr_poison(addr + copy_len, partitions[i].length +656				       TESTMGR_POISON_LEN - copy_len);657		} else {658			testmgr_poison(addr, partitions[i].length +659				       TESTMGR_POISON_LEN);660		}661	}662 663	sg_mark_end(&tsgl->sgl[tsgl->nents - 1]);664	tsgl->sgl_ptr = tsgl->sgl;665	memcpy(tsgl->sgl_saved, tsgl->sgl, tsgl->nents * sizeof(tsgl->sgl[0]));666	return 0;667}668 669/*670 * Verify that a scatterlist crypto operation produced the correct output.671 *672 * @tsgl: scatterlist containing the actual output673 * @expected_output: buffer containing the expected output674 * @len_to_check: length of @expected_output in bytes675 * @unchecked_prefix_len: number of ignored bytes in @tsgl prior to real result676 * @check_poison: verify that the poison bytes after each chunk are intact?677 *678 * Return: 0 if correct, -EINVAL if incorrect, -EOVERFLOW if buffer overrun.679 */680static int verify_correct_output(const struct test_sglist *tsgl,681				 const char *expected_output,682				 unsigned int len_to_check,683				 unsigned int unchecked_prefix_len,684				 bool check_poison)685{686	unsigned int i;687 688	for (i = 0; i < tsgl->nents; i++) {689		struct scatterlist *sg = &tsgl->sgl_ptr[i];690		unsigned int len = sg->length;691		unsigned int offset = sg->offset;692		const char *actual_output;693 694		if (unchecked_prefix_len) {695			if (unchecked_prefix_len >= len) {696				unchecked_prefix_len -= len;697				continue;698			}699			offset += unchecked_prefix_len;700			len -= unchecked_prefix_len;701			unchecked_prefix_len = 0;702		}703		len = min(len, len_to_check);704		actual_output = page_address(sg_page(sg)) + offset;705		if (memcmp(expected_output, actual_output, len) != 0)706			return -EINVAL;707		if (check_poison &&708		    !testmgr_is_poison(actual_output + len, TESTMGR_POISON_LEN))709			return -EOVERFLOW;710		len_to_check -= len;711		expected_output += len;712	}713	if (WARN_ON(len_to_check != 0))714		return -EINVAL;715	return 0;716}717 718static bool is_test_sglist_corrupted(const struct test_sglist *tsgl)719{720	unsigned int i;721 722	for (i = 0; i < tsgl->nents; i++) {723		if (tsgl->sgl[i].page_link != tsgl->sgl_saved[i].page_link)724			return true;725		if (tsgl->sgl[i].offset != tsgl->sgl_saved[i].offset)726			return true;727		if (tsgl->sgl[i].length != tsgl->sgl_saved[i].length)728			return true;729	}730	return false;731}732 733struct cipher_test_sglists {734	struct test_sglist src;735	struct test_sglist dst;736};737 738static struct cipher_test_sglists *alloc_cipher_test_sglists(void)739{740	struct cipher_test_sglists *tsgls;741 742	tsgls = kmalloc(sizeof(*tsgls), GFP_KERNEL);743	if (!tsgls)744		return NULL;745 746	if (init_test_sglist(&tsgls->src) != 0)747		goto fail_kfree;748	if (init_test_sglist(&tsgls->dst) != 0)749		goto fail_destroy_src;750 751	return tsgls;752 753fail_destroy_src:754	destroy_test_sglist(&tsgls->src);755fail_kfree:756	kfree(tsgls);757	return NULL;758}759 760static void free_cipher_test_sglists(struct cipher_test_sglists *tsgls)761{762	if (tsgls) {763		destroy_test_sglist(&tsgls->src);764		destroy_test_sglist(&tsgls->dst);765		kfree(tsgls);766	}767}768 769/* Build the src and dst scatterlists for an skcipher or AEAD test */770static int build_cipher_test_sglists(struct cipher_test_sglists *tsgls,771				     const struct testvec_config *cfg,772				     unsigned int alignmask,773				     unsigned int src_total_len,774				     unsigned int dst_total_len,775				     const struct kvec *inputs,776				     unsigned int nr_inputs)777{778	struct iov_iter input;779	int err;780 781	iov_iter_kvec(&input, ITER_SOURCE, inputs, nr_inputs, src_total_len);782	err = build_test_sglist(&tsgls->src, cfg->src_divs, alignmask,783				cfg->inplace_mode != OUT_OF_PLACE ?784					max(dst_total_len, src_total_len) :785					src_total_len,786				&input, NULL);787	if (err)788		return err;789 790	/*791	 * In-place crypto operations can use the same scatterlist for both the792	 * source and destination (req->src == req->dst), or can use separate793	 * scatterlists (req->src != req->dst) which point to the same794	 * underlying memory.  Make sure to test both cases.795	 */796	if (cfg->inplace_mode == INPLACE_ONE_SGLIST) {797		tsgls->dst.sgl_ptr = tsgls->src.sgl;798		tsgls->dst.nents = tsgls->src.nents;799		return 0;800	}801	if (cfg->inplace_mode == INPLACE_TWO_SGLISTS) {802		/*803		 * For now we keep it simple and only test the case where the804		 * two scatterlists have identical entries, rather than805		 * different entries that split up the same memory differently.806		 */807		memcpy(tsgls->dst.sgl, tsgls->src.sgl,808		       tsgls->src.nents * sizeof(tsgls->src.sgl[0]));809		memcpy(tsgls->dst.sgl_saved, tsgls->src.sgl,810		       tsgls->src.nents * sizeof(tsgls->src.sgl[0]));811		tsgls->dst.sgl_ptr = tsgls->dst.sgl;812		tsgls->dst.nents = tsgls->src.nents;813		return 0;814	}815	/* Out of place */816	return build_test_sglist(&tsgls->dst,817				 cfg->dst_divs[0].proportion_of_total ?818					cfg->dst_divs : cfg->src_divs,819				 alignmask, dst_total_len, NULL, NULL);820}821 822/*823 * Support for testing passing a misaligned key to setkey():824 *825 * If cfg->key_offset is set, copy the key into a new buffer at that offset,826 * optionally adding alignmask.  Else, just use the key directly.827 */828static int prepare_keybuf(const u8 *key, unsigned int ksize,829			  const struct testvec_config *cfg,830			  unsigned int alignmask,831			  const u8 **keybuf_ret, const u8 **keyptr_ret)832{833	unsigned int key_offset = cfg->key_offset;834	u8 *keybuf = NULL, *keyptr = (u8 *)key;835 836	if (key_offset != 0) {837		if (cfg->key_offset_relative_to_alignmask)838			key_offset += alignmask;839		keybuf = kmalloc(key_offset + ksize, GFP_KERNEL);840		if (!keybuf)841			return -ENOMEM;842		keyptr = keybuf + key_offset;843		memcpy(keyptr, key, ksize);844	}845	*keybuf_ret = keybuf;846	*keyptr_ret = keyptr;847	return 0;848}849 850/*851 * Like setkey_f(tfm, key, ksize), but sometimes misalign the key.852 * In addition, run the setkey function in no-SIMD context if requested.853 */854#define do_setkey(setkey_f, tfm, key, ksize, cfg, alignmask)		\855({									\856	const u8 *keybuf, *keyptr;					\857	int err;							\858									\859	err = prepare_keybuf((key), (ksize), (cfg), (alignmask),	\860			     &keybuf, &keyptr);				\861	if (err == 0) {							\862		if ((cfg)->nosimd_setkey)				\863			crypto_disable_simd_for_test();			\864		err = setkey_f((tfm), keyptr, (ksize));			\865		if ((cfg)->nosimd_setkey)				\866			crypto_reenable_simd_for_test();		\867		kfree(keybuf);						\868	}								\869	err;								\870})871 872#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS873 874/*875 * The fuzz tests use prandom instead of the normal Linux RNG since they don't876 * need cryptographically secure random numbers.  This greatly improves the877 * performance of these tests, especially if they are run before the Linux RNG878 * has been initialized or if they are run on a lockdep-enabled kernel.879 */880 881static inline void init_rnd_state(struct rnd_state *rng)882{883	prandom_seed_state(rng, get_random_u64());884}885 886static inline u8 prandom_u8(struct rnd_state *rng)887{888	return prandom_u32_state(rng);889}890 891static inline u32 prandom_u32_below(struct rnd_state *rng, u32 ceil)892{893	/*894	 * This is slightly biased for non-power-of-2 values of 'ceil', but this895	 * isn't important here.896	 */897	return prandom_u32_state(rng) % ceil;898}899 900static inline bool prandom_bool(struct rnd_state *rng)901{902	return prandom_u32_below(rng, 2);903}904 905static inline u32 prandom_u32_inclusive(struct rnd_state *rng,906					u32 floor, u32 ceil)907{908	return floor + prandom_u32_below(rng, ceil - floor + 1);909}910 911/* Generate a random length in range [0, max_len], but prefer smaller values */912static unsigned int generate_random_length(struct rnd_state *rng,913					   unsigned int max_len)914{915	unsigned int len = prandom_u32_below(rng, max_len + 1);916 917	switch (prandom_u32_below(rng, 4)) {918	case 0:919		len %= 64;920		break;921	case 1:922		len %= 256;923		break;924	case 2:925		len %= 1024;926		break;927	default:928		break;929	}930	if (len && prandom_u32_below(rng, 4) == 0)931		len = rounddown_pow_of_two(len);932	return len;933}934 935/* Flip a random bit in the given nonempty data buffer */936static void flip_random_bit(struct rnd_state *rng, u8 *buf, size_t size)937{938	size_t bitpos;939 940	bitpos = prandom_u32_below(rng, size * 8);941	buf[bitpos / 8] ^= 1 << (bitpos % 8);942}943 944/* Flip a random byte in the given nonempty data buffer */945static void flip_random_byte(struct rnd_state *rng, u8 *buf, size_t size)946{947	buf[prandom_u32_below(rng, size)] ^= 0xff;948}949 950/* Sometimes make some random changes to the given nonempty data buffer */951static void mutate_buffer(struct rnd_state *rng, u8 *buf, size_t size)952{953	size_t num_flips;954	size_t i;955 956	/* Sometimes flip some bits */957	if (prandom_u32_below(rng, 4) == 0) {958		num_flips = min_t(size_t, 1 << prandom_u32_below(rng, 8),959				  size * 8);960		for (i = 0; i < num_flips; i++)961			flip_random_bit(rng, buf, size);962	}963 964	/* Sometimes flip some bytes */965	if (prandom_u32_below(rng, 4) == 0) {966		num_flips = min_t(size_t, 1 << prandom_u32_below(rng, 8), size);967		for (i = 0; i < num_flips; i++)968			flip_random_byte(rng, buf, size);969	}970}971 972/* Randomly generate 'count' bytes, but sometimes make them "interesting" */973static void generate_random_bytes(struct rnd_state *rng, u8 *buf, size_t count)974{975	u8 b;976	u8 increment;977	size_t i;978 979	if (count == 0)980		return;981 982	switch (prandom_u32_below(rng, 8)) { /* Choose a generation strategy */983	case 0:984	case 1:985		/* All the same byte, plus optional mutations */986		switch (prandom_u32_below(rng, 4)) {987		case 0:988			b = 0x00;989			break;990		case 1:991			b = 0xff;992			break;993		default:994			b = prandom_u8(rng);995			break;996		}997		memset(buf, b, count);998		mutate_buffer(rng, buf, count);999		break;1000	case 2:1001		/* Ascending or descending bytes, plus optional mutations */1002		increment = prandom_u8(rng);1003		b = prandom_u8(rng);1004		for (i = 0; i < count; i++, b += increment)1005			buf[i] = b;1006		mutate_buffer(rng, buf, count);1007		break;1008	default:1009		/* Fully random bytes */1010		prandom_bytes_state(rng, buf, count);1011	}1012}1013 1014static char *generate_random_sgl_divisions(struct rnd_state *rng,1015					   struct test_sg_division *divs,1016					   size_t max_divs, char *p, char *end,1017					   bool gen_flushes, u32 req_flags)1018{1019	struct test_sg_division *div = divs;1020	unsigned int remaining = TEST_SG_TOTAL;1021 1022	do {1023		unsigned int this_len;1024		const char *flushtype_str;1025 1026		if (div == &divs[max_divs - 1] || prandom_bool(rng))1027			this_len = remaining;1028		else if (prandom_u32_below(rng, 4) == 0)1029			this_len = (remaining + 1) / 2;1030		else1031			this_len = prandom_u32_inclusive(rng, 1, remaining);1032		div->proportion_of_total = this_len;1033 1034		if (prandom_u32_below(rng, 4) == 0)1035			div->offset = prandom_u32_inclusive(rng,1036							    PAGE_SIZE - 128,1037							    PAGE_SIZE - 1);1038		else if (prandom_bool(rng))1039			div->offset = prandom_u32_below(rng, 32);1040		else1041			div->offset = prandom_u32_below(rng, PAGE_SIZE);1042		if (prandom_u32_below(rng, 8) == 0)1043			div->offset_relative_to_alignmask = true;1044 1045		div->flush_type = FLUSH_TYPE_NONE;1046		if (gen_flushes) {1047			switch (prandom_u32_below(rng, 4)) {1048			case 0:1049				div->flush_type = FLUSH_TYPE_REIMPORT;1050				break;1051			case 1:1052				div->flush_type = FLUSH_TYPE_FLUSH;1053				break;1054			}1055		}1056 1057		if (div->flush_type != FLUSH_TYPE_NONE &&1058		    !(req_flags & CRYPTO_TFM_REQ_MAY_SLEEP) &&1059		    prandom_bool(rng))1060			div->nosimd = true;1061 1062		switch (div->flush_type) {1063		case FLUSH_TYPE_FLUSH:1064			if (div->nosimd)1065				flushtype_str = "<flush,nosimd>";1066			else1067				flushtype_str = "<flush>";1068			break;1069		case FLUSH_TYPE_REIMPORT:1070			if (div->nosimd)1071				flushtype_str = "<reimport,nosimd>";1072			else1073				flushtype_str = "<reimport>";1074			break;1075		default:1076			flushtype_str = "";1077			break;1078		}1079 1080		BUILD_BUG_ON(TEST_SG_TOTAL != 10000); /* for "%u.%u%%" */1081		p += scnprintf(p, end - p, "%s%u.%u%%@%s+%u%s", flushtype_str,1082			       this_len / 100, this_len % 100,1083			       div->offset_relative_to_alignmask ?1084					"alignmask" : "",1085			       div->offset, this_len == remaining ? "" : ", ");1086		remaining -= this_len;1087		div++;1088	} while (remaining);1089 1090	return p;1091}1092 1093/* Generate a random testvec_config for fuzz testing */1094static void generate_random_testvec_config(struct rnd_state *rng,1095					   struct testvec_config *cfg,1096					   char *name, size_t max_namelen)1097{1098	char *p = name;1099	char * const end = name + max_namelen;1100 1101	memset(cfg, 0, sizeof(*cfg));1102 1103	cfg->name = name;1104 1105	p += scnprintf(p, end - p, "random:");1106 1107	switch (prandom_u32_below(rng, 4)) {1108	case 0:1109	case 1:1110		cfg->inplace_mode = OUT_OF_PLACE;1111		break;1112	case 2:1113		cfg->inplace_mode = INPLACE_ONE_SGLIST;1114		p += scnprintf(p, end - p, " inplace_one_sglist");1115		break;1116	default:1117		cfg->inplace_mode = INPLACE_TWO_SGLISTS;1118		p += scnprintf(p, end - p, " inplace_two_sglists");1119		break;1120	}1121 1122	if (prandom_bool(rng)) {1123		cfg->req_flags |= CRYPTO_TFM_REQ_MAY_SLEEP;1124		p += scnprintf(p, end - p, " may_sleep");1125	}1126 1127	switch (prandom_u32_below(rng, 4)) {1128	case 0:1129		cfg->finalization_type = FINALIZATION_TYPE_FINAL;1130		p += scnprintf(p, end - p, " use_final");1131		break;1132	case 1:1133		cfg->finalization_type = FINALIZATION_TYPE_FINUP;1134		p += scnprintf(p, end - p, " use_finup");1135		break;1136	default:1137		cfg->finalization_type = FINALIZATION_TYPE_DIGEST;1138		p += scnprintf(p, end - p, " use_digest");1139		break;1140	}1141 1142	if (!(cfg->req_flags & CRYPTO_TFM_REQ_MAY_SLEEP)) {1143		if (prandom_bool(rng)) {1144			cfg->nosimd = true;1145			p += scnprintf(p, end - p, " nosimd");1146		}1147		if (prandom_bool(rng)) {1148			cfg->nosimd_setkey = true;1149			p += scnprintf(p, end - p, " nosimd_setkey");1150		}1151	}1152 1153	p += scnprintf(p, end - p, " src_divs=[");1154	p = generate_random_sgl_divisions(rng, cfg->src_divs,1155					  ARRAY_SIZE(cfg->src_divs), p, end,1156					  (cfg->finalization_type !=1157					   FINALIZATION_TYPE_DIGEST),1158					  cfg->req_flags);1159	p += scnprintf(p, end - p, "]");1160 1161	if (cfg->inplace_mode == OUT_OF_PLACE && prandom_bool(rng)) {1162		p += scnprintf(p, end - p, " dst_divs=[");1163		p = generate_random_sgl_divisions(rng, cfg->dst_divs,1164						  ARRAY_SIZE(cfg->dst_divs),1165						  p, end, false,1166						  cfg->req_flags);1167		p += scnprintf(p, end - p, "]");1168	}1169 1170	if (prandom_bool(rng)) {1171		cfg->iv_offset = prandom_u32_inclusive(rng, 1,1172						       MAX_ALGAPI_ALIGNMASK);1173		p += scnprintf(p, end - p, " iv_offset=%u", cfg->iv_offset);1174	}1175 1176	if (prandom_bool(rng)) {1177		cfg->key_offset = prandom_u32_inclusive(rng, 1,1178							MAX_ALGAPI_ALIGNMASK);1179		p += scnprintf(p, end - p, " key_offset=%u", cfg->key_offset);1180	}1181 1182	WARN_ON_ONCE(!valid_testvec_config(cfg));1183}1184 1185static void crypto_disable_simd_for_test(void)1186{1187	migrate_disable();1188	__this_cpu_write(crypto_simd_disabled_for_test, true);1189}1190 1191static void crypto_reenable_simd_for_test(void)1192{1193	__this_cpu_write(crypto_simd_disabled_for_test, false);1194	migrate_enable();1195}1196 1197/*1198 * Given an algorithm name, build the name of the generic implementation of that1199 * algorithm, assuming the usual naming convention.  Specifically, this appends1200 * "-generic" to every part of the name that is not a template name.  Examples:1201 *1202 *	aes => aes-generic1203 *	cbc(aes) => cbc(aes-generic)1204 *	cts(cbc(aes)) => cts(cbc(aes-generic))1205 *	rfc7539(chacha20,poly1305) => rfc7539(chacha20-generic,poly1305-generic)1206 *1207 * Return: 0 on success, or -ENAMETOOLONG if the generic name would be too long1208 */1209static int build_generic_driver_name(const char *algname,1210				     char driver_name[CRYPTO_MAX_ALG_NAME])1211{1212	const char *in = algname;1213	char *out = driver_name;1214	size_t len = strlen(algname);1215 1216	if (len >= CRYPTO_MAX_ALG_NAME)1217		goto too_long;1218	do {1219		const char *in_saved = in;1220 1221		while (*in && *in != '(' && *in != ')' && *in != ',')1222			*out++ = *in++;1223		if (*in != '(' && in > in_saved) {1224			len += 8;1225			if (len >= CRYPTO_MAX_ALG_NAME)1226				goto too_long;1227			memcpy(out, "-generic", 8);1228			out += 8;1229		}1230	} while ((*out++ = *in++) != '\0');1231	return 0;1232 1233too_long:1234	pr_err("alg: generic driver name for \"%s\" would be too long\n",1235	       algname);1236	return -ENAMETOOLONG;1237}1238#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */1239static void crypto_disable_simd_for_test(void)1240{1241}1242 1243static void crypto_reenable_simd_for_test(void)1244{1245}1246#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */1247 1248static int build_hash_sglist(struct test_sglist *tsgl,1249			     const struct hash_testvec *vec,1250			     const struct testvec_config *cfg,1251			     unsigned int alignmask,1252			     const struct test_sg_division *divs[XBUFSIZE])1253{1254	struct kvec kv;1255	struct iov_iter input;1256 1257	kv.iov_base = (void *)vec->plaintext;1258	kv.iov_len = vec->psize;1259	iov_iter_kvec(&input, ITER_SOURCE, &kv, 1, vec->psize);1260	return build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize,1261				 &input, divs);1262}1263 1264static int check_hash_result(const char *type,1265			     const u8 *result, unsigned int digestsize,1266			     const struct hash_testvec *vec,1267			     const char *vec_name,1268			     const char *driver,1269			     const struct testvec_config *cfg)1270{1271	if (memcmp(result, vec->digest, digestsize) != 0) {1272		pr_err("alg: %s: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",1273		       type, driver, vec_name, cfg->name);1274		return -EINVAL;1275	}1276	if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) {1277		pr_err("alg: %s: %s overran result buffer on test vector %s, cfg=\"%s\"\n",1278		       type, driver, vec_name, cfg->name);1279		return -EOVERFLOW;1280	}1281	return 0;1282}1283 1284static inline int check_shash_op(const char *op, int err,1285				 const char *driver, const char *vec_name,1286				 const struct testvec_config *cfg)1287{1288	if (err)1289		pr_err("alg: shash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",1290		       driver, op, err, vec_name, cfg->name);1291	return err;1292}1293 1294/* Test one hash test vector in one configuration, using the shash API */1295static int test_shash_vec_cfg(const struct hash_testvec *vec,1296			      const char *vec_name,1297			      const struct testvec_config *cfg,1298			      struct shash_desc *desc,1299			      struct test_sglist *tsgl,1300			      u8 *hashstate)1301{1302	struct crypto_shash *tfm = desc->tfm;1303	const unsigned int digestsize = crypto_shash_digestsize(tfm);1304	const unsigned int statesize = crypto_shash_statesize(tfm);1305	const char *driver = crypto_shash_driver_name(tfm);1306	const struct test_sg_division *divs[XBUFSIZE];1307	unsigned int i;1308	u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN];1309	int err;1310 1311	/* Set the key, if specified */1312	if (vec->ksize) {1313		err = do_setkey(crypto_shash_setkey, tfm, vec->key, vec->ksize,1314				cfg, 0);1315		if (err) {1316			if (err == vec->setkey_error)1317				return 0;1318			pr_err("alg: shash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",1319			       driver, vec_name, vec->setkey_error, err,1320			       crypto_shash_get_flags(tfm));1321			return err;1322		}1323		if (vec->setkey_error) {1324			pr_err("alg: shash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",1325			       driver, vec_name, vec->setkey_error);1326			return -EINVAL;1327		}1328	}1329 1330	/* Build the scatterlist for the source data */1331	err = build_hash_sglist(tsgl, vec, cfg, 0, divs);1332	if (err) {1333		pr_err("alg: shash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",1334		       driver, vec_name, cfg->name);1335		return err;1336	}1337 1338	/* Do the actual hashing */1339 1340	testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm));1341	testmgr_poison(result, digestsize + TESTMGR_POISON_LEN);1342 1343	if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST ||1344	    vec->digest_error) {1345		/* Just using digest() */1346		if (tsgl->nents != 1)1347			return 0;1348		if (cfg->nosimd)1349			crypto_disable_simd_for_test();1350		err = crypto_shash_digest(desc, sg_virt(&tsgl->sgl[0]),1351					  tsgl->sgl[0].length, result);1352		if (cfg->nosimd)1353			crypto_reenable_simd_for_test();1354		if (err) {1355			if (err == vec->digest_error)1356				return 0;1357			pr_err("alg: shash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",1358			       driver, vec_name, vec->digest_error, err,1359			       cfg->name);1360			return err;1361		}1362		if (vec->digest_error) {1363			pr_err("alg: shash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",1364			       driver, vec_name, vec->digest_error, cfg->name);1365			return -EINVAL;1366		}1367		goto result_ready;1368	}1369 1370	/* Using init(), zero or more update(), then final() or finup() */1371 1372	if (cfg->nosimd)1373		crypto_disable_simd_for_test();1374	err = crypto_shash_init(desc);1375	if (cfg->nosimd)1376		crypto_reenable_simd_for_test();1377	err = check_shash_op("init", err, driver, vec_name, cfg);1378	if (err)1379		return err;1380 1381	for (i = 0; i < tsgl->nents; i++) {1382		if (i + 1 == tsgl->nents &&1383		    cfg->finalization_type == FINALIZATION_TYPE_FINUP) {1384			if (divs[i]->nosimd)1385				crypto_disable_simd_for_test();1386			err = crypto_shash_finup(desc, sg_virt(&tsgl->sgl[i]),1387						 tsgl->sgl[i].length, result);1388			if (divs[i]->nosimd)1389				crypto_reenable_simd_for_test();1390			err = check_shash_op("finup", err, driver, vec_name,1391					     cfg);1392			if (err)1393				return err;1394			goto result_ready;1395		}1396		if (divs[i]->nosimd)1397			crypto_disable_simd_for_test();1398		err = crypto_shash_update(desc, sg_virt(&tsgl->sgl[i]),1399					  tsgl->sgl[i].length);1400		if (divs[i]->nosimd)1401			crypto_reenable_simd_for_test();1402		err = check_shash_op("update", err, driver, vec_name, cfg);1403		if (err)1404			return err;1405		if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) {1406			/* Test ->export() and ->import() */1407			testmgr_poison(hashstate + statesize,1408				       TESTMGR_POISON_LEN);1409			err = crypto_shash_export(desc, hashstate);1410			err = check_shash_op("export", err, driver, vec_name,1411					     cfg);1412			if (err)1413				return err;1414			if (!testmgr_is_poison(hashstate + statesize,1415					       TESTMGR_POISON_LEN)) {1416				pr_err("alg: shash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",1417				       driver, vec_name, cfg->name);1418				return -EOVERFLOW;1419			}1420			testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm));1421			err = crypto_shash_import(desc, hashstate);1422			err = check_shash_op("import", err, driver, vec_name,1423					     cfg);1424			if (err)1425				return err;1426		}1427	}1428 1429	if (cfg->nosimd)1430		crypto_disable_simd_for_test();1431	err = crypto_shash_final(desc, result);1432	if (cfg->nosimd)1433		crypto_reenable_simd_for_test();1434	err = check_shash_op("final", err, driver, vec_name, cfg);1435	if (err)1436		return err;1437result_ready:1438	return check_hash_result("shash", result, digestsize, vec, vec_name,1439				 driver, cfg);1440}1441 1442static int do_ahash_op(int (*op)(struct ahash_request *req),1443		       struct ahash_request *req,1444		       struct crypto_wait *wait, bool nosimd)1445{1446	int err;1447 1448	if (nosimd)1449		crypto_disable_simd_for_test();1450 1451	err = op(req);1452 1453	if (nosimd)1454		crypto_reenable_simd_for_test();1455 1456	return crypto_wait_req(err, wait);1457}1458 1459static int check_nonfinal_ahash_op(const char *op, int err,1460				   u8 *result, unsigned int digestsize,1461				   const char *driver, const char *vec_name,1462				   const struct testvec_config *cfg)1463{1464	if (err) {1465		pr_err("alg: ahash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n",1466		       driver, op, err, vec_name, cfg->name);1467		return err;1468	}1469	if (!testmgr_is_poison(result, digestsize)) {1470		pr_err("alg: ahash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n",1471		       driver, op, vec_name, cfg->name);1472		return -EINVAL;1473	}1474	return 0;1475}1476 1477/* Test one hash test vector in one configuration, using the ahash API */1478static int test_ahash_vec_cfg(const struct hash_testvec *vec,1479			      const char *vec_name,1480			      const struct testvec_config *cfg,1481			      struct ahash_request *req,1482			      struct test_sglist *tsgl,1483			      u8 *hashstate)1484{1485	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);1486	const unsigned int digestsize = crypto_ahash_digestsize(tfm);1487	const unsigned int statesize = crypto_ahash_statesize(tfm);1488	const char *driver = crypto_ahash_driver_name(tfm);1489	const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;1490	const struct test_sg_division *divs[XBUFSIZE];1491	DECLARE_CRYPTO_WAIT(wait);1492	unsigned int i;1493	struct scatterlist *pending_sgl;1494	unsigned int pending_len;1495	u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN];1496	int err;1497 1498	/* Set the key, if specified */1499	if (vec->ksize) {1500		err = do_setkey(crypto_ahash_setkey, tfm, vec->key, vec->ksize,1501				cfg, 0);1502		if (err) {1503			if (err == vec->setkey_error)1504				return 0;1505			pr_err("alg: ahash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",1506			       driver, vec_name, vec->setkey_error, err,1507			       crypto_ahash_get_flags(tfm));1508			return err;1509		}1510		if (vec->setkey_error) {1511			pr_err("alg: ahash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",1512			       driver, vec_name, vec->setkey_error);1513			return -EINVAL;1514		}1515	}1516 1517	/* Build the scatterlist for the source data */1518	err = build_hash_sglist(tsgl, vec, cfg, 0, divs);1519	if (err) {1520		pr_err("alg: ahash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n",1521		       driver, vec_name, cfg->name);1522		return err;1523	}1524 1525	/* Do the actual hashing */1526 1527	testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm));1528	testmgr_poison(result, digestsize + TESTMGR_POISON_LEN);1529 1530	if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST ||1531	    vec->digest_error) {1532		/* Just using digest() */1533		ahash_request_set_callback(req, req_flags, crypto_req_done,1534					   &wait);1535		ahash_request_set_crypt(req, tsgl->sgl, result, vec->psize);1536		err = do_ahash_op(crypto_ahash_digest, req, &wait, cfg->nosimd);1537		if (err) {1538			if (err == vec->digest_error)1539				return 0;1540			pr_err("alg: ahash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",1541			       driver, vec_name, vec->digest_error, err,1542			       cfg->name);1543			return err;1544		}1545		if (vec->digest_error) {1546			pr_err("alg: ahash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",1547			       driver, vec_name, vec->digest_error, cfg->name);1548			return -EINVAL;1549		}1550		goto result_ready;1551	}1552 1553	/* Using init(), zero or more update(), then final() or finup() */1554 1555	ahash_request_set_callback(req, req_flags, crypto_req_done, &wait);1556	ahash_request_set_crypt(req, NULL, result, 0);1557	err = do_ahash_op(crypto_ahash_init, req, &wait, cfg->nosimd);1558	err = check_nonfinal_ahash_op("init", err, result, digestsize,1559				      driver, vec_name, cfg);1560	if (err)1561		return err;1562 1563	pending_sgl = NULL;1564	pending_len = 0;1565	for (i = 0; i < tsgl->nents; i++) {1566		if (divs[i]->flush_type != FLUSH_TYPE_NONE &&1567		    pending_sgl != NULL) {1568			/* update() with the pending data */1569			ahash_request_set_callback(req, req_flags,1570						   crypto_req_done, &wait);1571			ahash_request_set_crypt(req, pending_sgl, result,1572						pending_len);1573			err = do_ahash_op(crypto_ahash_update, req, &wait,1574					  divs[i]->nosimd);1575			err = check_nonfinal_ahash_op("update", err,1576						      result, digestsize,1577						      driver, vec_name, cfg);1578			if (err)1579				return err;1580			pending_sgl = NULL;1581			pending_len = 0;1582		}1583		if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) {1584			/* Test ->export() and ->import() */1585			testmgr_poison(hashstate + statesize,1586				       TESTMGR_POISON_LEN);1587			err = crypto_ahash_export(req, hashstate);1588			err = check_nonfinal_ahash_op("export", err,1589						      result, digestsize,1590						      driver, vec_name, cfg);1591			if (err)1592				return err;1593			if (!testmgr_is_poison(hashstate + statesize,1594					       TESTMGR_POISON_LEN)) {1595				pr_err("alg: ahash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n",1596				       driver, vec_name, cfg->name);1597				return -EOVERFLOW;1598			}1599 1600			testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm));1601			err = crypto_ahash_import(req, hashstate);1602			err = check_nonfinal_ahash_op("import", err,1603						      result, digestsize,1604						      driver, vec_name, cfg);1605			if (err)1606				return err;1607		}1608		if (pending_sgl == NULL)1609			pending_sgl = &tsgl->sgl[i];1610		pending_len += tsgl->sgl[i].length;1611	}1612 1613	ahash_request_set_callback(req, req_flags, crypto_req_done, &wait);1614	ahash_request_set_crypt(req, pending_sgl, result, pending_len);1615	if (cfg->finalization_type == FINALIZATION_TYPE_FINAL) {1616		/* finish with update() and final() */1617		err = do_ahash_op(crypto_ahash_update, req, &wait, cfg->nosimd);1618		err = check_nonfinal_ahash_op("update", err, result, digestsize,1619					      driver, vec_name, cfg);1620		if (err)1621			return err;1622		err = do_ahash_op(crypto_ahash_final, req, &wait, cfg->nosimd);1623		if (err) {1624			pr_err("alg: ahash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n",1625			       driver, err, vec_name, cfg->name);1626			return err;1627		}1628	} else {1629		/* finish with finup() */1630		err = do_ahash_op(crypto_ahash_finup, req, &wait, cfg->nosimd);1631		if (err) {1632			pr_err("alg: ahash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n",1633			       driver, err, vec_name, cfg->name);1634			return err;1635		}1636	}1637 1638result_ready:1639	return check_hash_result("ahash", result, digestsize, vec, vec_name,1640				 driver, cfg);1641}1642 1643static int test_hash_vec_cfg(const struct hash_testvec *vec,1644			     const char *vec_name,1645			     const struct testvec_config *cfg,1646			     struct ahash_request *req,1647			     struct shash_desc *desc,1648			     struct test_sglist *tsgl,1649			     u8 *hashstate)1650{1651	int err;1652 1653	/*1654	 * For algorithms implemented as "shash", most bugs will be detected by1655	 * both the shash and ahash tests.  Test the shash API first so that the1656	 * failures involve less indirection, so are easier to debug.1657	 */1658 1659	if (desc) {1660		err = test_shash_vec_cfg(vec, vec_name, cfg, desc, tsgl,1661					 hashstate);1662		if (err)1663			return err;1664	}1665 1666	return test_ahash_vec_cfg(vec, vec_name, cfg, req, tsgl, hashstate);1667}1668 1669static int test_hash_vec(const struct hash_testvec *vec, unsigned int vec_num,1670			 struct ahash_request *req, struct shash_desc *desc,1671			 struct test_sglist *tsgl, u8 *hashstate)1672{1673	char vec_name[16];1674	unsigned int i;1675	int err;1676 1677	sprintf(vec_name, "%u", vec_num);1678 1679	for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++) {1680		err = test_hash_vec_cfg(vec, vec_name,1681					&default_hash_testvec_configs[i],1682					req, desc, tsgl, hashstate);1683		if (err)1684			return err;1685	}1686 1687#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS1688	if (!noextratests) {1689		struct rnd_state rng;1690		struct testvec_config cfg;1691		char cfgname[TESTVEC_CONFIG_NAMELEN];1692 1693		init_rnd_state(&rng);1694 1695		for (i = 0; i < fuzz_iterations; i++) {1696			generate_random_testvec_config(&rng, &cfg, cfgname,1697						       sizeof(cfgname));1698			err = test_hash_vec_cfg(vec, vec_name, &cfg,1699						req, desc, tsgl, hashstate);1700			if (err)1701				return err;1702			cond_resched();1703		}1704	}1705#endif1706	return 0;1707}1708 1709#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS1710/*1711 * Generate a hash test vector from the given implementation.1712 * Assumes the buffers in 'vec' were already allocated.1713 */1714static void generate_random_hash_testvec(struct rnd_state *rng,1715					 struct shash_desc *desc,1716					 struct hash_testvec *vec,1717					 unsigned int maxkeysize,1718					 unsigned int maxdatasize,1719					 char *name, size_t max_namelen)1720{1721	/* Data */1722	vec->psize = generate_random_length(rng, maxdatasize);1723	generate_random_bytes(rng, (u8 *)vec->plaintext, vec->psize);1724 1725	/*1726	 * Key: length in range [1, maxkeysize], but usually choose maxkeysize.1727	 * If algorithm is unkeyed, then maxkeysize == 0 and set ksize = 0.1728	 */1729	vec->setkey_error = 0;1730	vec->ksize = 0;1731	if (maxkeysize) {1732		vec->ksize = maxkeysize;1733		if (prandom_u32_below(rng, 4) == 0)1734			vec->ksize = prandom_u32_inclusive(rng, 1, maxkeysize);1735		generate_random_bytes(rng, (u8 *)vec->key, vec->ksize);1736 1737		vec->setkey_error = crypto_shash_setkey(desc->tfm, vec->key,1738							vec->ksize);1739		/* If the key couldn't be set, no need to continue to digest. */1740		if (vec->setkey_error)1741			goto done;1742	}1743 1744	/* Digest */1745	vec->digest_error = crypto_shash_digest(desc, vec->plaintext,1746						vec->psize, (u8 *)vec->digest);1747done:1748	snprintf(name, max_namelen, "\"random: psize=%u ksize=%u\"",1749		 vec->psize, vec->ksize);1750}1751 1752/*1753 * Test the hash algorithm represented by @req against the corresponding generic1754 * implementation, if one is available.1755 */1756static int test_hash_vs_generic_impl(const char *generic_driver,1757				     unsigned int maxkeysize,1758				     struct ahash_request *req,1759				     struct shash_desc *desc,1760				     struct test_sglist *tsgl,1761				     u8 *hashstate)1762{1763	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);1764	const unsigned int digestsize = crypto_ahash_digestsize(tfm);1765	const unsigned int blocksize = crypto_ahash_blocksize(tfm);1766	const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;1767	const char *algname = crypto_hash_alg_common(tfm)->base.cra_name;1768	const char *driver = crypto_ahash_driver_name(tfm);1769	struct rnd_state rng;1770	char _generic_driver[CRYPTO_MAX_ALG_NAME];1771	struct crypto_shash *generic_tfm = NULL;1772	struct shash_desc *generic_desc = NULL;1773	unsigned int i;1774	struct hash_testvec vec = { 0 };1775	char vec_name[64];1776	struct testvec_config *cfg;1777	char cfgname[TESTVEC_CONFIG_NAMELEN];1778	int err;1779 1780	if (noextratests)1781		return 0;1782 1783	init_rnd_state(&rng);1784 1785	if (!generic_driver) { /* Use default naming convention? */1786		err = build_generic_driver_name(algname, _generic_driver);1787		if (err)1788			return err;1789		generic_driver = _generic_driver;1790	}1791 1792	if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */1793		return 0;1794 1795	generic_tfm = crypto_alloc_shash(generic_driver, 0, 0);1796	if (IS_ERR(generic_tfm)) {1797		err = PTR_ERR(generic_tfm);1798		if (err == -ENOENT) {1799			pr_warn("alg: hash: skipping comparison tests for %s because %s is unavailable\n",1800				driver, generic_driver);1801			return 0;1802		}1803		pr_err("alg: hash: error allocating %s (generic impl of %s): %d\n",1804		       generic_driver, algname, err);1805		return err;1806	}1807 1808	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);1809	if (!cfg) {1810		err = -ENOMEM;1811		goto out;1812	}1813 1814	generic_desc = kzalloc(sizeof(*desc) +1815			       crypto_shash_descsize(generic_tfm), GFP_KERNEL);1816	if (!generic_desc) {1817		err = -ENOMEM;1818		goto out;1819	}1820	generic_desc->tfm = generic_tfm;1821 1822	/* Check the algorithm properties for consistency. */1823 1824	if (digestsize != crypto_shash_digestsize(generic_tfm)) {1825		pr_err("alg: hash: digestsize for %s (%u) doesn't match generic impl (%u)\n",1826		       driver, digestsize,1827		       crypto_shash_digestsize(generic_tfm));1828		err = -EINVAL;1829		goto out;1830	}1831 1832	if (blocksize != crypto_shash_blocksize(generic_tfm)) {1833		pr_err("alg: hash: blocksize for %s (%u) doesn't match generic impl (%u)\n",1834		       driver, blocksize, crypto_shash_blocksize(generic_tfm));1835		err = -EINVAL;1836		goto out;1837	}1838 1839	/*1840	 * Now generate test vectors using the generic implementation, and test1841	 * the other implementation against them.1842	 */1843 1844	vec.key = kmalloc(maxkeysize, GFP_KERNEL);1845	vec.plaintext = kmalloc(maxdatasize, GFP_KERNEL);1846	vec.digest = kmalloc(digestsize, GFP_KERNEL);1847	if (!vec.key || !vec.plaintext || !vec.digest) {1848		err = -ENOMEM;1849		goto out;1850	}1851 1852	for (i = 0; i < fuzz_iterations * 8; i++) {1853		generate_random_hash_testvec(&rng, generic_desc, &vec,1854					     maxkeysize, maxdatasize,1855					     vec_name, sizeof(vec_name));1856		generate_random_testvec_config(&rng, cfg, cfgname,1857					       sizeof(cfgname));1858 1859		err = test_hash_vec_cfg(&vec, vec_name, cfg,1860					req, desc, tsgl, hashstate);1861		if (err)1862			goto out;1863		cond_resched();1864	}1865	err = 0;1866out:1867	kfree(cfg);1868	kfree(vec.key);1869	kfree(vec.plaintext);1870	kfree(vec.digest);1871	crypto_free_shash(generic_tfm);1872	kfree_sensitive(generic_desc);1873	return err;1874}1875#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */1876static int test_hash_vs_generic_impl(const char *generic_driver,1877				     unsigned int maxkeysize,1878				     struct ahash_request *req,1879				     struct shash_desc *desc,1880				     struct test_sglist *tsgl,1881				     u8 *hashstate)1882{1883	return 0;1884}1885#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */1886 1887static int alloc_shash(const char *driver, u32 type, u32 mask,1888		       struct crypto_shash **tfm_ret,1889		       struct shash_desc **desc_ret)1890{1891	struct crypto_shash *tfm;1892	struct shash_desc *desc;1893 1894	tfm = crypto_alloc_shash(driver, type, mask);1895	if (IS_ERR(tfm)) {1896		if (PTR_ERR(tfm) == -ENOENT) {1897			/*1898			 * This algorithm is only available through the ahash1899			 * API, not the shash API, so skip the shash tests.1900			 */1901			return 0;1902		}1903		pr_err("alg: hash: failed to allocate shash transform for %s: %ld\n",1904		       driver, PTR_ERR(tfm));1905		return PTR_ERR(tfm);1906	}1907 1908	desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(tfm), GFP_KERNEL);1909	if (!desc) {1910		crypto_free_shash(tfm);1911		return -ENOMEM;1912	}1913	desc->tfm = tfm;1914 1915	*tfm_ret = tfm;1916	*desc_ret = desc;1917	return 0;1918}1919 1920static int __alg_test_hash(const struct hash_testvec *vecs,1921			   unsigned int num_vecs, const char *driver,1922			   u32 type, u32 mask,1923			   const char *generic_driver, unsigned int maxkeysize)1924{1925	struct crypto_ahash *atfm = NULL;1926	struct ahash_request *req = NULL;1927	struct crypto_shash *stfm = NULL;1928	struct shash_desc *desc = NULL;1929	struct test_sglist *tsgl = NULL;1930	u8 *hashstate = NULL;1931	unsigned int statesize;1932	unsigned int i;1933	int err;1934 1935	/*1936	 * Always test the ahash API.  This works regardless of whether the1937	 * algorithm is implemented as ahash or shash.1938	 */1939 1940	atfm = crypto_alloc_ahash(driver, type, mask);1941	if (IS_ERR(atfm)) {1942		if (PTR_ERR(atfm) == -ENOENT)1943			return 0;1944		pr_err("alg: hash: failed to allocate transform for %s: %ld\n",1945		       driver, PTR_ERR(atfm));1946		return PTR_ERR(atfm);1947	}1948	driver = crypto_ahash_driver_name(atfm);1949 1950	req = ahash_request_alloc(atfm, GFP_KERNEL);1951	if (!req) {1952		pr_err("alg: hash: failed to allocate request for %s\n",1953		       driver);1954		err = -ENOMEM;1955		goto out;1956	}1957 1958	/*1959	 * If available also test the shash API, to cover corner cases that may1960	 * be missed by testing the ahash API only.1961	 */1962	err = alloc_shash(driver, type, mask, &stfm, &desc);1963	if (err)1964		goto out;1965 1966	tsgl = kmalloc(sizeof(*tsgl), GFP_KERNEL);1967	if (!tsgl || init_test_sglist(tsgl) != 0) {1968		pr_err("alg: hash: failed to allocate test buffers for %s\n",1969		       driver);1970		kfree(tsgl);1971		tsgl = NULL;1972		err = -ENOMEM;1973		goto out;1974	}1975 1976	statesize = crypto_ahash_statesize(atfm);1977	if (stfm)1978		statesize = max(statesize, crypto_shash_statesize(stfm));1979	hashstate = kmalloc(statesize + TESTMGR_POISON_LEN, GFP_KERNEL);1980	if (!hashstate) {1981		pr_err("alg: hash: failed to allocate hash state buffer for %s\n",1982		       driver);1983		err = -ENOMEM;1984		goto out;1985	}1986 1987	for (i = 0; i < num_vecs; i++) {1988		if (fips_enabled && vecs[i].fips_skip)1989			continue;1990 1991		err = test_hash_vec(&vecs[i], i, req, desc, tsgl, hashstate);1992		if (err)1993			goto out;1994		cond_resched();1995	}1996	err = test_hash_vs_generic_impl(generic_driver, maxkeysize, req,1997					desc, tsgl, hashstate);1998out:1999	kfree(hashstate);2000	if (tsgl) {2001		destroy_test_sglist(tsgl);2002		kfree(tsgl);2003	}2004	kfree(desc);2005	crypto_free_shash(stfm);2006	ahash_request_free(req);2007	crypto_free_ahash(atfm);2008	return err;2009}2010 2011static int alg_test_hash(const struct alg_test_desc *desc, const char *driver,2012			 u32 type, u32 mask)2013{2014	const struct hash_testvec *template = desc->suite.hash.vecs;2015	unsigned int tcount = desc->suite.hash.count;2016	unsigned int nr_unkeyed, nr_keyed;2017	unsigned int maxkeysize = 0;2018	int err;2019 2020	/*2021	 * For OPTIONAL_KEY algorithms, we have to do all the unkeyed tests2022	 * first, before setting a key on the tfm.  To make this easier, we2023	 * require that the unkeyed test vectors (if any) are listed first.2024	 */2025 2026	for (nr_unkeyed = 0; nr_unkeyed < tcount; nr_unkeyed++) {2027		if (template[nr_unkeyed].ksize)2028			break;2029	}2030	for (nr_keyed = 0; nr_unkeyed + nr_keyed < tcount; nr_keyed++) {2031		if (!template[nr_unkeyed + nr_keyed].ksize) {2032			pr_err("alg: hash: test vectors for %s out of order, "2033			       "unkeyed ones must come first\n", desc->alg);2034			return -EINVAL;2035		}2036		maxkeysize = max_t(unsigned int, maxkeysize,2037				   template[nr_unkeyed + nr_keyed].ksize);2038	}2039 2040	err = 0;2041	if (nr_unkeyed) {2042		err = __alg_test_hash(template, nr_unkeyed, driver, type, mask,2043				      desc->generic_driver, maxkeysize);2044		template += nr_unkeyed;2045	}2046 2047	if (!err && nr_keyed)2048		err = __alg_test_hash(template, nr_keyed, driver, type, mask,2049				      desc->generic_driver, maxkeysize);2050 2051	return err;2052}2053 2054static int test_aead_vec_cfg(int enc, const struct aead_testvec *vec,2055			     const char *vec_name,2056			     const struct testvec_config *cfg,2057			     struct aead_request *req,2058			     struct cipher_test_sglists *tsgls)2059{2060	struct crypto_aead *tfm = crypto_aead_reqtfm(req);2061	const unsigned int alignmask = crypto_aead_alignmask(tfm);2062	const unsigned int ivsize = crypto_aead_ivsize(tfm);2063	const unsigned int authsize = vec->clen - vec->plen;2064	const char *driver = crypto_aead_driver_name(tfm);2065	const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;2066	const char *op = enc ? "encryption" : "decryption";2067	DECLARE_CRYPTO_WAIT(wait);2068	u8 _iv[3 * (MAX_ALGAPI_ALIGNMASK + 1) + MAX_IVLEN];2069	u8 *iv = PTR_ALIGN(&_iv[0], 2 * (MAX_ALGAPI_ALIGNMASK + 1)) +2070		 cfg->iv_offset +2071		 (cfg->iv_offset_relative_to_alignmask ? alignmask : 0);2072	struct kvec input[2];2073	int err;2074 2075	/* Set the key */2076	if (vec->wk)2077		crypto_aead_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);2078	else2079		crypto_aead_clear_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);2080 2081	err = do_setkey(crypto_aead_setkey, tfm, vec->key, vec->klen,2082			cfg, alignmask);2083	if (err && err != vec->setkey_error) {2084		pr_err("alg: aead: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",2085		       driver, vec_name, vec->setkey_error, err,2086		       crypto_aead_get_flags(tfm));2087		return err;2088	}2089	if (!err && vec->setkey_error) {2090		pr_err("alg: aead: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",2091		       driver, vec_name, vec->setkey_error);2092		return -EINVAL;2093	}2094 2095	/* Set the authentication tag size */2096	err = crypto_aead_setauthsize(tfm, authsize);2097	if (err && err != vec->setauthsize_error) {2098		pr_err("alg: aead: %s setauthsize failed on test vector %s; expected_error=%d, actual_error=%d\n",2099		       driver, vec_name, vec->setauthsize_error, err);2100		return err;2101	}2102	if (!err && vec->setauthsize_error) {2103		pr_err("alg: aead: %s setauthsize unexpectedly succeeded on test vector %s; expected_error=%d\n",2104		       driver, vec_name, vec->setauthsize_error);2105		return -EINVAL;2106	}2107 2108	if (vec->setkey_error || vec->setauthsize_error)2109		return 0;2110 2111	/* The IV must be copied to a buffer, as the algorithm may modify it */2112	if (WARN_ON(ivsize > MAX_IVLEN))2113		return -EINVAL;2114	if (vec->iv)2115		memcpy(iv, vec->iv, ivsize);2116	else2117		memset(iv, 0, ivsize);2118 2119	/* Build the src/dst scatterlists */2120	input[0].iov_base = (void *)vec->assoc;2121	input[0].iov_len = vec->alen;2122	input[1].iov_base = enc ? (void *)vec->ptext : (void *)vec->ctext;2123	input[1].iov_len = enc ? vec->plen : vec->clen;2124	err = build_cipher_test_sglists(tsgls, cfg, alignmask,2125					vec->alen + (enc ? vec->plen :2126						     vec->clen),2127					vec->alen + (enc ? vec->clen :2128						     vec->plen),2129					input, 2);2130	if (err) {2131		pr_err("alg: aead: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n",2132		       driver, op, vec_name, cfg->name);2133		return err;2134	}2135 2136	/* Do the actual encryption or decryption */2137	testmgr_poison(req->__ctx, crypto_aead_reqsize(tfm));2138	aead_request_set_callback(req, req_flags, crypto_req_done, &wait);2139	aead_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr,2140			       enc ? vec->plen : vec->clen, iv);2141	aead_request_set_ad(req, vec->alen);2142	if (cfg->nosimd)2143		crypto_disable_simd_for_test();2144	err = enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req);2145	if (cfg->nosimd)2146		crypto_reenable_simd_for_test();2147	err = crypto_wait_req(err, &wait);2148 2149	/* Check that the algorithm didn't overwrite things it shouldn't have */2150	if (req->cryptlen != (enc ? vec->plen : vec->clen) ||2151	    req->assoclen != vec->alen ||2152	    req->iv != iv ||2153	    req->src != tsgls->src.sgl_ptr ||2154	    req->dst != tsgls->dst.sgl_ptr ||2155	    crypto_aead_reqtfm(req) != tfm ||2156	    req->base.complete != crypto_req_done ||2157	    req->base.flags != req_flags ||2158	    req->base.data != &wait) {2159		pr_err("alg: aead: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n",2160		       driver, op, vec_name, cfg->name);2161		if (req->cryptlen != (enc ? vec->plen : vec->clen))2162			pr_err("alg: aead: changed 'req->cryptlen'\n");2163		if (req->assoclen != vec->alen)2164			pr_err("alg: aead: changed 'req->assoclen'\n");2165		if (req->iv != iv)2166			pr_err("alg: aead: changed 'req->iv'\n");2167		if (req->src != tsgls->src.sgl_ptr)2168			pr_err("alg: aead: changed 'req->src'\n");2169		if (req->dst != tsgls->dst.sgl_ptr)2170			pr_err("alg: aead: changed 'req->dst'\n");2171		if (crypto_aead_reqtfm(req) != tfm)2172			pr_err("alg: aead: changed 'req->base.tfm'\n");2173		if (req->base.complete != crypto_req_done)2174			pr_err("alg: aead: changed 'req->base.complete'\n");2175		if (req->base.flags != req_flags)2176			pr_err("alg: aead: changed 'req->base.flags'\n");2177		if (req->base.data != &wait)2178			pr_err("alg: aead: changed 'req->base.data'\n");2179		return -EINVAL;2180	}2181	if (is_test_sglist_corrupted(&tsgls->src)) {2182		pr_err("alg: aead: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n",2183		       driver, op, vec_name, cfg->name);2184		return -EINVAL;2185	}2186	if (tsgls->dst.sgl_ptr != tsgls->src.sgl &&2187	    is_test_sglist_corrupted(&tsgls->dst)) {2188		pr_err("alg: aead: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n",2189		       driver, op, vec_name, cfg->name);2190		return -EINVAL;2191	}2192 2193	/* Check for unexpected success or failure, or wrong error code */2194	if ((err == 0 && vec->novrfy) ||2195	    (err != vec->crypt_error && !(err == -EBADMSG && vec->novrfy))) {2196		char expected_error[32];2197 2198		if (vec->novrfy &&2199		    vec->crypt_error != 0 && vec->crypt_error != -EBADMSG)2200			sprintf(expected_error, "-EBADMSG or %d",2201				vec->crypt_error);2202		else if (vec->novrfy)2203			sprintf(expected_error, "-EBADMSG");2204		else2205			sprintf(expected_error, "%d", vec->crypt_error);2206		if (err) {2207			pr_err("alg: aead: %s %s failed on test vector %s; expected_error=%s, actual_error=%d, cfg=\"%s\"\n",2208			       driver, op, vec_name, expected_error, err,2209			       cfg->name);2210			return err;2211		}2212		pr_err("alg: aead: %s %s unexpectedly succeeded on test vector %s; expected_error=%s, cfg=\"%s\"\n",2213		       driver, op, vec_name, expected_error, cfg->name);2214		return -EINVAL;2215	}2216	if (err) /* Expectedly failed. */2217		return 0;2218 2219	/* Check for the correct output (ciphertext or plaintext) */2220	err = verify_correct_output(&tsgls->dst, enc ? vec->ctext : vec->ptext,2221				    enc ? vec->clen : vec->plen,2222				    vec->alen,2223				    enc || cfg->inplace_mode == OUT_OF_PLACE);2224	if (err == -EOVERFLOW) {2225		pr_err("alg: aead: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n",2226		       driver, op, vec_name, cfg->name);2227		return err;2228	}2229	if (err) {2230		pr_err("alg: aead: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",2231		       driver, op, vec_name, cfg->name);2232		return err;2233	}2234 2235	return 0;2236}2237 2238static int test_aead_vec(int enc, const struct aead_testvec *vec,2239			 unsigned int vec_num, struct aead_request *req,2240			 struct cipher_test_sglists *tsgls)2241{2242	char vec_name[16];2243	unsigned int i;2244	int err;2245 2246	if (enc && vec->novrfy)2247		return 0;2248 2249	sprintf(vec_name, "%u", vec_num);2250 2251	for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) {2252		err = test_aead_vec_cfg(enc, vec, vec_name,2253					&default_cipher_testvec_configs[i],2254					req, tsgls);2255		if (err)2256			return err;2257	}2258 2259#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS2260	if (!noextratests) {2261		struct rnd_state rng;2262		struct testvec_config cfg;2263		char cfgname[TESTVEC_CONFIG_NAMELEN];2264 2265		init_rnd_state(&rng);2266 2267		for (i = 0; i < fuzz_iterations; i++) {2268			generate_random_testvec_config(&rng, &cfg, cfgname,2269						       sizeof(cfgname));2270			err = test_aead_vec_cfg(enc, vec, vec_name,2271						&cfg, req, tsgls);2272			if (err)2273				return err;2274			cond_resched();2275		}2276	}2277#endif2278	return 0;2279}2280 2281#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS2282 2283struct aead_extra_tests_ctx {2284	struct rnd_state rng;2285	struct aead_request *req;2286	struct crypto_aead *tfm;2287	const struct alg_test_desc *test_desc;2288	struct cipher_test_sglists *tsgls;2289	unsigned int maxdatasize;2290	unsigned int maxkeysize;2291 2292	struct aead_testvec vec;2293	char vec_name[64];2294	char cfgname[TESTVEC_CONFIG_NAMELEN];2295	struct testvec_config cfg;2296};2297 2298/*2299 * Make at least one random change to a (ciphertext, AAD) pair.  "Ciphertext"2300 * here means the full ciphertext including the authentication tag.  The2301 * authentication tag (and hence also the ciphertext) is assumed to be nonempty.2302 */2303static void mutate_aead_message(struct rnd_state *rng,2304				struct aead_testvec *vec, bool aad_iv,2305				unsigned int ivsize)2306{2307	const unsigned int aad_tail_size = aad_iv ? ivsize : 0;2308	const unsigned int authsize = vec->clen - vec->plen;2309 2310	if (prandom_bool(rng) && vec->alen > aad_tail_size) {2311		 /* Mutate the AAD */2312		flip_random_bit(rng, (u8 *)vec->assoc,2313				vec->alen - aad_tail_size);2314		if (prandom_bool(rng))2315			return;2316	}2317	if (prandom_bool(rng)) {2318		/* Mutate auth tag (assuming it's at the end of ciphertext) */2319		flip_random_bit(rng, (u8 *)vec->ctext + vec->plen, authsize);2320	} else {2321		/* Mutate any part of the ciphertext */2322		flip_random_bit(rng, (u8 *)vec->ctext, vec->clen);2323	}2324}2325 2326/*2327 * Minimum authentication tag size in bytes at which we assume that we can2328 * reliably generate inauthentic messages, i.e. not generate an authentic2329 * message by chance.2330 */2331#define MIN_COLLISION_FREE_AUTHSIZE 82332 2333static void generate_aead_message(struct rnd_state *rng,2334				  struct aead_request *req,2335				  const struct aead_test_suite *suite,2336				  struct aead_testvec *vec,2337				  bool prefer_inauthentic)2338{2339	struct crypto_aead *tfm = crypto_aead_reqtfm(req);2340	const unsigned int ivsize = crypto_aead_ivsize(tfm);2341	const unsigned int authsize = vec->clen - vec->plen;2342	const bool inauthentic = (authsize >= MIN_COLLISION_FREE_AUTHSIZE) &&2343				 (prefer_inauthentic ||2344				  prandom_u32_below(rng, 4) == 0);2345 2346	/* Generate the AAD. */2347	generate_random_bytes(rng, (u8 *)vec->assoc, vec->alen);2348	if (suite->aad_iv && vec->alen >= ivsize)2349		/* Avoid implementation-defined behavior. */2350		memcpy((u8 *)vec->assoc + vec->alen - ivsize, vec->iv, ivsize);2351 2352	if (inauthentic && prandom_bool(rng)) {2353		/* Generate a random ciphertext. */2354		generate_random_bytes(rng, (u8 *)vec->ctext, vec->clen);2355	} else {2356		int i = 0;2357		struct scatterlist src[2], dst;2358		u8 iv[MAX_IVLEN];2359		DECLARE_CRYPTO_WAIT(wait);2360 2361		/* Generate a random plaintext and encrypt it. */2362		sg_init_table(src, 2);2363		if (vec->alen)2364			sg_set_buf(&src[i++], vec->assoc, vec->alen);2365		if (vec->plen) {2366			generate_random_bytes(rng, (u8 *)vec->ptext, vec->plen);2367			sg_set_buf(&src[i++], vec->ptext, vec->plen);2368		}2369		sg_init_one(&dst, vec->ctext, vec->alen + vec->clen);2370		memcpy(iv, vec->iv, ivsize);2371		aead_request_set_callback(req, 0, crypto_req_done, &wait);2372		aead_request_set_crypt(req, src, &dst, vec->plen, iv);2373		aead_request_set_ad(req, vec->alen);2374		vec->crypt_error = crypto_wait_req(crypto_aead_encrypt(req),2375						   &wait);2376		/* If encryption failed, we're done. */2377		if (vec->crypt_error != 0)2378			return;2379		memmove((u8 *)vec->ctext, vec->ctext + vec->alen, vec->clen);2380		if (!inauthentic)2381			return;2382		/*2383		 * Mutate the authentic (ciphertext, AAD) pair to get an2384		 * inauthentic one.2385		 */2386		mutate_aead_message(rng, vec, suite->aad_iv, ivsize);2387	}2388	vec->novrfy = 1;2389	if (suite->einval_allowed)2390		vec->crypt_error = -EINVAL;2391}2392 2393/*2394 * Generate an AEAD test vector 'vec' using the implementation specified by2395 * 'req'.  The buffers in 'vec' must already be allocated.2396 *2397 * If 'prefer_inauthentic' is true, then this function will generate inauthentic2398 * test vectors (i.e. vectors with 'vec->novrfy=1') more often.2399 */2400static void generate_random_aead_testvec(struct rnd_state *rng,2401					 struct aead_request *req,2402					 struct aead_testvec *vec,2403					 const struct aead_test_suite *suite,2404					 unsigned int maxkeysize,2405					 unsigned int maxdatasize,2406					 char *name, size_t max_namelen,2407					 bool prefer_inauthentic)2408{2409	struct crypto_aead *tfm = crypto_aead_reqtfm(req);2410	const unsigned int ivsize = crypto_aead_ivsize(tfm);2411	const unsigned int maxauthsize = crypto_aead_maxauthsize(tfm);2412	unsigned int authsize;2413	unsigned int total_len;2414 2415	/* Key: length in [0, maxkeysize], but usually choose maxkeysize */2416	vec->klen = maxkeysize;2417	if (prandom_u32_below(rng, 4) == 0)2418		vec->klen = prandom_u32_below(rng, maxkeysize + 1);2419	generate_random_bytes(rng, (u8 *)vec->key, vec->klen);2420	vec->setkey_error = crypto_aead_setkey(tfm, vec->key, vec->klen);2421 2422	/* IV */2423	generate_random_bytes(rng, (u8 *)vec->iv, ivsize);2424 2425	/* Tag length: in [0, maxauthsize], but usually choose maxauthsize */2426	authsize = maxauthsize;2427	if (prandom_u32_below(rng, 4) == 0)2428		authsize = prandom_u32_below(rng, maxauthsize + 1);2429	if (prefer_inauthentic && authsize < MIN_COLLISION_FREE_AUTHSIZE)2430		authsize = MIN_COLLISION_FREE_AUTHSIZE;2431	if (WARN_ON(authsize > maxdatasize))2432		authsize = maxdatasize;2433	maxdatasize -= authsize;2434	vec->setauthsize_error = crypto_aead_setauthsize(tfm, authsize);2435 2436	/* AAD, plaintext, and ciphertext lengths */2437	total_len = generate_random_length(rng, maxdatasize);2438	if (prandom_u32_below(rng, 4) == 0)2439		vec->alen = 0;2440	else2441		vec->alen = generate_random_length(rng, total_len);2442	vec->plen = total_len - vec->alen;2443	vec->clen = vec->plen + authsize;2444 2445	/*2446	 * Generate the AAD, plaintext, and ciphertext.  Not applicable if the2447	 * key or the authentication tag size couldn't be set.2448	 */2449	vec->novrfy = 0;2450	vec->crypt_error = 0;2451	if (vec->setkey_error == 0 && vec->setauthsize_error == 0)2452		generate_aead_message(rng, req, suite, vec, prefer_inauthentic);2453	snprintf(name, max_namelen,2454		 "\"random: alen=%u plen=%u authsize=%u klen=%u novrfy=%d\"",2455		 vec->alen, vec->plen, authsize, vec->klen, vec->novrfy);2456}2457 2458static void try_to_generate_inauthentic_testvec(2459					struct aead_extra_tests_ctx *ctx)2460{2461	int i;2462 2463	for (i = 0; i < 10; i++) {2464		generate_random_aead_testvec(&ctx->rng, ctx->req, &ctx->vec,2465					     &ctx->test_desc->suite.aead,2466					     ctx->maxkeysize, ctx->maxdatasize,2467					     ctx->vec_name,2468					     sizeof(ctx->vec_name), true);2469		if (ctx->vec.novrfy)2470			return;2471	}2472}2473 2474/*2475 * Generate inauthentic test vectors (i.e. ciphertext, AAD pairs that aren't the2476 * result of an encryption with the key) and verify that decryption fails.2477 */2478static int test_aead_inauthentic_inputs(struct aead_extra_tests_ctx *ctx)2479{2480	unsigned int i;2481	int err;2482 2483	for (i = 0; i < fuzz_iterations * 8; i++) {2484		/*2485		 * Since this part of the tests isn't comparing the2486		 * implementation to another, there's no point in testing any2487		 * test vectors other than inauthentic ones (vec.novrfy=1) here.2488		 *2489		 * If we're having trouble generating such a test vector, e.g.2490		 * if the algorithm keeps rejecting the generated keys, don't2491		 * retry forever; just continue on.2492		 */2493		try_to_generate_inauthentic_testvec(ctx);2494		if (ctx->vec.novrfy) {2495			generate_random_testvec_config(&ctx->rng, &ctx->cfg,2496						       ctx->cfgname,2497						       sizeof(ctx->cfgname));2498			err = test_aead_vec_cfg(DECRYPT, &ctx->vec,2499						ctx->vec_name, &ctx->cfg,2500						ctx->req, ctx->tsgls);2501			if (err)2502				return err;2503		}2504		cond_resched();2505	}2506	return 0;2507}2508 2509/*2510 * Test the AEAD algorithm against the corresponding generic implementation, if2511 * one is available.2512 */2513static int test_aead_vs_generic_impl(struct aead_extra_tests_ctx *ctx)2514{2515	struct crypto_aead *tfm = ctx->tfm;2516	const char *algname = crypto_aead_alg(tfm)->base.cra_name;2517	const char *driver = crypto_aead_driver_name(tfm);2518	const char *generic_driver = ctx->test_desc->generic_driver;2519	char _generic_driver[CRYPTO_MAX_ALG_NAME];2520	struct crypto_aead *generic_tfm = NULL;2521	struct aead_request *generic_req = NULL;2522	unsigned int i;2523	int err;2524 2525	if (!generic_driver) { /* Use default naming convention? */2526		err = build_generic_driver_name(algname, _generic_driver);2527		if (err)2528			return err;2529		generic_driver = _generic_driver;2530	}2531 2532	if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */2533		return 0;2534 2535	generic_tfm = crypto_alloc_aead(generic_driver, 0, 0);2536	if (IS_ERR(generic_tfm)) {2537		err = PTR_ERR(generic_tfm);2538		if (err == -ENOENT) {2539			pr_warn("alg: aead: skipping comparison tests for %s because %s is unavailable\n",2540				driver, generic_driver);2541			return 0;2542		}2543		pr_err("alg: aead: error allocating %s (generic impl of %s): %d\n",2544		       generic_driver, algname, err);2545		return err;2546	}2547 2548	generic_req = aead_request_alloc(generic_tfm, GFP_KERNEL);2549	if (!generic_req) {2550		err = -ENOMEM;2551		goto out;2552	}2553 2554	/* Check the algorithm properties for consistency. */2555 2556	if (crypto_aead_maxauthsize(tfm) !=2557	    crypto_aead_maxauthsize(generic_tfm)) {2558		pr_err("alg: aead: maxauthsize for %s (%u) doesn't match generic impl (%u)\n",2559		       driver, crypto_aead_maxauthsize(tfm),2560		       crypto_aead_maxauthsize(generic_tfm));2561		err = -EINVAL;2562		goto out;2563	}2564 2565	if (crypto_aead_ivsize(tfm) != crypto_aead_ivsize(generic_tfm)) {2566		pr_err("alg: aead: ivsize for %s (%u) doesn't match generic impl (%u)\n",2567		       driver, crypto_aead_ivsize(tfm),2568		       crypto_aead_ivsize(generic_tfm));2569		err = -EINVAL;2570		goto out;2571	}2572 2573	if (crypto_aead_blocksize(tfm) != crypto_aead_blocksize(generic_tfm)) {2574		pr_err("alg: aead: blocksize for %s (%u) doesn't match generic impl (%u)\n",2575		       driver, crypto_aead_blocksize(tfm),2576		       crypto_aead_blocksize(generic_tfm));2577		err = -EINVAL;2578		goto out;2579	}2580 2581	/*2582	 * Now generate test vectors using the generic implementation, and test2583	 * the other implementation against them.2584	 */2585	for (i = 0; i < fuzz_iterations * 8; i++) {2586		generate_random_aead_testvec(&ctx->rng, generic_req, &ctx->vec,2587					     &ctx->test_desc->suite.aead,2588					     ctx->maxkeysize, ctx->maxdatasize,2589					     ctx->vec_name,2590					     sizeof(ctx->vec_name), false);2591		generate_random_testvec_config(&ctx->rng, &ctx->cfg,2592					       ctx->cfgname,2593					       sizeof(ctx->cfgname));2594		if (!ctx->vec.novrfy) {2595			err = test_aead_vec_cfg(ENCRYPT, &ctx->vec,2596						ctx->vec_name, &ctx->cfg,2597						ctx->req, ctx->tsgls);2598			if (err)2599				goto out;2600		}2601		if (ctx->vec.crypt_error == 0 || ctx->vec.novrfy) {2602			err = test_aead_vec_cfg(DECRYPT, &ctx->vec,2603						ctx->vec_name, &ctx->cfg,2604						ctx->req, ctx->tsgls);2605			if (err)2606				goto out;2607		}2608		cond_resched();2609	}2610	err = 0;2611out:2612	crypto_free_aead(generic_tfm);2613	aead_request_free(generic_req);2614	return err;2615}2616 2617static int test_aead_extra(const struct alg_test_desc *test_desc,2618			   struct aead_request *req,2619			   struct cipher_test_sglists *tsgls)2620{2621	struct aead_extra_tests_ctx *ctx;2622	unsigned int i;2623	int err;2624 2625	if (noextratests)2626		return 0;2627 2628	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);2629	if (!ctx)2630		return -ENOMEM;2631	init_rnd_state(&ctx->rng);2632	ctx->req = req;2633	ctx->tfm = crypto_aead_reqtfm(req);2634	ctx->test_desc = test_desc;2635	ctx->tsgls = tsgls;2636	ctx->maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;2637	ctx->maxkeysize = 0;2638	for (i = 0; i < test_desc->suite.aead.count; i++)2639		ctx->maxkeysize = max_t(unsigned int, ctx->maxkeysize,2640					test_desc->suite.aead.vecs[i].klen);2641 2642	ctx->vec.key = kmalloc(ctx->maxkeysize, GFP_KERNEL);2643	ctx->vec.iv = kmalloc(crypto_aead_ivsize(ctx->tfm), GFP_KERNEL);2644	ctx->vec.assoc = kmalloc(ctx->maxdatasize, GFP_KERNEL);2645	ctx->vec.ptext = kmalloc(ctx->maxdatasize, GFP_KERNEL);2646	ctx->vec.ctext = kmalloc(ctx->maxdatasize, GFP_KERNEL);2647	if (!ctx->vec.key || !ctx->vec.iv || !ctx->vec.assoc ||2648	    !ctx->vec.ptext || !ctx->vec.ctext) {2649		err = -ENOMEM;2650		goto out;2651	}2652 2653	err = test_aead_vs_generic_impl(ctx);2654	if (err)2655		goto out;2656 2657	err = test_aead_inauthentic_inputs(ctx);2658out:2659	kfree(ctx->vec.key);2660	kfree(ctx->vec.iv);2661	kfree(ctx->vec.assoc);2662	kfree(ctx->vec.ptext);2663	kfree(ctx->vec.ctext);2664	kfree(ctx);2665	return err;2666}2667#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */2668static int test_aead_extra(const struct alg_test_desc *test_desc,2669			   struct aead_request *req,2670			   struct cipher_test_sglists *tsgls)2671{2672	return 0;2673}2674#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */2675 2676static int test_aead(int enc, const struct aead_test_suite *suite,2677		     struct aead_request *req,2678		     struct cipher_test_sglists *tsgls)2679{2680	unsigned int i;2681	int err;2682 2683	for (i = 0; i < suite->count; i++) {2684		err = test_aead_vec(enc, &suite->vecs[i], i, req, tsgls);2685		if (err)2686			return err;2687		cond_resched();2688	}2689	return 0;2690}2691 2692static int alg_test_aead(const struct alg_test_desc *desc, const char *driver,2693			 u32 type, u32 mask)2694{2695	const struct aead_test_suite *suite = &desc->suite.aead;2696	struct crypto_aead *tfm;2697	struct aead_request *req = NULL;2698	struct cipher_test_sglists *tsgls = NULL;2699	int err;2700 2701	if (suite->count <= 0) {2702		pr_err("alg: aead: empty test suite for %s\n", driver);2703		return -EINVAL;2704	}2705 2706	tfm = crypto_alloc_aead(driver, type, mask);2707	if (IS_ERR(tfm)) {2708		if (PTR_ERR(tfm) == -ENOENT)2709			return 0;2710		pr_err("alg: aead: failed to allocate transform for %s: %ld\n",2711		       driver, PTR_ERR(tfm));2712		return PTR_ERR(tfm);2713	}2714	driver = crypto_aead_driver_name(tfm);2715 2716	req = aead_request_alloc(tfm, GFP_KERNEL);2717	if (!req) {2718		pr_err("alg: aead: failed to allocate request for %s\n",2719		       driver);2720		err = -ENOMEM;2721		goto out;2722	}2723 2724	tsgls = alloc_cipher_test_sglists();2725	if (!tsgls) {2726		pr_err("alg: aead: failed to allocate test buffers for %s\n",2727		       driver);2728		err = -ENOMEM;2729		goto out;2730	}2731 2732	err = test_aead(ENCRYPT, suite, req, tsgls);2733	if (err)2734		goto out;2735 2736	err = test_aead(DECRYPT, suite, req, tsgls);2737	if (err)2738		goto out;2739 2740	err = test_aead_extra(desc, req, tsgls);2741out:2742	free_cipher_test_sglists(tsgls);2743	aead_request_free(req);2744	crypto_free_aead(tfm);2745	return err;2746}2747 2748static int test_cipher(struct crypto_cipher *tfm, int enc,2749		       const struct cipher_testvec *template,2750		       unsigned int tcount)2751{2752	const char *algo = crypto_tfm_alg_driver_name(crypto_cipher_tfm(tfm));2753	unsigned int i, j, k;2754	char *q;2755	const char *e;2756	const char *input, *result;2757	void *data;2758	char *xbuf[XBUFSIZE];2759	int ret = -ENOMEM;2760 2761	if (testmgr_alloc_buf(xbuf))2762		goto out_nobuf;2763 2764	if (enc == ENCRYPT)2765	        e = "encryption";2766	else2767		e = "decryption";2768 2769	j = 0;2770	for (i = 0; i < tcount; i++) {2771 2772		if (fips_enabled && template[i].fips_skip)2773			continue;2774 2775		input  = enc ? template[i].ptext : template[i].ctext;2776		result = enc ? template[i].ctext : template[i].ptext;2777		j++;2778 2779		ret = -EINVAL;2780		if (WARN_ON(template[i].len > PAGE_SIZE))2781			goto out;2782 2783		data = xbuf[0];2784		memcpy(data, input, template[i].len);2785 2786		crypto_cipher_clear_flags(tfm, ~0);2787		if (template[i].wk)2788			crypto_cipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);2789 2790		ret = crypto_cipher_setkey(tfm, template[i].key,2791					   template[i].klen);2792		if (ret) {2793			if (ret == template[i].setkey_error)2794				continue;2795			pr_err("alg: cipher: %s setkey failed on test vector %u; expected_error=%d, actual_error=%d, flags=%#x\n",2796			       algo, j, template[i].setkey_error, ret,2797			       crypto_cipher_get_flags(tfm));2798			goto out;2799		}2800		if (template[i].setkey_error) {2801			pr_err("alg: cipher: %s setkey unexpectedly succeeded on test vector %u; expected_error=%d\n",2802			       algo, j, template[i].setkey_error);2803			ret = -EINVAL;2804			goto out;2805		}2806 2807		for (k = 0; k < template[i].len;2808		     k += crypto_cipher_blocksize(tfm)) {2809			if (enc)2810				crypto_cipher_encrypt_one(tfm, data + k,2811							  data + k);2812			else2813				crypto_cipher_decrypt_one(tfm, data + k,2814							  data + k);2815		}2816 2817		q = data;2818		if (memcmp(q, result, template[i].len)) {2819			printk(KERN_ERR "alg: cipher: Test %d failed "2820			       "on %s for %s\n", j, e, algo);2821			hexdump(q, template[i].len);2822			ret = -EINVAL;2823			goto out;2824		}2825	}2826 2827	ret = 0;2828 2829out:2830	testmgr_free_buf(xbuf);2831out_nobuf:2832	return ret;2833}2834 2835static int test_skcipher_vec_cfg(int enc, const struct cipher_testvec *vec,2836				 const char *vec_name,2837				 const struct testvec_config *cfg,2838				 struct skcipher_request *req,2839				 struct cipher_test_sglists *tsgls)2840{2841	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);2842	const unsigned int alignmask = crypto_skcipher_alignmask(tfm);2843	const unsigned int ivsize = crypto_skcipher_ivsize(tfm);2844	const char *driver = crypto_skcipher_driver_name(tfm);2845	const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags;2846	const char *op = enc ? "encryption" : "decryption";2847	DECLARE_CRYPTO_WAIT(wait);2848	u8 _iv[3 * (MAX_ALGAPI_ALIGNMASK + 1) + MAX_IVLEN];2849	u8 *iv = PTR_ALIGN(&_iv[0], 2 * (MAX_ALGAPI_ALIGNMASK + 1)) +2850		 cfg->iv_offset +2851		 (cfg->iv_offset_relative_to_alignmask ? alignmask : 0);2852	struct kvec input;2853	int err;2854 2855	/* Set the key */2856	if (vec->wk)2857		crypto_skcipher_set_flags(tfm, CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);2858	else2859		crypto_skcipher_clear_flags(tfm,2860					    CRYPTO_TFM_REQ_FORBID_WEAK_KEYS);2861	err = do_setkey(crypto_skcipher_setkey, tfm, vec->key, vec->klen,2862			cfg, alignmask);2863	if (err) {2864		if (err == vec->setkey_error)2865			return 0;2866		pr_err("alg: skcipher: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n",2867		       driver, vec_name, vec->setkey_error, err,2868		       crypto_skcipher_get_flags(tfm));2869		return err;2870	}2871	if (vec->setkey_error) {2872		pr_err("alg: skcipher: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n",2873		       driver, vec_name, vec->setkey_error);2874		return -EINVAL;2875	}2876 2877	/* The IV must be copied to a buffer, as the algorithm may modify it */2878	if (ivsize) {2879		if (WARN_ON(ivsize > MAX_IVLEN))2880			return -EINVAL;2881		if (vec->generates_iv && !enc)2882			memcpy(iv, vec->iv_out, ivsize);2883		else if (vec->iv)2884			memcpy(iv, vec->iv, ivsize);2885		else2886			memset(iv, 0, ivsize);2887	} else {2888		if (vec->generates_iv) {2889			pr_err("alg: skcipher: %s has ivsize=0 but test vector %s generates IV!\n",2890			       driver, vec_name);2891			return -EINVAL;2892		}2893		iv = NULL;2894	}2895 2896	/* Build the src/dst scatterlists */2897	input.iov_base = enc ? (void *)vec->ptext : (void *)vec->ctext;2898	input.iov_len = vec->len;2899	err = build_cipher_test_sglists(tsgls, cfg, alignmask,2900					vec->len, vec->len, &input, 1);2901	if (err) {2902		pr_err("alg: skcipher: %s %s: error preparing scatterlists for test vector %s, cfg=\"%s\"\n",2903		       driver, op, vec_name, cfg->name);2904		return err;2905	}2906 2907	/* Do the actual encryption or decryption */2908	testmgr_poison(req->__ctx, crypto_skcipher_reqsize(tfm));2909	skcipher_request_set_callback(req, req_flags, crypto_req_done, &wait);2910	skcipher_request_set_crypt(req, tsgls->src.sgl_ptr, tsgls->dst.sgl_ptr,2911				   vec->len, iv);2912	if (cfg->nosimd)2913		crypto_disable_simd_for_test();2914	err = enc ? crypto_skcipher_encrypt(req) : crypto_skcipher_decrypt(req);2915	if (cfg->nosimd)2916		crypto_reenable_simd_for_test();2917	err = crypto_wait_req(err, &wait);2918 2919	/* Check that the algorithm didn't overwrite things it shouldn't have */2920	if (req->cryptlen != vec->len ||2921	    req->iv != iv ||2922	    req->src != tsgls->src.sgl_ptr ||2923	    req->dst != tsgls->dst.sgl_ptr ||2924	    crypto_skcipher_reqtfm(req) != tfm ||2925	    req->base.complete != crypto_req_done ||2926	    req->base.flags != req_flags ||2927	    req->base.data != &wait) {2928		pr_err("alg: skcipher: %s %s corrupted request struct on test vector %s, cfg=\"%s\"\n",2929		       driver, op, vec_name, cfg->name);2930		if (req->cryptlen != vec->len)2931			pr_err("alg: skcipher: changed 'req->cryptlen'\n");2932		if (req->iv != iv)2933			pr_err("alg: skcipher: changed 'req->iv'\n");2934		if (req->src != tsgls->src.sgl_ptr)2935			pr_err("alg: skcipher: changed 'req->src'\n");2936		if (req->dst != tsgls->dst.sgl_ptr)2937			pr_err("alg: skcipher: changed 'req->dst'\n");2938		if (crypto_skcipher_reqtfm(req) != tfm)2939			pr_err("alg: skcipher: changed 'req->base.tfm'\n");2940		if (req->base.complete != crypto_req_done)2941			pr_err("alg: skcipher: changed 'req->base.complete'\n");2942		if (req->base.flags != req_flags)2943			pr_err("alg: skcipher: changed 'req->base.flags'\n");2944		if (req->base.data != &wait)2945			pr_err("alg: skcipher: changed 'req->base.data'\n");2946		return -EINVAL;2947	}2948	if (is_test_sglist_corrupted(&tsgls->src)) {2949		pr_err("alg: skcipher: %s %s corrupted src sgl on test vector %s, cfg=\"%s\"\n",2950		       driver, op, vec_name, cfg->name);2951		return -EINVAL;2952	}2953	if (tsgls->dst.sgl_ptr != tsgls->src.sgl &&2954	    is_test_sglist_corrupted(&tsgls->dst)) {2955		pr_err("alg: skcipher: %s %s corrupted dst sgl on test vector %s, cfg=\"%s\"\n",2956		       driver, op, vec_name, cfg->name);2957		return -EINVAL;2958	}2959 2960	/* Check for success or failure */2961	if (err) {2962		if (err == vec->crypt_error)2963			return 0;2964		pr_err("alg: skcipher: %s %s failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n",2965		       driver, op, vec_name, vec->crypt_error, err, cfg->name);2966		return err;2967	}2968	if (vec->crypt_error) {2969		pr_err("alg: skcipher: %s %s unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n",2970		       driver, op, vec_name, vec->crypt_error, cfg->name);2971		return -EINVAL;2972	}2973 2974	/* Check for the correct output (ciphertext or plaintext) */2975	err = verify_correct_output(&tsgls->dst, enc ? vec->ctext : vec->ptext,2976				    vec->len, 0, true);2977	if (err == -EOVERFLOW) {2978		pr_err("alg: skcipher: %s %s overran dst buffer on test vector %s, cfg=\"%s\"\n",2979		       driver, op, vec_name, cfg->name);2980		return err;2981	}2982	if (err) {2983		pr_err("alg: skcipher: %s %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n",2984		       driver, op, vec_name, cfg->name);2985		return err;2986	}2987 2988	/* If applicable, check that the algorithm generated the correct IV */2989	if (vec->iv_out && memcmp(iv, vec->iv_out, ivsize) != 0) {2990		pr_err("alg: skcipher: %s %s test failed (wrong output IV) on test vector %s, cfg=\"%s\"\n",2991		       driver, op, vec_name, cfg->name);2992		hexdump(iv, ivsize);2993		return -EINVAL;2994	}2995 2996	return 0;2997}2998 2999static int test_skcipher_vec(int enc, const struct cipher_testvec *vec,3000			     unsigned int vec_num,3001			     struct skcipher_request *req,3002			     struct cipher_test_sglists *tsgls)3003{3004	char vec_name[16];3005	unsigned int i;3006	int err;3007 3008	if (fips_enabled && vec->fips_skip)3009		return 0;3010 3011	sprintf(vec_name, "%u", vec_num);3012 3013	for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++) {3014		err = test_skcipher_vec_cfg(enc, vec, vec_name,3015					    &default_cipher_testvec_configs[i],3016					    req, tsgls);3017		if (err)3018			return err;3019	}3020 3021#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS3022	if (!noextratests) {3023		struct rnd_state rng;3024		struct testvec_config cfg;3025		char cfgname[TESTVEC_CONFIG_NAMELEN];3026 3027		init_rnd_state(&rng);3028 3029		for (i = 0; i < fuzz_iterations; i++) {3030			generate_random_testvec_config(&rng, &cfg, cfgname,3031						       sizeof(cfgname));3032			err = test_skcipher_vec_cfg(enc, vec, vec_name,3033						    &cfg, req, tsgls);3034			if (err)3035				return err;3036			cond_resched();3037		}3038	}3039#endif3040	return 0;3041}3042 3043#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS3044/*3045 * Generate a symmetric cipher test vector from the given implementation.3046 * Assumes the buffers in 'vec' were already allocated.3047 */3048static void generate_random_cipher_testvec(struct rnd_state *rng,3049					   struct skcipher_request *req,3050					   struct cipher_testvec *vec,3051					   unsigned int maxdatasize,3052					   char *name, size_t max_namelen)3053{3054	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);3055	const unsigned int maxkeysize = crypto_skcipher_max_keysize(tfm);3056	const unsigned int ivsize = crypto_skcipher_ivsize(tfm);3057	struct scatterlist src, dst;3058	u8 iv[MAX_IVLEN];3059	DECLARE_CRYPTO_WAIT(wait);3060 3061	/* Key: length in [0, maxkeysize], but usually choose maxkeysize */3062	vec->klen = maxkeysize;3063	if (prandom_u32_below(rng, 4) == 0)3064		vec->klen = prandom_u32_below(rng, maxkeysize + 1);3065	generate_random_bytes(rng, (u8 *)vec->key, vec->klen);3066	vec->setkey_error = crypto_skcipher_setkey(tfm, vec->key, vec->klen);3067 3068	/* IV */3069	generate_random_bytes(rng, (u8 *)vec->iv, ivsize);3070 3071	/* Plaintext */3072	vec->len = generate_random_length(rng, maxdatasize);3073	generate_random_bytes(rng, (u8 *)vec->ptext, vec->len);3074 3075	/* If the key couldn't be set, no need to continue to encrypt. */3076	if (vec->setkey_error)3077		goto done;3078 3079	/* Ciphertext */3080	sg_init_one(&src, vec->ptext, vec->len);3081	sg_init_one(&dst, vec->ctext, vec->len);3082	memcpy(iv, vec->iv, ivsize);3083	skcipher_request_set_callback(req, 0, crypto_req_done, &wait);3084	skcipher_request_set_crypt(req, &src, &dst, vec->len, iv);3085	vec->crypt_error = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);3086	if (vec->crypt_error != 0) {3087		/*3088		 * The only acceptable error here is for an invalid length, so3089		 * skcipher decryption should fail with the same error too.3090		 * We'll test for this.  But to keep the API usage well-defined,3091		 * explicitly initialize the ciphertext buffer too.3092		 */3093		memset((u8 *)vec->ctext, 0, vec->len);3094	}3095done:3096	snprintf(name, max_namelen, "\"random: len=%u klen=%u\"",3097		 vec->len, vec->klen);3098}3099 3100/*3101 * Test the skcipher algorithm represented by @req against the corresponding3102 * generic implementation, if one is available.3103 */3104static int test_skcipher_vs_generic_impl(const char *generic_driver,3105					 struct skcipher_request *req,3106					 struct cipher_test_sglists *tsgls)3107{3108	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);3109	const unsigned int maxkeysize = crypto_skcipher_max_keysize(tfm);3110	const unsigned int ivsize = crypto_skcipher_ivsize(tfm);3111	const unsigned int blocksize = crypto_skcipher_blocksize(tfm);3112	const unsigned int maxdatasize = (2 * PAGE_SIZE) - TESTMGR_POISON_LEN;3113	const char *algname = crypto_skcipher_alg(tfm)->base.cra_name;3114	const char *driver = crypto_skcipher_driver_name(tfm);3115	struct rnd_state rng;3116	char _generic_driver[CRYPTO_MAX_ALG_NAME];3117	struct crypto_skcipher *generic_tfm = NULL;3118	struct skcipher_request *generic_req = NULL;3119	unsigned int i;3120	struct cipher_testvec vec = { 0 };3121	char vec_name[64];3122	struct testvec_config *cfg;3123	char cfgname[TESTVEC_CONFIG_NAMELEN];3124	int err;3125 3126	if (noextratests)3127		return 0;3128 3129	/* Keywrap isn't supported here yet as it handles its IV differently. */3130	if (strncmp(algname, "kw(", 3) == 0)3131		return 0;3132 3133	init_rnd_state(&rng);3134 3135	if (!generic_driver) { /* Use default naming convention? */3136		err = build_generic_driver_name(algname, _generic_driver);3137		if (err)3138			return err;3139		generic_driver = _generic_driver;3140	}3141 3142	if (strcmp(generic_driver, driver) == 0) /* Already the generic impl? */3143		return 0;3144 3145	generic_tfm = crypto_alloc_skcipher(generic_driver, 0, 0);3146	if (IS_ERR(generic_tfm)) {3147		err = PTR_ERR(generic_tfm);3148		if (err == -ENOENT) {3149			pr_warn("alg: skcipher: skipping comparison tests for %s because %s is unavailable\n",3150				driver, generic_driver);3151			return 0;3152		}3153		pr_err("alg: skcipher: error allocating %s (generic impl of %s): %d\n",3154		       generic_driver, algname, err);3155		return err;3156	}3157 3158	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);3159	if (!cfg) {3160		err = -ENOMEM;3161		goto out;3162	}3163 3164	generic_req = skcipher_request_alloc(generic_tfm, GFP_KERNEL);3165	if (!generic_req) {3166		err = -ENOMEM;3167		goto out;3168	}3169 3170	/* Check the algorithm properties for consistency. */3171 3172	if (crypto_skcipher_min_keysize(tfm) !=3173	    crypto_skcipher_min_keysize(generic_tfm)) {3174		pr_err("alg: skcipher: min keysize for %s (%u) doesn't match generic impl (%u)\n",3175		       driver, crypto_skcipher_min_keysize(tfm),3176		       crypto_skcipher_min_keysize(generic_tfm));3177		err = -EINVAL;3178		goto out;3179	}3180 3181	if (maxkeysize != crypto_skcipher_max_keysize(generic_tfm)) {3182		pr_err("alg: skcipher: max keysize for %s (%u) doesn't match generic impl (%u)\n",3183		       driver, maxkeysize,3184		       crypto_skcipher_max_keysize(generic_tfm));3185		err = -EINVAL;3186		goto out;3187	}3188 3189	if (ivsize != crypto_skcipher_ivsize(generic_tfm)) {3190		pr_err("alg: skcipher: ivsize for %s (%u) doesn't match generic impl (%u)\n",3191		       driver, ivsize, crypto_skcipher_ivsize(generic_tfm));3192		err = -EINVAL;3193		goto out;3194	}3195 3196	if (blocksize != crypto_skcipher_blocksize(generic_tfm)) {3197		pr_err("alg: skcipher: blocksize for %s (%u) doesn't match generic impl (%u)\n",3198		       driver, blocksize,3199		       crypto_skcipher_blocksize(generic_tfm));3200		err = -EINVAL;3201		goto out;3202	}3203 3204	/*3205	 * Now generate test vectors using the generic implementation, and test3206	 * the other implementation against them.3207	 */3208 3209	vec.key = kmalloc(maxkeysize, GFP_KERNEL);3210	vec.iv = kmalloc(ivsize, GFP_KERNEL);3211	vec.ptext = kmalloc(maxdatasize, GFP_KERNEL);3212	vec.ctext = kmalloc(maxdatasize, GFP_KERNEL);3213	if (!vec.key || !vec.iv || !vec.ptext || !vec.ctext) {3214		err = -ENOMEM;3215		goto out;3216	}3217 3218	for (i = 0; i < fuzz_iterations * 8; i++) {3219		generate_random_cipher_testvec(&rng, generic_req, &vec,3220					       maxdatasize,3221					       vec_name, sizeof(vec_name));3222		generate_random_testvec_config(&rng, cfg, cfgname,3223					       sizeof(cfgname));3224 3225		err = test_skcipher_vec_cfg(ENCRYPT, &vec, vec_name,3226					    cfg, req, tsgls);3227		if (err)3228			goto out;3229		err = test_skcipher_vec_cfg(DECRYPT, &vec, vec_name,3230					    cfg, req, tsgls);3231		if (err)3232			goto out;3233		cond_resched();3234	}3235	err = 0;3236out:3237	kfree(cfg);3238	kfree(vec.key);3239	kfree(vec.iv);3240	kfree(vec.ptext);3241	kfree(vec.ctext);3242	crypto_free_skcipher(generic_tfm);3243	skcipher_request_free(generic_req);3244	return err;3245}3246#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */3247static int test_skcipher_vs_generic_impl(const char *generic_driver,3248					 struct skcipher_request *req,3249					 struct cipher_test_sglists *tsgls)3250{3251	return 0;3252}3253#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */3254 3255static int test_skcipher(int enc, const struct cipher_test_suite *suite,3256			 struct skcipher_request *req,3257			 struct cipher_test_sglists *tsgls)3258{3259	unsigned int i;3260	int err;3261 3262	for (i = 0; i < suite->count; i++) {3263		err = test_skcipher_vec(enc, &suite->vecs[i], i, req, tsgls);3264		if (err)3265			return err;3266		cond_resched();3267	}3268	return 0;3269}3270 3271static int alg_test_skcipher(const struct alg_test_desc *desc,3272			     const char *driver, u32 type, u32 mask)3273{3274	const struct cipher_test_suite *suite = &desc->suite.cipher;3275	struct crypto_skcipher *tfm;3276	struct skcipher_request *req = NULL;3277	struct cipher_test_sglists *tsgls = NULL;3278	int err;3279 3280	if (suite->count <= 0) {3281		pr_err("alg: skcipher: empty test suite for %s\n", driver);3282		return -EINVAL;3283	}3284 3285	tfm = crypto_alloc_skcipher(driver, type, mask);3286	if (IS_ERR(tfm)) {3287		if (PTR_ERR(tfm) == -ENOENT)3288			return 0;3289		pr_err("alg: skcipher: failed to allocate transform for %s: %ld\n",3290		       driver, PTR_ERR(tfm));3291		return PTR_ERR(tfm);3292	}3293	driver = crypto_skcipher_driver_name(tfm);3294 3295	req = skcipher_request_alloc(tfm, GFP_KERNEL);3296	if (!req) {3297		pr_err("alg: skcipher: failed to allocate request for %s\n",3298		       driver);3299		err = -ENOMEM;3300		goto out;3301	}3302 3303	tsgls = alloc_cipher_test_sglists();3304	if (!tsgls) {3305		pr_err("alg: skcipher: failed to allocate test buffers for %s\n",3306		       driver);3307		err = -ENOMEM;3308		goto out;3309	}3310 3311	err = test_skcipher(ENCRYPT, suite, req, tsgls);3312	if (err)3313		goto out;3314 3315	err = test_skcipher(DECRYPT, suite, req, tsgls);3316	if (err)3317		goto out;3318 3319	err = test_skcipher_vs_generic_impl(desc->generic_driver, req, tsgls);3320out:3321	free_cipher_test_sglists(tsgls);3322	skcipher_request_free(req);3323	crypto_free_skcipher(tfm);3324	return err;3325}3326 3327static int test_comp(struct crypto_comp *tfm,3328		     const struct comp_testvec *ctemplate,3329		     const struct comp_testvec *dtemplate,3330		     int ctcount, int dtcount)3331{3332	const char *algo = crypto_tfm_alg_driver_name(crypto_comp_tfm(tfm));3333	char *output, *decomp_output;3334	unsigned int i;3335	int ret;3336 3337	output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);3338	if (!output)3339		return -ENOMEM;3340 3341	decomp_output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);3342	if (!decomp_output) {3343		kfree(output);3344		return -ENOMEM;3345	}3346 3347	for (i = 0; i < ctcount; i++) {3348		int ilen;3349		unsigned int dlen = COMP_BUF_SIZE;3350 3351		memset(output, 0, COMP_BUF_SIZE);3352		memset(decomp_output, 0, COMP_BUF_SIZE);3353 3354		ilen = ctemplate[i].inlen;3355		ret = crypto_comp_compress(tfm, ctemplate[i].input,3356					   ilen, output, &dlen);3357		if (ret) {3358			printk(KERN_ERR "alg: comp: compression failed "3359			       "on test %d for %s: ret=%d\n", i + 1, algo,3360			       -ret);3361			goto out;3362		}3363 3364		ilen = dlen;3365		dlen = COMP_BUF_SIZE;3366		ret = crypto_comp_decompress(tfm, output,3367					     ilen, decomp_output, &dlen);3368		if (ret) {3369			pr_err("alg: comp: compression failed: decompress: on test %d for %s failed: ret=%d\n",3370			       i + 1, algo, -ret);3371			goto out;3372		}3373 3374		if (dlen != ctemplate[i].inlen) {3375			printk(KERN_ERR "alg: comp: Compression test %d "3376			       "failed for %s: output len = %d\n", i + 1, algo,3377			       dlen);3378			ret = -EINVAL;3379			goto out;3380		}3381 3382		if (memcmp(decomp_output, ctemplate[i].input,3383			   ctemplate[i].inlen)) {3384			pr_err("alg: comp: compression failed: output differs: on test %d for %s\n",3385			       i + 1, algo);3386			hexdump(decomp_output, dlen);3387			ret = -EINVAL;3388			goto out;3389		}3390	}3391 3392	for (i = 0; i < dtcount; i++) {3393		int ilen;3394		unsigned int dlen = COMP_BUF_SIZE;3395 3396		memset(decomp_output, 0, COMP_BUF_SIZE);3397 3398		ilen = dtemplate[i].inlen;3399		ret = crypto_comp_decompress(tfm, dtemplate[i].input,3400					     ilen, decomp_output, &dlen);3401		if (ret) {3402			printk(KERN_ERR "alg: comp: decompression failed "3403			       "on test %d for %s: ret=%d\n", i + 1, algo,3404			       -ret);3405			goto out;3406		}3407 3408		if (dlen != dtemplate[i].outlen) {3409			printk(KERN_ERR "alg: comp: Decompression test %d "3410			       "failed for %s: output len = %d\n", i + 1, algo,3411			       dlen);3412			ret = -EINVAL;3413			goto out;3414		}3415 3416		if (memcmp(decomp_output, dtemplate[i].output, dlen)) {3417			printk(KERN_ERR "alg: comp: Decompression test %d "3418			       "failed for %s\n", i + 1, algo);3419			hexdump(decomp_output, dlen);3420			ret = -EINVAL;3421			goto out;3422		}3423	}3424 3425	ret = 0;3426 3427out:3428	kfree(decomp_output);3429	kfree(output);3430	return ret;3431}3432 3433static int test_acomp(struct crypto_acomp *tfm,3434		      const struct comp_testvec *ctemplate,3435		      const struct comp_testvec *dtemplate,3436		      int ctcount, int dtcount)3437{3438	const char *algo = crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm));3439	unsigned int i;3440	char *output, *decomp_out;3441	int ret;3442	struct scatterlist src, dst;3443	struct acomp_req *req;3444	struct crypto_wait wait;3445 3446	output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);3447	if (!output)3448		return -ENOMEM;3449 3450	decomp_out = kmalloc(COMP_BUF_SIZE, GFP_KERNEL);3451	if (!decomp_out) {3452		kfree(output);3453		return -ENOMEM;3454	}3455 3456	for (i = 0; i < ctcount; i++) {3457		unsigned int dlen = COMP_BUF_SIZE;3458		int ilen = ctemplate[i].inlen;3459		void *input_vec;3460 3461		input_vec = kmemdup(ctemplate[i].input, ilen, GFP_KERNEL);3462		if (!input_vec) {3463			ret = -ENOMEM;3464			goto out;3465		}3466 3467		memset(output, 0, dlen);3468		crypto_init_wait(&wait);3469		sg_init_one(&src, input_vec, ilen);3470		sg_init_one(&dst, output, dlen);3471 3472		req = acomp_request_alloc(tfm);3473		if (!req) {3474			pr_err("alg: acomp: request alloc failed for %s\n",3475			       algo);3476			kfree(input_vec);3477			ret = -ENOMEM;3478			goto out;3479		}3480 3481		acomp_request_set_params(req, &src, &dst, ilen, dlen);3482		acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,3483					   crypto_req_done, &wait);3484 3485		ret = crypto_wait_req(crypto_acomp_compress(req), &wait);3486		if (ret) {3487			pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n",3488			       i + 1, algo, -ret);3489			kfree(input_vec);3490			acomp_request_free(req);3491			goto out;3492		}3493 3494		ilen = req->dlen;3495		dlen = COMP_BUF_SIZE;3496		sg_init_one(&src, output, ilen);3497		sg_init_one(&dst, decomp_out, dlen);3498		crypto_init_wait(&wait);3499		acomp_request_set_params(req, &src, &dst, ilen, dlen);3500 3501		ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);3502		if (ret) {3503			pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n",3504			       i + 1, algo, -ret);3505			kfree(input_vec);3506			acomp_request_free(req);3507			goto out;3508		}3509 3510		if (req->dlen != ctemplate[i].inlen) {3511			pr_err("alg: acomp: Compression test %d failed for %s: output len = %d\n",3512			       i + 1, algo, req->dlen);3513			ret = -EINVAL;3514			kfree(input_vec);3515			acomp_request_free(req);3516			goto out;3517		}3518 3519		if (memcmp(input_vec, decomp_out, req->dlen)) {3520			pr_err("alg: acomp: Compression test %d failed for %s\n",3521			       i + 1, algo);3522			hexdump(output, req->dlen);3523			ret = -EINVAL;3524			kfree(input_vec);3525			acomp_request_free(req);3526			goto out;3527		}3528 3529#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS3530		crypto_init_wait(&wait);3531		sg_init_one(&src, input_vec, ilen);3532		acomp_request_set_params(req, &src, NULL, ilen, 0);3533 3534		ret = crypto_wait_req(crypto_acomp_compress(req), &wait);3535		if (ret) {3536			pr_err("alg: acomp: compression failed on NULL dst buffer test %d for %s: ret=%d\n",3537			       i + 1, algo, -ret);3538			kfree(input_vec);3539			acomp_request_free(req);3540			goto out;3541		}3542#endif3543 3544		kfree(input_vec);3545		acomp_request_free(req);3546	}3547 3548	for (i = 0; i < dtcount; i++) {3549		unsigned int dlen = COMP_BUF_SIZE;3550		int ilen = dtemplate[i].inlen;3551		void *input_vec;3552 3553		input_vec = kmemdup(dtemplate[i].input, ilen, GFP_KERNEL);3554		if (!input_vec) {3555			ret = -ENOMEM;3556			goto out;3557		}3558 3559		memset(output, 0, dlen);3560		crypto_init_wait(&wait);3561		sg_init_one(&src, input_vec, ilen);3562		sg_init_one(&dst, output, dlen);3563 3564		req = acomp_request_alloc(tfm);3565		if (!req) {3566			pr_err("alg: acomp: request alloc failed for %s\n",3567			       algo);3568			kfree(input_vec);3569			ret = -ENOMEM;3570			goto out;3571		}3572 3573		acomp_request_set_params(req, &src, &dst, ilen, dlen);3574		acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,3575					   crypto_req_done, &wait);3576 3577		ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);3578		if (ret) {3579			pr_err("alg: acomp: decompression failed on test %d for %s: ret=%d\n",3580			       i + 1, algo, -ret);3581			kfree(input_vec);3582			acomp_request_free(req);3583			goto out;3584		}3585 3586		if (req->dlen != dtemplate[i].outlen) {3587			pr_err("alg: acomp: Decompression test %d failed for %s: output len = %d\n",3588			       i + 1, algo, req->dlen);3589			ret = -EINVAL;3590			kfree(input_vec);3591			acomp_request_free(req);3592			goto out;3593		}3594 3595		if (memcmp(output, dtemplate[i].output, req->dlen)) {3596			pr_err("alg: acomp: Decompression test %d failed for %s\n",3597			       i + 1, algo);3598			hexdump(output, req->dlen);3599			ret = -EINVAL;3600			kfree(input_vec);3601			acomp_request_free(req);3602			goto out;3603		}3604 3605#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS3606		crypto_init_wait(&wait);3607		acomp_request_set_params(req, &src, NULL, ilen, 0);3608 3609		ret = crypto_wait_req(crypto_acomp_decompress(req), &wait);3610		if (ret) {3611			pr_err("alg: acomp: decompression failed on NULL dst buffer test %d for %s: ret=%d\n",3612			       i + 1, algo, -ret);3613			kfree(input_vec);3614			acomp_request_free(req);3615			goto out;3616		}3617#endif3618 3619		kfree(input_vec);3620		acomp_request_free(req);3621	}3622 3623	ret = 0;3624 3625out:3626	kfree(decomp_out);3627	kfree(output);3628	return ret;3629}3630 3631static int test_cprng(struct crypto_rng *tfm,3632		      const struct cprng_testvec *template,3633		      unsigned int tcount)3634{3635	const char *algo = crypto_tfm_alg_driver_name(crypto_rng_tfm(tfm));3636	int err = 0, i, j, seedsize;3637	u8 *seed;3638	char result[32];3639 3640	seedsize = crypto_rng_seedsize(tfm);3641 3642	seed = kmalloc(seedsize, GFP_KERNEL);3643	if (!seed) {3644		printk(KERN_ERR "alg: cprng: Failed to allocate seed space "3645		       "for %s\n", algo);3646		return -ENOMEM;3647	}3648 3649	for (i = 0; i < tcount; i++) {3650		memset(result, 0, 32);3651 3652		memcpy(seed, template[i].v, template[i].vlen);3653		memcpy(seed + template[i].vlen, template[i].key,3654		       template[i].klen);3655		memcpy(seed + template[i].vlen + template[i].klen,3656		       template[i].dt, template[i].dtlen);3657 3658		err = crypto_rng_reset(tfm, seed, seedsize);3659		if (err) {3660			printk(KERN_ERR "alg: cprng: Failed to reset rng "3661			       "for %s\n", algo);3662			goto out;3663		}3664 3665		for (j = 0; j < template[i].loops; j++) {3666			err = crypto_rng_get_bytes(tfm, result,3667						   template[i].rlen);3668			if (err < 0) {3669				printk(KERN_ERR "alg: cprng: Failed to obtain "3670				       "the correct amount of random data for "3671				       "%s (requested %d)\n", algo,3672				       template[i].rlen);3673				goto out;3674			}3675		}3676 3677		err = memcmp(result, template[i].result,3678			     template[i].rlen);3679		if (err) {3680			printk(KERN_ERR "alg: cprng: Test %d failed for %s\n",3681			       i, algo);3682			hexdump(result, template[i].rlen);3683			err = -EINVAL;3684			goto out;3685		}3686	}3687 3688out:3689	kfree(seed);3690	return err;3691}3692 3693static int alg_test_cipher(const struct alg_test_desc *desc,3694			   const char *driver, u32 type, u32 mask)3695{3696	const struct cipher_test_suite *suite = &desc->suite.cipher;3697	struct crypto_cipher *tfm;3698	int err;3699 3700	tfm = crypto_alloc_cipher(driver, type, mask);3701	if (IS_ERR(tfm)) {3702		if (PTR_ERR(tfm) == -ENOENT)3703			return 0;3704		printk(KERN_ERR "alg: cipher: Failed to load transform for "3705		       "%s: %ld\n", driver, PTR_ERR(tfm));3706		return PTR_ERR(tfm);3707	}3708 3709	err = test_cipher(tfm, ENCRYPT, suite->vecs, suite->count);3710	if (!err)3711		err = test_cipher(tfm, DECRYPT, suite->vecs, suite->count);3712 3713	crypto_free_cipher(tfm);3714	return err;3715}3716 3717static int alg_test_comp(const struct alg_test_desc *desc, const char *driver,3718			 u32 type, u32 mask)3719{3720	struct crypto_comp *comp;3721	struct crypto_acomp *acomp;3722	int err;3723	u32 algo_type = type & CRYPTO_ALG_TYPE_ACOMPRESS_MASK;3724 3725	if (algo_type == CRYPTO_ALG_TYPE_ACOMPRESS) {3726		acomp = crypto_alloc_acomp(driver, type, mask);3727		if (IS_ERR(acomp)) {3728			if (PTR_ERR(acomp) == -ENOENT)3729				return 0;3730			pr_err("alg: acomp: Failed to load transform for %s: %ld\n",3731			       driver, PTR_ERR(acomp));3732			return PTR_ERR(acomp);3733		}3734		err = test_acomp(acomp, desc->suite.comp.comp.vecs,3735				 desc->suite.comp.decomp.vecs,3736				 desc->suite.comp.comp.count,3737				 desc->suite.comp.decomp.count);3738		crypto_free_acomp(acomp);3739	} else {3740		comp = crypto_alloc_comp(driver, type, mask);3741		if (IS_ERR(comp)) {3742			if (PTR_ERR(comp) == -ENOENT)3743				return 0;3744			pr_err("alg: comp: Failed to load transform for %s: %ld\n",3745			       driver, PTR_ERR(comp));3746			return PTR_ERR(comp);3747		}3748 3749		err = test_comp(comp, desc->suite.comp.comp.vecs,3750				desc->suite.comp.decomp.vecs,3751				desc->suite.comp.comp.count,3752				desc->suite.comp.decomp.count);3753 3754		crypto_free_comp(comp);3755	}3756	return err;3757}3758 3759static int alg_test_crc32c(const struct alg_test_desc *desc,3760			   const char *driver, u32 type, u32 mask)3761{3762	struct crypto_shash *tfm;3763	__le32 val;3764	int err;3765 3766	err = alg_test_hash(desc, driver, type, mask);3767	if (err)3768		return err;3769 3770	tfm = crypto_alloc_shash(driver, type, mask);3771	if (IS_ERR(tfm)) {3772		if (PTR_ERR(tfm) == -ENOENT) {3773			/*3774			 * This crc32c implementation is only available through3775			 * ahash API, not the shash API, so the remaining part3776			 * of the test is not applicable to it.3777			 */3778			return 0;3779		}3780		printk(KERN_ERR "alg: crc32c: Failed to load transform for %s: "3781		       "%ld\n", driver, PTR_ERR(tfm));3782		return PTR_ERR(tfm);3783	}3784	driver = crypto_shash_driver_name(tfm);3785 3786	do {3787		SHASH_DESC_ON_STACK(shash, tfm);3788		u32 *ctx = (u32 *)shash_desc_ctx(shash);3789 3790		shash->tfm = tfm;3791 3792		*ctx = 420553207;3793		err = crypto_shash_final(shash, (u8 *)&val);3794		if (err) {3795			printk(KERN_ERR "alg: crc32c: Operation failed for "3796			       "%s: %d\n", driver, err);3797			break;3798		}3799 3800		if (val != cpu_to_le32(~420553207)) {3801			pr_err("alg: crc32c: Test failed for %s: %u\n",3802			       driver, le32_to_cpu(val));3803			err = -EINVAL;3804		}3805	} while (0);3806 3807	crypto_free_shash(tfm);3808 3809	return err;3810}3811 3812static int alg_test_cprng(const struct alg_test_desc *desc, const char *driver,3813			  u32 type, u32 mask)3814{3815	struct crypto_rng *rng;3816	int err;3817 3818	rng = crypto_alloc_rng(driver, type, mask);3819	if (IS_ERR(rng)) {3820		if (PTR_ERR(rng) == -ENOENT)3821			return 0;3822		printk(KERN_ERR "alg: cprng: Failed to load transform for %s: "3823		       "%ld\n", driver, PTR_ERR(rng));3824		return PTR_ERR(rng);3825	}3826 3827	err = test_cprng(rng, desc->suite.cprng.vecs, desc->suite.cprng.count);3828 3829	crypto_free_rng(rng);3830 3831	return err;3832}3833 3834 3835static int drbg_cavs_test(const struct drbg_testvec *test, int pr,3836			  const char *driver, u32 type, u32 mask)3837{3838	int ret = -EAGAIN;3839	struct crypto_rng *drng;3840	struct drbg_test_data test_data;3841	struct drbg_string addtl, pers, testentropy;3842	unsigned char *buf = kzalloc(test->expectedlen, GFP_KERNEL);3843 3844	if (!buf)3845		return -ENOMEM;3846 3847	drng = crypto_alloc_rng(driver, type, mask);3848	if (IS_ERR(drng)) {3849		kfree_sensitive(buf);3850		if (PTR_ERR(drng) == -ENOENT)3851			return 0;3852		printk(KERN_ERR "alg: drbg: could not allocate DRNG handle for "3853		       "%s\n", driver);3854		return PTR_ERR(drng);3855	}3856 3857	test_data.testentropy = &testentropy;3858	drbg_string_fill(&testentropy, test->entropy, test->entropylen);3859	drbg_string_fill(&pers, test->pers, test->perslen);3860	ret = crypto_drbg_reset_test(drng, &pers, &test_data);3861	if (ret) {3862		printk(KERN_ERR "alg: drbg: Failed to reset rng\n");3863		goto outbuf;3864	}3865 3866	drbg_string_fill(&addtl, test->addtla, test->addtllen);3867	if (pr) {3868		drbg_string_fill(&testentropy, test->entpra, test->entprlen);3869		ret = crypto_drbg_get_bytes_addtl_test(drng,3870			buf, test->expectedlen, &addtl,	&test_data);3871	} else {3872		ret = crypto_drbg_get_bytes_addtl(drng,3873			buf, test->expectedlen, &addtl);3874	}3875	if (ret < 0) {3876		printk(KERN_ERR "alg: drbg: could not obtain random data for "3877		       "driver %s\n", driver);3878		goto outbuf;3879	}3880 3881	drbg_string_fill(&addtl, test->addtlb, test->addtllen);3882	if (pr) {3883		drbg_string_fill(&testentropy, test->entprb, test->entprlen);3884		ret = crypto_drbg_get_bytes_addtl_test(drng,3885			buf, test->expectedlen, &addtl, &test_data);3886	} else {3887		ret = crypto_drbg_get_bytes_addtl(drng,3888			buf, test->expectedlen, &addtl);3889	}3890	if (ret < 0) {3891		printk(KERN_ERR "alg: drbg: could not obtain random data for "3892		       "driver %s\n", driver);3893		goto outbuf;3894	}3895 3896	ret = memcmp(test->expected, buf, test->expectedlen);3897 3898outbuf:3899	crypto_free_rng(drng);3900	kfree_sensitive(buf);3901	return ret;3902}3903 3904 3905static int alg_test_drbg(const struct alg_test_desc *desc, const char *driver,3906			 u32 type, u32 mask)3907{3908	int err = 0;3909	int pr = 0;3910	int i = 0;3911	const struct drbg_testvec *template = desc->suite.drbg.vecs;3912	unsigned int tcount = desc->suite.drbg.count;3913 3914	if (0 == memcmp(driver, "drbg_pr_", 8))3915		pr = 1;3916 3917	for (i = 0; i < tcount; i++) {3918		err = drbg_cavs_test(&template[i], pr, driver, type, mask);3919		if (err) {3920			printk(KERN_ERR "alg: drbg: Test %d failed for %s\n",3921			       i, driver);3922			err = -EINVAL;3923			break;3924		}3925	}3926	return err;3927 3928}3929 3930static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec,3931		       const char *alg)3932{3933	struct kpp_request *req;3934	void *input_buf = NULL;3935	void *output_buf = NULL;3936	void *a_public = NULL;3937	void *a_ss = NULL;3938	void *shared_secret = NULL;3939	struct crypto_wait wait;3940	unsigned int out_len_max;3941	int err = -ENOMEM;3942	struct scatterlist src, dst;3943 3944	req = kpp_request_alloc(tfm, GFP_KERNEL);3945	if (!req)3946		return err;3947 3948	crypto_init_wait(&wait);3949 3950	err = crypto_kpp_set_secret(tfm, vec->secret, vec->secret_size);3951	if (err < 0)3952		goto free_req;3953 3954	out_len_max = crypto_kpp_maxsize(tfm);3955	output_buf = kzalloc(out_len_max, GFP_KERNEL);3956	if (!output_buf) {3957		err = -ENOMEM;3958		goto free_req;3959	}3960 3961	/* Use appropriate parameter as base */3962	kpp_request_set_input(req, NULL, 0);3963	sg_init_one(&dst, output_buf, out_len_max);3964	kpp_request_set_output(req, &dst, out_len_max);3965	kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,3966				 crypto_req_done, &wait);3967 3968	/* Compute party A's public key */3969	err = crypto_wait_req(crypto_kpp_generate_public_key(req), &wait);3970	if (err) {3971		pr_err("alg: %s: Party A: generate public key test failed. err %d\n",3972		       alg, err);3973		goto free_output;3974	}3975 3976	if (vec->genkey) {3977		/* Save party A's public key */3978		a_public = kmemdup(sg_virt(req->dst), out_len_max, GFP_KERNEL);3979		if (!a_public) {3980			err = -ENOMEM;3981			goto free_output;3982		}3983	} else {3984		/* Verify calculated public key */3985		if (memcmp(vec->expected_a_public, sg_virt(req->dst),3986			   vec->expected_a_public_size)) {3987			pr_err("alg: %s: Party A: generate public key test failed. Invalid output\n",3988			       alg);3989			err = -EINVAL;3990			goto free_output;3991		}3992	}3993 3994	/* Calculate shared secret key by using counter part (b) public key. */3995	input_buf = kmemdup(vec->b_public, vec->b_public_size, GFP_KERNEL);3996	if (!input_buf) {3997		err = -ENOMEM;3998		goto free_output;3999	}4000 4001	sg_init_one(&src, input_buf, vec->b_public_size);4002	sg_init_one(&dst, output_buf, out_len_max);4003	kpp_request_set_input(req, &src, vec->b_public_size);4004	kpp_request_set_output(req, &dst, out_len_max);4005	kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,4006				 crypto_req_done, &wait);4007	err = crypto_wait_req(crypto_kpp_compute_shared_secret(req), &wait);4008	if (err) {4009		pr_err("alg: %s: Party A: compute shared secret test failed. err %d\n",4010		       alg, err);4011		goto free_all;4012	}4013 4014	if (vec->genkey) {4015		/* Save the shared secret obtained by party A */4016		a_ss = kmemdup(sg_virt(req->dst), vec->expected_ss_size, GFP_KERNEL);4017		if (!a_ss) {4018			err = -ENOMEM;4019			goto free_all;4020		}4021 4022		/*4023		 * Calculate party B's shared secret by using party A's4024		 * public key.4025		 */4026		err = crypto_kpp_set_secret(tfm, vec->b_secret,4027					    vec->b_secret_size);4028		if (err < 0)4029			goto free_all;4030 4031		sg_init_one(&src, a_public, vec->expected_a_public_size);4032		sg_init_one(&dst, output_buf, out_len_max);4033		kpp_request_set_input(req, &src, vec->expected_a_public_size);4034		kpp_request_set_output(req, &dst, out_len_max);4035		kpp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,4036					 crypto_req_done, &wait);4037		err = crypto_wait_req(crypto_kpp_compute_shared_secret(req),4038				      &wait);4039		if (err) {4040			pr_err("alg: %s: Party B: compute shared secret failed. err %d\n",4041			       alg, err);4042			goto free_all;4043		}4044 4045		shared_secret = a_ss;4046	} else {4047		shared_secret = (void *)vec->expected_ss;4048	}4049 4050	/*4051	 * verify shared secret from which the user will derive4052	 * secret key by executing whatever hash it has chosen4053	 */4054	if (memcmp(shared_secret, sg_virt(req->dst),4055		   vec->expected_ss_size)) {4056		pr_err("alg: %s: compute shared secret test failed. Invalid output\n",4057		       alg);4058		err = -EINVAL;4059	}4060 4061free_all:4062	kfree(a_ss);4063	kfree(input_buf);4064free_output:4065	kfree(a_public);4066	kfree(output_buf);4067free_req:4068	kpp_request_free(req);4069	return err;4070}4071 4072static int test_kpp(struct crypto_kpp *tfm, const char *alg,4073		    const struct kpp_testvec *vecs, unsigned int tcount)4074{4075	int ret, i;4076 4077	for (i = 0; i < tcount; i++) {4078		ret = do_test_kpp(tfm, vecs++, alg);4079		if (ret) {4080			pr_err("alg: %s: test failed on vector %d, err=%d\n",4081			       alg, i + 1, ret);4082			return ret;4083		}4084	}4085	return 0;4086}4087 4088static int alg_test_kpp(const struct alg_test_desc *desc, const char *driver,4089			u32 type, u32 mask)4090{4091	struct crypto_kpp *tfm;4092	int err = 0;4093 4094	tfm = crypto_alloc_kpp(driver, type, mask);4095	if (IS_ERR(tfm)) {4096		if (PTR_ERR(tfm) == -ENOENT)4097			return 0;4098		pr_err("alg: kpp: Failed to load tfm for %s: %ld\n",4099		       driver, PTR_ERR(tfm));4100		return PTR_ERR(tfm);4101	}4102	if (desc->suite.kpp.vecs)4103		err = test_kpp(tfm, desc->alg, desc->suite.kpp.vecs,4104			       desc->suite.kpp.count);4105 4106	crypto_free_kpp(tfm);4107	return err;4108}4109 4110static u8 *test_pack_u32(u8 *dst, u32 val)4111{4112	memcpy(dst, &val, sizeof(val));4113	return dst + sizeof(val);4114}4115 4116static int test_akcipher_one(struct crypto_akcipher *tfm,4117			     const struct akcipher_testvec *vecs)4118{4119	char *xbuf[XBUFSIZE];4120	struct akcipher_request *req;4121	void *outbuf_enc = NULL;4122	void *outbuf_dec = NULL;4123	struct crypto_wait wait;4124	unsigned int out_len_max, out_len = 0;4125	int err = -ENOMEM;4126	struct scatterlist src, dst, src_tab[3];4127	const char *m, *c;4128	unsigned int m_size, c_size;4129	const char *op;4130	u8 *key, *ptr;4131 4132	if (testmgr_alloc_buf(xbuf))4133		return err;4134 4135	req = akcipher_request_alloc(tfm, GFP_KERNEL);4136	if (!req)4137		goto free_xbuf;4138 4139	crypto_init_wait(&wait);4140 4141	key = kmalloc(vecs->key_len + sizeof(u32) * 2 + vecs->param_len,4142		      GFP_KERNEL);4143	if (!key)4144		goto free_req;4145	memcpy(key, vecs->key, vecs->key_len);4146	ptr = key + vecs->key_len;4147	ptr = test_pack_u32(ptr, vecs->algo);4148	ptr = test_pack_u32(ptr, vecs->param_len);4149	memcpy(ptr, vecs->params, vecs->param_len);4150 4151	if (vecs->public_key_vec)4152		err = crypto_akcipher_set_pub_key(tfm, key, vecs->key_len);4153	else4154		err = crypto_akcipher_set_priv_key(tfm, key, vecs->key_len);4155	if (err)4156		goto free_key;4157 4158	/*4159	 * First run test which do not require a private key, such as4160	 * encrypt or verify.4161	 */4162	err = -ENOMEM;4163	out_len_max = crypto_akcipher_maxsize(tfm);4164	outbuf_enc = kzalloc(out_len_max, GFP_KERNEL);4165	if (!outbuf_enc)4166		goto free_key;4167 4168	if (!vecs->siggen_sigver_test) {4169		m = vecs->m;4170		m_size = vecs->m_size;4171		c = vecs->c;4172		c_size = vecs->c_size;4173		op = "encrypt";4174	} else {4175		/* Swap args so we could keep plaintext (digest)4176		 * in vecs->m, and cooked signature in vecs->c.4177		 */4178		m = vecs->c; /* signature */4179		m_size = vecs->c_size;4180		c = vecs->m; /* digest */4181		c_size = vecs->m_size;4182		op = "verify";4183	}4184 4185	err = -E2BIG;4186	if (WARN_ON(m_size > PAGE_SIZE))4187		goto free_all;4188	memcpy(xbuf[0], m, m_size);4189 4190	sg_init_table(src_tab, 3);4191	sg_set_buf(&src_tab[0], xbuf[0], 8);4192	sg_set_buf(&src_tab[1], xbuf[0] + 8, m_size - 8);4193	if (vecs->siggen_sigver_test) {4194		if (WARN_ON(c_size > PAGE_SIZE))4195			goto free_all;4196		memcpy(xbuf[1], c, c_size);4197		sg_set_buf(&src_tab[2], xbuf[1], c_size);4198		akcipher_request_set_crypt(req, src_tab, NULL, m_size, c_size);4199	} else {4200		sg_init_one(&dst, outbuf_enc, out_len_max);4201		akcipher_request_set_crypt(req, src_tab, &dst, m_size,4202					   out_len_max);4203	}4204	akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,4205				      crypto_req_done, &wait);4206 4207	err = crypto_wait_req(vecs->siggen_sigver_test ?4208			      /* Run asymmetric signature verification */4209			      crypto_akcipher_verify(req) :4210			      /* Run asymmetric encrypt */4211			      crypto_akcipher_encrypt(req), &wait);4212	if (err) {4213		pr_err("alg: akcipher: %s test failed. err %d\n", op, err);4214		goto free_all;4215	}4216	if (!vecs->siggen_sigver_test && c) {4217		if (req->dst_len != c_size) {4218			pr_err("alg: akcipher: %s test failed. Invalid output len\n",4219			       op);4220			err = -EINVAL;4221			goto free_all;4222		}4223		/* verify that encrypted message is equal to expected */4224		if (memcmp(c, outbuf_enc, c_size) != 0) {4225			pr_err("alg: akcipher: %s test failed. Invalid output\n",4226			       op);4227			hexdump(outbuf_enc, c_size);4228			err = -EINVAL;4229			goto free_all;4230		}4231	}4232 4233	/*4234	 * Don't invoke (decrypt or sign) test which require a private key4235	 * for vectors with only a public key.4236	 */4237	if (vecs->public_key_vec) {4238		err = 0;4239		goto free_all;4240	}4241	outbuf_dec = kzalloc(out_len_max, GFP_KERNEL);4242	if (!outbuf_dec) {4243		err = -ENOMEM;4244		goto free_all;4245	}4246 4247	if (!vecs->siggen_sigver_test && !c) {4248		c = outbuf_enc;4249		c_size = req->dst_len;4250	}4251 4252	err = -E2BIG;4253	op = vecs->siggen_sigver_test ? "sign" : "decrypt";4254	if (WARN_ON(c_size > PAGE_SIZE))4255		goto free_all;4256	memcpy(xbuf[0], c, c_size);4257 4258	sg_init_one(&src, xbuf[0], c_size);4259	sg_init_one(&dst, outbuf_dec, out_len_max);4260	crypto_init_wait(&wait);4261	akcipher_request_set_crypt(req, &src, &dst, c_size, out_len_max);4262 4263	err = crypto_wait_req(vecs->siggen_sigver_test ?4264			      /* Run asymmetric signature generation */4265			      crypto_akcipher_sign(req) :4266			      /* Run asymmetric decrypt */4267			      crypto_akcipher_decrypt(req), &wait);4268	if (err) {4269		pr_err("alg: akcipher: %s test failed. err %d\n", op, err);4270		goto free_all;4271	}4272	out_len = req->dst_len;4273	if (out_len < m_size) {4274		pr_err("alg: akcipher: %s test failed. Invalid output len %u\n",4275		       op, out_len);4276		err = -EINVAL;4277		goto free_all;4278	}4279	/* verify that decrypted message is equal to the original msg */4280	if (memchr_inv(outbuf_dec, 0, out_len - m_size) ||4281	    memcmp(m, outbuf_dec + out_len - m_size, m_size)) {4282		pr_err("alg: akcipher: %s test failed. Invalid output\n", op);4283		hexdump(outbuf_dec, out_len);4284		err = -EINVAL;4285	}4286free_all:4287	kfree(outbuf_dec);4288	kfree(outbuf_enc);4289free_key:4290	kfree(key);4291free_req:4292	akcipher_request_free(req);4293free_xbuf:4294	testmgr_free_buf(xbuf);4295	return err;4296}4297 4298static int test_akcipher(struct crypto_akcipher *tfm, const char *alg,4299			 const struct akcipher_testvec *vecs,4300			 unsigned int tcount)4301{4302	const char *algo =4303		crypto_tfm_alg_driver_name(crypto_akcipher_tfm(tfm));4304	int ret, i;4305 4306	for (i = 0; i < tcount; i++) {4307		ret = test_akcipher_one(tfm, vecs++);4308		if (!ret)4309			continue;4310 4311		pr_err("alg: akcipher: test %d failed for %s, err=%d\n",4312		       i + 1, algo, ret);4313		return ret;4314	}4315	return 0;4316}4317 4318static int alg_test_akcipher(const struct alg_test_desc *desc,4319			     const char *driver, u32 type, u32 mask)4320{4321	struct crypto_akcipher *tfm;4322	int err = 0;4323 4324	tfm = crypto_alloc_akcipher(driver, type, mask);4325	if (IS_ERR(tfm)) {4326		if (PTR_ERR(tfm) == -ENOENT)4327			return 0;4328		pr_err("alg: akcipher: Failed to load tfm for %s: %ld\n",4329		       driver, PTR_ERR(tfm));4330		return PTR_ERR(tfm);4331	}4332	if (desc->suite.akcipher.vecs)4333		err = test_akcipher(tfm, desc->alg, desc->suite.akcipher.vecs,4334				    desc->suite.akcipher.count);4335 4336	crypto_free_akcipher(tfm);4337	return err;4338}4339 4340static int alg_test_null(const struct alg_test_desc *desc,4341			     const char *driver, u32 type, u32 mask)4342{4343	return 0;4344}4345 4346#define ____VECS(tv)	.vecs = tv, .count = ARRAY_SIZE(tv)4347#define __VECS(tv)	{ ____VECS(tv) }4348 4349/* Please keep this list sorted by algorithm name. */4350static const struct alg_test_desc alg_test_descs[] = {4351	{4352		.alg = "adiantum(xchacha12,aes)",4353		.generic_driver = "adiantum(xchacha12-generic,aes-generic,nhpoly1305-generic)",4354		.test = alg_test_skcipher,4355		.suite = {4356			.cipher = __VECS(adiantum_xchacha12_aes_tv_template)4357		},4358	}, {4359		.alg = "adiantum(xchacha20,aes)",4360		.generic_driver = "adiantum(xchacha20-generic,aes-generic,nhpoly1305-generic)",4361		.test = alg_test_skcipher,4362		.suite = {4363			.cipher = __VECS(adiantum_xchacha20_aes_tv_template)4364		},4365	}, {4366		.alg = "aegis128",4367		.test = alg_test_aead,4368		.suite = {4369			.aead = __VECS(aegis128_tv_template)4370		}4371	}, {4372		.alg = "ansi_cprng",4373		.test = alg_test_cprng,4374		.suite = {4375			.cprng = __VECS(ansi_cprng_aes_tv_template)4376		}4377	}, {4378		.alg = "authenc(hmac(md5),ecb(cipher_null))",4379		.test = alg_test_aead,4380		.suite = {4381			.aead = __VECS(hmac_md5_ecb_cipher_null_tv_template)4382		}4383	}, {4384		.alg = "authenc(hmac(sha1),cbc(aes))",4385		.test = alg_test_aead,4386		.fips_allowed = 1,4387		.suite = {4388			.aead = __VECS(hmac_sha1_aes_cbc_tv_temp)4389		}4390	}, {4391		.alg = "authenc(hmac(sha1),cbc(des))",4392		.test = alg_test_aead,4393		.suite = {4394			.aead = __VECS(hmac_sha1_des_cbc_tv_temp)4395		}4396	}, {4397		.alg = "authenc(hmac(sha1),cbc(des3_ede))",4398		.test = alg_test_aead,4399		.suite = {4400			.aead = __VECS(hmac_sha1_des3_ede_cbc_tv_temp)4401		}4402	}, {4403		.alg = "authenc(hmac(sha1),ctr(aes))",4404		.test = alg_test_null,4405		.fips_allowed = 1,4406	}, {4407		.alg = "authenc(hmac(sha1),ecb(cipher_null))",4408		.test = alg_test_aead,4409		.suite = {4410			.aead = __VECS(hmac_sha1_ecb_cipher_null_tv_temp)4411		}4412	}, {4413		.alg = "authenc(hmac(sha1),rfc3686(ctr(aes)))",4414		.test = alg_test_null,4415		.fips_allowed = 1,4416	}, {4417		.alg = "authenc(hmac(sha224),cbc(des))",4418		.test = alg_test_aead,4419		.suite = {4420			.aead = __VECS(hmac_sha224_des_cbc_tv_temp)4421		}4422	}, {4423		.alg = "authenc(hmac(sha224),cbc(des3_ede))",4424		.test = alg_test_aead,4425		.suite = {4426			.aead = __VECS(hmac_sha224_des3_ede_cbc_tv_temp)4427		}4428	}, {4429		.alg = "authenc(hmac(sha256),cbc(aes))",4430		.test = alg_test_aead,4431		.fips_allowed = 1,4432		.suite = {4433			.aead = __VECS(hmac_sha256_aes_cbc_tv_temp)4434		}4435	}, {4436		.alg = "authenc(hmac(sha256),cbc(des))",4437		.test = alg_test_aead,4438		.suite = {4439			.aead = __VECS(hmac_sha256_des_cbc_tv_temp)4440		}4441	}, {4442		.alg = "authenc(hmac(sha256),cbc(des3_ede))",4443		.test = alg_test_aead,4444		.suite = {4445			.aead = __VECS(hmac_sha256_des3_ede_cbc_tv_temp)4446		}4447	}, {4448		.alg = "authenc(hmac(sha256),ctr(aes))",4449		.test = alg_test_null,4450		.fips_allowed = 1,4451	}, {4452		.alg = "authenc(hmac(sha256),rfc3686(ctr(aes)))",4453		.test = alg_test_null,4454		.fips_allowed = 1,4455	}, {4456		.alg = "authenc(hmac(sha384),cbc(des))",4457		.test = alg_test_aead,4458		.suite = {4459			.aead = __VECS(hmac_sha384_des_cbc_tv_temp)4460		}4461	}, {4462		.alg = "authenc(hmac(sha384),cbc(des3_ede))",4463		.test = alg_test_aead,4464		.suite = {4465			.aead = __VECS(hmac_sha384_des3_ede_cbc_tv_temp)4466		}4467	}, {4468		.alg = "authenc(hmac(sha384),ctr(aes))",4469		.test = alg_test_null,4470		.fips_allowed = 1,4471	}, {4472		.alg = "authenc(hmac(sha384),rfc3686(ctr(aes)))",4473		.test = alg_test_null,4474		.fips_allowed = 1,4475	}, {4476		.alg = "authenc(hmac(sha512),cbc(aes))",4477		.fips_allowed = 1,4478		.test = alg_test_aead,4479		.suite = {4480			.aead = __VECS(hmac_sha512_aes_cbc_tv_temp)4481		}4482	}, {4483		.alg = "authenc(hmac(sha512),cbc(des))",4484		.test = alg_test_aead,4485		.suite = {4486			.aead = __VECS(hmac_sha512_des_cbc_tv_temp)4487		}4488	}, {4489		.alg = "authenc(hmac(sha512),cbc(des3_ede))",4490		.test = alg_test_aead,4491		.suite = {4492			.aead = __VECS(hmac_sha512_des3_ede_cbc_tv_temp)4493		}4494	}, {4495		.alg = "authenc(hmac(sha512),ctr(aes))",4496		.test = alg_test_null,4497		.fips_allowed = 1,4498	}, {4499		.alg = "authenc(hmac(sha512),rfc3686(ctr(aes)))",4500		.test = alg_test_null,4501		.fips_allowed = 1,4502	}, {4503		.alg = "blake2b-160",4504		.test = alg_test_hash,4505		.fips_allowed = 0,4506		.suite = {4507			.hash = __VECS(blake2b_160_tv_template)4508		}4509	}, {4510		.alg = "blake2b-256",4511		.test = alg_test_hash,4512		.fips_allowed = 0,4513		.suite = {4514			.hash = __VECS(blake2b_256_tv_template)4515		}4516	}, {4517		.alg = "blake2b-384",4518		.test = alg_test_hash,4519		.fips_allowed = 0,4520		.suite = {4521			.hash = __VECS(blake2b_384_tv_template)4522		}4523	}, {4524		.alg = "blake2b-512",4525		.test = alg_test_hash,4526		.fips_allowed = 0,4527		.suite = {4528			.hash = __VECS(blake2b_512_tv_template)4529		}4530	}, {4531		.alg = "cbc(aes)",4532		.test = alg_test_skcipher,4533		.fips_allowed = 1,4534		.suite = {4535			.cipher = __VECS(aes_cbc_tv_template)4536		},4537	}, {4538		.alg = "cbc(anubis)",4539		.test = alg_test_skcipher,4540		.suite = {4541			.cipher = __VECS(anubis_cbc_tv_template)4542		},4543	}, {4544		.alg = "cbc(aria)",4545		.test = alg_test_skcipher,4546		.suite = {4547			.cipher = __VECS(aria_cbc_tv_template)4548		},4549	}, {4550		.alg = "cbc(blowfish)",4551		.test = alg_test_skcipher,4552		.suite = {4553			.cipher = __VECS(bf_cbc_tv_template)4554		},4555	}, {4556		.alg = "cbc(camellia)",4557		.test = alg_test_skcipher,4558		.suite = {4559			.cipher = __VECS(camellia_cbc_tv_template)4560		},4561	}, {4562		.alg = "cbc(cast5)",4563		.test = alg_test_skcipher,4564		.suite = {4565			.cipher = __VECS(cast5_cbc_tv_template)4566		},4567	}, {4568		.alg = "cbc(cast6)",4569		.test = alg_test_skcipher,4570		.suite = {4571			.cipher = __VECS(cast6_cbc_tv_template)4572		},4573	}, {4574		.alg = "cbc(des)",4575		.test = alg_test_skcipher,4576		.suite = {4577			.cipher = __VECS(des_cbc_tv_template)4578		},4579	}, {4580		.alg = "cbc(des3_ede)",4581		.test = alg_test_skcipher,4582		.suite = {4583			.cipher = __VECS(des3_ede_cbc_tv_template)4584		},4585	}, {4586		/* Same as cbc(aes) except the key is stored in4587		 * hardware secure memory which we reference by index4588		 */4589		.alg = "cbc(paes)",4590		.test = alg_test_null,4591		.fips_allowed = 1,4592	}, {4593		/* Same as cbc(sm4) except the key is stored in4594		 * hardware secure memory which we reference by index4595		 */4596		.alg = "cbc(psm4)",4597		.test = alg_test_null,4598	}, {4599		.alg = "cbc(serpent)",4600		.test = alg_test_skcipher,4601		.suite = {4602			.cipher = __VECS(serpent_cbc_tv_template)4603		},4604	}, {4605		.alg = "cbc(sm4)",4606		.test = alg_test_skcipher,4607		.suite = {4608			.cipher = __VECS(sm4_cbc_tv_template)4609		}4610	}, {4611		.alg = "cbc(twofish)",4612		.test = alg_test_skcipher,4613		.suite = {4614			.cipher = __VECS(tf_cbc_tv_template)4615		},4616	}, {4617#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)4618		.alg = "cbc-paes-s390",4619		.fips_allowed = 1,4620		.test = alg_test_skcipher,4621		.suite = {4622			.cipher = __VECS(aes_cbc_tv_template)4623		}4624	}, {4625#endif4626		.alg = "cbcmac(aes)",4627		.test = alg_test_hash,4628		.suite = {4629			.hash = __VECS(aes_cbcmac_tv_template)4630		}4631	}, {4632		.alg = "cbcmac(sm4)",4633		.test = alg_test_hash,4634		.suite = {4635			.hash = __VECS(sm4_cbcmac_tv_template)4636		}4637	}, {4638		.alg = "ccm(aes)",4639		.generic_driver = "ccm_base(ctr(aes-generic),cbcmac(aes-generic))",4640		.test = alg_test_aead,4641		.fips_allowed = 1,4642		.suite = {4643			.aead = {4644				____VECS(aes_ccm_tv_template),4645				.einval_allowed = 1,4646			}4647		}4648	}, {4649		.alg = "ccm(sm4)",4650		.generic_driver = "ccm_base(ctr(sm4-generic),cbcmac(sm4-generic))",4651		.test = alg_test_aead,4652		.suite = {4653			.aead = {4654				____VECS(sm4_ccm_tv_template),4655				.einval_allowed = 1,4656			}4657		}4658	}, {4659		.alg = "chacha20",4660		.test = alg_test_skcipher,4661		.suite = {4662			.cipher = __VECS(chacha20_tv_template)4663		},4664	}, {4665		.alg = "cmac(aes)",4666		.fips_allowed = 1,4667		.test = alg_test_hash,4668		.suite = {4669			.hash = __VECS(aes_cmac128_tv_template)4670		}4671	}, {4672		.alg = "cmac(camellia)",4673		.test = alg_test_hash,4674		.suite = {4675			.hash = __VECS(camellia_cmac128_tv_template)4676		}4677	}, {4678		.alg = "cmac(des3_ede)",4679		.test = alg_test_hash,4680		.suite = {4681			.hash = __VECS(des3_ede_cmac64_tv_template)4682		}4683	}, {4684		.alg = "cmac(sm4)",4685		.test = alg_test_hash,4686		.suite = {4687			.hash = __VECS(sm4_cmac128_tv_template)4688		}4689	}, {4690		.alg = "compress_null",4691		.test = alg_test_null,4692	}, {4693		.alg = "crc32",4694		.test = alg_test_hash,4695		.fips_allowed = 1,4696		.suite = {4697			.hash = __VECS(crc32_tv_template)4698		}4699	}, {4700		.alg = "crc32c",4701		.test = alg_test_crc32c,4702		.fips_allowed = 1,4703		.suite = {4704			.hash = __VECS(crc32c_tv_template)4705		}4706	}, {4707		.alg = "crc64-rocksoft",4708		.test = alg_test_hash,4709		.fips_allowed = 1,4710		.suite = {4711			.hash = __VECS(crc64_rocksoft_tv_template)4712		}4713	}, {4714		.alg = "crct10dif",4715		.test = alg_test_hash,4716		.fips_allowed = 1,4717		.suite = {4718			.hash = __VECS(crct10dif_tv_template)4719		}4720	}, {4721		.alg = "ctr(aes)",4722		.test = alg_test_skcipher,4723		.fips_allowed = 1,4724		.suite = {4725			.cipher = __VECS(aes_ctr_tv_template)4726		}4727	}, {4728		.alg = "ctr(aria)",4729		.test = alg_test_skcipher,4730		.suite = {4731			.cipher = __VECS(aria_ctr_tv_template)4732		}4733	}, {4734		.alg = "ctr(blowfish)",4735		.test = alg_test_skcipher,4736		.suite = {4737			.cipher = __VECS(bf_ctr_tv_template)4738		}4739	}, {4740		.alg = "ctr(camellia)",4741		.test = alg_test_skcipher,4742		.suite = {4743			.cipher = __VECS(camellia_ctr_tv_template)4744		}4745	}, {4746		.alg = "ctr(cast5)",4747		.test = alg_test_skcipher,4748		.suite = {4749			.cipher = __VECS(cast5_ctr_tv_template)4750		}4751	}, {4752		.alg = "ctr(cast6)",4753		.test = alg_test_skcipher,4754		.suite = {4755			.cipher = __VECS(cast6_ctr_tv_template)4756		}4757	}, {4758		.alg = "ctr(des)",4759		.test = alg_test_skcipher,4760		.suite = {4761			.cipher = __VECS(des_ctr_tv_template)4762		}4763	}, {4764		.alg = "ctr(des3_ede)",4765		.test = alg_test_skcipher,4766		.suite = {4767			.cipher = __VECS(des3_ede_ctr_tv_template)4768		}4769	}, {4770		/* Same as ctr(aes) except the key is stored in4771		 * hardware secure memory which we reference by index4772		 */4773		.alg = "ctr(paes)",4774		.test = alg_test_null,4775		.fips_allowed = 1,4776	}, {4777 4778		/* Same as ctr(sm4) except the key is stored in4779		 * hardware secure memory which we reference by index4780		 */4781		.alg = "ctr(psm4)",4782		.test = alg_test_null,4783	}, {4784		.alg = "ctr(serpent)",4785		.test = alg_test_skcipher,4786		.suite = {4787			.cipher = __VECS(serpent_ctr_tv_template)4788		}4789	}, {4790		.alg = "ctr(sm4)",4791		.test = alg_test_skcipher,4792		.suite = {4793			.cipher = __VECS(sm4_ctr_tv_template)4794		}4795	}, {4796		.alg = "ctr(twofish)",4797		.test = alg_test_skcipher,4798		.suite = {4799			.cipher = __VECS(tf_ctr_tv_template)4800		}4801	}, {4802#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)4803		.alg = "ctr-paes-s390",4804		.fips_allowed = 1,4805		.test = alg_test_skcipher,4806		.suite = {4807			.cipher = __VECS(aes_ctr_tv_template)4808		}4809	}, {4810#endif4811		.alg = "cts(cbc(aes))",4812		.test = alg_test_skcipher,4813		.fips_allowed = 1,4814		.suite = {4815			.cipher = __VECS(cts_mode_tv_template)4816		}4817	}, {4818		/* Same as cts(cbc((aes)) except the key is stored in4819		 * hardware secure memory which we reference by index4820		 */4821		.alg = "cts(cbc(paes))",4822		.test = alg_test_null,4823		.fips_allowed = 1,4824	}, {4825		.alg = "cts(cbc(sm4))",4826		.test = alg_test_skcipher,4827		.suite = {4828			.cipher = __VECS(sm4_cts_tv_template)4829		}4830	}, {4831		.alg = "curve25519",4832		.test = alg_test_kpp,4833		.suite = {4834			.kpp = __VECS(curve25519_tv_template)4835		}4836	}, {4837		.alg = "deflate",4838		.test = alg_test_comp,4839		.fips_allowed = 1,4840		.suite = {4841			.comp = {4842				.comp = __VECS(deflate_comp_tv_template),4843				.decomp = __VECS(deflate_decomp_tv_template)4844			}4845		}4846	}, {4847		.alg = "deflate-iaa",4848		.test = alg_test_comp,4849		.fips_allowed = 1,4850		.suite = {4851			.comp = {4852				.comp = __VECS(deflate_comp_tv_template),4853				.decomp = __VECS(deflate_decomp_tv_template)4854			}4855		}4856	}, {4857		.alg = "dh",4858		.test = alg_test_kpp,4859		.suite = {4860			.kpp = __VECS(dh_tv_template)4861		}4862	}, {4863		.alg = "digest_null",4864		.test = alg_test_null,4865	}, {4866		.alg = "drbg_nopr_ctr_aes128",4867		.test = alg_test_drbg,4868		.fips_allowed = 1,4869		.suite = {4870			.drbg = __VECS(drbg_nopr_ctr_aes128_tv_template)4871		}4872	}, {4873		.alg = "drbg_nopr_ctr_aes192",4874		.test = alg_test_drbg,4875		.fips_allowed = 1,4876		.suite = {4877			.drbg = __VECS(drbg_nopr_ctr_aes192_tv_template)4878		}4879	}, {4880		.alg = "drbg_nopr_ctr_aes256",4881		.test = alg_test_drbg,4882		.fips_allowed = 1,4883		.suite = {4884			.drbg = __VECS(drbg_nopr_ctr_aes256_tv_template)4885		}4886	}, {4887		.alg = "drbg_nopr_hmac_sha256",4888		.test = alg_test_drbg,4889		.fips_allowed = 1,4890		.suite = {4891			.drbg = __VECS(drbg_nopr_hmac_sha256_tv_template)4892		}4893	}, {4894		/*4895		 * There is no need to specifically test the DRBG with every4896		 * backend cipher -- covered by drbg_nopr_hmac_sha512 test4897		 */4898		.alg = "drbg_nopr_hmac_sha384",4899		.test = alg_test_null,4900	}, {4901		.alg = "drbg_nopr_hmac_sha512",4902		.test = alg_test_drbg,4903		.fips_allowed = 1,4904		.suite = {4905			.drbg = __VECS(drbg_nopr_hmac_sha512_tv_template)4906		}4907	}, {4908		.alg = "drbg_nopr_sha256",4909		.test = alg_test_drbg,4910		.fips_allowed = 1,4911		.suite = {4912			.drbg = __VECS(drbg_nopr_sha256_tv_template)4913		}4914	}, {4915		/* covered by drbg_nopr_sha256 test */4916		.alg = "drbg_nopr_sha384",4917		.test = alg_test_null,4918	}, {4919		.alg = "drbg_nopr_sha512",4920		.fips_allowed = 1,4921		.test = alg_test_null,4922	}, {4923		.alg = "drbg_pr_ctr_aes128",4924		.test = alg_test_drbg,4925		.fips_allowed = 1,4926		.suite = {4927			.drbg = __VECS(drbg_pr_ctr_aes128_tv_template)4928		}4929	}, {4930		/* covered by drbg_pr_ctr_aes128 test */4931		.alg = "drbg_pr_ctr_aes192",4932		.fips_allowed = 1,4933		.test = alg_test_null,4934	}, {4935		.alg = "drbg_pr_ctr_aes256",4936		.fips_allowed = 1,4937		.test = alg_test_null,4938	}, {4939		.alg = "drbg_pr_hmac_sha256",4940		.test = alg_test_drbg,4941		.fips_allowed = 1,4942		.suite = {4943			.drbg = __VECS(drbg_pr_hmac_sha256_tv_template)4944		}4945	}, {4946		/* covered by drbg_pr_hmac_sha256 test */4947		.alg = "drbg_pr_hmac_sha384",4948		.test = alg_test_null,4949	}, {4950		.alg = "drbg_pr_hmac_sha512",4951		.test = alg_test_null,4952		.fips_allowed = 1,4953	}, {4954		.alg = "drbg_pr_sha256",4955		.test = alg_test_drbg,4956		.fips_allowed = 1,4957		.suite = {4958			.drbg = __VECS(drbg_pr_sha256_tv_template)4959		}4960	}, {4961		/* covered by drbg_pr_sha256 test */4962		.alg = "drbg_pr_sha384",4963		.test = alg_test_null,4964	}, {4965		.alg = "drbg_pr_sha512",4966		.fips_allowed = 1,4967		.test = alg_test_null,4968	}, {4969		.alg = "ecb(aes)",4970		.test = alg_test_skcipher,4971		.fips_allowed = 1,4972		.suite = {4973			.cipher = __VECS(aes_tv_template)4974		}4975	}, {4976		.alg = "ecb(anubis)",4977		.test = alg_test_skcipher,4978		.suite = {4979			.cipher = __VECS(anubis_tv_template)4980		}4981	}, {4982		.alg = "ecb(arc4)",4983		.generic_driver = "arc4-generic",4984		.test = alg_test_skcipher,4985		.suite = {4986			.cipher = __VECS(arc4_tv_template)4987		}4988	}, {4989		.alg = "ecb(aria)",4990		.test = alg_test_skcipher,4991		.suite = {4992			.cipher = __VECS(aria_tv_template)4993		}4994	}, {4995		.alg = "ecb(blowfish)",4996		.test = alg_test_skcipher,4997		.suite = {4998			.cipher = __VECS(bf_tv_template)4999		}5000	}, {5001		.alg = "ecb(camellia)",5002		.test = alg_test_skcipher,5003		.suite = {5004			.cipher = __VECS(camellia_tv_template)5005		}5006	}, {5007		.alg = "ecb(cast5)",5008		.test = alg_test_skcipher,5009		.suite = {5010			.cipher = __VECS(cast5_tv_template)5011		}5012	}, {5013		.alg = "ecb(cast6)",5014		.test = alg_test_skcipher,5015		.suite = {5016			.cipher = __VECS(cast6_tv_template)5017		}5018	}, {5019		.alg = "ecb(cipher_null)",5020		.test = alg_test_null,5021		.fips_allowed = 1,5022	}, {5023		.alg = "ecb(des)",5024		.test = alg_test_skcipher,5025		.suite = {5026			.cipher = __VECS(des_tv_template)5027		}5028	}, {5029		.alg = "ecb(des3_ede)",5030		.test = alg_test_skcipher,5031		.suite = {5032			.cipher = __VECS(des3_ede_tv_template)5033		}5034	}, {5035		.alg = "ecb(fcrypt)",5036		.test = alg_test_skcipher,5037		.suite = {5038			.cipher = {5039				.vecs = fcrypt_pcbc_tv_template,5040				.count = 15041			}5042		}5043	}, {5044		.alg = "ecb(khazad)",5045		.test = alg_test_skcipher,5046		.suite = {5047			.cipher = __VECS(khazad_tv_template)5048		}5049	}, {5050		/* Same as ecb(aes) except the key is stored in5051		 * hardware secure memory which we reference by index5052		 */5053		.alg = "ecb(paes)",5054		.test = alg_test_null,5055		.fips_allowed = 1,5056	}, {5057		.alg = "ecb(seed)",5058		.test = alg_test_skcipher,5059		.suite = {5060			.cipher = __VECS(seed_tv_template)5061		}5062	}, {5063		.alg = "ecb(serpent)",5064		.test = alg_test_skcipher,5065		.suite = {5066			.cipher = __VECS(serpent_tv_template)5067		}5068	}, {5069		.alg = "ecb(sm4)",5070		.test = alg_test_skcipher,5071		.suite = {5072			.cipher = __VECS(sm4_tv_template)5073		}5074	}, {5075		.alg = "ecb(tea)",5076		.test = alg_test_skcipher,5077		.suite = {5078			.cipher = __VECS(tea_tv_template)5079		}5080	}, {5081		.alg = "ecb(twofish)",5082		.test = alg_test_skcipher,5083		.suite = {5084			.cipher = __VECS(tf_tv_template)5085		}5086	}, {5087		.alg = "ecb(xeta)",5088		.test = alg_test_skcipher,5089		.suite = {5090			.cipher = __VECS(xeta_tv_template)5091		}5092	}, {5093		.alg = "ecb(xtea)",5094		.test = alg_test_skcipher,5095		.suite = {5096			.cipher = __VECS(xtea_tv_template)5097		}5098	}, {5099#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)5100		.alg = "ecb-paes-s390",5101		.fips_allowed = 1,5102		.test = alg_test_skcipher,5103		.suite = {5104			.cipher = __VECS(aes_tv_template)5105		}5106	}, {5107#endif5108		.alg = "ecdh-nist-p192",5109		.test = alg_test_kpp,5110		.suite = {5111			.kpp = __VECS(ecdh_p192_tv_template)5112		}5113	}, {5114		.alg = "ecdh-nist-p256",5115		.test = alg_test_kpp,5116		.fips_allowed = 1,5117		.suite = {5118			.kpp = __VECS(ecdh_p256_tv_template)5119		}5120	}, {5121		.alg = "ecdh-nist-p384",5122		.test = alg_test_kpp,5123		.fips_allowed = 1,5124		.suite = {5125			.kpp = __VECS(ecdh_p384_tv_template)5126		}5127	}, {5128		.alg = "ecdsa-nist-p192",5129		.test = alg_test_akcipher,5130		.suite = {5131			.akcipher = __VECS(ecdsa_nist_p192_tv_template)5132		}5133	}, {5134		.alg = "ecdsa-nist-p256",5135		.test = alg_test_akcipher,5136		.fips_allowed = 1,5137		.suite = {5138			.akcipher = __VECS(ecdsa_nist_p256_tv_template)5139		}5140	}, {5141		.alg = "ecdsa-nist-p384",5142		.test = alg_test_akcipher,5143		.fips_allowed = 1,5144		.suite = {5145			.akcipher = __VECS(ecdsa_nist_p384_tv_template)5146		}5147	}, {5148		.alg = "ecdsa-nist-p521",5149		.test = alg_test_akcipher,5150		.fips_allowed = 1,5151		.suite = {5152			.akcipher = __VECS(ecdsa_nist_p521_tv_template)5153		}5154	}, {5155		.alg = "ecrdsa",5156		.test = alg_test_akcipher,5157		.suite = {5158			.akcipher = __VECS(ecrdsa_tv_template)5159		}5160	}, {5161		.alg = "essiv(authenc(hmac(sha256),cbc(aes)),sha256)",5162		.test = alg_test_aead,5163		.fips_allowed = 1,5164		.suite = {5165			.aead = __VECS(essiv_hmac_sha256_aes_cbc_tv_temp)5166		}5167	}, {5168		.alg = "essiv(cbc(aes),sha256)",5169		.test = alg_test_skcipher,5170		.fips_allowed = 1,5171		.suite = {5172			.cipher = __VECS(essiv_aes_cbc_tv_template)5173		}5174	}, {5175#if IS_ENABLED(CONFIG_CRYPTO_DH_RFC7919_GROUPS)5176		.alg = "ffdhe2048(dh)",5177		.test = alg_test_kpp,5178		.fips_allowed = 1,5179		.suite = {5180			.kpp = __VECS(ffdhe2048_dh_tv_template)5181		}5182	}, {5183		.alg = "ffdhe3072(dh)",5184		.test = alg_test_kpp,5185		.fips_allowed = 1,5186		.suite = {5187			.kpp = __VECS(ffdhe3072_dh_tv_template)5188		}5189	}, {5190		.alg = "ffdhe4096(dh)",5191		.test = alg_test_kpp,5192		.fips_allowed = 1,5193		.suite = {5194			.kpp = __VECS(ffdhe4096_dh_tv_template)5195		}5196	}, {5197		.alg = "ffdhe6144(dh)",5198		.test = alg_test_kpp,5199		.fips_allowed = 1,5200		.suite = {5201			.kpp = __VECS(ffdhe6144_dh_tv_template)5202		}5203	}, {5204		.alg = "ffdhe8192(dh)",5205		.test = alg_test_kpp,5206		.fips_allowed = 1,5207		.suite = {5208			.kpp = __VECS(ffdhe8192_dh_tv_template)5209		}5210	}, {5211#endif /* CONFIG_CRYPTO_DH_RFC7919_GROUPS */5212		.alg = "gcm(aes)",5213		.generic_driver = "gcm_base(ctr(aes-generic),ghash-generic)",5214		.test = alg_test_aead,5215		.fips_allowed = 1,5216		.suite = {5217			.aead = __VECS(aes_gcm_tv_template)5218		}5219	}, {5220		.alg = "gcm(aria)",5221		.generic_driver = "gcm_base(ctr(aria-generic),ghash-generic)",5222		.test = alg_test_aead,5223		.suite = {5224			.aead = __VECS(aria_gcm_tv_template)5225		}5226	}, {5227		.alg = "gcm(sm4)",5228		.generic_driver = "gcm_base(ctr(sm4-generic),ghash-generic)",5229		.test = alg_test_aead,5230		.suite = {5231			.aead = __VECS(sm4_gcm_tv_template)5232		}5233	}, {5234		.alg = "ghash",5235		.test = alg_test_hash,5236		.suite = {5237			.hash = __VECS(ghash_tv_template)5238		}5239	}, {5240		.alg = "hctr2(aes)",5241		.generic_driver =5242		    "hctr2_base(xctr(aes-generic),polyval-generic)",5243		.test = alg_test_skcipher,5244		.suite = {5245			.cipher = __VECS(aes_hctr2_tv_template)5246		}5247	}, {5248		.alg = "hmac(md5)",5249		.test = alg_test_hash,5250		.suite = {5251			.hash = __VECS(hmac_md5_tv_template)5252		}5253	}, {5254		.alg = "hmac(rmd160)",5255		.test = alg_test_hash,5256		.suite = {5257			.hash = __VECS(hmac_rmd160_tv_template)5258		}5259	}, {5260		.alg = "hmac(sha1)",5261		.test = alg_test_hash,5262		.fips_allowed = 1,5263		.suite = {5264			.hash = __VECS(hmac_sha1_tv_template)5265		}5266	}, {5267		.alg = "hmac(sha224)",5268		.test = alg_test_hash,5269		.fips_allowed = 1,5270		.suite = {5271			.hash = __VECS(hmac_sha224_tv_template)5272		}5273	}, {5274		.alg = "hmac(sha256)",5275		.test = alg_test_hash,5276		.fips_allowed = 1,5277		.suite = {5278			.hash = __VECS(hmac_sha256_tv_template)5279		}5280	}, {5281		.alg = "hmac(sha3-224)",5282		.test = alg_test_hash,5283		.fips_allowed = 1,5284		.suite = {5285			.hash = __VECS(hmac_sha3_224_tv_template)5286		}5287	}, {5288		.alg = "hmac(sha3-256)",5289		.test = alg_test_hash,5290		.fips_allowed = 1,5291		.suite = {5292			.hash = __VECS(hmac_sha3_256_tv_template)5293		}5294	}, {5295		.alg = "hmac(sha3-384)",5296		.test = alg_test_hash,5297		.fips_allowed = 1,5298		.suite = {5299			.hash = __VECS(hmac_sha3_384_tv_template)5300		}5301	}, {5302		.alg = "hmac(sha3-512)",5303		.test = alg_test_hash,5304		.fips_allowed = 1,5305		.suite = {5306			.hash = __VECS(hmac_sha3_512_tv_template)5307		}5308	}, {5309		.alg = "hmac(sha384)",5310		.test = alg_test_hash,5311		.fips_allowed = 1,5312		.suite = {5313			.hash = __VECS(hmac_sha384_tv_template)5314		}5315	}, {5316		.alg = "hmac(sha512)",5317		.test = alg_test_hash,5318		.fips_allowed = 1,5319		.suite = {5320			.hash = __VECS(hmac_sha512_tv_template)5321		}5322	}, {5323		.alg = "hmac(sm3)",5324		.test = alg_test_hash,5325		.suite = {5326			.hash = __VECS(hmac_sm3_tv_template)5327		}5328	}, {5329		.alg = "hmac(streebog256)",5330		.test = alg_test_hash,5331		.suite = {5332			.hash = __VECS(hmac_streebog256_tv_template)5333		}5334	}, {5335		.alg = "hmac(streebog512)",5336		.test = alg_test_hash,5337		.suite = {5338			.hash = __VECS(hmac_streebog512_tv_template)5339		}5340	}, {5341		.alg = "jitterentropy_rng",5342		.fips_allowed = 1,5343		.test = alg_test_null,5344	}, {5345		.alg = "kw(aes)",5346		.test = alg_test_skcipher,5347		.fips_allowed = 1,5348		.suite = {5349			.cipher = __VECS(aes_kw_tv_template)5350		}5351	}, {5352		.alg = "lrw(aes)",5353		.generic_driver = "lrw(ecb(aes-generic))",5354		.test = alg_test_skcipher,5355		.suite = {5356			.cipher = __VECS(aes_lrw_tv_template)5357		}5358	}, {5359		.alg = "lrw(camellia)",5360		.generic_driver = "lrw(ecb(camellia-generic))",5361		.test = alg_test_skcipher,5362		.suite = {5363			.cipher = __VECS(camellia_lrw_tv_template)5364		}5365	}, {5366		.alg = "lrw(cast6)",5367		.generic_driver = "lrw(ecb(cast6-generic))",5368		.test = alg_test_skcipher,5369		.suite = {5370			.cipher = __VECS(cast6_lrw_tv_template)5371		}5372	}, {5373		.alg = "lrw(serpent)",5374		.generic_driver = "lrw(ecb(serpent-generic))",5375		.test = alg_test_skcipher,5376		.suite = {5377			.cipher = __VECS(serpent_lrw_tv_template)5378		}5379	}, {5380		.alg = "lrw(twofish)",5381		.generic_driver = "lrw(ecb(twofish-generic))",5382		.test = alg_test_skcipher,5383		.suite = {5384			.cipher = __VECS(tf_lrw_tv_template)5385		}5386	}, {5387		.alg = "lz4",5388		.test = alg_test_comp,5389		.fips_allowed = 1,5390		.suite = {5391			.comp = {5392				.comp = __VECS(lz4_comp_tv_template),5393				.decomp = __VECS(lz4_decomp_tv_template)5394			}5395		}5396	}, {5397		.alg = "lz4hc",5398		.test = alg_test_comp,5399		.fips_allowed = 1,5400		.suite = {5401			.comp = {5402				.comp = __VECS(lz4hc_comp_tv_template),5403				.decomp = __VECS(lz4hc_decomp_tv_template)5404			}5405		}5406	}, {5407		.alg = "lzo",5408		.test = alg_test_comp,5409		.fips_allowed = 1,5410		.suite = {5411			.comp = {5412				.comp = __VECS(lzo_comp_tv_template),5413				.decomp = __VECS(lzo_decomp_tv_template)5414			}5415		}5416	}, {5417		.alg = "lzo-rle",5418		.test = alg_test_comp,5419		.fips_allowed = 1,5420		.suite = {5421			.comp = {5422				.comp = __VECS(lzorle_comp_tv_template),5423				.decomp = __VECS(lzorle_decomp_tv_template)5424			}5425		}5426	}, {5427		.alg = "md4",5428		.test = alg_test_hash,5429		.suite = {5430			.hash = __VECS(md4_tv_template)5431		}5432	}, {5433		.alg = "md5",5434		.test = alg_test_hash,5435		.suite = {5436			.hash = __VECS(md5_tv_template)5437		}5438	}, {5439		.alg = "michael_mic",5440		.test = alg_test_hash,5441		.suite = {5442			.hash = __VECS(michael_mic_tv_template)5443		}5444	}, {5445		.alg = "nhpoly1305",5446		.test = alg_test_hash,5447		.suite = {5448			.hash = __VECS(nhpoly1305_tv_template)5449		}5450	}, {5451		.alg = "pcbc(fcrypt)",5452		.test = alg_test_skcipher,5453		.suite = {5454			.cipher = __VECS(fcrypt_pcbc_tv_template)5455		}5456	}, {5457		.alg = "pkcs1pad(rsa,sha224)",5458		.test = alg_test_null,5459		.fips_allowed = 1,5460	}, {5461		.alg = "pkcs1pad(rsa,sha256)",5462		.test = alg_test_akcipher,5463		.fips_allowed = 1,5464		.suite = {5465			.akcipher = __VECS(pkcs1pad_rsa_tv_template)5466		}5467	}, {5468		.alg = "pkcs1pad(rsa,sha3-256)",5469		.test = alg_test_null,5470		.fips_allowed = 1,5471	}, {5472		.alg = "pkcs1pad(rsa,sha3-384)",5473		.test = alg_test_null,5474		.fips_allowed = 1,5475	}, {5476		.alg = "pkcs1pad(rsa,sha3-512)",5477		.test = alg_test_null,5478		.fips_allowed = 1,5479	}, {5480		.alg = "pkcs1pad(rsa,sha384)",5481		.test = alg_test_null,5482		.fips_allowed = 1,5483	}, {5484		.alg = "pkcs1pad(rsa,sha512)",5485		.test = alg_test_null,5486		.fips_allowed = 1,5487	}, {5488		.alg = "poly1305",5489		.test = alg_test_hash,5490		.suite = {5491			.hash = __VECS(poly1305_tv_template)5492		}5493	}, {5494		.alg = "polyval",5495		.test = alg_test_hash,5496		.suite = {5497			.hash = __VECS(polyval_tv_template)5498		}5499	}, {5500		.alg = "rfc3686(ctr(aes))",5501		.test = alg_test_skcipher,5502		.fips_allowed = 1,5503		.suite = {5504			.cipher = __VECS(aes_ctr_rfc3686_tv_template)5505		}5506	}, {5507		.alg = "rfc3686(ctr(sm4))",5508		.test = alg_test_skcipher,5509		.suite = {5510			.cipher = __VECS(sm4_ctr_rfc3686_tv_template)5511		}5512	}, {5513		.alg = "rfc4106(gcm(aes))",5514		.generic_driver = "rfc4106(gcm_base(ctr(aes-generic),ghash-generic))",5515		.test = alg_test_aead,5516		.fips_allowed = 1,5517		.suite = {5518			.aead = {5519				____VECS(aes_gcm_rfc4106_tv_template),5520				.einval_allowed = 1,5521				.aad_iv = 1,5522			}5523		}5524	}, {5525		.alg = "rfc4309(ccm(aes))",5526		.generic_driver = "rfc4309(ccm_base(ctr(aes-generic),cbcmac(aes-generic)))",5527		.test = alg_test_aead,5528		.fips_allowed = 1,5529		.suite = {5530			.aead = {5531				____VECS(aes_ccm_rfc4309_tv_template),5532				.einval_allowed = 1,5533				.aad_iv = 1,5534			}5535		}5536	}, {5537		.alg = "rfc4543(gcm(aes))",5538		.generic_driver = "rfc4543(gcm_base(ctr(aes-generic),ghash-generic))",5539		.test = alg_test_aead,5540		.suite = {5541			.aead = {5542				____VECS(aes_gcm_rfc4543_tv_template),5543				.einval_allowed = 1,5544				.aad_iv = 1,5545			}5546		}5547	}, {5548		.alg = "rfc7539(chacha20,poly1305)",5549		.test = alg_test_aead,5550		.suite = {5551			.aead = __VECS(rfc7539_tv_template)5552		}5553	}, {5554		.alg = "rfc7539esp(chacha20,poly1305)",5555		.test = alg_test_aead,5556		.suite = {5557			.aead = {5558				____VECS(rfc7539esp_tv_template),5559				.einval_allowed = 1,5560				.aad_iv = 1,5561			}5562		}5563	}, {5564		.alg = "rmd160",5565		.test = alg_test_hash,5566		.suite = {5567			.hash = __VECS(rmd160_tv_template)5568		}5569	}, {5570		.alg = "rsa",5571		.test = alg_test_akcipher,5572		.fips_allowed = 1,5573		.suite = {5574			.akcipher = __VECS(rsa_tv_template)5575		}5576	}, {5577		.alg = "sha1",5578		.test = alg_test_hash,5579		.fips_allowed = 1,5580		.suite = {5581			.hash = __VECS(sha1_tv_template)5582		}5583	}, {5584		.alg = "sha224",5585		.test = alg_test_hash,5586		.fips_allowed = 1,5587		.suite = {5588			.hash = __VECS(sha224_tv_template)5589		}5590	}, {5591		.alg = "sha256",5592		.test = alg_test_hash,5593		.fips_allowed = 1,5594		.suite = {5595			.hash = __VECS(sha256_tv_template)5596		}5597	}, {5598		.alg = "sha3-224",5599		.test = alg_test_hash,5600		.fips_allowed = 1,5601		.suite = {5602			.hash = __VECS(sha3_224_tv_template)5603		}5604	}, {5605		.alg = "sha3-256",5606		.test = alg_test_hash,5607		.fips_allowed = 1,5608		.suite = {5609			.hash = __VECS(sha3_256_tv_template)5610		}5611	}, {5612		.alg = "sha3-384",5613		.test = alg_test_hash,5614		.fips_allowed = 1,5615		.suite = {5616			.hash = __VECS(sha3_384_tv_template)5617		}5618	}, {5619		.alg = "sha3-512",5620		.test = alg_test_hash,5621		.fips_allowed = 1,5622		.suite = {5623			.hash = __VECS(sha3_512_tv_template)5624		}5625	}, {5626		.alg = "sha384",5627		.test = alg_test_hash,5628		.fips_allowed = 1,5629		.suite = {5630			.hash = __VECS(sha384_tv_template)5631		}5632	}, {5633		.alg = "sha512",5634		.test = alg_test_hash,5635		.fips_allowed = 1,5636		.suite = {5637			.hash = __VECS(sha512_tv_template)5638		}5639	}, {5640		.alg = "sm3",5641		.test = alg_test_hash,5642		.suite = {5643			.hash = __VECS(sm3_tv_template)5644		}5645	}, {5646		.alg = "streebog256",5647		.test = alg_test_hash,5648		.suite = {5649			.hash = __VECS(streebog256_tv_template)5650		}5651	}, {5652		.alg = "streebog512",5653		.test = alg_test_hash,5654		.suite = {5655			.hash = __VECS(streebog512_tv_template)5656		}5657	}, {5658		.alg = "vmac64(aes)",5659		.test = alg_test_hash,5660		.suite = {5661			.hash = __VECS(vmac64_aes_tv_template)5662		}5663	}, {5664		.alg = "wp256",5665		.test = alg_test_hash,5666		.suite = {5667			.hash = __VECS(wp256_tv_template)5668		}5669	}, {5670		.alg = "wp384",5671		.test = alg_test_hash,5672		.suite = {5673			.hash = __VECS(wp384_tv_template)5674		}5675	}, {5676		.alg = "wp512",5677		.test = alg_test_hash,5678		.suite = {5679			.hash = __VECS(wp512_tv_template)5680		}5681	}, {5682		.alg = "xcbc(aes)",5683		.test = alg_test_hash,5684		.suite = {5685			.hash = __VECS(aes_xcbc128_tv_template)5686		}5687	}, {5688		.alg = "xcbc(sm4)",5689		.test = alg_test_hash,5690		.suite = {5691			.hash = __VECS(sm4_xcbc128_tv_template)5692		}5693	}, {5694		.alg = "xchacha12",5695		.test = alg_test_skcipher,5696		.suite = {5697			.cipher = __VECS(xchacha12_tv_template)5698		},5699	}, {5700		.alg = "xchacha20",5701		.test = alg_test_skcipher,5702		.suite = {5703			.cipher = __VECS(xchacha20_tv_template)5704		},5705	}, {5706		.alg = "xctr(aes)",5707		.test = alg_test_skcipher,5708		.suite = {5709			.cipher = __VECS(aes_xctr_tv_template)5710		}5711	}, {5712		.alg = "xts(aes)",5713		.generic_driver = "xts(ecb(aes-generic))",5714		.test = alg_test_skcipher,5715		.fips_allowed = 1,5716		.suite = {5717			.cipher = __VECS(aes_xts_tv_template)5718		}5719	}, {5720		.alg = "xts(camellia)",5721		.generic_driver = "xts(ecb(camellia-generic))",5722		.test = alg_test_skcipher,5723		.suite = {5724			.cipher = __VECS(camellia_xts_tv_template)5725		}5726	}, {5727		.alg = "xts(cast6)",5728		.generic_driver = "xts(ecb(cast6-generic))",5729		.test = alg_test_skcipher,5730		.suite = {5731			.cipher = __VECS(cast6_xts_tv_template)5732		}5733	}, {5734		/* Same as xts(aes) except the key is stored in5735		 * hardware secure memory which we reference by index5736		 */5737		.alg = "xts(paes)",5738		.test = alg_test_null,5739		.fips_allowed = 1,5740	}, {5741		.alg = "xts(serpent)",5742		.generic_driver = "xts(ecb(serpent-generic))",5743		.test = alg_test_skcipher,5744		.suite = {5745			.cipher = __VECS(serpent_xts_tv_template)5746		}5747	}, {5748		.alg = "xts(sm4)",5749		.generic_driver = "xts(ecb(sm4-generic))",5750		.test = alg_test_skcipher,5751		.suite = {5752			.cipher = __VECS(sm4_xts_tv_template)5753		}5754	}, {5755		.alg = "xts(twofish)",5756		.generic_driver = "xts(ecb(twofish-generic))",5757		.test = alg_test_skcipher,5758		.suite = {5759			.cipher = __VECS(tf_xts_tv_template)5760		}5761	}, {5762#if IS_ENABLED(CONFIG_CRYPTO_PAES_S390)5763		.alg = "xts-paes-s390",5764		.fips_allowed = 1,5765		.test = alg_test_skcipher,5766		.suite = {5767			.cipher = __VECS(aes_xts_tv_template)5768		}5769	}, {5770#endif5771		.alg = "xxhash64",5772		.test = alg_test_hash,5773		.fips_allowed = 1,5774		.suite = {5775			.hash = __VECS(xxhash64_tv_template)5776		}5777	}, {5778		.alg = "zstd",5779		.test = alg_test_comp,5780		.fips_allowed = 1,5781		.suite = {5782			.comp = {5783				.comp = __VECS(zstd_comp_tv_template),5784				.decomp = __VECS(zstd_decomp_tv_template)5785			}5786		}5787	}5788};5789 5790static void alg_check_test_descs_order(void)5791{5792	int i;5793 5794	for (i = 1; i < ARRAY_SIZE(alg_test_descs); i++) {5795		int diff = strcmp(alg_test_descs[i - 1].alg,5796				  alg_test_descs[i].alg);5797 5798		if (WARN_ON(diff > 0)) {5799			pr_warn("testmgr: alg_test_descs entries in wrong order: '%s' before '%s'\n",5800				alg_test_descs[i - 1].alg,5801				alg_test_descs[i].alg);5802		}5803 5804		if (WARN_ON(diff == 0)) {5805			pr_warn("testmgr: duplicate alg_test_descs entry: '%s'\n",5806				alg_test_descs[i].alg);5807		}5808	}5809}5810 5811static void alg_check_testvec_configs(void)5812{5813	int i;5814 5815	for (i = 0; i < ARRAY_SIZE(default_cipher_testvec_configs); i++)5816		WARN_ON(!valid_testvec_config(5817				&default_cipher_testvec_configs[i]));5818 5819	for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++)5820		WARN_ON(!valid_testvec_config(5821				&default_hash_testvec_configs[i]));5822}5823 5824static void testmgr_onetime_init(void)5825{5826	alg_check_test_descs_order();5827	alg_check_testvec_configs();5828 5829#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS5830	pr_warn("alg: extra crypto tests enabled.  This is intended for developer use only.\n");5831#endif5832}5833 5834static int alg_find_test(const char *alg)5835{5836	int start = 0;5837	int end = ARRAY_SIZE(alg_test_descs);5838 5839	while (start < end) {5840		int i = (start + end) / 2;5841		int diff = strcmp(alg_test_descs[i].alg, alg);5842 5843		if (diff > 0) {5844			end = i;5845			continue;5846		}5847 5848		if (diff < 0) {5849			start = i + 1;5850			continue;5851		}5852 5853		return i;5854	}5855 5856	return -1;5857}5858 5859static int alg_fips_disabled(const char *driver, const char *alg)5860{5861	pr_info("alg: %s (%s) is disabled due to FIPS\n", alg, driver);5862 5863	return -ECANCELED;5864}5865 5866int alg_test(const char *driver, const char *alg, u32 type, u32 mask)5867{5868	int i;5869	int j;5870	int rc;5871 5872	if (!fips_enabled && notests) {5873		printk_once(KERN_INFO "alg: self-tests disabled\n");5874		return 0;5875	}5876 5877	DO_ONCE(testmgr_onetime_init);5878 5879	if ((type & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_CIPHER) {5880		char nalg[CRYPTO_MAX_ALG_NAME];5881 5882		if (snprintf(nalg, sizeof(nalg), "ecb(%s)", alg) >=5883		    sizeof(nalg))5884			return -ENAMETOOLONG;5885 5886		i = alg_find_test(nalg);5887		if (i < 0)5888			goto notest;5889 5890		if (fips_enabled && !alg_test_descs[i].fips_allowed)5891			goto non_fips_alg;5892 5893		rc = alg_test_cipher(alg_test_descs + i, driver, type, mask);5894		goto test_done;5895	}5896 5897	i = alg_find_test(alg);5898	j = alg_find_test(driver);5899	if (i < 0 && j < 0)5900		goto notest;5901 5902	if (fips_enabled) {5903		if (j >= 0 && !alg_test_descs[j].fips_allowed)5904			return -EINVAL;5905 5906		if (i >= 0 && !alg_test_descs[i].fips_allowed)5907			goto non_fips_alg;5908	}5909 5910	rc = 0;5911	if (i >= 0)5912		rc |= alg_test_descs[i].test(alg_test_descs + i, driver,5913					     type, mask);5914	if (j >= 0 && j != i)5915		rc |= alg_test_descs[j].test(alg_test_descs + j, driver,5916					     type, mask);5917 5918test_done:5919	if (rc) {5920		if (fips_enabled || panic_on_fail) {5921			fips_fail_notify();5922			panic("alg: self-tests for %s (%s) failed in %s mode!\n",5923			      driver, alg,5924			      fips_enabled ? "fips" : "panic_on_fail");5925		}5926		pr_warn("alg: self-tests for %s using %s failed (rc=%d)",5927			alg, driver, rc);5928		WARN(rc != -ENOENT,5929		     "alg: self-tests for %s using %s failed (rc=%d)",5930		     alg, driver, rc);5931	} else {5932		if (fips_enabled)5933			pr_info("alg: self-tests for %s (%s) passed\n",5934				driver, alg);5935	}5936 5937	return rc;5938 5939notest:5940	if ((type & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_LSKCIPHER) {5941		char nalg[CRYPTO_MAX_ALG_NAME];5942 5943		if (snprintf(nalg, sizeof(nalg), "ecb(%s)", alg) >=5944		    sizeof(nalg))5945			goto notest2;5946 5947		i = alg_find_test(nalg);5948		if (i < 0)5949			goto notest2;5950 5951		if (fips_enabled && !alg_test_descs[i].fips_allowed)5952			goto non_fips_alg;5953 5954		rc = alg_test_skcipher(alg_test_descs + i, driver, type, mask);5955		goto test_done;5956	}5957 5958notest2:5959	printk(KERN_INFO "alg: No test for %s (%s)\n", alg, driver);5960 5961	if (type & CRYPTO_ALG_FIPS_INTERNAL)5962		return alg_fips_disabled(driver, alg);5963 5964	return 0;5965non_fips_alg:5966	return alg_fips_disabled(driver, alg);5967}5968 5969#endif /* CONFIG_CRYPTO_MANAGER_DISABLE_TESTS */5970 5971EXPORT_SYMBOL_GPL(alg_test);5972