Merge tag 'libcrypto-tests-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux

Pull crypto library test updates from Eric Biggers:

 - Add comprehensive KUnit test suites for the new AES-GCM and AES-CCM
   library APIs

 - Add FIPS self-tests for all the AES encryption modes. This is needed
   for parity with the traditional crypto API

 - Fix a couple more issues in the IRQ test helper

* tag 'libcrypto-tests-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux:
  lib/crypto: aes-cmac: Use __cleanup() instead of memzero_explicit()
  kunit: irq: Unregister on-stack timer and work from debugobjects
  kunit: irq: Continue increasing hrtimer interval for longer
  lib/crypto: tests: Add KUnit test suite for AES-GCM
  lib/crypto: tests: Add KUnit test suite for AES-CCM
  lib/crypto: tests: Add aead-test-template.h
  lib/crypto: tests: Use per-test-case buffers in hash tests
  lib/crypto: tests: Create test-utils.h
  lib/crypto: aes: Add FIPS self-tests for GCM and CCM
  lib/crypto: aes: Add FIPS self-tests for unauthenticated modes
  lib/crypto: fips: Split fips.h into fips-aes.h and fips-sha.h

32 files changed, 2533 insertions, 367 deletions

diff --git a/lib/crypto/.kunitconfig b/lib/crypto/.kunitconfig
index 3efc854a2c08..60e0a77f9889 100644
--- a/lib/crypto/.kunitconfig
+++ b/lib/crypto/.kunitconfig
@@ -3,6 +3,8 @@ CONFIG_KUNIT=y
 CONFIG_CRYPTO_LIB_ENABLE_ALL_FOR_KUNIT=y
 
 CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST=y
+CONFIG_CRYPTO_LIB_AES_CCM_KUNIT_TEST=y
+CONFIG_CRYPTO_LIB_AES_GCM_KUNIT_TEST=y
 CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST=y
 CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST=y
 CONFIG_CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST=y
diff --git a/lib/crypto/aes.c b/lib/crypto/aes.c
index 4222a4cec2f2..f1549839b3de 100644
--- a/lib/crypto/aes.c
+++ b/lib/crypto/aes.c
@@ -19,7 +19,7 @@
 #include <linux/export.h>
 #include <linux/module.h>
 #include <linux/unaligned.h>
-#include "fips.h"
+#include "fips-aes.h"
 
 static const u8 ____cacheline_aligned aes_sbox[] = {
 	0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5,
@@ -522,6 +522,26 @@ void aes_decrypt(const struct aes_key *key, u8 out[AES_BLOCK_SIZE],
 }
 EXPORT_SYMBOL(aes_decrypt);
 
+/* FIPS cryptographic algorithm self-test for "bare" AES */
+static void __init aes_fips_test(void)
+{
+	struct aes_key key;
+	u8 data[AES_BLOCK_SIZE];
+
+	if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0)
+		panic("aes: FIPS self-test failed (preparekey)\n");
+
+	aes_encrypt(&key, data, fips_test_data);
+	if (memcmp(fips_test_aes_ecb_ctext, data, sizeof(data)) != 0)
+		panic("aes: FIPS self-test failed (wrong ciphertext)\n");
+
+	aes_decrypt(&key, data, data);
+	if (memcmp(fips_test_data, data, sizeof(data)) != 0)
+		panic("aes: FIPS self-test failed (wrong plaintext)\n");
+
+	memzero_explicit(&key, sizeof(key));
+}
+
 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS)
 
 #ifndef aes_cbcmac_blocks_arch
@@ -717,17 +737,10 @@ void aes_cbcmac_final(struct aes_cbcmac_ctx *ctx, u8 out[AES_BLOCK_SIZE])
 }
 EXPORT_SYMBOL_NS_GPL(aes_cbcmac_final, "CRYPTO_INTERNAL");
 
-/*
- * FIPS cryptographic algorithm self-test for AES-CMAC.  As per the FIPS 140-3
- * Implementation Guidance, a cryptographic algorithm self-test for at least one
- * of AES-GCM, AES-CCM, AES-CMAC, or AES-GMAC is required if any of those modes
- * is implemented.  This fulfills that requirement via AES-CMAC.
- *
- * This is just for FIPS.  The full tests are in the KUnit test suite.
- */
+/* FIPS cryptographic algorithm self-test for AES-CMAC */
 static void __init aes_cmac_fips_test(void)
 {
-	struct aes_cmac_key key;
+	struct aes_cmac_key key __cleanup(aes_cmac_zeroize_key);
 	u8 mac[AES_BLOCK_SIZE];
 
 	if (aes_cmac_preparekey(&key, fips_test_key, sizeof(fips_test_key)) !=
@@ -736,7 +749,6 @@ static void __init aes_cmac_fips_test(void)
 	aes_cmac(&key, fips_test_data, sizeof(fips_test_data), mac);
 	if (memcmp(fips_test_aes_cmac_value, mac, sizeof(mac)) != 0)
 		panic("aes: CMAC FIPS self-test failed (wrong MAC)\n");
-	memzero_explicit(&key, sizeof(key));
 }
 #else /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */
 static inline void aes_cmac_fips_test(void)
@@ -797,7 +809,31 @@ void aes_ecb_decrypt(u8 *dst, const u8 *src, size_t len,
 		aes_decrypt(key, &dst[i], &src[i]);
 }
 EXPORT_SYMBOL_GPL(aes_ecb_decrypt);
-#endif /* CONFIG_CRYPTO_LIB_AES_ECB */
+
+/* FIPS cryptographic algorithm self-test for AES-ECB */
+static void __init aes_ecb_fips_test(void)
+{
+	struct aes_key key;
+	u8 data[sizeof(fips_test_data)];
+
+	if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0)
+		panic("aes: ECB FIPS self-test failed (preparekey)\n");
+
+	aes_ecb_encrypt(data, fips_test_data, sizeof(data), &key);
+	if (memcmp(fips_test_aes_ecb_ctext, data, sizeof(data)) != 0)
+		panic("aes: ECB FIPS self-test failed (wrong ciphertext)\n");
+
+	aes_ecb_decrypt(data, data, sizeof(data), &key);
+	if (memcmp(fips_test_data, data, sizeof(data)) != 0)
+		panic("aes: ECB FIPS self-test failed (wrong plaintext)\n");
+
+	memzero_explicit(&key, sizeof(key));
+}
+#else /* CONFIG_CRYPTO_LIB_AES_ECB */
+static inline void aes_ecb_fips_test(void)
+{
+}
+#endif /* !CONFIG_CRYPTO_LIB_AES_ECB */
 
 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC)
 /*
@@ -983,7 +1019,66 @@ void aes_cbc_cts_decrypt(u8 *dst, const u8 *src, size_t len,
 	crypto_xor(pad, iv, AES_BLOCK_SIZE); /* P[n - 1] */
 }
 EXPORT_SYMBOL_GPL(aes_cbc_cts_decrypt);
-#endif /* CONFIG_CRYPTO_LIB_AES_CBC */
+
+/* FIPS cryptographic algorithm self-test for AES-CBC */
+static void __init aes_cbc_fips_test(void)
+{
+	struct aes_key key;
+	u8 iv[AES_BLOCK_SIZE];
+	u8 data[sizeof(fips_test_data)];
+
+	if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0)
+		panic("aes: CBC FIPS self-test failed (preparekey)\n");
+
+	memcpy(iv, fips_test_iv, sizeof(iv));
+	aes_cbc_encrypt(data, fips_test_data, sizeof(data), iv, &key);
+	if (memcmp(fips_test_aes_cbc_ctext, data, sizeof(data)) != 0)
+		panic("aes: CBC FIPS self-test failed (wrong ciphertext)\n");
+
+	memcpy(iv, fips_test_iv, sizeof(iv));
+	aes_cbc_decrypt(data, data, sizeof(data), iv, &key);
+	if (memcmp(fips_test_data, data, sizeof(data)) != 0)
+		panic("aes: CBC FIPS self-test failed (wrong plaintext)\n");
+
+	memzero_explicit(&key, sizeof(key));
+}
+
+/* FIPS cryptographic algorithm self-test for AES-CBC-CTS */
+static void __init aes_cbc_cts_fips_test(void)
+{
+	struct aes_key key;
+	u8 iv[AES_BLOCK_SIZE];
+	const size_t data_len = 2 * AES_BLOCK_SIZE;
+	u8 ptext[2 * AES_BLOCK_SIZE];
+	u8 data[2 * AES_BLOCK_SIZE];
+
+	/* ptext = fips_test_data || fips_test_data */
+	memcpy(ptext, fips_test_data, AES_BLOCK_SIZE);
+	memcpy(&ptext[AES_BLOCK_SIZE], ptext, AES_BLOCK_SIZE);
+
+	if (aes_preparekey(&key, fips_test_key, sizeof(fips_test_key)) != 0)
+		panic("aes: CBC-CTS FIPS self-test failed (preparekey)\n");
+
+	memcpy(iv, fips_test_iv, sizeof(iv));
+	aes_cbc_cts_encrypt(data, ptext, data_len, iv, &key);
+	if (memcmp(fips_test_aes_cbc_cts_ctext, data, data_len) != 0)
+		panic("aes: CBC-CTS FIPS self-test failed (wrong ciphertext)\n");
+
+	memcpy(iv, fips_test_iv, sizeof(iv));
+	aes_cbc_cts_decrypt(data, data, data_len, iv, &key);
+	if (memcmp(ptext, data, data_len) != 0)
+		panic("aes: CBC-CTS FIPS self-test failed (wrong plaintext)\n");
+
+	memzero_explicit(&key, sizeof(key));
+}
+#else /* CONFIG_CRYPTO_LIB_AES_CBC */
+static inline void aes_cbc_fips_test(void)
+{
+}
+static inline void aes_cbc_cts_fips_test(void)
+{
+}
+#endif /* !CONFIG_CRYPTO_LIB_AES_CBC */
 
 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR)
 /*
@@ -1078,7 +1173,34 @@ void aes_xctr(u8 *dst, const u8 *src, size_t len, u64 *ctr,
 	memzero_explicit(aes_input, sizeof(aes_input));
 }
 EXPORT_SYMBOL_GPL(aes_xctr);
-#endif /* CONFIG_CRYPTO_LIB_AES_CTR */
+
+/* FIPS cryptographic algorithm self-test for AES-CTR */
+static void __init aes_ctr_fips_test(void)
+{
+	struct aes_enckey key;
+	u8 ctr[AES_BLOCK_SIZE];
+	u8 data[sizeof(fips_test_data)];
+
+	if (aes_prepareenckey(&key, fips_test_key, sizeof(fips_test_key)) != 0)
+		panic("aes: CTR FIPS self-test failed (preparekey)\n");
+
+	memcpy(ctr, fips_test_iv, sizeof(ctr));
+	aes_ctr(data, fips_test_data, sizeof(data), ctr, &key);
+	if (memcmp(fips_test_aes_ctr_ctext, data, sizeof(data)) != 0)
+		panic("aes: CTR FIPS self-test failed (wrong ciphertext)\n");
+
+	memcpy(ctr, fips_test_iv, sizeof(ctr));
+	aes_ctr(data, data, sizeof(data), ctr, &key);
+	if (memcmp(fips_test_data, data, sizeof(data)) != 0)
+		panic("aes: CTR FIPS self-test failed (wrong plaintext)\n");
+
+	memzero_explicit(&key, sizeof(key));
+}
+#else /* CONFIG_CRYPTO_LIB_AES_CTR */
+static inline void aes_ctr_fips_test(void)
+{
+}
+#endif /* !CONFIG_CRYPTO_LIB_AES_CTR */
 
 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS)
 int aes_xts_preparekey(struct aes_xts_key *key, const u8 *in_key,
@@ -1307,7 +1429,36 @@ void aes_xts_decrypt(u8 *dst, const u8 *src, size_t len,
 	aes_xts_decrypt_nocts(dst, src, len, tweak, key, cont);
 }
 EXPORT_SYMBOL_GPL(aes_xts_decrypt);
-#endif /* CONFIG_CRYPTO_LIB_AES_XTS */
+
+/* FIPS cryptographic algorithm self-test for AES-XTS */
+static void __init aes_xts_fips_test(void)
+{
+	struct aes_xts_key *key __free(kfree_sensitive) = kmalloc_obj(*key);
+	u8 tweak[AES_BLOCK_SIZE];
+	u8 data[sizeof(fips_test_data)];
+
+	if (key == NULL)
+		panic("aes: XTS FIPS self-test failed (kmalloc)\n");
+
+	if (aes_xts_preparekey(key, fips_test_xts_key,
+			       sizeof(fips_test_xts_key), 0) != 0)
+		panic("aes: XTS FIPS self-test failed (preparekey)\n");
+
+	memcpy(tweak, fips_test_iv, sizeof(tweak));
+	aes_xts_encrypt(data, fips_test_data, sizeof(data), tweak, key, false);
+	if (memcmp(fips_test_aes_xts_ctext, data, sizeof(data)) != 0)
+		panic("aes: XTS FIPS self-test failed (wrong ciphertext)\n");
+
+	memcpy(tweak, fips_test_iv, sizeof(tweak));
+	aes_xts_decrypt(data, data, sizeof(data), tweak, key, false);
+	if (memcmp(fips_test_data, data, sizeof(data)) != 0)
+		panic("aes: XTS FIPS self-test failed (wrong plaintext)\n");
+}
+#else /* CONFIG_CRYPTO_LIB_AES_XTS */
+static inline void aes_xts_fips_test(void)
+{
+}
+#endif /* !CONFIG_CRYPTO_LIB_AES_XTS */
 
 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_GCM)
 /*
@@ -1586,7 +1737,37 @@ int aes_gcm_decrypt(u8 *dst, const u8 *src, size_t data_len, const u8 *authtag,
 }
 EXPORT_SYMBOL_GPL(aes_gcm_decrypt);
 
-#endif /* CONFIG_CRYPTO_LIB_AES_GCM */
+/* FIPS cryptographic algorithm self-test for AES-GCM */
+static void __init aes_gcm_fips_test(void)
+{
+	const size_t data_len = sizeof(fips_test_data);
+	u8 buf[sizeof(fips_test_data) + AES_BLOCK_SIZE];
+	struct aes_gcm_key key;
+	int err;
+
+	if (aes_gcm_preparekey(&key, fips_test_key, sizeof(fips_test_key),
+			       AES_BLOCK_SIZE) != 0)
+		panic("aes: GCM FIPS self-test failed (preparekey)\n");
+
+	aes_gcm_encrypt(buf, fips_test_data, data_len, &buf[data_len],
+			fips_test_ad, sizeof(fips_test_ad), fips_test_iv, &key);
+	if (memcmp(fips_test_aes_gcm_ctext_and_tag, buf, sizeof(buf)) != 0)
+		panic("aes: GCM FIPS self-test failed (wrong ciphertext and/or tag)\n");
+
+	err = aes_gcm_decrypt(buf, buf, data_len, &buf[data_len], fips_test_ad,
+			      sizeof(fips_test_ad), fips_test_iv, &key);
+	if (err != 0)
+		panic("aes: GCM FIPS self-test failed (decryption failed)\n");
+	if (memcmp(fips_test_data, buf, data_len) != 0)
+		panic("aes: GCM FIPS self-test failed (wrong plaintext)\n");
+
+	memzero_explicit(&key, sizeof(key));
+}
+#else /* CONFIG_CRYPTO_LIB_AES_GCM */
+static inline void aes_gcm_fips_test(void)
+{
+}
+#endif /* !CONFIG_CRYPTO_LIB_AES_GCM */
 
 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CCM)
 int aes_ccm_preparekey(struct aes_ccm_key *key, const u8 *in_key,
@@ -1898,15 +2079,60 @@ int aes_ccm_decrypt(u8 *dst, const u8 *src, size_t data_len, const u8 *authtag,
 	return err;
 }
 EXPORT_SYMBOL_GPL(aes_ccm_decrypt);
-#endif /* CONFIG_CRYPTO_LIB_AES_CCM */
+
+/* FIPS cryptographic algorithm self-test for AES-CCM */
+static void __init aes_ccm_fips_test(void)
+{
+	const size_t data_len = sizeof(fips_test_data);
+	const size_t nonce_len = 13;
+	u8 buf[sizeof(fips_test_data) + AES_BLOCK_SIZE];
+	struct aes_ccm_key key;
+	int err;
+
+	if (aes_ccm_preparekey(&key, fips_test_key, sizeof(fips_test_key),
+			       AES_BLOCK_SIZE) != 0)
+		panic("aes: CCM FIPS self-test failed (preparekey)\n");
+
+	err = aes_ccm_encrypt(buf, fips_test_data, data_len, &buf[data_len],
+			      fips_test_ad, sizeof(fips_test_ad), fips_test_iv,
+			      nonce_len, &key);
+	if (err != 0)
+		panic("aes: CCM FIPS self-test failed (encryption failed)\n");
+	if (memcmp(fips_test_aes_ccm_ctext_and_tag, buf, sizeof(buf)) != 0)
+		panic("aes: CCM FIPS self-test failed (wrong ciphertext and/or tag)\n");
+
+	err = aes_ccm_decrypt(buf, buf, data_len, &buf[data_len], fips_test_ad,
+			      sizeof(fips_test_ad), fips_test_iv, nonce_len,
+			      &key);
+	if (err != 0)
+		panic("aes: CCM FIPS self-test failed (decryption failed)\n");
+	if (memcmp(fips_test_data, buf, data_len) != 0)
+		panic("aes: CCM FIPS self-test failed (wrong plaintext)\n");
+
+	memzero_explicit(&key, sizeof(key));
+}
+#else /* CONFIG_CRYPTO_LIB_AES_CCM */
+static inline void aes_ccm_fips_test(void)
+{
+}
+#endif /* !CONFIG_CRYPTO_LIB_AES_CCM */
 
 static int __init aes_mod_init(void)
 {
 #ifdef aes_mod_init_arch
 	aes_mod_init_arch();
 #endif
-	if (fips_enabled)
+	if (fips_enabled) {
+		aes_fips_test();
 		aes_cmac_fips_test();
+		aes_ecb_fips_test();
+		aes_cbc_fips_test();
+		aes_cbc_cts_fips_test();
+		aes_ctr_fips_test();
+		aes_xts_fips_test();
+		aes_gcm_fips_test();
+		aes_ccm_fips_test();
+	}
 	return 0;
 }
 subsys_initcall(aes_mod_init);
diff --git a/lib/crypto/fips-aes.h b/lib/crypto/fips-aes.h
new file mode 100644
index 000000000000..2a1746606533
--- /dev/null
+++ b/lib/crypto/fips-aes.h
@@ -0,0 +1,78 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* This file was generated by: gen-fips-testvecs.py */
+/* clang-format off */
+
+#include <linux/fips.h>
+
+static const u8 fips_test_data[] __initconst __maybe_unused = {
+	0x66, 0x69, 0x70, 0x73, 0x20, 0x74, 0x65, 0x73,
+	0x74, 0x20, 0x64, 0x61, 0x74, 0x61, 0x00, 0x00,
+};
+
+static const u8 fips_test_ad[] __initconst __maybe_unused = {
+	0x66, 0x69, 0x70, 0x73, 0x20, 0x74, 0x65, 0x73,
+	0x74, 0x20, 0x61, 0x64, 0x00, 0x00, 0x00, 0x00,
+};
+
+static const u8 fips_test_iv[] __initconst __maybe_unused = {
+	0x66, 0x69, 0x70, 0x73, 0x20, 0x74, 0x65, 0x73,
+	0x74, 0x20, 0x69, 0x76, 0x00, 0x00, 0x00, 0x00,
+};
+
+static const u8 fips_test_key[] __initconst __maybe_unused = {
+	0x66, 0x69, 0x70, 0x73, 0x20, 0x74, 0x65, 0x73,
+	0x74, 0x20, 0x6b, 0x65, 0x79, 0x00, 0x00, 0x00,
+};
+
+static const u8 fips_test_xts_key[] __initconst __maybe_unused = {
+	0x6b, 0x65, 0x79, 0x31, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+	0x6b, 0x65, 0x79, 0x32, 0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+};
+
+static const u8 fips_test_aes_cmac_value[] __initconst __maybe_unused = {
+	0xc5, 0x88, 0x28, 0x55, 0xd7, 0x2c, 0x00, 0xb6,
+	0x6a, 0xa7, 0xfc, 0x82, 0x90, 0x81, 0xcf, 0x18,
+};
+
+static const u8 fips_test_aes_ecb_ctext[] __initconst __maybe_unused = {
+	0x47, 0x76, 0x48, 0xaf, 0x1b, 0xd8, 0x4c, 0xe6,
+	0xb5, 0xa7, 0x20, 0x8d, 0x64, 0x88, 0xbc, 0x3f,
+};
+
+static const u8 fips_test_aes_cbc_ctext[] __initconst __maybe_unused = {
+	0xc8, 0x7d, 0x7c, 0x25, 0xba, 0x15, 0xf7, 0xe1,
+	0x08, 0xa0, 0xd0, 0x7a, 0x20, 0x37, 0xaf, 0x5e,
+};
+
+static const u8 fips_test_aes_cbc_cts_ctext[] __initconst __maybe_unused = {
+	0x36, 0x8e, 0x37, 0xb4, 0x78, 0xe2, 0x88, 0x59,
+	0xd5, 0xe8, 0x17, 0x65, 0x5c, 0xa1, 0x25, 0xe6,
+	0xc8, 0x7d, 0x7c, 0x25, 0xba, 0x15, 0xf7, 0xe1,
+	0x08, 0xa0, 0xd0, 0x7a, 0x20, 0x37, 0xaf, 0x5e,
+};
+
+static const u8 fips_test_aes_ctr_ctext[] __initconst __maybe_unused = {
+	0x95, 0xf4, 0xf4, 0x7a, 0xc8, 0xa2, 0x53, 0x73,
+	0x53, 0x8f, 0x95, 0xfc, 0x18, 0xfe, 0x58, 0x2f,
+};
+
+static const u8 fips_test_aes_xts_ctext[] __initconst __maybe_unused = {
+	0xd4, 0x51, 0x7f, 0x01, 0x14, 0x91, 0x16, 0x29,
+	0x26, 0xbe, 0xec, 0x9b, 0x90, 0xed, 0x59, 0x30,
+};
+
+static const u8 fips_test_aes_gcm_ctext_and_tag[] __initconst __maybe_unused = {
+	0x12, 0x0c, 0x5d, 0x03, 0x32, 0x93, 0x13, 0x44,
+	0x06, 0x35, 0x26, 0x9d, 0xe0, 0xea, 0xbc, 0xe2,
+	0x30, 0xa9, 0xa4, 0x15, 0xc5, 0x3d, 0xb3, 0xf9,
+	0x30, 0x82, 0xdf, 0x9c, 0xd8, 0xc4, 0x3f, 0x2f,
+};
+
+static const u8 fips_test_aes_ccm_ctext_and_tag[] __initconst __maybe_unused = {
+	0x11, 0x8e, 0x01, 0xcb, 0xb5, 0x22, 0x6d, 0xb4,
+	0x66, 0x98, 0x97, 0x1d, 0x35, 0x53, 0x78, 0xdd,
+	0xd1, 0xc5, 0xff, 0xb6, 0x90, 0xcf, 0xb1, 0xf2,
+	0x87, 0x99, 0xd6, 0x1e, 0xd5, 0xd1, 0xed, 0x63,
+};
diff --git a/lib/crypto/fips.h b/lib/crypto/fips-sha.h
index 9fc49747db64..68af7e14e09c 100644
--- a/lib/crypto/fips.h
+++ b/lib/crypto/fips-sha.h
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-or-later */
 /* This file was generated by: gen-fips-testvecs.py */
+/* clang-format off */
 
 #include <linux/fips.h>
 
@@ -43,8 +44,3 @@ static const u8 fips_test_sha3_256_value[] __initconst __maybe_unused = {
 	0xba, 0x9b, 0xb6, 0xaa, 0x32, 0xa7, 0x97, 0x00,
 	0x98, 0xdb, 0xff, 0xe7, 0xc6, 0xde, 0xb5, 0x82,
 };
-
-static const u8 fips_test_aes_cmac_value[] __initconst __maybe_unused = {
-	0xc5, 0x88, 0x28, 0x55, 0xd7, 0x2c, 0x00, 0xb6,
-	0x6a, 0xa7, 0xfc, 0x82, 0x90, 0x81, 0xcf, 0x18,
-};
diff --git a/lib/crypto/sha1.c b/lib/crypto/sha1.c
index daf18c862fdf..b687b89d97cb 100644
--- a/lib/crypto/sha1.c
+++ b/lib/crypto/sha1.c
@@ -12,7 +12,7 @@
 #include <linux/string.h>
 #include <linux/unaligned.h>
 #include <linux/wordpart.h>
-#include "fips.h"
+#include "fips-sha.h"
 
 static const struct sha1_block_state sha1_iv = {
 	.h = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 },
diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c
index 5d6b77e7e141..e8c346f563c5 100644
--- a/lib/crypto/sha256.c
+++ b/lib/crypto/sha256.c
@@ -17,7 +17,7 @@
 #include <linux/string.h>
 #include <linux/unaligned.h>
 #include <linux/wordpart.h>
-#include "fips.h"
+#include "fips-sha.h"
 
 static const struct sha256_block_state sha224_iv = {
 	.h = {
diff --git a/lib/crypto/sha3.c b/lib/crypto/sha3.c
index 32b7074de792..286a2373c156 100644
--- a/lib/crypto/sha3.c
+++ b/lib/crypto/sha3.c
@@ -17,7 +17,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/unaligned.h>
-#include "fips.h"
+#include "fips-sha.h"
 
 /*
  * On some 32-bit architectures, such as h8300, GCC ends up using over 1 KB of
diff --git a/lib/crypto/sha512.c b/lib/crypto/sha512.c
index 605eab51aabd..0dd6fb4ca15b 100644
--- a/lib/crypto/sha512.c
+++ b/lib/crypto/sha512.c
@@ -17,7 +17,7 @@
 #include <linux/string.h>
 #include <linux/unaligned.h>
 #include <linux/wordpart.h>
-#include "fips.h"
+#include "fips-sha.h"
 
 static const struct sha512_block_state sha384_iv = {
 	.h = {
diff --git a/lib/crypto/tests/Kconfig b/lib/crypto/tests/Kconfig
index bc084dde424f..e121114624da 100644
--- a/lib/crypto/tests/Kconfig
+++ b/lib/crypto/tests/Kconfig
@@ -9,6 +9,24 @@ config CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST
 	  KUnit tests for the AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC message
 	  authentication codes.
 
+config CRYPTO_LIB_AES_CCM_KUNIT_TEST
+	tristate "KUnit tests for AES-CCM" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_AES_CCM
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	# This test uses BLAKE2s, but that's always built-in.
+	help
+	  KUnit tests for the AES-CCM authenticated encryption algorithm.
+
+config CRYPTO_LIB_AES_GCM_KUNIT_TEST
+	tristate "KUnit tests for AES-GCM" if !KUNIT_ALL_TESTS
+	depends on KUNIT && CRYPTO_LIB_AES_GCM
+	default KUNIT_ALL_TESTS
+	select CRYPTO_LIB_BENCHMARK_VISIBLE
+	# This test uses BLAKE2s, but that's always built-in.
+	help
+	  KUnit tests for the AES-GCM authenticated encryption algorithm.
+
 config CRYPTO_LIB_BLAKE2B_KUNIT_TEST
 	tristate "KUnit tests for BLAKE2b" if !KUNIT_ALL_TESTS
 	depends on KUNIT && CRYPTO_LIB_BLAKE2B
diff --git a/lib/crypto/tests/Makefile b/lib/crypto/tests/Makefile
index a739413500b6..c97c1d78784a 100644
--- a/lib/crypto/tests/Makefile
+++ b/lib/crypto/tests/Makefile
@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 obj-$(CONFIG_CRYPTO_LIB_AES_CBC_MACS_KUNIT_TEST) += aes_cbc_macs_kunit.o
+obj-$(CONFIG_CRYPTO_LIB_AES_CCM_KUNIT_TEST) += aes_ccm_kunit.o
+obj-$(CONFIG_CRYPTO_LIB_AES_GCM_KUNIT_TEST) += aes_gcm_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_BLAKE2B_KUNIT_TEST) += blake2b_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_BLAKE2S_KUNIT_TEST) += blake2s_kunit.o
 obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305_KUNIT_TEST) += chacha20poly1305_kunit.o
diff --git a/lib/crypto/tests/aead-test-template.h b/lib/crypto/tests/aead-test-template.h
new file mode 100644
index 000000000000..0c0138d2fa8a
--- /dev/null
+++ b/lib/crypto/tests/aead-test-template.h
@@ -0,0 +1,1039 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Shared KUnit test cases for AEAD algorithms, including a benchmark
+ *
+ * Copyright 2026 Google LLC
+ */
+
+/*
+ * This file implements KUnit test cases shared by the different KUnit test
+ * suites for Authenticated Encryption with Associated Data (AEAD) algorithms.
+ *
+ * Test suites including this file must #define the following:
+ *
+ * Data structs:
+ * - AEAD_KEY: name of key struct
+ * - AEAD_CTX: name of context for incremental computation
+ *
+ * Constants:
+ * - AEAD_VALID_KEY_LENS: array of all valid key lengths in bytes
+ * - AEAD_VALID_NONCE_LENS: array of all valid nonce lengths in bytes
+ * - AEAD_VALID_TAG_LENS: array of all valid authtag lengths in bytes
+ * - AEAD_MAX_KEY_LEN: max key length in bytes (assumed to fit on stack)
+ * - AEAD_MAX_NONCE_LEN: max nonce length in bytes (assumed to fit on stack)
+ * - AEAD_MAX_TAG_LEN: max authtag length in bytes (assumed to fit on stack)
+ * - AEAD_MONTE_CARLO_CHECKSUM: checksum of a deterministically generated series
+ *   of (ciphertext, authtag) pairs (see test_aead_monte_carlo())
+ *
+ * Functions:
+ * - AEAD_PREPAREKEY: key preparation
+ * - AEAD_ENCRYPT and AEAD_DECRYPT: one-shot encryption and decryption
+ * - AEAD_INIT, AEAD_AUTH_UPDATE, AEAD_ENCRYPT_UPDATE, AEAD_ENCRYPT_FINAL,
+ *   AEAD_DECRYPT_UPDATE, AEAD_DECRYPT_FINAL: functions for incremental
+ *   encryption and decryption
+ *
+ * Function prototypes and their behavior must match the AES-CCM API.
+ */
+
+#include <crypto/blake2s.h>
+#include <kunit/run-in-irq-context.h>
+#include <kunit/test.h>
+#include <linux/ktime.h>
+#include <linux/preempt.h>
+#include "test-utils.h"
+
+/*
+ * Allocate a KUnit-managed struct AEAD_KEY and prepare it with a random key,
+ * using a random key length and random authentication tag length.
+ */
+static struct AEAD_KEY *aead_alloc_random_key(struct kunit *test,
+					      size_t *tag_len_ret)
+{
+	size_t key_len =
+		AEAD_VALID_KEY_LENS[rand32() % ARRAY_SIZE(AEAD_VALID_KEY_LENS)];
+	size_t tag_len =
+		AEAD_VALID_TAG_LENS[rand32() % ARRAY_SIZE(AEAD_VALID_TAG_LENS)];
+	u8 raw_key[AEAD_MAX_KEY_LEN];
+	struct AEAD_KEY *key = alloc_buf(test, sizeof(*key));
+	int err;
+
+	rand_bytes(raw_key, key_len);
+	err = AEAD_PREPAREKEY(key, raw_key, key_len, tag_len);
+	KUNIT_ASSERT_EQ(test, 0, err);
+	*tag_len_ret = tag_len;
+	return key;
+}
+
+/*
+ * Allocate a KUnit-managed slab buffer of length @len bytes and initialize it
+ * with random data.
+ */
+static u8 *aead_alloc_random_data(struct kunit *test, size_t len)
+{
+	u8 *buf = alloc_buf(test, len);
+
+	rand_bytes(buf, len);
+	return buf;
+}
+
+/*
+ * Allocate a KUnit-managed guarded buffer of length @len bytes and initialize
+ * it with random data.
+ */
+static u8 *aead_alloc_random_data_guarded(struct kunit *test, size_t len)
+{
+	u8 *buf = alloc_guarded_buf(test, len);
+
+	rand_bytes(buf, len);
+	return buf;
+}
+
+/* Process the given associated data using a random incremental strategy. */
+static size_t aead_auth_incrementally(struct AEAD_CTX *ctx, const u8 *ad,
+				      size_t ad_len)
+{
+	size_t num_parts = 0;
+	size_t pos = 0;
+
+	while (rand_bool()) {
+		size_t part_len = rand_length(ad_len - pos);
+
+		AEAD_AUTH_UPDATE(ctx, &ad[pos], part_len);
+		pos += part_len;
+		num_parts++;
+	}
+	if (pos < ad_len || rand_bool()) {
+		AEAD_AUTH_UPDATE(ctx, &ad[pos], ad_len - pos);
+		num_parts++;
+	}
+	return num_parts;
+}
+
+/* Process the given en/decrypted data using a random incremental strategy. */
+static size_t aead_crypt_incrementally(struct AEAD_CTX *ctx, u8 *dst,
+				       const u8 *src, size_t data_len, bool enc)
+{
+	size_t num_parts = 0;
+	size_t pos = 0;
+
+	while (rand_bool()) {
+		size_t part_len = rand_length(data_len - pos);
+
+		if (enc)
+			AEAD_ENCRYPT_UPDATE(ctx, &dst[pos], &src[pos],
+					    part_len);
+		else
+			AEAD_DECRYPT_UPDATE(ctx, &dst[pos], &src[pos],
+					    part_len);
+		pos += part_len;
+		num_parts++;
+	}
+	if (pos < data_len || rand_bool()) {
+		if (enc)
+			AEAD_ENCRYPT_UPDATE(ctx, &dst[pos], &src[pos],
+					    data_len - pos);
+		else
+			AEAD_DECRYPT_UPDATE(ctx, &dst[pos], &src[pos],
+					    data_len - pos);
+		num_parts++;
+	}
+	return num_parts;
+}
+
+struct aead_incremental_info {
+	size_t num_data_parts;
+	size_t num_ad_parts;
+};
+
+static const char *aead_incr_info_str(struct kunit *test,
+				      const struct aead_incremental_info *info)
+{
+	const size_t max_str_len = 64;
+	char *str = alloc_buf(test, max_str_len);
+
+	snprintf(str, max_str_len, "num_data_parts=%zu num_ad_parts=%zu",
+		 info->num_data_parts, info->num_ad_parts);
+	return str;
+}
+
+/*
+ * Encrypt data using a random incremental strategy.
+ * Return information about the incremental strategy used.
+ */
+static struct aead_incremental_info
+aead_encrypt_incrementally(struct kunit *test, struct AEAD_CTX *ctx, u8 *dst,
+			   const u8 *src, size_t data_len, u8 *tag,
+			   const u8 *ad, size_t ad_len, const u8 *nonce,
+			   size_t nonce_len, const struct AEAD_KEY *key)
+{
+	struct aead_incremental_info info;
+	int err;
+
+	err = AEAD_INIT(ctx, data_len, ad_len, nonce, nonce_len, key);
+	KUNIT_ASSERT_EQ(test, 0, err);
+	info.num_ad_parts = aead_auth_incrementally(ctx, ad, ad_len);
+	info.num_data_parts = aead_crypt_incrementally(ctx, dst, src, data_len,
+						       /* enc= */ true);
+	AEAD_ENCRYPT_FINAL(ctx, tag);
+	KUNIT_ASSERT_TRUE_MSG(test, mem_is_zero(ctx, sizeof(*ctx)),
+			      "encrypt_final didn't zeroize context");
+	return info;
+}
+
+/*
+ * Decrypt authentic data using a random incremental strategy.
+ * Return information about the incremental strategy used.
+ */
+static struct aead_incremental_info
+aead_decrypt_incrementally(struct kunit *test, struct AEAD_CTX *ctx, u8 *dst,</