diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2026-08-24 08:46:03 -0700 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2026-08-24 08:46:03 -0700 |
| commit | cf9610f9116d55c5a66ef9f1faecacabd93a9322 (patch) | |
| tree | 757a3d280760c654767631cab5ad8af2d86d1848 /lib/crypto/arm64/aes.h | |
| download | linux-stable-cf9610f9116d55c5a66ef9f1faecacabd93a9322.tar.gz linux-stable-cf9610f9116d55c5a66ef9f1faecacabd93a9322.zip | |
Merge tag 'pinctrl-v7.3-1' of git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrlgrafted
Pull pin control updates from Linus Walleij:
"Core changes:
- Use the non-blocking mux_state_try_select() in the generic MUX pin
control back-end
- Free pin maps on pinctrl_generic_to_map() failure in the core
helpers
New hardware pin control:
- Qualcomm Maili TLMM SoC, PMG1110 PMIC, and Eliza LPASS LPI TLMM SoC
(this is for the low power audio, LPASS = Low Power Audio SubSystem
portions)
- Upboard support on top of the Intel pin controller
- Apple T6030 and T6031 support (just compatible strings)
- Samsung Exynos 8855 SoC
- Mediatek MT6858 SoC
- Rockchip RK3308B and RV1106 SoCs
- Airoha EN7523 and AN7563 SoCs
Improvements:
- Qualcomm TLMM GPIOs are unconditionally marked as wakeup capable
- Qualcomm improvements on top of the Qualcomm PDC (power domain
controller) irqchip improvements from tglx:s tree
- Qualcomm IPQ806x and IPQ9650 non-urgent pin grop fixups
- The NPCM8xx driver has been put into shape fixing many pin and
group definitions
- Some non-urgent Tegra 264 pin assignments are fixed up
- Some non-urgent fixups to Airoha AN7581 and AN7583 registers and
pin assignments, missing features etc"
* tag 'pinctrl-v7.3-1' of git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl: (130 commits)
dt-bindings: pinctrl: Convert TI DA850 pupd to DT schema
pinctrl: meson: sync some modify from A4
pinctrl: meson: a4: Add input enable pin configuration
pinctrl: realtek: rtd1625: remove unused group name spdif_sel
pinctrl: realtek: rtd1625: fix base_bit for VE4 GPIO 13
dt-bindings: pinctrl: microchip,pic32mzda-pinctrl: Convert to DT schema
Revert "Merge branch 'ib-rsk7204' into devel"
pinctrl: rockchip: Add RV1106 pinctrl support
dt-bindings: pinctrl: rockchip: Add RV1106 compatible
pinctrl: rockchip: Decode drive strength in the get function
pinctrl: fix PINCTRL_GENERIC_MUX not always being selectable
pinctrl: airoha: add support of an7563 SoC
dt-bindings: pinctrl: airoha: add support of an7563 pin controller
pinctrl: airoha: try to find chip scu node by phandle first
pinctrl: airoha: add support of en7523 SoC
dt-bindings: pinctrl: airoha: add support of en7523 pin controller
pinctrl: airoha: an7583: add support for olt pinmux
pinctrl: airoha: an7583: add support for pon_alt pinmux
pinctrl: airoha: an7583: add support for npu_uart pinmux
dt-bindings: pinctrl: airoha: an7583: add missed features
...
Diffstat (limited to 'lib/crypto/arm64/aes.h')
| -rw-r--r-- | lib/crypto/arm64/aes.h | 219 |
1 files changed, 219 insertions, 0 deletions
diff --git a/lib/crypto/arm64/aes.h b/lib/crypto/arm64/aes.h new file mode 100644 index 000000000..9e9e45a6f --- /dev/null +++ b/lib/crypto/arm64/aes.h @@ -0,0 +1,219 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * AES block cipher, optimized for ARM64 + * + * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> + * Copyright 2026 Google LLC + */ + +#include <asm/neon.h> +#include <asm/simd.h> +#include <linux/unaligned.h> +#include <linux/cpufeature.h> + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes); + +struct aes_block { + u8 b[AES_BLOCK_SIZE]; +}; + +asmlinkage void __aes_arm64_encrypt(const u32 rk[], u8 out[AES_BLOCK_SIZE], + const u8 in[AES_BLOCK_SIZE], int rounds); +asmlinkage void __aes_arm64_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE], + const u8 in[AES_BLOCK_SIZE], int rounds); +asmlinkage void __aes_ce_encrypt(const u32 rk[], u8 out[AES_BLOCK_SIZE], + const u8 in[AES_BLOCK_SIZE], int rounds); +asmlinkage void __aes_ce_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE], + const u8 in[AES_BLOCK_SIZE], int rounds); +asmlinkage u32 __aes_ce_sub(u32 l); +asmlinkage void __aes_ce_invert(struct aes_block *out, + const struct aes_block *in); +asmlinkage void neon_aes_mac_update(u8 const in[], u32 const rk[], int rounds, + size_t blocks, u8 dg[], int enc_before, + int enc_after); + +/* + * Expand an AES key using the crypto extensions if supported and usable or + * generic code otherwise. The expanded key format is compatible between the + * two cases. The outputs are @rndkeys (required) and @inv_rndkeys (optional). + */ +static void aes_expandkey_arm64(u32 rndkeys[], u32 *inv_rndkeys, + const u8 *in_key, int key_len, int nrounds) +{ + /* + * The AES key schedule round constants + */ + static u8 const rcon[] = { + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, + }; + + u32 kwords = key_len / sizeof(u32); + struct aes_block *key_enc, *key_dec; + int i, j; + + if (!static_branch_likely(&have_aes) || unlikely(!may_use_simd())) { + aes_expandkey_generic(rndkeys, inv_rndkeys, in_key, key_len); + return; + } + + for (i = 0; i < kwords; i++) + rndkeys[i] = get_unaligned_le32(&in_key[i * sizeof(u32)]); + + scoped_ksimd() { + for (i = 0; i < sizeof(rcon); i++) { + u32 *rki = &rndkeys[i * kwords]; + u32 *rko = rki + kwords; + + rko[0] = ror32(__aes_ce_sub(rki[kwords - 1]), 8) ^ + rcon[i] ^ rki[0]; + rko[1] = rko[0] ^ rki[1]; + rko[2] = rko[1] ^ rki[2]; + rko[3] = rko[2] ^ rki[3]; + + if (key_len == AES_KEYSIZE_192) { + if (i >= 7) + break; + rko[4] = rko[3] ^ rki[4]; + rko[5] = rko[4] ^ rki[5]; + } else if (key_len == AES_KEYSIZE_256) { + if (i >= 6) + break; + rko[4] = __aes_ce_sub(rko[3]) ^ rki[4]; + rko[5] = rko[4] ^ rki[5]; + rko[6] = rko[5] ^ rki[6]; + rko[7] = rko[6] ^ rki[7]; + } + } + + /* + * Generate the decryption keys for the Equivalent Inverse + * Cipher. This involves reversing the order of the round + * keys, and applying the Inverse Mix Columns transformation on + * all but the first and the last one. + */ + if (inv_rndkeys) { + key_enc = (struct aes_block *)rndkeys; + key_dec = (struct aes_block *)inv_rndkeys; + j = nrounds; + + key_dec[0] = key_enc[j]; + for (i = 1, j--; j > 0; i++, j--) + __aes_ce_invert(key_dec + i, key_enc + j); + key_dec[i] = key_enc[0]; + } + } +} + +static void aes_preparekey_arch(union aes_enckey_arch *k, + union aes_invkey_arch *inv_k, + const u8 *in_key, int key_len, int nrounds) +{ + aes_expandkey_arm64(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL, + in_key, key_len, nrounds); +} + +/* + * This is here temporarily until the remaining AES mode implementations are + * migrated from arch/arm64/crypto/ to lib/crypto/arm64/. + */ +int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, + unsigned int key_len) +{ + if (aes_check_keylen(key_len) != 0) + return -EINVAL; + ctx->key_length = key_len; + aes_expandkey_arm64(ctx->key_enc, ctx->key_dec, in_key, key_len, + 6 + key_len / 4); + return 0; +} +EXPORT_SYMBOL(ce_aes_expandkey); + +EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_cts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_cts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_ctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_xctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_xts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_xts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL"); + +EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_cts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_cts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_ctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_xctr_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_xts_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_xts_decrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL"); +EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL"); +#if IS_MODULE(CONFIG_CRYPTO_AES_ARM64_CE_CCM) +EXPORT_SYMBOL_NS_GPL(ce_aes_mac_update, "CRYPTO_INTERNAL"); +#endif + +static void aes_encrypt_arch(const struct aes_enckey *key, + u8 out[AES_BLOCK_SIZE], + const u8 in[AES_BLOCK_SIZE]) +{ + if (static_branch_likely(&have_aes) && likely(may_use_simd())) { + scoped_ksimd() + __aes_ce_encrypt(key->k.rndkeys, out, in, key->nrounds); + } else { + __aes_arm64_encrypt(key->k.rndkeys, out, in, key->nrounds); + } +} + +static void aes_decrypt_arch(const struct aes_key *key, + u8 out[AES_BLOCK_SIZE], + const u8 in[AES_BLOCK_SIZE]) +{ + if (static_branch_likely(&have_aes) && likely(may_use_simd())) { + scoped_ksimd() + __aes_ce_decrypt(key->inv_k.inv_rndkeys, out, in, + key->nrounds); + } else { + __aes_arm64_decrypt(key->inv_k.inv_rndkeys, out, in, + key->nrounds); + } +} + +#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS) +#define aes_cbcmac_blocks_arch aes_cbcmac_blocks_arch +static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE], + const struct aes_enckey *key, const u8 *data, + size_t nblocks, bool enc_before, + bool enc_after) +{ + if (static_branch_likely(&have_neon) && likely(may_use_simd())) { + scoped_ksimd() { + if (static_branch_likely(&have_aes)) + ce_aes_mac_update(data, key->k.rndkeys, + key->nrounds, nblocks, h, + enc_before, enc_after); + else + neon_aes_mac_update(data, key->k.rndkeys, + key->nrounds, nblocks, h, + enc_before, enc_after); + } + return true; + } + return false; +} +#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */ + +#define aes_mod_init_arch aes_mod_init_arch +static void aes_mod_init_arch(void) +{ + if (cpu_have_named_feature(ASIMD)) { + static_branch_enable(&have_neon); + if (cpu_have_named_feature(AES)) + static_branch_enable(&have_aes); + } +} |
