summaryrefslogtreecommitdiffstats
path: root/arch/sparc/crypto
diff options
context:
space:
mode:
Diffstat (limited to 'arch/sparc/crypto')
-rw-r--r--arch/sparc/crypto/Kconfig26
-rw-r--r--arch/sparc/crypto/Makefile10
-rw-r--r--arch/sparc/crypto/aes_glue.c344
-rw-r--r--arch/sparc/crypto/camellia_asm.S563
-rw-r--r--arch/sparc/crypto/camellia_glue.c293
-rw-r--r--arch/sparc/crypto/crop_devid.c15
6 files changed, 1251 insertions, 0 deletions
diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig
new file mode 100644
index 000000000..8db3f6eea
--- /dev/null
+++ b/arch/sparc/crypto/Kconfig
@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: GPL-2.0
+
+menu "Accelerated Cryptographic Algorithms for CPU (sparc64)"
+
+config CRYPTO_AES_SPARC64
+ tristate "Ciphers: AES, modes: ECB, CBC, CTR"
+ depends on SPARC64
+ select CRYPTO_LIB_AES
+ select CRYPTO_SKCIPHER
+ help
+ Length-preseving ciphers: AES with ECB, CBC, and CTR modes
+
+ Architecture: sparc64 using crypto instructions
+
+config CRYPTO_CAMELLIA_SPARC64
+ tristate "Ciphers: Camellia, modes: ECB, CBC"
+ depends on SPARC64
+ select CRYPTO_ALGAPI
+ select CRYPTO_SKCIPHER
+ help
+ Block ciphers: Camellia cipher algorithms
+ Length-preserving ciphers: Camellia with ECB and CBC modes
+
+ Architecture: sparc64
+
+endmenu
diff --git a/arch/sparc/crypto/Makefile b/arch/sparc/crypto/Makefile
new file mode 100644
index 000000000..ab4a7765b
--- /dev/null
+++ b/arch/sparc/crypto/Makefile
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Arch-specific CryptoAPI modules.
+#
+
+obj-$(CONFIG_CRYPTO_AES_SPARC64) += aes-sparc64.o
+obj-$(CONFIG_CRYPTO_CAMELLIA_SPARC64) += camellia-sparc64.o
+
+aes-sparc64-y := aes_glue.o
+camellia-sparc64-y := camellia_asm.o camellia_glue.o
diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c
new file mode 100644
index 000000000..661561837
--- /dev/null
+++ b/arch/sparc/crypto/aes_glue.c
@@ -0,0 +1,344 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Glue code for AES encryption optimized for sparc64 crypto opcodes.
+ *
+ * This is based largely upon arch/x86/crypto/aesni-intel_glue.c
+ *
+ * Copyright (C) 2008, Intel Corp.
+ * Author: Huang Ying <ying.huang@intel.com>
+ *
+ * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
+ * interface for 64-bit kernels.
+ * Authors: Adrian Hoban <adrian.hoban@intel.com>
+ * Gabriele Paoloni <gabriele.paoloni@intel.com>
+ * Tadeusz Struk (tadeusz.struk@intel.com)
+ * Aidan O'Mahony (aidan.o.mahony@intel.com)
+ * Copyright (c) 2010, Intel Corporation.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/crypto.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/mm.h>
+#include <linux/types.h>
+#include <crypto/algapi.h>
+#include <crypto/aes.h>
+#include <crypto/internal/skcipher.h>
+
+#include <asm/fpumacro.h>
+#include <asm/opcodes.h>
+#include <asm/pstate.h>
+#include <asm/elf.h>
+
+struct aes_ops {
+ void (*load_encrypt_keys)(const u64 *key);
+ void (*load_decrypt_keys)(const u64 *key);
+ void (*ecb_encrypt)(const u64 *key, const u64 *input, u64 *output,
+ unsigned int len);
+ void (*ecb_decrypt)(const u64 *key, const u64 *input, u64 *output,
+ unsigned int len);
+ void (*cbc_encrypt)(const u64 *key, const u64 *input, u64 *output,
+ unsigned int len, u64 *iv);
+ void (*cbc_decrypt)(const u64 *key, const u64 *input, u64 *output,
+ unsigned int len, u64 *iv);
+ void (*ctr_crypt)(const u64 *key, const u64 *input, u64 *output,
+ unsigned int len, u64 *iv);
+};
+
+struct crypto_sparc64_aes_ctx {
+ struct aes_ops *ops;
+ u64 key[AES_MAX_KEYLENGTH / sizeof(u64)];
+ u32 key_length;
+ u32 expanded_key_length;
+};
+
+static struct aes_ops aes128_ops = {
+ .load_encrypt_keys = aes_sparc64_load_encrypt_keys_128,
+ .load_decrypt_keys = aes_sparc64_load_decrypt_keys_128,
+ .ecb_encrypt = aes_sparc64_ecb_encrypt_128,
+ .ecb_decrypt = aes_sparc64_ecb_decrypt_128,
+ .cbc_encrypt = aes_sparc64_cbc_encrypt_128,
+ .cbc_decrypt = aes_sparc64_cbc_decrypt_128,
+ .ctr_crypt = aes_sparc64_ctr_crypt_128,
+};
+
+static struct aes_ops aes192_ops = {
+ .load_encrypt_keys = aes_sparc64_load_encrypt_keys_192,
+ .load_decrypt_keys = aes_sparc64_load_decrypt_keys_192,
+ .ecb_encrypt = aes_sparc64_ecb_encrypt_192,
+ .ecb_decrypt = aes_sparc64_ecb_decrypt_192,
+ .cbc_encrypt = aes_sparc64_cbc_encrypt_192,
+ .cbc_decrypt = aes_sparc64_cbc_decrypt_192,
+ .ctr_crypt = aes_sparc64_ctr_crypt_192,
+};
+
+static struct aes_ops aes256_ops = {
+ .load_encrypt_keys = aes_sparc64_load_encrypt_keys_256,
+ .load_decrypt_keys = aes_sparc64_load_decrypt_keys_256,
+ .ecb_encrypt = aes_sparc64_ecb_encrypt_256,
+ .ecb_decrypt = aes_sparc64_ecb_decrypt_256,
+ .cbc_encrypt = aes_sparc64_cbc_encrypt_256,
+ .cbc_decrypt = aes_sparc64_cbc_decrypt_256,
+ .ctr_crypt = aes_sparc64_ctr_crypt_256,
+};
+
+static int aes_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+ switch (key_len) {
+ case AES_KEYSIZE_128:
+ ctx->expanded_key_length = 0xb0;
+ ctx->ops = &aes128_ops;
+ break;
+
+ case AES_KEYSIZE_192:
+ ctx->expanded_key_length = 0xd0;
+ ctx->ops = &aes192_ops;
+ break;
+
+ case AES_KEYSIZE_256:
+ ctx->expanded_key_length = 0xf0;
+ ctx->ops = &aes256_ops;
+ break;
+
+ default:
+ return -EINVAL;
+ }
+
+ aes_sparc64_key_expand((const u32 *)in_key, &ctx->key[0], key_len);
+ ctx->key_length = key_len;
+
+ return 0;
+}
+
+static int ecb_encrypt(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ unsigned int nbytes;
+ int err;
+
+ err = skcipher_walk_virt(&walk, req, true);
+ if (err)
+ return err;
+
+ ctx->ops->load_encrypt_keys(&ctx->key[0]);
+ while ((nbytes = walk.nbytes) != 0) {
+ ctx->ops->ecb_encrypt(&ctx->key[0], walk.src.virt.addr,
+ walk.dst.virt.addr,
+ round_down(nbytes, AES_BLOCK_SIZE));
+ err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
+ }
+ fprs_write(0);
+ return err;
+}
+
+static int ecb_decrypt(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
+ const u64 *key_end;
+ struct skcipher_walk walk;
+ unsigned int nbytes;
+ int err;
+
+ err = skcipher_walk_virt(&walk, req, true);
+ if (err)
+ return err;
+
+ ctx->ops->load_decrypt_keys(&ctx->key[0]);
+ key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
+ while ((nbytes = walk.nbytes) != 0) {
+ ctx->ops->ecb_decrypt(key_end, walk.src.virt.addr,
+ walk.dst.virt.addr,
+ round_down(nbytes, AES_BLOCK_SIZE));
+ err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
+ }
+ fprs_write(0);
+
+ return err;
+}
+
+static int cbc_encrypt(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ unsigned int nbytes;
+ int err;
+
+ err = skcipher_walk_virt(&walk, req, true);
+ if (err)
+ return err;
+
+ ctx->ops->load_encrypt_keys(&ctx->key[0]);
+ while ((nbytes = walk.nbytes) != 0) {
+ ctx->ops->cbc_encrypt(&ctx->key[0], walk.src.virt.addr,
+ walk.dst.virt.addr,
+ round_down(nbytes, AES_BLOCK_SIZE),
+ walk.iv);
+ err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
+ }
+ fprs_write(0);
+ return err;
+}
+
+static int cbc_decrypt(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
+ const u64 *key_end;
+ struct skcipher_walk walk;
+ unsigned int nbytes;
+ int err;
+
+ err = skcipher_walk_virt(&walk, req, true);
+ if (err)
+ return err;
+
+ ctx->ops->load_decrypt_keys(&ctx->key[0]);
+ key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
+ while ((nbytes = walk.nbytes) != 0) {
+ ctx->ops->cbc_decrypt(key_end, walk.src.virt.addr,
+ walk.dst.virt.addr,
+ round_down(nbytes, AES_BLOCK_SIZE),
+ walk.iv);
+ err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
+ }
+ fprs_write(0);
+
+ return err;
+}
+
+static void ctr_crypt_final(const struct crypto_sparc64_aes_ctx *ctx,
+ struct skcipher_walk *walk)
+{
+ u8 *ctrblk = walk->iv;
+ u64 keystream[AES_BLOCK_SIZE / sizeof(u64)];
+ const u8 *src = walk->src.virt.addr;
+ u8 *dst = walk->dst.virt.addr;
+ unsigned int nbytes = walk->nbytes;
+
+ ctx->ops->ecb_encrypt(&ctx->key[0], (const u64 *)ctrblk,
+ keystream, AES_BLOCK_SIZE);
+ crypto_xor_cpy(dst, (u8 *) keystream, src, nbytes);
+ crypto_inc(ctrblk, AES_BLOCK_SIZE);
+}
+
+static int ctr_crypt(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ unsigned int nbytes;
+ int err;
+
+ err = skcipher_walk_virt(&walk, req, true);
+ if (err)
+ return err;
+
+ ctx->ops->load_encrypt_keys(&ctx->key[0]);
+ while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
+ ctx->ops->ctr_crypt(&ctx->key[0], walk.src.virt.addr,
+ walk.dst.virt.addr,
+ round_down(nbytes, AES_BLOCK_SIZE),
+ walk.iv);
+ err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
+ }
+ if (walk.nbytes) {
+ ctr_crypt_final(ctx, &walk);
+ err = skcipher_walk_done(&walk, 0);
+ }
+ fprs_write(0);
+ return err;
+}
+
+static struct skcipher_alg skcipher_algs[] = {
+ {
+ .base.cra_name = "ecb(aes)",
+ .base.cra_driver_name = "ecb-aes-sparc64",
+ .base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx),
+ .base.cra_alignmask = 7,
+ .base.cra_module = THIS_MODULE,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = aes_set_key_skcipher,
+ .encrypt = ecb_encrypt,
+ .decrypt = ecb_decrypt,
+ }, {
+ .base.cra_name = "cbc(aes)",
+ .base.cra_driver_name = "cbc-aes-sparc64",
+ .base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx),
+ .base.cra_alignmask = 7,
+ .base.cra_module = THIS_MODULE,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = aes_set_key_skcipher,
+ .encrypt = cbc_encrypt,
+ .decrypt = cbc_decrypt,
+ }, {
+ .base.cra_name = "ctr(aes)",
+ .base.cra_driver_name = "ctr-aes-sparc64",
+ .base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
+ .base.cra_blocksize = 1,
+ .base.cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx),
+ .base.cra_alignmask = 7,
+ .base.cra_module = THIS_MODULE,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = aes_set_key_skcipher,
+ .encrypt = ctr_crypt,
+ .decrypt = ctr_crypt,
+ .chunksize = AES_BLOCK_SIZE,
+ }
+};
+
+static bool __init sparc64_has_aes_opcode(void)
+{
+ unsigned long cfr;
+
+ if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
+ return false;
+
+ __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
+ if (!(cfr & CFR_AES))
+ return false;
+
+ return true;
+}
+
+static int __init aes_sparc64_mod_init(void)
+{
+ if (!sparc64_has_aes_opcode()) {
+ pr_info("sparc64 aes opcodes not available.\n");
+ return -ENODEV;
+ }
+ pr_info("Using sparc64 aes opcodes optimized AES implementation\n");
+ return crypto_register_skciphers(skcipher_algs,
+ ARRAY_SIZE(skcipher_algs));
+}
+
+static void __exit aes_sparc64_mod_fini(void)
+{
+ crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
+}
+
+module_init(aes_sparc64_mod_init);
+module_exit(aes_sparc64_mod_fini);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, sparc64 aes opcode accelerated");
+
+MODULE_ALIAS_CRYPTO("aes");
+
+#include "crop_devid.c"
diff --git a/arch/sparc/crypto/camellia_asm.S b/arch/sparc/crypto/camellia_asm.S
new file mode 100644
index 000000000..8471b346e
--- /dev/null
+++ b/arch/sparc/crypto/camellia_asm.S
@@ -0,0 +1,563 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#include <linux/linkage.h>
+#include <asm/opcodes.h>
+#include <asm/visasm.h>
+
+#define CAMELLIA_6ROUNDS(KEY_BASE, I0, I1) \
+ CAMELLIA_F(KEY_BASE + 0, I1, I0, I1) \
+ CAMELLIA_F(KEY_BASE + 2, I0, I1, I0) \
+ CAMELLIA_F(KEY_BASE + 4, I1, I0, I1) \
+ CAMELLIA_F(KEY_BASE + 6, I0, I1, I0) \
+ CAMELLIA_F(KEY_BASE + 8, I1, I0, I1) \
+ CAMELLIA_F(KEY_BASE + 10, I0, I1, I0)
+
+#define CAMELLIA_6ROUNDS_FL_FLI(KEY_BASE, I0, I1) \
+ CAMELLIA_6ROUNDS(KEY_BASE, I0, I1) \
+ CAMELLIA_FL(KEY_BASE + 12, I0, I0) \
+ CAMELLIA_FLI(KEY_BASE + 14, I1, I1)
+
+ .data
+
+ .align 8
+SIGMA: .xword 0xA09E667F3BCC908B
+ .xword 0xB67AE8584CAA73B2
+ .xword 0xC6EF372FE94F82BE
+ .xword 0x54FF53A5F1D36F1C
+ .xword 0x10E527FADE682D1D
+ .xword 0xB05688C2B3E6C1FD
+
+ .text
+
+ .align 32
+ENTRY(camellia_sparc64_key_expand)
+ /* %o0=in_key, %o1=encrypt_key, %o2=key_len, %o3=decrypt_key */
+ VISEntry
+ ld [%o0 + 0x00], %f0 ! i0, k[0]
+ ld [%o0 + 0x04], %f1 ! i1, k[1]
+ ld [%o0 + 0x08], %f2 ! i2, k[2]
+ ld [%o0 + 0x0c], %f3 ! i3, k[3]
+ std %f0, [%o1 + 0x00] ! k[0, 1]
+ fsrc2 %f0, %f28
+ std %f2, [%o1 + 0x08] ! k[2, 3]
+ cmp %o2, 16
+ be 10f
+ fsrc2 %f2, %f30
+
+ ld [%o0 + 0x10], %f0
+ ld [%o0 + 0x14], %f1
+ std %f0, [%o1 + 0x20] ! k[8, 9]
+ cmp %o2, 24
+ fone %f10
+ be,a 1f
+ fxor %f10, %f0, %f2
+ ld [%o0 + 0x18], %f2
+ ld [%o0 + 0x1c], %f3
+1:
+ std %f2, [%o1 + 0x28] ! k[10, 11]
+ fxor %f28, %f0, %f0
+ fxor %f30, %f2, %f2
+
+10:
+ sethi %hi(SIGMA), %g3
+ or %g3, %lo(SIGMA), %g3
+ ldd [%g3 + 0x00], %f16
+ ldd [%g3 + 0x08], %f18
+ ldd [%g3 + 0x10], %f20
+ ldd [%g3 + 0x18], %f22
+ ldd [%g3 + 0x20], %f24
+ ldd [%g3 + 0x28], %f26
+ CAMELLIA_F(16, 2, 0, 2)
+ CAMELLIA_F(18, 0, 2, 0)
+ fxor %f28, %f0, %f0
+ fxor %f30, %f2, %f2
+ CAMELLIA_F(20, 2, 0, 2)
+ CAMELLIA_F(22, 0, 2, 0)
+
+#define ROTL128(S01, S23, TMP1, TMP2, N) \
+ srlx S01, (64 - N), TMP1; \
+ sllx S01, N, S01; \
+ srlx S23, (64 - N), TMP2; \
+ sllx S23, N, S23; \
+ or S01, TMP2, S01; \
+ or S23, TMP1, S23
+
+ cmp %o2, 16
+ bne 1f
+ nop
+ /* 128-bit key */
+ std %f0, [%o1 + 0x10] ! k[ 4, 5]
+ std %f2, [%o1 + 0x18] ! k[ 6, 7]
+ MOVDTOX_F0_O4
+ MOVDTOX_F2_O5
+ ROTL128(%o4, %o5, %g2, %g3, 15)
+ stx %o4, [%o1 + 0x30] ! k[12, 13]
+ stx %o5, [%o1 + 0x38] ! k[14, 15]
+ ROTL128(%o4, %o5, %g2, %g3, 15)
+ stx %o4, [%o1 + 0x40] ! k[16, 17]
+ stx %o5, [%o1 + 0x48] ! k[18, 19]
+ ROTL128(%o4, %o5, %g2, %g3, 15)
+ stx %o4, [%o1 + 0x60] ! k[24, 25]
+ ROTL128(%o4, %o5, %g2, %g3, 15)
+ stx %o4, [%o1 + 0x70] ! k[28, 29]
+ stx %o5, [%o1 + 0x78] ! k[30, 31]
+ ROTL128(%o4, %o5, %g2, %g3, 34)
+ stx %o4, [%o1 + 0xa0] ! k[40, 41]
+ stx %o5, [%o1 + 0xa8] ! k[42, 43]
+ ROTL128(%o4, %o5, %g2, %g3, 17)
+ stx %o4, [%o1 + 0xc0] ! k[48, 49]
+ stx %o5, [%o1 + 0xc8] ! k[50, 51]
+
+ ldx [%o1 + 0x00], %o4 ! k[ 0, 1]
+ ldx [%o1 + 0x08], %o5 ! k[ 2, 3]
+ ROTL128(%o4, %o5, %g2, %g3, 15)
+ stx %o4, [%o1 + 0x20] ! k[ 8, 9]
+ stx %o5, [%o1 + 0x28] ! k[10, 11]
+ ROTL128(%o4, %o5, %g2, %g3, 30)
+ stx %o4, [%o1 + 0x50] ! k[20, 21]
+ stx %o5, [%o1 + 0x58] ! k[22, 23]
+ ROTL128(%o4, %o5, %g2, %g3, 15)
+ stx %o5, [%o1 + 0x68] ! k[26, 27]
+ ROTL128(%o4, %o5, %g2, %g3, 17)
+ stx %o4, [%o1 + 0x80] ! k[32, 33]
+ stx %o5, [%o1 + 0x88] ! k[34, 35]
+ ROTL128(%o4, %o5, %g2, %g3, 17)
+ stx %o4, [%o1 + 0x90] ! k[36, 37]
+ stx %o5, [%o1 + 0x98] ! k[38, 39]
+ ROTL128(%o4, %o5, %g2, %g3, 17)
+ stx %o4, [%o1 + 0xb0] ! k[44, 45]
+ stx %o5, [%o1 + 0xb8] ! k[46, 47]
+
+ ba,pt %xcc, 2f
+ mov (3 * 16 * 4), %o0
+
+1:
+ /* 192-bit or 256-bit key */
+ std %f0, [%o1 + 0x30] ! k[12, 13]
+ std %f2, [%o1 + 0x38] ! k[14, 15]
+ ldd [%o1 + 0x20], %f4 ! k[ 8, 9]
+ ldd [%o1 + 0x28], %f6 ! k[10, 11]
+ fxor %f0, %f4, %f0
+ fxor %f2, %f6, %f2
+ CAMELLIA_F(24, 2, 0, 2)
+ CAMELLIA_F(26, 0, 2, 0)
+ std %f0, [%o1 + 0x10] ! k[ 4, 5]
+ std %f2, [%o1 + 0x18] ! k[ 6, 7]
+ MOVDTOX_F0_O4
+ MOVDTOX_F2_O5
+ ROTL128(%o4, %o5, %g2, %g3, 30)
+ stx %o4, [%o1 + 0x50] ! k[20, 21]
+ stx %o5, [%o1 + 0x58] ! k[22, 23]
+ ROTL128(%o4, %o5, %g2, %g3, 30)
+ stx %o4, [%o1 + 0xa0] ! k[40, 41]
+ stx %o5, [%o1 + 0xa8] ! k[42, 43]
+ ROTL128(%o4, %o5, %g2, %g3, 51)
+ stx %o4, [%o1 + 0x100] ! k[64, 65]
+ stx %o5, [%o1 + 0x108] ! k[66, 67]
+ ldx [%o1 + 0x20], %o4 ! k[ 8, 9]
+ ldx [%o1 + 0x28], %o5 ! k[10, 11]
+ ROTL128(%o4, %o5, %g2, %g3, 15)
+ stx %o4, [%o1 + 0x20] ! k[ 8, 9]
+ stx %o5, [%o1 + 0x28] ! k[10, 11]
+ ROTL128(%o4, %o5, %g2, %g3, 15)
+ stx %o4, [%o1 + 0x40] ! k[16, 17]
+ stx %o5, [%o1 + 0x48] ! k[18, 19]
+ ROTL128(%o4, %o5, %g2, %g3, 30)
+ stx %o4, [%o1 + 0x90] ! k[36, 37]
+ stx %o5, [%o1 + 0x98] ! k[38, 39]
+ ROTL128(%o4, %o5, %g2, %g3, 34)
+ stx %o4, [%o1 + 0xd0] ! k[52, 53]
+ stx %o5, [%o1 + 0xd8] ! k[54, 55]
+ ldx [%o1 + 0x30], %o4 ! k[12, 13]
+ ldx [%o1 + 0x38], %o5 ! k[14, 15]
+ ROTL128(%o4, %o5, %g2, %g3, 15)
+ stx %o4, [%o1 + 0x30] ! k[12, 13]
+ stx %o5, [%o1 + 0x38] ! k[14, 15]
+ ROTL128(%o4, %o5, %g2, %g3, 30)
+ stx %o4, [%o1 + 0x70] ! k[28, 29]
+ stx %o5, [%o1 + 0x78] ! k[30, 31]
+ srlx %o4, 32, %g2
+ srlx %o5, 32, %g3
+ stw %o4, [%o1 + 0xc0] ! k[48]
+ stw %g3, [%o1 + 0xc4] ! k[49]
+ stw %o5, [%o1 + 0xc8] ! k[50]
+ stw %g2, [%o1 + 0xcc] ! k[51]
+ ROTL128(%o4, %o5, %g2, %g3, 49)
+ stx %o4, [%o1 + 0xe0] ! k[56, 57]
+ stx %o5, [%o1 + 0xe8] ! k[58, 59]
+ ldx [%o1 + 0x00], %o4 ! k[ 0, 1]
+ ldx [%o1 + 0x08], %o5 ! k[ 2, 3]
+ ROTL128(%o4, %o5, %g2, %g3, 45)
+ stx %o4, [%o1 + 0x60] ! k[24, 25]
+ stx %o5, [%o1 + 0x68] ! k[26, 27]
+ ROTL128(%o4, %o5, %g2, %g3, 15)
+ stx %o4, [%o1 + 0x80] ! k[32, 33]
+ stx %o5, [%o1 + 0x88] ! k[34, 35]
+ ROTL128(%o4, %o5, %g2, %g3, 17)
+ stx %o4, [%o1 + 0xb0] ! k[44, 45]
+ stx %o5, [%o1 + 0xb8] ! k[46, 47]
+ ROTL128(%o4, %o5, %g2, %g3, 34)
+ stx %o4, [%o1 + 0xf0] ! k[60, 61]
+ stx %o5, [%o1 + 0xf8] ! k[62, 63]
+ mov (4 * 16 * 4), %o0
+2:
+ add %o1, %o0, %o1
+ ldd [%o1 + 0x00], %f0
+ ldd [%o1 + 0x08], %f2
+ std %f0, [%o3 + 0x00]
+ std %f2, [%o3 + 0x08]
+ add %o3, 0x10, %o3
+1:
+ sub %o1, (16 * 4), %o1
+ ldd [%o1 + 0x38], %f0
+ ldd [%o1 + 0x30], %f2
+ ldd [%o1 + 0x28], %f4
+ ldd [%o1 + 0x20], %f6
+ ldd [%o1 + 0x18], %f8
+ ldd [%o1 + 0x10], %f10
+ std %f0, [%o3 + 0x00]
+ std %f2, [%o3 + 0x08]
+ std %f4, [%o3 + 0x10]
+ std %f6, [%o3 + 0x18]
+ std %f8, [%o3 + 0x20]
+ std %f10, [%o3 + 0x28]
+
+ ldd [%o1 + 0x08], %f0
+ ldd [%o1 + 0x00], %f2
+ std %f0, [%o3 + 0x30]
+ std %f2, [%o3 + 0x38]
+ subcc %o0, (16 * 4), %o0
+ bne,pt %icc, 1b
+ add %o3, (16 * 4), %o3
+
+ std %f2, [%o3 - 0x10]
+ std %f0, [%o3 - 0x08]
+
+ retl
+ VISExit
+ENDPROC(camellia_sparc64_key_expand)
+
+ .align 32
+ENTRY(camellia_sparc64_crypt)
+ /* %o0=key, %o1=input, %o2=output, %o3=key_len */
+ VISEntry
+
+ ld [%o1 + 0x00], %f0
+ ld [%o1 + 0x04], %f1
+ ld [%o1 + 0x08], %f2
+ ld [%o1 + 0x0c], %f3
+
+ ldd [%o0 + 0x00], %f4
+ ldd [%o0 + 0x08], %f6
+
+ cmp %o3, 16
+ fxor %f4, %f0, %f0
+ be 1f
+ fxor %f6, %f2, %f2
+
+ ldd [%o0 + 0x10], %f8
+ ldd [%o0 + 0x18], %f10
+ ldd [%o0 + 0x20], %f12
+ ldd [%o0 + 0x28], %f14
+ ldd [%o0 + 0x30], %f16
+ ldd [%o0 + 0x38], %f18
+ ldd [%o0 + 0x40], %f20
+ ldd [%o0 + 0x48], %f22
+ add %o0, 0x40, %o0
+
+ CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2)
+
+1:
+ ldd [%o0 + 0x10], %f8
+ ldd [%o0 + 0x18], %f10
+ ldd [%o0 + 0x20], %f12
+ ldd [%o0 + 0x28], %f14
+ ldd [%o0 + 0x30], %f16
+ ldd [%o0 + 0x38], %f18
+ ldd [%o0 + 0x40], %f20
+ ldd [%o0 + 0x48], %f22
+ ldd [%o0 + 0x50], %f24
+ ldd [%o0 + 0x58], %f26
+ ldd [%o0 + 0x60], %f28
+ ldd [%o0 + 0x68], %f30
+ ldd [%o0 + 0x70], %f32
+ ldd [%o0 + 0x78], %f34
+ ldd [%o0 + 0x80], %f36
+ ldd [%o0 + 0x88], %f38
+ ldd [%o0 + 0x90], %f40
+ ldd [%o0 + 0x98], %f42
+ ldd [%o0 + 0xa0], %f44
+ ldd [%o0 + 0xa8], %f46
+ ldd [%o0 + 0xb0], %f48
+ ldd [%o0 + 0xb8], %f50
+ ldd [%o0 + 0xc0], %f52
+ ldd [%o0 + 0xc8], %f54
+
+ CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2)
+ CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2)
+ CAMELLIA_6ROUNDS(40, 0, 2)
+ fxor %f52, %f2, %f2
+ fxor %f54, %f0, %f0
+
+ st %f2, [%o2 + 0x00]
+ st %f3, [%o2 + 0x04]
+ st %f0, [%o2 + 0x08]
+ st %f1, [%o2 + 0x0c]
+
+ retl
+ VISExit
+ENDPROC(camellia_sparc64_crypt)
+
+ .align 32
+ENTRY(camellia_sparc64_load_keys)
+ /* %o0=key, %o1=key_len */
+ VISEntry
+ ldd [%o0 + 0x00], %f4
+ ldd [%o0 + 0x08], %f6
+ ldd [%o0 + 0x10], %f8
+ ldd [%o0 + 0x18], %f10
+ ldd [%o0 + 0x20], %f12
+ ldd [%o0 + 0x28], %f14
+ ldd [%o0 + 0x30], %f16
+ ldd [%o0 + 0x38], %f18
+ ldd [%o0 + 0x40], %f20
+ ldd [%o0 + 0x48], %f22
+ ldd [%o0 + 0x50], %f24
+ ldd [%o0 + 0x58], %f26
+ ldd [%o0 + 0x60], %f28
+ ldd [%o0 + 0x68], %f30
+ ldd [%o0 + 0x70], %f32
+ ldd [%o0 + 0x78], %f34
+ ldd [%o0 + 0x80], %f36
+ ldd [%o0 + 0x88], %f38
+ ldd [%o0 + 0x90], %f40
+ ldd [%o0 + 0x98], %f42
+ ldd [%o0 + 0xa0], %f44
+ ldd [%o0 + 0xa8], %f46
+ ldd [%o0 + 0xb0], %f48
+ ldd [%o0 + 0xb8], %f50
+ ldd [%o0 + 0xc0], %f52
+ retl
+ ldd [%o0 + 0xc8], %f54
+ENDPROC(camellia_sparc64_load_keys)
+
+ .align 32
+ENTRY(camellia_sparc64_ecb_crypt_3_grand_rounds)
+ /* %o0=input, %o1=output, %o2=len, %o3=key */
+1: ldd [%o0 + 0x00], %f0
+ ldd [%o0 + 0x08], %f2
+ add %o0, 0x10, %o0
+ fxor %f4, %f0, %f0
+ fxor %f6, %f2, %f2
+ CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2)
+ CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2)
+ CAMELLIA_6ROUNDS(40, 0, 2)
+ fxor %f52, %f2, %f2
+ fxor %f54, %f0, %f0
+ std %f2, [%o1 + 0x00]
+ std %f0, [%o1 + 0x08]
+ subcc %o2, 0x10, %o2
+ bne,pt %icc, 1b
+ add %o1, 0x10, %o1
+ retl
+ nop
+ENDPROC(camellia_sparc64_ecb_crypt_3_grand_rounds)
+
+ .align 32
+ENTRY(camellia_sparc64_ecb_crypt_4_grand_rounds)
+ /* %o0=input, %o1=output, %o2=len, %o3=key */
+1: ldd [%o0 + 0x00], %f0
+ ldd [%o0 + 0x08], %f2
+ add %o0, 0x10, %o0
+ fxor %f4, %f0, %f0
+ fxor %f6, %f2, %f2
+ CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2)
+ ldd [%o3 + 0xd0], %f8
+ ldd [%o3 + 0xd8], %f10
+ ldd [%o3 + 0xe0], %f12
+ ldd [%o3 + 0xe8], %f14
+ ldd [%o3 + 0xf0], %f16
+ ldd [%o3 + 0xf8], %f18
+ ldd [%o3 + 0x100], %f20
+ ldd [%o3 + 0x108], %f22
+ CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2)
+ CAMELLIA_6ROUNDS_FL_FLI(40, 0, 2)
+ CAMELLIA_F(8, 2, 0, 2)
+ CAMELLIA_F(10, 0, 2, 0)
+ ldd [%o3 + 0x10], %f8
+ ldd [%o3 + 0x18], %f10
+ CAMELLIA_F(12, 2, 0, 2)
+ CAMELLIA_F(14, 0, 2, 0)
+ ldd [%o3 + 0x20], %f12
+ ldd [%o3 + 0x28], %f14
+ CAMELLIA_F(16, 2, 0, 2)
+ CAMELLIA_F(18, 0, 2, 0)
+ ldd [%o3 + 0x30], %f16
+ ldd [%o3 + 0x38], %f18
+ fxor %f20, %f2, %f2
+ fxor %f22, %f0, %f0
+ ldd [%o3 + 0x40], %f20
+ ldd [%o3 + 0x48], %f22
+ std %f2, [%o1 + 0x00]
+ std %f0, [%o1 + 0x08]
+ subcc %o2, 0x10, %o2
+ bne,pt %icc, 1b
+ add %o1, 0x10, %o1
+ retl
+ nop
+ENDPROC(camellia_sparc64_ecb_crypt_4_grand_rounds)
+
+ .align 32
+ENTRY(camellia_sparc64_cbc_encrypt_3_grand_rounds)
+ /* %o0=input, %o1=output, %o2=len, %o3=key, %o4=IV */
+ ldd [%o4 + 0x00], %f60
+ ldd [%o4 + 0x08], %f62
+1: ldd [%o0 + 0x00], %f0
+ ldd [%o0 + 0x08], %f2
+ add %o0, 0x10, %o0
+ fxor %f60, %f0, %f0
+ fxor %f62, %f2, %f2
+ fxor %f4, %f0, %f0
+ fxor %f6, %f2, %f2
+ CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2)
+ CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2)
+ CAMELLIA_6ROUNDS(40, 0, 2)
+ fxor %f52, %f2, %f60
+ fxor %f54, %f0, %f62
+ std %f60, [%o1 + 0x00]
+ std %f62, [%o1 + 0x08]
+ subcc %o2, 0x10, %o2
+ bne,pt %icc, 1b
+ add %o1, 0x10, %o1
+ std %f60, [%o4 + 0x00]
+ retl
+ std %f62, [%o4 + 0x08]
+ENDPROC(camellia_sparc64_cbc_encrypt_3_grand_rounds)
+
+ .align 32
+ENTRY(camellia_sparc64_cbc_encrypt_4_grand_rounds)
+ /* %o0=input, %o1=output, %o2=len, %o3=key, %o4=IV */
+ ldd [%o4 + 0x00], %f60
+ ldd [%o4 + 0x08], %f62
+1: ldd [%o0 + 0x00], %f0
+ ldd [%o0 + 0x08], %f2
+ add %o0, 0x10, %o0
+ fxor %f60, %f0, %f0
+ fxor %f62, %f2, %f2
+ fxor %f4, %f0, %f0
+ fxor %f6, %f2, %f2
+ CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2)
+ ldd [%o3 + 0xd0], %f8
+ ldd [%o3 + 0xd8], %f10
+ ldd [%o3 + 0xe0], %f12
+ ldd [%o3 + 0xe8], %f14
+ ldd [%o3 + 0xf0], %f16
+ ldd [%o3 + 0xf8], %f18
+ ldd [%o3 + 0x100], %f20
+ ldd [%o3 + 0x108], %f22
+ CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2)
+ CAMELLIA_6ROUNDS_FL_FLI(40, 0, 2)
+ CAMELLIA_F(8, 2, 0, 2)
+ CAMELLIA_F(10, 0, 2, 0)
+ ldd [%o3 + 0x10], %f8
+ ldd [%o3 + 0x18], %f10
+ CAMELLIA_F(12, 2, 0, 2)
+ CAMELLIA_F(14, 0, 2, 0)
+ ldd [%o3 + 0x20], %f12
+ ldd [%o3 + 0x28], %f14
+ CAMELLIA_F(16, 2, 0, 2)
+ CAMELLIA_F(18, 0, 2, 0)
+ ldd [%o3 + 0x30], %f16
+ ldd [%o3 + 0x38], %f18
+ fxor %f20, %f2, %f60
+ fxor %f22, %f0, %f62
+ ldd [%o3 + 0x40], %f20
+ ldd [%o3 + 0x48], %f22
+ std %f60, [%o1 + 0x00]
+ std %f62, [%o1 + 0x08]
+ subcc %o2, 0x10, %o2
+ bne,pt %icc, 1b
+ add %o1, 0x10, %o1
+ std %f60, [%o4 + 0x00]
+ retl
+ std %f62, [%o4 + 0x08]
+ENDPROC(camellia_sparc64_cbc_encrypt_4_grand_rounds)
+
+ .align 32
+ENTRY(camellia_sparc64_cbc_decrypt_3_grand_rounds)
+ /* %o0=input, %o1=output, %o2=len, %o3=key, %o4=IV */
+ ldd [%o4 + 0x00], %f60
+ ldd [%o4 + 0x08], %f62
+1: ldd [%o0 + 0x00], %f56
+ ldd [%o0 + 0x08], %f58
+ add %o0, 0x10, %o0
+ fxor %f4, %f56, %f0
+ fxor %f6, %f58, %f2
+ CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2)
+ CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2)
+ CAMELLIA_6ROUNDS(40, 0, 2)
+ fxor %f52, %f2, %f2
+ fxor %f54, %f0, %f0
+ fxor %f60, %f2, %f2
+ fxor %f62, %f0, %f0
+ fsrc2 %f56, %f60
+ fsrc2 %f58, %f62
+ std %f2, [%o1 + 0x00]
+ std %f0, [%o1 + 0x08]
+ subcc %o2, 0x10, %o2
+ bne,pt %icc, 1b
+ add %o1, 0x10, %o1
+ std %f60, [%o4 + 0x00]
+ retl
+ std %f62, [%o4 + 0x08]
+ENDPROC(camellia_sparc64_cbc_decrypt_3_grand_rounds)
+
+ .align 32
+ENTRY(camellia_sparc64_cbc_decrypt_4_grand_rounds)
+ /* %o0=input, %o1=output, %o2=len, %o3=key, %o4=IV */
+ ldd [%o4 + 0x00], %f60
+ ldd [%o4 + 0x08], %f62
+1: ldd [%o0 + 0x00], %f56
+ ldd [%o0 + 0x08], %f58
+ add %o0, 0x10, %o0
+ fxor %f4, %f56, %f0
+ fxor %f6, %f58, %f2
+ CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2)
+ ldd [%o3 + 0xd0], %f8
+ ldd [%o3 + 0xd8], %f10
+ ldd [%o3 + 0xe0], %f12
+ ldd [%o3 + 0xe8], %f14
+ ldd [%o3 + 0xf0], %f16
+ ldd [%o3 + 0xf8], %f18
+ ldd [%o3 + 0x100], %f20
+ ldd [%o3 + 0x108], %f22
+ CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2)
+ CAMELLIA_6ROUNDS_FL_FLI(40, 0, 2)
+ CAMELLIA_F(8, 2, 0, 2)
+ CAMELLIA_F(10, 0, 2, 0)
+ ldd [%o3 + 0x10], %f8
+ ldd [%o3 + 0x18], %f10
+ CAMELLIA_F(12, 2, 0, 2)
+ CAMELLIA_F(14, 0, 2, 0)
+ ldd [%o3 + 0x20], %f12
+ ldd [%o3 + 0x28], %f14
+ CAMELLIA_F(16, 2, 0, 2)
+ CAMELLIA_F(18, 0, 2, 0)
+ ldd [%o3 + 0x30], %f16
+ ldd [%o3 + 0x38], %f18
+ fxor %f20, %f2, %f2
+ fxor %f22, %f0, %f0
+ ldd [%o3 + 0x40], %f20
+ ldd [%o3 + 0x48], %f22
+ fxor %f60, %f2, %f2
+ fxor %f62, %f0, %f0
+ fsrc2 %f56, %f60
+ fsrc2 %f58, %f62
+ std %f2, [%o1 + 0x00]
+ std %f0, [%o1 + 0x08]
+ subcc %o2, 0x10, %o2
+ bne,pt %icc, 1b
+ add %o1, 0x10, %o1
+ std %f60, [%o4 + 0x00]
+ retl
+ std %f62, [%o4 + 0x08]
+ENDPROC(camellia_sparc64_cbc_decrypt_4_grand_rounds)
diff --git a/arch/sparc/crypto/camellia_glue.c b/arch/sparc/crypto/camellia_glue.c
new file mode 100644
index 000000000..e7a1e1c42
--- /dev/null
+++ b/arch/sparc/crypto/camellia_glue.c
@@ -0,0 +1,293 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Glue code for CAMELLIA encryption optimized for sparc64 crypto opcodes.
+ *
+ * Copyright (C) 2012 David S. Miller <davem@davemloft.net>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/crypto.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/mm.h>
+#include <linux/types.h>
+#include <crypto/algapi.h>
+#include <crypto/internal/skcipher.h>
+
+#include <asm/fpumacro.h>
+#include <asm/opcodes.h>
+#include <asm/pstate.h>
+#include <asm/elf.h>
+
+#define CAMELLIA_MIN_KEY_SIZE 16
+#define CAMELLIA_MAX_KEY_SIZE 32
+#define CAMELLIA_BLOCK_SIZE 16
+#define CAMELLIA_TABLE_BYTE_LEN 272
+
+struct camellia_sparc64_ctx {
+ u64 encrypt_key[CAMELLIA_TABLE_BYTE_LEN / sizeof(u64)];
+ u64 decrypt_key[CAMELLIA_TABLE_BYTE_LEN / sizeof(u64)];
+ int key_len;
+};
+
+extern void camellia_sparc64_key_expand(const u32 *in_key, u64 *encrypt_key,
+ unsigned int key_len, u64 *decrypt_key);
+
+static int camellia_set_key(struct crypto_tfm *tfm, const u8 *_in_key,
+ unsigned int key_len)
+{
+ struct camellia_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
+ const u32 *in_key = (const u32 *) _in_key;
+
+ if (key_len != 16 && key_len != 24 && key_len != 32)
+ return -EINVAL;
+
+ ctx->key_len = key_len;
+
+ camellia_sparc64_key_expand(in_key, &ctx->encrypt_key[0],
+ key_len, &ctx->decrypt_key[0]);
+ return 0;
+}
+
+static int camellia_set_key_skcipher(struct crypto_skcipher *tfm,
+ const u8 *in_key, unsigned int key_len)
+{
+ return camellia_set_key(crypto_skcipher_tfm(tfm), in_key, key_len);
+}
+
+extern void camellia_sparc64_crypt(const u64 *key, const u32 *input,
+ u32 *output, unsigned int key_len);
+
+static void camellia_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
+{
+ struct camellia_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ camellia_sparc64_crypt(&ctx->encrypt_key[0],
+ (const u32 *) src,
+ (u32 *) dst, ctx->key_len);
+}
+
+static void camellia_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
+{
+ struct camellia_sparc64_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ camellia_sparc64_crypt(&ctx->decrypt_key[0],
+ (const u32 *) src,
+ (u32 *) dst, ctx->key_len);
+}
+
+extern void camellia_sparc64_load_keys(const u64 *key, unsigned int key_len);
+
+typedef void ecb_crypt_op(const u64 *input, u64 *output, unsigned int len,
+ const u64 *key);
+
+extern ecb_crypt_op camellia_sparc64_ecb_crypt_3_grand_rounds;
+extern ecb_crypt_op camellia_sparc64_ecb_crypt_4_grand_rounds;
+
+static int __ecb_crypt(struct skcipher_request *req, bool encrypt)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ const struct camellia_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ ecb_crypt_op *op;
+ const u64 *key;
+ unsigned int nbytes;
+ int err;
+
+ op = camellia_sparc64_ecb_crypt_3_grand_rounds;
+ if (ctx->key_len != 16)
+ op = camellia_sparc64_ecb_crypt_4_grand_rounds;
+
+ err = skcipher_walk_virt(&walk, req, true);
+ if (err)
+ return err;
+
+ if (encrypt)
+ key = &ctx->encrypt_key[0];
+ else
+ key = &ctx->decrypt_key[0];
+ camellia_sparc64_load_keys(key, ctx->key_len);
+ while ((nbytes = walk.nbytes) != 0) {
+ op(walk.src.virt.addr, walk.dst.virt.addr,
+ round_down(nbytes, CAMELLIA_BLOCK_SIZE), key);
+ err = skcipher_walk_done(&walk, nbytes % CAMELLIA_BLOCK_SIZE);
+ }
+ fprs_write(0);
+ return err;
+}
+
+static int ecb_encrypt(struct skcipher_request *req)
+{
+ return __ecb_crypt(req, true);
+}
+
+static int ecb_decrypt(struct skcipher_request *req)
+{
+ return __ecb_crypt(req, false);
+}
+
+typedef void cbc_crypt_op(const u64 *input, u64 *output, unsigned int len,
+ const u64 *key, u64 *iv);
+
+extern cbc_crypt_op camellia_sparc64_cbc_encrypt_3_grand_rounds;
+extern cbc_crypt_op camellia_sparc64_cbc_encrypt_4_grand_rounds;
+extern cbc_crypt_op camellia_sparc64_cbc_decrypt_3_grand_rounds;
+extern cbc_crypt_op camellia_sparc64_cbc_decrypt_4_grand_rounds;
+
+static int cbc_encrypt(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ const struct camellia_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ cbc_crypt_op *op;
+ const u64 *key;
+ unsigned int nbytes;
+ int err;
+
+ op = camellia_sparc64_cbc_encrypt_3_grand_rounds;
+ if (ctx->key_len != 16)
+ op = camellia_sparc64_cbc_encrypt_4_grand_rounds;
+
+ err = skcipher_walk_virt(&walk, req, true);
+ if (err)
+ return err;
+
+ key = &ctx->encrypt_key[0];
+ camellia_sparc64_load_keys(key, ctx->key_len);
+ while ((nbytes = walk.nbytes) != 0) {
+ op(walk.src.virt.addr, walk.dst.virt.addr,
+ round_down(nbytes, CAMELLIA_BLOCK_SIZE), key, walk.iv);
+ err = skcipher_walk_done(&walk, nbytes % CAMELLIA_BLOCK_SIZE);
+ }
+ fprs_write(0);
+ return err;
+}
+
+static int cbc_decrypt(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ const struct camellia_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ cbc_crypt_op *op;
+ const u64 *key;
+ unsigned int nbytes;
+ int err;
+
+ op = camellia_sparc64_cbc_decrypt_3_grand_rounds;
+ if (ctx->key_len != 16)
+ op = camellia_sparc64_cbc_decrypt_4_grand_rounds;
+
+ err = skcipher_walk_virt(&walk, req, true);
+ if (err)
+ return err;
+
+ key = &ctx->decrypt_key[0];
+ camellia_sparc64_load_keys(key, ctx->key_len);
+ while ((nbytes = walk.nbytes) != 0) {
+ op(walk.src.virt.addr, walk.dst.virt.addr,
+ round_down(nbytes, CAMELLIA_BLOCK_SIZE), key, walk.iv);
+ err = skcipher_walk_done(&walk, nbytes % CAMELLIA_BLOCK_SIZE);
+ }
+ fprs_write(0);
+ return err;
+}
+
+static struct crypto_alg cipher_alg = {
+ .cra_name = "camellia",
+ .cra_driver_name = "camellia-sparc64",
+ .cra_priority = SPARC_CR_OPCODE_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
+ .cra_blocksize = CAMELLIA_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct camellia_sparc64_ctx),
+ .cra_alignmask = 3,
+ .cra_module = THIS_MODULE,
+ .cra_u = {
+ .cipher = {
+ .cia_min_keysize = CAMELLIA_MIN_KEY_SIZE,
+ .cia_max_keysize = CAMELLIA_MAX_KEY_SIZE,
+ .cia_setkey = camellia_set_key,
+ .cia_encrypt = camellia_encrypt,
+ .cia_decrypt = camellia_decrypt
+ }
+ }
+};
+
+static struct skcipher_alg skcipher_algs[] = {
+ {
+ .base.cra_name = "ecb(camellia)",
+ .base.cra_driver_name = "ecb-camellia-sparc64",
+ .base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
+ .base.cra_blocksize = CAMELLIA_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct camellia_sparc64_ctx),
+ .base.cra_alignmask = 7,
+ .base.cra_module = THIS_MODULE,
+ .min_keysize = CAMELLIA_MIN_KEY_SIZE,
+ .max_keysize = CAMELLIA_MAX_KEY_SIZE,
+ .setkey = camellia_set_key_skcipher,
+ .encrypt = ecb_encrypt,
+ .decrypt = ecb_decrypt,
+ }, {
+ .base.cra_name = "cbc(camellia)",
+ .base.cra_driver_name = "cbc-camellia-sparc64",
+ .base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
+ .base.cra_blocksize = CAMELLIA_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct camellia_sparc64_ctx),
+ .base.cra_alignmask = 7,
+ .base.cra_module = THIS_MODULE,
+ .min_keysize = CAMELLIA_MIN_KEY_SIZE,
+ .max_keysize = CAMELLIA_MAX_KEY_SIZE,
+ .ivsize = CAMELLIA_BLOCK_SIZE,
+ .setkey = camellia_set_key_skcipher,
+ .encrypt = cbc_encrypt,
+ .decrypt = cbc_decrypt,
+ }
+};
+
+static bool __init sparc64_has_camellia_opcode(void)
+{
+ unsigned long cfr;
+
+ if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
+ return false;
+
+ __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
+ if (!(cfr & CFR_CAMELLIA))
+ return false;
+
+ return true;
+}
+
+static int __init camellia_sparc64_mod_init(void)
+{
+ int err;
+
+ if (!sparc64_has_camellia_opcode()) {
+ pr_info("sparc64 camellia opcodes not available.\n");
+ return -ENODEV;
+ }
+ pr_info("Using sparc64 camellia opcodes optimized CAMELLIA implementation\n");
+ err = crypto_register_alg(&cipher_alg);
+ if (err)
+ return err;
+ err = crypto_register_skciphers(skcipher_algs,
+ ARRAY_SIZE(skcipher_algs));
+ if (err)
+ crypto_unregister_alg(&cipher_alg);
+ return err;
+}
+
+static void __exit camellia_sparc64_mod_fini(void)
+{
+ crypto_unregister_alg(&cipher_alg);
+ crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
+}
+
+module_init(camellia_sparc64_mod_init);
+module_exit(camellia_sparc64_mod_fini);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Camellia Cipher Algorithm, sparc64 camellia opcode accelerated");
+
+MODULE_ALIAS_CRYPTO("camellia");
+
+#include "crop_devid.c"
diff --git a/arch/sparc/crypto/crop_devid.c b/arch/sparc/crypto/crop_devid.c
new file mode 100644
index 000000000..79a059829
--- /dev/null
+++ b/arch/sparc/crypto/crop_devid.c
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/device-id/of.h>
+#include <linux/module.h>
+
+/* This is a dummy device table linked into all of the crypto
+ * opcode drivers. It serves to trigger the module autoloading
+ * mechanisms in userspace which scan the OF device tree and
+ * load any modules which have device table entries that
+ * match OF device nodes.
+ */
+static const struct of_device_id crypto_opcode_match[] = {
+ { .name = "cpu", .compatible = "sun4v", },
+ {},
+};
+MODULE_DEVICE_TABLE(of, crypto_opcode_match);
generated by cgit v1.3.1 (git 2.54.0) at 2026-10-26 20:00:42 +0000