x86/kprobes: Fix crash when probing CS CALL instructionsgrafted

When using eBPF to probe CS CALL instructions within a function,
a crash can be triggered.

The eBPF tool probes offset 257 of the __hrtimer_run_queues()
function:

<__hrtimer_run_queues+249>:  nopl   0x0(%rax,%rax,1)
<__hrtimer_run_queues+254>:  mov    %r14,%rdi
<__hrtimer_run_queues+257>:  cs call <__x86_indirect_thunk_r12>
<__hrtimer_run_queues+263>:  mov    %eax,%r12d
<__hrtimer_run_queues+266>:  xchg   %ax,%ax
<__hrtimer_run_queues+268>:  mov    %r13,%rdi

Which triggers this crash:

  BUG: unable to handle page fault for address: 00000000000f41c9
  #PF: supervisor write access in kernel mode
  #PF: error_code(0x0002) - not-present page
  PGD 0 P4D 0
  Oops: 0002 [#1] SMP NOPTI
  CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P
  RIP: 0010:__hrtimer_run_queues+0x106/0x230

Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is
at the 6th byte of the above CS CALL instruction. Since the CS CALL
instruction occupies 6 bytes, the exception occurred in the middle of that
call instruction.

The root cause is that when using eBPF tools to probe in the middle of a
function, a kprobe with INT3 is used as the underlying implementation.

During single-step emulation of the original CALL instruction,
int3_emulate_call() assumes that the probed CALL instruction is 5 bytes
long. However, the actual CS-prefixed CALL instruction occupies 6 bytes,
so it constructs an incorrect exception return address. When the CPU
returns from the kprobe handler, the next instruction to be executed is at
the address of the last byte of that CS CALL instruction. Coincidentally,
starting from that address, the CPU fetches and decodes a completely
different instruction, which ultimately triggers a kernel crash.

Fix the issue by using the actual instruction length obtained from
the instruction decoder when constructing the exception return
address, rather than relying on the hardcoded CALL_INSN_SIZE macro.

[ mingo: Refined the changelog ]

Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step")
Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Jinke Han <jinkehan@didiglobal.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Yafang Shao <laoar.shao@gmail.com>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000

1 files changed, 310 insertions, 0 deletions

diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
new file mode 100644
index 000000000..56f308dd0
--- /dev/null
+++ b/lib/crypto/Kconfig
@@ -0,0 +1,310 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config CRYPTO_HASH_INFO
+	bool
+
+config CRYPTO_LIB_UTILS
+	tristate
+
+config CRYPTO_LIB_AES
+	tristate
+	# Select dependencies of modes that are part of libaes.
+	select CRYPTO_LIB_GF128HASH if CRYPTO_LIB_AES_GCM != n
+	select CRYPTO_LIB_UTILS
+
+config CRYPTO_LIB_AES_ARCH
+	bool
+	depends on CRYPTO_LIB_AES && !UML && !KMSAN
+	default y if ARM
+	default y if ARM64
+	default y if PPC && (SPE || (PPC64 && VSX))
+	default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
+		     RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
+	default y if S390
+	default y if SPARC64
+	default y if X86
+
+config CRYPTO_LIB_AESCFB
+	tristate
+	select CRYPTO_LIB_AES
+	select CRYPTO_LIB_UTILS
+
+config CRYPTO_LIB_AES_CBC
+	tristate
+	select CRYPTO_LIB_AES
+	help
+	  The AES-CBC and AES-CBC-CTS library functions.
+
+config CRYPTO_LIB_AES_CBC_MACS
+	tristate
+	select CRYPTO_LIB_AES
+	help
+	  The AES-CMAC, AES-XCBC-MAC, and AES-CBC-MAC library functions.  Select
+	  this if your module uses any of the functions from
+	  <crypto/aes-cbc-macs.h>.
+
+config CRYPTO_LIB_AES_CCM
+	tristate
+	select CRYPTO_LIB_AES
+	select CRYPTO_LIB_AES_CBC_MACS
+	select CRYPTO_LIB_AES_CTR
+	help
+	  The AES-CCM library functions.
+
+config CRYPTO_LIB_AES_CTR
+	tristate
+	select CRYPTO_LIB_AES
+	help
+	  The AES-CTR and AES-XCTR library functions.
+
+config CRYPTO_LIB_AES_ECB
+	tristate
+	select CRYPTO_LIB_AES
+	help
+	  The AES-ECB library functions.
+
+config CRYPTO_LIB_AES_GCM
+	tristate
+	select CRYPTO_LIB_AES
+	select CRYPTO_LIB_AES_CTR
+	help
+	  The AES-GCM library functions.
+
+config CRYPTO_LIB_AES_XTS
+	tristate
+	select CRYPTO_LIB_AES
+	help
+	  The AES-XTS library functions.
+
+config CRYPTO_LIB_ARC4
+	tristate
+
+config CRYPTO_LIB_GF128MUL
+	tristate
+
+config CRYPTO_LIB_BLAKE2B
+	tristate
+	help
+	  The BLAKE2b library functions.  Select this if your module uses any of
+	  the functions from <crypto/blake2b.h>.
+
+config CRYPTO_LIB_BLAKE2B_ARCH
+	bool
+	depends on CRYPTO_LIB_BLAKE2B && !UML
+	default y if ARM && KERNEL_MODE_NEON
+
+# BLAKE2s support is always built-in, so there's no CRYPTO_LIB_BLAKE2S option.
+
+config CRYPTO_LIB_BLAKE2S_ARCH
+	bool
+	depends on !UML
+	default y if ARM
+	default y if X86_64
+
+config CRYPTO_LIB_CHACHA
+	tristate
+	select CRYPTO_LIB_UTILS
+	help
+	  Enable the ChaCha library interface.  Select this if your module uses
+	  chacha_crypt() or hchacha_block().
+
+config CRYPTO_LIB_CHACHA_ARCH
+	bool
+	depends on CRYPTO_LIB_CHACHA && !UML && !KMSAN
+	default y if ARM
+	default y if ARM64
+	default y if MIPS && CPU_MIPS32_R2
+	default y if PPC64 && CPU_LITTLE_ENDIAN && VSX
+	default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
+		     RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
+	default y if S390
+	default y if X86_64
+
+config CRYPTO_LIB_CURVE25519
+	tristate
+	select CRYPTO_LIB_UTILS
+	help
+	  The Curve25519 library functions.  Select this if your module uses any
+	  of the functions from <crypto/curve25519.h>.
+
+config CRYPTO_LIB_CURVE25519_ARCH
+	bool
+	depends on CRYPTO_LIB_CURVE25519 && !UML && !KMSAN
+	default y if ARM && KERNEL_MODE_NEON && !CPU_BIG_ENDIAN
+	default y if PPC64 && CPU_LITTLE_ENDIAN
+	default y if X86_64
+
+config CRYPTO_LIB_CURVE25519_GENERIC
+	bool
+	depends on CRYPTO_LIB_CURVE25519
+	default y if !CRYPTO_LIB_CURVE25519_ARCH || ARM || X86_64
+
+config CRYPTO_LIB_DES
+	tristate
+
+config CRYPTO_LIB_GF128HASH
+	tristate
+	help
+	  The GHASH and POLYVAL library functions.  Select this if your module
+	  uses any of the functions from <crypto/gf128hash.h>.
+
+config CRYPTO_LIB_GF128HASH_ARCH
+	bool
+	depends on CRYPTO_LIB_GF128HASH && !UML
+	default y if ARM && KERNEL_MODE_NEON
+	default y if ARM64
+	default y if PPC64 && VSX
+	default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
+		     RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
+	default y if S390
+	default y if X86_64
+
+config CRYPTO_LIB_MD5
+	tristate
+	help
+	  The MD5 and HMAC-MD5 library functions.  Select this if your module
+	  uses any of the functions from <crypto/md5.h>.
+
+config CRYPTO_LIB_MLDSA
+	tristate
+	select CRYPTO_LIB_SHA3
+	help
+	  The ML-DSA library functions.  Select this if your module uses any of
+	  the functions from <crypto/mldsa.h>.
+
+config CRYPTO_LIB_NH
+	tristate
+	help
+	  Implementation of the NH almost-universal hash function, specifically
+	  the variant of NH used in Adiantum.
+
+config CRYPTO_LIB_NH_ARCH
+	bool
+	depends on CRYPTO_LIB_NH && !UML && !KMSAN
+	default y if ARM && KERNEL_MODE_NEON
+	default y if ARM64
+	default y if X86_64
+
+config CRYPTO_LIB_POLY1305
+	tristate
+	help
+	  The Poly1305 library functions.  Select this if your module uses any
+	  of the functions from <crypto/poly1305.h>.
+
+config CRYPTO_LIB_POLY1305_ARCH
+	bool
+	depends on CRYPTO_LIB_POLY1305 && !UML && !KMSAN
+	default y if ARM
+	default y if ARM64
+	default y if MIPS
+	# The PPC64 code needs to be fixed to work in softirq context.
+	default y if PPC64 && CPU_LITTLE_ENDIAN && VSX && BROKEN
+	default y if RISCV
+	default y if X86_64
+
+# This symbol controls the inclusion of the Poly1305 generic code.  This differs
+# from most of the other algorithms, which handle the generic code
+# "automatically" via __maybe_unused.  This is needed so that the Adiantum code,
+# which calls the poly1305_core_*() functions directly, can enable them.
+config CRYPTO_LIB_POLY1305_GENERIC
+	bool
+	depends on CRYPTO_LIB_POLY1305
+	# Enable if there's no arch impl or the arch impl requires the generic
+	# impl as a fallback.  (Or if selected explicitly.)
+	default y if !CRYPTO_LIB_POLY1305_ARCH || PPC64
+
+config CRYPTO_LIB_POLY1305_RSIZE
+	int
+	default 2 if MIPS || RISCV
+	default 11 if X86_64
+	default 9 if ARM || ARM64
+	default 1
+
+config CRYPTO_LIB_CHACHA20POLY1305
+	tristate
+	select CRYPTO_LIB_CHACHA
+	select CRYPTO_LIB_POLY1305
+	select CRYPTO_LIB_UTILS
+
+config CRYPTO_LIB_SHA1
+	tristate
+	help
+	  The SHA-1 and HMAC-SHA1 library functions.  Select this if your module
+	  uses any of the functions from <crypto/sha1.h>.
+
+config CRYPTO_LIB_SHA1_ARCH
+	bool
+	depends on CRYPTO_LIB_SHA1 && !UML
+	default y if ARM
+	default y if ARM64
+	default y if MIPS && CPU_CAVIUM_OCTEON
+	default y if PPC
+	default y if S390
+	default y if SPARC64
+	default y if X86_64
+
+config CRYPTO_LIB_SHA256
+	tristate
+	help
+	  The SHA-224, SHA-256, HMAC-SHA224, and HMAC-SHA256 library functions.
+	  Select this if your module uses any of these functions from
+	  <crypto/sha2.h>.
+
+config CRYPTO_LIB_SHA256_ARCH
+	bool
+	depends on CRYPTO_LIB_SHA256 && !UML
+	default y if ARM && !CPU_V7M
+	default y if ARM64
+	default y if MIPS && CPU_CAVIUM_OCTEON
+	default y if PPC && SPE
+	default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
+		     RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
+	default y if S390
+	default y if SPARC64
+	default y if X86_64
+
+config CRYPTO_LIB_SHA512
+	tristate
+	help
+	  The SHA-384, SHA-512, HMAC-SHA384, and HMAC-SHA512 library functions.
+	  Select this if your module uses any of these functions from
+	  <crypto/sha2.h>.
+
+config CRYPTO_LIB_SHA512_ARCH
+	bool
+	depends on CRYPTO_LIB_SHA512 && !UML
+	default y if ARM && !CPU_V7M
+	default y if ARM64
+	default y if MIPS && CPU_CAVIUM_OCTEON
+	default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
+		     RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
+	default y if S390
+	default y if SPARC64
+	default y if X86_64
+
+config CRYPTO_LIB_SHA3
+	tristate
+	select CRYPTO_LIB_UTILS
+	help
+	  The SHA3 library functions.  Select this if your module uses any of
+	  the functions from <crypto/sha3.h>.
+
+config CRYPTO_LIB_SHA3_ARCH
+	bool
+	depends on CRYPTO_LIB_SHA3 && !UML
+	default y if ARM64
+	default y if S390
+
+config CRYPTO_LIB_SM3
+	tristate
+	help
+	  The SM3 library functions.  Select this if your module uses any of the
+	  functions from <crypto/sm3.h>.
+
+config CRYPTO_LIB_SM3_ARCH
+	bool
+	depends on CRYPTO_LIB_SM3 && !UML
+	default y if ARM64
+	default y if RISCV && 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
+		     RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
+	default y if X86_64