x86/kprobes: Fix crash when probing CS CALL instructionsgrafted

When using eBPF to probe CS CALL instructions within a function,
a crash can be triggered.

The eBPF tool probes offset 257 of the __hrtimer_run_queues()
function:

<__hrtimer_run_queues+249>:  nopl   0x0(%rax,%rax,1)
<__hrtimer_run_queues+254>:  mov    %r14,%rdi
<__hrtimer_run_queues+257>:  cs call <__x86_indirect_thunk_r12>
<__hrtimer_run_queues+263>:  mov    %eax,%r12d
<__hrtimer_run_queues+266>:  xchg   %ax,%ax
<__hrtimer_run_queues+268>:  mov    %r13,%rdi

Which triggers this crash:

  BUG: unable to handle page fault for address: 00000000000f41c9
  #PF: supervisor write access in kernel mode
  #PF: error_code(0x0002) - not-present page
  PGD 0 P4D 0
  Oops: 0002 [#1] SMP NOPTI
  CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P
  RIP: 0010:__hrtimer_run_queues+0x106/0x230

Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is
at the 6th byte of the above CS CALL instruction. Since the CS CALL
instruction occupies 6 bytes, the exception occurred in the middle of that
call instruction.

The root cause is that when using eBPF tools to probe in the middle of a
function, a kprobe with INT3 is used as the underlying implementation.

During single-step emulation of the original CALL instruction,
int3_emulate_call() assumes that the probed CALL instruction is 5 bytes
long. However, the actual CS-prefixed CALL instruction occupies 6 bytes,
so it constructs an incorrect exception return address. When the CPU
returns from the kprobe handler, the next instruction to be executed is at
the address of the last byte of that CS CALL instruction. Coincidentally,
starting from that address, the CPU fetches and decodes a completely
different instruction, which ultimately triggers a kernel crash.

Fix the issue by using the actual instruction length obtained from
the instruction decoder when constructing the exception return
address, rather than relying on the hardcoded CALL_INSN_SIZE macro.

[ mingo: Refined the changelog ]

Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step")
Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Jinke Han <jinkehan@didiglobal.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Yafang Shao <laoar.shao@gmail.com>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000

1 files changed, 313 insertions, 0 deletions

diff --git a/lib/bug.c b/lib/bug.c
new file mode 100644
index 000000000..7c1c2c27f
--- /dev/null
+++ b/lib/bug.c
@@ -0,0 +1,313 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  Generic support for BUG()
+ *
+ *  This respects the following config options:
+ *
+ *  CONFIG_BUG - emit BUG traps.  Nothing happens without this.
+ *  CONFIG_GENERIC_BUG - enable this code.
+ *  CONFIG_GENERIC_BUG_RELATIVE_POINTERS - use 32-bit relative pointers for bug_addr and file
+ *  CONFIG_DEBUG_BUGVERBOSE - emit full file+line information for each BUG
+ *
+ *  CONFIG_BUG and CONFIG_DEBUG_BUGVERBOSE are potentially user-settable
+ *  (though they're generally always on).
+ *
+ *  CONFIG_GENERIC_BUG is set by each architecture using this code.
+ *
+ *  To use this, your architecture must:
+ *
+ *  1. Set up the config options:
+ *     - Enable CONFIG_GENERIC_BUG if CONFIG_BUG
+ *
+ *  2. Implement BUG (and optionally BUG_ON, WARN, WARN_ON)
+ *     - Define HAVE_ARCH_BUG
+ *     - Implement BUG() to generate a faulting instruction
+ *     - NOTE: struct bug_entry does not have "file" or "line" entries
+ *       when CONFIG_DEBUG_BUGVERBOSE is not enabled, so you must generate
+ *       the values accordingly.
+ *
+ *  3. Implement the trap
+ *     - In the illegal instruction trap handler (typically), verify
+ *       that the fault was in kernel mode, and call report_bug()
+ *     - report_bug() will return whether it was a false alarm, a warning,
+ *       or an actual bug.
+ *     - You must implement the is_valid_bugaddr(bugaddr) callback which
+ *       returns true if the eip is a real kernel address, and it points
+ *       to the expected BUG trap instruction.
+ *
+ *    Jeremy Fitzhardinge <jeremy@goop.org> 2006
+ */
+
+#define pr_fmt(fmt) fmt
+
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/bug.h>
+#include <linux/sched.h>
+#include <linux/rculist.h>
+#include <linux/ftrace.h>
+#include <linux/context_tracking.h>
+#include <kunit/test-bug.h>
+
+extern struct bug_entry __start___bug_table[], __stop___bug_table[];
+
+static inline unsigned long bug_addr(const struct bug_entry *bug)
+{
+#ifdef CONFIG_GENERIC_BUG_RELATIVE_POINTERS
+	return (unsigned long)&bug->bug_addr_disp + bug->bug_addr_disp;
+#else
+	return bug->bug_addr;
+#endif
+}
+
+#ifdef CONFIG_MODULES
+/* Updates are protected by module mutex */
+static LIST_HEAD(module_bug_list);
+
+static struct bug_entry *module_find_bug(unsigned long bugaddr)
+{
+	struct bug_entry *bug;
+	struct module *mod;
+
+	guard(rcu)();
+	list_for_each_entry_rcu(mod, &module_bug_list, bug_list) {
+		unsigned int i;
+
+		bug = mod->bug_table;
+		for (i = 0; i < mod->num_bugs; ++i, ++bug)
+			if (bugaddr == bug_addr(bug))
+				return bug;
+	}
+	return NULL;
+}
+
+void module_bug_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs,
+			 struct module *mod)
+{
+	char *secstrings;
+	unsigned int i;
+
+	mod->bug_table = NULL;
+	mod->num_bugs = 0;
+
+	/* Find the __bug_table section, if present */
+	secstrings = (char *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
+	for (i = 1; i < hdr->e_shnum; i++) {
+		if (strcmp(secstrings+sechdrs[i].sh_name, "__bug_table"))
+			continue;
+		mod->bug_table = (void *) sechdrs[i].sh_addr;
+		mod->num_bugs = sechdrs[i].sh_size / sizeof(struct bug_entry);
+		break;
+	}
+
+	/*
+	 * Strictly speaking this should have a spinlock to protect against
+	 * traversals, but since we only traverse on BUG()s, a spinlock
+	 * could potentially lead to deadlock and thus be counter-productive.
+	 * Thus, this uses RCU to safely manipulate the bug list, since BUG
+	 * must run in non-interruptive state.
+	 */
+	list_add_rcu(&mod->bug_list, &module_bug_list);
+}
+
+void module_bug_cleanup(struct module *mod)
+{
+	list_del_rcu(&mod->bug_list);
+}
+
+#else
+
+static inline struct bug_entry *module_find_bug(unsigned long bugaddr)
+{
+	return NULL;
+}
+#endif
+
+void bug_get_file_line(struct bug_entry *bug, const char **file,
+		       unsigned int *line)
+{
+#ifdef CONFIG_DEBUG_BUGVERBOSE
+#ifdef CONFIG_GENERIC_BUG_RELATIVE_POINTERS
+	*file = (const char *)&bug->file_disp + bug->file_disp;
+#else
+	*file = bug->file;
+#endif
+	*line = bug->line;
+#else
+	*file = NULL;
+	*line = 0;
+#endif
+}
+
+static const char *bug_get_format(struct bug_entry *bug)
+{
+	const char *format = NULL;
+#ifdef HAVE_ARCH_BUG_FORMAT
+#ifdef CONFIG_GENERIC_BUG_RELATIVE_POINTERS
+	/*
+	 * Allow an architecture to:
+	 *  - relative encode NULL (difficult vs KASLR);
+	 *  - use a literal 0 (there are no valid objects inside
+	 *    the __bug_table itself to refer to after all);
+	 *  - use an empty string.
+	 */
+	if (bug->format_disp)
+		format = (const char *)&bug->format_disp + bug->format_disp;
+	if (format && format[0] == '\0')
+		format = NULL;
+#else
+	format = bug->format;
+#endif
+#endif
+	return format;
+}
+
+struct bug_entry *find_bug(unsigned long bugaddr)
+{
+	struct bug_entry *bug;
+
+	for (bug = __start___bug_table; bug < __stop___bug_table; ++bug)
+		if (bugaddr == bug_addr(bug))
+			return bug;
+
+	return module_find_bug(bugaddr);
+}
+
+static __printf(1, 0)
+void __warn_printf(const char *fmt, struct pt_regs *regs)
+{
+	if (!fmt)
+		return;
+
+#ifdef HAVE_ARCH_BUG_FORMAT_ARGS
+	if (regs) {
+		struct arch_va_list _args;
+		va_list *args = __warn_args(&_args, regs);
+
+		if (args) {
+			vprintk(fmt, *args);
+			return;
+		}
+	}
+#endif
+
+	pr_warn("%s", fmt);
+}
+
+static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long bugaddr, struct pt_regs *regs)
+{
+	bool warning, once, done, no_cut, has_args;
+	const char *file, *fmt;
+	unsigned int line;
+
+	if (!bug) {
+		if (!is_valid_bugaddr(bugaddr))
+			return BUG_TRAP_TYPE_NONE;
+
+		bug = find_bug(bugaddr);
+		if (!bug)
+			return BUG_TRAP_TYPE_NONE;
+	}
+
+	bug_get_file_line(bug, &file, &line);
+	fmt = bug_get_format(bug);
+
+	warning  = bug->flags & BUGFLAG_WARNING;
+	once     = bug->flags & BUGFLAG_ONCE;
+	done     = bug->flags & BUGFLAG_DONE;
+	no_cut   = bug->flags & BUGFLAG_NO_CUT_HERE;
+	has_args = bug->flags & BUGFLAG_ARGS;
+
+	/*
+	 * Before the once logic so suppressed warnings do not consume
+	 * the single-fire budget of WARN_ON_ONCE().
+	 */
+	if (warning && kunit_is_suppressed_warning(true))
+		return BUG_TRAP_TYPE_WARN;
+
+	disable_trace_on_warning();
+
+	if (warning && once) {
+		if (done)
+			return BUG_TRAP_TYPE_WARN;
+
+		/*
+		 * Since this is the only store, concurrency is not an issue.
+		 */
+		bug->flags |= BUGFLAG_DONE;
+	}
+
+	/*
+	 * BUG() and WARN_ON() families don't print a custom debug message
+	 * before triggering the exception handler, so we must add the
+	 * "cut here" line now. WARN() issues its own "cut here" before the
+	 * extra debugging message it writes before triggering the handler.
+	 */
+	if (!no_cut) {
+		pr_info(CUT_HERE);
+		__warn_printf(fmt, has_args ? regs : NULL);
+	}
+
+	if (warning) {
+		/* this is a WARN_ON rather than BUG/BUG_ON */
+		__warn(file, line, (void *)bugaddr, BUG_GET_TAINT(bug), regs,
+		       NULL);
+		return BUG_TRAP_TYPE_WARN;
+	}
+
+	if (file)
+		pr_crit("kernel BUG at %s:%u!\n", file, line);
+	else
+		pr_crit("kernel BUG at %pB [verbose debug info unavailable]\n",
+			(void *)bugaddr);
+
+	return BUG_TRAP_TYPE_BUG;
+}
+
+enum bug_trap_type report_bug_entry(struct bug_entry *bug, struct pt_regs *regs)
+{
+	enum bug_trap_type ret;
+	bool rcu;
+
+	rcu = warn_rcu_enter();
+	ret = __report_bug(bug, bug_addr(bug), regs);
+	warn_rcu_exit(rcu);
+
+	return ret;
+}
+
+enum bug_trap_type report_bug(unsigned long bugaddr, struct pt_regs *regs)
+{
+	enum bug_trap_type ret;
+	bool rcu;
+
+	rcu = warn_rcu_enter();
+	ret = __report_bug(NULL, bugaddr, regs);
+	warn_rcu_exit(rcu);
+
+	return ret;
+}
+
+static void clear_once_table(struct bug_entry *start, struct bug_entry *end)
+{
+	struct bug_entry *bug;
+
+	for (bug = start; bug < end; bug++)
+		bug->flags &= ~BUGFLAG_DONE;
+}
+
+void generic_bug_clear_once(void)
+{
+#ifdef CONFIG_MODULES
+	struct module *mod;
+
+	scoped_guard(rcu) {
+		list_for_each_entry_rcu(mod, &module_bug_list, bug_list)
+			clear_once_table(mod->bug_table,
+					 mod->bug_table + mod->num_bugs);
+	}
+#endif
+
+	clear_once_table(__start___bug_table, __stop___bug_table);
+}