x86/kprobes: Fix crash when probing CS CALL instructionsgrafted

When using eBPF to probe CS CALL instructions within a function,
a crash can be triggered.

The eBPF tool probes offset 257 of the __hrtimer_run_queues()
function:

<__hrtimer_run_queues+249>:  nopl   0x0(%rax,%rax,1)
<__hrtimer_run_queues+254>:  mov    %r14,%rdi
<__hrtimer_run_queues+257>:  cs call <__x86_indirect_thunk_r12>
<__hrtimer_run_queues+263>:  mov    %eax,%r12d
<__hrtimer_run_queues+266>:  xchg   %ax,%ax
<__hrtimer_run_queues+268>:  mov    %r13,%rdi

Which triggers this crash:

  BUG: unable to handle page fault for address: 00000000000f41c9
  #PF: supervisor write access in kernel mode
  #PF: error_code(0x0002) - not-present page
  PGD 0 P4D 0
  Oops: 0002 [#1] SMP NOPTI
  CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P
  RIP: 0010:__hrtimer_run_queues+0x106/0x230

Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is
at the 6th byte of the above CS CALL instruction. Since the CS CALL
instruction occupies 6 bytes, the exception occurred in the middle of that
call instruction.

The root cause is that when using eBPF tools to probe in the middle of a
function, a kprobe with INT3 is used as the underlying implementation.

During single-step emulation of the original CALL instruction,
int3_emulate_call() assumes that the probed CALL instruction is 5 bytes
long. However, the actual CS-prefixed CALL instruction occupies 6 bytes,
so it constructs an incorrect exception return address. When the CPU
returns from the kprobe handler, the next instruction to be executed is at
the address of the last byte of that CS CALL instruction. Coincidentally,
starting from that address, the CPU fetches and decodes a completely
different instruction, which ultimately triggers a kernel crash.

Fix the issue by using the actual instruction length obtained from
the instruction decoder when constructing the exception return
address, rather than relying on the hardcoded CALL_INSN_SIZE macro.

[ mingo: Refined the changelog ]

Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step")
Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Jinke Han <jinkehan@didiglobal.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Yafang Shao <laoar.shao@gmail.com>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000

1 files changed, 346 insertions, 0 deletions

diff --git a/drivers/tty/serial/earlycon.c b/drivers/tty/serial/earlycon.c
new file mode 100644
index 000000000..ce740cdc7
--- /dev/null
+++ b/drivers/tty/serial/earlycon.c
@@ -0,0 +1,346 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2014 Linaro Ltd.
+ * Author: Rob Herring <robh@kernel.org>
+ *
+ * Based on 8250 earlycon:
+ * (c) Copyright 2004 Hewlett-Packard Development Company, L.P.
+ *	Bjorn Helgaas <bjorn.helgaas@hp.com>
+ */
+
+#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
+
+#include <linux/console.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/serial_core.h>
+#include <linux/sizes.h>
+#include <linux/of.h>
+#include <linux/of_fdt.h>
+#include <linux/acpi.h>
+
+#ifdef CONFIG_FIX_EARLYCON_MEM
+#include <asm/fixmap.h>
+#endif
+
+#include <asm/serial.h>
+
+static struct console early_con = {
+	.name =		"uart",		/* fixed up at earlycon registration */
+	.flags =	CON_PRINTBUFFER | CON_BOOT,
+	.index =	0,
+};
+
+static struct earlycon_device early_console_dev = {
+	.con = &early_con,
+};
+
+static void __iomem * __init earlycon_map(resource_size_t paddr, size_t size)
+{
+	void __iomem *base;
+#ifdef CONFIG_FIX_EARLYCON_MEM
+	set_fixmap_io(FIX_EARLYCON_MEM_BASE, paddr & PAGE_MASK);
+	base = (void __iomem *)__fix_to_virt(FIX_EARLYCON_MEM_BASE);
+	base += paddr & ~PAGE_MASK;
+#else
+	base = ioremap(paddr, size);
+#endif
+	if (!base)
+		pr_err("%s: Couldn't map %pa\n", __func__, &paddr);
+
+	return base;
+}
+
+static void __init earlycon_init(struct earlycon_device *device,
+				 const char *name)
+{
+	struct console *earlycon = device->con;
+	const char *s;
+	size_t len;
+
+	/* scan backwards from end of string for first non-numeral */
+	for (s = name + strlen(name);
+	     s > name && s[-1] >= '0' && s[-1] <= '9';
+	     s--)
+		;
+	if (*s)
+		earlycon->index = simple_strtoul(s, NULL, 10);
+	len = s - name;
+	strscpy(earlycon->name, name, min(len + 1, sizeof(earlycon->name)));
+	earlycon->data = &early_console_dev;
+}
+
+static void __init earlycon_print_info(struct earlycon_device *device)
+{
+	struct console *earlycon = device->con;
+	struct uart_port *port = &device->port;
+	char ioinfos[64];
+
+	uart_get_ioinfos(port, ioinfos, sizeof(ioinfos));
+
+	pr_info("%s%d%s (options '%s')\n", earlycon->name, earlycon->index,
+		ioinfos, device->options);
+}
+
+static int __init parse_options(struct earlycon_device *device, char *options)
+{
+	struct uart_port *port = &device->port;
+	int length;
+	resource_size_t addr;
+
+	if (uart_parse_earlycon(options, &port->iotype, &addr, &options))
+		return -EINVAL;
+
+	switch (port->iotype) {
+	case UPIO_MEM:
+		port->mapbase = addr;
+		break;
+	case UPIO_MEM16:
+		port->regshift = 1;
+		port->mapbase = addr;
+		break;
+	case UPIO_MEM32:
+	case UPIO_MEM32BE:
+		port->regshift = 2;
+		port->mapbase = addr;
+		break;
+	case UPIO_PORT:
+		port->iobase = addr;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (options) {
+		char *uartclk;
+
+		device->baud = simple_strtoul(options, NULL, 0);
+		uartclk = strchr(options, ',');
+		if (uartclk && kstrtouint(uartclk + 1, 0, &port->uartclk) < 0)
+			pr_warn("[%s] unsupported earlycon uart clkrate option\n",
+				options);
+		length = min(strcspn(options, " ") + 1,
+			     (size_t)(sizeof(device->options)));
+		strscpy(device->options, options, length);
+	}
+
+	return 0;
+}
+
+static int __init register_earlycon(char *buf, const struct earlycon_id *match)
+{
+	int err;
+	struct uart_port *port = &early_console_dev.port;
+
+	/* On parsing error, pass the options buf to the setup function */
+	if (buf && !parse_options(&early_console_dev, buf))
+		buf = NULL;
+
+	spin_lock_init(&port->lock);
+	if (!port->uartclk)
+		port->uartclk = BASE_BAUD * 16;
+	if (port->mapbase)
+		port->membase = earlycon_map(port->mapbase, 64);
+
+	earlycon_init(&early_console_dev, match->name);
+	err = match->setup(&early_console_dev, buf);
+	earlycon_print_info(&early_console_dev);
+	if (err < 0)
+		return err;
+	if (!early_console_dev.con->write)
+		return -ENODEV;
+
+	register_console(early_console_dev.con);
+	return 0;
+}
+
+/**
+ *	setup_earlycon - match and register earlycon console
+ *	@buf:	earlycon param string
+ *
+ *	Registers the earlycon console matching the earlycon specified
+ *	in the param string @buf. Acceptable param strings are of the form
+ *	   <name>,io|mmio|mmio32|mmio32be,<addr>,<options>
+ *	   <name>,0x<addr>,<options>
+ *	   <name>,<options>
+ *	   <name>
+ *
+ *	Only for the third form does the earlycon setup() method receive the
+ *	<options> string in the 'options' parameter; all other forms set
+ *	the parameter to NULL.
+ *
+ *	Returns 0 if an attempt to register the earlycon was made,
+ *	otherwise negative error code
+ */
+int __init setup_earlycon(char *buf)
+{
+	const struct earlycon_id *match;
+	bool empty_compatible = true;
+
+	if (!buf || !buf[0])
+		return -EINVAL;
+
+	if (console_is_registered(&early_con))
+		return -EALREADY;
+
+again:
+	for (match = __earlycon_table; match < __earlycon_table_end; match++) {
+		size_t len = strlen(match->name);
+
+		if (strncmp(buf, match->name, len))
+			continue;
+
+		/* prefer entries with empty compatible */
+		if (empty_compatible && *match->compatible)
+			continue;
+
+		if (buf[len]) {
+			if (buf[len] != ',')
+				continue;
+			buf += len + 1;
+		} else
+			buf = NULL;
+
+		return register_earlycon(buf, match);
+	}
+
+	if (empty_compatible) {
+		empty_compatible = false;
+		goto again;
+	}
+
+	return -ENOENT;
+}
+
+/*
+ * This defers the initialization of the early console until after ACPI has
+ * been initialized.
+ */
+bool earlycon_acpi_spcr_enable __initdata;
+
+/* early_param wrapper for setup_earlycon() */
+static int __init param_setup_earlycon(char *buf)
+{
+	int err;
+
+	/* Just 'earlycon' is a valid param for devicetree and ACPI SPCR. */
+	if (!buf || !buf[0]) {
+		if (IS_ENABLED(CONFIG_ACPI_SPCR_TABLE)) {
+			earlycon_acpi_spcr_enable = true;
+			return 0;
+		} else if (!buf) {
+			return early_init_dt_scan_chosen_stdout();
+		}
+	}
+
+	err = setup_earlycon(buf);
+	if (err == -ENOENT || err == -EALREADY)
+		return 0;
+	return err;
+}
+early_param("earlycon", param_setup_earlycon);
+
+/*
+ * The `console` parameter is overloaded. It's handled here as an early param
+ * and in `printk.c` as a late param. It's possible to specify an early
+ * `bootconsole` using `earlycon=uartXXXX` (handled above), or via
+ * the `console=uartXXX` alias. See the comment in `8250_early.c`.
+ */
+static int __init param_setup_earlycon_console_alias(char *buf)
+{
+	/*
+	 * A plain `console` parameter must not enable the SPCR `bootconsole`
+	 * like a plain `earlycon` does.
+	 *
+	 * A `console=` parameter that specifies an empty value is used to
+	 * disable the `console`, not the `earlycon` `bootconsole`. The
+	 * disabling of the `console` is handled by `printk.c`.
+	 */
+	if (!buf || !buf[0])
+		return 0;
+
+	return param_setup_earlycon(buf);
+}
+early_param("console", param_setup_earlycon_console_alias);
+
+#ifdef CONFIG_OF_EARLY_FLATTREE
+
+int __init of_setup_earlycon(const struct earlycon_id *match,
+			     unsigned long node,
+			     const char *options)
+{
+	int err;
+	struct uart_port *port = &early_console_dev.port;
+	const __be32 *val;
+	bool big_endian;
+	u64 addr;
+
+	if (console_is_registered(&early_con))
+		return -EALREADY;
+
+	spin_lock_init(&port->lock);
+	port->iotype = UPIO_MEM;
+	addr = of_flat_dt_translate_address(node);
+	if (addr == OF_BAD_ADDR) {
+		pr_warn("[%s] bad address\n", match->name);
+		return -ENXIO;
+	}
+	port->mapbase = addr;
+
+	val = of_get_flat_dt_prop(node, "reg-offset", NULL);
+	if (val)
+		port->mapbase += be32_to_cpu(*val);
+	port->membase = earlycon_map(port->mapbase, SZ_4K);
+
+	val = of_get_flat_dt_prop(node, "reg-shift", NULL);
+	if (val)
+		port->regshift = be32_to_cpu(*val);
+	big_endian = of_get_flat_dt_prop(node, "big-endian", NULL) != NULL ||
+		(IS_ENABLED(CONFIG_CPU_BIG_ENDIAN) &&
+		 of_get_flat_dt_prop(node, "native-endian", NULL) != NULL);
+	val = of_get_flat_dt_prop(node, "reg-io-width", NULL);
+	if (val) {
+		switch (be32_to_cpu(*val)) {
+		case 1:
+			port->iotype = UPIO_MEM;
+			break;
+		case 2:
+			port->iotype = UPIO_MEM16;
+			break;
+		case 4:
+			port->iotype = (big_endian) ? UPIO_MEM32BE : UPIO_MEM32;
+			break;
+		default:
+			pr_warn("[%s] unsupported reg-io-width\n", match->name);
+			return -EINVAL;
+		}
+	}
+
+	val = of_get_flat_dt_prop(node, "current-speed", NULL);
+	if (val)
+		early_console_dev.baud = be32_to_cpu(*val);
+
+	val = of_get_flat_dt_prop(node, "clock-frequency", NULL);
+	if (val)
+		port->uartclk = be32_to_cpu(*val);
+
+	if (options) {
+		early_console_dev.baud = simple_strtoul(options, NULL, 0);
+		strscpy(early_console_dev.options, options,
+			sizeof(early_console_dev.options));
+	}
+	earlycon_init(&early_console_dev, match->name);
+	err = match->setup(&early_console_dev, options);
+	earlycon_print_info(&early_console_dev);
+	if (err < 0)
+		return err;
+	if (!early_console_dev.con->write)
+		return -ENODEV;
+
+
+	register_console(early_console_dev.con);
+	return 0;
+}
+
+#endif /* CONFIG_OF_EARLY_FLATTREE */