x86/kprobes: Fix crash when probing CS CALL instructionsgrafted

When using eBPF to probe CS CALL instructions within a function,
a crash can be triggered.

The eBPF tool probes offset 257 of the __hrtimer_run_queues()
function:

<__hrtimer_run_queues+249>:  nopl   0x0(%rax,%rax,1)
<__hrtimer_run_queues+254>:  mov    %r14,%rdi
<__hrtimer_run_queues+257>:  cs call <__x86_indirect_thunk_r12>
<__hrtimer_run_queues+263>:  mov    %eax,%r12d
<__hrtimer_run_queues+266>:  xchg   %ax,%ax
<__hrtimer_run_queues+268>:  mov    %r13,%rdi

Which triggers this crash:

  BUG: unable to handle page fault for address: 00000000000f41c9
  #PF: supervisor write access in kernel mode
  #PF: error_code(0x0002) - not-present page
  PGD 0 P4D 0
  Oops: 0002 [#1] SMP NOPTI
  CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P
  RIP: 0010:__hrtimer_run_queues+0x106/0x230

Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is
at the 6th byte of the above CS CALL instruction. Since the CS CALL
instruction occupies 6 bytes, the exception occurred in the middle of that
call instruction.

The root cause is that when using eBPF tools to probe in the middle of a
function, a kprobe with INT3 is used as the underlying implementation.

During single-step emulation of the original CALL instruction,
int3_emulate_call() assumes that the probed CALL instruction is 5 bytes
long. However, the actual CS-prefixed CALL instruction occupies 6 bytes,
so it constructs an incorrect exception return address. When the CPU
returns from the kprobe handler, the next instruction to be executed is at
the address of the last byte of that CS CALL instruction. Coincidentally,
starting from that address, the CPU fetches and decodes a completely
different instruction, which ultimately triggers a kernel crash.

Fix the issue by using the actual instruction length obtained from
the instruction decoder when constructing the exception return
address, rather than relying on the hardcoded CALL_INSN_SIZE macro.

[ mingo: Refined the changelog ]

Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step")
Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Jinke Han <jinkehan@didiglobal.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Yafang Shao <laoar.shao@gmail.com>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000

1 files changed, 476 insertions, 0 deletions

diff --git a/drivers/tty/goldfish.c b/drivers/tty/goldfish.c
new file mode 100644
index 000000000..fa723a4ba
--- /dev/null
+++ b/drivers/tty/goldfish.c
@@ -0,0 +1,476 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2007 Google, Inc.
+ * Copyright (C) 2012 Intel, Inc.
+ * Copyright (C) 2017 Imagination Technologies Ltd.
+ */
+
+#include <linux/console.h>
+#include <linux/interrupt.h>
+#include <linux/platform_device.h>
+#include <linux/tty.h>
+#include <linux/tty_flip.h>
+#include <linux/slab.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/goldfish.h>
+#include <linux/mm.h>
+#include <linux/dma-mapping.h>
+#include <linux/serial_core.h>
+#include <linux/wordpart.h>
+
+/* Goldfish tty register's offsets */
+#define	GOLDFISH_TTY_REG_BYTES_READY	0x04
+#define	GOLDFISH_TTY_REG_CMD		0x08
+#define	GOLDFISH_TTY_REG_DATA_PTR	0x10
+#define	GOLDFISH_TTY_REG_DATA_LEN	0x14
+#define	GOLDFISH_TTY_REG_DATA_PTR_HIGH	0x18
+#define	GOLDFISH_TTY_REG_VERSION	0x20
+
+/* Goldfish tty commands */
+#define	GOLDFISH_TTY_CMD_INT_DISABLE	0
+#define	GOLDFISH_TTY_CMD_INT_ENABLE	1
+#define	GOLDFISH_TTY_CMD_WRITE_BUFFER	2
+#define	GOLDFISH_TTY_CMD_READ_BUFFER	3
+
+struct goldfish_tty {
+	struct tty_port port;
+	spinlock_t lock;
+	void __iomem *base;
+	u32 irq;
+	struct console console;
+	u32 version;
+	struct device *dev;
+};
+
+static DEFINE_MUTEX(goldfish_tty_lock);
+static struct tty_driver *goldfish_tty_driver;
+static u32 goldfish_tty_line_count = 8;
+static u32 goldfish_tty_current_line_count;
+static struct goldfish_tty *goldfish_ttys;
+
+static inline void gf_write_addr(unsigned long addr, void __iomem *portl, void __iomem *porth)
+{
+	gf_iowrite32(lower_32_bits(addr), portl);
+#ifdef CONFIG_64BIT
+	gf_iowrite32(upper_32_bits(addr), porth);
+#endif
+}
+
+static void do_rw_io(struct goldfish_tty *qtty, unsigned long address,
+		     size_t count, bool is_write)
+{
+	void __iomem *base = qtty->base;
+
+	guard(spinlock_irqsave)(&qtty->lock);
+
+	gf_write_addr(address, base + GOLDFISH_TTY_REG_DATA_PTR,
+		      base + GOLDFISH_TTY_REG_DATA_PTR_HIGH);
+	gf_iowrite32(count, base + GOLDFISH_TTY_REG_DATA_LEN);
+
+	if (is_write)
+		gf_iowrite32(GOLDFISH_TTY_CMD_WRITE_BUFFER,
+		       base + GOLDFISH_TTY_REG_CMD);
+	else
+		gf_iowrite32(GOLDFISH_TTY_CMD_READ_BUFFER,
+		       base + GOLDFISH_TTY_REG_CMD);
+}
+
+static void goldfish_tty_rw(struct goldfish_tty *qtty, unsigned long addr,
+			    size_t count, bool is_write)
+{
+	dma_addr_t dma_handle;
+	enum dma_data_direction dma_dir;
+
+	dma_dir = (is_write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
+	if (qtty->version > 0) {
+		/*
+		 * Goldfish TTY for Ranchu platform uses
+		 * physical addresses and DMA for read/write operations
+		 */
+		unsigned long addr_end = addr + count;
+
+		while (addr < addr_end) {
+			unsigned long pg_end = (addr & PAGE_MASK) + PAGE_SIZE;
+			unsigned long next =
+					pg_end < addr_end ? pg_end : addr_end;
+			unsigned long avail = next - addr;
+
+			/*
+			 * Map the buffer's virtual address to the DMA address
+			 * so the buffer can be accessed by the device.
+			 */
+			dma_handle = dma_map_single(qtty->dev, (void *)addr,
+						    avail, dma_dir);
+
+			if (dma_mapping_error(qtty->dev, dma_handle)) {
+				dev_err(qtty->dev, "tty: DMA mapping error.\n");
+				return;
+			}
+			do_rw_io(qtty, dma_handle, avail, is_write);
+
+			/*
+			 * Unmap the previously mapped region after
+			 * the completion of the read/write operation.
+			 */
+			dma_unmap_single(qtty->dev, dma_handle, avail, dma_dir);
+
+			addr += avail;
+		}
+	} else {
+		/*
+		 * Old style Goldfish TTY used on the Goldfish platform
+		 * uses virtual addresses.
+		 */
+		do_rw_io(qtty, addr, count, is_write);
+	}
+}
+
+static void goldfish_tty_do_write(int line, const u8 *buf, size_t count)
+{
+	struct goldfish_tty *qtty = &goldfish_ttys[line];
+
+	goldfish_tty_rw(qtty, (unsigned long)buf, count, true);
+}
+
+static irqreturn_t goldfish_tty_interrupt(int irq, void *dev_id)
+{
+	struct goldfish_tty *qtty = dev_id;
+	void __iomem *base = qtty->base;
+	u8 *buf;
+	u32 count;
+
+	count = gf_ioread32(base + GOLDFISH_TTY_REG_BYTES_READY);
+	if (count == 0)
+		return IRQ_NONE;
+
+	count = tty_prepare_flip_string(&qtty->port, &buf, count);
+
+	goldfish_tty_rw(qtty, (unsigned long)buf, count, false);
+
+	tty_flip_buffer_push(&qtty->port);
+	return IRQ_HANDLED;
+}
+
+static int goldfish_tty_activate(struct tty_port *port, struct tty_struct *tty)
+{
+	struct goldfish_tty *qtty = container_of(port, struct goldfish_tty,
+									port);
+	gf_iowrite32(GOLDFISH_TTY_CMD_INT_ENABLE, qtty->base + GOLDFISH_TTY_REG_CMD);
+	return 0;
+}
+
+static void goldfish_tty_shutdown(struct tty_port *port)
+{
+	struct goldfish_tty *qtty = container_of(port, struct goldfish_tty,
+									port);
+	gf_iowrite32(GOLDFISH_TTY_CMD_INT_DISABLE, qtty->base + GOLDFISH_TTY_REG_CMD);
+}
+
+static int goldfish_tty_open(struct tty_struct *tty, struct file *filp)
+{
+	struct goldfish_tty *qtty = &goldfish_ttys[tty->index];
+	return tty_port_open(&qtty->port, tty, filp);
+}
+
+static void goldfish_tty_close(struct tty_struct *tty, struct file *filp)
+{
+	tty_port_close(tty->port, tty, filp);
+}
+
+static void goldfish_tty_hangup(struct tty_struct *tty)
+{
+	tty_port_hangup(tty->port);
+}
+
+static ssize_t goldfish_tty_write(struct tty_struct *tty, const u8 *buf,
+				  size_t count)
+{
+	goldfish_tty_do_write(tty->index, buf, count);
+	return count;
+}
+
+static unsigned int goldfish_tty_write_room(struct tty_struct *tty)
+{
+	return 0x10000;
+}
+
+static unsigned int goldfish_tty_chars_in_buffer(struct tty_struct *tty)
+{
+	struct goldfish_tty *qtty = &goldfish_ttys[tty->index];
+	void __iomem *base = qtty->base;
+	return gf_ioread32(base + GOLDFISH_TTY_REG_BYTES_READY);
+}
+
+static void goldfish_tty_console_write(struct console *co, const char *b,
+								unsigned count)
+{
+	goldfish_tty_do_write(co->index, b, count);
+}
+
+static struct tty_driver *goldfish_tty_console_device(struct console *c,
+								int *index)
+{
+	*index = c->index;
+	return goldfish_tty_driver;
+}
+
+static int goldfish_tty_console_setup(struct console *co, char *options)
+{
+	if ((unsigned)co->index >= goldfish_tty_line_count)
+		return -ENODEV;
+	if (!goldfish_ttys[co->index].base)
+		return -ENODEV;
+	return 0;
+}
+
+static const struct tty_port_operations goldfish_port_ops = {
+	.activate = goldfish_tty_activate,
+	.shutdown = goldfish_tty_shutdown
+};
+
+static const struct tty_operations goldfish_tty_ops = {
+	.open = goldfish_tty_open,
+	.close = goldfish_tty_close,
+	.hangup = goldfish_tty_hangup,
+	.write = goldfish_tty_write,
+	.write_room = goldfish_tty_write_room,
+	.chars_in_buffer = goldfish_tty_chars_in_buffer,
+};
+
+static int goldfish_tty_create_driver(void)
+{
+	int ret;
+	struct tty_driver *tty;
+
+	goldfish_ttys = kzalloc_objs(*goldfish_ttys, goldfish_tty_line_count);
+	if (goldfish_ttys == NULL) {
+		ret = -ENOMEM;
+		goto err_alloc_goldfish_ttys_failed;
+	}
+	tty = tty_alloc_driver(goldfish_tty_line_count,
+			TTY_DRIVER_RESET_TERMIOS | TTY_DRIVER_REAL_RAW |
+			TTY_DRIVER_DYNAMIC_DEV);
+	if (IS_ERR(tty)) {
+		ret = PTR_ERR(tty);
+		goto err_tty_alloc_driver_failed;
+	}
+	tty->driver_name = "goldfish";
+	tty->name = "ttyGF";
+	tty->type = TTY_DRIVER_TYPE_SERIAL;
+	tty->subtype = SERIAL_TYPE_NORMAL;
+	tty->init_termios = tty_std_termios;
+	tty_set_operations(tty, &goldfish_tty_ops);
+	ret = tty_register_driver(tty);
+	if (ret)
+		goto err_tty_register_driver_failed;
+
+	goldfish_tty_driver = tty;
+	return 0;
+
+err_tty_register_driver_failed:
+	tty_driver_kref_put(tty);
+err_tty_alloc_driver_failed:
+	kfree(goldfish_ttys);
+	goldfish_ttys = NULL;
+err_alloc_goldfish_ttys_failed:
+	return ret;
+}
+
+static void goldfish_tty_delete_driver(void)
+{
+	tty_unregister_driver(goldfish_tty_driver);
+	tty_driver_kref_put(goldfish_tty_driver);
+	goldfish_tty_driver = NULL;
+	kfree(goldfish_ttys);
+	goldfish_ttys = NULL;
+}
+
+static int goldfish_tty_probe(struct platform_device *pdev)
+{
+	struct goldfish_tty *qtty;
+	int ret = -ENODEV;
+	struct resource *r;
+	struct device *ttydev;
+	void __iomem *base;
+	int irq;
+	unsigned int line;
+
+	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!r) {
+		pr_err("goldfish_tty: No MEM resource available!\n");
+		return -ENOMEM;
+	}
+
+	base = ioremap(r->start, 0x1000);
+	if (!base) {
+		pr_err("goldfish_tty: Unable to ioremap base!\n");
+		return -ENOMEM;
+	}
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		ret = irq;
+		goto err_unmap;
+	}
+
+	mutex_lock(&goldfish_tty_lock);
+
+	if (pdev->id == PLATFORM_DEVID_NONE)
+		line = goldfish_tty_current_line_count;
+	else
+		line = pdev->id;
+
+	if (line >= goldfish_tty_line_count) {
+		pr_err("goldfish_tty: Reached maximum tty number of %d.\n",
+		       goldfish_tty_current_line_count);
+		ret = -ENOMEM;
+		goto err_unlock;
+	}
+
+	if (goldfish_tty_current_line_count == 0) {
+		ret = goldfish_tty_create_driver();
+		if (ret)
+			goto err_unlock;
+	}
+	goldfish_tty_current_line_count++;
+
+	qtty = &goldfish_ttys[line];
+	spin_lock_init(&qtty->lock);
+	tty_port_init(&qtty->port);
+	qtty->port.ops = &goldfish_port_ops;
+	qtty->base = base;
+	qtty->irq = irq;
+	qtty->dev = &pdev->dev;
+
+	/*
+	 * Goldfish TTY device used by the Goldfish emulator
+	 * should identify itself with 0, forcing the driver
+	 * to use virtual addresses. Goldfish TTY device
+	 * on Ranchu emulator (qemu2) returns 1 here and
+	 * driver will use physical addresses.
+	 */
+	qtty->version = gf_ioread32(base + GOLDFISH_TTY_REG_VERSION);
+
+	/*
+	 * Goldfish TTY device on Ranchu emulator (qemu2)
+	 * will use DMA for read/write IO operations.
+	 */
+	if (qtty->version > 0) {
+		/*
+		 * Initialize dma_mask to 32-bits.
+		 */
+		if (!pdev->dev.dma_mask)
+			pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask;
+		ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
+		if (ret) {
+			dev_err(&pdev->dev, "No suitable DMA available.\n");
+			goto err_dec_line_count;
+		}
+	}
+
+	gf_iowrite32(GOLDFISH_TTY_CMD_INT_DISABLE, base + GOLDFISH_TTY_REG_CMD);
+
+	ret = request_irq(irq, goldfish_tty_interrupt, IRQF_SHARED,
+			  "goldfish_tty", qtty);
+	if (ret) {
+		pr_err("goldfish_tty: No IRQ available!\n");
+		goto err_dec_line_count;
+	}
+
+	ttydev = tty_port_register_device(&qtty->port, goldfish_tty_driver,
+					  line, &pdev->dev);
+	if (IS_ERR(ttydev)) {
+		ret = PTR_ERR(ttydev);
+		goto err_tty_register_device_failed;
+	}
+
+	strcpy(qtty->console.name, "ttyGF");
+	qtty->console.write = goldfish_tty_console_write;
+	qtty->console.device = goldfish_tty_console_device;
+	qtty->console.setup = goldfish_tty_console_setup;
+	qtty->console.flags = CON_PRINTBUFFER;
+	qtty->console.index = line;
+	register_console(&qtty->console);
+	platform_set_drvdata(pdev, qtty);
+
+	mutex_unlock(&goldfish_tty_lock);
+	return 0;
+
+err_tty_register_device_failed:
+	free_irq(irq, qtty);
+err_dec_line_count:
+	tty_port_destroy(&qtty->port);
+	goldfish_tty_current_line_count--;
+	if (goldfish_tty_current_line_count == 0)
+		goldfish_tty_delete_driver();
+err_unlock:
+	mutex_unlock(&goldfish_tty_lock);
+err_unmap:
+	iounmap(base);
+	return ret;
+}
+
+static void goldfish_tty_remove(struct platform_device *pdev)
+{
+	struct goldfish_tty *qtty = platform_get_drvdata(pdev);
+
+	guard(mutex)(&goldfish_tty_lock);
+
+	unregister_console(&qtty->console);
+	tty_unregister_device(goldfish_tty_driver, qtty->console.index);
+	iounmap(qtty->base);
+	qtty->base = NULL;
+	free_irq(qtty->irq, qtty);
+	tty_port_destroy(&qtty->port);
+	goldfish_tty_current_line_count--;
+	if (goldfish_tty_current_line_count == 0)
+		goldfish_tty_delete_driver();
+}
+
+#ifdef CONFIG_GOLDFISH_TTY_EARLY_CONSOLE
+static void gf_early_console_putchar(struct uart_port *port, unsigned char ch)
+{
+	gf_iowrite32(ch, port->membase);
+}
+
+static void gf_early_write(struct console *con, const char *s, unsigned int n)
+{
+	struct earlycon_device *dev = con->data;
+
+	uart_console_write(&dev->port, s, n, gf_early_console_putchar);
+}
+
+static int __init gf_earlycon_setup(struct earlycon_device *device,
+				    const char *opt)
+{
+	if (!device->port.membase)
+		return -ENODEV;
+
+	device->con->write = gf_early_write;
+	return 0;
+}
+
+OF_EARLYCON_DECLARE(early_gf_tty, "google,goldfish-tty", gf_earlycon_setup);
+#endif
+
+static const struct of_device_id goldfish_tty_of_match[] = {
+	{ .compatible = "google,goldfish-tty", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, goldfish_tty_of_match);
+
+static struct platform_driver goldfish_tty_platform_driver = {
+	.probe = goldfish_tty_probe,
+	.remove = goldfish_tty_remove,
+	.driver = {
+		.name = "goldfish_tty",
+		.of_match_table = goldfish_tty_of_match,
+	}
+};
+
+module_platform_driver(goldfish_tty_platform_driver);
+
+MODULE_DESCRIPTION("Goldfish TTY Driver");
+MODULE_LICENSE("GPL v2");