x86/kprobes: Fix crash when probing CS CALL instructionsgrafted

When using eBPF to probe CS CALL instructions within a function,
a crash can be triggered.

The eBPF tool probes offset 257 of the __hrtimer_run_queues()
function:

<__hrtimer_run_queues+249>:  nopl   0x0(%rax,%rax,1)
<__hrtimer_run_queues+254>:  mov    %r14,%rdi
<__hrtimer_run_queues+257>:  cs call <__x86_indirect_thunk_r12>
<__hrtimer_run_queues+263>:  mov    %eax,%r12d
<__hrtimer_run_queues+266>:  xchg   %ax,%ax
<__hrtimer_run_queues+268>:  mov    %r13,%rdi

Which triggers this crash:

  BUG: unable to handle page fault for address: 00000000000f41c9
  #PF: supervisor write access in kernel mode
  #PF: error_code(0x0002) - not-present page
  PGD 0 P4D 0
  Oops: 0002 [#1] SMP NOPTI
  CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P
  RIP: 0010:__hrtimer_run_queues+0x106/0x230

Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is
at the 6th byte of the above CS CALL instruction. Since the CS CALL
instruction occupies 6 bytes, the exception occurred in the middle of that
call instruction.

The root cause is that when using eBPF tools to probe in the middle of a
function, a kprobe with INT3 is used as the underlying implementation.

During single-step emulation of the original CALL instruction,
int3_emulate_call() assumes that the probed CALL instruction is 5 bytes
long. However, the actual CS-prefixed CALL instruction occupies 6 bytes,
so it constructs an incorrect exception return address. When the CPU
returns from the kprobe handler, the next instruction to be executed is at
the address of the last byte of that CS CALL instruction. Coincidentally,
starting from that address, the CPU fetches and decodes a completely
different instruction, which ultimately triggers a kernel crash.

Fix the issue by using the actual instruction length obtained from
the instruction decoder when constructing the exception return
address, rather than relying on the hardcoded CALL_INSN_SIZE macro.

[ mingo: Refined the changelog ]

Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step")
Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Jinke Han <jinkehan@didiglobal.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Yafang Shao <laoar.shao@gmail.com>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000

1 files changed, 463 insertions, 0 deletions

diff --git a/drivers/mailbox/mailbox-test.c b/drivers/mailbox/mailbox-test.c
new file mode 100644
index 000000000..7b6ef033e
--- /dev/null
+++ b/drivers/mailbox/mailbox-test.c
@@ -0,0 +1,463 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2015 ST Microelectronics
+ *
+ * Author: Lee Jones <lee.jones@linaro.org>
+ */
+
+#include <linux/debugfs.h>
+#include <linux/err.h>
+#include <linux/fs.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/mailbox_client.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/poll.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/uaccess.h>
+#include <linux/sched/signal.h>
+
+#define MBOX_MAX_SIG_LEN	8
+#define MBOX_MAX_MSG_LEN	128
+#define MBOX_BYTES_PER_LINE	16
+#define MBOX_HEXDUMP_LINE_LEN	((MBOX_BYTES_PER_LINE * 4) + 2)
+#define MBOX_HEXDUMP_MAX_LEN	(MBOX_HEXDUMP_LINE_LEN *		\
+				 (MBOX_MAX_MSG_LEN / MBOX_BYTES_PER_LINE))
+
+struct mbox_test_device {
+	struct device		*dev;
+	void __iomem		*tx_mmio;
+	void __iomem		*rx_mmio;
+	struct mbox_chan	*tx_channel;
+	struct mbox_chan	*rx_channel;
+	char			*rx_buffer;
+	char			*signal;
+	char			*message;
+	spinlock_t		lock;
+	struct mutex		mutex;
+	wait_queue_head_t	waitq;
+	bool			data_ready;
+	struct fasync_struct	*async_queue;
+	struct dentry		*root_debugfs_dir;
+};
+
+static ssize_t mbox_test_signal_write(struct file *filp,
+				       const char __user *userbuf,
+				       size_t count, loff_t *ppos)
+{
+	struct mbox_test_device *tdev = filp->private_data;
+
+	if (!tdev->tx_channel) {
+		dev_err(tdev->dev, "Channel cannot do Tx\n");
+		return -EINVAL;
+	}
+
+	if (count > MBOX_MAX_SIG_LEN) {
+		dev_err(tdev->dev,
+			"Signal length %zd greater than max allowed %d\n",
+			count, MBOX_MAX_SIG_LEN);
+		return -EINVAL;
+	}
+
+	/* Only allocate memory if we need to */
+	if (!tdev->signal) {
+		tdev->signal = kzalloc(MBOX_MAX_SIG_LEN, GFP_KERNEL);
+		if (!tdev->signal)
+			return -ENOMEM;
+	}
+
+	if (copy_from_user(tdev->signal, userbuf, count)) {
+		kfree(tdev->signal);
+		tdev->signal = NULL;
+		return -EFAULT;
+	}
+
+	return count;
+}
+
+static const struct file_operations mbox_test_signal_ops = {
+	.write	= mbox_test_signal_write,
+	.open	= simple_open,
+	.llseek	= generic_file_llseek,
+};
+
+static int mbox_test_message_fasync(int fd, struct file *filp, int on)
+{
+	struct mbox_test_device *tdev = filp->private_data;
+
+	return fasync_helper(fd, filp, on, &tdev->async_queue);
+}
+
+static ssize_t mbox_test_message_write(struct file *filp,
+				       const char __user *userbuf,
+				       size_t count, loff_t *ppos)
+{
+	struct mbox_test_device *tdev = filp->private_data;
+	char *message;
+	void *data;
+	int ret;
+
+	if (!tdev->tx_channel) {
+		dev_err(tdev->dev, "Channel cannot do Tx\n");
+		return -EINVAL;
+	}
+
+	if (count > MBOX_MAX_MSG_LEN) {
+		dev_err(tdev->dev,
+			"Message length %zd greater than max allowed %d\n",
+			count, MBOX_MAX_MSG_LEN);
+		return -EINVAL;
+	}
+
+	message = kzalloc(MBOX_MAX_MSG_LEN, GFP_KERNEL);
+	if (!message)
+		return -ENOMEM;
+
+	mutex_lock(&tdev->mutex);
+
+	tdev->message = message;
+	ret = copy_from_user(tdev->message, userbuf, count);
+	if (ret) {
+		ret = -EFAULT;
+		goto out;
+	}
+
+	/*
+	 * A separate signal is only of use if there is
+	 * MMIO to subsequently pass the message through
+	 */
+	if (tdev->tx_mmio && tdev->signal) {
+		print_hex_dump_bytes("Client: Sending: Signal: ", DUMP_PREFIX_ADDRESS,
+				     tdev->signal, MBOX_MAX_SIG_LEN);
+
+		data = tdev->signal;
+	} else
+		data = tdev->message;
+
+	print_hex_dump_bytes("Client: Sending: Message: ", DUMP_PREFIX_ADDRESS,
+			     tdev->message, MBOX_MAX_MSG_LEN);
+
+	ret = mbox_send_message(tdev->tx_channel, data);
+	if (ret < 0)
+		dev_err(tdev->dev, "Failed to send message via mailbox\n");
+
+out:
+	kfree(tdev->signal);
+	kfree(tdev->message);
+	tdev->signal = NULL;
+
+	mutex_unlock(&tdev->mutex);
+
+	return ret < 0 ? ret : count;
+}
+
+static bool mbox_test_message_data_ready(struct mbox_test_device *tdev)
+{
+	bool data_ready;
+	unsigned long flags;
+
+	spin_lock_irqsave(&tdev->lock, flags);
+	data_ready = tdev->data_ready;
+	spin_unlock_irqrestore(&tdev->lock, flags);
+
+	return data_ready;
+}
+
+static ssize_t mbox_test_message_read(struct file *filp, char __user *userbuf,
+				      size_t count, loff_t *ppos)
+{
+	struct mbox_test_device *tdev = filp->private_data;
+	unsigned long flags;
+	char *touser, *ptr;
+	int l = 0;
+	int ret;
+
+	DECLARE_WAITQUEUE(wait, current);
+
+	touser = kzalloc(MBOX_HEXDUMP_MAX_LEN + 1, GFP_KERNEL);
+	if (!touser)
+		return -ENOMEM;
+
+	if (!tdev->rx_channel) {
+		ret = snprintf(touser, 20, "<NO RX CAPABILITY>\n");
+		ret = simple_read_from_buffer(userbuf, count, ppos,
+					      touser, ret);
+		goto kfree_err;
+	}
+
+	add_wait_queue(&tdev->waitq, &wait);
+
+	do {
+		__set_current_state(TASK_INTERRUPTIBLE);
+
+		if (mbox_test_message_data_ready(tdev))
+			break;
+
+		if (filp->f_flags & O_NONBLOCK) {
+			ret = -EAGAIN;
+			goto waitq_err;
+		}
+
+		if (signal_pending(current)) {
+			ret = -ERESTARTSYS;
+			goto waitq_err;
+		}
+		schedule();
+
+	} while (1);
+
+	spin_lock_irqsave(&tdev->lock, flags);
+
+	ptr = tdev->rx_buffer;
+	while (l < MBOX_HEXDUMP_MAX_LEN) {
+		hex_dump_to_buffer(ptr,
+				   MBOX_BYTES_PER_LINE,
+				   MBOX_BYTES_PER_LINE, 1, touser + l,
+				   MBOX_HEXDUMP_LINE_LEN, true);
+
+		ptr += MBOX_BYTES_PER_LINE;
+		l += MBOX_HEXDUMP_LINE_LEN;
+		*(touser + (l - 1)) = '\n';
+	}
+	*(touser + l) = '\0';
+
+	memset(tdev->rx_buffer, 0, MBOX_MAX_MSG_LEN);
+	tdev->data_ready = false;
+
+	spin_unlock_irqrestore(&tdev->lock, flags);
+
+	ret = simple_read_from_buffer(userbuf, count, ppos, touser, MBOX_HEXDUMP_MAX_LEN);
+waitq_err:
+	__set_current_state(TASK_RUNNING);
+	remove_wait_queue(&tdev->waitq, &wait);
+kfree_err:
+	kfree(touser);
+	return ret;
+}
+
+static __poll_t
+mbox_test_message_poll(struct file *filp, struct poll_table_struct *wait)
+{
+	struct mbox_test_device *tdev = filp->private_data;
+
+	poll_wait(filp, &tdev->waitq, wait);
+
+	if (mbox_test_message_data_ready(tdev))
+		return EPOLLIN | EPOLLRDNORM;
+	return 0;
+}
+
+static const struct file_operations mbox_test_message_ops = {
+	.write	= mbox_test_message_write,
+	.read	= mbox_test_message_read,
+	.fasync	= mbox_test_message_fasync,
+	.poll	= mbox_test_message_poll,
+	.open	= simple_open,
+	.llseek	= generic_file_llseek,
+};
+
+static int mbox_test_add_debugfs(struct platform_device *pdev,
+				 struct mbox_test_device *tdev)
+{
+	if (!debugfs_initialized())
+		return 0;
+
+	tdev->root_debugfs_dir = debugfs_create_dir(dev_name(&pdev->dev), NULL);
+	if (IS_ERR(tdev->root_debugfs_dir)) {
+		dev_err(&pdev->dev, "Failed to create Mailbox debugfs\n");
+		return -EINVAL;
+	}
+
+	debugfs_create_file("message", 0600, tdev->root_debugfs_dir,
+			    tdev, &mbox_test_message_ops);
+
+	debugfs_create_file("signal", 0200, tdev->root_debugfs_dir,
+			    tdev, &mbox_test_signal_ops);
+
+	return 0;
+}
+
+static void mbox_test_receive_message(struct mbox_client *client, void *message)
+{
+	struct mbox_test_device *tdev = dev_get_drvdata(client->dev);
+	unsigned long flags;
+
+	spin_lock_irqsave(&tdev->lock, flags);
+	if (tdev->rx_mmio) {
+		memcpy_fromio(tdev->rx_buffer, tdev->rx_mmio, MBOX_MAX_MSG_LEN);
+		print_hex_dump_bytes("Client: Received [MMIO]: ", DUMP_PREFIX_ADDRESS,
+				     tdev->rx_buffer, MBOX_MAX_MSG_LEN);
+	} else if (message) {
+		print_hex_dump_bytes("Client: Received [API]: ", DUMP_PREFIX_ADDRESS,
+				     message, MBOX_MAX_MSG_LEN);
+		memcpy(tdev->rx_buffer, message, MBOX_MAX_MSG_LEN);
+	}
+	tdev->data_ready = true;
+	spin_unlock_irqrestore(&tdev->lock, flags);
+
+	wake_up_interruptible(&tdev->waitq);
+
+	kill_fasync(&tdev->async_queue, SIGIO, POLL_IN);
+}
+
+static void mbox_test_prepare_message(struct mbox_client *client, void *message)
+{
+	struct mbox_test_device *tdev = dev_get_drvdata(client->dev);
+
+	if (tdev->tx_mmio) {
+		if (tdev->signal)
+			memcpy_toio(tdev->tx_mmio, tdev->message, MBOX_MAX_MSG_LEN);
+		else
+			memcpy_toio(tdev->tx_mmio, message, MBOX_MAX_MSG_LEN);
+	}
+}
+
+static void mbox_test_message_sent(struct mbox_client *client,
+				   void *message, int r)
+{
+	if (r)
+		dev_warn(client->dev,
+			 "Client: Message could not be sent: %d\n", r);
+	else
+		dev_info(client->dev,
+			 "Client: Message sent\n");
+}
+
+static struct mbox_chan *
+mbox_test_request_channel(struct platform_device *pdev, const char *name)
+{
+	struct mbox_client *client;
+	struct mbox_chan *channel;
+
+	client = devm_kzalloc(&pdev->dev, sizeof(*client), GFP_KERNEL);
+	if (!client)
+		return NULL;
+
+	client->dev		= &pdev->dev;
+	client->rx_callback	= mbox_test_receive_message;
+	client->tx_prepare	= mbox_test_prepare_message;
+	client->tx_done		= mbox_test_message_sent;
+	client->tx_block	= true;
+	client->knows_txdone	= false;
+	client->tx_tout		= 500;
+
+	channel = mbox_request_channel_byname(client, name);
+	if (IS_ERR(channel)) {
+		dev_warn(&pdev->dev, "Failed to request %s channel\n", name);
+		return NULL;
+	}
+
+	return channel;
+}
+
+static void __iomem *mbox_test_ioremap(struct platform_device *pdev, unsigned int res_num)
+{
+	struct resource *res;
+	void __iomem *mmio;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, res_num);
+	if (!res)
+		return NULL;
+
+	mmio = devm_ioremap_resource(&pdev->dev, res);
+	if (PTR_ERR(mmio) == -EBUSY) {
+		dev_info(&pdev->dev, "trying workaround with plain ioremap\n");
+		return devm_ioremap(&pdev->dev, res->start, resource_size(res));
+	}
+
+	return IS_ERR(mmio) ? NULL : mmio;
+}
+
+static int mbox_test_probe(struct platform_device *pdev)
+{
+	struct mbox_test_device *tdev;
+	int ret;
+
+	tdev = devm_kzalloc(&pdev->dev, sizeof(*tdev), GFP_KERNEL);
+	if (!tdev)
+		return -ENOMEM;
+
+	tdev->dev = &pdev->dev;
+	spin_lock_init(&tdev->lock);
+	mutex_init(&tdev->mutex);
+	init_waitqueue_head(&tdev->waitq);
+	platform_set_drvdata(pdev, tdev);
+
+	/* It's okay for MMIO to be NULL */
+	tdev->tx_mmio = mbox_test_ioremap(pdev, 0);
+
+	/* If specified, second reg entry is Rx MMIO */
+	tdev->rx_mmio = mbox_test_ioremap(pdev, 1);
+	if (!tdev->rx_mmio)
+		tdev->rx_mmio = tdev->tx_mmio;
+
+	tdev->tx_channel = mbox_test_request_channel(pdev, "tx");
+	tdev->rx_channel = mbox_test_request_channel(pdev, "rx");
+
+	if (!tdev->tx_channel && !tdev->rx_channel)
+		return -EPROBE_DEFER;
+
+	/* If Rx is not specified but has Rx MMIO, then Rx = Tx */
+	if (!tdev->rx_channel && (tdev->rx_mmio != tdev->tx_mmio))
+		tdev->rx_channel = tdev->tx_channel;
+
+	if (tdev->rx_channel) {
+		tdev->rx_buffer = devm_kzalloc(&pdev->dev,
+					       MBOX_MAX_MSG_LEN, GFP_KERNEL);
+		if (!tdev->rx_buffer) {
+			ret = -ENOMEM;
+			goto err_free_chans;
+		}
+	}
+
+	ret = mbox_test_add_debugfs(pdev, tdev);
+	if (ret)
+		goto err_free_chans;
+
+	dev_info(&pdev->dev, "Successfully registered\n");
+
+	return 0;
+
+err_free_chans:
+	if (tdev->tx_channel)
+		mbox_free_channel(tdev->tx_channel);
+	if (tdev->rx_channel && tdev->rx_channel != tdev->tx_channel)
+		mbox_free_channel(tdev->rx_channel);
+	return ret;
+}
+
+static void mbox_test_remove(struct platform_device *pdev)
+{
+	struct mbox_test_device *tdev = platform_get_drvdata(pdev);
+
+	debugfs_remove_recursive(tdev->root_debugfs_dir);
+
+	if (tdev->tx_channel)
+		mbox_free_channel(tdev->tx_channel);
+	if (tdev->rx_channel && tdev->rx_channel != tdev->tx_channel)
+		mbox_free_channel(tdev->rx_channel);
+}
+
+static const struct of_device_id mbox_test_match[] = {
+	{ .compatible = "mailbox-test" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, mbox_test_match);
+
+static struct platform_driver mbox_test_driver = {
+	.driver = {
+		.name = "mailbox_test",
+		.of_match_table = mbox_test_match,
+	},
+	.probe = mbox_test_probe,
+	.remove = mbox_test_remove,
+};
+module_platform_driver(mbox_test_driver);
+
+MODULE_DESCRIPTION("Generic Mailbox Testing Facility");
+MODULE_AUTHOR("Lee Jones <lee.jones@linaro.org");
+MODULE_LICENSE("GPL v2");