x86/kprobes: Fix crash when probing CS CALL instructionsgrafted

When using eBPF to probe CS CALL instructions within a function,
a crash can be triggered.

The eBPF tool probes offset 257 of the __hrtimer_run_queues()
function:

<__hrtimer_run_queues+249>:  nopl   0x0(%rax,%rax,1)
<__hrtimer_run_queues+254>:  mov    %r14,%rdi
<__hrtimer_run_queues+257>:  cs call <__x86_indirect_thunk_r12>
<__hrtimer_run_queues+263>:  mov    %eax,%r12d
<__hrtimer_run_queues+266>:  xchg   %ax,%ax
<__hrtimer_run_queues+268>:  mov    %r13,%rdi

Which triggers this crash:

  BUG: unable to handle page fault for address: 00000000000f41c9
  #PF: supervisor write access in kernel mode
  #PF: error_code(0x0002) - not-present page
  PGD 0 P4D 0
  Oops: 0002 [#1] SMP NOPTI
  CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P
  RIP: 0010:__hrtimer_run_queues+0x106/0x230

Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is
at the 6th byte of the above CS CALL instruction. Since the CS CALL
instruction occupies 6 bytes, the exception occurred in the middle of that
call instruction.

The root cause is that when using eBPF tools to probe in the middle of a
function, a kprobe with INT3 is used as the underlying implementation.

During single-step emulation of the original CALL instruction,
int3_emulate_call() assumes that the probed CALL instruction is 5 bytes
long. However, the actual CS-prefixed CALL instruction occupies 6 bytes,
so it constructs an incorrect exception return address. When the CPU
returns from the kprobe handler, the next instruction to be executed is at
the address of the last byte of that CS CALL instruction. Coincidentally,
starting from that address, the CPU fetches and decodes a completely
different instruction, which ultimately triggers a kernel crash.

Fix the issue by using the actual instruction length obtained from
the instruction decoder when constructing the exception return
address, rather than relying on the hardcoded CALL_INSN_SIZE macro.

[ mingo: Refined the changelog ]

Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step")
Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Jinke Han <jinkehan@didiglobal.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Yafang Shao <laoar.shao@gmail.com>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000

1 files changed, 491 insertions, 0 deletions

diff --git a/drivers/mailbox/mailbox-sti.c b/drivers/mailbox/mailbox-sti.c
new file mode 100644
index 000000000..4fb5d1b55
--- /dev/null
+++ b/drivers/mailbox/mailbox-sti.c
@@ -0,0 +1,491 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * STi Mailbox
+ *
+ * Copyright (C) 2015 ST Microelectronics
+ *
+ * Author: Lee Jones <lee.jones@linaro.org> for ST Microelectronics
+ *
+ * Based on the original driver written by;
+ *   Alexandre Torgue, Olivier Lebreton and Loic Pallardy
+ */
+
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/mailbox_controller.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/slab.h>
+
+#define STI_MBOX_INST_MAX	4      /* RAM saving: Max supported instances */
+#define STI_MBOX_CHAN_MAX	20     /* RAM saving: Max supported channels  */
+
+#define STI_IRQ_VAL_OFFSET	0x04   /* Read interrupt status	              */
+#define STI_IRQ_SET_OFFSET	0x24   /* Generate a Tx channel interrupt     */
+#define STI_IRQ_CLR_OFFSET	0x44   /* Clear pending Rx interrupts	      */
+#define STI_ENA_VAL_OFFSET	0x64   /* Read enable status		      */
+#define STI_ENA_SET_OFFSET	0x84   /* Enable a channel		      */
+#define STI_ENA_CLR_OFFSET	0xa4   /* Disable a channel		      */
+
+#define MBOX_BASE(mdev, inst)   ((mdev)->base + ((inst) * 4))
+
+/**
+ * struct sti_mbox_device - STi Mailbox device data
+ *
+ * @dev:	Device to which it is attached
+ * @mbox:	Representation of a communication channel controller
+ * @base:	Base address of the register mapping region
+ * @name:	Name of the mailbox
+ * @enabled:	Local copy of enabled channels
+ * @lock:	Mutex protecting enabled status
+ *
+ * An IP Mailbox is currently composed of 4 instances
+ * Each instance is currently composed of 32 channels
+ * This means that we have 128 channels per Mailbox
+ * A channel an be used for TX or RX
+ */
+struct sti_mbox_device {
+	struct device		*dev;
+	struct mbox_controller	*mbox;
+	void __iomem		*base;
+	const char		*name;
+	u32			enabled[STI_MBOX_INST_MAX];
+	spinlock_t		lock;
+};
+
+/**
+ * struct sti_mbox_pdata - STi Mailbox platform specific configuration
+ *
+ * @num_inst:	Maximum number of instances in one HW Mailbox
+ * @num_chan:	Maximum number of channel per instance
+ */
+struct sti_mbox_pdata {
+	unsigned int		num_inst;
+	unsigned int		num_chan;
+};
+
+/**
+ * struct sti_channel - STi Mailbox allocated channel information
+ *
+ * @mdev:	Pointer to parent Mailbox device
+ * @instance:	Instance number channel resides in
+ * @channel:	Channel number pertaining to this container
+ */
+struct sti_channel {
+	struct sti_mbox_device	*mdev;
+	unsigned int		instance;
+	unsigned int		channel;
+};
+
+static inline bool sti_mbox_channel_is_enabled(struct mbox_chan *chan)
+{
+	struct sti_channel *chan_info = chan->con_priv;
+	struct sti_mbox_device *mdev = chan_info->mdev;
+	unsigned int instance = chan_info->instance;
+	unsigned int channel = chan_info->channel;
+
+	return mdev->enabled[instance] & BIT(channel);
+}
+
+static inline
+struct mbox_chan *sti_mbox_to_channel(struct mbox_controller *mbox,
+				      unsigned int instance,
+				      unsigned int channel)
+{
+	struct sti_channel *chan_info;
+	int i;
+
+	for (i = 0; i < mbox->num_chans; i++) {
+		chan_info = mbox->chans[i].con_priv;
+		if (chan_info &&
+		    chan_info->instance == instance &&
+		    chan_info->channel == channel)
+			return &mbox->chans[i];
+	}
+
+	dev_err(mbox->dev,
+		"Channel not registered: instance: %d channel: %d\n",
+		instance, channel);
+
+	return NULL;
+}
+
+static void sti_mbox_enable_channel(struct mbox_chan *chan)
+{
+	struct sti_channel *chan_info = chan->con_priv;
+	struct sti_mbox_device *mdev = chan_info->mdev;
+	unsigned int instance = chan_info->instance;
+	unsigned int channel = chan_info->channel;
+	unsigned long flags;
+	void __iomem *base = MBOX_BASE(mdev, instance);
+
+	spin_lock_irqsave(&mdev->lock, flags);
+	mdev->enabled[instance] |= BIT(channel);
+	writel_relaxed(BIT(channel), base + STI_ENA_SET_OFFSET);
+	spin_unlock_irqrestore(&mdev->lock, flags);
+}
+
+static void sti_mbox_disable_channel(struct mbox_chan *chan)
+{
+	struct sti_channel *chan_info = chan->con_priv;
+	struct sti_mbox_device *mdev = chan_info->mdev;
+	unsigned int instance = chan_info->instance;
+	unsigned int channel = chan_info->channel;
+	unsigned long flags;
+	void __iomem *base = MBOX_BASE(mdev, instance);
+
+	spin_lock_irqsave(&mdev->lock, flags);
+	mdev->enabled[instance] &= ~BIT(channel);
+	writel_relaxed(BIT(channel), base + STI_ENA_CLR_OFFSET);
+	spin_unlock_irqrestore(&mdev->lock, flags);
+}
+
+static void sti_mbox_clear_irq(struct mbox_chan *chan)
+{
+	struct sti_channel *chan_info = chan->con_priv;
+	struct sti_mbox_device *mdev = chan_info->mdev;
+	unsigned int instance = chan_info->instance;
+	unsigned int channel = chan_info->channel;
+	void __iomem *base = MBOX_BASE(mdev, instance);
+
+	writel_relaxed(BIT(channel), base + STI_IRQ_CLR_OFFSET);
+}
+
+static struct mbox_chan *sti_mbox_irq_to_channel(struct sti_mbox_device *mdev,
+						 unsigned int instance)
+{
+	struct mbox_controller *mbox = mdev->mbox;
+	struct mbox_chan *chan = NULL;
+	unsigned int channel;
+	unsigned long bits;
+	void __iomem *base = MBOX_BASE(mdev, instance);
+
+	bits = readl_relaxed(base + STI_IRQ_VAL_OFFSET);
+	if (!bits)
+		/* No IRQs fired in specified instance */
+		return NULL;
+
+	/* An IRQ has fired, find the associated channel */
+	for (channel = 0; bits; channel++) {
+		if (!test_and_clear_bit(channel, &bits))
+			continue;
+
+		chan = sti_mbox_to_channel(mbox, instance, channel);
+		if (chan) {
+			dev_dbg(mbox->dev,
+				"IRQ fired on instance: %d channel: %d\n",
+				instance, channel);
+			break;
+		}
+	}
+
+	return chan;
+}
+
+static irqreturn_t sti_mbox_thread_handler(int irq, void *data)
+{
+	struct sti_mbox_device *mdev = data;
+	struct sti_mbox_pdata *pdata = dev_get_platdata(mdev->dev);
+	struct mbox_chan *chan;
+	unsigned int instance;
+
+	for (instance = 0; instance < pdata->num_inst; instance++) {
+keep_looking:
+		chan = sti_mbox_irq_to_channel(mdev, instance);
+		if (!chan)
+			continue;
+
+		mbox_chan_received_data(chan, NULL);
+		sti_mbox_clear_irq(chan);
+		sti_mbox_enable_channel(chan);
+		goto keep_looking;
+	}
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t sti_mbox_irq_handler(int irq, void *data)
+{
+	struct sti_mbox_device *mdev = data;
+	struct sti_mbox_pdata *pdata = dev_get_platdata(mdev->dev);
+	struct sti_channel *chan_info;
+	struct mbox_chan *chan;
+	unsigned int instance;
+	int ret = IRQ_NONE;
+
+	for (instance = 0; instance < pdata->num_inst; instance++) {
+		chan = sti_mbox_irq_to_channel(mdev, instance);
+		if (!chan)
+			continue;
+		chan_info = chan->con_priv;
+
+		if (!sti_mbox_channel_is_enabled(chan)) {
+			dev_warn(mdev->dev,
+				 "Unexpected IRQ: %s\n"
+				 "  instance: %d: channel: %d [enabled: %x]\n",
+				 mdev->name, chan_info->instance,
+				 chan_info->channel, mdev->enabled[instance]);
+
+			/* Only handle IRQ if no other valid IRQs were found */
+			if (ret == IRQ_NONE)
+				ret = IRQ_HANDLED;
+			continue;
+		}
+
+		sti_mbox_disable_channel(chan);
+		ret = IRQ_WAKE_THREAD;
+	}
+
+	if (ret == IRQ_NONE)
+		dev_err(mdev->dev, "Spurious IRQ - was a channel requested?\n");
+
+	return ret;
+}
+
+static bool sti_mbox_tx_is_ready(struct mbox_chan *chan)
+{
+	struct sti_channel *chan_info = chan->con_priv;
+	struct sti_mbox_device *mdev = chan_info->mdev;
+	unsigned int instance = chan_info->instance;
+	unsigned int channel = chan_info->channel;
+	void __iomem *base = MBOX_BASE(mdev, instance);
+
+	if (!(readl_relaxed(base + STI_ENA_VAL_OFFSET) & BIT(channel))) {
+		dev_dbg(mdev->dev, "Mbox: %s: inst: %d, chan: %d disabled\n",
+			mdev->name, instance, channel);
+		return false;
+	}
+
+	if (readl_relaxed(base + STI_IRQ_VAL_OFFSET) & BIT(channel)) {
+		dev_dbg(mdev->dev, "Mbox: %s: inst: %d, chan: %d not ready\n",
+			mdev->name, instance, channel);
+		return false;
+	}
+
+	return true;
+}
+
+static int sti_mbox_send_data(struct mbox_chan *chan, void *data)
+{
+	struct sti_channel *chan_info = chan->con_priv;
+	struct sti_mbox_device *mdev = chan_info->mdev;
+	unsigned int instance = chan_info->instance;
+	unsigned int channel = chan_info->channel;
+	void __iomem *base = MBOX_BASE(mdev, instance);
+
+	/* Send event to co-processor */
+	writel_relaxed(BIT(channel), base + STI_IRQ_SET_OFFSET);
+
+	dev_dbg(mdev->dev,
+		"Sent via Mailbox %s: instance: %d channel: %d\n",
+		mdev->name, instance, channel);
+
+	return 0;
+}
+
+static int sti_mbox_startup_chan(struct mbox_chan *chan)
+{
+	sti_mbox_clear_irq(chan);
+	sti_mbox_enable_channel(chan);
+
+	return 0;
+}
+
+static void sti_mbox_shutdown_chan(struct mbox_chan *chan)
+{
+	struct sti_channel *chan_info = chan->con_priv;
+	struct mbox_controller *mbox = chan_info->mdev->mbox;
+	int i;
+
+	for (i = 0; i < mbox->num_chans; i++)
+		if (chan == &mbox->chans[i])
+			break;
+
+	if (mbox->num_chans == i) {
+		dev_warn(mbox->dev, "Request to free non-existent channel\n");
+		return;
+	}
+
+	/* Reset channel */
+	sti_mbox_disable_channel(chan);
+	sti_mbox_clear_irq(chan);
+	chan->con_priv = NULL;
+}
+
+static struct mbox_chan *sti_mbox_xlate(struct mbox_controller *mbox,
+					const struct of_phandle_args *spec)
+{
+	struct sti_mbox_device *mdev = dev_get_drvdata(mbox->dev);
+	struct sti_mbox_pdata *pdata = dev_get_platdata(mdev->dev);
+	struct sti_channel *chan_info;
+	struct mbox_chan *chan = NULL;
+	unsigned int instance  = spec->args[0];
+	unsigned int channel   = spec->args[1];
+	int i;
+
+	/* Bounds checking */
+	if (instance >= pdata->num_inst || channel  >= pdata->num_chan) {
+		dev_err(mbox->dev,
+			"Invalid channel requested instance: %d channel: %d\n",
+			instance, channel);
+		return ERR_PTR(-EINVAL);
+	}
+
+	for (i = 0; i < mbox->num_chans; i++) {
+		chan_info = mbox->chans[i].con_priv;
+
+		/* Is requested channel free? */
+		if (chan_info &&
+		    mbox->dev == chan_info->mdev->dev &&
+		    instance == chan_info->instance &&
+		    channel == chan_info->channel) {
+
+			dev_err(mbox->dev, "Channel in use\n");
+			return ERR_PTR(-EBUSY);
+		}
+
+		/*
+		 * Find the first free slot, then continue checking
+		 * to see if requested channel is in use
+		 */
+		if (!chan && !chan_info)
+			chan = &mbox->chans[i];
+	}
+
+	if (!chan) {
+		dev_err(mbox->dev, "No free channels left\n");
+		return ERR_PTR(-EBUSY);
+	}
+
+	chan_info = devm_kzalloc(mbox->dev, sizeof(*chan_info), GFP_KERNEL);
+	if (!chan_info)
+		return ERR_PTR(-ENOMEM);
+
+	chan_info->mdev		= mdev;
+	chan_info->instance	= instance;
+	chan_info->channel	= channel;
+
+	chan->con_priv = chan_info;
+
+	dev_info(mbox->dev,
+		 "Mbox: %s: Created channel: instance: %d channel: %d\n",
+		 mdev->name, instance, channel);
+
+	return chan;
+}
+
+static const struct mbox_chan_ops sti_mbox_ops = {
+	.startup	= sti_mbox_startup_chan,
+	.shutdown	= sti_mbox_shutdown_chan,
+	.send_data	= sti_mbox_send_data,
+	.last_tx_done	= sti_mbox_tx_is_ready,
+};
+
+static const struct sti_mbox_pdata mbox_stih407_pdata = {
+	.num_inst	= 4,
+	.num_chan	= 32,
+};
+
+static const struct of_device_id sti_mailbox_match[] = {
+	{
+		.compatible = "st,stih407-mailbox",
+		.data = (void *)&mbox_stih407_pdata
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, sti_mailbox_match);
+
+static int sti_mbox_probe(struct platform_device *pdev)
+{
+	struct mbox_controller *mbox;
+	struct sti_mbox_device *mdev;
+	struct device_node *np = pdev->dev.of_node;
+	struct mbox_chan *chans;
+	int irq;
+	int ret;
+
+	pdev->dev.platform_data = (struct sti_mbox_pdata *)device_get_match_data(&pdev->dev);
+	if (!pdev->dev.platform_data) {
+		dev_err(&pdev->dev, "No configuration found\n");
+		return -ENODEV;
+	}
+
+	mdev = devm_kzalloc(&pdev->dev, sizeof(*mdev), GFP_KERNEL);
+	if (!mdev)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, mdev);
+
+	mdev->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(mdev->base))
+		return PTR_ERR(mdev->base);
+
+	ret = of_property_read_string(np, "mbox-name", &mdev->name);
+	if (ret)
+		mdev->name = np->full_name;
+
+	mbox = devm_kzalloc(&pdev->dev, sizeof(*mbox), GFP_KERNEL);
+	if (!mbox)
+		return -ENOMEM;
+
+	chans = devm_kcalloc(&pdev->dev,
+			     STI_MBOX_CHAN_MAX, sizeof(*chans), GFP_KERNEL);
+	if (!chans)
+		return -ENOMEM;
+
+	mdev->dev		= &pdev->dev;
+	mdev->mbox		= mbox;
+
+	spin_lock_init(&mdev->lock);
+
+	/* STi Mailbox does not have a Tx-Done or Tx-Ready IRQ */
+	mbox->txdone_irq	= false;
+	mbox->txdone_poll	= true;
+	mbox->txpoll_period	= 100;
+	mbox->ops		= &sti_mbox_ops;
+	mbox->dev		= mdev->dev;
+	mbox->of_xlate		= sti_mbox_xlate;
+	mbox->chans		= chans;
+	mbox->num_chans		= STI_MBOX_CHAN_MAX;
+
+	ret = devm_mbox_controller_register(&pdev->dev, mbox);
+	if (ret)
+		return ret;
+
+	/* It's okay for Tx Mailboxes to not supply IRQs */
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0) {
+		dev_info(&pdev->dev,
+			 "%s: Registered Tx only Mailbox\n", mdev->name);
+		return 0;
+	}
+
+	ret = devm_request_threaded_irq(&pdev->dev, irq,
+					sti_mbox_irq_handler,
+					sti_mbox_thread_handler,
+					IRQF_ONESHOT, mdev->name, mdev);
+	if (ret)
+		return -EINVAL;
+
+	dev_info(&pdev->dev, "%s: Registered Tx/Rx Mailbox\n", mdev->name);
+
+	return 0;
+}
+
+static struct platform_driver sti_mbox_driver = {
+	.probe = sti_mbox_probe,
+	.driver = {
+		.name = "sti-mailbox",
+		.of_match_table = sti_mailbox_match,
+	},
+};
+module_platform_driver(sti_mbox_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("STMicroelectronics Mailbox Controller");
+MODULE_AUTHOR("Lee Jones <lee.jones@linaro.org");
+MODULE_ALIAS("platform:mailbox-sti");