x86/kprobes: Fix crash when probing CS CALL instructionsgrafted

When using eBPF to probe CS CALL instructions within a function,
a crash can be triggered.

The eBPF tool probes offset 257 of the __hrtimer_run_queues()
function:

<__hrtimer_run_queues+249>:  nopl   0x0(%rax,%rax,1)
<__hrtimer_run_queues+254>:  mov    %r14,%rdi
<__hrtimer_run_queues+257>:  cs call <__x86_indirect_thunk_r12>
<__hrtimer_run_queues+263>:  mov    %eax,%r12d
<__hrtimer_run_queues+266>:  xchg   %ax,%ax
<__hrtimer_run_queues+268>:  mov    %r13,%rdi

Which triggers this crash:

  BUG: unable to handle page fault for address: 00000000000f41c9
  #PF: supervisor write access in kernel mode
  #PF: error_code(0x0002) - not-present page
  PGD 0 P4D 0
  Oops: 0002 [#1] SMP NOPTI
  CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P
  RIP: 0010:__hrtimer_run_queues+0x106/0x230

Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is
at the 6th byte of the above CS CALL instruction. Since the CS CALL
instruction occupies 6 bytes, the exception occurred in the middle of that
call instruction.

The root cause is that when using eBPF tools to probe in the middle of a
function, a kprobe with INT3 is used as the underlying implementation.

During single-step emulation of the original CALL instruction,
int3_emulate_call() assumes that the probed CALL instruction is 5 bytes
long. However, the actual CS-prefixed CALL instruction occupies 6 bytes,
so it constructs an incorrect exception return address. When the CPU
returns from the kprobe handler, the next instruction to be executed is at
the address of the last byte of that CS CALL instruction. Coincidentally,
starting from that address, the CPU fetches and decodes a completely
different instruction, which ultimately triggers a kernel crash.

Fix the issue by using the actual instruction length obtained from
the instruction decoder when constructing the exception return
address, rather than relying on the hardcoded CALL_INSN_SIZE macro.

[ mingo: Refined the changelog ]

Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step")
Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Jinke Han <jinkehan@didiglobal.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Yafang Shao <laoar.shao@gmail.com>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000

1 files changed, 230 insertions, 0 deletions

diff --git a/drivers/clk/imx/clk.c b/drivers/clk/imx/clk.c
new file mode 100644
index 000000000..f38506830
--- /dev/null
+++ b/drivers/clk/imx/clk.c
@@ -0,0 +1,230 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/bits.h>
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include "clk.h"
+
+#define CCM_CCDR			0x4
+#define CCDR_MMDC_CH0_MASK		BIT(17)
+#define CCDR_MMDC_CH1_MASK		BIT(16)
+
+DEFINE_SPINLOCK(imx_ccm_lock);
+EXPORT_SYMBOL_GPL(imx_ccm_lock);
+
+bool mcore_booted;
+EXPORT_SYMBOL_GPL(mcore_booted);
+
+void imx_unregister_hw_clocks(struct clk_hw *hws[], unsigned int count)
+{
+	unsigned int i;
+
+	for (i = 0; i < count; i++)
+		clk_hw_unregister(hws[i]);
+}
+EXPORT_SYMBOL_GPL(imx_unregister_hw_clocks);
+
+void imx_mmdc_mask_handshake(void __iomem *ccm_base,
+				    unsigned int chn)
+{
+	unsigned int reg;
+
+	reg = readl_relaxed(ccm_base + CCM_CCDR);
+	reg |= chn == 0 ? CCDR_MMDC_CH0_MASK : CCDR_MMDC_CH1_MASK;
+	writel_relaxed(reg, ccm_base + CCM_CCDR);
+}
+
+void imx_check_clocks(struct clk *clks[], unsigned int count)
+{
+	unsigned i;
+
+	for (i = 0; i < count; i++)
+		if (IS_ERR(clks[i]))
+			pr_err("i.MX clk %u: register failed with %ld\n",
+			       i, PTR_ERR(clks[i]));
+}
+
+void imx_check_clk_hws(struct clk_hw *clks[], unsigned int count)
+{
+	unsigned int i;
+
+	for (i = 0; i < count; i++)
+		if (IS_ERR(clks[i]))
+			pr_err("i.MX clk %u: register failed with %ld\n",
+			       i, PTR_ERR(clks[i]));
+}
+EXPORT_SYMBOL_GPL(imx_check_clk_hws);
+
+static struct clk *imx_obtain_fixed_clock_from_dt(const char *name)
+{
+	struct of_phandle_args phandle;
+	struct clk *clk = ERR_PTR(-ENODEV);
+	char *path;
+
+	path = kasprintf(GFP_KERNEL, "/clocks/%s", name);
+	if (!path)
+		return ERR_PTR(-ENOMEM);
+
+	phandle.np = of_find_node_by_path(path);
+	kfree(path);
+
+	if (phandle.np) {
+		clk = of_clk_get_from_provider(&phandle);
+		of_node_put(phandle.np);
+	}
+	return clk;
+}
+
+struct clk *imx_obtain_fixed_clock(
+			const char *name, unsigned long rate)
+{
+	struct clk *clk;
+
+	clk = imx_obtain_fixed_clock_from_dt(name);
+	if (IS_ERR(clk))
+		clk = imx_clk_fixed(name, rate);
+	return clk;
+}
+
+struct clk_hw *imx_obtain_fixed_clock_hw(
+			const char *name, unsigned long rate)
+{
+	struct clk *clk;
+
+	clk = imx_obtain_fixed_clock_from_dt(name);
+	if (IS_ERR(clk))
+		clk = imx_clk_fixed(name, rate);
+	return __clk_get_hw(clk);
+}
+
+struct clk_hw *imx_obtain_fixed_of_clock(struct device_node *np,
+					 const char *name, unsigned long rate)
+{
+	struct clk *clk = of_clk_get_by_name(np, name);
+	struct clk_hw *hw;
+
+	if (IS_ERR(clk))
+		hw = imx_obtain_fixed_clock_hw(name, rate);
+	else
+		hw = __clk_get_hw(clk);
+
+	return hw;
+}
+
+struct clk_hw *imx_get_clk_hw_by_name(struct device_node *np, const char *name)
+{
+	struct clk *clk;
+
+	clk = of_clk_get_by_name(np, name);
+	if (IS_ERR(clk))
+		return ERR_PTR(-ENOENT);
+
+	return __clk_get_hw(clk);
+}
+EXPORT_SYMBOL_GPL(imx_get_clk_hw_by_name);
+
+/*
+ * This fixups the register CCM_CSCMR1 write value.
+ * The write/read/divider values of the aclk_podf field
+ * of that register have the relationship described by
+ * the following table:
+ *
+ * write value       read value        divider
+ * 3b'000            3b'110            7
+ * 3b'001            3b'111            8
+ * 3b'010            3b'100            5
+ * 3b'011            3b'101            6
+ * 3b'100            3b'010            3
+ * 3b'101            3b'011            4
+ * 3b'110            3b'000            1
+ * 3b'111            3b'001            2(default)
+ *
+ * That's why we do the xor operation below.
+ */
+#define CSCMR1_FIXUP	0x00600000
+
+void imx_cscmr1_fixup(u32 *val)
+{
+	*val ^= CSCMR1_FIXUP;
+	return;
+}
+
+#ifndef MODULE
+
+static bool imx_keep_uart_clocks;
+static int imx_enabled_uart_clocks;
+static struct clk **imx_uart_clocks;
+
+static int __init imx_keep_uart_clocks_param(char *str)
+{
+	imx_keep_uart_clocks = 1;
+
+	return 0;
+}
+__setup_param("earlycon", imx_keep_uart_earlycon,
+	      imx_keep_uart_clocks_param, 0);
+__setup_param("earlyprintk", imx_keep_uart_earlyprintk,
+	      imx_keep_uart_clocks_param, 0);
+
+void imx_register_uart_clocks(void)
+{
+	unsigned int num __maybe_unused;
+
+	imx_enabled_uart_clocks = 0;
+
+/* i.MX boards use device trees now.  For build tests without CONFIG_OF, do nothing */
+#ifdef CONFIG_OF
+	if (imx_keep_uart_clocks) {
+		int i;
+
+		num = of_clk_get_parent_count(of_stdout);
+		if (!num)
+			return;
+
+		if (!of_stdout)
+			return;
+
+		imx_uart_clocks = kzalloc_objs(struct clk *, num);
+		if (!imx_uart_clocks)
+			return;
+
+		for (i = 0; i < num; i++) {
+			imx_uart_clocks[imx_enabled_uart_clocks] = of_clk_get(of_stdout, i);
+
+			/* Stop if there are no more of_stdout references */
+			if (IS_ERR(imx_uart_clocks[imx_enabled_uart_clocks]))
+				return;
+
+			/* Only enable the clock if it's not NULL */
+			if (imx_uart_clocks[imx_enabled_uart_clocks])
+				clk_prepare_enable(imx_uart_clocks[imx_enabled_uart_clocks++]);
+		}
+	}
+#endif
+}
+
+static int __init imx_clk_disable_uart(void)
+{
+	if (imx_keep_uart_clocks && imx_enabled_uart_clocks) {
+		int i;
+
+		for (i = 0; i < imx_enabled_uart_clocks; i++) {
+			clk_disable_unprepare(imx_uart_clocks[i]);
+			clk_put(imx_uart_clocks[i]);
+		}
+	}
+
+	kfree(imx_uart_clocks);
+
+	return 0;
+}
+late_initcall_sync(imx_clk_disable_uart);
+#endif
+
+MODULE_DESCRIPTION("Common clock support for NXP i.MX SoC family");
+MODULE_LICENSE("GPL v2");