x86/kprobes: Fix crash when probing CS CALL instructionsgrafted

When using eBPF to probe CS CALL instructions within a function,
a crash can be triggered.

The eBPF tool probes offset 257 of the __hrtimer_run_queues()
function:

<__hrtimer_run_queues+249>:  nopl   0x0(%rax,%rax,1)
<__hrtimer_run_queues+254>:  mov    %r14,%rdi
<__hrtimer_run_queues+257>:  cs call <__x86_indirect_thunk_r12>
<__hrtimer_run_queues+263>:  mov    %eax,%r12d
<__hrtimer_run_queues+266>:  xchg   %ax,%ax
<__hrtimer_run_queues+268>:  mov    %r13,%rdi

Which triggers this crash:

  BUG: unable to handle page fault for address: 00000000000f41c9
  #PF: supervisor write access in kernel mode
  #PF: error_code(0x0002) - not-present page
  PGD 0 P4D 0
  Oops: 0002 [#1] SMP NOPTI
  CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P
  RIP: 0010:__hrtimer_run_queues+0x106/0x230

Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is
at the 6th byte of the above CS CALL instruction. Since the CS CALL
instruction occupies 6 bytes, the exception occurred in the middle of that
call instruction.

The root cause is that when using eBPF tools to probe in the middle of a
function, a kprobe with INT3 is used as the underlying implementation.

During single-step emulation of the original CALL instruction,
int3_emulate_call() assumes that the probed CALL instruction is 5 bytes
long. However, the actual CS-prefixed CALL instruction occupies 6 bytes,
so it constructs an incorrect exception return address. When the CPU
returns from the kprobe handler, the next instruction to be executed is at
the address of the last byte of that CS CALL instruction. Coincidentally,
starting from that address, the CPU fetches and decodes a completely
different instruction, which ultimately triggers a kernel crash.

Fix the issue by using the actual instruction length obtained from
the instruction decoder when constructing the exception return
address, rather than relying on the hardcoded CALL_INSN_SIZE macro.

[ mingo: Refined the changelog ]

Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step")
Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Jinke Han <jinkehan@didiglobal.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Yafang Shao <laoar.shao@gmail.com>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000

1 files changed, 241 insertions, 0 deletions

diff --git a/arch/arm/mach-imx/src.c b/arch/arm/mach-imx/src.c
new file mode 100644
index 000000000..c3c80b4c3
--- /dev/null
+++ b/arch/arm/mach-imx/src.c
@@ -0,0 +1,241 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright 2011 Freescale Semiconductor, Inc.
+ * Copyright 2011 Linaro Ltd.
+ */
+
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/platform_device.h>
+#include <linux/reset-controller.h>
+#include <linux/smp.h>
+#include <asm/smp_plat.h>
+#include "common.h"
+#include "hardware.h"
+
+#define SRC_SCR				0x000
+#define SRC_GPR1_V1			0x020
+#define SRC_GPR1_V2			0x074
+#define SRC_GPR1(gpr_v2)		((gpr_v2) ? SRC_GPR1_V2 : SRC_GPR1_V1)
+#define BP_SRC_SCR_WARM_RESET_ENABLE	0
+#define BP_SRC_SCR_SW_GPU_RST		1
+#define BP_SRC_SCR_SW_VPU_RST		2
+#define BP_SRC_SCR_SW_IPU1_RST		3
+#define BP_SRC_SCR_SW_OPEN_VG_RST	4
+#define BP_SRC_SCR_SW_IPU2_RST		12
+#define BP_SRC_SCR_CORE1_RST		14
+#define BP_SRC_SCR_CORE1_ENABLE		22
+/* below is for i.MX7D */
+#define SRC_A7RCR1			0x008
+#define BP_SRC_A7RCR1_A7_CORE1_ENABLE	1
+#define GPC_CPU_PGC_SW_PUP_REQ		0xf0
+#define GPC_CPU_PGC_SW_PDN_REQ		0xfc
+#define GPC_PGC_C1			0x840
+#define BM_CPU_PGC_SW_PDN_PUP_REQ_CORE1_A7	0x2
+
+static void __iomem *src_base;
+static DEFINE_SPINLOCK(scr_lock);
+static bool gpr_v2;
+static void __iomem *gpc_base;
+
+static const int sw_reset_bits[5] = {
+	BP_SRC_SCR_SW_GPU_RST,
+	BP_SRC_SCR_SW_VPU_RST,
+	BP_SRC_SCR_SW_IPU1_RST,
+	BP_SRC_SCR_SW_OPEN_VG_RST,
+	BP_SRC_SCR_SW_IPU2_RST
+};
+
+static int imx_src_reset_module(struct reset_controller_dev *rcdev,
+		unsigned long sw_reset_idx)
+{
+	unsigned long timeout;
+	unsigned long flags;
+	int bit;
+	u32 val;
+
+	if (sw_reset_idx >= ARRAY_SIZE(sw_reset_bits))
+		return -EINVAL;
+
+	bit = 1 << sw_reset_bits[sw_reset_idx];
+
+	spin_lock_irqsave(&scr_lock, flags);
+	val = readl_relaxed(src_base + SRC_SCR);
+	val |= bit;
+	writel_relaxed(val, src_base + SRC_SCR);
+	spin_unlock_irqrestore(&scr_lock, flags);
+
+	timeout = jiffies + msecs_to_jiffies(1000);
+	while (readl(src_base + SRC_SCR) & bit) {
+		if (time_after(jiffies, timeout))
+			return -ETIME;
+		cpu_relax();
+	}
+
+	return 0;
+}
+
+static const struct reset_control_ops imx_src_ops = {
+	.reset = imx_src_reset_module,
+};
+
+static void imx_gpcv2_set_m_core_pgc(bool enable, u32 offset)
+{
+	writel_relaxed(enable, gpc_base + offset);
+}
+
+/*
+ * The motivation for bringing up the second i.MX7D core inside the kernel
+ * is that legacy vendor bootloaders usually do not implement PSCI support.
+ * This is a significant blocker for systems in the field that are running old
+ * bootloader versions to upgrade to a modern mainline kernel version, as only
+ * one CPU of the i.MX7D would be brought up.
+ * Bring up the second i.MX7D core inside the kernel to make the migration
+ * path to mainline kernel easier for the existing iMX7D users.
+ */
+void imx_gpcv2_set_core1_pdn_pup_by_software(bool pdn)
+{
+	u32 reg = pdn ? GPC_CPU_PGC_SW_PDN_REQ : GPC_CPU_PGC_SW_PUP_REQ;
+	u32 val, pup;
+	int ret;
+
+	imx_gpcv2_set_m_core_pgc(true, GPC_PGC_C1);
+	val = readl_relaxed(gpc_base + reg);
+	val |= BM_CPU_PGC_SW_PDN_PUP_REQ_CORE1_A7;
+	writel_relaxed(val, gpc_base + reg);
+
+	ret = readl_relaxed_poll_timeout_atomic(gpc_base + reg, pup,
+				!(pup & BM_CPU_PGC_SW_PDN_PUP_REQ_CORE1_A7),
+				5, 1000000);
+	if (ret < 0) {
+		pr_err("i.MX7D: CORE1_A7 power up timeout\n");
+		val &= ~BM_CPU_PGC_SW_PDN_PUP_REQ_CORE1_A7;
+		writel_relaxed(val, gpc_base + reg);
+	}
+
+	imx_gpcv2_set_m_core_pgc(false, GPC_PGC_C1);
+}
+
+void imx_enable_cpu(int cpu, bool enable)
+{
+	u32 mask, val;
+
+	cpu = cpu_logical_map(cpu);
+	spin_lock(&scr_lock);
+	if (gpr_v2) {
+		if (enable)
+			imx_gpcv2_set_core1_pdn_pup_by_software(false);
+
+		mask = 1 << (BP_SRC_A7RCR1_A7_CORE1_ENABLE + cpu - 1);
+		val = readl_relaxed(src_base + SRC_A7RCR1);
+		val = enable ? val | mask : val & ~mask;
+		writel_relaxed(val, src_base + SRC_A7RCR1);
+	} else {
+		mask = 1 << (BP_SRC_SCR_CORE1_ENABLE + cpu - 1);
+		val = readl_relaxed(src_base + SRC_SCR);
+		val = enable ? val | mask : val & ~mask;
+		val |= 1 << (BP_SRC_SCR_CORE1_RST + cpu - 1);
+		writel_relaxed(val, src_base + SRC_SCR);
+	}
+	spin_unlock(&scr_lock);
+}
+
+void imx_set_cpu_jump(int cpu, void *jump_addr)
+{
+	cpu = cpu_logical_map(cpu);
+	writel_relaxed(__pa_symbol(jump_addr),
+		       src_base + SRC_GPR1(gpr_v2) + cpu * 8);
+}
+
+u32 imx_get_cpu_arg(int cpu)
+{
+	cpu = cpu_logical_map(cpu);
+	return readl_relaxed(src_base + SRC_GPR1(gpr_v2) + cpu * 8 + 4);
+}
+
+void imx_set_cpu_arg(int cpu, u32 arg)
+{
+	cpu = cpu_logical_map(cpu);
+	writel_relaxed(arg, src_base + SRC_GPR1(gpr_v2) + cpu * 8 + 4);
+}
+
+void __init imx_src_init(void)
+{
+	struct device_node *np;
+	u32 val;
+
+	np = of_find_compatible_node(NULL, NULL, "fsl,imx51-src");
+	if (!np)
+		return;
+	src_base = of_iomap(np, 0);
+	of_node_put(np);
+	WARN_ON(!src_base);
+
+	/*
+	 * force warm reset sources to generate cold reset
+	 * for a more reliable restart
+	 */
+	spin_lock(&scr_lock);
+	val = readl_relaxed(src_base + SRC_SCR);
+	val &= ~(1 << BP_SRC_SCR_WARM_RESET_ENABLE);
+	writel_relaxed(val, src_base + SRC_SCR);
+	spin_unlock(&scr_lock);
+}
+
+void __init imx7_src_init(void)
+{
+	struct device_node *np;
+
+	gpr_v2 = true;
+
+	np = of_find_compatible_node(NULL, NULL, "fsl,imx7d-src");
+	if (!np)
+		return;
+
+	src_base = of_iomap(np, 0);
+	of_node_put(np);
+	if (!src_base)
+		return;
+
+	np = of_find_compatible_node(NULL, NULL, "fsl,imx7d-gpc");
+	if (!np)
+		return;
+
+	gpc_base = of_iomap(np, 0);
+	of_node_put(np);
+	if (!gpc_base)
+		return;
+}
+
+static const struct of_device_id imx_src_dt_ids[] = {
+	{ .compatible = "fsl,imx51-src" },
+	{ /* sentinel */ }
+};
+
+static int imx_src_probe(struct platform_device *pdev)
+{
+	struct reset_controller_dev *rcdev;
+
+	rcdev = devm_kzalloc(&pdev->dev, sizeof(*rcdev), GFP_KERNEL);
+	if (!rcdev)
+		return -ENOMEM;
+
+	rcdev->ops = &imx_src_ops;
+	rcdev->dev = &pdev->dev;
+	rcdev->of_node = pdev->dev.of_node;
+	rcdev->nr_resets = ARRAY_SIZE(sw_reset_bits);
+
+	return devm_reset_controller_register(&pdev->dev, rcdev);
+}
+
+static struct platform_driver imx_src_driver = {
+	.driver = {
+		.name = "imx-src",
+		.of_match_table = imx_src_dt_ids,
+	},
+	.probe = imx_src_probe,
+};
+builtin_platform_driver(imx_src_driver);