x86/kprobes: Fix crash when probing CS CALL instructionsgrafted

When using eBPF to probe CS CALL instructions within a function,
a crash can be triggered.

The eBPF tool probes offset 257 of the __hrtimer_run_queues()
function:

<__hrtimer_run_queues+249>:  nopl   0x0(%rax,%rax,1)
<__hrtimer_run_queues+254>:  mov    %r14,%rdi
<__hrtimer_run_queues+257>:  cs call <__x86_indirect_thunk_r12>
<__hrtimer_run_queues+263>:  mov    %eax,%r12d
<__hrtimer_run_queues+266>:  xchg   %ax,%ax
<__hrtimer_run_queues+268>:  mov    %r13,%rdi

Which triggers this crash:

  BUG: unable to handle page fault for address: 00000000000f41c9
  #PF: supervisor write access in kernel mode
  #PF: error_code(0x0002) - not-present page
  PGD 0 P4D 0
  Oops: 0002 [#1] SMP NOPTI
  CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P
  RIP: 0010:__hrtimer_run_queues+0x106/0x230

Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is
at the 6th byte of the above CS CALL instruction. Since the CS CALL
instruction occupies 6 bytes, the exception occurred in the middle of that
call instruction.

The root cause is that when using eBPF tools to probe in the middle of a
function, a kprobe with INT3 is used as the underlying implementation.

During single-step emulation of the original CALL instruction,
int3_emulate_call() assumes that the probed CALL instruction is 5 bytes
long. However, the actual CS-prefixed CALL instruction occupies 6 bytes,
so it constructs an incorrect exception return address. When the CPU
returns from the kprobe handler, the next instruction to be executed is at
the address of the last byte of that CS CALL instruction. Coincidentally,
starting from that address, the CPU fetches and decodes a completely
different instruction, which ultimately triggers a kernel crash.

Fix the issue by using the actual instruction length obtained from
the instruction decoder when constructing the exception return
address, rather than relying on the hardcoded CALL_INSN_SIZE macro.

[ mingo: Refined the changelog ]

Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step")
Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Jinke Han <jinkehan@didiglobal.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Yafang Shao <laoar.shao@gmail.com>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000

1 files changed, 377 insertions, 0 deletions

diff --git a/arch/powerpc/boot/devtree.c b/arch/powerpc/boot/devtree.c
new file mode 100644
index 000000000..58fbcfcc9
--- /dev/null
+++ b/arch/powerpc/boot/devtree.c
@@ -0,0 +1,377 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * devtree.c - convenience functions for device tree manipulation
+ * Copyright 2007 David Gibson, IBM Corporation.
+ * Copyright (c) 2007 Freescale Semiconductor, Inc.
+ *
+ * Authors: David Gibson <david@gibson.dropbear.id.au>
+ *	    Scott Wood <scottwood@freescale.com>
+ */
+#include <stdarg.h>
+#include <stddef.h>
+#include "types.h"
+#include "string.h"
+#include "stdio.h"
+#include "ops.h"
+#include "of.h"
+
+void dt_fixup_memory(u64 start, u64 size)
+{
+	void *root, *memory;
+	int naddr, nsize, i;
+	u32 memreg[4];
+
+	root = finddevice("/");
+	if (getprop(root, "#address-cells", &naddr, sizeof(naddr)) < 0)
+		naddr = 2;
+	else
+		naddr = be32_to_cpu(naddr);
+	if (naddr < 1 || naddr > 2)
+		fatal("Can't cope with #address-cells == %d in /\n\r", naddr);
+
+	if (getprop(root, "#size-cells", &nsize, sizeof(nsize)) < 0)
+		nsize = 1;
+	else
+		nsize = be32_to_cpu(nsize);
+	if (nsize < 1 || nsize > 2)
+		fatal("Can't cope with #size-cells == %d in /\n\r", nsize);
+
+	i = 0;
+	if (naddr == 2)
+		memreg[i++] = cpu_to_be32(start >> 32);
+	memreg[i++] = cpu_to_be32(start & 0xffffffff);
+	if (nsize == 2)
+		memreg[i++] = cpu_to_be32(size >> 32);
+	memreg[i++] = cpu_to_be32(size & 0xffffffff);
+
+	memory = finddevice("/memory");
+	if (! memory) {
+		memory = create_node(NULL, "memory");
+		setprop_str(memory, "device_type", "memory");
+	}
+
+	printf("Memory <- <0x%x", be32_to_cpu(memreg[0]));
+	for (i = 1; i < (naddr + nsize); i++)
+		printf(" 0x%x", be32_to_cpu(memreg[i]));
+	printf("> (%ldMB)\n\r", (unsigned long)(size >> 20));
+
+	setprop(memory, "reg", memreg, (naddr + nsize)*sizeof(u32));
+}
+
+#define MHZ(x)	((x + 500000) / 1000000)
+
+void dt_fixup_cpu_clocks(u32 cpu, u32 tb, u32 bus)
+{
+	void *devp = NULL;
+
+	printf("CPU clock-frequency <- 0x%x (%dMHz)\n\r", cpu, MHZ(cpu));
+	printf("CPU timebase-frequency <- 0x%x (%dMHz)\n\r", tb, MHZ(tb));
+	if (bus > 0)
+		printf("CPU bus-frequency <- 0x%x (%dMHz)\n\r", bus, MHZ(bus));
+
+	while ((devp = find_node_by_devtype(devp, "cpu"))) {
+		setprop_val(devp, "clock-frequency", cpu_to_be32(cpu));
+		setprop_val(devp, "timebase-frequency", cpu_to_be32(tb));
+		if (bus > 0)
+			setprop_val(devp, "bus-frequency", cpu_to_be32(bus));
+	}
+
+	timebase_period_ns = 1000000000 / tb;
+}
+
+void dt_fixup_clock(const char *path, u32 freq)
+{
+	void *devp = finddevice(path);
+
+	if (devp) {
+		printf("%s: clock-frequency <- %x (%dMHz)\n\r", path, freq, MHZ(freq));
+		setprop_val(devp, "clock-frequency", cpu_to_be32(freq));
+	}
+}
+
+void dt_fixup_mac_address_by_alias(const char *alias, const u8 *addr)
+{
+	void *devp = find_node_by_alias(alias);
+
+	if (devp) {
+		printf("%s: local-mac-address <-"
+		       " %02x:%02x:%02x:%02x:%02x:%02x\n\r", alias,
+		       addr[0], addr[1], addr[2],
+		       addr[3], addr[4], addr[5]);
+
+		setprop(devp, "local-mac-address", addr, 6);
+	}
+}
+
+void dt_fixup_mac_address(u32 index, const u8 *addr)
+{
+	void *devp = find_node_by_prop_value(NULL, "linux,network-index",
+	                                     (void*)&index, sizeof(index));
+
+	if (devp) {
+		printf("ENET%d: local-mac-address <-"
+		       " %02x:%02x:%02x:%02x:%02x:%02x\n\r", index,
+		       addr[0], addr[1], addr[2],
+		       addr[3], addr[4], addr[5]);
+
+		setprop(devp, "local-mac-address", addr, 6);
+	}
+}
+
+void __dt_fixup_mac_addresses(u32 startindex, ...)
+{
+	va_list ap;
+	u32 index = startindex;
+	const u8 *addr;
+
+	va_start(ap, startindex);
+
+	while ((addr = va_arg(ap, const u8 *)))
+		dt_fixup_mac_address(index++, addr);
+
+	va_end(ap);
+}
+
+#define MAX_ADDR_CELLS 4
+
+void dt_get_reg_format(void *node, u32 *naddr, u32 *nsize)
+{
+	if (getprop(node, "#address-cells", naddr, 4) != 4)
+		*naddr = 2;
+	else
+		*naddr = be32_to_cpu(*naddr);
+	if (getprop(node, "#size-cells", nsize, 4) != 4)
+		*nsize = 1;
+	else
+		*nsize = be32_to_cpu(*nsize);
+}
+
+static void copy_val(u32 *dest, u32 *src, int naddr)
+{
+	int pad = MAX_ADDR_CELLS - naddr;
+
+	memset(dest, 0, pad * 4);
+	memcpy(dest + pad, src, naddr * 4);
+}
+
+static int sub_reg(u32 *reg, u32 *sub)
+{
+	int i, borrow = 0;
+
+	for (i = MAX_ADDR_CELLS - 1; i >= 0; i--) {
+		int prev_borrow = borrow;
+		borrow = reg[i] < sub[i] + prev_borrow;
+		reg[i] -= sub[i] + prev_borrow;
+	}
+
+	return !borrow;
+}
+
+static int add_reg(u32 *reg, u32 *add, int naddr)
+{
+	int i, carry = 0;
+
+	for (i = MAX_ADDR_CELLS - 1; i >= MAX_ADDR_CELLS - naddr; i--) {
+		u64 tmp = (u64)be32_to_cpu(reg[i]) + be32_to_cpu(add[i]) + carry;
+		carry = tmp >> 32;
+		reg[i] = cpu_to_be32((u32)tmp);
+	}
+
+	return !carry;
+}
+
+/* It is assumed that if the first byte of reg fits in a
+ * range, then the whole reg block fits.
+ */
+static int compare_reg(u32 *reg, u32 *range, u32 *rangesize)
+{
+	int i;
+	u32 end;
+
+	for (i = 0; i < MAX_ADDR_CELLS; i++) {
+		if (be32_to_cpu(reg[i]) < be32_to_cpu(range[i]))
+			return 0;
+		if (be32_to_cpu(reg[i]) > be32_to_cpu(range[i]))
+			break;
+	}
+
+	for (i = 0; i < MAX_ADDR_CELLS; i++) {
+		end = be32_to_cpu(range[i]) + be32_to_cpu(rangesize[i]);
+
+		if (be32_to_cpu(reg[i]) < end)
+			break;
+		if (be32_to_cpu(reg[i]) > end)
+			return 0;
+	}
+
+	return reg[i] != end;
+}
+
+/* reg must be MAX_ADDR_CELLS */
+static int find_range(u32 *reg, u32 *ranges, int nregaddr,
+                      int naddr, int nsize, int buflen)
+{
+	int nrange = nregaddr + naddr + nsize;
+	int i;
+
+	for (i = 0; i + nrange <= buflen; i += nrange) {
+		u32 range_addr[MAX_ADDR_CELLS];
+		u32 range_size[MAX_ADDR_CELLS];
+
+		copy_val(range_addr, ranges + i, nregaddr);
+		copy_val(range_size, ranges + i + nregaddr + naddr, nsize);
+
+		if (compare_reg(reg, range_addr, range_size))
+			return i;
+	}
+
+	return -1;
+}
+
+/* Currently only generic buses without special encodings are supported.
+ * In particular, PCI is not supported.  Also, only the beginning of the
+ * reg block is tracked; size is ignored except in ranges.
+ */
+static u32 prop_buf[MAX_PROP_LEN / 4];
+
+static int dt_xlate(void *node, int res, int reglen, unsigned long *addr,
+		unsigned long *size)
+{
+	u32 last_addr[MAX_ADDR_CELLS];
+	u32 this_addr[MAX_ADDR_CELLS];
+	void *parent;
+	u64 ret_addr, ret_size;
+	u32 naddr, nsize, prev_naddr, prev_nsize;
+	int buflen, offset;
+
+	parent = get_parent(node);
+	if (!parent)
+		return 0;
+
+	dt_get_reg_format(parent, &naddr, &nsize);
+	if (nsize > 2)
+		return 0;
+
+	offset = (naddr + nsize) * res;
+
+	if (reglen < offset + naddr + nsize ||
+	    MAX_PROP_LEN < (offset + naddr + nsize) * 4)
+		return 0;
+
+	copy_val(last_addr, prop_buf + offset, naddr);
+
+	ret_size = be32_to_cpu(prop_buf[offset + naddr]);
+	if (nsize == 2) {
+		ret_size <<= 32;
+		ret_size |= be32_to_cpu(prop_buf[offset + naddr + 1]);
+	}
+
+	for (;;) {
+		prev_naddr = naddr;
+		prev_nsize = nsize;
+		node = parent;
+
+		parent = get_parent(node);
+		if (!parent)
+			break;
+
+		dt_get_reg_format(parent, &naddr, &nsize);
+
+		buflen = getprop(node, "ranges", prop_buf,
+				sizeof(prop_buf));
+		if (buflen == 0)
+			continue;
+		if (buflen < 0 || buflen > sizeof(prop_buf))
+			return 0;
+
+		offset = find_range(last_addr, prop_buf, prev_naddr,
+		                    naddr, prev_nsize, buflen / 4);
+		if (offset < 0)
+			return 0;
+
+		copy_val(this_addr, prop_buf + offset, prev_naddr);
+
+		if (!sub_reg(last_addr, this_addr))
+			return 0;
+
+		copy_val(this_addr, prop_buf + offset + prev_naddr, naddr);
+
+		if (!add_reg(last_addr, this_addr, naddr))
+			return 0;
+	}
+
+	if (naddr > 2)
+		return 0;
+
+	ret_addr = ((u64)be32_to_cpu(last_addr[2]) << 32) | be32_to_cpu(last_addr[3]);
+	if (sizeof(void *) == 4 &&
+	    (ret_addr >= 0x100000000ULL || ret_size > 0x100000000ULL ||
+	     ret_addr + ret_size > 0x100000000ULL))
+		return 0;
+
+	*addr = ret_addr;
+	if (size)
+		*size = ret_size;
+
+	return 1;
+}
+
+int dt_xlate_reg(void *node, int res, unsigned long *addr, unsigned long *size)
+{
+	int reglen;
+
+	reglen = getprop(node, "reg", prop_buf, sizeof(prop_buf)) / 4;
+	return dt_xlate(node, res, reglen, addr, size);
+}
+
+int dt_xlate_addr(void *node, u32 *buf, int buflen, unsigned long *xlated_addr)
+{
+
+	if (buflen > sizeof(prop_buf))
+		return 0;
+
+	memcpy(prop_buf, buf, buflen);
+	return dt_xlate(node, 0, buflen / 4, xlated_addr, NULL);
+}
+
+int dt_is_compatible(void *node, const char *compat)
+{
+	char *buf = (char *)prop_buf;
+	int len, pos;
+
+	len = getprop(node, "compatible", buf, MAX_PROP_LEN);
+	if (len < 0)
+		return 0;
+
+	for (pos = 0; pos < len; pos++) {
+		if (!strcmp(buf + pos, compat))
+			return 1;
+
+		pos += strnlen(&buf[pos], len - pos);
+	}
+
+	return 0;
+}
+
+int dt_get_virtual_reg(void *node, void **addr, int nres)
+{
+	unsigned long xaddr;
+	int n, i;
+
+	n = getprop(node, "virtual-reg", addr, nres * 4);
+	if (n > 0) {
+		for (i = 0; i < n/4; i ++)
+			((u32 *)addr)[i] = be32_to_cpu(((u32 *)addr)[i]);
+		return n / 4;
+	}
+
+	for (n = 0; n < nres; n++) {
+		if (!dt_xlate_reg(node, n, &xaddr, NULL))
+			break;
+
+		addr[n] = (void *)xaddr;
+	}
+
+	return n;
+}
+