x86/kprobes: Fix crash when probing CS CALL instructionsgrafted

When using eBPF to probe CS CALL instructions within a function,
a crash can be triggered.

The eBPF tool probes offset 257 of the __hrtimer_run_queues()
function:

<__hrtimer_run_queues+249>:  nopl   0x0(%rax,%rax,1)
<__hrtimer_run_queues+254>:  mov    %r14,%rdi
<__hrtimer_run_queues+257>:  cs call <__x86_indirect_thunk_r12>
<__hrtimer_run_queues+263>:  mov    %eax,%r12d
<__hrtimer_run_queues+266>:  xchg   %ax,%ax
<__hrtimer_run_queues+268>:  mov    %r13,%rdi

Which triggers this crash:

  BUG: unable to handle page fault for address: 00000000000f41c9
  #PF: supervisor write access in kernel mode
  #PF: error_code(0x0002) - not-present page
  PGD 0 P4D 0
  Oops: 0002 [#1] SMP NOPTI
  CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P
  RIP: 0010:__hrtimer_run_queues+0x106/0x230

Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is
at the 6th byte of the above CS CALL instruction. Since the CS CALL
instruction occupies 6 bytes, the exception occurred in the middle of that
call instruction.

The root cause is that when using eBPF tools to probe in the middle of a
function, a kprobe with INT3 is used as the underlying implementation.

During single-step emulation of the original CALL instruction,
int3_emulate_call() assumes that the probed CALL instruction is 5 bytes
long. However, the actual CS-prefixed CALL instruction occupies 6 bytes,
so it constructs an incorrect exception return address. When the CPU
returns from the kprobe handler, the next instruction to be executed is at
the address of the last byte of that CS CALL instruction. Coincidentally,
starting from that address, the CPU fetches and decodes a completely
different instruction, which ultimately triggers a kernel crash.

Fix the issue by using the actual instruction length obtained from
the instruction decoder when constructing the exception return
address, rather than relying on the hardcoded CALL_INSN_SIZE macro.

[ mingo: Refined the changelog ]

Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step")
Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Jinke Han <jinkehan@didiglobal.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Yafang Shao <laoar.shao@gmail.com>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000

1 files changed, 293 insertions, 0 deletions

diff --git a/scripts/dtc/data.c b/scripts/dtc/data.c
new file mode 100644
index 000000000..5b25aa060
--- /dev/null
+++ b/scripts/dtc/data.c
@@ -0,0 +1,293 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * (C) Copyright David Gibson <dwg@au1.ibm.com>, IBM Corporation.  2005.
+ */
+
+#include "dtc.h"
+
+void data_free(struct data d)
+{
+	struct marker *m, *nm;
+
+	m = d.markers;
+	while (m) {
+		nm = m->next;
+		free(m->ref);
+		free(m);
+		m = nm;
+	}
+
+	if (d.val)
+		free(d.val);
+}
+
+struct data data_grow_for(struct data d, unsigned int xlen)
+{
+	struct data nd;
+	unsigned int newsize;
+
+	if (xlen == 0)
+		return d;
+
+	nd = d;
+
+	newsize = xlen;
+
+	while ((d.len + xlen) > newsize)
+		newsize *= 2;
+
+	nd.val = xrealloc(d.val, newsize);
+
+	return nd;
+}
+
+struct data data_copy_mem(const char *mem, int len)
+{
+	struct data d;
+
+	d = data_grow_for(empty_data, len);
+
+	d.len = len;
+	memcpy(d.val, mem, len);
+
+	return d;
+}
+
+struct data data_copy_escape_string(const char *s, int len)
+{
+	int i = 0;
+	struct data d;
+	char *q;
+
+	d = data_add_marker(empty_data, TYPE_STRING, NULL);
+	d = data_grow_for(d, len + 1);
+
+	q = d.val;
+	while (i < len) {
+		char c = s[i++];
+
+		if (c == '\\')
+			c = get_escape_char(s, &i);
+
+		q[d.len++] = c;
+	}
+
+	q[d.len++] = '\0';
+	return d;
+}
+
+struct data data_copy_file(FILE *f, size_t maxlen)
+{
+	struct data d = empty_data;
+
+	d = data_add_marker(d, TYPE_NONE, NULL);
+	while (!feof(f) && (d.len < maxlen)) {
+		size_t chunksize, ret;
+
+		if (maxlen == (size_t)-1)
+			chunksize = 4096;
+		else
+			chunksize = maxlen - d.len;
+
+		d = data_grow_for(d, chunksize);
+		ret = fread(d.val + d.len, 1, chunksize, f);
+
+		if (ferror(f))
+			die("Error reading file into data: %s", strerror(errno));
+
+		if (d.len + ret < d.len)
+			die("Overflow reading file into data\n");
+
+		d.len += ret;
+	}
+
+	return d;
+}
+
+struct data data_append_data(struct data d, const void *p, int len)
+{
+	d = data_grow_for(d, len);
+	memcpy(d.val + d.len, p, len);
+	d.len += len;
+	return d;
+}
+
+struct data data_insert_at_marker(struct data d, struct marker *m,
+				  const void *p, int len)
+{
+	d = data_grow_for(d, len);
+	memmove(d.val + m->offset + len, d.val + m->offset, d.len - m->offset);
+	memcpy(d.val + m->offset, p, len);
+	d.len += len;
+
+	/* Adjust all markers after the one we're inserting at */
+	m = m->next;
+	for_each_marker(m)
+		m->offset += len;
+	return d;
+}
+
+static struct data data_append_markers(struct data d, struct marker *m)
+{
+	struct marker **mp = &d.markers;
+
+	/* Find the end of the markerlist */
+	while (*mp)
+		mp = &((*mp)->next);
+	*mp = m;
+	return d;
+}
+
+struct data data_merge(struct data d1, struct data d2)
+{
+	struct data d;
+	struct marker *m2 = d2.markers;
+
+	d = data_append_markers(data_append_data(d1, d2.val, d2.len), m2);
+
+	/* Adjust for the length of d1 */
+	for_each_marker(m2)
+		m2->offset += d1.len;
+
+	d2.markers = NULL; /* So data_free() doesn't clobber them */
+	data_free(d2);
+
+	return d;
+}
+
+struct data data_append_integer(struct data d, uint64_t value, int bits)
+{
+	uint8_t value_8;
+	fdt16_t value_16;
+	fdt32_t value_32;
+	fdt64_t value_64;
+
+	switch (bits) {
+	case 8:
+		value_8 = value;
+		return data_append_data(d, &value_8, 1);
+
+	case 16:
+		value_16 = cpu_to_fdt16(value);
+		return data_append_data(d, &value_16, 2);
+
+	case 32:
+		value_32 = cpu_to_fdt32(value);
+		return data_append_data(d, &value_32, 4);
+
+	case 64:
+		value_64 = cpu_to_fdt64(value);
+		return data_append_data(d, &value_64, 8);
+
+	default:
+		die("Invalid literal size (%d)\n", bits);
+	}
+}
+
+struct data data_append_re(struct data d, uint64_t address, uint64_t size)
+{
+	struct fdt_reserve_entry re;
+
+	re.address = cpu_to_fdt64(address);
+	re.size = cpu_to_fdt64(size);
+
+	return data_append_data(d, &re, sizeof(re));
+}
+
+struct data data_append_cell(struct data d, cell_t word)
+{
+	return data_append_integer(d, word, sizeof(word) * 8);
+}
+
+struct data data_append_addr(struct data d, uint64_t addr)
+{
+	return data_append_integer(d, addr, sizeof(addr) * 8);
+}
+
+struct data data_append_byte(struct data d, uint8_t byte)
+{
+	return data_append_data(d, &byte, 1);
+}
+
+struct data data_append_zeroes(struct data d, int len)
+{
+	d = data_grow_for(d, len);
+
+	memset(d.val + d.len, 0, len);
+	d.len += len;
+	return d;
+}
+
+struct data data_append_align(struct data d, int align)
+{
+	int newlen = ALIGN(d.len, align);
+	return data_append_zeroes(d, newlen - d.len);
+}
+
+struct data data_add_marker(struct data d, enum markertype type, char *ref)
+{
+	struct marker *m;
+
+	m = alloc_marker(d.len, type, ref);
+
+	return data_append_markers(d, m);
+}
+
+bool data_is_one_string(struct data d)
+{
+	int i;
+	int len = d.len;
+
+	if (len == 0)
+		return false;
+
+	for (i = 0; i < len-1; i++)
+		if (d.val[i] == '\0')
+			return false;
+
+	if (d.val[len-1] != '\0')
+		return false;
+
+	return true;
+}
+
+struct data data_insert_data(struct data d, struct marker *m, struct data old)
+{
+	unsigned int offset = m->offset;
+	struct marker *next = m->next;
+	struct marker *marker;
+	struct data new_data;
+	char *ref;
+
+	new_data = data_insert_at_marker(d, m, old.val, old.len);
+
+	/* Copy all markers from old value */
+	marker = old.markers;
+	for_each_marker(marker) {
+		ref = NULL;
+
+		if (marker->ref)
+			ref = xstrdup(marker->ref);
+
+		m->next = alloc_marker(marker->offset + offset, marker->type,
+				       ref);
+		m = m->next;
+	}
+	m->next = next;
+
+	return new_data;
+}
+
+struct marker *alloc_marker(unsigned int offset, enum markertype type,
+			    char *ref)
+{
+	struct marker *m;
+
+	m = xmalloc(sizeof(*m));
+	m->offset = offset;
+	m->type = type;
+	m->ref = ref;
+	m->next = NULL;
+
+	return m;
+}