x86/kprobes: Fix crash when probing CS CALL instructionsgrafted

When using eBPF to probe CS CALL instructions within a function,
a crash can be triggered.

The eBPF tool probes offset 257 of the __hrtimer_run_queues()
function:

<__hrtimer_run_queues+249>:  nopl   0x0(%rax,%rax,1)
<__hrtimer_run_queues+254>:  mov    %r14,%rdi
<__hrtimer_run_queues+257>:  cs call <__x86_indirect_thunk_r12>
<__hrtimer_run_queues+263>:  mov    %eax,%r12d
<__hrtimer_run_queues+266>:  xchg   %ax,%ax
<__hrtimer_run_queues+268>:  mov    %r13,%rdi

Which triggers this crash:

  BUG: unable to handle page fault for address: 00000000000f41c9
  #PF: supervisor write access in kernel mode
  #PF: error_code(0x0002) - not-present page
  PGD 0 P4D 0
  Oops: 0002 [#1] SMP NOPTI
  CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P
  RIP: 0010:__hrtimer_run_queues+0x106/0x230

Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is
at the 6th byte of the above CS CALL instruction. Since the CS CALL
instruction occupies 6 bytes, the exception occurred in the middle of that
call instruction.

The root cause is that when using eBPF tools to probe in the middle of a
function, a kprobe with INT3 is used as the underlying implementation.

During single-step emulation of the original CALL instruction,
int3_emulate_call() assumes that the probed CALL instruction is 5 bytes
long. However, the actual CS-prefixed CALL instruction occupies 6 bytes,
so it constructs an incorrect exception return address. When the CPU
returns from the kprobe handler, the next instruction to be executed is at
the address of the last byte of that CS CALL instruction. Coincidentally,
starting from that address, the CPU fetches and decodes a completely
different instruction, which ultimately triggers a kernel crash.

Fix the issue by using the actual instruction length obtained from
the instruction decoder when constructing the exception return
address, rather than relying on the hardcoded CALL_INSN_SIZE macro.

[ mingo: Refined the changelog ]

Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step")
Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Jinke Han <jinkehan@didiglobal.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Yafang Shao <laoar.shao@gmail.com>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000

6 files changed, 568 insertions, 0 deletions

diff --git a/arch/m68k/sun3x/Makefile b/arch/m68k/sun3x/Makefile
new file mode 100644
index 000000000..f36020e9d
--- /dev/null
+++ b/arch/m68k/sun3x/Makefile
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Makefile for Linux arch/m68k/sun3x source directory
+#
+
+obj-y		:= config.o time.o dvma.o prom.o
diff --git a/arch/m68k/sun3x/config.c b/arch/m68k/sun3x/config.c
new file mode 100644
index 000000000..798ea72a0
--- /dev/null
+++ b/arch/m68k/sun3x/config.c
@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Setup kernel for a Sun3x machine
+ *
+ * (C) 1999 Thomas Bogendoerfer (tsbogend@alpha.franken.de)
+ *
+ * based on code from Oliver Jowett <oliver@jowett.manawatu.gen.nz>
+ */
+
+#include <linux/types.h>
+#include <linux/mm.h>
+#include <linux/seq_file.h>
+#include <linux/console.h>
+#include <linux/init.h>
+
+#include <asm/machdep.h>
+#include <asm/irq.h>
+#include <asm/sun3xprom.h>
+#include <asm/sun3ints.h>
+#include <asm/setup.h>
+#include <asm/oplib.h>
+#include <asm/config.h>
+
+#include "time.h"
+#include "../sun3/sun3.h"
+
+volatile char *clock_va;
+
+void sun3_leds(unsigned char byte)
+{
+
+}
+
+static void sun3x_get_hardware_list(struct seq_file *m)
+{
+	seq_printf(m, "PROM Revision:\t%s\n", romvec->pv_monid);
+}
+
+/*
+ *  Setup the sun3x configuration info
+ */
+void __init config_sun3x(void)
+{
+
+	sun3x_prom_init();
+
+	mach_sched_init      = sun3x_sched_init;
+	mach_init_IRQ        = sun3_init_IRQ;
+
+	mach_reset           = sun3x_reboot;
+
+	mach_hwclk           = sun3x_hwclk;
+	mach_get_model       = sun3_get_model;
+	mach_get_hardware_list = sun3x_get_hardware_list;
+
+	sun3_intreg = (unsigned char *)SUN3X_INTREG;
+
+	/* only the serial console is known to work anyway... */
+#if 0
+	switch (*(unsigned char *)SUN3X_EEPROM_CONS) {
+	case 0x10:
+		serial_console = 1;
+		conswitchp = NULL;
+		break;
+	case 0x11:
+		serial_console = 2;
+		conswitchp = NULL;
+		break;
+	default:
+		serial_console = 0;
+		break;
+	}
+#endif
+
+}
+
diff --git a/arch/m68k/sun3x/dvma.c b/arch/m68k/sun3x/dvma.c
new file mode 100644
index 000000000..5185b4818
--- /dev/null
+++ b/arch/m68k/sun3x/dvma.c
@@ -0,0 +1,200 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Virtual DMA allocation
+ *
+ * (C) 1999 Thomas Bogendoerfer (tsbogend@alpha.franken.de)
+ *
+ * 11/26/2000 -- disabled the existing code because it didn't work for
+ * me in 2.4.  Replaced with a significantly more primitive version
+ * similar to the sun3 code.  the old functionality was probably more
+ * desirable, but....   -- Sam Creasey (sammy@oh.verio.com)
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/bitops.h>
+#include <linux/mm.h>
+#include <linux/memblock.h>
+#include <linux/vmalloc.h>
+
+#include <asm/sun3x.h>
+#include <asm/dvma.h>
+#include <asm/io.h>
+#include <asm/page.h>
+#include <asm/tlbflush.h>
+
+/* IOMMU support */
+
+#define IOMMU_ADDR_MASK            0x03ffe000
+#define IOMMU_CACHE_INHIBIT        0x00000040
+#define IOMMU_FULL_BLOCK           0x00000020
+#define IOMMU_MODIFIED             0x00000010
+#define IOMMU_USED                 0x00000008
+#define IOMMU_WRITE_PROTECT        0x00000004
+#define IOMMU_DT_MASK              0x00000003
+#define IOMMU_DT_INVALID           0x00000000
+#define IOMMU_DT_VALID             0x00000001
+#define IOMMU_DT_BAD               0x00000002
+
+
+static volatile unsigned long *iommu_pte = (unsigned long *)SUN3X_IOMMU;
+
+
+#define dvma_entry_paddr(index)		(iommu_pte[index] & IOMMU_ADDR_MASK)
+#define dvma_entry_vaddr(index,paddr)	((index << DVMA_PAGE_SHIFT) |  \
+					 (paddr & (DVMA_PAGE_SIZE-1)))
+#if 0
+#define dvma_entry_set(index,addr)	(iommu_pte[index] =            \
+					    (addr & IOMMU_ADDR_MASK) | \
+				             IOMMU_DT_VALID | IOMMU_CACHE_INHIBIT)
+#else
+#define dvma_entry_set(index,addr)	(iommu_pte[index] =            \
+					    (addr & IOMMU_ADDR_MASK) | \
+				             IOMMU_DT_VALID)
+#endif
+#define dvma_entry_clr(index)		(iommu_pte[index] = IOMMU_DT_INVALID)
+#define dvma_entry_hash(addr)		((addr >> DVMA_PAGE_SHIFT) ^ \
+					 ((addr & 0x03c00000) >>     \
+						(DVMA_PAGE_SHIFT+4)))
+
+#ifdef DEBUG
+/* code to print out a dvma mapping for debugging purposes */
+static void dvma_print (unsigned long dvma_addr)
+{
+
+	unsigned long index;
+
+	index = dvma_addr >> DVMA_PAGE_SHIFT;
+
+	pr_info("idx %lx dvma_addr %08lx paddr %08lx\n", index, dvma_addr,
+		dvma_entry_paddr(index));
+}
+#endif
+
+
+/* create a virtual mapping for a page assigned within the IOMMU
+   so that the cpu can reach it easily */
+inline int dvma_map_cpu(unsigned long kaddr,
+			       unsigned long vaddr, int len)
+{
+	pgd_t *pgd;
+	p4d_t *p4d;
+	pud_t *pud;
+	unsigned long end;
+	int ret = 0;
+
+	kaddr &= PAGE_MASK;
+	vaddr &= PAGE_MASK;
+
+	end = PAGE_ALIGN(vaddr + len);
+
+	pr_debug("dvma: mapping kern %08lx to virt %08lx\n", kaddr, vaddr);
+	pgd = pgd_offset_k(vaddr);
+	p4d = p4d_offset(pgd, vaddr);
+	pud = pud_offset(p4d, vaddr);
+
+	do {
+		pmd_t *pmd;
+		unsigned long end2;
+
+		if((pmd = pmd_alloc(&init_mm, pud, vaddr)) == NULL) {
+			ret = -ENOMEM;
+			goto out;
+		}
+
+		if((end & PGDIR_MASK) > (vaddr & PGDIR_MASK))
+			end2 = (vaddr + (PGDIR_SIZE-1)) & PGDIR_MASK;
+		else
+			end2 = end;
+
+		do {
+			pte_t *pte;
+			unsigned long end3;
+
+			if((pte = pte_alloc_kernel(pmd, vaddr)) == NULL) {
+				ret = -ENOMEM;
+				goto out;
+			}
+
+			if((end2 & PMD_MASK) > (vaddr & PMD_MASK))
+				end3 = (vaddr + (PMD_SIZE-1)) & PMD_MASK;
+			else
+				end3 = end2;
+
+			do {
+				pr_debug("mapping %08lx phys to %08lx\n",
+					 __pa(kaddr), vaddr);
+				set_pte(pte, pfn_pte(virt_to_pfn((void *)kaddr),
+						     PAGE_KERNEL));
+				pte++;
+				kaddr += PAGE_SIZE;
+				vaddr += PAGE_SIZE;
+			} while(vaddr < end3);
+
+		} while(vaddr < end2);
+
+	} while(vaddr < end);
+
+	flush_tlb_all();
+
+ out:
+	return ret;
+}
+
+
+int dvma_map_iommu(unsigned long kaddr, unsigned long baddr, int len)
+{
+	unsigned long end, index;
+
+	index = baddr >> DVMA_PAGE_SHIFT;
+	end = ((baddr+len) >> DVMA_PAGE_SHIFT);
+
+	if(len & ~DVMA_PAGE_MASK)
+		end++;
+
+	for(; index < end ; index++) {
+//		if(dvma_entry_use(index))
+//			BUG();
+//		pr_info("mapping pa %lx to ba %lx\n", __pa(kaddr),
+//			index << DVMA_PAGE_SHIFT);
+
+		dvma_entry_set(index, __pa(kaddr));
+
+		iommu_pte[index] |= IOMMU_FULL_BLOCK;
+//		dvma_entry_inc(index);
+
+		kaddr += DVMA_PAGE_SIZE;
+	}
+
+#ifdef DEBUG
+	for(index = (baddr >> DVMA_PAGE_SHIFT); index < end; index++)
+		dvma_print(index << DVMA_PAGE_SHIFT);
+#endif
+	return 0;
+
+}
+
+void dvma_unmap_iommu(unsigned long baddr, int len)
+{
+
+	int index, end;
+
+
+	index = baddr >> DVMA_PAGE_SHIFT;
+	end = (DVMA_PAGE_ALIGN(baddr+len) >> DVMA_PAGE_SHIFT);
+
+	for(; index < end ; index++) {
+		pr_debug("freeing bus mapping %08x\n",
+			 index << DVMA_PAGE_SHIFT);
+#if 0
+		if(!dvma_entry_use(index))
+			pr_info("dvma_unmap freeing unused entry %04x\n",
+				index);
+		else
+			dvma_entry_dec(index);
+#endif
+		dvma_entry_clr(index);
+	}
+
+}
diff --git a/arch/m68k/sun3x/prom.c b/arch/m68k/sun3x/prom.c
new file mode 100644
index 000000000..8ac87d3dc
--- /dev/null
+++ b/arch/m68k/sun3x/prom.c
@@ -0,0 +1,164 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Prom access routines for the sun3x */
+
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/tty.h>
+#include <linux/console.h>
+#include <linux/init.h>
+#include <linux/mm.h>
+#include <linux/string.h>
+
+#include <asm/page.h>
+#include <asm/setup.h>
+#include <asm/traps.h>
+#include <asm/sun3xprom.h>
+#include <asm/idprom.h>
+#include <asm/sun3ints.h>
+#include <asm/openprom.h>
+#include <asm/machines.h>
+
+void (*sun3x_putchar)(int);
+int (*sun3x_getchar)(void);
+int (*sun3x_mayget)(void);
+int (*sun3x_mayput)(int);
+void (*sun3x_prom_reboot)(void);
+e_vector sun3x_prom_abort;
+struct linux_romvec *romvec;
+
+/* prom vector table */
+e_vector *sun3x_prom_vbr;
+
+/* Handle returning to the prom */
+static void sun3x_halt(void)
+{
+	unsigned long flags;
+
+	/* Disable interrupts while we mess with things */
+	local_irq_save(flags);
+
+	/* Restore prom vbr */
+	asm volatile ("movec %0,%%vbr" : : "r" ((void*)sun3x_prom_vbr));
+
+	/* Restore prom NMI clock */
+//	sun3x_disable_intreg(5);
+	sun3_enable_irq(7);
+
+	/* Let 'er rip */
+	asm volatile ("trap #14");
+
+	/* Restore everything */
+	sun3_disable_irq(7);
+	sun3_enable_irq(5);
+
+	asm volatile ("movec %0,%%vbr" : : "r" ((void*)vectors));
+	local_irq_restore(flags);
+}
+
+void sun3x_reboot(void)
+{
+	/* This never returns, don't bother saving things */
+	local_irq_disable();
+
+	/* Restore prom vbr */
+	asm volatile ("movec %0,%%vbr" : : "r" ((void*)sun3x_prom_vbr));
+
+	/* Restore prom NMI clock */
+	sun3_disable_irq(5);
+	sun3_enable_irq(7);
+
+	/* Let 'er rip */
+	(*romvec->pv_reboot)("vmlinux");
+}
+
+static void sun3x_prom_write(struct console *co, const char *s,
+                             unsigned int count)
+{
+	while (count--) {
+		if (*s == '\n')
+			sun3x_putchar('\r');
+		sun3x_putchar(*s++);
+	}
+}
+
+/* debug console - write-only */
+
+static struct console sun3x_debug = {
+	.name	= "debug",
+	.write	= sun3x_prom_write,
+	.flags	= CON_PRINTBUFFER,
+	.index	= -1,
+};
+
+void __init sun3x_prom_init(void)
+{
+	/* Read the vector table */
+
+	sun3x_putchar = *(void (**)(int)) (SUN3X_P_PUTCHAR);
+	sun3x_getchar = *(int (**)(void)) (SUN3X_P_GETCHAR);
+	sun3x_mayget = *(int (**)(void))  (SUN3X_P_MAYGET);
+	sun3x_mayput = *(int (**)(int))   (SUN3X_P_MAYPUT);
+	sun3x_prom_reboot = *(void (**)(void)) (SUN3X_P_REBOOT);
+	sun3x_prom_abort = *(e_vector *)  (SUN3X_P_ABORT);
+	romvec = (struct linux_romvec *)SUN3X_PROM_BASE;
+
+	idprom_init();
+
+	if (!((idprom->id_machtype & SM_ARCH_MASK) == SM_SUN3X)) {
+		pr_warn("Machine reports strange type %02x\n",
+			idprom->id_machtype);
+		pr_warn("Pretending it's a 3/80, but very afraid...\n");
+		idprom->id_machtype = SM_SUN3X | SM_3_80;
+	}
+
+	/* point trap #14 at abort.
+	 * XXX this is futile since we restore the vbr first - oops
+	 */
+	vectors[VEC_TRAP14] = sun3x_prom_abort;
+}
+
+static int __init sun3x_debug_setup(char *arg)
+{
+	/* If debug=prom was specified, start the debug console */
+	if (MACH_IS_SUN3X && !strcmp(arg, "prom"))
+		register_console(&sun3x_debug);
+	return 0;
+}
+
+early_param("debug", sun3x_debug_setup);
+
+/* some prom functions to export */
+int prom_getintdefault(int node, char *property, int deflt)
+{
+	return deflt;
+}
+
+int prom_getbool (int node, char *prop)
+{
+	return 1;
+}
+
+void prom_printf(char *fmt, ...)
+{
+}
+
+void prom_halt (void)
+{
+	sun3x_halt();
+}
+
+/* Get the idprom and stuff it into buffer 'idbuf'.  Returns the
+ * format type.  'num_bytes' is the number of bytes that your idbuf
+ * has space for.  Returns 0xff on error.
+ */
+unsigned char
+prom_get_idprom(char *idbuf, int num_bytes)
+{
+        int i;
+
+	/* make a copy of the idprom structure */
+	for (i = 0; i < num_bytes; i++)
+		idbuf[i] = ((char *)SUN3X_IDPROM)[i];
+
+        return idbuf[0];
+}
diff --git a/arch/m68k/sun3x/time.c b/arch/m68k/sun3x/time.c
new file mode 100644
index 000000000..a2c97821f
--- /dev/null
+++ b/arch/m68k/sun3x/time.c
@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  linux/arch/m68k/sun3x/time.c
+ *
+ *  Sun3x-specific time handling
+ */
+
+#include <linux/types.h>
+#include <linux/kd.h>
+#include <linux/init.h>
+#include <linux/sched.h>
+#include <linux/kernel_stat.h>
+#include <linux/interrupt.h>
+#include <linux/rtc.h>
+#include <linux/bcd.h>
+
+#include <asm/irq.h>
+#include <asm/io.h>
+#include <asm/machdep.h>
+#include <asm/traps.h>
+#include <asm/sun3x.h>
+#include <asm/sun3ints.h>
+
+#include "time.h"
+
+#define M_CONTROL 0xf8
+#define M_SEC     0xf9
+#define M_MIN     0xfa
+#define M_HOUR    0xfb
+#define M_DAY     0xfc
+#define M_DATE    0xfd
+#define M_MONTH   0xfe
+#define M_YEAR    0xff
+
+#define C_WRITE   0x80
+#define C_READ    0x40
+#define C_SIGN    0x20
+#define C_CALIB   0x1f
+
+int sun3x_hwclk(int set, struct rtc_time *t)
+{
+	volatile struct mostek_dt *h =
+		(struct mostek_dt *)(SUN3X_EEPROM+M_CONTROL);
+	unsigned long flags;
+
+	local_irq_save(flags);
+
+	if(set) {
+		h->csr |= C_WRITE;
+		h->sec = bin2bcd(t->tm_sec);
+		h->min = bin2bcd(t->tm_min);
+		h->hour = bin2bcd(t->tm_hour);
+		h->wday = bin2bcd(t->tm_wday);
+		h->mday = bin2bcd(t->tm_mday);
+		h->month = bin2bcd(t->tm_mon + 1);
+		h->year = bin2bcd(t->tm_year % 100);
+		h->csr &= ~C_WRITE;
+	} else {
+		h->csr |= C_READ;
+		t->tm_sec = bcd2bin(h->sec);
+		t->tm_min = bcd2bin(h->min);
+		t->tm_hour = bcd2bin(h->hour);
+		t->tm_wday = bcd2bin(h->wday);
+		t->tm_mday = bcd2bin(h->mday);
+		t->tm_mon = bcd2bin(h->month) - 1;
+		t->tm_year = bcd2bin(h->year);
+		h->csr &= ~C_READ;
+		if (t->tm_year < 70)
+			t->tm_year += 100;
+	}
+
+	local_irq_restore(flags);
+
+	return 0;
+}
+
+#if 0
+static irqreturn_t sun3x_timer_tick(int irq, void *dev_id)
+{
+	unsigned long flags;
+
+	local_irq_save(flags);
+	/* Clear the pending interrupt - pulse the enable line low */
+	disable_irq(5);
+	enable_irq(5);
+	legacy_timer_tick(1);
+	local_irq_restore(flags);
+
+	return IRQ_HANDLED;
+}
+#endif
+
+void __init sun3x_sched_init(void)
+{
+
+	sun3_disable_interrupts();
+
+
+    /* Pulse enable low to get the clock started */
+	sun3_disable_irq(5);
+	sun3_enable_irq(5);
+	sun3_enable_interrupts();
+}
diff --git a/arch/m68k/sun3x/time.h b/arch/m68k/sun3x/time.h
new file mode 100644
index 000000000..7cfff22e4
--- /dev/null
+++ b/arch/m68k/sun3x/time.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef SUN3X_TIME_H
+#define SUN3X_TIME_H
+
+extern int sun3x_hwclk(int set, struct rtc_time *t);
+void sun3x_sched_init(void);
+
+struct mostek_dt {
+	volatile unsigned char csr;
+	volatile unsigned char sec;
+	volatile unsigned char min;
+	volatile unsigned char hour;
+	volatile unsigned char wday;
+	volatile unsigned char mday;
+	volatile unsigned char month;
+	volatile unsigned char year;
+};
+
+#endif