x86/kprobes: Fix crash when probing CS CALL instructionsgrafted

When using eBPF to probe CS CALL instructions within a function,
a crash can be triggered.

The eBPF tool probes offset 257 of the __hrtimer_run_queues()
function:

<__hrtimer_run_queues+249>:  nopl   0x0(%rax,%rax,1)
<__hrtimer_run_queues+254>:  mov    %r14,%rdi
<__hrtimer_run_queues+257>:  cs call <__x86_indirect_thunk_r12>
<__hrtimer_run_queues+263>:  mov    %eax,%r12d
<__hrtimer_run_queues+266>:  xchg   %ax,%ax
<__hrtimer_run_queues+268>:  mov    %r13,%rdi

Which triggers this crash:

  BUG: unable to handle page fault for address: 00000000000f41c9
  #PF: supervisor write access in kernel mode
  #PF: error_code(0x0002) - not-present page
  PGD 0 P4D 0
  Oops: 0002 [#1] SMP NOPTI
  CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P
  RIP: 0010:__hrtimer_run_queues+0x106/0x230

Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is
at the 6th byte of the above CS CALL instruction. Since the CS CALL
instruction occupies 6 bytes, the exception occurred in the middle of that
call instruction.

The root cause is that when using eBPF tools to probe in the middle of a
function, a kprobe with INT3 is used as the underlying implementation.

During single-step emulation of the original CALL instruction,
int3_emulate_call() assumes that the probed CALL instruction is 5 bytes
long. However, the actual CS-prefixed CALL instruction occupies 6 bytes,
so it constructs an incorrect exception return address. When the CPU
returns from the kprobe handler, the next instruction to be executed is at
the address of the last byte of that CS CALL instruction. Coincidentally,
starting from that address, the CPU fetches and decodes a completely
different instruction, which ultimately triggers a kernel crash.

Fix the issue by using the actual instruction length obtained from
the instruction decoder when constructing the exception return
address, rather than relying on the hardcoded CALL_INSN_SIZE macro.

[ mingo: Refined the changelog ]

Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step")
Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Jinke Han <jinkehan@didiglobal.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Yafang Shao <laoar.shao@gmail.com>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000

1 files changed, 401 insertions, 0 deletions

diff --git a/drivers/s390/cio/trace.h b/drivers/s390/cio/trace.h
new file mode 100644
index 000000000..6bca5315e
--- /dev/null
+++ b/drivers/s390/cio/trace.h
@@ -0,0 +1,401 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Tracepoint header for the s390 Common I/O layer (CIO)
+ *
+ * Copyright IBM Corp. 2015
+ * Author(s): Peter Oberparleiter <oberpar@linux.vnet.ibm.com>
+ */
+
+#include <linux/kernel.h>
+#include <asm/crw.h>
+#include <uapi/asm/chpid.h>
+#include <uapi/asm/schid.h>
+#include "cio.h"
+#include "orb.h"
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM s390
+
+#if !defined(_TRACE_S390_CIO_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_S390_CIO_H
+
+#include <linux/tracepoint.h>
+
+DECLARE_EVENT_CLASS(s390_class_schib,
+	TP_PROTO(struct subchannel_id schid, struct schib *schib, int cc),
+	TP_ARGS(schid, schib, cc),
+	TP_STRUCT__entry(
+		__field(u8, cssid)
+		__field(u8, ssid)
+		__field(u16, schno)
+		__field(u16, devno)
+		__field_struct(struct schib, schib)
+		__field(u8, pmcw_ena)
+		__field(u8, pmcw_st)
+		__field(u8, pmcw_dnv)
+		__field(u16, pmcw_dev)
+		__field(u8, pmcw_lpm)
+		__field(u8, pmcw_pnom)
+		__field(u8, pmcw_lpum)
+		__field(u8, pmcw_pim)
+		__field(u8, pmcw_pam)
+		__field(u8, pmcw_pom)
+		__field(u64, pmcw_chpid)
+		__field(int, cc)
+	),
+	TP_fast_assign(
+		__entry->cssid = schid.cssid;
+		__entry->ssid = schid.ssid;
+		__entry->schno = schid.sch_no;
+		__entry->devno = schib->pmcw.dev;
+		__entry->schib = *schib;
+		__entry->pmcw_ena = schib->pmcw.ena;
+		__entry->pmcw_st = schib->pmcw.st;
+		__entry->pmcw_dnv = schib->pmcw.dnv;
+		__entry->pmcw_dev = schib->pmcw.dev;
+		__entry->pmcw_lpm = schib->pmcw.lpm;
+		__entry->pmcw_pnom = schib->pmcw.pnom;
+		__entry->pmcw_lpum = schib->pmcw.lpum;
+		__entry->pmcw_pim = schib->pmcw.pim;
+		__entry->pmcw_pam = schib->pmcw.pam;
+		__entry->pmcw_pom = schib->pmcw.pom;
+		memcpy(&__entry->pmcw_chpid, &schib->pmcw.chpid, 8);
+		__entry->cc = cc;
+	),
+	TP_printk("schid=%x.%x.%04x cc=%d ena=%d st=%d dnv=%d dev=%04x "
+		  "lpm=0x%02x pnom=0x%02x lpum=0x%02x pim=0x%02x pam=0x%02x "
+		  "pom=0x%02x chpids=%016llx",
+		  __entry->cssid, __entry->ssid, __entry->schno, __entry->cc,
+		  __entry->pmcw_ena, __entry->pmcw_st,
+		  __entry->pmcw_dnv, __entry->pmcw_dev,
+		  __entry->pmcw_lpm, __entry->pmcw_pnom,
+		  __entry->pmcw_lpum, __entry->pmcw_pim,
+		  __entry->pmcw_pam, __entry->pmcw_pom,
+		  __entry->pmcw_chpid
+	)
+);
+
+/**
+ * s390_cio_stsch -  Store Subchannel instruction (STSCH) was performed
+ * @schid: Subchannel ID
+ * @schib: Subchannel-Information block
+ * @cc: Condition code
+ */
+DEFINE_EVENT(s390_class_schib, s390_cio_stsch,
+	TP_PROTO(struct subchannel_id schid, struct schib *schib, int cc),
+	TP_ARGS(schid, schib, cc)
+);
+
+/**
+ * s390_cio_msch -  Modify Subchannel instruction (MSCH) was performed
+ * @schid: Subchannel ID
+ * @schib: Subchannel-Information block
+ * @cc: Condition code
+ */
+DEFINE_EVENT(s390_class_schib, s390_cio_msch,
+	TP_PROTO(struct subchannel_id schid, struct schib *schib, int cc),
+	TP_ARGS(schid, schib, cc)
+);
+
+/**
+ * s390_cio_tsch - Test Subchannel instruction (TSCH) was performed
+ * @schid: Subchannel ID
+ * @irb: Interruption-Response Block
+ * @cc: Condition code
+ */
+TRACE_EVENT(s390_cio_tsch,
+	TP_PROTO(struct subchannel_id schid, struct irb *irb, int cc),
+	TP_ARGS(schid, irb, cc),
+	TP_STRUCT__entry(
+		__field(u8, cssid)
+		__field(u8, ssid)
+		__field(u16, schno)
+		__field_struct(struct irb, irb)
+		__field(u8, scsw_dcc)
+		__field(u8, scsw_pno)
+		__field(u8, scsw_fctl)
+		__field(u8, scsw_actl)
+		__field(u8, scsw_stctl)
+		__field(u8, scsw_dstat)
+		__field(u8, scsw_cstat)
+		__field(int, cc)
+	),
+	TP_fast_assign(
+		__entry->cssid = schid.cssid;
+		__entry->ssid = schid.ssid;
+		__entry->schno = schid.sch_no;
+		__entry->irb = *irb;
+		__entry->scsw_dcc = scsw_cc(&irb->scsw);
+		__entry->scsw_pno = scsw_pno(&irb->scsw);
+		__entry->scsw_fctl = scsw_fctl(&irb->scsw);
+		__entry->scsw_actl = scsw_actl(&irb->scsw);
+		__entry->scsw_stctl = scsw_stctl(&irb->scsw);
+		__entry->scsw_dstat = scsw_dstat(&irb->scsw);
+		__entry->scsw_cstat = scsw_cstat(&irb->scsw);
+		__entry->cc = cc;
+	),
+	TP_printk("schid=%x.%x.%04x cc=%d dcc=%d pno=%d fctl=0x%x actl=0x%x "
+		  "stctl=0x%x dstat=0x%x cstat=0x%x",
+		  __entry->cssid, __entry->ssid, __entry->schno, __entry->cc,
+		  __entry->scsw_dcc, __entry->scsw_pno,
+		  __entry->scsw_fctl, __entry->scsw_actl,
+		  __entry->scsw_stctl,
+		  __entry->scsw_dstat, __entry->scsw_cstat
+	)
+);
+
+/**
+ * s390_cio_tpi - Test Pending Interruption instruction (TPI) was performed
+ * @addr: Address of the I/O interruption code or %NULL
+ * @cc: Condition code
+ */
+TRACE_EVENT(s390_cio_tpi,
+	TP_PROTO(struct tpi_info *addr, int cc),
+	TP_ARGS(addr, cc),
+	TP_STRUCT__entry(
+		__field(int, cc)
+		__field_struct(struct tpi_info, tpi_info)
+		__field(u8, cssid)
+		__field(u8, ssid)
+		__field(u16, schno)
+		__field(u8, adapter_IO)
+		__field(u8, isc)
+		__field(u8, type)
+	),
+	TP_fast_assign(
+		__entry->cc = cc;
+		if (cc != 0)
+			memset(&__entry->tpi_info, 0, sizeof(struct tpi_info));
+		else if (addr)
+			__entry->tpi_info = *addr;
+		else
+			__entry->tpi_info = get_lowcore()->tpi_info;
+		__entry->cssid = __entry->tpi_info.schid.cssid;
+		__entry->ssid = __entry->tpi_info.schid.ssid;
+		__entry->schno = __entry->tpi_info.schid.sch_no;
+		__entry->adapter_IO = __entry->tpi_info.adapter_IO;
+		__entry->isc = __entry->tpi_info.isc;
+		__entry->type = __entry->tpi_info.type;
+	),
+	TP_printk("schid=%x.%x.%04x cc=%d a=%d isc=%d type=%d",
+		  __entry->cssid, __entry->ssid, __entry->schno, __entry->cc,
+		  __entry->adapter_IO, __entry->isc,
+		  __entry->type
+	)
+);
+
+/**
+ * s390_cio_ssch - Start Subchannel instruction (SSCH) was performed
+ * @schid: Subchannel ID
+ * @orb: Operation-Request Block
+ * @cc: Condition code
+ */
+TRACE_EVENT(s390_cio_ssch,
+	TP_PROTO(struct subchannel_id schid, union orb *orb, int cc),
+	TP_ARGS(schid, orb, cc),
+	TP_STRUCT__entry(
+		__field(u8, cssid)
+		__field(u8, ssid)
+		__field(u16, schno)
+		__field_struct(union orb, orb)
+		__field(int, cc)
+	),
+	TP_fast_assign(
+		__entry->cssid = schid.cssid;
+		__entry->ssid = schid.ssid;
+		__entry->schno = schid.sch_no;
+		__entry->orb = *orb;
+		__entry->cc = cc;
+	),
+	TP_printk("schid=%x.%x.%04x cc=%d", __entry->cssid, __entry->ssid,
+		  __entry->schno, __entry->cc
+	)
+);
+
+DECLARE_EVENT_CLASS(s390_class_schid,
+	TP_PROTO(struct subchannel_id schid, int cc),
+	TP_ARGS(schid, cc),
+	TP_STRUCT__entry(
+		__field(u8, cssid)
+		__field(u8, ssid)
+		__field(u16, schno)
+		__field(int, cc)
+	),
+	TP_fast_assign(
+		__entry->cssid = schid.cssid;
+		__entry->ssid = schid.ssid;
+		__entry->schno = schid.sch_no;
+		__entry->cc = cc;
+	),
+	TP_printk("schid=%x.%x.%04x cc=%d", __entry->cssid, __entry->ssid,
+		  __entry->schno, __entry->cc
+	)
+);
+
+/**
+ * s390_cio_csch - Clear Subchannel instruction (CSCH) was performed
+ * @schid: Subchannel ID
+ * @cc: Condition code
+ */
+DEFINE_EVENT(s390_class_schid, s390_cio_csch,
+	TP_PROTO(struct subchannel_id schid, int cc),
+	TP_ARGS(schid, cc)
+);
+
+/**
+ * s390_cio_hsch - Halt Subchannel instruction (HSCH) was performed
+ * @schid: Subchannel ID
+ * @cc: Condition code
+ */
+DEFINE_EVENT(s390_class_schid, s390_cio_hsch,
+	TP_PROTO(struct subchannel_id schid, int cc),
+	TP_ARGS(schid, cc)
+);
+
+/**
+ * s390_cio_xsch - Cancel Subchannel instruction (XSCH) was performed
+ * @schid: Subchannel ID
+ * @cc: Condition code
+ */
+DEFINE_EVENT(s390_class_schid, s390_cio_xsch,
+	TP_PROTO(struct subchannel_id schid, int cc),
+	TP_ARGS(schid, cc)
+);
+
+/**
+ * s390_cio_rsch - Resume Subchannel instruction (RSCH) was performed
+ * @schid: Subchannel ID
+ * @cc: Condition code
+ */
+DEFINE_EVENT(s390_class_schid, s390_cio_rsch,
+	TP_PROTO(struct subchannel_id schid, int cc),
+	TP_ARGS(schid, cc)
+);
+
+#define CHSC_MAX_REQUEST_LEN		64
+#define CHSC_MAX_RESPONSE_LEN		64
+
+/**
+ * s390_cio_chsc - Channel Subsystem Call (CHSC) instruction was performed
+ * @chsc: CHSC block
+ * @cc: Condition code
+ */
+TRACE_EVENT(s390_cio_chsc,
+	TP_PROTO(struct chsc_header *chsc, int cc),
+	TP_ARGS(chsc, cc),
+	TP_STRUCT__entry(
+		__field(int, cc)
+		__field(u16, code)
+		__field(u16, rcode)
+		__array(u8, request, CHSC_MAX_REQUEST_LEN)
+		__array(u8, response, CHSC_MAX_RESPONSE_LEN)
+	),
+	TP_fast_assign(
+		__entry->cc = cc;
+		__entry->code = chsc->code;
+		memcpy(&entry->request, chsc,
+		       min_t(u16, chsc->length, CHSC_MAX_REQUEST_LEN));
+		chsc = (struct chsc_header *) ((char *) chsc + chsc->length);
+		__entry->rcode = chsc->code;
+		memcpy(&entry->response, chsc,
+		       min_t(u16, chsc->length, CHSC_MAX_RESPONSE_LEN));
+	),
+	TP_printk("code=0x%04x cc=%d rcode=0x%04x", __entry->code,
+		  __entry->cc, __entry->rcode)
+);
+
+/**
+ * s390_cio_interrupt - An I/O interrupt occurred
+ * @tpi_info: Address of the I/O interruption code
+ */
+TRACE_EVENT(s390_cio_interrupt,
+	TP_PROTO(struct tpi_info *tpi_info),
+	TP_ARGS(tpi_info),
+	TP_STRUCT__entry(
+		__field_struct(struct tpi_info, tpi_info)
+		__field(u8, cssid)
+		__field(u8, ssid)
+		__field(u16, schno)
+		__field(u8, isc)
+		__field(u8, type)
+	),
+	TP_fast_assign(
+		__entry->tpi_info = *tpi_info;
+		__entry->cssid = tpi_info->schid.cssid;
+		__entry->ssid = tpi_info->schid.ssid;
+		__entry->schno = tpi_info->schid.sch_no;
+		__entry->isc = tpi_info->isc;
+		__entry->type = tpi_info->type;
+	),
+	TP_printk("schid=%x.%x.%04x isc=%d type=%d",
+		  __entry->cssid, __entry->ssid, __entry->schno,
+		  __entry->isc, __entry->type
+	)
+);
+
+/**
+ * s390_cio_adapter_int - An adapter interrupt occurred
+ * @tpi_info: Address of the I/O interruption code
+ */
+TRACE_EVENT(s390_cio_adapter_int,
+	TP_PROTO(struct tpi_info *tpi_info),
+	TP_ARGS(tpi_info),
+	TP_STRUCT__entry(
+		__field_struct(struct tpi_info, tpi_info)
+		__field(u8, isc)
+	),
+	TP_fast_assign(
+		__entry->tpi_info = *tpi_info;
+		__entry->isc = tpi_info->isc;
+	),
+	TP_printk("isc=%d", __entry->isc)
+);
+
+/**
+ * s390_cio_stcrw - Store Channel Report Word (STCRW) was performed
+ * @crw: Channel Report Word
+ * @cc: Condition code
+ */
+TRACE_EVENT(s390_cio_stcrw,
+	TP_PROTO(struct crw *crw, int cc),
+	TP_ARGS(crw, cc),
+	TP_STRUCT__entry(
+		__field_struct(struct crw, crw)
+		__field(int, cc)
+		__field(u8, slct)
+		__field(u8, oflw)
+		__field(u8, chn)
+		__field(u8, rsc)
+		__field(u8, anc)
+		__field(u8, erc)
+		__field(u16, rsid)
+	),
+	TP_fast_assign(
+		__entry->crw = *crw;
+		__entry->cc = cc;
+		__entry->slct = crw->slct;
+		__entry->oflw = crw->oflw;
+		__entry->chn = crw->chn;
+		__entry->rsc = crw->rsc;
+		__entry->anc = crw->anc;
+		__entry->erc = crw->erc;
+		__entry->rsid = crw->rsid;
+	),
+	TP_printk("cc=%d slct=%d oflw=%d chn=%d rsc=%d anc=%d erc=0x%x "
+		  "rsid=0x%x",
+		  __entry->cc, __entry->slct, __entry->oflw,
+		  __entry->chn, __entry->rsc,  __entry->anc,
+		  __entry->erc, __entry->rsid
+	)
+);
+
+#endif /* _TRACE_S390_CIO_H */
+
+/* This part must be outside protection */
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_FILE trace
+
+#include <trace/define_trace.h>