From a5f7a5bb3b7f28ba7e4fa246775b29a0e5537255 Mon Sep 17 00:00:00 2001 From: Jinke Han Date: Tue, 8 Sep 2026 15:37:42 +0800 Subject: x86/kprobes: Fix crash when probing CS CALL instructions When using eBPF to probe CS CALL instructions within a function, a crash can be triggered. The eBPF tool probes offset 257 of the __hrtimer_run_queues() function: <__hrtimer_run_queues+249>: nopl 0x0(%rax,%rax,1) <__hrtimer_run_queues+254>: mov %r14,%rdi <__hrtimer_run_queues+257>: cs call <__x86_indirect_thunk_r12> <__hrtimer_run_queues+263>: mov %eax,%r12d <__hrtimer_run_queues+266>: xchg %ax,%ax <__hrtimer_run_queues+268>: mov %r13,%rdi Which triggers this crash: BUG: unable to handle page fault for address: 00000000000f41c9 #PF: supervisor write access in kernel mode #PF: error_code(0x0002) - not-present page PGD 0 P4D 0 Oops: 0002 [#1] SMP NOPTI CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P RIP: 0010:__hrtimer_run_queues+0x106/0x230 Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is at the 6th byte of the above CS CALL instruction. Since the CS CALL instruction occupies 6 bytes, the exception occurred in the middle of that call instruction. The root cause is that when using eBPF tools to probe in the middle of a function, a kprobe with INT3 is used as the underlying implementation. During single-step emulation of the original CALL instruction, int3_emulate_call() assumes that the probed CALL instruction is 5 bytes long. However, the actual CS-prefixed CALL instruction occupies 6 bytes, so it constructs an incorrect exception return address. When the CPU returns from the kprobe handler, the next instruction to be executed is at the address of the last byte of that CS CALL instruction. Coincidentally, starting from that address, the CPU fetches and decodes a completely different instruction, which ultimately triggers a kernel crash. Fix the issue by using the actual instruction length obtained from the instruction decoder when constructing the exception return address, rather than relying on the hardcoded CALL_INSN_SIZE macro. [ mingo: Refined the changelog ] Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step") Suggested-by: Masami Hiramatsu (Google) Signed-off-by: Jinke Han Signed-off-by: Ingo Molnar Reviewed-by: Masami Hiramatsu (Google) Acked-by: Yafang Shao Acked-by: Borislav Petkov Cc: Peter Zijlstra Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000 --- arch/mips/tools/.gitignore | 3 + arch/mips/tools/Makefile | 10 ++ arch/mips/tools/elf-entry.c | 103 +++++++++++ arch/mips/tools/generic-board-config.sh | 84 +++++++++ arch/mips/tools/loongson3-llsc-check.c | 309 ++++++++++++++++++++++++++++++++ 5 files changed, 509 insertions(+) create mode 100644 arch/mips/tools/.gitignore create mode 100644 arch/mips/tools/Makefile create mode 100644 arch/mips/tools/elf-entry.c create mode 100755 arch/mips/tools/generic-board-config.sh create mode 100644 arch/mips/tools/loongson3-llsc-check.c (limited to 'arch/mips/tools') diff --git a/arch/mips/tools/.gitignore b/arch/mips/tools/.gitignore new file mode 100644 index 000000000..794817dfb --- /dev/null +++ b/arch/mips/tools/.gitignore @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +elf-entry +loongson3-llsc-check diff --git a/arch/mips/tools/Makefile b/arch/mips/tools/Makefile new file mode 100644 index 000000000..b851e5dcc --- /dev/null +++ b/arch/mips/tools/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0 +hostprogs := elf-entry +PHONY += elf-entry +elf-entry: $(obj)/elf-entry + @: + +hostprogs += loongson3-llsc-check +PHONY += loongson3-llsc-check +loongson3-llsc-check: $(obj)/loongson3-llsc-check + @: diff --git a/arch/mips/tools/elf-entry.c b/arch/mips/tools/elf-entry.c new file mode 100644 index 000000000..dbd14ff05 --- /dev/null +++ b/arch/mips/tools/elf-entry.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef be32toh +/* If libc provides [bl]e{32,64}toh() then we'll use them */ +#elif BYTE_ORDER == LITTLE_ENDIAN +# define be32toh(x) bswap_32(x) +# define le32toh(x) (x) +# define be64toh(x) bswap_64(x) +# define le64toh(x) (x) +#elif BYTE_ORDER == BIG_ENDIAN +# define be32toh(x) (x) +# define le32toh(x) bswap_32(x) +# define be64toh(x) (x) +# define le64toh(x) bswap_64(x) +#endif + +__attribute__((noreturn)) +static void die(const char *msg) +{ + fputs(msg, stderr); + exit(EXIT_FAILURE); +} + +int main(int argc, const char *argv[]) +{ + uint64_t entry; + size_t nread; + FILE *file; + union { + Elf32_Ehdr ehdr32; + Elf64_Ehdr ehdr64; + } hdr; + + if (argc != 2) + die("Usage: elf-entry \n"); + + file = fopen(argv[1], "r"); + if (!file) { + perror("Unable to open input file"); + return EXIT_FAILURE; + } + + nread = fread(&hdr, 1, sizeof(hdr), file); + if (nread != sizeof(hdr)) { + perror("Unable to read input file"); + fclose(file); + return EXIT_FAILURE; + } + + if (memcmp(hdr.ehdr32.e_ident, ELFMAG, SELFMAG)) { + fclose(file); + die("Input is not an ELF\n"); + } + + switch (hdr.ehdr32.e_ident[EI_CLASS]) { + case ELFCLASS32: + switch (hdr.ehdr32.e_ident[EI_DATA]) { + case ELFDATA2LSB: + entry = le32toh(hdr.ehdr32.e_entry); + break; + case ELFDATA2MSB: + entry = be32toh(hdr.ehdr32.e_entry); + break; + default: + fclose(file); + die("Invalid ELF encoding\n"); + } + + /* Sign extend to form a canonical address */ + entry = (int64_t)(int32_t)entry; + break; + + case ELFCLASS64: + switch (hdr.ehdr32.e_ident[EI_DATA]) { + case ELFDATA2LSB: + entry = le64toh(hdr.ehdr64.e_entry); + break; + case ELFDATA2MSB: + entry = be64toh(hdr.ehdr64.e_entry); + break; + default: + fclose(file); + die("Invalid ELF encoding\n"); + } + break; + + default: + fclose(file); + die("Invalid ELF class\n"); + } + + printf("0x%016" PRIx64 "\n", entry); + fclose(file); + return EXIT_SUCCESS; +} diff --git a/arch/mips/tools/generic-board-config.sh b/arch/mips/tools/generic-board-config.sh new file mode 100755 index 000000000..dfa5f31f5 --- /dev/null +++ b/arch/mips/tools/generic-board-config.sh @@ -0,0 +1,84 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Copyright (C) 2017 Imagination Technologies +# Author: Paul Burton +# +# This script merges configuration fragments for boards supported by the +# generic MIPS kernel. It checks each for requirements specified using +# formatted comments, and then calls merge_config.sh to merge those +# fragments which have no unmet requirements. +# +# An example of requirements in your board config fragment might be: +# +# # require CONFIG_CPU_MIPS32_R2=y +# # require CONFIG_CPU_LITTLE_ENDIAN=y +# +# This would mean that your board is only included in kernels which are +# configured for little endian MIPS32r2 CPUs, and not for example in kernels +# configured for 64 bit or big endian systems. +# + +srctree="$1" +objtree="$2" +ref_cfg="$3" +cfg="$4" +boards_origin="$5" +shift 5 + +# Only print Skipping... lines if the user explicitly specified BOARDS=. In the +# general case it only serves to obscure the useful output about what actually +# was included. +case ${boards_origin} in +"command line") + print_skipped=1 + ;; +environment*) + print_skipped=1 + ;; +*) + print_skipped=0 + ;; +esac + +for board in $@; do + board_cfg="${srctree}/arch/mips/configs/generic/board-${board}.config" + if [ ! -f "${board_cfg}" ]; then + echo "WARNING: Board config '${board_cfg}' not found" + continue + fi + + # For each line beginning with # require, cut out the field following + # it & search for that in the reference config file. If the requirement + # is not found then the subshell will exit with code 1, and we'll + # continue on to the next board. + grep -E '^# require ' "${board_cfg}" | \ + cut -d' ' -f 3- | \ + while read req; do + case ${req} in + *=y) + # If we require something =y then we check that a line + # containing it is present in the reference config. + grep -Eq "^${req}\$" "${ref_cfg}" && continue + ;; + *=n) + # If we require something =n then we just invert that + # check, considering the requirement met if there isn't + # a line containing the value =y in the reference + # config. + grep -Eq "^${req/%=n/=y}\$" "${ref_cfg}" || continue + ;; + *) + echo "WARNING: Unhandled requirement '${req}'" + ;; + esac + + [ ${print_skipped} -eq 1 ] && echo "Skipping ${board_cfg}" + exit 1 + done || continue + + # Merge this board config fragment into our final config file + ${srctree}/scripts/kconfig/merge_config.sh \ + -m -O ${objtree} ${cfg} ${board_cfg} \ + | grep -Ev '^(#|Using)' +done diff --git a/arch/mips/tools/loongson3-llsc-check.c b/arch/mips/tools/loongson3-llsc-check.c new file mode 100644 index 000000000..5f68a4fa8 --- /dev/null +++ b/arch/mips/tools/loongson3-llsc-check.c @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef be32toh +/* If libc provides le{16,32,64}toh() then we'll use them */ +#elif BYTE_ORDER == LITTLE_ENDIAN +# define le16toh(x) (x) +# define le32toh(x) (x) +# define le64toh(x) (x) +#elif BYTE_ORDER == BIG_ENDIAN +# define le16toh(x) bswap_16(x) +# define le32toh(x) bswap_32(x) +# define le64toh(x) bswap_64(x) +#endif + +/* MIPS opcodes, in bits 31:26 of an instruction */ +#define OP_SPECIAL 0x00 +#define OP_REGIMM 0x01 +#define OP_BEQ 0x04 +#define OP_BNE 0x05 +#define OP_BLEZ 0x06 +#define OP_BGTZ 0x07 +#define OP_BEQL 0x14 +#define OP_BNEL 0x15 +#define OP_BLEZL 0x16 +#define OP_BGTZL 0x17 +#define OP_LL 0x30 +#define OP_LLD 0x34 +#define OP_SC 0x38 +#define OP_SCD 0x3c + +/* Bits 20:16 of OP_REGIMM instructions */ +#define REGIMM_BLTZ 0x00 +#define REGIMM_BGEZ 0x01 +#define REGIMM_BLTZL 0x02 +#define REGIMM_BGEZL 0x03 +#define REGIMM_BLTZAL 0x10 +#define REGIMM_BGEZAL 0x11 +#define REGIMM_BLTZALL 0x12 +#define REGIMM_BGEZALL 0x13 + +/* Bits 5:0 of OP_SPECIAL instructions */ +#define SPECIAL_SYNC 0x0f + +static void usage(FILE *f) +{ + fprintf(f, "Usage: loongson3-llsc-check /path/to/vmlinux\n"); +} + +static int se16(uint16_t x) +{ + return (int16_t)x; +} + +static bool is_ll(uint32_t insn) +{ + switch (insn >> 26) { + case OP_LL: + case OP_LLD: + return true; + + default: + return false; + } +} + +static bool is_sc(uint32_t insn) +{ + switch (insn >> 26) { + case OP_SC: + case OP_SCD: + return true; + + default: + return false; + } +} + +static bool is_sync(uint32_t insn) +{ + /* Bits 31:11 should all be zeroes */ + if (insn >> 11) + return false; + + /* Bits 5:0 specify the SYNC special encoding */ + if ((insn & 0x3f) != SPECIAL_SYNC) + return false; + + return true; +} + +static bool is_branch(uint32_t insn, int *off) +{ + switch (insn >> 26) { + case OP_BEQ: + case OP_BEQL: + case OP_BNE: + case OP_BNEL: + case OP_BGTZ: + case OP_BGTZL: + case OP_BLEZ: + case OP_BLEZL: + *off = se16(insn) + 1; + return true; + + case OP_REGIMM: + switch ((insn >> 16) & 0x1f) { + case REGIMM_BGEZ: + case REGIMM_BGEZL: + case REGIMM_BGEZAL: + case REGIMM_BGEZALL: + case REGIMM_BLTZ: + case REGIMM_BLTZL: + case REGIMM_BLTZAL: + case REGIMM_BLTZALL: + *off = se16(insn) + 1; + return true; + + default: + return false; + } + + default: + return false; + } +} + +static int check_ll(uint64_t pc, uint32_t *code, size_t sz) +{ + ssize_t i, max, sc_pos; + int off; + + /* + * Every LL must be preceded by a sync instruction in order to ensure + * that instruction reordering doesn't allow a prior memory access to + * execute after the LL & cause erroneous results. + */ + if (!is_sync(le32toh(code[-1]))) { + fprintf(stderr, "%" PRIx64 ": LL not preceded by sync\n", pc); + return -EINVAL; + } + + /* Find the matching SC instruction */ + max = sz / 4; + for (sc_pos = 0; sc_pos < max; sc_pos++) { + if (is_sc(le32toh(code[sc_pos]))) + break; + } + if (sc_pos >= max) { + fprintf(stderr, "%" PRIx64 ": LL has no matching SC\n", pc); + return -EINVAL; + } + + /* + * Check branches within the LL/SC loop target sync instructions, + * ensuring that speculative execution can't generate memory accesses + * due to instructions outside of the loop. + */ + for (i = 0; i < sc_pos; i++) { + if (!is_branch(le32toh(code[i]), &off)) + continue; + + /* + * If the branch target is within the LL/SC loop then we don't + * need to worry about it. + */ + if ((off >= -i) && (off <= sc_pos)) + continue; + + /* If the branch targets a sync instruction we're all good... */ + if (is_sync(le32toh(code[i + off]))) + continue; + + /* ...but if not, we have a problem */ + fprintf(stderr, "%" PRIx64 ": Branch target not a sync\n", + pc + (i * 4)); + return -EINVAL; + } + + return 0; +} + +static int check_code(uint64_t pc, uint32_t *code, size_t sz) +{ + int err = 0; + + if (sz % 4) { + fprintf(stderr, "%" PRIx64 ": Section size not a multiple of 4\n", + pc); + err = -EINVAL; + sz -= (sz % 4); + } + + if (is_ll(le32toh(code[0]))) { + fprintf(stderr, "%" PRIx64 ": First instruction in section is an LL\n", + pc); + err = -EINVAL; + } + +#define advance() ( \ + code++, \ + pc += 4, \ + sz -= 4 \ +) + + /* + * Skip the first instruction, allowing check_ll to look backwards + * unconditionally. + */ + advance(); + + /* Now scan through the code looking for LL instructions */ + for (; sz; advance()) { + if (is_ll(le32toh(code[0]))) + err |= check_ll(pc, code, sz); + } + + return err; +} + +int main(int argc, char *argv[]) +{ + int vmlinux_fd, status, err, i; + const char *vmlinux_path; + struct stat st; + Elf64_Ehdr *eh; + Elf64_Shdr *sh; + void *vmlinux; + + status = EXIT_FAILURE; + + if (argc < 2) { + usage(stderr); + goto out_ret; + } + + vmlinux_path = argv[1]; + vmlinux_fd = open(vmlinux_path, O_RDONLY); + if (vmlinux_fd == -1) { + perror("Unable to open vmlinux"); + goto out_ret; + } + + err = fstat(vmlinux_fd, &st); + if (err) { + perror("Unable to stat vmlinux"); + goto out_close; + } + + vmlinux = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, vmlinux_fd, 0); + if (vmlinux == MAP_FAILED) { + perror("Unable to mmap vmlinux"); + goto out_close; + } + + eh = vmlinux; + if (memcmp(eh->e_ident, ELFMAG, SELFMAG)) { + fprintf(stderr, "vmlinux is not an ELF?\n"); + goto out_munmap; + } + + if (eh->e_ident[EI_CLASS] != ELFCLASS64) { + fprintf(stderr, "vmlinux is not 64b?\n"); + goto out_munmap; + } + + if (eh->e_ident[EI_DATA] != ELFDATA2LSB) { + fprintf(stderr, "vmlinux is not little endian?\n"); + goto out_munmap; + } + + for (i = 0; i < le16toh(eh->e_shnum); i++) { + sh = vmlinux + le64toh(eh->e_shoff) + (i * le16toh(eh->e_shentsize)); + + if (sh->sh_type != SHT_PROGBITS) + continue; + if (!(sh->sh_flags & SHF_EXECINSTR)) + continue; + + err = check_code(le64toh(sh->sh_addr), + vmlinux + le64toh(sh->sh_offset), + le64toh(sh->sh_size)); + if (err) + goto out_munmap; + } + + status = EXIT_SUCCESS; +out_munmap: + munmap(vmlinux, st.st_size); +out_close: + close(vmlinux_fd); +out_ret: + fprintf(stdout, "loongson3-llsc-check returns %s\n", + status ? "failure" : "success"); + return status; +} -- cgit v1.3.1