From a5f7a5bb3b7f28ba7e4fa246775b29a0e5537255 Mon Sep 17 00:00:00 2001 From: Jinke Han Date: Tue, 8 Sep 2026 15:37:42 +0800 Subject: x86/kprobes: Fix crash when probing CS CALL instructions When using eBPF to probe CS CALL instructions within a function, a crash can be triggered. The eBPF tool probes offset 257 of the __hrtimer_run_queues() function: <__hrtimer_run_queues+249>: nopl 0x0(%rax,%rax,1) <__hrtimer_run_queues+254>: mov %r14,%rdi <__hrtimer_run_queues+257>: cs call <__x86_indirect_thunk_r12> <__hrtimer_run_queues+263>: mov %eax,%r12d <__hrtimer_run_queues+266>: xchg %ax,%ax <__hrtimer_run_queues+268>: mov %r13,%rdi Which triggers this crash: BUG: unable to handle page fault for address: 00000000000f41c9 #PF: supervisor write access in kernel mode #PF: error_code(0x0002) - not-present page PGD 0 P4D 0 Oops: 0002 [#1] SMP NOPTI CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P RIP: 0010:__hrtimer_run_queues+0x106/0x230 Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is at the 6th byte of the above CS CALL instruction. Since the CS CALL instruction occupies 6 bytes, the exception occurred in the middle of that call instruction. The root cause is that when using eBPF tools to probe in the middle of a function, a kprobe with INT3 is used as the underlying implementation. During single-step emulation of the original CALL instruction, int3_emulate_call() assumes that the probed CALL instruction is 5 bytes long. However, the actual CS-prefixed CALL instruction occupies 6 bytes, so it constructs an incorrect exception return address. When the CPU returns from the kprobe handler, the next instruction to be executed is at the address of the last byte of that CS CALL instruction. Coincidentally, starting from that address, the CPU fetches and decodes a completely different instruction, which ultimately triggers a kernel crash. Fix the issue by using the actual instruction length obtained from the instruction decoder when constructing the exception return address, rather than relying on the hardcoded CALL_INSN_SIZE macro. [ mingo: Refined the changelog ] Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step") Suggested-by: Masami Hiramatsu (Google) Signed-off-by: Jinke Han Signed-off-by: Ingo Molnar Reviewed-by: Masami Hiramatsu (Google) Acked-by: Yafang Shao Acked-by: Borislav Petkov Cc: Peter Zijlstra Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000 --- fs/char_dev.c | 689 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 689 insertions(+) create mode 100644 fs/char_dev.c (limited to 'fs/char_dev.c') diff --git a/fs/char_dev.c b/fs/char_dev.c new file mode 100644 index 000000000..00229e25c --- /dev/null +++ b/fs/char_dev.c @@ -0,0 +1,689 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/char_dev.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "internal.h" + +static struct kobj_map *cdev_map __ro_after_init; + +static DEFINE_MUTEX(chrdevs_lock); + +#define CHRDEV_MAJOR_HASH_SIZE 255 + +static struct char_device_struct { + struct char_device_struct *next; + unsigned int major; + unsigned int baseminor; + int minorct; + char name[64]; + struct cdev *cdev; /* will die */ +} *chrdevs[CHRDEV_MAJOR_HASH_SIZE]; + +/* index in the above */ +static inline int major_to_index(unsigned major) +{ + return major % CHRDEV_MAJOR_HASH_SIZE; +} + +#ifdef CONFIG_PROC_FS + +void chrdev_show(struct seq_file *f, off_t offset) +{ + struct char_device_struct *cd; + + mutex_lock(&chrdevs_lock); + for (cd = chrdevs[major_to_index(offset)]; cd; cd = cd->next) { + if (cd->major == offset) + seq_printf(f, "%3d %s\n", cd->major, cd->name); + } + mutex_unlock(&chrdevs_lock); +} + +#endif /* CONFIG_PROC_FS */ + +static int find_dynamic_major(void) +{ + int i; + struct char_device_struct *cd; + + for (i = ARRAY_SIZE(chrdevs)-1; i >= CHRDEV_MAJOR_DYN_END; i--) { + if (chrdevs[i] == NULL) + return i; + } + + for (i = CHRDEV_MAJOR_DYN_EXT_START; + i >= CHRDEV_MAJOR_DYN_EXT_END; i--) { + for (cd = chrdevs[major_to_index(i)]; cd; cd = cd->next) + if (cd->major == i) + break; + + if (cd == NULL) + return i; + } + + return -EBUSY; +} + +/* + * Register a single major with a specified minor range. + * + * If major == 0 this function will dynamically allocate an unused major. + * If major > 0 this function will attempt to reserve the range of minors + * with given major. + * + */ +static struct char_device_struct * +__register_chrdev_region(unsigned int major, unsigned int baseminor, + int minorct, const char *name) +{ + struct char_device_struct *cd __free(kfree) = NULL; + struct char_device_struct *curr, *prev = NULL; + int ret; + int i; + + if (major >= CHRDEV_MAJOR_MAX) { + pr_err("CHRDEV \"%s\" major requested (%u) is greater than the maximum (%u)\n", + name, major, CHRDEV_MAJOR_MAX-1); + return ERR_PTR(-EINVAL); + } + + if (minorct > MINORMASK + 1 - baseminor) { + pr_err("CHRDEV \"%s\" minor range requested (%u-%u) is out of range of maximum range (%u-%u) for a single major\n", + name, baseminor, baseminor + minorct - 1, 0, MINORMASK); + return ERR_PTR(-EINVAL); + } + + cd = kzalloc_obj(struct char_device_struct); + if (cd == NULL) + return ERR_PTR(-ENOMEM); + + guard(mutex)(&chrdevs_lock); + + if (major == 0) { + ret = find_dynamic_major(); + if (ret < 0) { + pr_err("CHRDEV \"%s\" dynamic allocation region is full\n", + name); + return ERR_PTR(ret); + } + major = ret; + } + + ret = -EBUSY; + i = major_to_index(major); + for (curr = chrdevs[i]; curr; prev = curr, curr = curr->next) { + if (curr->major < major) + continue; + + if (curr->major > major) + break; + + if (curr->baseminor + curr->minorct <= baseminor) + continue; + + if (curr->baseminor >= baseminor + minorct) + break; + + return ERR_PTR(ret); + } + + cd->major = major; + cd->baseminor = baseminor; + cd->minorct = minorct; + strscpy(cd->name, name, sizeof(cd->name)); + + if (!prev) { + cd->next = curr; + chrdevs[i] = cd; + } else { + cd->next = prev->next; + prev->next = cd; + } + + return_ptr(cd); +} + +static struct char_device_struct * +__unregister_chrdev_region(unsigned major, unsigned baseminor, int minorct) +{ + struct char_device_struct *cd = NULL, **cp; + int i = major_to_index(major); + + mutex_lock(&chrdevs_lock); + for (cp = &chrdevs[i]; *cp; cp = &(*cp)->next) + if ((*cp)->major == major && + (*cp)->baseminor == baseminor && + (*cp)->minorct == minorct) + break; + if (*cp) { + cd = *cp; + *cp = cd->next; + } + mutex_unlock(&chrdevs_lock); + return cd; +} + +/** + * register_chrdev_region() - register a range of device numbers + * @from: the first in the desired range of device numbers; must include + * the major number. + * @count: the number of consecutive device numbers required + * @name: the name of the device or driver. + * + * Return value is zero on success, a negative error code on failure. + */ +int register_chrdev_region(dev_t from, unsigned count, const char *name) +{ + struct char_device_struct *cd; + dev_t to = from + count; + dev_t n, next; + + for (n = from; n < to; n = next) { + next = MKDEV(MAJOR(n)+1, 0); + if (next > to) + next = to; + cd = __register_chrdev_region(MAJOR(n), MINOR(n), + next - n, name); + if (IS_ERR(cd)) + goto fail; + } + return 0; +fail: + to = n; + for (n = from; n < to; n = next) { + next = MKDEV(MAJOR(n)+1, 0); + kfree(__unregister_chrdev_region(MAJOR(n), MINOR(n), next - n)); + } + return PTR_ERR(cd); +} + +/** + * alloc_chrdev_region() - register a range of char device numbers + * @dev: output parameter for first assigned number + * @baseminor: first of the requested range of minor numbers + * @count: the number of minor numbers required + * @name: the name of the associated device or driver + * + * Allocates a range of char device numbers. The major number will be + * chosen dynamically, and returned (along with the first minor number) + * in @dev. Returns zero or a negative error code. + */ +int alloc_chrdev_region(dev_t *dev, unsigned baseminor, unsigned count, + const char *name) +{ + struct char_device_struct *cd; + cd = __register_chrdev_region(0, baseminor, count, name); + if (IS_ERR(cd)) + return PTR_ERR(cd); + *dev = MKDEV(cd->major, cd->baseminor); + return 0; +} + +/** + * __register_chrdev() - create and register a cdev occupying a range of minors + * @major: major device number or 0 for dynamic allocation + * @baseminor: first of the requested range of minor numbers + * @count: the number of minor numbers required + * @name: name of this range of devices + * @fops: file operations associated with this devices + * + * If @major == 0 this functions will dynamically allocate a major and return + * its number. + * + * If @major > 0 this function will attempt to reserve a device with the given + * major number and will return zero on success. + * + * Returns a -ve errno on failure. + * + * The name of this device has nothing to do with the name of the device in + * /dev. It only helps to keep track of the different owners of devices. If + * your module name has only one type of devices it's ok to use e.g. the name + * of the module here. + */ +int __register_chrdev(unsigned int major, unsigned int baseminor, + unsigned int count, const char *name, + const struct file_operations *fops) +{ + struct char_device_struct *cd; + struct cdev *cdev; + int err = -ENOMEM; + + cd = __register_chrdev_region(major, baseminor, count, name); + if (IS_ERR(cd)) + return PTR_ERR(cd); + + cdev = cdev_alloc(); + if (!cdev) + goto out2; + + cdev->owner = fops->owner; + cdev->ops = fops; + kobject_set_name(&cdev->kobj, "%s", name); + + err = cdev_add(cdev, MKDEV(cd->major, baseminor), count); + if (err) + goto out; + + cd->cdev = cdev; + + return major ? 0 : cd->major; +out: + kobject_put(&cdev->kobj); +out2: + kfree(__unregister_chrdev_region(cd->major, baseminor, count)); + return err; +} + +/** + * unregister_chrdev_region() - unregister a range of device numbers + * @from: the first in the range of numbers to unregister + * @count: the number of device numbers to unregister + * + * This function will unregister a range of @count device numbers, + * starting with @from. The caller should normally be the one who + * allocated those numbers in the first place... + */ +void unregister_chrdev_region(dev_t from, unsigned count) +{ + dev_t to = from + count; + dev_t n, next; + + for (n = from; n < to; n = next) { + next = MKDEV(MAJOR(n)+1, 0); + if (next > to) + next = to; + kfree(__unregister_chrdev_region(MAJOR(n), MINOR(n), next - n)); + } +} + +/** + * __unregister_chrdev - unregister and destroy a cdev + * @major: major device number + * @baseminor: first of the range of minor numbers + * @count: the number of minor numbers this cdev is occupying + * @name: name of this range of devices + * + * Unregister and destroy the cdev occupying the region described by + * @major, @baseminor and @count. This function undoes what + * __register_chrdev() did. + */ +void __unregister_chrdev(unsigned int major, unsigned int baseminor, + unsigned int count, const char *name) +{ + struct char_device_struct *cd; + + cd = __unregister_chrdev_region(major, baseminor, count); + if (cd && cd->cdev) + cdev_del(cd->cdev); + kfree(cd); +} + +static __cacheline_aligned_in_smp DEFINE_SPINLOCK(cdev_lock); + +static struct kobject *cdev_get(struct cdev *p) +{ + struct module *owner = p->owner; + struct kobject *kobj; + + if (!try_module_get(owner)) + return NULL; + kobj = kobject_get_unless_zero(&p->kobj); + if (!kobj) + module_put(owner); + return kobj; +} + +void cdev_put(struct cdev *p) +{ + if (p) { + struct module *owner = p->owner; + kobject_put(&p->kobj); + module_put(owner); + } +} + +/* + * Called every time a character special file is opened + */ +static int chrdev_open(struct inode *inode, struct file *filp) +{ + const struct file_operations *fops; + struct cdev *p; + struct cdev *new = NULL; + int ret = 0; + + spin_lock(&cdev_lock); + p = inode->i_cdev; + if (!p) { + struct kobject *kobj; + int idx; + spin_unlock(&cdev_lock); + kobj = kobj_lookup(cdev_map, inode->i_rdev, &idx); + if (!kobj) + return -ENXIO; + new = container_of(kobj, struct cdev, kobj); + spin_lock(&cdev_lock); + /* Check i_cdev again in case somebody beat us to it while + we dropped the lock. */ + p = inode->i_cdev; + if (!p) { + inode->i_cdev = p = new; + list_add(&inode->i_devices, &p->list); + new = NULL; + } else if (!cdev_get(p)) + ret = -ENXIO; + } else if (!cdev_get(p)) + ret = -ENXIO; + spin_unlock(&cdev_lock); + cdev_put(new); + if (ret) + return ret; + + ret = -ENXIO; + fops = fops_get(p->ops); + if (!fops) + goto out_cdev_put; + + replace_fops(filp, fops); + if (filp->f_op->open) { + ret = filp->f_op->open(inode, filp); + if (ret) + goto out_cdev_put; + } + + return 0; + + out_cdev_put: + cdev_put(p); + return ret; +} + +void cd_forget(struct inode *inode) +{ + spin_lock(&cdev_lock); + list_del_init(&inode->i_devices); + inode->i_cdev = NULL; + inode->i_mapping = &inode->i_data; + spin_unlock(&cdev_lock); +} + +static void cdev_purge(struct cdev *cdev) +{ + spin_lock(&cdev_lock); + while (!list_empty(&cdev->list)) { + struct inode *inode; + inode = container_of(cdev->list.next, struct inode, i_devices); + list_del_init(&inode->i_devices); + inode->i_cdev = NULL; + } + spin_unlock(&cdev_lock); +} + +/* + * Dummy default file-operations: the only thing this does + * is contain the open that then fills in the correct operations + * depending on the special file... + */ +const struct file_operations def_chr_fops = { + .open = chrdev_open, + .llseek = noop_llseek, +}; + +static struct kobject *exact_match(dev_t dev, int *part, void *data) +{ + struct cdev *p = data; + return &p->kobj; +} + +static int exact_lock(dev_t dev, void *data) +{ + struct cdev *p = data; + return cdev_get(p) ? 0 : -1; +} + +/** + * cdev_add() - add a char device to the system + * @p: the cdev structure for the device + * @dev: the first device number for which this device is responsible + * @count: the number of consecutive minor numbers corresponding to this + * device + * + * cdev_add() adds the device represented by @p to the system, making it + * live immediately. A negative error code is returned on failure. + */ +int cdev_add(struct cdev *p, dev_t dev, unsigned count) +{ + int error; + + p->dev = dev; + p->count = count; + + if (WARN_ON(dev == WHITEOUT_DEV)) { + error = -EBUSY; + goto err; + } + + error = kobj_map(cdev_map, dev, count, NULL, + exact_match, exact_lock, p); + if (error) + goto err; + + kobject_get(p->kobj.parent); + + return 0; + +err: + kfree_const(p->kobj.name); + p->kobj.name = NULL; + return error; +} + +/** + * cdev_set_parent() - set the parent kobject for a char device + * @p: the cdev structure + * @kobj: the kobject to take a reference to + * + * cdev_set_parent() sets a parent kobject which will be referenced + * appropriately so the parent is not freed before the cdev. This + * should be called before cdev_add. + */ +void cdev_set_parent(struct cdev *p, struct kobject *kobj) +{ + WARN_ON(!kobj->state_initialized); + p->kobj.parent = kobj; +} + +/** + * cdev_device_add() - add a char device and it's corresponding + * struct device, linkink + * @dev: the device structure + * @cdev: the cdev structure + * + * cdev_device_add() adds the char device represented by @cdev to the system, + * just as cdev_add does. It then adds @dev to the system using device_add + * The dev_t for the char device will be taken from the struct device which + * needs to be initialized first. This helper function correctly takes a + * reference to the parent device so the parent will not get released until + * all references to the cdev are released. + * + * This helper uses dev->devt for the device number. If it is not set + * it will not add the cdev and it will be equivalent to device_add. + * + * This function should be used whenever the struct cdev and the + * struct device are members of the same structure whose lifetime is + * managed by the struct device. + * + * NOTE: Callers must assume that userspace was able to open the cdev and + * can call cdev fops callbacks at any time, even if this function fails. + */ +int cdev_device_add(struct cdev *cdev, struct device *dev) +{ + int rc = 0; + + if (dev->devt) { + cdev_set_parent(cdev, &dev->kobj); + + rc = cdev_add(cdev, dev->devt, 1); + if (rc) + return rc; + } + + rc = device_add(dev); + if (rc && dev->devt) + cdev_del(cdev); + + return rc; +} + +/** + * cdev_device_del() - inverse of cdev_device_add + * @cdev: the cdev structure + * @dev: the device structure + * + * cdev_device_del() is a helper function to call cdev_del and device_del. + * It should be used whenever cdev_device_add is used. + * + * If dev->devt is not set it will not remove the cdev and will be equivalent + * to device_del. + * + * NOTE: This guarantees that associated sysfs callbacks are not running + * or runnable, however any cdevs already open will remain and their fops + * will still be callable even after this function returns. + */ +void cdev_device_del(struct cdev *cdev, struct device *dev) +{ + device_del(dev); + if (dev->devt) + cdev_del(cdev); +} + +static void cdev_unmap(dev_t dev, unsigned count) +{ + kobj_unmap(cdev_map, dev, count); +} + +/** + * cdev_del() - remove a cdev from the system + * @p: the cdev structure to be removed + * + * cdev_del() removes @p from the system, possibly freeing the structure + * itself. + * + * NOTE: This guarantees that cdev device will no longer be able to be + * opened, however any cdevs already open will remain and their fops will + * still be callable even after cdev_del returns. + */ +void cdev_del(struct cdev *p) +{ + cdev_unmap(p->dev, p->count); + kobject_put(&p->kobj); +} + + +static void cdev_default_release(struct kobject *kobj) +{ + struct cdev *p = container_of(kobj, struct cdev, kobj); + struct kobject *parent = kobj->parent; + + cdev_purge(p); + kobject_put(parent); +} + +static void cdev_dynamic_release(struct kobject *kobj) +{ + struct cdev *p = container_of(kobj, struct cdev, kobj); + struct kobject *parent = kobj->parent; + + cdev_purge(p); + kfree(p); + kobject_put(parent); +} + +static struct kobj_type ktype_cdev_default = { + .release = cdev_default_release, +}; + +static struct kobj_type ktype_cdev_dynamic = { + .release = cdev_dynamic_release, +}; + +/** + * cdev_alloc() - allocate a cdev structure + * + * Allocates and returns a cdev structure, or NULL on failure. + */ +struct cdev *cdev_alloc(void) +{ + struct cdev *p = kzalloc_obj(struct cdev); + if (p) { + INIT_LIST_HEAD(&p->list); + kobject_init(&p->kobj, &ktype_cdev_dynamic); + } + return p; +} + +/** + * cdev_init() - initialize a cdev structure + * @cdev: the structure to initialize + * @fops: the file_operations for this device + * + * Initializes @cdev, remembering @fops, making it ready to add to the + * system with cdev_add(). + */ +void cdev_init(struct cdev *cdev, const struct file_operations *fops) +{ + memset(cdev, 0, sizeof *cdev); + INIT_LIST_HEAD(&cdev->list); + kobject_init(&cdev->kobj, &ktype_cdev_default); + cdev->ops = fops; +} + +static struct kobject *base_probe(dev_t dev, int *part, void *data) +{ + if (request_module("char-major-%d-%d", MAJOR(dev), MINOR(dev)) > 0) + /* Make old-style 2.4 aliases work */ + request_module("char-major-%d", MAJOR(dev)); + return NULL; +} + +void __init chrdev_init(void) +{ + cdev_map = kobj_map_init(base_probe, &chrdevs_lock); +} + + +/* Let modules do char dev stuff */ +EXPORT_SYMBOL(register_chrdev_region); +EXPORT_SYMBOL(unregister_chrdev_region); +EXPORT_SYMBOL(alloc_chrdev_region); +EXPORT_SYMBOL(cdev_init); +EXPORT_SYMBOL(cdev_alloc); +EXPORT_SYMBOL(cdev_del); +EXPORT_SYMBOL(cdev_add); +EXPORT_SYMBOL(cdev_set_parent); +EXPORT_SYMBOL(cdev_device_add); +EXPORT_SYMBOL(cdev_device_del); +EXPORT_SYMBOL(__register_chrdev); +EXPORT_SYMBOL(__unregister_chrdev); -- cgit v1.3.1