From a5f7a5bb3b7f28ba7e4fa246775b29a0e5537255 Mon Sep 17 00:00:00 2001 From: Jinke Han Date: Tue, 8 Sep 2026 15:37:42 +0800 Subject: x86/kprobes: Fix crash when probing CS CALL instructions When using eBPF to probe CS CALL instructions within a function, a crash can be triggered. The eBPF tool probes offset 257 of the __hrtimer_run_queues() function: <__hrtimer_run_queues+249>: nopl 0x0(%rax,%rax,1) <__hrtimer_run_queues+254>: mov %r14,%rdi <__hrtimer_run_queues+257>: cs call <__x86_indirect_thunk_r12> <__hrtimer_run_queues+263>: mov %eax,%r12d <__hrtimer_run_queues+266>: xchg %ax,%ax <__hrtimer_run_queues+268>: mov %r13,%rdi Which triggers this crash: BUG: unable to handle page fault for address: 00000000000f41c9 #PF: supervisor write access in kernel mode #PF: error_code(0x0002) - not-present page PGD 0 P4D 0 Oops: 0002 [#1] SMP NOPTI CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P RIP: 0010:__hrtimer_run_queues+0x106/0x230 Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is at the 6th byte of the above CS CALL instruction. Since the CS CALL instruction occupies 6 bytes, the exception occurred in the middle of that call instruction. The root cause is that when using eBPF tools to probe in the middle of a function, a kprobe with INT3 is used as the underlying implementation. During single-step emulation of the original CALL instruction, int3_emulate_call() assumes that the probed CALL instruction is 5 bytes long. However, the actual CS-prefixed CALL instruction occupies 6 bytes, so it constructs an incorrect exception return address. When the CPU returns from the kprobe handler, the next instruction to be executed is at the address of the last byte of that CS CALL instruction. Coincidentally, starting from that address, the CPU fetches and decodes a completely different instruction, which ultimately triggers a kernel crash. Fix the issue by using the actual instruction length obtained from the instruction decoder when constructing the exception return address, rather than relying on the hardcoded CALL_INSN_SIZE macro. [ mingo: Refined the changelog ] Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step") Suggested-by: Masami Hiramatsu (Google) Signed-off-by: Jinke Han Signed-off-by: Ingo Molnar Reviewed-by: Masami Hiramatsu (Google) Acked-by: Yafang Shao Acked-by: Borislav Petkov Cc: Peter Zijlstra Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000 --- fs/notify/inotify/Kconfig | 17 + fs/notify/inotify/Makefile | 2 + fs/notify/inotify/inotify.h | 69 +++ fs/notify/inotify/inotify_fsnotify.c | 209 +++++++++ fs/notify/inotify/inotify_user.c | 848 +++++++++++++++++++++++++++++++++++ 5 files changed, 1145 insertions(+) create mode 100644 fs/notify/inotify/Kconfig create mode 100644 fs/notify/inotify/Makefile create mode 100644 fs/notify/inotify/inotify.h create mode 100644 fs/notify/inotify/inotify_fsnotify.c create mode 100644 fs/notify/inotify/inotify_user.c (limited to 'fs/notify/inotify') diff --git a/fs/notify/inotify/Kconfig b/fs/notify/inotify/Kconfig new file mode 100644 index 000000000..1cc8be25d --- /dev/null +++ b/fs/notify/inotify/Kconfig @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: GPL-2.0-only +config INOTIFY_USER + bool "Inotify support for userspace" + select FSNOTIFY + default y + help + Say Y here to enable inotify support for userspace, including the + associated system calls. Inotify allows monitoring of both files and + directories via a single open fd. Events are read from the file + descriptor, which is also select()- and poll()-able. + Inotify fixes numerous shortcomings in dnotify and introduces several + new features including multiple file events, one-shot support, and + unmount notification. + + For more information, see + + If unsure, say Y. diff --git a/fs/notify/inotify/Makefile b/fs/notify/inotify/Makefile new file mode 100644 index 000000000..812237eec --- /dev/null +++ b/fs/notify/inotify/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_INOTIFY_USER) += inotify_fsnotify.o inotify_user.o diff --git a/fs/notify/inotify/inotify.h b/fs/notify/inotify/inotify.h new file mode 100644 index 000000000..7d5df7a21 --- /dev/null +++ b/fs/notify/inotify/inotify.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include /* struct kmem_cache */ + +struct inotify_event_info { + struct fsnotify_event fse; + u32 mask; + int wd; + u32 sync_cookie; + int name_len; + char name[]; +}; + +struct inotify_inode_mark { + struct fsnotify_mark fsn_mark; + int wd; +}; + +static inline struct inotify_event_info *INOTIFY_E(struct fsnotify_event *fse) +{ + return container_of(fse, struct inotify_event_info, fse); +} + +/* + * INOTIFY_USER_FLAGS represents all of the mask bits that we expose to + * userspace. There is at least one bit (FS_EVENT_ON_CHILD) which is + * used only internally to the kernel. + */ +#define INOTIFY_USER_MASK (IN_ALL_EVENTS) + +static inline __u32 inotify_mark_user_mask(struct fsnotify_mark *fsn_mark) +{ + __u32 mask = fsn_mark->mask & INOTIFY_USER_MASK; + + if (fsn_mark->flags & FSNOTIFY_MARK_FLAG_EXCL_UNLINK) + mask |= IN_EXCL_UNLINK; + if (fsn_mark->flags & FSNOTIFY_MARK_FLAG_IN_ONESHOT) + mask |= IN_ONESHOT; + + return mask; +} + +extern void inotify_ignored_and_remove_idr(struct fsnotify_mark *fsn_mark, + struct fsnotify_group *group); +extern int inotify_handle_inode_event(struct fsnotify_mark *inode_mark, + u32 mask, struct inode *inode, + struct inode *dir, + const struct qstr *name, u32 cookie); + +extern const struct fsnotify_ops inotify_fsnotify_ops; +extern struct kmem_cache *inotify_inode_mark_cachep; + +#ifdef CONFIG_INOTIFY_USER +static inline void dec_inotify_instances(struct ucounts *ucounts) +{ + dec_ucount(ucounts, UCOUNT_INOTIFY_INSTANCES); +} + +static inline struct ucounts *inc_inotify_watches(struct ucounts *ucounts) +{ + return inc_ucount(ucounts->ns, ucounts->uid, UCOUNT_INOTIFY_WATCHES); +} + +static inline void dec_inotify_watches(struct ucounts *ucounts) +{ + dec_ucount(ucounts, UCOUNT_INOTIFY_WATCHES); +} +#endif diff --git a/fs/notify/inotify/inotify_fsnotify.c b/fs/notify/inotify/inotify_fsnotify.c new file mode 100644 index 000000000..7c326ec2e --- /dev/null +++ b/fs/notify/inotify/inotify_fsnotify.c @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * fs/inotify_user.c - inotify support for userspace + * + * Authors: + * John McCutchan + * Robert Love + * + * Copyright (C) 2005 John McCutchan + * Copyright 2006 Hewlett-Packard Development Company, L.P. + * + * Copyright (C) 2009 Eric Paris + * inotify was largely rewritten to make use of the fsnotify infrastructure + */ + +#include /* d_unlinked */ +#include /* struct inode */ +#include +#include +#include /* struct path */ +#include /* kmem_* */ +#include +#include +#include +#include + +#include "inotify.h" + +/* + * Check if 2 events contain the same information. + */ +static bool event_compare(struct fsnotify_event *old_fsn, + struct fsnotify_event *new_fsn) +{ + struct inotify_event_info *old, *new; + + old = INOTIFY_E(old_fsn); + new = INOTIFY_E(new_fsn); + if (old->mask & FS_IN_IGNORED) + return false; + if ((old->mask == new->mask) && + (old->wd == new->wd) && + (old->name_len == new->name_len) && + (!old->name_len || !strcmp(old->name, new->name))) + return true; + return false; +} + +static int inotify_merge(struct fsnotify_group *group, + struct fsnotify_event *event) +{ + struct list_head *list = &group->notification_list; + struct fsnotify_event *last_event; + + last_event = list_entry(list->prev, struct fsnotify_event, list); + return event_compare(last_event, event); +} + +int inotify_handle_inode_event(struct fsnotify_mark *inode_mark, u32 mask, + struct inode *inode, struct inode *dir, + const struct qstr *name, u32 cookie) +{ + struct inotify_inode_mark *i_mark; + struct inotify_event_info *event; + struct fsnotify_event *fsn_event; + struct fsnotify_group *group = inode_mark->group; + int ret; + int len = 0, wd; + int alloc_len = sizeof(struct inotify_event_info); + struct mem_cgroup *old_memcg; + + if (name) { + len = name->len; + alloc_len += len + 1; + } + + pr_debug("%s: group=%p mark=%p mask=%x\n", __func__, group, inode_mark, + mask); + + i_mark = container_of(inode_mark, struct inotify_inode_mark, + fsn_mark); + + /* + * We can be racing with mark being detached. Don't report event with + * invalid wd. + */ + wd = READ_ONCE(i_mark->wd); + if (wd == -1) + return 0; + /* + * Whoever is interested in the event, pays for the allocation. Do not + * trigger OOM killer in the target monitoring memcg as it may have + * security repercussion. + */ + old_memcg = set_active_memcg(group->memcg); + event = kmalloc(alloc_len, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL); + set_active_memcg(old_memcg); + + if (unlikely(!event)) { + /* + * Treat lost event due to ENOMEM the same way as queue + * overflow to let userspace know event was lost. + */ + fsnotify_queue_overflow(group); + return -ENOMEM; + } + + /* + * We now report FS_ISDIR flag with MOVE_SELF and DELETE_SELF events + * for fanotify. inotify never reported IN_ISDIR with those events. + * It looks like an oversight, but to avoid the risk of breaking + * existing inotify programs, mask the flag out from those events. + */ + if (mask & (IN_MOVE_SELF | IN_DELETE_SELF)) + mask &= ~IN_ISDIR; + + fsn_event = &event->fse; + fsnotify_init_event(fsn_event); + event->mask = mask; + event->wd = wd; + event->sync_cookie = cookie; + event->name_len = len; + if (len) + strscpy(event->name, name->name, event->name_len + 1); + + ret = fsnotify_add_event(group, fsn_event, inotify_merge); + if (ret) { + /* Our event wasn't used in the end. Free it. */ + fsnotify_destroy_event(group, fsn_event); + } + + if (inode_mark->flags & FSNOTIFY_MARK_FLAG_IN_ONESHOT) + fsnotify_destroy_mark(inode_mark, group); + + return 0; +} + +static void inotify_freeing_mark(struct fsnotify_mark *fsn_mark, struct fsnotify_group *group) +{ + inotify_ignored_and_remove_idr(fsn_mark, group); +} + +/* + * This is NEVER supposed to be called. Inotify marks should either have been + * removed from the idr when the watch was removed or in the + * fsnotify_destroy_mark_by_group() call when the inotify instance was being + * torn down. This is only called if the idr is about to be freed but there + * are still marks in it. + */ +static int idr_callback(int id, void *p, void *data) +{ + struct fsnotify_mark *fsn_mark; + struct inotify_inode_mark *i_mark; + static bool warned = false; + + if (warned) + return 0; + + warned = true; + fsn_mark = p; + i_mark = container_of(fsn_mark, struct inotify_inode_mark, fsn_mark); + + WARN(1, "inotify closing but id=%d for fsn_mark=%p in group=%p still in " + "idr. Probably leaking memory\n", id, p, data); + + /* + * I'm taking the liberty of assuming that the mark in question is a + * valid address and I'm dereferencing it. This might help to figure + * out why we got here and the panic is no worse than the original + * BUG() that was here. + */ + if (fsn_mark) + printk(KERN_WARNING "fsn_mark->group=%p wd=%d\n", + fsn_mark->group, i_mark->wd); + return 0; +} + +static void inotify_free_group_priv(struct fsnotify_group *group) +{ + /* ideally the idr is empty and we won't hit the BUG in the callback */ + idr_for_each(&group->inotify_data.idr, idr_callback, group); + idr_destroy(&group->inotify_data.idr); + if (group->inotify_data.ucounts) + dec_inotify_instances(group->inotify_data.ucounts); +} + +static void inotify_free_event(struct fsnotify_group *group, + struct fsnotify_event *fsn_event) +{ + kfree(INOTIFY_E(fsn_event)); +} + +/* ding dong the mark is dead */ +static void inotify_free_mark(struct fsnotify_mark *fsn_mark) +{ + struct inotify_inode_mark *i_mark; + + i_mark = container_of(fsn_mark, struct inotify_inode_mark, fsn_mark); + + kmem_cache_free(inotify_inode_mark_cachep, i_mark); +} + +const struct fsnotify_ops inotify_fsnotify_ops = { + .handle_inode_event = inotify_handle_inode_event, + .free_group_priv = inotify_free_group_priv, + .free_event = inotify_free_event, + .freeing_mark = inotify_freeing_mark, + .free_mark = inotify_free_mark, +}; diff --git a/fs/notify/inotify/inotify_user.c b/fs/notify/inotify/inotify_user.c new file mode 100644 index 000000000..5f19c24ec --- /dev/null +++ b/fs/notify/inotify/inotify_user.c @@ -0,0 +1,848 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * fs/inotify_user.c - inotify support for userspace + * + * Authors: + * John McCutchan + * Robert Love + * + * Copyright (C) 2005 John McCutchan + * Copyright 2006 Hewlett-Packard Development Company, L.P. + * + * Copyright (C) 2009 Eric Paris + * inotify was largely rewriten to make use of the fsnotify infrastructure + */ + +#include +#include /* struct inode */ +#include +#include +#include /* fs_initcall */ +#include +#include /* roundup() */ +#include /* LOOKUP_FOLLOW */ +#include +#include /* struct kmem_cache */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "inotify.h" +#include "../fdinfo.h" + +#include + +/* + * An inotify watch requires allocating an inotify_inode_mark structure as + * well as pinning the watched inode. Doubling the size of a VFS inode + * should be more than enough to cover the additional filesystem inode + * size increase. + */ +#define INOTIFY_WATCH_COST (sizeof(struct inotify_inode_mark) + \ + 2 * sizeof(struct inode)) + +/* configurable via /proc/sys/fs/inotify/ */ +static int inotify_max_queued_events __read_mostly; + +struct kmem_cache *inotify_inode_mark_cachep __ro_after_init; + +#ifdef CONFIG_SYSCTL + +#include + +static long it_zero = 0; +static long it_int_max = INT_MAX; + +static const struct ctl_table inotify_table[] = { + { + .procname = "max_user_instances", + .data = &init_user_ns.ucount_max[UCOUNT_INOTIFY_INSTANCES], + .maxlen = sizeof(long), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + .extra1 = &it_zero, + .extra2 = &it_int_max, + }, + { + .procname = "max_user_watches", + .data = &init_user_ns.ucount_max[UCOUNT_INOTIFY_WATCHES], + .maxlen = sizeof(long), + .mode = 0644, + .proc_handler = proc_doulongvec_minmax, + .extra1 = &it_zero, + .extra2 = &it_int_max, + }, + { + .procname = "max_queued_events", + .data = &inotify_max_queued_events, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO + }, +}; + +static void __init inotify_sysctls_init(void) +{ + register_sysctl("fs/inotify", inotify_table); +} + +#else +#define inotify_sysctls_init() do { } while (0) +#endif /* CONFIG_SYSCTL */ + +static inline __u32 inotify_arg_to_mask(struct inode *inode, u32 arg) +{ + __u32 mask; + + /* + * Everything should receive events when the inode is unmounted. + * All directories care about children. + */ + mask = (FS_UNMOUNT); + if (S_ISDIR(inode->i_mode)) + mask |= FS_EVENT_ON_CHILD; + + /* mask off the flags used to open the fd */ + mask |= (arg & INOTIFY_USER_MASK); + + return mask; +} + +#define INOTIFY_MARK_FLAGS \ + (FSNOTIFY_MARK_FLAG_EXCL_UNLINK | FSNOTIFY_MARK_FLAG_IN_ONESHOT) + +static inline unsigned int inotify_arg_to_flags(u32 arg) +{ + unsigned int flags = 0; + + if (arg & IN_EXCL_UNLINK) + flags |= FSNOTIFY_MARK_FLAG_EXCL_UNLINK; + if (arg & IN_ONESHOT) + flags |= FSNOTIFY_MARK_FLAG_IN_ONESHOT; + + return flags; +} + +static inline u32 inotify_mask_to_arg(__u32 mask) +{ + return mask & (IN_ALL_EVENTS | IN_ISDIR | IN_UNMOUNT | IN_IGNORED | + IN_Q_OVERFLOW); +} + +/* inotify userspace file descriptor functions */ +static __poll_t inotify_poll(struct file *file, poll_table *wait) +{ + struct fsnotify_group *group = file->private_data; + __poll_t ret = 0; + + poll_wait(file, &group->notification_waitq, wait); + spin_lock(&group->notification_lock); + if (!fsnotify_notify_queue_is_empty(group)) + ret = EPOLLIN | EPOLLRDNORM; + spin_unlock(&group->notification_lock); + + return ret; +} + +static int round_event_name_len(struct fsnotify_event *fsn_event) +{ + struct inotify_event_info *event; + + event = INOTIFY_E(fsn_event); + if (!event->name_len) + return 0; + return roundup(event->name_len + 1, sizeof(struct inotify_event)); +} + +/* + * Get an inotify_kernel_event if one exists and is small + * enough to fit in "count". Return an error pointer if + * not large enough. + * + * Called with the group->notification_lock held. + */ +static struct fsnotify_event *get_one_event(struct fsnotify_group *group, + size_t count) +{ + size_t event_size = sizeof(struct inotify_event); + struct fsnotify_event *event; + + event = fsnotify_peek_first_event(group); + if (!event) + return NULL; + + pr_debug("%s: group=%p event=%p\n", __func__, group, event); + + event_size += round_event_name_len(event); + if (event_size > count) + return ERR_PTR(-EINVAL); + + /* held the notification_lock the whole time, so this is the + * same event we peeked above */ + fsnotify_remove_first_event(group); + + return event; +} + +/* + * Copy an event to user space, returning how much we copied. + * + * We already checked that the event size is smaller than the + * buffer we had in "get_one_event()" above. + */ +static ssize_t copy_event_to_user(struct fsnotify_group *group, + struct fsnotify_event *fsn_event, + char __user *buf) +{ + struct inotify_event inotify_event; + struct inotify_event_info *event; + size_t event_size = sizeof(struct inotify_event); + size_t name_len; + size_t pad_name_len; + + pr_debug("%s: group=%p event=%p\n", __func__, group, fsn_event); + + event = INOTIFY_E(fsn_event); + name_len = event->name_len; + /* + * round up name length so it is a multiple of event_size + * plus an extra byte for the terminating '\0'. + */ + pad_name_len = round_event_name_len(fsn_event); + inotify_event.len = pad_name_len; + inotify_event.mask = inotify_mask_to_arg(event->mask); + inotify_event.wd = event->wd; + inotify_event.cookie = event->sync_cookie; + + /* send the main event */ + if (copy_to_user(buf, &inotify_event, event_size)) + return -EFAULT; + + buf += event_size; + + /* + * fsnotify only stores the pathname, so here we have to send the pathname + * and then pad that pathname out to a multiple of sizeof(inotify_event) + * with zeros. + */ + if (pad_name_len) { + /* copy the path name */ + if (copy_to_user(buf, event->name, name_len)) + return -EFAULT; + buf += name_len; + + /* fill userspace with 0's */ + if (clear_user(buf, pad_name_len - name_len)) + return -EFAULT; + event_size += pad_name_len; + } + + return event_size; +} + +static ssize_t inotify_read(struct file *file, char __user *buf, + size_t count, loff_t *pos) +{ + struct fsnotify_group *group; + struct fsnotify_event *kevent; + char __user *start; + int ret; + DEFINE_WAIT_FUNC(wait, woken_wake_function); + + start = buf; + group = file->private_data; + + add_wait_queue(&group->notification_waitq, &wait); + while (1) { + spin_lock(&group->notification_lock); + kevent = get_one_event(group, count); + spin_unlock(&group->notification_lock); + + pr_debug("%s: group=%p kevent=%p\n", __func__, group, kevent); + + if (kevent) { + ret = PTR_ERR(kevent); + if (IS_ERR(kevent)) + break; + ret = copy_event_to_user(group, kevent, buf); + fsnotify_destroy_event(group, kevent); + if (ret < 0) + break; + buf += ret; + count -= ret; + continue; + } + + ret = -EAGAIN; + if (file->f_flags & O_NONBLOCK) + break; + ret = -ERESTARTSYS; + if (signal_pending(current)) + break; + + if (start != buf) + break; + + wait_woken(&wait, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); + } + remove_wait_queue(&group->notification_waitq, &wait); + + if (start != buf && ret != -EFAULT) + ret = buf - start; + return ret; +} + +static int inotify_release(struct inode *ignored, struct file *file) +{ + struct fsnotify_group *group = file->private_data; + + pr_debug("%s: group=%p\n", __func__, group); + + /* free this group, matching get was inotify_init->fsnotify_obtain_group */ + fsnotify_destroy_group(group); + + return 0; +} + +static long inotify_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct fsnotify_group *group; + struct fsnotify_event *fsn_event; + void __user *p; + int ret = -ENOTTY; + size_t send_len = 0; + + group = file->private_data; + p = (void __user *) arg; + + pr_debug("%s: group=%p cmd=%u\n", __func__, group, cmd); + + switch (cmd) { + case FIONREAD: + spin_lock(&group->notification_lock); + list_for_each_entry(fsn_event, &group->notification_list, + list) { + send_len += sizeof(struct inotify_event); + send_len += round_event_name_len(fsn_event); + } + spin_unlock(&group->notification_lock); + ret = put_user(send_len, (int __user *) p); + break; +#ifdef CONFIG_CHECKPOINT_RESTORE + case INOTIFY_IOC_SETNEXTWD: + ret = -EINVAL; + if (arg >= 1 && arg <= INT_MAX) { + struct inotify_group_private_data *data; + + data = &group->inotify_data; + spin_lock(&data->idr_lock); + idr_set_cursor(&data->idr, (unsigned int)arg); + spin_unlock(&data->idr_lock); + ret = 0; + } + break; +#endif /* CONFIG_CHECKPOINT_RESTORE */ + } + + return ret; +} + +static const struct file_operations inotify_fops = { + .show_fdinfo = inotify_show_fdinfo, + .poll = inotify_poll, + .read = inotify_read, + .fasync = fsnotify_fasync, + .release = inotify_release, + .unlocked_ioctl = inotify_ioctl, + .compat_ioctl = inotify_ioctl, + .llseek = noop_llseek, +}; + + +/* + * find_inode - resolve a user-given path to a specific inode + */ +static int inotify_find_inode(const char __user *dirname, struct path *path, + unsigned int flags, __u64 mask) +{ + int error; + + error = user_path_at(AT_FDCWD, dirname, flags, path); + if (error) + return error; + /* you can only watch an inode if you have read permissions on it */ + error = path_permission(path, MAY_READ); + if (error) { + path_put(path); + return error; + } + error = security_path_notify(path, mask, + FSNOTIFY_OBJ_TYPE_INODE); + if (error) + path_put(path); + + return error; +} + +static int inotify_add_to_idr(struct idr *idr, spinlock_t *idr_lock, + struct inotify_inode_mark *i_mark) +{ + int ret; + + idr_preload(GFP_KERNEL); + spin_lock(idr_lock); + + ret = idr_alloc_cyclic(idr, i_mark, 1, 0, GFP_NOWAIT); + if (ret >= 0) { + /* we added the mark to the idr, take a reference */ + i_mark->wd = ret; + fsnotify_get_mark(&i_mark->fsn_mark); + } + + spin_unlock(idr_lock); + idr_preload_end(); + return ret < 0 ? ret : 0; +} + +static struct inotify_inode_mark *inotify_idr_find_locked(struct fsnotify_group *group, + int wd) +{ + struct idr *idr = &group->inotify_data.idr; + spinlock_t *idr_lock = &group->inotify_data.idr_lock; + struct inotify_inode_mark *i_mark; + + assert_spin_locked(idr_lock); + + i_mark = idr_find(idr, wd); + if (i_mark) { + struct fsnotify_mark *fsn_mark = &i_mark->fsn_mark; + + fsnotify_get_mark(fsn_mark); + /* One ref for being in the idr, one ref we just took */ + BUG_ON(refcount_read(&fsn_mark->refcnt) < 2); + } + + return i_mark; +} + +static struct inotify_inode_mark *inotify_idr_find(struct fsnotify_group *group, + int wd) +{ + struct inotify_inode_mark *i_mark; + spinlock_t *idr_lock = &group->inotify_data.idr_lock; + + spin_lock(idr_lock); + i_mark = inotify_idr_find_locked(group, wd); + spin_unlock(idr_lock); + + return i_mark; +} + +/* + * Remove the mark from the idr (if present) and drop the reference + * on the mark because it was in the idr. + */ +static void inotify_remove_from_idr(struct fsnotify_group *group, + struct inotify_inode_mark *i_mark) +{ + struct idr *idr = &group->inotify_data.idr; + spinlock_t *idr_lock = &group->inotify_data.idr_lock; + struct inotify_inode_mark *found_i_mark = NULL; + int wd; + + spin_lock(idr_lock); + wd = i_mark->wd; + + /* + * does this i_mark think it is in the idr? we shouldn't get called + * if it wasn't.... + */ + if (wd == -1) { + WARN_ONCE(1, "%s: i_mark=%p i_mark->wd=%d i_mark->group=%p\n", + __func__, i_mark, i_mark->wd, i_mark->fsn_mark.group); + goto out; + } + + /* Lets look in the idr to see if we find it */ + found_i_mark = inotify_idr_find_locked(group, wd); + if (unlikely(!found_i_mark)) { + WARN_ONCE(1, "%s: i_mark=%p i_mark->wd=%d i_mark->group=%p\n", + __func__, i_mark, i_mark->wd, i_mark->fsn_mark.group); + goto out; + } + + /* + * We found an mark in the idr at the right wd, but it's + * not the mark we were told to remove. eparis seriously + * fucked up somewhere. + */ + if (unlikely(found_i_mark != i_mark)) { + WARN_ONCE(1, "%s: i_mark=%p i_mark->wd=%d i_mark->group=%p " + "found_i_mark=%p found_i_mark->wd=%d " + "found_i_mark->group=%p\n", __func__, i_mark, + i_mark->wd, i_mark->fsn_mark.group, found_i_mark, + found_i_mark->wd, found_i_mark->fsn_mark.group); + goto out; + } + + /* + * One ref for being in the idr + * one ref grabbed by inotify_idr_find + */ + if (unlikely(refcount_read(&i_mark->fsn_mark.refcnt) < 2)) { + printk(KERN_ERR "%s: i_mark=%p i_mark->wd=%d i_mark->group=%p\n", + __func__, i_mark, i_mark->wd, i_mark->fsn_mark.group); + /* we can't really recover with bad ref cnting.. */ + BUG(); + } + + idr_remove(idr, wd); + /* Removed from the idr, drop that ref. */ + fsnotify_put_mark(&i_mark->fsn_mark); +out: + i_mark->wd = -1; + spin_unlock(idr_lock); + /* match the ref taken by inotify_idr_find_locked() */ + if (found_i_mark) + fsnotify_put_mark(&found_i_mark->fsn_mark); +} + +/* + * Send IN_IGNORED for this wd, remove this wd from the idr. + */ +void inotify_ignored_and_remove_idr(struct fsnotify_mark *fsn_mark, + struct fsnotify_group *group) +{ + struct inotify_inode_mark *i_mark; + + /* Queue ignore event for the watch */ + inotify_handle_inode_event(fsn_mark, FS_IN_IGNORED, NULL, NULL, NULL, + 0); + + i_mark = container_of(fsn_mark, struct inotify_inode_mark, fsn_mark); + /* remove this mark from the idr */ + inotify_remove_from_idr(group, i_mark); + + dec_inotify_watches(group->inotify_data.ucounts); +} + +static int inotify_update_existing_watch(struct fsnotify_group *group, + struct inode *inode, + u32 arg) +{ + struct fsnotify_mark *fsn_mark; + struct inotify_inode_mark *i_mark; + int replace = !(arg & IN_MASK_ADD); + int create = (arg & IN_MASK_CREATE); + int ret; + + fsn_mark = fsnotify_find_inode_mark(inode, group); + if (!fsn_mark) + return -ENOENT; + else if (create) { + ret = -EEXIST; + goto out; + } + + i_mark = container_of(fsn_mark, struct inotify_inode_mark, fsn_mark); + + spin_lock(&fsn_mark->lock); + if (replace) { + fsn_mark->mask = 0; + fsn_mark->flags &= ~INOTIFY_MARK_FLAGS; + } + fsn_mark->mask |= inotify_arg_to_mask(inode, arg); + fsn_mark->flags |= inotify_arg_to_flags(arg); + spin_unlock(&fsn_mark->lock); + + fsnotify_recalc_mask(fsn_mark->connector); + + /* return the wd */ + ret = i_mark->wd; + +out: + /* match the get from fsnotify_find_mark() */ + fsnotify_put_mark(fsn_mark); + + return ret; +} + +static int inotify_new_watch(struct fsnotify_group *group, + struct inode *inode, + u32 arg) +{ + struct inotify_inode_mark *tmp_i_mark; + int ret; + struct idr *idr = &group->inotify_data.idr; + spinlock_t *idr_lock = &group->inotify_data.idr_lock; + + tmp_i_mark = kmem_cache_alloc(inotify_inode_mark_cachep, GFP_KERNEL); + if (unlikely(!tmp_i_mark)) + return -ENOMEM; + + fsnotify_init_mark(&tmp_i_mark->fsn_mark, group); + tmp_i_mark->fsn_mark.mask = inotify_arg_to_mask(inode, arg); + tmp_i_mark->fsn_mark.flags = inotify_arg_to_flags(arg); + tmp_i_mark->wd = -1; + + ret = inotify_add_to_idr(idr, idr_lock, tmp_i_mark); + if (ret) + goto out_err; + + /* increment the number of watches the user has */ + if (!inc_inotify_watches(group->inotify_data.ucounts)) { + inotify_remove_from_idr(group, tmp_i_mark); + ret = -ENOSPC; + goto out_err; + } + + /* we are on the idr, now get on the inode */ + ret = fsnotify_add_inode_mark_locked(&tmp_i_mark->fsn_mark, inode, 0); + if (ret) { + /* we failed to get on the inode, get off the idr */ + inotify_remove_from_idr(group, tmp_i_mark); + dec_inotify_watches(group->inotify_data.ucounts); + goto out_err; + } + + + /* return the watch descriptor for this new mark */ + ret = tmp_i_mark->wd; + +out_err: + /* match the ref from fsnotify_init_mark() */ + fsnotify_put_mark(&tmp_i_mark->fsn_mark); + + return ret; +} + +static int inotify_update_watch(struct fsnotify_group *group, struct inode *inode, u32 arg) +{ + int ret = 0; + + fsnotify_group_lock(group); + /* try to update and existing watch with the new arg */ + ret = inotify_update_existing_watch(group, inode, arg); + /* no mark present, try to add a new one */ + if (ret == -ENOENT) + ret = inotify_new_watch(group, inode, arg); + fsnotify_group_unlock(group); + + return ret; +} + +static struct fsnotify_group *inotify_new_group(unsigned int max_events) +{ + struct fsnotify_group *group; + struct inotify_event_info *oevent; + + group = fsnotify_alloc_group(&inotify_fsnotify_ops, + FSNOTIFY_GROUP_USER); + if (IS_ERR(group)) + return group; + + oevent = kmalloc_obj(struct inotify_event_info, GFP_KERNEL_ACCOUNT); + if (unlikely(!oevent)) { + fsnotify_destroy_group(group); + return ERR_PTR(-ENOMEM); + } + group->overflow_event = &oevent->fse; + fsnotify_init_event(group->overflow_event); + oevent->mask = FS_Q_OVERFLOW; + oevent->wd = -1; + oevent->sync_cookie = 0; + oevent->name_len = 0; + + group->max_events = max_events; + group->memcg = get_mem_cgroup_from_mm(current->mm); + + spin_lock_init(&group->inotify_data.idr_lock); + idr_init(&group->inotify_data.idr); + group->inotify_data.ucounts = inc_ucount(current_user_ns(), + current_euid(), + UCOUNT_INOTIFY_INSTANCES); + + if (!group->inotify_data.ucounts) { + fsnotify_destroy_group(group); + return ERR_PTR(-EMFILE); + } + + return group; +} + + +/* inotify syscalls */ +static int do_inotify_init(int flags) +{ + struct fsnotify_group *group; + int ret; + + /* Check the IN_* constants for consistency. */ + BUILD_BUG_ON(IN_CLOEXEC != O_CLOEXEC); + BUILD_BUG_ON(IN_NONBLOCK != O_NONBLOCK); + + if (flags & ~(IN_CLOEXEC | IN_NONBLOCK)) + return -EINVAL; + + /* fsnotify_obtain_group took a reference to group, we put this when we kill the file in the end */ + group = inotify_new_group(inotify_max_queued_events); + if (IS_ERR(group)) + return PTR_ERR(group); + + ret = anon_inode_getfd("inotify", &inotify_fops, group, + O_RDONLY | flags); + if (ret < 0) + fsnotify_destroy_group(group); + + return ret; +} + +SYSCALL_DEFINE1(inotify_init1, int, flags) +{ + return do_inotify_init(flags); +} + +SYSCALL_DEFINE0(inotify_init) +{ + return do_inotify_init(0); +} + +SYSCALL_DEFINE3(inotify_add_watch, int, fd, const char __user *, pathname, + u32, mask) +{ + struct fsnotify_group *group; + struct inode *inode; + struct path path; + int ret; + unsigned flags = 0; + + /* + * We share a lot of code with fs/dnotify. We also share + * the bit layout between inotify's IN_* and the fsnotify + * FS_*. This check ensures that only the inotify IN_* + * bits get passed in and set in watches/events. + */ + if (unlikely(mask & ~ALL_INOTIFY_BITS)) + return -EINVAL; + /* + * Require at least one valid bit set in the mask. + * Without _something_ set, we would have no events to + * watch for. + */ + if (unlikely(!(mask & ALL_INOTIFY_BITS))) + return -EINVAL; + + CLASS(fd, f)(fd); + if (fd_empty(f)) + return -EBADF; + + /* IN_MASK_ADD and IN_MASK_CREATE don't make sense together */ + if (unlikely((mask & IN_MASK_ADD) && (mask & IN_MASK_CREATE))) + return -EINVAL; + + /* verify that this is indeed an inotify instance */ + if (unlikely(fd_file(f)->f_op != &inotify_fops)) + return -EINVAL; + + if (!(mask & IN_DONT_FOLLOW)) + flags |= LOOKUP_FOLLOW; + if (mask & IN_ONLYDIR) + flags |= LOOKUP_DIRECTORY; + + ret = inotify_find_inode(pathname, &path, flags, + (mask & IN_ALL_EVENTS)); + if (ret) + return ret; + + /* inode held in place by reference to path; group by fget on fd */ + inode = path.dentry->d_inode; + group = fd_file(f)->private_data; + + /* create/update an inode mark */ + ret = inotify_update_watch(group, inode, mask); + path_put(&path); + return ret; +} + +SYSCALL_DEFINE2(inotify_rm_watch, int, fd, __s32, wd) +{ + struct fsnotify_group *group; + struct inotify_inode_mark *i_mark; + CLASS(fd, f)(fd); + + if (fd_empty(f)) + return -EBADF; + + /* verify that this is indeed an inotify instance */ + if (unlikely(fd_file(f)->f_op != &inotify_fops)) + return -EINVAL; + + group = fd_file(f)->private_data; + + i_mark = inotify_idr_find(group, wd); + if (unlikely(!i_mark)) + return -EINVAL; + + fsnotify_destroy_mark(&i_mark->fsn_mark, group); + + /* match ref taken by inotify_idr_find */ + fsnotify_put_mark(&i_mark->fsn_mark); + return 0; +} + +/* + * inotify_user_setup - Our initialization function. Note that we cannot return + * error because we have compiled-in VFS hooks. So an (unlikely) failure here + * must result in panic(). + */ +static int __init inotify_user_setup(void) +{ + unsigned long watches_max; + struct sysinfo si; + + si_meminfo(&si); + /* + * Allow up to 1% of addressable memory to be allocated for inotify + * watches (per user) limited to the range [8192, 1048576]. + */ + watches_max = (((si.totalram - si.totalhigh) / 100) << PAGE_SHIFT) / + INOTIFY_WATCH_COST; + watches_max = clamp(watches_max, 8192UL, 1048576UL); + + BUILD_BUG_ON(IN_ACCESS != FS_ACCESS); + BUILD_BUG_ON(IN_MODIFY != FS_MODIFY); + BUILD_BUG_ON(IN_ATTRIB != FS_ATTRIB); + BUILD_BUG_ON(IN_CLOSE_WRITE != FS_CLOSE_WRITE); + BUILD_BUG_ON(IN_CLOSE_NOWRITE != FS_CLOSE_NOWRITE); + BUILD_BUG_ON(IN_OPEN != FS_OPEN); + BUILD_BUG_ON(IN_MOVED_FROM != FS_MOVED_FROM); + BUILD_BUG_ON(IN_MOVED_TO != FS_MOVED_TO); + BUILD_BUG_ON(IN_CREATE != FS_CREATE); + BUILD_BUG_ON(IN_DELETE != FS_DELETE); + BUILD_BUG_ON(IN_DELETE_SELF != FS_DELETE_SELF); + BUILD_BUG_ON(IN_MOVE_SELF != FS_MOVE_SELF); + BUILD_BUG_ON(IN_UNMOUNT != FS_UNMOUNT); + BUILD_BUG_ON(IN_Q_OVERFLOW != FS_Q_OVERFLOW); + BUILD_BUG_ON(IN_IGNORED != FS_IN_IGNORED); + BUILD_BUG_ON(IN_ISDIR != FS_ISDIR); + + BUILD_BUG_ON(HWEIGHT32(ALL_INOTIFY_BITS) != 22); + + inotify_inode_mark_cachep = KMEM_CACHE(inotify_inode_mark, + SLAB_PANIC|SLAB_ACCOUNT); + + inotify_max_queued_events = 16384; + init_user_ns.ucount_max[UCOUNT_INOTIFY_INSTANCES] = 128; + init_user_ns.ucount_max[UCOUNT_INOTIFY_WATCHES] = watches_max; + inotify_sysctls_init(); + + return 0; +} +fs_initcall(inotify_user_setup); -- cgit v1.3.1