From a5f7a5bb3b7f28ba7e4fa246775b29a0e5537255 Mon Sep 17 00:00:00 2001 From: Jinke Han Date: Tue, 8 Sep 2026 15:37:42 +0800 Subject: x86/kprobes: Fix crash when probing CS CALL instructions When using eBPF to probe CS CALL instructions within a function, a crash can be triggered. The eBPF tool probes offset 257 of the __hrtimer_run_queues() function: <__hrtimer_run_queues+249>: nopl 0x0(%rax,%rax,1) <__hrtimer_run_queues+254>: mov %r14,%rdi <__hrtimer_run_queues+257>: cs call <__x86_indirect_thunk_r12> <__hrtimer_run_queues+263>: mov %eax,%r12d <__hrtimer_run_queues+266>: xchg %ax,%ax <__hrtimer_run_queues+268>: mov %r13,%rdi Which triggers this crash: BUG: unable to handle page fault for address: 00000000000f41c9 #PF: supervisor write access in kernel mode #PF: error_code(0x0002) - not-present page PGD 0 P4D 0 Oops: 0002 [#1] SMP NOPTI CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P RIP: 0010:__hrtimer_run_queues+0x106/0x230 Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is at the 6th byte of the above CS CALL instruction. Since the CS CALL instruction occupies 6 bytes, the exception occurred in the middle of that call instruction. The root cause is that when using eBPF tools to probe in the middle of a function, a kprobe with INT3 is used as the underlying implementation. During single-step emulation of the original CALL instruction, int3_emulate_call() assumes that the probed CALL instruction is 5 bytes long. However, the actual CS-prefixed CALL instruction occupies 6 bytes, so it constructs an incorrect exception return address. When the CPU returns from the kprobe handler, the next instruction to be executed is at the address of the last byte of that CS CALL instruction. Coincidentally, starting from that address, the CPU fetches and decodes a completely different instruction, which ultimately triggers a kernel crash. Fix the issue by using the actual instruction length obtained from the instruction decoder when constructing the exception return address, rather than relying on the hardcoded CALL_INSN_SIZE macro. [ mingo: Refined the changelog ] Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step") Suggested-by: Masami Hiramatsu (Google) Signed-off-by: Jinke Han Signed-off-by: Ingo Molnar Reviewed-by: Masami Hiramatsu (Google) Acked-by: Yafang Shao Acked-by: Borislav Petkov Cc: Peter Zijlstra Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000 --- net/unix/Kconfig | 32 + net/unix/Makefile | 13 + net/unix/af_unix.c | 3982 ++++++++++++++++++++++++++++++++++++++++++++ net/unix/af_unix.h | 71 + net/unix/diag.c | 330 ++++ net/unix/garbage.c | 660 ++++++++ net/unix/sysctl_net_unix.c | 73 + net/unix/unix_bpf.c | 205 +++ 8 files changed, 5366 insertions(+) create mode 100644 net/unix/Kconfig create mode 100644 net/unix/Makefile create mode 100644 net/unix/af_unix.c create mode 100644 net/unix/af_unix.h create mode 100644 net/unix/diag.c create mode 100644 net/unix/garbage.c create mode 100644 net/unix/sysctl_net_unix.c create mode 100644 net/unix/unix_bpf.c (limited to 'net/unix') diff --git a/net/unix/Kconfig b/net/unix/Kconfig new file mode 100644 index 000000000..6f1783c16 --- /dev/null +++ b/net/unix/Kconfig @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Unix Domain Sockets +# + +config UNIX + bool "Unix domain sockets" + help + If you say Y here, you will include support for Unix domain sockets; + sockets are the standard Unix mechanism for establishing and + accessing network connections. Many commonly used programs such as + the X Window system and syslog use these sockets even if your + machine is not connected to any network. Unless you are working on + an embedded system or something similar, you therefore definitely + want to say Y here. + + Say Y unless you know what you are doing. + +config AF_UNIX_OOB + bool "UNIX: out-of-bound messages" + depends on UNIX + default y + help + Support for MSG_OOB in UNIX domain sockets. If unsure, say Y. + +config UNIX_DIAG + tristate "UNIX: socket monitoring interface" + depends on UNIX + default n + help + Support for UNIX socket monitoring interface used by the ss tool. + If unsure, say Y. diff --git a/net/unix/Makefile b/net/unix/Makefile new file mode 100644 index 000000000..4ddd125c4 --- /dev/null +++ b/net/unix/Makefile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the Linux unix domain socket layer. +# + +obj-$(CONFIG_UNIX) += unix.o + +unix-y := af_unix.o garbage.o +unix-$(CONFIG_SYSCTL) += sysctl_net_unix.o +unix-$(CONFIG_BPF_SYSCALL) += unix_bpf.o + +obj-$(CONFIG_UNIX_DIAG) += unix_diag.o +unix_diag-y := diag.o diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c new file mode 100644 index 000000000..42cffeafc --- /dev/null +++ b/net/unix/af_unix.c @@ -0,0 +1,3982 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NET4: Implementation of BSD Unix domain sockets. + * + * Authors: Alan Cox, + * + * Fixes: + * Linus Torvalds : Assorted bug cures. + * Niibe Yutaka : async I/O support. + * Carsten Paeth : PF_UNIX check, address fixes. + * Alan Cox : Limit size of allocated blocks. + * Alan Cox : Fixed the stupid socketpair bug. + * Alan Cox : BSD compatibility fine tuning. + * Alan Cox : Fixed a bug in connect when interrupted. + * Alan Cox : Sorted out a proper draft version of + * file descriptor passing hacked up from + * Mike Shaver's work. + * Marty Leisner : Fixes to fd passing + * Nick Nevin : recvmsg bugfix. + * Alan Cox : Started proper garbage collector + * Heiko EiBfeldt : Missing verify_area check + * Alan Cox : Started POSIXisms + * Andreas Schwab : Replace inode by dentry for proper + * reference counting + * Kirk Petersen : Made this a module + * Christoph Rohland : Elegant non-blocking accept/connect algorithm. + * Lots of bug fixes. + * Alexey Kuznetosv : Repaired (I hope) bugs introduces + * by above two patches. + * Andrea Arcangeli : If possible we block in connect(2) + * if the max backlog of the listen socket + * is been reached. This won't break + * old apps and it will avoid huge amount + * of socks hashed (this for unix_gc() + * performances reasons). + * Security fix that limits the max + * number of socks to 2*max_files and + * the number of skb queueable in the + * dgram receiver. + * Artur Skawina : Hash function optimizations + * Alexey Kuznetsov : Full scale SMP. Lot of bugs are introduced 8) + * Malcolm Beattie : Set peercred for socketpair + * Michal Ostrowski : Module initialization cleanup. + * Arnaldo C. Melo : Remove MOD_{INC,DEC}_USE_COUNT, + * the core infrastructure is doing that + * for all net proto families now (2.5.69+) + * + * Known differences from reference BSD that was tested: + * + * [TO FIX] + * ECONNREFUSED is not returned from one end of a connected() socket to the + * other the moment one end closes. + * fstat() doesn't return st_dev=0, and give the blksize as high water mark + * and a fake inode identifier (nor the BSD first socket fstat twice bug). + * [NOT TO FIX] + * accept() returns a path name even if the connecting socket has closed + * in the meantime (BSD loses the path and gives up). + * accept() returns 0 length path for an unbound connector. BSD returns 16 + * and a null first byte in the path (but not for gethost/peername - BSD bug ??) + * socketpair(...SOCK_RAW..) doesn't panic the kernel. + * BSD af_unix apparently has connect forgetting to block properly. + * (need to check this with the POSIX spec in detail) + * + * Differences from 2.0.0-11-... (ANK) + * Bug fixes and improvements. + * - client shutdown killed server socket. + * - removed all useless cli/sti pairs. + * + * Semantic changes/extensions. + * - generic control message passing. + * - SCM_CREDENTIALS control message. + * - "Abstract" (not FS based) socket bindings. + * Abstract names are sequences of bytes (not zero terminated) + * started by 0, so that this name space does not intersect + * with BSD names. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "af_unix.h" + +static atomic_long_t unix_nr_socks; +static struct hlist_head bsd_socket_buckets[UNIX_HASH_SIZE / 2]; +static spinlock_t bsd_socket_locks[UNIX_HASH_SIZE / 2]; + +/* SMP locking strategy: + * hash table is protected with spinlock. + * each socket state is protected by separate spinlock. + */ +#ifdef CONFIG_PROVE_LOCKING +#define cmp_ptr(l, r) (((l) > (r)) - ((l) < (r))) + +static int unix_table_lock_cmp_fn(const struct lockdep_map *a, + const struct lockdep_map *b) +{ + return cmp_ptr(a, b); +} + +static int unix_state_lock_cmp_fn(const struct lockdep_map *_a, + const struct lockdep_map *_b) +{ + const struct unix_sock *a, *b; + + a = container_of(_a, struct unix_sock, lock.dep_map); + b = container_of(_b, struct unix_sock, lock.dep_map); + + if (a->sk.sk_state == TCP_LISTEN) { + /* unix_stream_connect(): Before the 2nd unix_state_lock(), + * + * 1. a is TCP_LISTEN. + * 2. b is not a. + * 3. concurrent connect(b -> a) must fail. + * + * Except for 2. & 3., the b's state can be any possible + * value due to concurrent connect() or listen(). + * + * 2. is detected in debug_spin_lock_before(), and 3. cannot + * be expressed as lock_cmp_fn. + */ + switch (b->sk.sk_state) { + case TCP_CLOSE: + case TCP_ESTABLISHED: + case TCP_LISTEN: + return -1; + default: + /* Invalid case. */ + return 0; + } + } + + /* Should never happen. Just to be symmetric. */ + if (b->sk.sk_state == TCP_LISTEN) { + switch (b->sk.sk_state) { + case TCP_CLOSE: + case TCP_ESTABLISHED: + return 1; + default: + return 0; + } + } + + /* unix_state_double_lock(): ascending address order. */ + return cmp_ptr(a, b); +} + +static int unix_recvq_lock_cmp_fn(const struct lockdep_map *_a, + const struct lockdep_map *_b) +{ + const struct sock *a, *b; + + a = container_of(_a, struct sock, sk_receive_queue.lock.dep_map); + b = container_of(_b, struct sock, sk_receive_queue.lock.dep_map); + + /* unix_collect_skb(): listener -> embryo order. */ + if (a->sk_state == TCP_LISTEN && unix_sk(b)->listener == a) + return -1; + + /* Should never happen. Just to be symmetric. */ + if (b->sk_state == TCP_LISTEN && unix_sk(a)->listener == b) + return 1; + + return 0; +} +#endif + +static unsigned int unix_unbound_hash(struct sock *sk) +{ + unsigned long hash = (unsigned long)sk; + + hash ^= hash >> 16; + hash ^= hash >> 8; + hash ^= sk->sk_type; + + return hash & UNIX_HASH_MOD; +} + +static unsigned int unix_bsd_hash(struct inode *i) +{ + return i->i_ino & UNIX_HASH_MOD; +} + +static unsigned int unix_abstract_hash(struct sockaddr_un *sunaddr, + int addr_len, int type) +{ + __wsum csum = csum_partial(sunaddr, addr_len, 0); + unsigned int hash; + + hash = (__force unsigned int)csum_fold(csum); + hash ^= hash >> 8; + hash ^= type; + + return UNIX_HASH_MOD + 1 + (hash & UNIX_HASH_MOD); +} + +static void unix_table_double_lock(struct net *net, + unsigned int hash1, unsigned int hash2) +{ + if (hash1 == hash2) { + spin_lock(&net->unx.table.locks[hash1]); + return; + } + + if (hash1 > hash2) + swap(hash1, hash2); + + spin_lock(&net->unx.table.locks[hash1]); + spin_lock(&net->unx.table.locks[hash2]); +} + +static void unix_table_double_unlock(struct net *net, + unsigned int hash1, unsigned int hash2) +{ + if (hash1 == hash2) { + spin_unlock(&net->unx.table.locks[hash1]); + return; + } + + spin_unlock(&net->unx.table.locks[hash1]); + spin_unlock(&net->unx.table.locks[hash2]); +} + +#ifdef CONFIG_SECURITY_NETWORK +static void unix_get_secdata(struct scm_cookie *scm, struct sk_buff *skb) +{ + UNIXCB(skb).secid = scm->secid; +} + +static inline void unix_set_secdata(struct scm_cookie *scm, struct sk_buff *skb) +{ + scm->secid = UNIXCB(skb).secid; +} + +static inline bool unix_secdata_eq(struct scm_cookie *scm, struct sk_buff *skb) +{ + return (scm->secid == UNIXCB(skb).secid); +} +#else +static inline void unix_get_secdata(struct scm_cookie *scm, struct sk_buff *skb) +{ } + +static inline void unix_set_secdata(struct scm_cookie *scm, struct sk_buff *skb) +{ } + +static inline bool unix_secdata_eq(struct scm_cookie *scm, struct sk_buff *skb) +{ + return true; +} +#endif /* CONFIG_SECURITY_NETWORK */ + +static inline int unix_may_send(struct sock *sk, struct sock *osk) +{ + return !unix_peer(osk) || unix_peer(osk) == sk; +} + +static inline int unix_recvq_full_lockless(const struct sock *sk) +{ + return skb_queue_len_lockless(&sk->sk_receive_queue) > sk->sk_max_ack_backlog; +} + +struct sock *unix_peer_get(struct sock *s) +{ + struct sock *peer; + + unix_state_lock(s); + peer = unix_peer(s); + if (peer) + sock_hold(peer); + unix_state_unlock(s); + return peer; +} +EXPORT_SYMBOL_GPL(unix_peer_get); + +static struct unix_address *unix_create_addr(struct sockaddr_un *sunaddr, + int addr_len) +{ + struct unix_address *addr; + + addr = kmalloc(sizeof(*addr) + addr_len, GFP_KERNEL); + if (!addr) + return NULL; + + refcount_set(&addr->refcnt, 1); + addr->len = addr_len; + memcpy(addr->name, sunaddr, addr_len); + + return addr; +} + +static inline void unix_release_addr(struct unix_address *addr) +{ + if (refcount_dec_and_test(&addr->refcnt)) + kfree(addr); +} + +/* + * Check unix socket name: + * - should be not zero length. + * - if started by not zero, should be NULL terminated (FS object) + * - if started by zero, it is abstract name. + */ + +static int unix_validate_addr(struct sockaddr_un *sunaddr, int addr_len) +{ + if (addr_len <= offsetof(struct sockaddr_un, sun_path) || + addr_len > sizeof(*sunaddr)) + return -EINVAL; + + if (sunaddr->sun_family != AF_UNIX) + return -EINVAL; + + return 0; +} + +static int unix_mkname_bsd(struct sockaddr_un *sunaddr, int addr_len) +{ + struct sockaddr_storage *addr = (struct sockaddr_storage *)sunaddr; + short offset = offsetof(struct sockaddr_storage, __data); + + BUILD_BUG_ON(offset != offsetof(struct sockaddr_un, sun_path)); + + /* This may look like an off by one error but it is a bit more + * subtle. 108 is the longest valid AF_UNIX path for a binding. + * sun_path[108] doesn't as such exist. However in kernel space + * we are guaranteed that it is a valid memory location in our + * kernel address buffer because syscall functions always pass + * a pointer of struct sockaddr_storage which has a bigger buffer + * than 108. Also, we must terminate sun_path for strlen() in + * getname_kernel(). + */ + addr->__data[addr_len - offset] = 0; + + /* Don't pass sunaddr->sun_path to strlen(). Otherwise, 108 will + * cause panic if CONFIG_FORTIFY_SOURCE=y. Let __fortify_strlen() + * know the actual buffer. + */ + return strlen(addr->__data) + offset + 1; +} + +static void __unix_remove_socket(struct sock *sk) +{ + sk_del_node_init(sk); +} + +static void __unix_insert_socket(struct net *net, struct sock *sk) +{ + DEBUG_NET_WARN_ON_ONCE(!sk_unhashed(sk)); + sk_add_node(sk, &net->unx.table.buckets[sk->sk_hash]); +} + +static void __unix_set_addr_hash(struct net *net, struct sock *sk, + struct unix_address *addr, unsigned int hash) +{ + __unix_remove_socket(sk); + smp_store_release(&unix_sk(sk)->addr, addr); + + sk->sk_hash = hash; + __unix_insert_socket(net, sk); +} + +static void unix_remove_socket(struct net *net, struct sock *sk) +{ + spin_lock(&net->unx.table.locks[sk->sk_hash]); + __unix_remove_socket(sk); + spin_unlock(&net->unx.table.locks[sk->sk_hash]); +} + +static void unix_insert_unbound_socket(struct net *net, struct sock *sk) +{ + spin_lock(&net->unx.table.locks[sk->sk_hash]); + __unix_insert_socket(net, sk); + spin_unlock(&net->unx.table.locks[sk->sk_hash]); +} + +static void unix_insert_bsd_socket(struct sock *sk) +{ + spin_lock(&bsd_socket_locks[sk->sk_hash]); + sk_add_bind_node(sk, &bsd_socket_buckets[sk->sk_hash]); + spin_unlock(&bsd_socket_locks[sk->sk_hash]); +} + +static void unix_remove_bsd_socket(struct sock *sk) +{ + if (!hlist_unhashed(&sk->sk_bind_node)) { + spin_lock(&bsd_socket_locks[sk->sk_hash]); + __sk_del_bind_node(sk); + spin_unlock(&bsd_socket_locks[sk->sk_hash]); + + sk_node_init(&sk->sk_bind_node); + } +} + +static struct sock *__unix_find_socket_byname(struct net *net, + struct sockaddr_un *sunname, + int len, unsigned int hash) +{ + struct sock *s; + + sk_for_each(s, &net->unx.table.buckets[hash]) { + struct unix_sock *u = unix_sk(s); + + if (u->addr->len == len && + !memcmp(u->addr->name, sunname, len)) + return s; + } + return NULL; +} + +static inline struct sock *unix_find_socket_byname(struct net *net, + struct sockaddr_un *sunname, + int len, unsigned int hash) +{ + struct sock *s; + + spin_lock(&net->unx.table.locks[hash]); + s = __unix_find_socket_byname(net, sunname, len, hash); + if (s) + sock_hold(s); + spin_unlock(&net->unx.table.locks[hash]); + return s; +} + +static struct sock *unix_find_socket_byinode(struct inode *i) +{ + unsigned int hash = unix_bsd_hash(i); + struct sock *s; + + spin_lock(&bsd_socket_locks[hash]); + sk_for_each_bound(s, &bsd_socket_buckets[hash]) { + struct dentry *dentry = unix_sk(s)->path.dentry; + + if (dentry && d_backing_inode(dentry) == i) { + sock_hold(s); + spin_unlock(&bsd_socket_locks[hash]); + return s; + } + } + spin_unlock(&bsd_socket_locks[hash]); + return NULL; +} + +/* Support code for asymmetrically connected dgram sockets + * + * If a datagram socket is connected to a socket not itself connected + * to the first socket (eg, /dev/log), clients may only enqueue more + * messages if the present receive queue of the server socket is not + * "too large". This means there's a second writeability condition + * poll and sendmsg need to test. The dgram recv code will do a wake + * up on the peer_wait wait queue of a socket upon reception of a + * datagram which needs to be propagated to sleeping would-be writers + * since these might not have sent anything so far. This can't be + * accomplished via poll_wait because the lifetime of the server + * socket might be less than that of its clients if these break their + * association with it or if the server socket is closed while clients + * are still connected to it and there's no way to inform "a polling + * implementation" that it should let go of a certain wait queue + * + * In order to propagate a wake up, a wait_queue_entry_t of the client + * socket is enqueued on the peer_wait queue of the server socket + * whose wake function does a wake_up on the ordinary client socket + * wait queue. This connection is established whenever a write (or + * poll for write) hit the flow control condition and broken when the + * association to the server socket is dissolved or after a wake up + * was relayed. + */ + +static int unix_dgram_peer_wake_relay(wait_queue_entry_t *q, unsigned mode, int flags, + void *key) +{ + struct unix_sock *u; + wait_queue_head_t *u_sleep; + + u = container_of(q, struct unix_sock, peer_wake); + + __remove_wait_queue(&unix_sk(u->peer_wake.private)->peer_wait, + q); + u->peer_wake.private = NULL; + + /* relaying can only happen while the wq still exists */ + u_sleep = sk_sleep(&u->sk); + if (u_sleep) + wake_up_interruptible_poll(u_sleep, key_to_poll(key)); + + return 0; +} + +static int unix_dgram_peer_wake_connect(struct sock *sk, struct sock *other) +{ + struct unix_sock *u, *u_other; + int rc; + + u = unix_sk(sk); + u_other = unix_sk(other); + rc = 0; + spin_lock(&u_other->peer_wait.lock); + + if (!u->peer_wake.private) { + u->peer_wake.private = other; + __add_wait_queue(&u_other->peer_wait, &u->peer_wake); + + rc = 1; + } + + spin_unlock(&u_other->peer_wait.lock); + return rc; +} + +static void unix_dgram_peer_wake_disconnect(struct sock *sk, + struct sock *other) +{ + struct unix_sock *u, *u_other; + + u = unix_sk(sk); + u_other = unix_sk(other); + spin_lock(&u_other->peer_wait.lock); + + if (u->peer_wake.private == other) { + __remove_wait_queue(&u_other->peer_wait, &u->peer_wake); + u->peer_wake.private = NULL; + } + + spin_unlock(&u_other->peer_wait.lock); +} + +static void unix_dgram_peer_wake_disconnect_wakeup(struct sock *sk, + struct sock *other) +{ + unix_dgram_peer_wake_disconnect(sk, other); + wake_up_interruptible_poll(sk_sleep(sk), + EPOLLOUT | + EPOLLWRNORM | + EPOLLWRBAND); +} + +/* preconditions: + * - unix_peer(sk) == other + * - association is stable + */ +static int unix_dgram_peer_wake_me(struct sock *sk, struct sock *other) +{ + int connected; + + connected = unix_dgram_peer_wake_connect(sk, other); + + /* If other is SOCK_DEAD, we want to make sure we signal + * POLLOUT, such that a subsequent write() can get a + * -ECONNREFUSED. Otherwise, if we haven't queued any skbs + * to other and its full, we will hang waiting for POLLOUT. + */ + if (unix_recvq_full_lockless(other) && !sock_flag(other, SOCK_DEAD)) + return 1; + + if (connected) + unix_dgram_peer_wake_disconnect(sk, other); + + return 0; +} + +static int unix_writable(const struct sock *sk, unsigned char state) +{ + return state != TCP_LISTEN && + (refcount_read(&sk->sk_wmem_alloc) << 2) <= READ_ONCE(sk->sk_sndbuf); +} + +static void unix_write_space(struct sock *sk) +{ + struct socket_wq *wq; + + rcu_read_lock(); + if (unix_writable(sk, READ_ONCE(sk->sk_state))) { + wq = rcu_dereference(sk->sk_wq); + if (skwq_has_sleeper(wq)) + wake_up_interruptible_sync_poll(&wq->wait, + EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND); + sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT); + } + rcu_read_unlock(); +} + +/* When dgram socket disconnects (or changes its peer), we clear its receive + * queue of packets arrived from previous peer. First, it allows to do + * flow control based only on wmem_alloc; second, sk connected to peer + * may receive messages only from that peer. */ +static void unix_dgram_disconnected(struct sock *sk, struct sock *other) +{ + if (!skb_queue_empty(&sk->sk_receive_queue)) { + skb_queue_purge_reason(&sk->sk_receive_queue, + SKB_DROP_REASON_UNIX_DISCONNECT); + + wake_up_interruptible_all(&unix_sk(sk)->peer_wait); + + /* If one link of bidirectional dgram pipe is disconnected, + * we signal error. Messages are lost. Do not make this, + * when peer was not connected to us. + */ + if (!sock_flag(other, SOCK_DEAD) && unix_peer(other) == sk) { + WRITE_ONCE(other->sk_err, ECONNRESET); + sk_error_report(other); + } + } +} + +static void unix_sock_destructor(struct sock *sk) +{ + struct unix_sock *u = unix_sk(sk); + + skb_queue_purge_reason(&sk->sk_receive_queue, SKB_DROP_REASON_SOCKET_CLOSE); + + DEBUG_NET_WARN_ON_ONCE(refcount_read(&sk->sk_wmem_alloc)); + DEBUG_NET_WARN_ON_ONCE(!sk_unhashed(sk)); + DEBUG_NET_WARN_ON_ONCE(sk->sk_socket); + if (!sock_flag(sk, SOCK_DEAD)) { + pr_info("Attempt to release alive unix socket: %p\n", sk); + return; + } + + if (u->addr) + unix_release_addr(u->addr); + + atomic_long_dec(&unix_nr_socks); + sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); +#ifdef UNIX_REFCNT_DEBUG + pr_debug("UNIX %p is destroyed, %ld are still alive.\n", sk, + atomic_long_read(&unix_nr_socks)); +#endif +} + +static unsigned int unix_skb_len(const struct sk_buff *skb) +{ + return skb->len - UNIXCB(skb).consumed; +} + +static void unix_release_sock(struct sock *sk, int embrion) +{ + struct unix_sock *u = unix_sk(sk); + struct sock *skpair; + struct sk_buff *skb; + struct path path; + int state; + + unix_remove_socket(sock_net(sk), sk); + unix_remove_bsd_socket(sk); + + /* Clear state */ + unix_state_lock(sk); + sock_orphan(sk); + WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK); + path = u->path; + u->path.dentry = NULL; + u->path.mnt = NULL; + state = sk->sk_state; + WRITE_ONCE(sk->sk_state, TCP_CLOSE); + + skpair = unix_peer(sk); + unix_peer(sk) = NULL; + + unix_state_unlock(sk); + +#if IS_ENABLED(CONFIG_AF_UNIX_OOB) + u->oob_skb = NULL; +#endif + + wake_up_interruptible_all(&u->peer_wait); + + if (skpair != NULL) { + if (sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET) { + struct sk_buff *skb = skb_peek(&sk->sk_receive_queue); + +#if IS_ENABLED(CONFIG_AF_UNIX_OOB) + if (skb && !unix_skb_len(skb)) + skb = skb_peek_next(skb, &sk->sk_receive_queue); +#endif + unix_state_lock(skpair); + /* No more writes */ + WRITE_ONCE(skpair->sk_shutdown, SHUTDOWN_MASK); + if (skb || embrion) + WRITE_ONCE(skpair->sk_err, ECONNRESET); + unix_state_unlock(skpair); + skpair->sk_state_change(skpair); + sk_wake_async(skpair, SOCK_WAKE_WAITD, POLL_HUP); + } + + unix_dgram_peer_wake_disconnect(sk, skpair); + sock_put(skpair); /* It may now die */ + } + + /* Try to flush out this socket. Throw out buffers at least */ + + while ((skb = skb_dequeue(&sk->sk_receive_queue)) != NULL) { + if (state == TCP_LISTEN) + unix_release_sock(skb->sk, 1); + + /* passed fds are erased in the kfree_skb hook */ + kfree_skb_reason(skb, SKB_DROP_REASON_SOCKET_CLOSE); + } + + if (path.dentry) + path_put(&path); + + sock_put(sk); + + /* ---- Socket is dead now and most probably destroyed ---- */ + + unix_schedule_gc(NULL); +} + +struct unix_peercred { + struct pid *peer_pid; + const struct cred *peer_cred; +}; + +static inline int prepare_peercred(struct unix_peercred *peercred) +{ + struct pid *pid; + int err; + + pid = task_tgid(current); + err = pidfs_register_pid(pid); + if (likely(!err)) { + peercred->peer_pid = get_pid(pid); + peercred->peer_cred = get_current_cred(); + } + return err; +} + +static void drop_peercred(struct unix_peercred *peercred) +{ + const struct cred *cred = NULL; + struct pid *pid = NULL; + + might_sleep(); + + swap(peercred->peer_pid, pid); + swap(peercred->peer_cred, cred); + + put_pid(pid); + put_cred(cred); +} + +static inline void init_peercred(struct sock *sk, + const struct unix_peercred *peercred) +{ + sk->sk_peer_pid = peercred->peer_pid; + sk->sk_peer_cred = peercred->peer_cred; +} + +static void update_peercred(struct sock *sk, struct unix_peercred *peercred) +{ + const struct cred *old_cred; + struct pid *old_pid; + + spin_lock(&sk->sk_peer_lock); + old_pid = sk->sk_peer_pid; + old_cred = sk->sk_peer_cred; + init_peercred(sk, peercred); + spin_unlock(&sk->sk_peer_lock); + + peercred->peer_pid = old_pid; + peercred->peer_cred = old_cred; +} + +static void copy_peercred(struct sock *sk, struct sock *peersk) +{ + lockdep_assert_held(&unix_sk(peersk)->lock); + + spin_lock(&sk->sk_peer_lock); + sk->sk_peer_pid = get_pid(peersk->sk_peer_pid); + sk->sk_peer_cred = get_cred(peersk->sk_peer_cred); + spin_unlock(&sk->sk_peer_lock); +} + +static bool unix_may_passcred(const struct sock *sk) +{ + return sk->sk_scm_credentials || sk->sk_scm_pidfd; +} + +static int unix_listen(struct socket *sock, int backlog) +{ + int err; + struct sock *sk = sock->sk; + struct unix_sock *u = unix_sk(sk); + struct unix_peercred peercred = {}; + + err = -EOPNOTSUPP; + if (sock->type != SOCK_STREAM && sock->type != SOCK_SEQPACKET) + goto out; /* Only stream/seqpacket sockets accept */ + err = -EINVAL; + if (!READ_ONCE(u->addr)) + goto out; /* No listens on an unbound socket */ + err = prepare_peercred(&peercred); + if (err) + goto out; + unix_state_lock(sk); + err = -EINVAL; + if (sk->sk_state != TCP_CLOSE && sk->sk_state != TCP_LISTEN) + goto out_unlock; + if (backlog > sk->sk_max_ack_backlog) + wake_up_interruptible_all(&u->peer_wait); + sk->sk_max_ack_backlog = backlog; + WRITE_ONCE(sk->sk_state, TCP_LISTEN); + + /* set credentials so connect can copy them */ + update_peercred(sk, &peercred); + err = 0; + +out_unlock: + unix_state_unlock(sk); + drop_peercred(&peercred); +out: + return err; +} + +static int unix_release(struct socket *); +static int unix_bind(struct socket *, struct sockaddr_unsized *, int); +static int unix_stream_connect(struct socket *, struct sockaddr_unsized *, + int addr_len, int flags); +static int unix_socketpair(struct socket *, struct socket *); +static int unix_accept(struct socket *, struct socket *, struct proto_accept_arg *arg); +static int unix_getname(struct socket *, struct sockaddr *, int); +static __poll_t unix_poll(struct file *, struct socket *, poll_table *); +static __poll_t unix_dgram_poll(struct file *, struct socket *, + poll_table *); +static int unix_ioctl(struct socket *, unsigned int, unsigned long); +#ifdef CONFIG_COMPAT +static int unix_compat_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg); +#endif +static int unix_shutdown(struct socket *, int); +static int unix_stream_sendmsg(struct socket *, struct msghdr *, size_t); +static int unix_stream_recvmsg(struct socket *, struct msghdr *, size_t, int); +static ssize_t unix_stream_splice_read(struct socket *, loff_t *ppos, + struct pipe_inode_info *, size_t size, + unsigned int flags); +static int unix_dgram_sendmsg(struct socket *, struct msghdr *, size_t); +static int unix_dgram_recvmsg(struct socket *, struct msghdr *, size_t, int); +static int unix_read_skb(struct sock *sk, skb_read_actor_t recv_actor); +static int unix_stream_read_skb(struct sock *sk, skb_read_actor_t recv_actor); +static int unix_dgram_connect(struct socket *, struct sockaddr_unsized *, + int, int); +static int unix_seqpacket_sendmsg(struct socket *, struct msghdr *, size_t); +static int unix_seqpacket_recvmsg(struct socket *, struct msghdr *, size_t, + int); + +#ifdef CONFIG_PROC_FS +static int unix_count_nr_fds(struct sock *sk) +{ + struct sk_buff *skb; + struct unix_sock *u; + int nr_fds = 0; + + spin_lock(&sk->sk_receive_queue.lock); + skb = skb_peek(&sk->sk_receive_queue); + while (skb) { + u = unix_sk(skb->sk); + nr_fds += atomic_read(&u->scm_stat.nr_fds); + skb = skb_peek_next(skb, &sk->sk_receive_queue); + } + spin_unlock(&sk->sk_receive_queue.lock); + + return nr_fds; +} + +static void unix_show_fdinfo(struct seq_file *m, struct socket *sock) +{ + struct sock *sk = sock->sk; + unsigned char s_state; + struct unix_sock *u; + int nr_fds = 0; + + if (sk) { + s_state = READ_ONCE(sk->sk_state); + u = unix_sk(sk); + + /* SOCK_STREAM and SOCK_SEQPACKET sockets never change their + * sk_state after switching to TCP_ESTABLISHED or TCP_LISTEN. + * SOCK_DGRAM is ordinary. So, no lock is needed. + */ + if (sock->type == SOCK_DGRAM || s_state == TCP_ESTABLISHED) + nr_fds = atomic_read(&u->scm_stat.nr_fds); + else if (s_state == TCP_LISTEN) + nr_fds = unix_count_nr_fds(sk); + + seq_printf(m, "scm_fds: %u\n", nr_fds); + } +} +#else +#define unix_show_fdinfo NULL +#endif + +static bool unix_custom_sockopt(int optname) +{ + switch (optname) { + case SO_INQ: + case SO_RIGHTS_NOTRUNC: + return true; + default: + return false; + } +} + +static int unix_setsockopt(struct socket *sock, int level, int optname, + sockptr_t optval, unsigned int optlen) +{ + struct unix_sock *u = unix_sk(sock->sk); + struct sock *sk = sock->sk; + int val; + + if (level != SOL_SOCKET) + return -EOPNOTSUPP; + + if (!unix_custom_sockopt(optname)) + return sock_setsockopt(sock, level, optname, optval, optlen); + + if (optlen != sizeof(int)) + return -EINVAL; + + if (copy_from_sockptr(&val, optval, sizeof(val))) + return -EFAULT; + + switch (optname) { + case SO_INQ: + if (sk->sk_type != SOCK_STREAM) + return -ENOPROTOOPT; + + if (val > 1 || val < 0) + return -EINVAL; + + WRITE_ONCE(u->recvmsg_inq, val); + break; + + case SO_RIGHTS_NOTRUNC: + if (val > 1 || val < 0) + return -EINVAL; + + WRITE_ONCE(u->scm_rights_notrunc, val); + break; + + default: + return -ENOPROTOOPT; + } + + return 0; +} + +static const struct proto_ops unix_stream_ops = { + .family = PF_UNIX, + .owner = THIS_MODULE, + .release = unix_release, + .bind = unix_bind, + .connect = unix_stream_connect, + .socketpair = unix_socketpair, + .accept = unix_accept, + .getname = unix_getname, + .poll = unix_poll, + .ioctl = unix_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = unix_compat_ioctl, +#endif + .listen = unix_listen, + .shutdown = unix_shutdown, + .setsockopt = unix_setsockopt, + .sendmsg = unix_stream_sendmsg, + .recvmsg = unix_stream_recvmsg, + .read_skb = unix_stream_read_skb, + .mmap = sock_no_mmap, + .splice_read = unix_stream_splice_read, + .set_peek_off = sk_set_peek_off, + .show_fdinfo = unix_show_fdinfo, +}; + +static const struct proto_ops unix_dgram_ops = { + .family = PF_UNIX, + .owner = THIS_MODULE, + .release = unix_release, + .bind = unix_bind, + .connect = unix_dgram_connect, + .socketpair = unix_socketpair, + .accept = sock_no_accept, + .getname = unix_getname, + .poll = unix_dgram_poll, + .ioctl = unix_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = unix_compat_ioctl, +#endif + .listen = sock_no_listen, + .shutdown = unix_shutdown, + .setsockopt = unix_setsockopt, + .sendmsg = unix_dgram_sendmsg, + .read_skb = unix_read_skb, + .recvmsg = unix_dgram_recvmsg, + .mmap = sock_no_mmap, + .set_peek_off = sk_set_peek_off, + .show_fdinfo = unix_show_fdinfo, +}; + +static const struct proto_ops unix_seqpacket_ops = { + .family = PF_UNIX, + .owner = THIS_MODULE, + .release = unix_release, + .bind = unix_bind, + .connect = unix_stream_connect, + .socketpair = unix_socketpair, + .accept = unix_accept, + .getname = unix_getname, + .poll = unix_dgram_poll, + .ioctl = unix_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = unix_compat_ioctl, +#endif + .listen = unix_listen, + .shutdown = unix_shutdown, + .setsockopt = unix_setsockopt, + .sendmsg = unix_seqpacket_sendmsg, + .recvmsg = unix_seqpacket_recvmsg, + .mmap = sock_no_mmap, + .set_peek_off = sk_set_peek_off, + .show_fdinfo = unix_show_fdinfo, +}; + +static void unix_close(struct sock *sk, long timeout) +{ + /* Nothing to do here, unix socket does not need a ->close(). + * This is merely for sockmap. + */ +} + +static bool unix_bpf_bypass_getsockopt(int level, int optname) +{ + if (level == SOL_SOCKET) { + switch (optname) { + case SO_PEERPIDFD: + return true; + default: + return false; + } + } + + return false; +} + +struct proto unix_dgram_proto = { + .name = "UNIX", + .owner = THIS_MODULE, + .obj_size = sizeof(struct unix_sock), + .close = unix_close, + .bpf_bypass_getsockopt = unix_bpf_bypass_getsockopt, +#ifdef CONFIG_BPF_SYSCALL + .psock_update_sk_prot = unix_dgram_bpf_update_proto, +#endif +}; + +struct proto unix_stream_proto = { + .name = "UNIX-STREAM", + .owner = THIS_MODULE, + .obj_size = sizeof(struct unix_sock), + .close = unix_close, + .bpf_bypass_getsockopt = unix_bpf_bypass_getsockopt, +#ifdef CONFIG_BPF_SYSCALL + .psock_update_sk_prot = unix_stream_bpf_update_proto, +#endif +}; + +static struct sock *unix_create1(struct net *net, struct socket *sock, int kern, int type) +{ + struct unix_sock *u; + struct sock *sk; + int err; + + atomic_long_inc(&unix_nr_socks); + if (atomic_long_read(&unix_nr_socks) > 2 * get_max_files()) { + err = -ENFILE; + goto err; + } + + if (type == SOCK_STREAM) + sk = sk_alloc(net, PF_UNIX, GFP_KERNEL, &unix_stream_proto, kern); + else /*dgram and seqpacket */ + sk = sk_alloc(net, PF_UNIX, GFP_KERNEL, &unix_dgram_proto, kern); + + if (!sk) { + err = -ENOMEM; + goto err; + } + + sock_init_data(sock, sk); + + sk->sk_scm_rights = 1; + sk->sk_hash = unix_unbound_hash(sk); + sk->sk_allocation = GFP_KERNEL_ACCOUNT; + sk->sk_write_space = unix_write_space; + sk->sk_max_ack_backlog = READ_ONCE(net->unx.sysctl_max_dgram_qlen); + sk->sk_destruct = unix_sock_destructor; + lock_set_cmp_fn(&sk->sk_receive_queue.lock, unix_recvq_lock_cmp_fn, NULL); + + u = unix_sk(sk); + u->listener = NULL; + u->vertex = NULL; + u->path.dentry = NULL; + u->path.mnt = NULL; + spin_lock_init(&u->lock); + lock_set_cmp_fn(&u->lock, unix_state_lock_cmp_fn, NULL); + mutex_init(&u->iolock); /* single task reading lock */ + mutex_init(&u->bindlock); /* single task binding lock */ + init_waitqueue_head(&u->peer_wait); + init_waitqueue_func_entry(&u->peer_wake, unix_dgram_peer_wake_relay); + memset(&u->scm_stat, 0, sizeof(struct scm_stat)); + unix_insert_unbound_socket(net, sk); + + sock_prot_inuse_add(net, sk->sk_prot, 1); + + return sk; + +err: + atomic_long_dec(&unix_nr_socks); + return ERR_PTR(err); +} + +static int unix_create(struct net *net, struct socket *sock, int protocol, + int kern) +{ + struct sock *sk; + + if (protocol && protocol != PF_UNIX) + return -EPROTONOSUPPORT; + + set_bit(SOCK_CUSTOM_SOCKOPT, &sock->flags); + + switch (sock->type) { + case SOCK_STREAM: + sock->ops = &unix_stream_ops; + break; + /* + * Believe it or not BSD has AF_UNIX, SOCK_RAW though + * nothing uses it. + */ + case SOCK_RAW: + sock->type = SOCK_DGRAM; + fallthrough; + case SOCK_DGRAM: + sock->ops = &unix_dgram_ops; + break; + case SOCK_SEQPACKET: + sock->ops = &unix_seqpacket_ops; + break; + default: + return -ESOCKTNOSUPPORT; + } + + sk = unix_create1(net, sock, kern, sock->type); + if (IS_ERR(sk)) + return PTR_ERR(sk); + + return 0; +} + +static int unix_release(struct socket *sock) +{ + struct sock *sk = sock->sk; + + if (!sk) + return 0; + + sk->sk_prot->close(sk, 0); + unix_release_sock(sk, 0); + sock->sk = NULL; + + return 0; +} + +static struct sock *unix_find_bsd(struct sockaddr_un *sunaddr, int addr_len, + int type, int flags) +{ + struct inode *inode; + struct path path; + struct sock *sk; + int err; + + unix_mkname_bsd(sunaddr, addr_len); + + if (flags & SOCK_COREDUMP) { + scoped_with_init_fs() { + scoped_with_kernel_creds() + err = kern_path(sunaddr->sun_path, + LOOKUP_NO_SYMLINKS | + LOOKUP_NO_MAGICLINKS, &path); + } + if (err) + goto fail; + } else { + err = kern_path(sunaddr->sun_path, LOOKUP_FOLLOW, &path); + if (err) + goto fail; + + err = path_permission(&path, MAY_WRITE); + if (err) + goto path_put; + } + + err = -ECONNREFUSED; + inode = d_backing_inode(path.dentry); + if (!S_ISSOCK(inode->i_mode)) + goto path_put; + + sk = unix_find_socket_byinode(inode); + if (!sk) + goto path_put; + + err = -EPROTOTYPE; + if (sk->sk_type != type) + goto sock_put; + + err = security_unix_find(&path, sk, flags); + if (err) + goto sock_put; + + touch_atime(&path); + + path_put(&path); + + return sk; + +sock_put: + sock_put(sk); +path_put: + path_put(&path); +fail: + return ERR_PTR(err); +} + +static struct sock *unix_find_abstract(struct net *net, + struct sockaddr_un *sunaddr, + int addr_len, int type) +{ + unsigned int hash = unix_abstract_hash(sunaddr, addr_len, type); + struct dentry *dentry; + struct sock *sk; + + sk = unix_find_socket_byname(net, sunaddr, addr_len, hash); + if (!sk) + return ERR_PTR(-ECONNREFUSED); + + dentry = unix_sk(sk)->path.dentry; + if (dentry) + touch_atime(&unix_sk(sk)->path); + + return sk; +} + +static struct sock *unix_find_other(struct net *net, + struct sockaddr_un *sunaddr, + int addr_len, int type, int flags) +{ + struct sock *sk; + + if (sunaddr->sun_path[0]) + sk = unix_find_bsd(sunaddr, addr_len, type, flags); + else + sk = unix_find_abstract(net, sunaddr, addr_len, type); + + return sk; +} + +static int unix_autobind(struct sock *sk) +{ + struct unix_sock *u = unix_sk(sk); + unsigned int new_hash, old_hash; + struct net *net = sock_net(sk); + struct unix_address *addr; + u32 lastnum, ordernum; + int err; + + err = mutex_lock_interruptible(&u->bindlock); + if (err) + return err; + + if (u->addr) + goto out; + + err = -ENOMEM; + addr = kzalloc(sizeof(*addr) + + offsetof(struct sockaddr_un, sun_path) + 16, GFP_KERNEL); + if (!addr) + goto out; + + addr->len = offsetof(struct sockaddr_un, sun_path) + 6; + addr->name->sun_family = AF_UNIX; + refcount_set(&addr->refcnt, 1); + + old_hash = sk->sk_hash; + ordernum = get_random_u32(); + lastnum = ordernum & 0xFFFFF; +retry: + ordernum = (ordernum + 1) & 0xFFFFF; + sprintf(addr->name->sun_path + 1, "%05x", ordernum); + + new_hash = unix_abstract_hash(addr->name, addr->len, sk->sk_type); + unix_table_double_lock(net, old_hash, new_hash); + + if (__unix_find_socket_byname(net, addr->name, addr->len, new_hash)) { + unix_table_double_unlock(net, old_hash, new_hash); + + /* __unix_find_socket_byname() may take long time if many names + * are already in use. + */ + cond_resched(); + + if (ordernum == lastnum) { + /* Give up if all names seems to be in use. */ + err = -ENOSPC; + unix_release_addr(addr); + goto out; + } + + goto retry; + } + + __unix_set_addr_hash(net, sk, addr, new_hash); + unix_table_double_unlock(net, old_hash, new_hash); + err = 0; + +out: mutex_unlock(&u->bindlock); + return err; +} + +static int unix_bind_bsd(struct sock *sk, struct sockaddr_un *sunaddr, + int addr_len) +{ + umode_t mode = S_IFSOCK | + (SOCK_INODE(sk->sk_socket)->i_mode & ~current_umask()); + struct unix_sock *u = unix_sk(sk); + unsigned int new_hash, old_hash; + struct net *net = sock_net(sk); + struct mnt_idmap *idmap; + struct unix_address *addr; + struct dentry *dentry; + struct path parent; + int err; + + addr_len = unix_mkname_bsd(sunaddr, addr_len); + addr = unix_create_addr(sunaddr, addr_len); + if (!addr) + return -ENOMEM; + + /* + * Get the parent directory, calculate the hash for last + * component. + */ + dentry = start_creating_path(AT_FDCWD, addr->name->sun_path, &parent, 0); + if (IS_ERR(dentry)) { + err = PTR_ERR(dentry); + goto out; + } + + /* + * All right, let's create it. + */ + idmap = mnt_idmap(parent.mnt); + err = security_path_mknod(&parent, dentry, mode, 0); + if (!err) + err = vfs_mknod(idmap, d_inode(parent.dentry), dentry, mode, 0, NULL); + if (err) + goto out_path; + err = mutex_lock_interruptible(&u->bindlock); + if (err) + goto out_unlink; + if (u->addr) + goto out_unlock; + + old_hash = sk->sk_hash; + new_hash = unix_bsd_hash(d_backing_inode(dentry)); + unix_table_double_lock(net, old_hash, new_hash); + u->path.mnt = mntget(parent.mnt); + u->path.dentry = dget(dentry); + __unix_set_addr_hash(net, sk, addr, new_hash); + unix_table_double_unlock(net, old_hash, new_hash); + unix_insert_bsd_socket(sk); + mutex_unlock(&u->bindlock); + end_creating_path(&parent, dentry); + return 0; + +out_unlock: + mutex_unlock(&u->bindlock); + err = -EINVAL; +out_unlink: + /* failed after successful mknod? unlink what we'd created... */ + vfs_unlink(idmap, d_inode(parent.dentry), dentry, NULL); +out_path: + end_creating_path(&parent, dentry); +out: + unix_release_addr(addr); + return err == -EEXIST ? -EADDRINUSE : err; +} + +static int unix_bind_abstract(struct sock *sk, struct sockaddr_un *sunaddr, + int addr_len) +{ + struct unix_sock *u = unix_sk(sk); + unsigned int new_hash, old_hash; + struct net *net = sock_net(sk); + struct unix_address *addr; + int err; + + addr = unix_create_addr(sunaddr, addr_len); + if (!addr) + return -ENOMEM; + + err = mutex_lock_interruptible(&u->bindlock); + if (err) + goto out; + + if (u->addr) { + err = -EINVAL; + goto out_mutex; + } + + old_hash = sk->sk_hash; + new_hash = unix_abstract_hash(addr->name, addr->len, sk->sk_type); + unix_table_double_lock(net, old_hash, new_hash); + + if (__unix_find_socket_byname(net, addr->name, addr->len, new_hash)) + goto out_spin; + + __unix_set_addr_hash(net, sk, addr, new_hash); + unix_table_double_unlock(net, old_hash, new_hash); + mutex_unlock(&u->bindlock); + return 0; + +out_spin: + unix_table_double_unlock(net, old_hash, new_hash); + err = -EADDRINUSE; +out_mutex: + mutex_unlock(&u->bindlock); +out: + unix_release_addr(addr); + return err; +} + +static int unix_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int addr_len) +{ + struct sockaddr_un *sunaddr = (struct sockaddr_un *)uaddr; + struct sock *sk = sock->sk; + int err; + + if (addr_len == offsetof(struct sockaddr_un, sun_path) && + sunaddr->sun_family == AF_UNIX) + return unix_autobind(sk); + + err = unix_validate_addr(sunaddr, addr_len); + if (err) + return err; + + if (sunaddr->sun_path[0]) + err = unix_bind_bsd(sk, sunaddr, addr_len); + else + err = unix_bind_abstract(sk, sunaddr, addr_len); + + return err; +} + +static void unix_state_double_lock(struct sock *sk1, struct sock *sk2) +{ + if (unlikely(sk1 == sk2) || !sk2) { + unix_state_lock(sk1); + return; + } + + if (sk1 > sk2) + swap(sk1, sk2); + + unix_state_lock(sk1); + unix_state_lock(sk2); +} + +static void unix_state_double_unlock(struct sock *sk1, struct sock *sk2) +{ + if (unlikely(sk1 == sk2) || !sk2) { + unix_state_unlock(sk1); + return; + } + unix_state_unlock(sk1); + unix_state_unlock(sk2); +} + +static int unix_dgram_connect(struct socket *sock, struct sockaddr_unsized *addr, + int alen, int flags) +{ + struct sockaddr_un *sunaddr = (struct sockaddr_un *)addr; + struct sock *sk = sock->sk; + struct sock *other; + int err; + + err = -EINVAL; + if (alen < offsetofend(struct sockaddr, sa_family)) + goto out; + + if (addr->sa_family != AF_UNSPEC) { + err = unix_validate_addr(sunaddr, alen); + if (err) + goto out; + + err = BPF_CGROUP_RUN_PROG_UNIX_CONNECT_LOCK(sk, addr, &alen); + if (err) + goto out; + + if (unix_may_passcred(sk) && !READ_ONCE(unix_sk(sk)->addr)) { + err = unix_autobind(sk); + if (err) + goto out; + } + +restart: + other = unix_find_other(sock_net(sk), sunaddr, alen, sock->type, 0); + if (IS_ERR(other)) { + err = PTR_ERR(other); + goto out; + } + + unix_state_double_lock(sk, other); + + /* Apparently VFS overslept socket death. Retry. */ + if (sock_flag(other, SOCK_DEAD)) { + unix_state_double_unlock(sk, other); + sock_put(other); + goto restart; + } + + err = -EPERM; + if (!unix_may_send(sk, other)) + goto out_unlock; + + err = security_unix_may_send(sk->sk_socket, other->sk_socket); + if (err) + goto out_unlock; + + WRITE_ONCE(sk->sk_state, TCP_ESTABLISHED); + WRITE_ONCE(other->sk_state, TCP_ESTABLISHED); + } else { + /* + * 1003.1g breaking connected state with AF_UNSPEC + */ + other = NULL; + unix_state_double_lock(sk, other); + } + + /* + * If it was connected, reconnect. + */ + if (unix_peer(sk)) { + struct sock *old_peer = unix_peer(sk); + + unix_peer(sk) = other; + if (!other) + WRITE_ONCE(sk->sk_state, TCP_CLOSE); + unix_dgram_peer_wake_disconnect_wakeup(sk, old_peer); + + unix_state_double_unlock(sk, other); + + if (other != old_peer) { + unix_dgram_disconnected(sk, old_peer); + + unix_state_lock(old_peer); + if (!unix_peer(old_peer)) + WRITE_ONCE(old_peer->sk_state, TCP_CLOSE); + unix_state_unlock(old_peer); + } + + sock_put(old_peer); + } else { + unix_peer(sk) = other; + unix_state_double_unlock(sk, other); + } + + return 0; + +out_unlock: + unix_state_double_unlock(sk, other); + sock_put(other); +out: + return err; +} + +static long unix_wait_for_peer(struct sock *other, long timeo) +{ + struct unix_sock *u = unix_sk(other); + int sched; + DEFINE_WAIT(wait); + + prepare_to_wait_exclusive(&u->peer_wait, &wait, TASK_INTERRUPTIBLE); + + sched = !sock_flag(other, SOCK_DEAD) && + !(other->sk_shutdown & RCV_SHUTDOWN) && + unix_recvq_full_lockless(other); + + unix_state_unlock(other); + + if (sched) + timeo = schedule_timeout(timeo); + + finish_wait(&u->peer_wait, &wait); + return timeo; +} + +static int unix_stream_connect(struct socket *sock, struct sockaddr_unsized *uaddr, + int addr_len, int flags) +{ + struct sockaddr_un *sunaddr = (struct sockaddr_un *)uaddr; + struct sock *sk = sock->sk, *newsk = NULL, *other = NULL; + struct unix_sock *u = unix_sk(sk), *newu, *otheru; + struct unix_peercred peercred = {}; + struct net *net = sock_net(sk); + struct sk_buff *skb = NULL; + unsigned char state; + long timeo; + int err; + + err = unix_validate_addr(sunaddr, addr_len); + if (err) + goto out; + + err = BPF_CGROUP_RUN_PROG_UNIX_CONNECT_LOCK(sk, uaddr, &addr_len); + if (err) + goto out; + + if (unix_may_passcred(sk) && !READ_ONCE(u->addr)) { + err = unix_autobind(sk); + if (err) + goto out; + } + + timeo = sock_sndtimeo(sk, flags & O_NONBLOCK); + + err = prepare_peercred(&peercred); + if (err) + goto out; + + /* create new sock for complete connection */ + newsk = unix_create1(net, NULL, 0, sock->type); + if (IS_ERR(newsk)) { + err = PTR_ERR(newsk); + goto out; + } + + /* Allocate skb for sending to listening sock */ + skb = sock_wmalloc(newsk, 1, 0, GFP_KERNEL); + if (!skb) { + err = -ENOMEM; + goto out_free_sk; + } + +restart: + /* Find listening sock. */ + other = unix_find_other(net, sunaddr, addr_len, sk->sk_type, flags); + if (IS_ERR(other)) { + err = PTR_ERR(other); + goto out_free_skb; + } + + unix_state_lock(other); + + /* Apparently VFS overslept socket death. Retry. */ + if (sock_flag(other, SOCK_DEAD)) { + unix_state_unlock(other); + sock_put(other); + goto restart; + } + + if (other->sk_state != TCP_LISTEN || + other->sk_shutdown & RCV_SHUTDOWN) { + err = -ECONNREFUSED; + goto out_unlock; + } + + if (unix_recvq_full_lockless(other)) { + if (!timeo) { + err = -EAGAIN; + goto out_unlock; + } + + timeo = unix_wait_for_peer(other, timeo); + sock_put(other); + + err = sock_intr_errno(timeo); + if (signal_pending(current)) + goto out_free_skb; + + goto restart; + } + + /* self connect and simultaneous connect are eliminated + * by rejecting TCP_LISTEN socket to avoid deadlock. + */ + state = READ_ONCE(sk->sk_state); + if (unlikely(state != TCP_CLOSE)) { + err = state == TCP_ESTABLISHED ? -EISCONN : -EINVAL; + goto out_unlock; + } + + unix_state_lock(sk); + + if (unlikely(sk->sk_state != TCP_CLOSE)) { + err = sk->sk_state == TCP_ESTABLISHED ? -EISCONN : -EINVAL; + unix_state_unlock(sk); + goto out_unlock; + } + + err = security_unix_stream_connect(sk, other, newsk); + if (err) { + unix_state_unlock(sk); + goto out_unlock; + } + + /* The way is open! Fastly set all the necessary fields... */ + + sock_hold(sk); + unix_peer(newsk) = sk; + newsk->sk_state = TCP_ESTABLISHED; + newsk->sk_type = sk->sk_type; + newsk->sk_scm_recv_flags = other->sk_scm_recv_flags; + init_peercred(newsk, &peercred); + + newu = unix_sk(newsk); + otheru = unix_sk(other); + newu->listener = other; + newu->scm_rights_notrunc = READ_ONCE(otheru->scm_rights_notrunc); + RCU_INIT_POINTER(newsk->sk_wq, &newu->peer_wq); + + /* copy address information from listening to new sock + * + * The contents of *(otheru->addr) and otheru->path + * are seen fully set up here, since we have found + * otheru in hash under its lock. Insertion into the + * hash chain we'd found it in had been done in an + * earlier critical area protected by the chain's lock, + * the same one where we'd set *(otheru->addr) contents, + * as well as otheru->path and otheru->addr itself. + * + * Using smp_store_release() here to set newu->addr + * is enough to make those stores, as well as stores + * to newu->path visible to anyone who gets newu->addr + * by smp_load_acquire(). IOW, the same warranties + * as for unix_sock instances bound in unix_bind() or + * in unix_autobind(). + */ + if (otheru->path.dentry) { + path_get(&otheru->path); + newu->path = otheru->path; + } + refcount_inc(&otheru->addr->refcnt); + smp_store_release(&newu->addr, otheru->addr); + + /* Set credentials */ + copy_peercred(sk, other); + + WRITE_ONCE(sk->sk_state, TCP_ESTABLISHED); + sock_hold(newsk); + + smp_mb__after_atomic(); /* sock_hold() does an atomic_inc() */ + unix_peer(sk) = newsk; + + unix_state_unlock(sk); + + /* take ten and send info to listening sock */ + spin_lock(&other->sk_receive_queue.lock); + __skb_queue_tail(&other->sk_receive_queue, skb); + spin_unlock(&other->sk_receive_queue.lock); + unix_state_unlock(other); + READ_ONCE(other->sk_data_ready)(other); + sock_put(other); + return 0; + +out_unlock: + unix_state_unlock(other); + sock_put(other); +out_free_skb: + consume_skb(skb); +out_free_sk: + unix_release_sock(newsk, 0); +out: + drop_peercred(&peercred); + return err; +} + +static int unix_socketpair(struct socket *socka, struct socket *sockb) +{ + struct unix_peercred ska_peercred = {}, skb_peercred = {}; + struct sock *ska = socka->sk, *skb = sockb->sk; + int err; + + err = prepare_peercred(&ska_peercred); + if (err) + return err; + + err = prepare_peercred(&skb_peercred); + if (err) { + drop_peercred(&ska_peercred); + return err; + } + + /* Join our sockets back to back */ + sock_hold(ska); + sock_hold(skb); + unix_peer(ska) = skb; + unix_peer(skb) = ska; + init_peercred(ska, &ska_peercred); + init_peercred(skb, &skb_peercred); + + ska->sk_state = TCP_ESTABLISHED; + skb->sk_state = TCP_ESTABLISHED; + + return 0; +} + +static int unix_accept(struct socket *sock, struct socket *newsock, + struct proto_accept_arg *arg) +{ + struct sock *sk = sock->sk; + struct sk_buff *skb; + struct sock *tsk; + + arg->err = -EOPNOTSUPP; + if (sock->type != SOCK_STREAM && sock->type != SOCK_SEQPACKET) + goto out; + + arg->err = -EINVAL; + if (READ_ONCE(sk->sk_state) != TCP_LISTEN) + goto out; + + /* If socket state is TCP_LISTEN it cannot change (for now...), + * so that no locks are necessary. + */ + + skb = skb_recv_datagram(sk, (arg->flags & O_NONBLOCK) ? MSG_DONTWAIT : 0, + &arg->err); + if (!skb) { + /* This means receive shutdown. */ + if (arg->err == 0) + arg->err = -EINVAL; + goto out; + } + + tsk = skb->sk; + skb_free_datagram(sk, skb); + wake_up_interruptible(&unix_sk(sk)->peer_wait); + + set_bit(SOCK_CUSTOM_SOCKOPT, &newsock->flags); + + /* attach accepted sock to socket */ + unix_state_lock(tsk); + unix_update_edges(unix_sk(tsk)); + sock_graft(tsk, newsock); + unix_state_unlock(tsk); + return 0; + +out: + return arg->err; +} + + +static int unix_getname(struct socket *sock, struct sockaddr *uaddr, int peer) +{ + struct sock *sk = sock->sk; + struct unix_address *addr; + DECLARE_SOCKADDR(struct sockaddr_un *, sunaddr, uaddr); + int err = 0; + + if (peer) { + sk = unix_peer_get(sk); + + err = -ENOTCONN; + if (!sk) + goto out; + err = 0; + } else { + sock_hold(sk); + } + + addr = smp_load_acquire(&unix_sk(sk)->addr); + if (!addr) { + sunaddr->sun_family = AF_UNIX; + sunaddr->sun_path[0] = 0; + err = offsetof(struct sockaddr_un, sun_path); + } else { + err = addr->len; + memcpy(sunaddr, addr->name, addr->len); + + if (peer) + BPF_CGROUP_RUN_SA_PROG(sk, uaddr, &err, + CGROUP_UNIX_GETPEERNAME); + else + BPF_CGROUP_RUN_SA_PROG(sk, uaddr, &err, + CGROUP_UNIX_GETSOCKNAME); + } + sock_put(sk); +out: + return err; +} + +/* The "user->unix_inflight" variable is protected by the garbage + * collection lock, and we just read it locklessly here. If you go + * over the limit, there might be a tiny race in actually noticing + * it across threads. Tough. + */ +static inline bool too_many_unix_fds(struct task_struct *p) +{ + struct user_struct *user = current_user(); + + if (unlikely(READ_ONCE(user->unix_inflight) > task_rlimit(p, RLIMIT_NOFILE))) + return !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN); + return false; +} + +static int unix_attach_fds(struct scm_cookie *scm, struct sk_buff *skb) +{ + if (too_many_unix_fds(current)) + return -ETOOMANYREFS; + + UNIXCB(skb).fp = scm->fp; + scm->fp = NULL; + + if (unix_prepare_fpl(UNIXCB(skb).fp)) + return -ENOMEM; + + return 0; +} + +static void unix_detach_fds(struct scm_cookie *scm, struct sk_buff *skb) +{ + scm->fp = UNIXCB(skb).fp; + UNIXCB(skb).fp = NULL; + + unix_destroy_fpl(scm->fp); +} + +static void unix_peek_fds(struct scm_cookie *scm, struct sk_buff *skb) +{ + scm->fp = scm_fp_dup(UNIXCB(skb).fp); + + unix_peek_fpl(scm->fp); +} + +static void unix_destruct_scm(struct sk_buff *skb) +{ + struct scm_cookie scm = {}; + + swap(scm.pid, UNIXCB(skb).pid); + + if (UNIXCB(skb).fp) + unix_detach_fds(&scm, skb); + + scm_destroy(&scm); +} + +static void unix_wfree(struct sk_buff *skb) +{ + unix_destruct_scm(skb); + sock_wfree(skb); +} + +static int unix_scm_to_skb(struct scm_cookie *scm, struct sk_buff *skb, bool send_fds) +{ + int err = 0; + + UNIXCB(skb).pid = get_pid(scm->pid); + UNIXCB(skb).uid = scm->creds.uid; + UNIXCB(skb).gid = scm->creds.gid; + UNIXCB(skb).fp = NULL; + unix_get_secdata(scm, skb); + if (scm->fp && send_fds) + err = unix_attach_fds(scm, skb); + + skb->destructor = unix_wfree; + return err; +} + +static void unix_skb_to_scm(struct sk_buff *skb, struct scm_cookie *scm) +{ + scm_set_cred(scm, UNIXCB(skb).pid, UNIXCB(skb).uid, UNIXCB(skb).gid); + unix_set_secdata(scm, skb); +} + +/** + * unix_maybe_add_creds() - Adds current task uid/gid and struct pid to skb if needed. + * @skb: skb to attach creds to. + * @sk: Sender sock. + * @other: Receiver sock. + * + * Some apps rely on write() giving SCM_CREDENTIALS + * We include credentials if source or destination socket + * asserted SOCK_PASSCRED. + * + * Context: May sleep. + * Return: On success zero, on error a negative error code is returned. + */ +static int unix_maybe_add_creds(struct sk_buff *skb, const struct sock *sk, + const struct sock *other) +{ + if (UNIXCB(skb).pid) + return 0; + + if (unix_may_passcred(sk) || unix_may_passcred(other) || + !other->sk_socket) { + struct pid *pid; + int err; + + pid = task_tgid(current); + err = pidfs_register_pid(pid); + if (unlikely(err)) + return err; + + UNIXCB(skb).pid = get_pid(pid); + current_uid_gid(&UNIXCB(skb).uid, &UNIXCB(skb).gid); + } + + return 0; +} + +static bool unix_skb_scm_eq(struct sk_buff *skb, + struct scm_cookie *scm) +{ + return UNIXCB(skb).pid == scm->pid && + uid_eq(UNIXCB(skb).uid, scm->creds.uid) && + gid_eq(UNIXCB(skb).gid, scm->creds.gid) && + unix_secdata_eq(scm, skb); +} + +static void scm_stat_add(struct sock *sk, struct sk_buff *skb) +{ + struct scm_fp_list *fp = UNIXCB(skb).fp; + struct unix_sock *u = unix_sk(sk); + + if (unlikely(fp && fp->count)) { + atomic_add(fp->count, &u->scm_stat.nr_fds); + unix_add_edges(fp, u); + } +} + +static void scm_stat_del(struct sock *sk, struct sk_buff *skb) +{ + struct scm_fp_list *fp = UNIXCB(skb).fp; + struct unix_sock *u = unix_sk(sk); + + if (unlikely(fp && fp->count)) { + atomic_sub(fp->count, &u->scm_stat.nr_fds); + unix_del_edges(fp); + } +} + +static void unix_orphan_scm(struct sock *sk, struct sk_buff *skb) +{ + scm_stat_del(sk, skb); + unix_destruct_scm(skb); + skb->destructor = sock_wfree; +} + +/* + * Send AF_UNIX data. + */ + +static int unix_dgram_sendmsg(struct socket *sock, struct msghdr *msg, + size_t len) +{ + struct sock *sk = sock->sk, *other = NULL; + struct unix_sock *u = unix_sk(sk); + struct scm_cookie scm; + struct sk_buff *skb; + int data_len = 0; + int sk_locked; + long timeo; + int err; + + err = scm_send(sock, msg, &scm, false); + if (err < 0) + return err; + + if (msg->msg_flags & MSG_OOB) { + err = -EOPNOTSUPP; + goto out; + } + + if (msg->msg_namelen) { + err = unix_validate_addr(msg->msg_name, msg->msg_namelen); + if (err) + goto out; + + err = BPF_CGROUP_RUN_PROG_UNIX_SENDMSG_LOCK(sk, + msg->msg_name, + &msg->msg_namelen, + NULL); + if (err) + goto out; + } + + if (unix_may_passcred(sk) && !READ_ONCE(u->addr)) { + err = unix_autobind(sk); + if (err) + goto out; + } + + if (len > READ_ONCE(sk->sk_sndbuf) - 32) { + err = -EMSGSIZE; + goto out; + } + + if (len > SKB_MAX_ALLOC) { + data_len = min_t(size_t, + len - SKB_MAX_ALLOC, + MAX_SKB_FRAGS * PAGE_SIZE); + data_len = PAGE_ALIGN(data_len); + + BUILD_BUG_ON(SKB_MAX_ALLOC < PAGE_SIZE); + } + + skb = sock_alloc_send_pskb(sk, len - data_len, data_len, + msg->msg_flags & MSG_DONTWAIT, &err, + PAGE_ALLOC_COSTLY_ORDER); + if (!skb) + goto out; + + err = unix_scm_to_skb(&scm, skb, true); + if (err < 0) + goto out_free; + + skb_put(skb, len - data_len); + skb->data_len = data_len; + skb->len = len; + err = skb_copy_datagram_from_iter(skb, 0, &msg->msg_iter, len); + if (err) + goto out_free; + + timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); + + if (msg->msg_namelen) { +lookup: + other = unix_find_other(sock_net(sk), msg->msg_name, + msg->msg_namelen, sk->sk_type, 0); + if (IS_ERR(other)) { + err = PTR_ERR(other); + goto out_free; + } + } else { + other = unix_peer_get(sk); + if (!other) { + err = -ENOTCONN; + goto out_free; + } + } + + if (sk_filter(other, skb) < 0) { + /* Toss the packet but do not return any error to the sender */ + err = len; + goto out_sock_put; + } + + err = unix_maybe_add_creds(skb, sk, other); + if (err) + goto out_sock_put; + +restart: + sk_locked = 0; + unix_state_lock(other); +restart_locked: + + if (!unix_may_send(sk, other)) { + err = -EPERM; + goto out_unlock; + } + + if (unlikely(sock_flag(other, SOCK_DEAD))) { + /* Check with 1003.1g - what should datagram error */ + + unix_state_unlock(other); + + if (sk->sk_type == SOCK_SEQPACKET) { + /* We are here only when racing with unix_release_sock() + * is clearing @other. Never change state to TCP_CLOSE + * unlike SOCK_DGRAM wants. + */ + err = -EPIPE; + goto out_sock_put; + } + + if (!sk_locked) + unix_state_lock(sk); + + if (unix_peer(sk) == other) { + unix_peer(sk) = NULL; + unix_dgram_peer_wake_disconnect_wakeup(sk, other); + + WRITE_ONCE(sk->sk_state, TCP_CLOSE); + unix_state_unlock(sk); + + unix_dgram_disconnected(sk, other); + sock_put(other); + err = -ECONNREFUSED; + goto out_sock_put; + } + + unix_state_unlock(sk); + + if (!msg->msg_namelen) { + err = -ECONNRESET; + goto out_sock_put; + } + + sock_put(other); + goto lookup; + } + + if (other->sk_shutdown & RCV_SHUTDOWN) { + err = -EPIPE; + goto out_unlock; + } + + if (UNIXCB(skb).fp && !other->sk_scm_rights) { + err = -EPERM; + goto out_unlock; + } + + if (sk->sk_type != SOCK_SEQPACKET) { + err = security_unix_may_send(sk->sk_socket, other->sk_socket); + if (err) + goto out_unlock; + } + + /* other == sk && unix_peer(other) != sk if + * - unix_peer(sk) == NULL, destination address bound to sk + * - unix_peer(sk) == sk by time of get but disconnected before lock + */ + if (other != sk && + unlikely(unix_peer(other) != sk && + unix_recvq_full_lockless(other))) { + if (timeo) { + timeo = unix_wait_for_peer(other, timeo); + + err = sock_intr_errno(timeo); + if (signal_pending(current)) + goto out_sock_put; + + goto restart; + } + + if (!sk_locked) { + unix_state_unlock(other); + unix_state_double_lock(sk, other); + } + + if (unix_peer(sk) != other || + unix_dgram_peer_wake_me(sk, other)) { + err = -EAGAIN; + sk_locked = 1; + goto out_unlock; + } + + if (!sk_locked) { + sk_locked = 1; + goto restart_locked; + } + } + + if (unlikely(sk_locked)) + unix_state_unlock(sk); + + if (sock_flag(other, SOCK_RCVTSTAMP)) + __net_timestamp(skb); + + scm_stat_add(other, skb); + skb_queue_tail(&other->sk_receive_queue, skb); + unix_state_unlock(other); + READ_ONCE(other->sk_data_ready)(other); + sock_put(other); + scm_destroy(&scm); + return len; + +out_unlock: + if (sk_locked) + unix_state_unlock(sk); + unix_state_unlock(other); +out_sock_put: + sock_put(other); +out_free: + consume_skb(skb); +out: + scm_destroy(&scm); + return err; +} + +/* We use paged skbs for stream sockets, and limit occupancy to 32768 + * bytes, and a minimum of a full page. + */ +#define UNIX_SKB_FRAGS_SZ (PAGE_SIZE << get_order(32768)) + +#if IS_ENABLED(CONFIG_AF_UNIX_OOB) +static int queue_oob(struct sock *sk, struct msghdr *msg, struct sock *other, + struct scm_cookie *scm, bool fds_sent) +{ + struct unix_sock *ousk = unix_sk(other); + struct sk_buff *skb; + int err; + + skb = sock_alloc_send_skb(sk, 1, msg->msg_flags & MSG_DONTWAIT, &err); + + if (!skb) + return err; + + err = unix_scm_to_skb(scm, skb, !fds_sent); + if (err < 0) + goto out; + + err = unix_maybe_add_creds(skb, sk, other); + if (err) + goto out; + + skb_put(skb, 1); + err = skb_copy_datagram_from_iter(skb, 0, &msg->msg_iter, 1); + + if (err) + goto out; + + unix_state_lock(other); + + if (sock_flag(other, SOCK_DEAD) || + (other->sk_shutdown & RCV_SHUTDOWN)) { + err = -EPIPE; + goto out_unlock; + } + + if (UNIXCB(skb).fp && !other->sk_scm_rights) { + err = -EPERM; + goto out_unlock; + } + + scm_stat_add(other, skb); + + spin_lock(&other->sk_receive_queue.lock); + WRITE_ONCE(ousk->oob_skb, skb); + WRITE_ONCE(ousk->inq_len, ousk->inq_len + 1); + __skb_queue_tail(&other->sk_receive_queue, skb); + spin_unlock(&other->sk_receive_queue.lock); + + sk_send_sigurg(other); + unix_state_unlock(other); + READ_ONCE(other->sk_data_ready)(other); + + return 0; +out_unlock: + unix_state_unlock(other); +out: + consume_skb(skb); + return err; +} +#endif + +static int unix_stream_sendmsg(struct socket *sock, struct msghdr *msg, + size_t len) +{ + struct sock *sk = sock->sk; + struct sk_buff *skb = NULL; + struct sock *other = NULL; + struct unix_sock *otheru; + struct scm_cookie scm; + bool fds_sent = false; + int err, sent = 0; + + err = scm_send(sock, msg, &scm, false); + if (err < 0) + return err; + + if (msg->msg_flags & MSG_OOB) { + err = -EOPNOTSUPP; +#if IS_ENABLED(CONFIG_AF_UNIX_OOB) + if (len) + len--; + else +#endif + goto out_err; + } + + if (msg->msg_namelen) { + err = READ_ONCE(sk->sk_state) == TCP_ESTABLISHED ? -EISCONN : -EOPNOTSUPP; + goto out_err; + } + + other = unix_peer(sk); + if (!other) { + err = -ENOTCONN; + goto out_err; + } + + otheru = unix_sk(other); + + if (READ_ONCE(sk->sk_shutdown) & SEND_SHUTDOWN) + goto out_pipe; + + while (sent < len) { + int size = len - sent; + int data_len; + + if (unlikely(msg->msg_flags & MSG_SPLICE_PAGES)) { + skb = sock_alloc_send_pskb(sk, 0, 0, + msg->msg_flags & MSG_DONTWAIT, + &err, 0); + } else { + /* Keep two messages in the pipe so it schedules better */ + size = min_t(int, size, (READ_ONCE(sk->sk_sndbuf) >> 1) - 64); + + /* allow fallback to order-0 allocations */ + size = min_t(int, size, SKB_MAX_HEAD(0) + UNIX_SKB_FRAGS_SZ); + + data_len = max_t(int, 0, size - SKB_MAX_HEAD(0)); + + data_len = min_t(size_t, size, PAGE_ALIGN(data_len)); + + skb = sock_alloc_send_pskb(sk, size - data_len, data_len, + msg->msg_flags & MSG_DONTWAIT, &err, + get_order(UNIX_SKB_FRAGS_SZ)); + } + if (!skb) + goto out_err; + + /* Only send the fds in the first buffer */ + err = unix_scm_to_skb(&scm, skb, !fds_sent); + if (err < 0) + goto out_free; + + fds_sent = true; + + err = unix_maybe_add_creds(skb, sk, other); + if (err) + goto out_free; + + if (unlikely(msg->msg_flags & MSG_SPLICE_PAGES)) { + skb->ip_summed = CHECKSUM_UNNECESSARY; + err = skb_splice_from_iter(skb, &msg->msg_iter, size); + if (err < 0) + goto out_free; + + size = err; + refcount_add(size, &sk->sk_wmem_alloc); + } else { + skb_put(skb, size - data_len); + skb->data_len = data_len; + skb->len = size; + err = skb_copy_datagram_from_iter(skb, 0, &msg->msg_iter, size); + if (err) + goto out_free; + } + + unix_state_lock(other); + + if (sock_flag(other, SOCK_DEAD) || + (other->sk_shutdown & RCV_SHUTDOWN)) + goto out_pipe_unlock; + + if (UNIXCB(skb).fp && !other->sk_scm_rights) { + unix_state_unlock(other); + err = -EPERM; + goto out_free; + } + + scm_stat_add(other, skb); + + spin_lock(&other->sk_receive_queue.lock); + WRITE_ONCE(otheru->inq_len, otheru->inq_len + skb->len); + __skb_queue_tail(&other->sk_receive_queue, skb); + spin_unlock(&other->sk_receive_queue.lock); + + unix_state_unlock(other); + READ_ONCE(other->sk_data_ready)(other); + sent += size; + } + +#if IS_ENABLED(CONFIG_AF_UNIX_OOB) + if (msg->msg_flags & MSG_OOB) { + err = queue_oob(sk, msg, other, &scm, fds_sent); + if (err) + goto out_err; + sent++; + } +#endif + + scm_destroy(&scm); + + return sent; + +out_pipe_unlock: + unix_state_unlock(other); +out_pipe: + if (!sent && !(msg->msg_flags & MSG_NOSIGNAL)) + send_sig(SIGPIPE, current, 0); + err = -EPIPE; +out_free: + consume_skb(skb); +out_err: + scm_destroy(&scm); + return sent ? : err; +} + +static int unix_seqpacket_sendmsg(struct socket *sock, struct msghdr *msg, + size_t len) +{ + int err; + struct sock *sk = sock->sk; + + err = sock_error(sk); + if (err) + return err; + + if (READ_ONCE(sk->sk_state) != TCP_ESTABLISHED) + return -ENOTCONN; + + if (msg->msg_namelen) + msg->msg_namelen = 0; + + return unix_dgram_sendmsg(sock, msg, len); +} + +static int unix_seqpacket_recvmsg(struct socket *sock, struct msghdr *msg, + size_t size, int flags) +{ + struct sock *sk = sock->sk; + + if (READ_ONCE(sk->sk_state) != TCP_ESTABLISHED) + return -ENOTCONN; + + return unix_dgram_recvmsg(sock, msg, size, flags); +} + +static void unix_copy_addr(struct msghdr *msg, struct sock *sk) +{ + struct unix_address *addr = smp_load_acquire(&unix_sk(sk)->addr); + + if (addr) { + msg->msg_namelen = addr->len; + memcpy(msg->msg_name, addr->name, addr->len); + } +} + +int __unix_dgram_recvmsg(struct sock *sk, struct msghdr *msg, size_t size, + int flags) +{ + struct scm_cookie scm; + struct socket *sock = sk->sk_socket; + struct unix_sock *u = unix_sk(sk); + struct sk_buff *skb, *last; + long timeo; + int skip; + int err; + + err = -EOPNOTSUPP; + if (flags&MSG_OOB) + goto out; + + timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); + + do { + mutex_lock(&u->iolock); + + skip = sk_peek_offset(sk, flags); + skb = __skb_try_recv_datagram(sk, &sk->sk_receive_queue, flags, + &skip, &err, &last); + if (skb) { + if (!(flags & MSG_PEEK)) + scm_stat_del(sk, skb); + break; + } + + mutex_unlock(&u->iolock); + + if (err != -EAGAIN) + break; + } while (timeo && + !__skb_wait_for_more_packets(sk, &sk->sk_receive_queue, + &err, &timeo, last)); + + if (!skb) { /* implies iolock unlocked */ + /* Signal EOF on disconnected non-blocking SEQPACKET socket. */ + if (sk->sk_type == SOCK_SEQPACKET && err == -EAGAIN && + (READ_ONCE(sk->sk_shutdown) & RCV_SHUTDOWN)) + err = 0; + goto out; + } + + if (wq_has_sleeper(&u->peer_wait)) + wake_up_interruptible_sync_poll(&u->peer_wait, + EPOLLOUT | EPOLLWRNORM | + EPOLLWRBAND); + + if (msg->msg_name) { + unix_copy_addr(msg, skb->sk); + + BPF_CGROUP_RUN_PROG_UNIX_RECVMSG_LOCK(sk, + msg->msg_name, + &msg->msg_namelen); + } + + if (size > skb->len - skip) + size = skb->len - skip; + else if (size < skb->len - skip) + msg->msg_flags |= MSG_TRUNC; + + err = skb_copy_datagram_msg(skb, skip, msg, size); + if (err) + goto out_free; + + if (sock_flag(sk, SOCK_RCVTSTAMP)) + __sock_recv_timestamp(msg, sk, skb); + + memset(&scm, 0, sizeof(scm)); + + unix_skb_to_scm(skb, &scm); + + if (!(flags & MSG_PEEK)) { + if (UNIXCB(skb).fp) + unix_detach_fds(&scm, skb); + + sk_peek_offset_bwd(sk, skb->len); + } else { + /* It is questionable: on PEEK we could: + - do not return fds - good, but too simple 8) + - return fds, and do not return them on read (old strategy, + apparently wrong) + - clone fds (I chose it for now, it is the most universal + solution) + + POSIX 1003.1g does not actually define this clearly + at all. POSIX 1003.1g doesn't define a lot of things + clearly however! + + */ + + sk_peek_offset_fwd(sk, size); + + if (UNIXCB(skb).fp) + unix_peek_fds(&scm, skb); + } + err = (flags & MSG_TRUNC) ? skb->len - skip : size; + + scm_recv_unix(sock, msg, &scm, flags); + +out_free: + skb_free_datagram(sk, skb); + mutex_unlock(&u->iolock); +out: + return err; +} + +static int unix_dgram_recvmsg(struct socket *sock, struct msghdr *msg, size_t size, + int flags) +{ + struct sock *sk = sock->sk; + +#ifdef CONFIG_BPF_SYSCALL + const struct proto *prot = READ_ONCE(sk->sk_prot); + + if (prot != &unix_dgram_proto) + return prot->recvmsg(sk, msg, size, flags); +#endif + return __unix_dgram_recvmsg(sk, msg, size, flags); +} + +static int unix_read_skb(struct sock *sk, skb_read_actor_t recv_actor) +{ + struct unix_sock *u = unix_sk(sk); + struct sk_buff *skb; + int err; + + mutex_lock(&u->iolock); + + skb = skb_recv_datagram(sk, MSG_DONTWAIT, &err); + if (!skb) { + mutex_unlock(&u->iolock); + return err; + } + + unix_orphan_scm(sk, skb); + + mutex_unlock(&u->iolock); + + return recv_actor(sk, skb); +} + +/* + * Sleep until more data has arrived. But check for races.. + */ +static long unix_stream_data_wait(struct sock *sk, long timeo, + struct sk_buff *last, bool freezable) +{ + unsigned int state = TASK_INTERRUPTIBLE | freezable * TASK_FREEZABLE; + struct sk_buff *tail; + DEFINE_WAIT(wait); + + unix_state_lock(sk); + + for (;;) { + prepare_to_wait(sk_sleep(sk), &wait, state); + + tail = skb_peek_tail(&sk->sk_receive_queue); + if (tail != last || + sk->sk_err || + (sk->sk_shutdown & RCV_SHUTDOWN) || + signal_pending(current) || + !timeo) + break; + + sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); + unix_state_unlock(sk); + timeo = schedule_timeout(timeo); + unix_state_lock(sk); + + if (sock_flag(sk, SOCK_DEAD)) + break; + + sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); + } + + finish_wait(sk_sleep(sk), &wait); + unix_state_unlock(sk); + return timeo; +} + +struct unix_stream_read_state { + int (*recv_actor)(struct sk_buff *, int, int, + struct unix_stream_read_state *); + struct socket *socket; + struct msghdr *msg; + struct pipe_inode_info *pipe; + size_t size; + int flags; + unsigned int splice_flags; +}; + +#if IS_ENABLED(CONFIG_AF_UNIX_OOB) +static int unix_stream_recv_urg(struct unix_stream_read_state *state) +{ + struct sk_buff *oob_skb, *read_skb = NULL; + struct socket *sock = state->socket; + struct sock *sk = sock->sk; + struct unix_sock *u = unix_sk(sk); + int chunk = 1; + + mutex_lock(&u->iolock); + unix_state_lock(sk); + spin_lock(&sk->sk_receive_queue.lock); + + if (sock_flag(sk, SOCK_URGINLINE) || !u->oob_skb) { + spin_unlock(&sk->sk_receive_queue.lock); + unix_state_unlock(sk); + mutex_unlock(&u->iolock); + return -EINVAL; + } + + oob_skb = u->oob_skb; + + if (!(state->flags & MSG_PEEK)) { + WRITE_ONCE(u->oob_skb, NULL); + WRITE_ONCE(u->inq_len, u->inq_len - 1); + + if (oob_skb->prev != (struct sk_buff *)&sk->sk_receive_queue && + !unix_skb_len(oob_skb->prev)) { + read_skb = oob_skb->prev; + __skb_unlink(read_skb, &sk->sk_receive_queue); + } + } + + spin_unlock(&sk->sk_receive_queue.lock); + unix_state_unlock(sk); + + chunk = state->recv_actor(oob_skb, 0, chunk, state); + + if (!(state->flags & MSG_PEEK)) + UNIXCB(oob_skb).consumed += 1; + + mutex_unlock(&u->iolock); + + consume_skb(read_skb); + + if (chunk < 0) + return -EFAULT; + + state->msg->msg_flags |= MSG_OOB; + return 1; +} + +static struct sk_buff *manage_oob(struct sk_buff *skb, struct sk_buff **last, + struct sock *sk, int flags, int copied) +{ + struct sk_buff *read_skb = NULL, *unread_skb = NULL; + struct unix_sock *u = unix_sk(sk); + + if (likely(unix_skb_len(skb) && skb != READ_ONCE(u->oob_skb))) + return skb; + + spin_lock(&sk->sk_receive_queue.lock); + + if (!unix_skb_len(skb)) { + if (copied && (!u->oob_skb || skb == u->oob_skb)) { + skb = NULL; + } else if (flags & MSG_PEEK) { + *last = skb; + skb = skb_peek_next(skb, &sk->sk_receive_queue); + } else { + read_skb = skb; + skb = skb_peek_next(skb, &sk->sk_receive_queue); + __skb_unlink(read_skb, &sk->sk_receive_queue); + *last = skb; + } + + if (!skb) + goto unlock; + } + + if (skb != u->oob_skb) + goto unlock; + + if (copied) { + skb = NULL; + } else if (!(flags & MSG_PEEK)) { + WRITE_ONCE(u->oob_skb, NULL); + + if (!sock_flag(sk, SOCK_URGINLINE)) { + __skb_unlink(skb, &sk->sk_receive_queue); + unread_skb = skb; + skb = skb_peek(&sk->sk_receive_queue); + *last = skb; + } + } else if (!sock_flag(sk, SOCK_URGINLINE)) { + *last = skb; + skb = skb_peek_next(skb, &sk->sk_receive_queue); + } + +unlock: + spin_unlock(&sk->sk_receive_queue.lock); + + consume_skb(read_skb); + kfree_skb_reason(unread_skb, SKB_DROP_REASON_UNIX_SKIP_OOB); + + return skb; +} +#endif + +static int unix_stream_read_skb(struct sock *sk, skb_read_actor_t recv_actor) +{ + struct sk_buff_head *queue = &sk->sk_receive_queue; + struct unix_sock *u = unix_sk(sk); + struct sk_buff *skb; + int err; + + if (unlikely(READ_ONCE(sk->sk_state) != TCP_ESTABLISHED)) + return -ENOTCONN; + + err = sock_error(sk); + if (err) + return err; + + mutex_lock(&u->iolock); + spin_lock(&queue->lock); + + skb = __skb_dequeue(queue); + if (!skb) { + spin_unlock(&queue->lock); + mutex_unlock(&u->iolock); + return -EAGAIN; + } + + WRITE_ONCE(u->inq_len, u->inq_len - unix_skb_len(skb)); + +#if IS_ENABLED(CONFIG_AF_UNIX_OOB) + if (skb == u->oob_skb) { + WRITE_ONCE(u->oob_skb, NULL); + spin_unlock(&queue->lock); + mutex_unlock(&u->iolock); + + kfree_skb_reason(skb, SKB_DROP_REASON_UNIX_SKIP_OOB); + return -EAGAIN; + } +#endif + + spin_unlock(&queue->lock); + + unix_orphan_scm(sk, skb); + + mutex_unlock(&u->iolock); + + return recv_actor(sk, skb); +} + +static int unix_stream_read_generic(struct unix_stream_read_state *state, + bool freezable) +{ + int noblock = state->flags & MSG_DONTWAIT; + struct socket *sock = state->socket; + struct msghdr *msg = state->msg; + struct sock *sk = sock->sk; + size_t size = state->size; + int flags = state->flags; + bool check_creds = false; + struct scm_cookie scm; + struct unix_sock *u; + int copied = 0; + int err = 0; + long timeo; + int target; + int skip; + + if (unlikely(READ_ONCE(sk->sk_state) != TCP_ESTABLISHED)) { + err = -EINVAL; + goto out; + } + + if (unlikely(flags & MSG_OOB)) { + err = -EOPNOTSUPP; +#if IS_ENABLED(CONFIG_AF_UNIX_OOB) + err = unix_stream_recv_urg(state); +#endif + goto out; + } + + target = sock_rcvlowat(sk, flags & MSG_WAITALL, size); + timeo = sock_rcvtimeo(sk, noblock); + + memset(&scm, 0, sizeof(scm)); + + u = unix_sk(sk); + +redo: + /* Lock the socket to prevent queue disordering + * while sleeps in memcpy_tomsg + */ + mutex_lock(&u->iolock); + + skip = max(sk_peek_offset(sk, flags), 0); + + do { + struct sk_buff *skb, *last; + int chunk; + + unix_state_lock(sk); + if (sock_flag(sk, SOCK_DEAD)) { + err = -ECONNRESET; + goto unlock; + } + last = skb = skb_peek(&sk->sk_receive_queue); + +again: +#if IS_ENABLED(CONFIG_AF_UNIX_OOB) + if (skb) { + skb = manage_oob(skb, &last, sk, flags, copied); + if (!skb && (copied || !state->size)) { + unix_state_unlock(sk); + break; + } + } +#endif + if (skb == NULL) { + if (copied >= target) + goto unlock; + + /* + * POSIX 1003.1g mandates this order. + */ + + err = sock_error(sk); + if (err) + goto unlock; + if (sk->sk_shutdown & RCV_SHUTDOWN) + goto unlock; + + unix_state_unlock(sk); + if (!timeo) { + err = -EAGAIN; + break; + } + + mutex_unlock(&u->iolock); + + timeo = unix_stream_data_wait(sk, timeo, last, freezable); + + if (signal_pending(current)) { + err = sock_intr_errno(timeo); + scm_destroy(&scm); + goto out; + } + + goto redo; +unlock: + unix_state_unlock(sk); + break; + } + + while (skip >= unix_skb_len(skb)) { + skip -= unix_skb_len(skb); + last = skb; + skb = skb_peek_next(skb, &sk->sk_receive_queue); + if (!skb) + goto again; + } + + unix_state_unlock(sk); + + if (check_creds) { + /* Never glue messages from different writers */ + if (!unix_skb_scm_eq(skb, &scm)) + break; + } else if (unix_may_passcred(sk)) { + /* Copy credentials */ + unix_skb_to_scm(skb, &scm); + check_creds = true; + } + + /* Copy address just once */ + if (msg && msg->msg_name) { + DECLARE_SOCKADDR(struct sockaddr_un *, sunaddr, msg->msg_name); + + unix_copy_addr(msg, skb->sk); + BPF_CGROUP_RUN_PROG_UNIX_RECVMSG_LOCK(sk, msg->msg_name, + &msg->msg_namelen); + + sunaddr = NULL; + } + + chunk = min_t(unsigned int, unix_skb_len(skb) - skip, size); + chunk = state->recv_actor(skb, skip, chunk, state); + if (chunk < 0) { + if (copied == 0) + copied = -EFAULT; + break; + } + copied += chunk; + size -= chunk; + + /* Mark read part of skb as used */ + if (!(flags & MSG_PEEK)) { + UNIXCB(skb).consumed += chunk; + + sk_peek_offset_bwd(sk, chunk); + + if (UNIXCB(skb).fp) { + scm_stat_del(sk, skb); + unix_detach_fds(&scm, skb); + } + + spin_lock(&sk->sk_receive_queue.lock); + WRITE_ONCE(u->inq_len, u->inq_len - chunk); + if (unix_skb_len(skb)) { + spin_unlock(&sk->sk_receive_queue.lock); + break; + } + __skb_unlink(skb, &sk->sk_receive_queue); + spin_unlock(&sk->sk_receive_queue.lock); + + consume_skb(skb); + + if (scm.fp) + break; + } else { + /* It is questionable, see note in unix_dgram_recvmsg. + */ + if (UNIXCB(skb).fp) + unix_peek_fds(&scm, skb); + + sk_peek_offset_fwd(sk, chunk); + + if (UNIXCB(skb).fp) + break; + + skip = 0; + last = skb; + unix_state_lock(sk); + skb = skb_peek_next(skb, &sk->sk_receive_queue); + if (skb) + goto again; + unix_state_unlock(sk); + break; + } + } while (size); + + mutex_unlock(&u->iolock); + if (msg) { + bool do_cmsg = READ_ONCE(u->recvmsg_inq); + + scm_recv_unix(sock, msg, &scm, flags); + + if ((do_cmsg | msg->msg_get_inq) && (copied ?: err) >= 0) { + msg->msg_inq = READ_ONCE(u->inq_len); + if (do_cmsg) + put_cmsg(msg, SOL_SOCKET, SCM_INQ, + sizeof(msg->msg_inq), &msg->msg_inq); + } + } else { + scm_destroy(&scm); + } +out: + return copied ? : err; +} + +static int unix_stream_read_actor(struct sk_buff *skb, + int skip, int chunk, + struct unix_stream_read_state *state) +{ + int ret; + + ret = skb_copy_datagram_msg(skb, UNIXCB(skb).consumed + skip, + state->msg, chunk); + return ret ?: chunk; +} + +int __unix_stream_recvmsg(struct sock *sk, struct msghdr *msg, + size_t size, int flags) +{ + struct unix_stream_read_state state = { + .recv_actor = unix_stream_read_actor, + .socket = sk->sk_socket, + .msg = msg, + .size = size, + .flags = flags + }; + + return unix_stream_read_generic(&state, true); +} + +static int unix_stream_recvmsg(struct socket *sock, struct msghdr *msg, + size_t size, int flags) +{ + struct unix_stream_read_state state = { + .recv_actor = unix_stream_read_actor, + .socket = sock, + .msg = msg, + .size = size, + .flags = flags + }; + +#ifdef CONFIG_BPF_SYSCALL + struct sock *sk = sock->sk; + const struct proto *prot = READ_ONCE(sk->sk_prot); + + if (prot != &unix_stream_proto) + return prot->recvmsg(sk, msg, size, flags); +#endif + return unix_stream_read_generic(&state, true); +} + +static int unix_stream_splice_actor(struct sk_buff *skb, + int skip, int chunk, + struct unix_stream_read_state *state) +{ + return skb_splice_bits(skb, state->socket->sk, + UNIXCB(skb).consumed + skip, + state->pipe, chunk, state->splice_flags); +} + +static ssize_t unix_stream_splice_read(struct socket *sock, loff_t *ppos, + struct pipe_inode_info *pipe, + size_t size, unsigned int flags) +{ + struct unix_stream_read_state state = { + .recv_actor = unix_stream_splice_actor, + .socket = sock, + .pipe = pipe, + .size = size, + .splice_flags = flags, + }; + + if (unlikely(*ppos)) + return -ESPIPE; + + if (sock->file->f_flags & O_NONBLOCK || + flags & SPLICE_F_NONBLOCK) + state.flags = MSG_DONTWAIT; + + return unix_stream_read_generic(&state, false); +} + +static int unix_shutdown(struct socket *sock, int mode) +{ + struct sock *sk = sock->sk; + struct sock *other; + + if (mode < SHUT_RD || mode > SHUT_RDWR) + return -EINVAL; + /* This maps: + * SHUT_RD (0) -> RCV_SHUTDOWN (1) + * SHUT_WR (1) -> SEND_SHUTDOWN (2) + * SHUT_RDWR (2) -> SHUTDOWN_MASK (3) + */ + ++mode; + + unix_state_lock(sk); + WRITE_ONCE(sk->sk_shutdown, sk->sk_shutdown | mode); + other = unix_peer(sk); + if (other) + sock_hold(other); + unix_state_unlock(sk); + sk->sk_state_change(sk); + + if (other && + (sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET)) { + + int peer_mode = 0; + const struct proto *prot = READ_ONCE(other->sk_prot); + + if (prot->unhash) + prot->unhash(other); + if (mode&RCV_SHUTDOWN) + peer_mode |= SEND_SHUTDOWN; + if (mode&SEND_SHUTDOWN) + peer_mode |= RCV_SHUTDOWN; + unix_state_lock(other); + WRITE_ONCE(other->sk_shutdown, other->sk_shutdown | peer_mode); + unix_state_unlock(other); + other->sk_state_change(other); + if (peer_mode == SHUTDOWN_MASK) + sk_wake_async(other, SOCK_WAKE_WAITD, POLL_HUP); + else if (peer_mode & RCV_SHUTDOWN) + sk_wake_async(other, SOCK_WAKE_WAITD, POLL_IN); + } + if (other) + sock_put(other); + + return 0; +} + +long unix_inq_len(struct sock *sk) +{ + struct sk_buff *skb; + long amount = 0; + + if (READ_ONCE(sk->sk_state) == TCP_LISTEN) + return -EINVAL; + + if (sk->sk_type == SOCK_STREAM) + return READ_ONCE(unix_sk(sk)->inq_len); + + spin_lock(&sk->sk_receive_queue.lock); + if (sk->sk_type == SOCK_SEQPACKET) { + skb_queue_walk(&sk->sk_receive_queue, skb) + amount += unix_skb_len(skb); + } else { + skb = skb_peek(&sk->sk_receive_queue); + if (skb) + amount = skb->len; + } + spin_unlock(&sk->sk_receive_queue.lock); + + return amount; +} +EXPORT_SYMBOL_GPL(unix_inq_len); + +long unix_outq_len(struct sock *sk) +{ + return sk_wmem_alloc_get(sk); +} +EXPORT_SYMBOL_GPL(unix_outq_len); + +static int unix_open_file(struct sock *sk) +{ + if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) + return -EPERM; + + if (!smp_load_acquire(&unix_sk(sk)->addr)) + return -ENOENT; + + if (!unix_sk(sk)->path.dentry) + return -ENOENT; + + return FD_ADD(O_CLOEXEC, dentry_open(&unix_sk(sk)->path, O_PATH, current_cred())); +} + +static int unix_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) +{ + struct sock *sk = sock->sk; + long amount = 0; + int err; + + switch (cmd) { + case SIOCOUTQ: + amount = unix_outq_len(sk); + err = put_user(amount, (int __user *)arg); + break; + case SIOCINQ: + amount = unix_inq_len(sk); + if (amount < 0) + err = amount; + else + err = put_user(amount, (int __user *)arg); + break; + case SIOCUNIXFILE: + err = unix_open_file(sk); + break; +#if IS_ENABLED(CONFIG_AF_UNIX_OOB) + case SIOCATMARK: + { + struct unix_sock *u = unix_sk(sk); + struct sk_buff *skb; + int answ = 0; + + if (sk->sk_type != SOCK_STREAM) + return -EOPNOTSUPP; + + mutex_lock(&u->iolock); + + skb = skb_peek(&sk->sk_receive_queue); + if (skb) { + struct sk_buff *oob_skb = READ_ONCE(u->oob_skb); + struct sk_buff *next_skb; + + next_skb = skb_peek_next(skb, &sk->sk_receive_queue); + + if (skb == oob_skb || + (!unix_skb_len(skb) && + (!oob_skb || next_skb == oob_skb))) + answ = 1; + } + + mutex_unlock(&u->iolock); + + err = put_user(answ, (int __user *)arg); + } + break; +#endif + default: + err = -ENOIOCTLCMD; + break; + } + return err; +} + +#ifdef CONFIG_COMPAT +static int unix_compat_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) +{ + return unix_ioctl(sock, cmd, (unsigned long)compat_ptr(arg)); +} +#endif + +static __poll_t unix_poll(struct file *file, struct socket *sock, poll_table *wait) +{ + struct sock *sk = sock->sk; + unsigned char state; + __poll_t mask; + u8 shutdown; + + sock_poll_wait(file, sock, wait); + mask = 0; + shutdown = READ_ONCE(sk->sk_shutdown); + state = READ_ONCE(sk->sk_state); + + /* exceptional events? */ + if (READ_ONCE(sk->sk_err)) + mask |= EPOLLERR; + if (shutdown == SHUTDOWN_MASK) + mask |= EPOLLHUP; + if (shutdown & RCV_SHUTDOWN) + mask |= EPOLLRDHUP | EPOLLIN | EPOLLRDNORM; + + /* readable? */ + if (!skb_queue_empty_lockless(&sk->sk_receive_queue)) + mask |= EPOLLIN | EPOLLRDNORM; + if (sk_is_readable(sk)) + mask |= EPOLLIN | EPOLLRDNORM; +#if IS_ENABLED(CONFIG_AF_UNIX_OOB) + if (READ_ONCE(unix_sk(sk)->oob_skb)) + mask |= EPOLLPRI; +#endif + + /* Connection-based need to check for termination and startup */ + if ((sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET) && + state == TCP_CLOSE) + mask |= EPOLLHUP; + + /* + * we set writable also when the other side has shut down the + * connection. This prevents stuck sockets. + */ + if (unix_writable(sk, state)) + mask |= EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND; + + return mask; +} + +static __poll_t unix_dgram_poll(struct file *file, struct socket *sock, + poll_table *wait) +{ + struct sock *sk = sock->sk, *other; + unsigned int writable; + unsigned char state; + __poll_t mask; + u8 shutdown; + + sock_poll_wait(file, sock, wait); + mask = 0; + shutdown = READ_ONCE(sk->sk_shutdown); + state = READ_ONCE(sk->sk_state); + + /* exceptional events? */ + if (READ_ONCE(sk->sk_err) || + !skb_queue_empty_lockless(&sk->sk_error_queue)) + mask |= EPOLLERR | + (sock_flag(sk, SOCK_SELECT_ERR_QUEUE) ? EPOLLPRI : 0); + + if (shutdown & RCV_SHUTDOWN) + mask |= EPOLLRDHUP | EPOLLIN | EPOLLRDNORM; + if (shutdown == SHUTDOWN_MASK) + mask |= EPOLLHUP; + + /* readable? */ + if (!skb_queue_empty_lockless(&sk->sk_receive_queue)) + mask |= EPOLLIN | EPOLLRDNORM; + if (sk_is_readable(sk)) + mask |= EPOLLIN | EPOLLRDNORM; + + /* Connection-based need to check for termination and startup */ + if (sk->sk_type == SOCK_SEQPACKET && state == TCP_CLOSE) + mask |= EPOLLHUP; + + /* No write status requested, avoid expensive OUT tests. */ + if (!(poll_requested_events(wait) & (EPOLLWRBAND|EPOLLWRNORM|EPOLLOUT))) + return mask; + + writable = unix_writable(sk, state); + if (writable) { + unix_state_lock(sk); + + other = unix_peer(sk); + if (other && unix_peer(other) != sk && + unix_recvq_full_lockless(other) && + unix_dgram_peer_wake_me(sk, other)) + writable = 0; + + unix_state_unlock(sk); + } + + if (writable) + mask |= EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND; + else + sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); + + return mask; +} + +#ifdef CONFIG_PROC_FS + +#define BUCKET_SPACE (BITS_PER_LONG - (UNIX_HASH_BITS + 1) - 1) + +#define get_bucket(x) ((x) >> BUCKET_SPACE) +#define get_offset(x) ((x) & ((1UL << BUCKET_SPACE) - 1)) +#define set_bucket_offset(b, o) ((b) << BUCKET_SPACE | (o)) + +static struct sock *unix_from_bucket(struct seq_file *seq, loff_t *pos) +{ + unsigned long offset = get_offset(*pos); + unsigned long bucket = get_bucket(*pos); + unsigned long count = 0; + struct sock *sk; + + for (sk = sk_head(&seq_file_net(seq)->unx.table.buckets[bucket]); + sk; sk = sk_next(sk)) { + if (++count == offset) + break; + } + + return sk; +} + +static struct sock *unix_get_first(struct seq_file *seq, loff_t *pos) +{ + unsigned long bucket = get_bucket(*pos); + struct net *net = seq_file_net(seq); + struct sock *sk; + + while (bucket < UNIX_HASH_SIZE) { + spin_lock(&net->unx.table.locks[bucket]); + + sk = unix_from_bucket(seq, pos); + if (sk) + return sk; + + spin_unlock(&net->unx.table.locks[bucket]); + + *pos = set_bucket_offset(++bucket, 1); + } + + return NULL; +} + +static struct sock *unix_get_next(struct seq_file *seq, struct sock *sk, + loff_t *pos) +{ + unsigned long bucket = get_bucket(*pos); + + sk = sk_next(sk); + if (sk) + return sk; + + + spin_unlock(&seq_file_net(seq)->unx.table.locks[bucket]); + + *pos = set_bucket_offset(++bucket, 1); + + return unix_get_first(seq, pos); +} + +static void *unix_seq_start(struct seq_file *seq, loff_t *pos) +{ + if (!*pos) + return SEQ_START_TOKEN; + + return unix_get_first(seq, pos); +} + +static void *unix_seq_next(struct seq_file *seq, void *v, loff_t *pos) +{ + ++*pos; + + if (v == SEQ_START_TOKEN) + return unix_get_first(seq, pos); + + return unix_get_next(seq, v, pos); +} + +static void unix_seq_stop(struct seq_file *seq, void *v) +{ + struct sock *sk = v; + + if (sk) + spin_unlock(&seq_file_net(seq)->unx.table.locks[sk->sk_hash]); +} + +static int unix_seq_show(struct seq_file *seq, void *v) +{ + + if (v == SEQ_START_TOKEN) + seq_puts(seq, "Num RefCount Protocol Flags Type St " + "Inode Path\n"); + else { + struct sock *s = v; + struct unix_sock *u = unix_sk(s); + unix_state_lock(s); + + seq_printf(seq, "%pK: %08X %08X %08X %04X %02X %5llu", + s, + refcount_read(&s->sk_refcnt), + 0, + s->sk_state == TCP_LISTEN ? __SO_ACCEPTCON : 0, + s->sk_type, + s->sk_socket ? + (s->sk_state == TCP_ESTABLISHED ? SS_CONNECTED : SS_UNCONNECTED) : + (s->sk_state == TCP_ESTABLISHED ? SS_CONNECTING : SS_DISCONNECTING), + sock_i_ino(s)); + + if (u->addr) { // under a hash table lock here + int i, len; + seq_putc(seq, ' '); + + i = 0; + len = u->addr->len - + offsetof(struct sockaddr_un, sun_path); + if (u->addr->name->sun_path[0]) { + len--; + } else { + seq_putc(seq, '@'); + i++; + } + for ( ; i < len; i++) + seq_putc(seq, u->addr->name->sun_path[i] ?: + '@'); + } + unix_state_unlock(s); + seq_putc(seq, '\n'); + } + + return 0; +} + +static const struct seq_operations unix_seq_ops = { + .start = unix_seq_start, + .next = unix_seq_next, + .stop = unix_seq_stop, + .show = unix_seq_show, +}; + +#ifdef CONFIG_BPF_SYSCALL +struct bpf_unix_iter_state { + struct seq_net_private p; + unsigned int cur_sk; + unsigned int end_sk; + unsigned int max_sk; + struct sock **batch; + bool st_bucket_done; +}; + +struct bpf_iter__unix { + __bpf_md_ptr(struct bpf_iter_meta *, meta); + __bpf_md_ptr(struct unix_sock *, unix_sk); + uid_t uid __aligned(8); +}; + +static int unix_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta, + struct unix_sock *unix_sk, uid_t uid) +{ + struct bpf_iter__unix ctx; + + meta->seq_num--; /* skip SEQ_START_TOKEN */ + ctx.meta = meta; + ctx.unix_sk = unix_sk; + ctx.uid = uid; + return bpf_iter_run_prog(prog, &ctx); +} + +static int bpf_iter_unix_hold_batch(struct seq_file *seq, struct sock *start_sk) + +{ + struct bpf_unix_iter_state *iter = seq->private; + unsigned int expected = 1; + struct sock *sk; + + sock_hold(start_sk); + iter->batch[iter->end_sk++] = start_sk; + + for (sk = sk_next(start_sk); sk; sk = sk_next(sk)) { + if (iter->end_sk < iter->max_sk) { + sock_hold(sk); + iter->batch[iter->end_sk++] = sk; + } + + expected++; + } + + spin_unlock(&seq_file_net(seq)->unx.table.locks[start_sk->sk_hash]); + + return expected; +} + +static void bpf_iter_unix_put_batch(struct bpf_unix_iter_state *iter) +{ + while (iter->cur_sk < iter->end_sk) + sock_put(iter->batch[iter->cur_sk++]); +} + +static int bpf_iter_unix_realloc_batch(struct bpf_unix_iter_state *iter, + unsigned int new_batch_sz) +{ + struct sock **new_batch; + + new_batch = kvmalloc_objs(*new_batch, new_batch_sz, + GFP_USER | __GFP_NOWARN); + if (!new_batch) + return -ENOMEM; + + bpf_iter_unix_put_batch(iter); + kvfree(iter->batch); + iter->batch = new_batch; + iter->max_sk = new_batch_sz; + + return 0; +} + +static struct sock *bpf_iter_unix_batch(struct seq_file *seq, + loff_t *pos) +{ + struct bpf_unix_iter_state *iter = seq->private; + unsigned int expected; + bool resized = false; + struct sock *sk; + + if (iter->st_bucket_done) + *pos = set_bucket_offset(get_bucket(*pos) + 1, 1); + +again: + /* Get a new batch */ + iter->cur_sk = 0; + iter->end_sk = 0; + + sk = unix_get_first(seq, pos); + if (!sk) + return NULL; /* Done */ + + expected = bpf_iter_unix_hold_batch(seq, sk); + + if (iter->end_sk == expected) { + iter->st_bucket_done = true; + return sk; + } + + if (!resized && !bpf_iter_unix_realloc_batch(iter, expected * 3 / 2)) { + resized = true; + goto again; + } + + return sk; +} + +static void *bpf_iter_unix_seq_start(struct seq_file *seq, loff_t *pos) +{ + if (!*pos) + return SEQ_START_TOKEN; + + /* bpf iter does not support lseek, so it always + * continue from where it was stop()-ped. + */ + return bpf_iter_unix_batch(seq, pos); +} + +static void *bpf_iter_unix_seq_next(struct seq_file *seq, void *v, loff_t *pos) +{ + struct bpf_unix_iter_state *iter = seq->private; + struct sock *sk; + + /* Whenever seq_next() is called, the iter->cur_sk is + * done with seq_show(), so advance to the next sk in + * the batch. + */ + if (iter->cur_sk < iter->end_sk) + sock_put(iter->batch[iter->cur_sk++]); + + ++*pos; + + if (iter->cur_sk < iter->end_sk) + sk = iter->batch[iter->cur_sk]; + else + sk = bpf_iter_unix_batch(seq, pos); + + return sk; +} + +static int bpf_iter_unix_seq_show(struct seq_file *seq, void *v) +{ + struct bpf_iter_meta meta; + struct bpf_prog *prog; + struct sock *sk = v; + uid_t uid; + int ret; + + if (v == SEQ_START_TOKEN) + return 0; + + lock_sock(sk); + unix_state_lock(sk); + + if (unlikely(sock_flag(sk, SOCK_DEAD))) { + ret = SEQ_SKIP; + goto unlock; + } + + uid = from_kuid_munged(seq_user_ns(seq), sk_uid(sk)); + meta.seq = seq; + prog = bpf_iter_get_info(&meta, false); + ret = unix_prog_seq_show(prog, &meta, v, uid); +unlock: + unix_state_unlock(sk); + release_sock(sk); + return ret; +} + +static void bpf_iter_unix_seq_stop(struct seq_file *seq, void *v) +{ + struct bpf_unix_iter_state *iter = seq->private; + struct bpf_iter_meta meta; + struct bpf_prog *prog; + + if (!v) { + meta.seq = seq; + prog = bpf_iter_get_info(&meta, true); + if (prog) + (void)unix_prog_seq_show(prog, &meta, v, 0); + } + + if (iter->cur_sk < iter->end_sk) + bpf_iter_unix_put_batch(iter); +} + +static const struct seq_operations bpf_iter_unix_seq_ops = { + .start = bpf_iter_unix_seq_start, + .next = bpf_iter_unix_seq_next, + .stop = bpf_iter_unix_seq_stop, + .show = bpf_iter_unix_seq_show, +}; +#endif +#endif + +static const struct net_proto_family unix_family_ops = { + .family = PF_UNIX, + .create = unix_create, + .owner = THIS_MODULE, +}; + + +static int __net_init unix_net_init(struct net *net) +{ + int i; + + net->unx.sysctl_max_dgram_qlen = 10; + if (unix_sysctl_register(net)) + goto out; + +#ifdef CONFIG_PROC_FS + if (!proc_create_net("unix", 0, net->proc_net, &unix_seq_ops, + sizeof(struct seq_net_private))) + goto err_sysctl; +#endif + + net->unx.table.locks = kvmalloc_objs(spinlock_t, UNIX_HASH_SIZE); + if (!net->unx.table.locks) + goto err_proc; + + net->unx.table.buckets = kvmalloc_objs(struct hlist_head, + UNIX_HASH_SIZE); + if (!net->unx.table.buckets) + goto free_locks; + + for (i = 0; i < UNIX_HASH_SIZE; i++) { + spin_lock_init(&net->unx.table.locks[i]); + lock_set_cmp_fn(&net->unx.table.locks[i], unix_table_lock_cmp_fn, NULL); + INIT_HLIST_HEAD(&net->unx.table.buckets[i]); + } + + return 0; + +free_locks: + kvfree(net->unx.table.locks); +err_proc: +#ifdef CONFIG_PROC_FS + remove_proc_entry("unix", net->proc_net); +err_sysctl: +#endif + unix_sysctl_unregister(net); +out: + return -ENOMEM; +} + +static void __net_exit unix_net_exit(struct net *net) +{ + kvfree(net->unx.table.buckets); + kvfree(net->unx.table.locks); + unix_sysctl_unregister(net); + remove_proc_entry("unix", net->proc_net); +} + +static struct pernet_operations unix_net_ops = { + .init = unix_net_init, + .exit = unix_net_exit, +}; + +#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) +DEFINE_BPF_ITER_FUNC(unix, struct bpf_iter_meta *meta, + struct unix_sock *unix_sk, uid_t uid) + +#define INIT_BATCH_SZ 16 + +static int bpf_iter_init_unix(void *priv_data, struct bpf_iter_aux_info *aux) +{ + struct bpf_unix_iter_state *iter = priv_data; + int err; + + err = bpf_iter_init_seq_net(priv_data, aux); + if (err) + return err; + + err = bpf_iter_unix_realloc_batch(iter, INIT_BATCH_SZ); + if (err) { + bpf_iter_fini_seq_net(priv_data); + return err; + } + + return 0; +} + +static void bpf_iter_fini_unix(void *priv_data) +{ + struct bpf_unix_iter_state *iter = priv_data; + + bpf_iter_fini_seq_net(priv_data); + kvfree(iter->batch); +} + +static const struct bpf_iter_seq_info unix_seq_info = { + .seq_ops = &bpf_iter_unix_seq_ops, + .init_seq_private = bpf_iter_init_unix, + .fini_seq_private = bpf_iter_fini_unix, + .seq_priv_size = sizeof(struct bpf_unix_iter_state), +}; + +static const struct bpf_func_proto * +bpf_iter_unix_get_func_proto(enum bpf_func_id func_id, + const struct bpf_prog *prog) +{ + switch (func_id) { + case BPF_FUNC_setsockopt: + return &bpf_sk_setsockopt_proto; + case BPF_FUNC_getsockopt: + return &bpf_sk_getsockopt_proto; + default: + return NULL; + } +} + +static struct bpf_iter_reg unix_reg_info = { + .target = "unix", + .ctx_arg_info_size = 1, + .ctx_arg_info = { + { offsetof(struct bpf_iter__unix, unix_sk), + PTR_TO_BTF_ID_OR_NULL }, + }, + .get_func_proto = bpf_iter_unix_get_func_proto, + .seq_info = &unix_seq_info, +}; + +static void __init bpf_iter_register(void) +{ + unix_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_UNIX]; + if (bpf_iter_reg_target(&unix_reg_info)) + pr_warn("Warning: could not register bpf iterator unix\n"); +} +#endif + +static int __init af_unix_init(void) +{ + int i, rc = -1; + + BUILD_BUG_ON(sizeof(struct unix_skb_parms) > sizeof_field(struct sk_buff, cb)); + + for (i = 0; i < UNIX_HASH_SIZE / 2; i++) { + spin_lock_init(&bsd_socket_locks[i]); + INIT_HLIST_HEAD(&bsd_socket_buckets[i]); + } + + rc = proto_register(&unix_dgram_proto, 1); + if (rc != 0) { + pr_crit("%s: Cannot create unix_sock SLAB cache!\n", __func__); + goto out; + } + + rc = proto_register(&unix_stream_proto, 1); + if (rc != 0) { + pr_crit("%s: Cannot create unix_sock SLAB cache!\n", __func__); + proto_unregister(&unix_dgram_proto); + goto out; + } + + sock_register(&unix_family_ops); + register_pernet_subsys(&unix_net_ops); + unix_bpf_build_proto(); + +#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS) + bpf_iter_register(); +#endif + +out: + return rc; +} + +/* Later than subsys_initcall() because we depend on stuff initialised there */ +fs_initcall(af_unix_init); diff --git a/net/unix/af_unix.h b/net/unix/af_unix.h new file mode 100644 index 000000000..8119dbeef --- /dev/null +++ b/net/unix/af_unix.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __AF_UNIX_H +#define __AF_UNIX_H + +#include + +#define UNIX_HASH_MOD (256 - 1) +#define UNIX_HASH_SIZE (256 * 2) +#define UNIX_HASH_BITS 8 + +struct sock *unix_peer_get(struct sock *sk); + +struct unix_skb_parms { + struct pid *pid; /* skb credentials */ + kuid_t uid; + kgid_t gid; + struct scm_fp_list *fp; /* Passed files */ +#ifdef CONFIG_SECURITY_NETWORK + u32 secid; /* Security ID */ +#endif + u32 consumed; +} __randomize_layout; + +#define UNIXCB(skb) (*(struct unix_skb_parms *)&((skb)->cb)) + +/* GC for SCM_RIGHTS */ +void unix_add_edges(struct scm_fp_list *fpl, struct unix_sock *receiver); +void unix_del_edges(struct scm_fp_list *fpl); +void unix_update_edges(struct unix_sock *receiver); +int unix_prepare_fpl(struct scm_fp_list *fpl); +void unix_destroy_fpl(struct scm_fp_list *fpl); +void unix_peek_fpl(struct scm_fp_list *fpl); +void unix_schedule_gc(struct user_struct *user); + +/* SOCK_DIAG */ +long unix_inq_len(struct sock *sk); +long unix_outq_len(struct sock *sk); + +/* sysctl */ +#ifdef CONFIG_SYSCTL +int unix_sysctl_register(struct net *net); +void unix_sysctl_unregister(struct net *net); +#else +static inline int unix_sysctl_register(struct net *net) +{ + return 0; +} + +static inline void unix_sysctl_unregister(struct net *net) +{ +} +#endif + +/* BPF SOCKMAP */ +int __unix_dgram_recvmsg(struct sock *sk, struct msghdr *msg, size_t size, int flags); +int __unix_stream_recvmsg(struct sock *sk, struct msghdr *msg, size_t size, int flags); + +#ifdef CONFIG_BPF_SYSCALL +extern struct proto unix_dgram_proto; +extern struct proto unix_stream_proto; + +int unix_dgram_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore); +int unix_stream_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore); +void __init unix_bpf_build_proto(void); +#else +static inline void __init unix_bpf_build_proto(void) +{ +} +#endif + +#endif diff --git a/net/unix/diag.c b/net/unix/diag.c new file mode 100644 index 000000000..cca7dea05 --- /dev/null +++ b/net/unix/diag.c @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "af_unix.h" + +static int sk_diag_dump_name(struct sock *sk, struct sk_buff *nlskb) +{ + /* might or might not have a hash table lock */ + struct unix_address *addr = smp_load_acquire(&unix_sk(sk)->addr); + + if (!addr) + return 0; + + return nla_put(nlskb, UNIX_DIAG_NAME, + addr->len - offsetof(struct sockaddr_un, sun_path), + addr->name->sun_path); +} + +static int sk_diag_dump_vfs(struct sock *sk, struct sk_buff *nlskb) +{ + struct unix_diag_vfs uv; + struct dentry *dentry; + bool have_vfs = false; + + unix_state_lock(sk); + dentry = unix_sk(sk)->path.dentry; + if (dentry) { + uv.udiag_vfs_ino = d_backing_inode(dentry)->i_ino; + uv.udiag_vfs_dev = dentry->d_sb->s_dev; + have_vfs = true; + } + unix_state_unlock(sk); + + if (!have_vfs) + return 0; + + return nla_put(nlskb, UNIX_DIAG_VFS, sizeof(uv), &uv); +} + +static int sk_diag_dump_peer(struct sock *sk, struct sk_buff *nlskb) +{ + struct sock *peer; + u64 ino; + + peer = unix_peer_get(sk); + if (peer) { + ino = sock_i_ino(peer); + sock_put(peer); + + return nla_put_u32(nlskb, UNIX_DIAG_PEER, ino); + } + + return 0; +} + +static int sk_diag_dump_icons(struct sock *sk, struct sk_buff *nlskb) +{ + struct sk_buff *skb; + struct nlattr *attr; + u32 *buf; + int i; + + if (READ_ONCE(sk->sk_state) == TCP_LISTEN) { + spin_lock(&sk->sk_receive_queue.lock); + + attr = nla_reserve(nlskb, UNIX_DIAG_ICONS, + sk->sk_receive_queue.qlen * sizeof(u32)); + if (!attr) + goto errout; + + buf = nla_data(attr); + i = 0; + skb_queue_walk(&sk->sk_receive_queue, skb) + buf[i++] = sock_i_ino(unix_peer(skb->sk)); + + spin_unlock(&sk->sk_receive_queue.lock); + } + + return 0; + +errout: + spin_unlock(&sk->sk_receive_queue.lock); + return -EMSGSIZE; +} + +static int sk_diag_show_rqlen(struct sock *sk, struct sk_buff *nlskb) +{ + struct unix_diag_rqlen rql; + + if (READ_ONCE(sk->sk_state) == TCP_LISTEN) { + rql.udiag_rqueue = skb_queue_len_lockless(&sk->sk_receive_queue); + rql.udiag_wqueue = sk->sk_max_ack_backlog; + } else { + rql.udiag_rqueue = (u32) unix_inq_len(sk); + rql.udiag_wqueue = (u32) unix_outq_len(sk); + } + + return nla_put(nlskb, UNIX_DIAG_RQLEN, sizeof(rql), &rql); +} + +static int sk_diag_dump_uid(struct sock *sk, struct sk_buff *nlskb, + struct user_namespace *user_ns) +{ + uid_t uid = from_kuid_munged(user_ns, sk_uid(sk)); + return nla_put(nlskb, UNIX_DIAG_UID, sizeof(uid_t), &uid); +} + +static int sk_diag_fill(struct sock *sk, struct sk_buff *skb, struct unix_diag_req *req, + struct user_namespace *user_ns, + u32 portid, u32 seq, u32 flags, u64 sk_ino) +{ + struct nlmsghdr *nlh; + struct unix_diag_msg *rep; + + nlh = nlmsg_put(skb, portid, seq, SOCK_DIAG_BY_FAMILY, sizeof(*rep), + flags); + if (!nlh) + return -EMSGSIZE; + + rep = nlmsg_data(nlh); + rep->udiag_family = AF_UNIX; + rep->udiag_type = sk->sk_type; + rep->udiag_state = READ_ONCE(sk->sk_state); + rep->pad = 0; + rep->udiag_ino = sk_ino; + sock_diag_save_cookie(sk, rep->udiag_cookie); + + if ((req->udiag_show & UDIAG_SHOW_NAME) && + sk_diag_dump_name(sk, skb)) + goto out_nlmsg_trim; + + if ((req->udiag_show & UDIAG_SHOW_VFS) && + sk_diag_dump_vfs(sk, skb)) + goto out_nlmsg_trim; + + if ((req->udiag_show & UDIAG_SHOW_PEER) && + sk_diag_dump_peer(sk, skb)) + goto out_nlmsg_trim; + + if ((req->udiag_show & UDIAG_SHOW_ICONS) && + sk_diag_dump_icons(sk, skb)) + goto out_nlmsg_trim; + + if ((req->udiag_show & UDIAG_SHOW_RQLEN) && + sk_diag_show_rqlen(sk, skb)) + goto out_nlmsg_trim; + + if ((req->udiag_show & UDIAG_SHOW_MEMINFO) && + sock_diag_put_meminfo(sk, skb, UNIX_DIAG_MEMINFO)) + goto out_nlmsg_trim; + + if (nla_put_u8(skb, UNIX_DIAG_SHUTDOWN, READ_ONCE(sk->sk_shutdown))) + goto out_nlmsg_trim; + + if ((req->udiag_show & UDIAG_SHOW_UID) && + sk_diag_dump_uid(sk, skb, user_ns)) + goto out_nlmsg_trim; + + nlmsg_end(skb, nlh); + return 0; + +out_nlmsg_trim: + nlmsg_cancel(skb, nlh); + return -EMSGSIZE; +} + +static int unix_diag_dump(struct sk_buff *skb, struct netlink_callback *cb) +{ + struct net *net = sock_net(skb->sk); + int num, s_num, slot, s_slot; + struct unix_diag_req *req; + + req = nlmsg_data(cb->nlh); + + s_slot = cb->args[0]; + num = s_num = cb->args[1]; + + for (slot = s_slot; slot < UNIX_HASH_SIZE; s_num = 0, slot++) { + struct sock *sk; + + num = 0; + spin_lock(&net->unx.table.locks[slot]); + sk_for_each(sk, &net->unx.table.buckets[slot]) { + u64 sk_ino; + + if (num < s_num) + goto next; + + if (!(req->udiag_states & (1 << READ_ONCE(sk->sk_state)))) + goto next; + + sk_ino = sock_i_ino(sk); + if (!sk_ino) + goto next; + + if (sk_diag_fill(sk, skb, req, sk_user_ns(skb->sk), + NETLINK_CB(cb->skb).portid, + cb->nlh->nlmsg_seq, + NLM_F_MULTI, sk_ino) < 0) { + spin_unlock(&net->unx.table.locks[slot]); + goto done; + } +next: + num++; + } + spin_unlock(&net->unx.table.locks[slot]); + } +done: + cb->args[0] = slot; + cb->args[1] = num; + + return skb->len; +} + +static struct sock *unix_lookup_by_ino(struct net *net, unsigned int ino) +{ + struct sock *sk; + int i; + + for (i = 0; i < UNIX_HASH_SIZE; i++) { + spin_lock(&net->unx.table.locks[i]); + sk_for_each(sk, &net->unx.table.buckets[i]) { + if (ino == sock_i_ino(sk)) { + sock_hold(sk); + spin_unlock(&net->unx.table.locks[i]); + return sk; + } + } + spin_unlock(&net->unx.table.locks[i]); + } + return NULL; +} + +static int unix_diag_get_exact(struct sk_buff *in_skb, + const struct nlmsghdr *nlh, + struct unix_diag_req *req) +{ + struct net *net = sock_net(in_skb->sk); + unsigned int extra_len; + struct sk_buff *rep; + struct sock *sk; + int err; + + err = -EINVAL; + if (req->udiag_ino == 0) + goto out_nosk; + + sk = unix_lookup_by_ino(net, req->udiag_ino); + err = -ENOENT; + if (sk == NULL) + goto out_nosk; + + err = sock_diag_check_cookie(sk, req->udiag_cookie); + if (err) + goto out; + + extra_len = 256; +again: + err = -ENOMEM; + rep = nlmsg_new(sizeof(struct unix_diag_msg) + extra_len, GFP_KERNEL); + if (!rep) + goto out; + + err = sk_diag_fill(sk, rep, req, sk_user_ns(NETLINK_CB(in_skb).sk), + NETLINK_CB(in_skb).portid, + nlh->nlmsg_seq, 0, req->udiag_ino); + if (err < 0) { + nlmsg_free(rep); + extra_len += 256; + if (extra_len >= PAGE_SIZE) + goto out; + + goto again; + } + err = nlmsg_unicast(net->diag_nlsk, rep, NETLINK_CB(in_skb).portid); + +out: + if (sk) + sock_put(sk); +out_nosk: + return err; +} + +static int unix_diag_handler_dump(struct sk_buff *skb, struct nlmsghdr *h) +{ + int hdrlen = sizeof(struct unix_diag_req); + + if (nlmsg_len(h) < hdrlen) + return -EINVAL; + + if (h->nlmsg_flags & NLM_F_DUMP) { + struct netlink_dump_control c = { + .dump = unix_diag_dump, + }; + return netlink_dump_start(sock_net(skb->sk)->diag_nlsk, skb, h, &c); + } else + return unix_diag_get_exact(skb, h, nlmsg_data(h)); +} + +static const struct sock_diag_handler unix_diag_handler = { + .owner = THIS_MODULE, + .family = AF_UNIX, + .dump = unix_diag_handler_dump, +}; + +static int __init unix_diag_init(void) +{ + return sock_diag_register(&unix_diag_handler); +} + +static void __exit unix_diag_exit(void) +{ + sock_diag_unregister(&unix_diag_handler); +} + +module_init(unix_diag_init); +module_exit(unix_diag_exit); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("UNIX socket monitoring via SOCK_DIAG"); +MODULE_ALIAS_NET_PF_PROTO_TYPE(PF_NETLINK, NETLINK_SOCK_DIAG, 1 /* AF_LOCAL */); diff --git a/net/unix/garbage.c b/net/unix/garbage.c new file mode 100644 index 000000000..9fcaaf55c --- /dev/null +++ b/net/unix/garbage.c @@ -0,0 +1,660 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NET3: Garbage Collector For AF_UNIX sockets + * + * Garbage Collector: + * Copyright (C) Barak A. Pearlmutter. + * + * Chopped about by Alan Cox 22/3/96 to make it fit the AF_UNIX socket problem. + * If it doesn't work blame me, it worked when Barak sent it. + * + * Assumptions: + * + * - object w/ a bit + * - free list + * + * Current optimizations: + * + * - explicit stack instead of recursion + * - tail recurse on first born instead of immediate push/pop + * - we gather the stuff that should not be killed into tree + * and stack is just a path from root to the current pointer. + * + * Future optimizations: + * + * - don't just push entire root set; process in place + * + * Fixes: + * Alan Cox 07 Sept 1997 Vmalloc internal stack as needed. + * Cope with changing max_files. + * Al Viro 11 Oct 1998 + * Graph may have cycles. That is, we can send the descriptor + * of foo to bar and vice versa. Current code chokes on that. + * Fix: move SCM_RIGHTS ones into the separate list and then + * skb_free() them all instead of doing explicit fput's. + * Another problem: since fput() may block somebody may + * create a new unix_socket when we are in the middle of sweep + * phase. Fix: revert the logic wrt MARKED. Mark everything + * upon the beginning and unmark non-junk ones. + * + * [12 Oct 1998] AAARGH! New code purges all SCM_RIGHTS + * sent to connect()'ed but still not accept()'ed sockets. + * Fixed. Old code had slightly different problem here: + * extra fput() in situation when we passed the descriptor via + * such socket and closed it (descriptor). That would happen on + * each unix_gc() until the accept(). Since the struct file in + * question would go to the free list and might be reused... + * That might be the reason of random oopses on filp_close() + * in unrelated processes. + * + * AV 28 Feb 1999 + * Kill the explicit allocation of stack. Now we keep the tree + * with root in dummy + pointer (gc_current) to one of the nodes. + * Stack is represented as path from gc_current to dummy. Unmark + * now means "add to tree". Push == "make it a son of gc_current". + * Pop == "move gc_current to parent". We keep only pointers to + * parents (->gc_tree). + * AV 1 Mar 1999 + * Damn. Added missing check for ->dead in listen queues scanning. + * + * Miklos Szeredi 25 Jun 2007 + * Reimplement with a cycle collecting algorithm. This should + * solve several problems with the previous code, like being racy + * wrt receive and holding up unrelated socket operations. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "af_unix.h" + +struct unix_vertex { + struct list_head edges; + struct list_head entry; + struct list_head scc_entry; + unsigned long out_degree; + unsigned long index; + unsigned long scc_index; +}; + +struct unix_edge { + struct unix_sock *predecessor; + struct unix_sock *successor; + struct list_head vertex_entry; + struct list_head stack_entry; +}; + +struct unix_sock *unix_get_socket(struct file *filp) +{ + struct inode *inode = file_inode(filp); + + /* Socket ? */ + if (S_ISSOCK(inode->i_mode) && !(filp->f_mode & FMODE_PATH)) { + struct socket *sock = SOCKET_I(inode); + const struct proto_ops *ops; + struct sock *sk = sock->sk; + + ops = READ_ONCE(sock->ops); + + /* PF_UNIX ? */ + if (sk && ops && ops->family == PF_UNIX) + return unix_sk(sk); + } + + return NULL; +} + +static struct unix_vertex *unix_edge_successor(struct unix_edge *edge) +{ + /* If an embryo socket has a fd, + * the listener indirectly holds the fd's refcnt. + */ + if (edge->successor->listener) + return unix_sk(edge->successor->listener)->vertex; + + return edge->successor->vertex; +} + +enum { + UNIX_GRAPH_NOT_CYCLIC, + UNIX_GRAPH_MAYBE_CYCLIC, + UNIX_GRAPH_CYCLIC, +}; + +static unsigned char unix_graph_state; + +static void unix_update_graph(struct unix_vertex *vertex) +{ + /* If the receiver socket is not inflight, no cyclic + * reference could be formed. + */ + if (!vertex) + return; + + WRITE_ONCE(unix_graph_state, UNIX_GRAPH_MAYBE_CYCLIC); +} + +static LIST_HEAD(unix_unvisited_vertices); + +enum unix_vertex_index { + UNIX_VERTEX_INDEX_MARK1, + UNIX_VERTEX_INDEX_MARK2, + UNIX_VERTEX_INDEX_START, +}; + +static unsigned long unix_vertex_unvisited_index = UNIX_VERTEX_INDEX_MARK1; +static unsigned long unix_vertex_max_scc_index = UNIX_VERTEX_INDEX_START; + +static void unix_add_edge(struct scm_fp_list *fpl, struct unix_edge *edge) +{ + struct unix_vertex *vertex = edge->predecessor->vertex; + + if (!vertex) { + vertex = list_first_entry(&fpl->vertices, typeof(*vertex), entry); + vertex->index = unix_vertex_unvisited_index; + vertex->scc_index = ++unix_vertex_max_scc_index; + vertex->out_degree = 0; + INIT_LIST_HEAD(&vertex->edges); + INIT_LIST_HEAD(&vertex->scc_entry); + + list_move_tail(&vertex->entry, &unix_unvisited_vertices); + edge->predecessor->vertex = vertex; + } + + vertex->out_degree++; + list_add_tail(&edge->vertex_entry, &vertex->edges); + + unix_update_graph(unix_edge_successor(edge)); +} + +static void unix_del_edge(struct scm_fp_list *fpl, struct unix_edge *edge) +{ + struct unix_vertex *vertex = edge->predecessor->vertex; + + if (!fpl->dead) + unix_update_graph(unix_edge_successor(edge)); + + list_del(&edge->vertex_entry); + vertex->out_degree--; + + if (!vertex->out_degree) { + edge->predecessor->vertex = NULL; + list_move_tail(&vertex->entry, &fpl->vertices); + list_del(&vertex->scc_entry); + } +} + +static void unix_free_vertices(struct scm_fp_list *fpl) +{ + struct unix_vertex *vertex, *next_vertex; + + list_for_each_entry_safe(vertex, next_vertex, &fpl->vertices, entry) { + list_del(&vertex->entry); + kfree(vertex); + } +} + +static __cacheline_aligned_in_smp DEFINE_SPINLOCK(unix_gc_lock); + +void unix_add_edges(struct scm_fp_list *fpl, struct unix_sock *receiver) +{ + int i = 0, j = 0; + + spin_lock(&unix_gc_lock); + + if (!fpl->count_unix) + goto out; + + do { + struct unix_sock *inflight = unix_get_socket(fpl->fp[j++]); + struct unix_edge *edge; + + if (!inflight) + continue; + + edge = fpl->edges + i++; + edge->predecessor = inflight; + edge->successor = receiver; + + unix_add_edge(fpl, edge); + } while (i < fpl->count_unix); + + receiver->scm_stat.nr_unix_fds += fpl->count_unix; +out: + WRITE_ONCE(fpl->user->unix_inflight, fpl->user->unix_inflight + fpl->count); + + spin_unlock(&unix_gc_lock); + + fpl->inflight = true; + + unix_free_vertices(fpl); +} + +void unix_del_edges(struct scm_fp_list *fpl) +{ + struct unix_sock *receiver; + int i = 0; + + spin_lock(&unix_gc_lock); + + if (!fpl->count_unix) + goto out; + + do { + struct unix_edge *edge = fpl->edges + i++; + + unix_del_edge(fpl, edge); + } while (i < fpl->count_unix); + + if (!fpl->dead) { + receiver = fpl->edges[0].successor; + receiver->scm_stat.nr_unix_fds -= fpl->count_unix; + } +out: + WRITE_ONCE(fpl->user->unix_inflight, fpl->user->unix_inflight - fpl->count); + + spin_unlock(&unix_gc_lock); + + fpl->inflight = false; +} + +void unix_update_edges(struct unix_sock *receiver) +{ + /* nr_unix_fds is only updated under unix_state_lock(). + * If it's 0 here, the embryo socket is not part of the + * inflight graph, and GC will not see it, so no lock needed. + */ + if (!receiver->scm_stat.nr_unix_fds) { + receiver->listener = NULL; + } else { + spin_lock(&unix_gc_lock); + unix_update_graph(unix_sk(receiver->listener)->vertex); + receiver->listener = NULL; + spin_unlock(&unix_gc_lock); + } +} + +int unix_prepare_fpl(struct scm_fp_list *fpl) +{ + struct unix_vertex *vertex; + int i; + + if (!fpl->count_unix) + return 0; + + for (i = 0; i < fpl->count_unix; i++) { + vertex = kmalloc_obj(*vertex); + if (!vertex) + goto err; + + list_add(&vertex->entry, &fpl->vertices); + } + + fpl->edges = kvmalloc_objs(*fpl->edges, fpl->count_unix, + GFP_KERNEL_ACCOUNT); + if (!fpl->edges) + goto err; + + unix_schedule_gc(fpl->user); + + return 0; + +err: + unix_free_vertices(fpl); + return -ENOMEM; +} + +void unix_destroy_fpl(struct scm_fp_list *fpl) +{ + if (fpl->inflight) + unix_del_edges(fpl); + + kvfree(fpl->edges); + unix_free_vertices(fpl); +} + +static bool gc_in_progress; +static seqcount_t unix_peek_seq = SEQCNT_ZERO(unix_peek_seq); + +void unix_peek_fpl(struct scm_fp_list *fpl) +{ + static DEFINE_SPINLOCK(unix_peek_lock); + + if (!fpl || !fpl->count_unix) + return; + + if (!READ_ONCE(gc_in_progress)) + return; + + /* Invalidate the final refcnt check in unix_vertex_dead(). */ + spin_lock(&unix_peek_lock); + raw_write_seqcount_barrier(&unix_peek_seq); + spin_unlock(&unix_peek_lock); +} + +static bool unix_vertex_dead(struct unix_vertex *vertex) +{ + struct unix_edge *edge; + struct unix_sock *u; + long total_ref; + + list_for_each_entry(edge, &vertex->edges, vertex_entry) { + struct unix_vertex *next_vertex = unix_edge_successor(edge); + + /* The vertex's fd can be received by a non-inflight socket. */ + if (!next_vertex) + return false; + + /* The vertex's fd can be received by an inflight socket in + * another SCC. + */ + if (next_vertex->scc_index != vertex->scc_index) + return false; + } + + /* No receiver exists out of the same SCC. */ + + edge = list_first_entry(&vertex->edges, typeof(*edge), vertex_entry); + u = edge->predecessor; + total_ref = file_count(u->sk.sk_socket->file); + + /* If not close()d, total_ref > out_degree. */ + if (total_ref != vertex->out_degree) + return false; + + return true; +} + +static LIST_HEAD(unix_visited_vertices); +static unsigned long unix_vertex_grouped_index = UNIX_VERTEX_INDEX_MARK2; + +static bool unix_scc_dead(struct list_head *scc, bool fast) +{ + struct unix_vertex *vertex; + bool scc_dead = true; + unsigned int seq; + + seq = read_seqcount_begin(&unix_peek_seq); + + list_for_each_entry_reverse(vertex, scc, scc_entry) { + /* Don't restart DFS from this vertex. */ + list_move_tail(&vertex->entry, &unix_visited_vertices); + + /* Mark vertex as off-stack for __unix_walk_scc(). */ + if (!fast) + vertex->index = unix_vertex_grouped_index; + + if (scc_dead) + scc_dead = unix_vertex_dead(vertex); + } + + /* If MSG_PEEK intervened, defer this SCC to the next round. */ + if (read_seqcount_retry(&unix_peek_seq, seq)) + return false; + + return scc_dead; +} + +static void unix_collect_skb(struct list_head *scc, struct sk_buff_head *hitlist) +{ + struct unix_vertex *vertex; + + list_for_each_entry_reverse(vertex, scc, scc_entry) { + struct sk_buff_head *queue; + struct unix_edge *edge; + struct unix_sock *u; + + edge = list_first_entry(&vertex->edges, typeof(*edge), vertex_entry); + u = edge->predecessor; + queue = &u->sk.sk_receive_queue; + + spin_lock(&queue->lock); + + if (u->sk.sk_state == TCP_LISTEN) { + struct sk_buff *skb; + + skb_queue_walk(queue, skb) { + struct sk_buff_head *embryo_queue = &skb->sk->sk_receive_queue; + + spin_lock(&embryo_queue->lock); + skb_queue_splice_init(embryo_queue, hitlist); + spin_unlock(&embryo_queue->lock); + } + } else { + skb_queue_splice_init(queue, hitlist); + } + + spin_unlock(&queue->lock); + } +} + +static bool unix_scc_cyclic(struct list_head *scc) +{ + struct unix_vertex *vertex; + struct unix_edge *edge; + + /* SCC containing multiple vertices ? */ + if (!list_is_singular(scc)) + return true; + + vertex = list_first_entry(scc, typeof(*vertex), scc_entry); + + /* Self-reference or a embryo-listener circle ? */ + list_for_each_entry(edge, &vertex->edges, vertex_entry) { + if (unix_edge_successor(edge) == vertex) + return true; + } + + return false; +} + +static unsigned long __unix_walk_scc(struct unix_vertex *vertex, + unsigned long *last_index, + struct sk_buff_head *hitlist) +{ + unsigned long cyclic_sccs = 0; + LIST_HEAD(vertex_stack); + struct unix_edge *edge; + LIST_HEAD(edge_stack); + +next_vertex: + /* Push vertex to vertex_stack and mark it as on-stack + * (index >= UNIX_VERTEX_INDEX_START). + * The vertex will be popped when finalising SCC later. + */ + list_add(&vertex->scc_entry, &vertex_stack); + + vertex->index = *last_index; + vertex->scc_index = *last_index; + (*last_index)++; + + /* Explore neighbour vertices (receivers of the current vertex's fd). */ + list_for_each_entry(edge, &vertex->edges, vertex_entry) { + struct unix_vertex *next_vertex = unix_edge_successor(edge); + + if (!next_vertex) + continue; + + if (next_vertex->index == unix_vertex_unvisited_index) { + /* Iterative deepening depth first search + * + * 1. Push a forward edge to edge_stack and set + * the successor to vertex for the next iteration. + */ + list_add(&edge->stack_entry, &edge_stack); + + vertex = next_vertex; + goto next_vertex; + + /* 2. Pop the edge directed to the current vertex + * and restore the ancestor for backtracking. + */ +prev_vertex: + edge = list_first_entry(&edge_stack, typeof(*edge), stack_entry); + list_del_init(&edge->stack_entry); + + next_vertex = vertex; + vertex = edge->predecessor->vertex; + + /* If the successor has a smaller scc_index, two vertices + * are in the same SCC, so propagate the smaller scc_index + * to skip SCC finalisation. + */ + vertex->scc_index = min(vertex->scc_index, next_vertex->scc_index); + } else if (next_vertex->index != unix_vertex_grouped_index) { + /* Loop detected by a back/cross edge. + * + * The successor is on vertex_stack, so two vertices are in + * the same SCC. If the successor has a smaller *scc_index*, + * propagate it to skip SCC finalisation. + */ + vertex->scc_index = min(vertex->scc_index, next_vertex->scc_index); + } else { + /* The successor was already grouped as another SCC */ + } + } + + if (vertex->index == vertex->scc_index) { + struct list_head scc; + + /* SCC finalised. + * + * If the scc_index was not updated, all the vertices above on + * vertex_stack are in the same SCC. Group them using scc_entry. + */ + __list_cut_position(&scc, &vertex_stack, &vertex->scc_entry); + + if (unix_scc_dead(&scc, false)) { + unix_collect_skb(&scc, hitlist); + } else { + if (unix_vertex_max_scc_index < vertex->scc_index) + unix_vertex_max_scc_index = vertex->scc_index; + + if (unix_scc_cyclic(&scc)) + cyclic_sccs++; + } + + list_del(&scc); + } + + /* Need backtracking ? */ + if (!list_empty(&edge_stack)) + goto prev_vertex; + + return cyclic_sccs; +} + +static unsigned long unix_graph_cyclic_sccs; + +static void unix_walk_scc(struct sk_buff_head *hitlist) +{ + unsigned long last_index = UNIX_VERTEX_INDEX_START; + unsigned long cyclic_sccs = 0; + + unix_vertex_max_scc_index = UNIX_VERTEX_INDEX_START; + + /* Visit every vertex exactly once. + * __unix_walk_scc() moves visited vertices to unix_visited_vertices. + */ + while (!list_empty(&unix_unvisited_vertices)) { + struct unix_vertex *vertex; + + vertex = list_first_entry(&unix_unvisited_vertices, typeof(*vertex), entry); + cyclic_sccs += __unix_walk_scc(vertex, &last_index, hitlist); + } + + list_replace_init(&unix_visited_vertices, &unix_unvisited_vertices); + swap(unix_vertex_unvisited_index, unix_vertex_grouped_index); + + WRITE_ONCE(unix_graph_cyclic_sccs, cyclic_sccs); + WRITE_ONCE(unix_graph_state, + cyclic_sccs ? UNIX_GRAPH_CYCLIC : UNIX_GRAPH_NOT_CYCLIC); +} + +static void unix_walk_scc_fast(struct sk_buff_head *hitlist) +{ + unsigned long cyclic_sccs = unix_graph_cyclic_sccs; + + while (!list_empty(&unix_unvisited_vertices)) { + struct unix_vertex *vertex; + struct list_head scc; + + vertex = list_first_entry(&unix_unvisited_vertices, typeof(*vertex), entry); + list_add(&scc, &vertex->scc_entry); + + if (unix_scc_dead(&scc, true)) { + cyclic_sccs--; + unix_collect_skb(&scc, hitlist); + } + + list_del(&scc); + } + + list_replace_init(&unix_visited_vertices, &unix_unvisited_vertices); + + WRITE_ONCE(unix_graph_cyclic_sccs, cyclic_sccs); + WRITE_ONCE(unix_graph_state, + cyclic_sccs ? UNIX_GRAPH_CYCLIC : UNIX_GRAPH_NOT_CYCLIC); +} + +static void unix_gc(struct work_struct *work) +{ + struct sk_buff_head hitlist; + struct sk_buff *skb; + + WRITE_ONCE(gc_in_progress, true); + + spin_lock(&unix_gc_lock); + + if (unix_graph_state == UNIX_GRAPH_NOT_CYCLIC) { + spin_unlock(&unix_gc_lock); + goto skip_gc; + } + + __skb_queue_head_init(&hitlist); + + if (unix_graph_state == UNIX_GRAPH_CYCLIC) + unix_walk_scc_fast(&hitlist); + else + unix_walk_scc(&hitlist); + + spin_unlock(&unix_gc_lock); + + skb_queue_walk(&hitlist, skb) { + if (UNIXCB(skb).fp) + UNIXCB(skb).fp->dead = true; + } + + __skb_queue_purge_reason(&hitlist, SKB_DROP_REASON_SOCKET_CLOSE); +skip_gc: + WRITE_ONCE(gc_in_progress, false); +} + +static DECLARE_WORK(unix_gc_work, unix_gc); + +#define UNIX_INFLIGHT_SANE_USER (SCM_MAX_FD * 8) + +void unix_schedule_gc(struct user_struct *user) +{ + if (READ_ONCE(unix_graph_state) == UNIX_GRAPH_NOT_CYCLIC) + return; + + /* Penalise users who want to send AF_UNIX sockets + * but whose sockets have not been received yet. + */ + if (user && + READ_ONCE(user->unix_inflight) < UNIX_INFLIGHT_SANE_USER) + return; + + if (!READ_ONCE(gc_in_progress)) + queue_work(system_dfl_wq, &unix_gc_work); + + if (user && READ_ONCE(unix_graph_cyclic_sccs)) + flush_work(&unix_gc_work); +} diff --git a/net/unix/sysctl_net_unix.c b/net/unix/sysctl_net_unix.c new file mode 100644 index 000000000..47660d572 --- /dev/null +++ b/net/unix/sysctl_net_unix.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * NET4: Sysctl interface to net af_unix subsystem. + * + * Authors: Mike Shaver. + */ + +#include +#include +#include +#include +#include + +#include "af_unix.h" + +static const struct ctl_table unix_table[] = { + { + .procname = "max_dgram_qlen", + .data = &init_net.unx.sysctl_max_dgram_qlen, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec + }, +}; + +static const struct ctl_table *unix_table_dup(struct net *net) +{ + struct ctl_table *table; + + table = kmemdup(unix_table, sizeof(unix_table), GFP_KERNEL); + if (!table) + return NULL; + + table[0].data = &net->unx.sysctl_max_dgram_qlen; + + return table; +} + +int __net_init unix_sysctl_register(struct net *net) +{ + const struct ctl_table *table; + + if (net_eq(net, &init_net)) { + table = unix_table; + } else { + table = unix_table_dup(net); + if (!table) + goto err_alloc; + } + + net->unx.ctl = register_net_sysctl_sz(net, "net/unix", table, + ARRAY_SIZE(unix_table)); + if (net->unx.ctl == NULL) + goto err_reg; + + return 0; + +err_reg: + if (!net_eq(net, &init_net)) + kfree(table); +err_alloc: + return -ENOMEM; +} + +void unix_sysctl_unregister(struct net *net) +{ + const struct ctl_table *table; + + table = net->unx.ctl->ctl_table_arg; + unregister_net_sysctl_table(net->unx.ctl); + if (!net_eq(net, &init_net)) + kfree(table); +} diff --git a/net/unix/unix_bpf.c b/net/unix/unix_bpf.c new file mode 100644 index 000000000..f86ff19e9 --- /dev/null +++ b/net/unix/unix_bpf.c @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2021 Cong Wang */ + +#include +#include +#include + +#include "af_unix.h" + +#define unix_sk_has_data(__sk, __psock) \ + ({ !skb_queue_empty(&__sk->sk_receive_queue) || \ + !skb_queue_empty(&__psock->ingress_skb) || \ + !list_empty(&__psock->ingress_msg); \ + }) + +static int unix_msg_wait_data(struct sock *sk, struct sk_psock *psock, + long timeo) +{ + DEFINE_WAIT_FUNC(wait, woken_wake_function); + struct unix_sock *u = unix_sk(sk); + int ret = 0; + + if (sk->sk_shutdown & RCV_SHUTDOWN) + return 1; + + if (!timeo) + return ret; + + add_wait_queue(sk_sleep(sk), &wait); + sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); + if (!unix_sk_has_data(sk, psock)) { + mutex_unlock(&u->iolock); + wait_woken(&wait, TASK_INTERRUPTIBLE, timeo); + mutex_lock(&u->iolock); + ret = unix_sk_has_data(sk, psock); + } + sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); + remove_wait_queue(sk_sleep(sk), &wait); + return ret; +} + +static int __unix_recvmsg(struct sock *sk, struct msghdr *msg, + size_t len, int flags) +{ + if (sk->sk_type == SOCK_DGRAM) + return __unix_dgram_recvmsg(sk, msg, len, flags); + else + return __unix_stream_recvmsg(sk, msg, len, flags); +} + +static int unix_bpf_recvmsg(struct sock *sk, struct msghdr *msg, + size_t len, int flags) +{ + struct unix_sock *u = unix_sk(sk); + struct sk_psock *psock; + int copied; + + if (flags & MSG_OOB) + return -EOPNOTSUPP; + + if (!len) + return 0; + + psock = sk_psock_get(sk); + if (unlikely(!psock)) + return __unix_recvmsg(sk, msg, len, flags); + + mutex_lock(&u->iolock); + if (!skb_queue_empty(&sk->sk_receive_queue) && + sk_psock_queue_empty(psock)) { + mutex_unlock(&u->iolock); + sk_psock_put(sk, psock); + return __unix_recvmsg(sk, msg, len, flags); + } + +msg_bytes_ready: + copied = sk_msg_recvmsg(sk, psock, msg, len, flags); + if (!copied) { + long timeo; + int data; + + timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); + data = unix_msg_wait_data(sk, psock, timeo); + if (data) { + if (!sk_psock_queue_empty(psock)) + goto msg_bytes_ready; + mutex_unlock(&u->iolock); + sk_psock_put(sk, psock); + return __unix_recvmsg(sk, msg, len, flags); + } + copied = -EAGAIN; + } + mutex_unlock(&u->iolock); + sk_psock_put(sk, psock); + return copied; +} + +static struct proto *unix_dgram_prot_saved __read_mostly; +static DEFINE_SPINLOCK(unix_dgram_prot_lock); +static struct proto unix_dgram_bpf_prot; + +static struct proto *unix_stream_prot_saved __read_mostly; +static DEFINE_SPINLOCK(unix_stream_prot_lock); +static struct proto unix_stream_bpf_prot; + +static void unix_dgram_bpf_rebuild_protos(struct proto *prot, const struct proto *base) +{ + *prot = *base; + prot->close = sock_map_close; + prot->recvmsg = unix_bpf_recvmsg; + prot->sock_is_readable = sk_msg_is_readable; +} + +static void unix_stream_bpf_rebuild_protos(struct proto *prot, + const struct proto *base) +{ + *prot = *base; + prot->close = sock_map_close; + prot->recvmsg = unix_bpf_recvmsg; + prot->sock_is_readable = sk_msg_is_readable; + prot->unhash = sock_map_unhash; +} + +static void unix_dgram_bpf_check_needs_rebuild(struct proto *ops) +{ + if (unlikely(ops != smp_load_acquire(&unix_dgram_prot_saved))) { + spin_lock_bh(&unix_dgram_prot_lock); + if (likely(ops != unix_dgram_prot_saved)) { + unix_dgram_bpf_rebuild_protos(&unix_dgram_bpf_prot, ops); + smp_store_release(&unix_dgram_prot_saved, ops); + } + spin_unlock_bh(&unix_dgram_prot_lock); + } +} + +static void unix_stream_bpf_check_needs_rebuild(struct proto *ops) +{ + if (unlikely(ops != smp_load_acquire(&unix_stream_prot_saved))) { + spin_lock_bh(&unix_stream_prot_lock); + if (likely(ops != unix_stream_prot_saved)) { + unix_stream_bpf_rebuild_protos(&unix_stream_bpf_prot, ops); + smp_store_release(&unix_stream_prot_saved, ops); + } + spin_unlock_bh(&unix_stream_prot_lock); + } +} + +int unix_dgram_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) +{ + if (sk->sk_type != SOCK_DGRAM) + return -EOPNOTSUPP; + + if (restore) { + sk->sk_write_space = psock->saved_write_space; + sock_replace_proto(sk, psock->sk_proto); + return 0; + } + + unix_dgram_bpf_check_needs_rebuild(psock->sk_proto); + sock_replace_proto(sk, &unix_dgram_bpf_prot); + return 0; +} + +int unix_stream_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore) +{ + struct sock *sk_pair; + + /* Restore does not decrement the sk_pair reference yet because we must + * keep the a reference to the socket until after an RCU grace period + * and any pending sends have completed. + */ + if (restore) { + sk->sk_write_space = psock->saved_write_space; + sock_replace_proto(sk, psock->sk_proto); + return 0; + } + + /* psock_update_sk_prot can be called multiple times if psock is + * added to multiple maps and/or slots in the same map. There is + * also an edge case where replacing a psock with itself can trigger + * an extra psock_update_sk_prot during the insert process. So it + * must be safe to do multiple calls. Here we need to ensure we don't + * increment the refcnt through sock_hold many times. There will only + * be a single matching destroy operation. + */ + if (!psock->sk_pair) { + sk_pair = unix_peer(sk); + if (unlikely(!sk_pair)) + return -EINVAL; + + sock_hold(sk_pair); + psock->sk_pair = sk_pair; + } + + unix_stream_bpf_check_needs_rebuild(psock->sk_proto); + sock_replace_proto(sk, &unix_stream_bpf_prot); + return 0; +} + +void __init unix_bpf_build_proto(void) +{ + unix_dgram_bpf_rebuild_protos(&unix_dgram_bpf_prot, &unix_dgram_proto); + unix_stream_bpf_rebuild_protos(&unix_stream_bpf_prot, &unix_stream_proto); + +} -- cgit v1.3.1