From a5f7a5bb3b7f28ba7e4fa246775b29a0e5537255 Mon Sep 17 00:00:00 2001 From: Jinke Han Date: Tue, 8 Sep 2026 15:37:42 +0800 Subject: x86/kprobes: Fix crash when probing CS CALL instructions When using eBPF to probe CS CALL instructions within a function, a crash can be triggered. The eBPF tool probes offset 257 of the __hrtimer_run_queues() function: <__hrtimer_run_queues+249>: nopl 0x0(%rax,%rax,1) <__hrtimer_run_queues+254>: mov %r14,%rdi <__hrtimer_run_queues+257>: cs call <__x86_indirect_thunk_r12> <__hrtimer_run_queues+263>: mov %eax,%r12d <__hrtimer_run_queues+266>: xchg %ax,%ax <__hrtimer_run_queues+268>: mov %r13,%rdi Which triggers this crash: BUG: unable to handle page fault for address: 00000000000f41c9 #PF: supervisor write access in kernel mode #PF: error_code(0x0002) - not-present page PGD 0 P4D 0 Oops: 0002 [#1] SMP NOPTI CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P RIP: 0010:__hrtimer_run_queues+0x106/0x230 Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is at the 6th byte of the above CS CALL instruction. Since the CS CALL instruction occupies 6 bytes, the exception occurred in the middle of that call instruction. The root cause is that when using eBPF tools to probe in the middle of a function, a kprobe with INT3 is used as the underlying implementation. During single-step emulation of the original CALL instruction, int3_emulate_call() assumes that the probed CALL instruction is 5 bytes long. However, the actual CS-prefixed CALL instruction occupies 6 bytes, so it constructs an incorrect exception return address. When the CPU returns from the kprobe handler, the next instruction to be executed is at the address of the last byte of that CS CALL instruction. Coincidentally, starting from that address, the CPU fetches and decodes a completely different instruction, which ultimately triggers a kernel crash. Fix the issue by using the actual instruction length obtained from the instruction decoder when constructing the exception return address, rather than relying on the hardcoded CALL_INSN_SIZE macro. [ mingo: Refined the changelog ] Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step") Suggested-by: Masami Hiramatsu (Google) Signed-off-by: Jinke Han Signed-off-by: Ingo Molnar Reviewed-by: Masami Hiramatsu (Google) Acked-by: Yafang Shao Acked-by: Borislav Petkov Cc: Peter Zijlstra Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000 --- net/ipv4/fib_rules.c | 530 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 530 insertions(+) create mode 100644 net/ipv4/fib_rules.c (limited to 'net/ipv4/fib_rules.c') diff --git a/net/ipv4/fib_rules.c b/net/ipv4/fib_rules.c new file mode 100644 index 000000000..060501b37 --- /dev/null +++ b/net/ipv4/fib_rules.c @@ -0,0 +1,530 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * INET An implementation of the TCP/IP protocol suite for the LINUX + * operating system. INET is implemented using the BSD Socket + * interface as the means of communication with the user level. + * + * IPv4 Forwarding Information Base: policy rules. + * + * Authors: Alexey Kuznetsov, + * Thomas Graf + * + * Fixes: + * Rani Assaf : local_rule cannot be deleted + * Marc Boucher : routing by fwmark + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct fib4_rule { + struct fib_rule common; + u8 dst_len; + u8 src_len; + dscp_t dscp; + dscp_t dscp_mask; + u8 dscp_full:1; /* DSCP or TOS selector */ + __be32 src; + __be32 srcmask; + __be32 dst; + __be32 dstmask; +#ifdef CONFIG_IP_ROUTE_CLASSID + u32 tclassid; +#endif +}; + +static bool fib4_rule_matchall(const struct fib_rule *rule) +{ + struct fib4_rule *r = container_of(rule, struct fib4_rule, common); + + if (r->dst_len || r->src_len || r->dscp) + return false; + return fib_rule_matchall(rule); +} + +bool fib4_rule_default(const struct fib_rule *rule) +{ + if (!fib4_rule_matchall(rule) || rule->action != FR_ACT_TO_TBL || + rule->l3mdev) + return false; + if (rule->table != RT_TABLE_LOCAL && rule->table != RT_TABLE_MAIN && + rule->table != RT_TABLE_DEFAULT) + return false; + return true; +} +EXPORT_SYMBOL_GPL(fib4_rule_default); + +int fib4_rules_dump(struct net *net, struct notifier_block *nb, + struct netlink_ext_ack *extack) +{ + return fib_rules_dump(net, nb, AF_INET, extack); +} + +unsigned int fib4_rules_seq_read(const struct net *net) +{ + return fib_rules_seq_read(net, AF_INET); +} + +int __fib_lookup(struct net *net, struct flowi4 *flp, + struct fib_result *res, unsigned int flags) +{ + struct fib_lookup_arg arg = { + .result = res, + .flags = flags, + }; + int err; + + /* update flow if oif or iif point to device enslaved to l3mdev */ + l3mdev_update_flow(net, flowi4_to_flowi(flp)); + + err = fib_rules_lookup(net->ipv4.rules_ops, flowi4_to_flowi(flp), 0, &arg); +#ifdef CONFIG_IP_ROUTE_CLASSID + if (arg.rule) + res->tclassid = ((struct fib4_rule *)arg.rule)->tclassid; + else + res->tclassid = 0; +#endif + + if (err == -ESRCH) + err = -ENETUNREACH; + + return err; +} +EXPORT_SYMBOL_GPL(__fib_lookup); + +INDIRECT_CALLABLE_SCOPE int fib4_rule_action(struct fib_rule *rule, + struct flowi *flp, int flags, + struct fib_lookup_arg *arg) +{ + int err = -EAGAIN; + struct fib_table *tbl; + u32 tb_id; + + switch (rule->action) { + case FR_ACT_TO_TBL: + break; + + case FR_ACT_UNREACHABLE: + return -ENETUNREACH; + + case FR_ACT_PROHIBIT: + return -EACCES; + + case FR_ACT_BLACKHOLE: + default: + return -EINVAL; + } + + rcu_read_lock(); + + tb_id = fib_rule_get_table(rule, arg); + tbl = fib_get_table(rule->fr_net, tb_id); + if (tbl) + err = fib_table_lookup(tbl, &flp->u.ip4, + (struct fib_result *)arg->result, + arg->flags); + + rcu_read_unlock(); + return err; +} + +INDIRECT_CALLABLE_SCOPE bool fib4_rule_suppress(struct fib_rule *rule, + int flags, + struct fib_lookup_arg *arg) +{ + struct fib_result *result = arg->result; + struct net_device *dev = NULL; + + if (result->fi) { + struct fib_nh_common *nhc = fib_info_nhc(result->fi, 0); + + dev = nhc->nhc_dev; + } + + /* do not accept result if the route does + * not meet the required prefix length + */ + if (result->prefixlen <= rule->suppress_prefixlen) + goto suppress_route; + + /* do not accept result if the route uses a device + * belonging to a forbidden interface group + */ + if (rule->suppress_ifgroup != -1 && dev && dev->group == rule->suppress_ifgroup) + goto suppress_route; + + return false; + +suppress_route: + if (!(arg->flags & FIB_LOOKUP_NOREF)) + fib_info_put(result->fi); + return true; +} + +INDIRECT_CALLABLE_SCOPE int fib4_rule_match(struct fib_rule *rule, + struct flowi *fl, int flags) +{ + struct fib4_rule *r = (struct fib4_rule *) rule; + struct flowi4 *fl4 = &fl->u.ip4; + __be32 daddr = fl4->daddr; + __be32 saddr = fl4->saddr; + + if (((saddr ^ r->src) & r->srcmask) || + ((daddr ^ r->dst) & r->dstmask)) + return 0; + + /* When DSCP selector is used we need to match on the entire DSCP field + * in the flow information structure. When TOS selector is used we need + * to mask the upper three DSCP bits prior to matching to maintain + * legacy behavior. + */ + if (r->dscp_full && (r->dscp ^ fl4->flowi4_dscp) & r->dscp_mask) + return 0; + else if (!r->dscp_full && r->dscp && + !fib_dscp_masked_match(r->dscp, fl4)) + return 0; + + if (rule->ip_proto && (rule->ip_proto != fl4->flowi4_proto)) + return 0; + + if (!fib_rule_port_match(&rule->sport_range, rule->sport_mask, + fl4->fl4_sport)) + return 0; + + if (!fib_rule_port_match(&rule->dport_range, rule->dport_mask, + fl4->fl4_dport)) + return 0; + + return 1; +} + +#define FIB_MAX_AUTO_TABLE_ID 4096 + +static struct fib_table *fib_empty_table(struct net *net) +{ + u32 id = 1; + + while (1) { + if (!fib_get_table(net, id)) + return fib_new_table(net, id); + + if (id++ == FIB_MAX_AUTO_TABLE_ID) + break; + } + return NULL; +} + +static int fib4_nl2rule_dscp(const struct nlattr *nla, struct fib4_rule *rule4, + struct netlink_ext_ack *extack) +{ + if (rule4->dscp) { + NL_SET_ERR_MSG(extack, "Cannot specify both TOS and DSCP"); + return -EINVAL; + } + + rule4->dscp = inet_dsfield_to_dscp(nla_get_u8(nla) << 2); + rule4->dscp_mask = inet_dsfield_to_dscp(INET_DSCP_MASK); + rule4->dscp_full = true; + + return 0; +} + +static int fib4_nl2rule_dscp_mask(const struct nlattr *nla, + struct fib4_rule *rule4, + struct netlink_ext_ack *extack) +{ + dscp_t dscp_mask; + + if (!rule4->dscp_full) { + NL_SET_ERR_MSG_ATTR(extack, nla, + "Cannot specify DSCP mask without DSCP value"); + return -EINVAL; + } + + dscp_mask = inet_dsfield_to_dscp(nla_get_u8(nla) << 2); + if (rule4->dscp & ~dscp_mask) { + NL_SET_ERR_MSG_ATTR(extack, nla, "Invalid DSCP mask"); + return -EINVAL; + } + + rule4->dscp_mask = dscp_mask; + + return 0; +} + +static int fib4_rule_configure(struct fib_rule *rule, struct sk_buff *skb, + struct fib_rule_hdr *frh, + struct nlattr **tb, + struct netlink_ext_ack *extack) +{ + struct fib4_rule *rule4 = (struct fib4_rule *)rule; + struct net *net = rule->fr_net; + int err = -EINVAL; + + if (tb[FRA_FLOWLABEL] || tb[FRA_FLOWLABEL_MASK]) { + NL_SET_ERR_MSG(extack, + "Flow label cannot be specified for IPv4 FIB rules"); + goto errout; + } + + if (!inet_validate_dscp(frh->tos)) { + NL_SET_ERR_MSG(extack, + "Invalid dsfield (tos): ECN bits must be 0"); + goto errout; + } + /* IPv4 currently doesn't handle high order DSCP bits correctly */ + if (frh->tos & ~IPTOS_TOS_MASK) { + NL_SET_ERR_MSG(extack, "Invalid tos"); + goto errout; + } + rule4->dscp = inet_dsfield_to_dscp(frh->tos); + + if (tb[FRA_DSCP] && + fib4_nl2rule_dscp(tb[FRA_DSCP], rule4, extack) < 0) + goto errout; + + if (tb[FRA_DSCP_MASK] && + fib4_nl2rule_dscp_mask(tb[FRA_DSCP_MASK], rule4, extack) < 0) + goto errout; + + if (!net->ipv4.fib_has_custom_rules) { + /* split local/main if they are not already split */ + err = fib_unmerge(net); + if (err) + goto errout; + } + + if (rule->table == RT_TABLE_UNSPEC && !rule->l3mdev) { + if (rule->action == FR_ACT_TO_TBL) { + struct fib_table *table; + + table = fib_empty_table(net); + if (!table) { + err = -ENOBUFS; + goto errout; + } + + rule->table = table->tb_id; + } + } + + if (frh->src_len) + rule4->src = nla_get_in_addr(tb[FRA_SRC]); + + if (frh->dst_len) + rule4->dst = nla_get_in_addr(tb[FRA_DST]); + +#ifdef CONFIG_IP_ROUTE_CLASSID + if (tb[FRA_FLOW]) { + rule4->tclassid = nla_get_u32(tb[FRA_FLOW]); + if (rule4->tclassid) + atomic_inc(&net->ipv4.fib_num_tclassid_users); + } +#endif + + if (fib_rule_requires_fldissect(rule)) + net->ipv4.fib_rules_require_fldissect++; + + rule4->src_len = frh->src_len; + rule4->srcmask = inet_make_mask(rule4->src_len); + rule4->dst_len = frh->dst_len; + rule4->dstmask = inet_make_mask(rule4->dst_len); + + net->ipv4.fib_has_custom_rules = true; + + err = 0; +errout: + return err; +} + +static void fib4_rule_delete(struct fib_rule *rule) +{ + struct net *net = rule->fr_net; + +#ifdef CONFIG_IP_ROUTE_CLASSID + if (((struct fib4_rule *)rule)->tclassid) + atomic_dec(&net->ipv4.fib_num_tclassid_users); +#endif + + if (net->ipv4.fib_rules_require_fldissect && + fib_rule_requires_fldissect(rule)) + net->ipv4.fib_rules_require_fldissect--; +} + +static int fib4_rule_compare(struct fib_rule *rule, struct fib_rule_hdr *frh, + struct nlattr **tb) +{ + struct fib4_rule *rule4 = (struct fib4_rule *) rule; + + if (frh->src_len && (rule4->src_len != frh->src_len)) + return 0; + + if (frh->dst_len && (rule4->dst_len != frh->dst_len)) + return 0; + + if (frh->tos && + (rule4->dscp_full || + inet_dscp_to_dsfield(rule4->dscp) != frh->tos)) + return 0; + + if (tb[FRA_DSCP]) { + dscp_t dscp; + + dscp = inet_dsfield_to_dscp(nla_get_u8(tb[FRA_DSCP]) << 2); + if (!rule4->dscp_full || rule4->dscp != dscp) + return 0; + } + + if (tb[FRA_DSCP_MASK]) { + dscp_t dscp_mask; + + dscp_mask = inet_dsfield_to_dscp(nla_get_u8(tb[FRA_DSCP_MASK]) << 2); + if (!rule4->dscp_full || rule4->dscp_mask != dscp_mask) + return 0; + } + +#ifdef CONFIG_IP_ROUTE_CLASSID + if (tb[FRA_FLOW] && (rule4->tclassid != nla_get_u32(tb[FRA_FLOW]))) + return 0; +#endif + + if (frh->src_len && (rule4->src != nla_get_in_addr(tb[FRA_SRC]))) + return 0; + + if (frh->dst_len && (rule4->dst != nla_get_in_addr(tb[FRA_DST]))) + return 0; + + return 1; +} + +static int fib4_rule_fill(struct fib_rule *rule, struct sk_buff *skb, + struct fib_rule_hdr *frh) +{ + struct fib4_rule *rule4 = (struct fib4_rule *) rule; + + frh->dst_len = rule4->dst_len; + frh->src_len = rule4->src_len; + + if (rule4->dscp_full) { + frh->tos = 0; + if (nla_put_u8(skb, FRA_DSCP, + inet_dscp_to_dsfield(rule4->dscp) >> 2) || + nla_put_u8(skb, FRA_DSCP_MASK, + inet_dscp_to_dsfield(rule4->dscp_mask) >> 2)) + goto nla_put_failure; + } else { + frh->tos = inet_dscp_to_dsfield(rule4->dscp); + } + + if ((rule4->dst_len && + nla_put_in_addr(skb, FRA_DST, rule4->dst)) || + (rule4->src_len && + nla_put_in_addr(skb, FRA_SRC, rule4->src))) + goto nla_put_failure; +#ifdef CONFIG_IP_ROUTE_CLASSID + if (rule4->tclassid && + nla_put_u32(skb, FRA_FLOW, rule4->tclassid)) + goto nla_put_failure; +#endif + return 0; + +nla_put_failure: + return -ENOBUFS; +} + +static size_t fib4_rule_nlmsg_payload(struct fib_rule *rule) +{ + return nla_total_size(4) /* dst */ + + nla_total_size(4) /* src */ + + nla_total_size(4) /* flow */ + + nla_total_size(1) /* dscp */ + + nla_total_size(1); /* dscp mask */ +} + +static void fib4_rule_flush_cache(struct fib_rules_ops *ops) +{ + rt_cache_flush(ops->fro_net); +} + +static bool fib4_rule_need_rtnl(struct net *net) +{ + return !net->ipv4.fib_has_custom_rules; +} + +static const struct fib_rules_ops __net_initconst fib4_rules_ops_template = { + .family = AF_INET, + .rule_size = sizeof(struct fib4_rule), + .addr_size = sizeof(u32), + .action = fib4_rule_action, + .suppress = fib4_rule_suppress, + .match = fib4_rule_match, + .configure = fib4_rule_configure, + .delete = fib4_rule_delete, + .compare = fib4_rule_compare, + .fill = fib4_rule_fill, + .nlmsg_payload = fib4_rule_nlmsg_payload, + .flush_cache = fib4_rule_flush_cache, + .need_rtnl = fib4_rule_need_rtnl, + .nlgroup = RTNLGRP_IPV4_RULE, + .owner = THIS_MODULE, +}; + +static int fib_default_rules_init(struct fib_rules_ops *ops) +{ + int err; + + err = fib_default_rule_add(ops, 0, RT_TABLE_LOCAL); + if (err < 0) + return err; + err = fib_default_rule_add(ops, 0x7FFE, RT_TABLE_MAIN); + if (err < 0) + return err; + err = fib_default_rule_add(ops, 0x7FFF, RT_TABLE_DEFAULT); + if (err < 0) + return err; + return 0; +} + +int __net_init fib4_rules_init(struct net *net) +{ + int err; + struct fib_rules_ops *ops; + + ops = fib_rules_register(&fib4_rules_ops_template, net); + if (IS_ERR(ops)) + return PTR_ERR(ops); + + err = fib_default_rules_init(ops); + if (err < 0) + goto fail; + net->ipv4.rules_ops = ops; + net->ipv4.fib_has_custom_rules = false; + net->ipv4.fib_rules_require_fldissect = 0; + return 0; + +fail: + /* also cleans all rules already added */ + fib_rules_unregister(ops); + return err; +} + +void __net_exit fib4_rules_exit(struct net *net) +{ + fib_rules_unregister(net->ipv4.rules_ops); +} -- cgit v1.3.1