From 0d6526f82c3cdefcca47f73f5fc08dc6f335eac6 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Mon, 21 Sep 2026 17:37:22 -0700 Subject: sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling bug alb_break_llc() decides whether to break LLC preference during active load balance. It does so by testing that every runnable fair task on the source rq prefers its LLC: env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable But the two counters cover different sets. nr_pref_llc_running is updated in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued, so it follows queued tasks. h_nr_runnable is updated in set_delayed()/ clear_delayed() and drops delay-dequeued tasks. So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks, alb_break_llc() returns false, and active balance is free to pull a task off its preferred LLC. Active balance only moves runnable tasks, and this is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB skips the per-task test in can_migrate_task(). The runnable set is the one we want. Fix it on the counter side. A task should be counted in nr_pref_llc_running exactly while it is both queued on its preferred LLC (pref_llc_queued) and runnable (!sched_delayed). Define that membership once in task_pref_llc_runnable(), and adjust the counter only through pref_llc_running_inc()/pref_llc_running_dec() from the four sites that change either input: account_llc_enqueue(), account_llc_dequeue(), set_delayed() and clear_delayed(). Gating every update on the same predicate keeps the delay, wake and dequeue paths from double-counting or underflowing; see the comments at those sites for the ordering. nr_llc_running and sd->llc_counts are not touched and stay on queued semantics. Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC") Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/ Reported-by: Zhan Xusheng Suggested-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Kayra Cizmeci Cc: # v7.2.x Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com --- net/sched/cls_basic.c | 348 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 348 insertions(+) create mode 100644 net/sched/cls_basic.c (limited to 'net/sched/cls_basic.c') diff --git a/net/sched/cls_basic.c b/net/sched/cls_basic.c new file mode 100644 index 000000000..e2a94ba9f --- /dev/null +++ b/net/sched/cls_basic.c @@ -0,0 +1,348 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * net/sched/cls_basic.c Basic Packet Classifier. + * + * Authors: Thomas Graf + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct basic_head { + struct list_head flist; + struct idr handle_idr; + struct rcu_head rcu; +}; + +struct basic_filter { + u32 handle; + struct tcf_exts exts; + struct tcf_ematch_tree ematches; + struct tcf_result res; + struct tcf_proto *tp; + struct list_head link; + struct tc_basic_pcnt __percpu *pf; + struct rcu_work rwork; +}; + +TC_INDIRECT_SCOPE int basic_classify(struct sk_buff *skb, + const struct tcf_proto *tp, + struct tcf_result *res) +{ + int r; + struct basic_head *head = rcu_dereference_bh(tp->root); + struct basic_filter *f; + + list_for_each_entry_rcu(f, &head->flist, link) { + __this_cpu_inc(f->pf->rcnt); + if (!tcf_em_tree_match(skb, &f->ematches, NULL)) + continue; + __this_cpu_inc(f->pf->rhit); + *res = f->res; + r = tcf_exts_exec(skb, &f->exts, res); + if (r < 0) + continue; + return r; + } + return -1; +} + +static void *basic_get(struct tcf_proto *tp, u32 handle) +{ + struct basic_head *head = rtnl_dereference(tp->root); + struct basic_filter *f; + + list_for_each_entry(f, &head->flist, link) { + if (f->handle == handle) { + return f; + } + } + + return NULL; +} + +static int basic_init(struct tcf_proto *tp) +{ + struct basic_head *head; + + head = kzalloc_obj(*head); + if (head == NULL) + return -ENOBUFS; + INIT_LIST_HEAD(&head->flist); + idr_init(&head->handle_idr); + rcu_assign_pointer(tp->root, head); + return 0; +} + +static void __basic_delete_filter(struct basic_filter *f) +{ + tcf_exts_destroy(&f->exts); + tcf_em_tree_destroy(&f->ematches); + tcf_exts_put_net(&f->exts); + free_percpu(f->pf); + kfree(f); +} + +static void basic_delete_filter_work(struct work_struct *work) +{ + struct basic_filter *f = container_of(to_rcu_work(work), + struct basic_filter, + rwork); + rtnl_lock(); + __basic_delete_filter(f); + rtnl_unlock(); +} + +static void basic_destroy(struct tcf_proto *tp, bool rtnl_held, + struct netlink_ext_ack *extack) +{ + struct basic_head *head = rtnl_dereference(tp->root); + struct basic_filter *f, *n; + + list_for_each_entry_safe(f, n, &head->flist, link) { + list_del_rcu(&f->link); + tcf_unbind_filter(tp, &f->res); + idr_remove(&head->handle_idr, f->handle); + if (tcf_exts_get_net(&f->exts)) + tcf_queue_work(&f->rwork, basic_delete_filter_work); + else + __basic_delete_filter(f); + } + idr_destroy(&head->handle_idr); + kfree_rcu(head, rcu); +} + +static int basic_delete(struct tcf_proto *tp, void *arg, bool *last, + bool rtnl_held, struct netlink_ext_ack *extack) +{ + struct basic_head *head = rtnl_dereference(tp->root); + struct basic_filter *f = arg; + + list_del_rcu(&f->link); + tcf_unbind_filter(tp, &f->res); + idr_remove(&head->handle_idr, f->handle); + tcf_exts_get_net(&f->exts); + tcf_queue_work(&f->rwork, basic_delete_filter_work); + *last = list_empty(&head->flist); + return 0; +} + +static const struct nla_policy basic_policy[TCA_BASIC_MAX + 1] = { + [TCA_BASIC_CLASSID] = { .type = NLA_U32 }, + [TCA_BASIC_EMATCHES] = { .type = NLA_NESTED }, +}; + +static int basic_set_parms(struct net *net, struct tcf_proto *tp, + struct basic_filter *f, unsigned long base, + struct nlattr **tb, + struct nlattr *est, u32 flags, + struct netlink_ext_ack *extack) +{ + int err; + + err = tcf_exts_validate(net, tp, tb, est, &f->exts, flags, extack); + if (err < 0) + return err; + + err = tcf_em_tree_validate(tp, tb[TCA_BASIC_EMATCHES], &f->ematches); + if (err < 0) + return err; + + if (tb[TCA_BASIC_CLASSID]) { + f->res.classid = nla_get_u32(tb[TCA_BASIC_CLASSID]); + tcf_bind_filter(tp, &f->res, base); + } + + f->tp = tp; + return 0; +} + +static int basic_change(struct net *net, struct sk_buff *in_skb, + struct tcf_proto *tp, unsigned long base, u32 handle, + struct nlattr **tca, void **arg, + u32 flags, struct netlink_ext_ack *extack) +{ + int err; + struct basic_head *head = rtnl_dereference(tp->root); + struct nlattr *tb[TCA_BASIC_MAX + 1]; + struct basic_filter *fold = (struct basic_filter *) *arg; + struct basic_filter *fnew; + + if (tca[TCA_OPTIONS] == NULL) + return -EINVAL; + + err = nla_parse_nested_deprecated(tb, TCA_BASIC_MAX, tca[TCA_OPTIONS], + basic_policy, NULL); + if (err < 0) + return err; + + if (fold != NULL) { + if (handle && fold->handle != handle) + return -EINVAL; + } + + fnew = kzalloc_obj(*fnew, GFP_KERNEL_ACCOUNT); + if (!fnew) + return -ENOBUFS; + + err = tcf_exts_init(&fnew->exts, net, TCA_BASIC_ACT, TCA_BASIC_POLICE); + if (err < 0) + goto errout; + + if (!handle) { + handle = 1; + err = idr_alloc_u32(&head->handle_idr, fnew, &handle, + INT_MAX, GFP_KERNEL); + } else if (!fold) { + err = idr_alloc_u32(&head->handle_idr, fnew, &handle, + handle, GFP_KERNEL); + } + if (err) + goto errout; + fnew->handle = handle; + fnew->pf = alloc_percpu_gfp(struct tc_basic_pcnt, GFP_KERNEL_ACCOUNT); + if (!fnew->pf) { + err = -ENOMEM; + if (!fold) + idr_remove(&head->handle_idr, fnew->handle); + goto errout; + } + + err = basic_set_parms(net, tp, fnew, base, tb, tca[TCA_RATE], flags, + extack); + if (err < 0) { + if (!fold) + idr_remove(&head->handle_idr, fnew->handle); + goto errout; + } + + *arg = fnew; + + if (fold) { + idr_replace(&head->handle_idr, fnew, fnew->handle); + list_replace_rcu(&fold->link, &fnew->link); + tcf_unbind_filter(tp, &fold->res); + tcf_exts_get_net(&fold->exts); + tcf_queue_work(&fold->rwork, basic_delete_filter_work); + } else { + list_add_rcu(&fnew->link, &head->flist); + } + + return 0; +errout: + free_percpu(fnew->pf); + tcf_exts_destroy(&fnew->exts); + kfree(fnew); + return err; +} + +static void basic_walk(struct tcf_proto *tp, struct tcf_walker *arg, + bool rtnl_held) +{ + struct basic_head *head = rtnl_dereference(tp->root); + struct basic_filter *f; + + list_for_each_entry(f, &head->flist, link) { + if (!tc_cls_stats_dump(tp, arg, f)) + break; + } +} + +static void basic_bind_class(void *fh, u32 classid, unsigned long cl, void *q, + unsigned long base) +{ + struct basic_filter *f = fh; + + tc_cls_bind_class(classid, cl, q, &f->res, base); +} + +static int basic_dump(struct net *net, struct tcf_proto *tp, void *fh, + struct sk_buff *skb, struct tcmsg *t, bool rtnl_held) +{ + struct tc_basic_pcnt gpf = {}; + struct basic_filter *f = fh; + struct nlattr *nest; + int cpu; + + if (f == NULL) + return skb->len; + + t->tcm_handle = f->handle; + + nest = nla_nest_start_noflag(skb, TCA_OPTIONS); + if (nest == NULL) + goto nla_put_failure; + + if (f->res.classid && + nla_put_u32(skb, TCA_BASIC_CLASSID, f->res.classid)) + goto nla_put_failure; + + for_each_possible_cpu(cpu) { + struct tc_basic_pcnt *pf = per_cpu_ptr(f->pf, cpu); + + gpf.rcnt += pf->rcnt; + gpf.rhit += pf->rhit; + } + + if (nla_put_64bit(skb, TCA_BASIC_PCNT, + sizeof(struct tc_basic_pcnt), + &gpf, TCA_BASIC_PAD)) + goto nla_put_failure; + + if (tcf_exts_dump(skb, &f->exts) < 0 || + tcf_em_tree_dump(skb, &f->ematches, TCA_BASIC_EMATCHES) < 0) + goto nla_put_failure; + + nla_nest_end(skb, nest); + + if (tcf_exts_dump_stats(skb, &f->exts) < 0) + goto nla_put_failure; + + return skb->len; + +nla_put_failure: + nla_nest_cancel(skb, nest); + return -1; +} + +static struct tcf_proto_ops cls_basic_ops __read_mostly = { + .kind = "basic", + .classify = basic_classify, + .init = basic_init, + .destroy = basic_destroy, + .get = basic_get, + .change = basic_change, + .delete = basic_delete, + .walk = basic_walk, + .dump = basic_dump, + .bind_class = basic_bind_class, + .owner = THIS_MODULE, +}; +MODULE_ALIAS_NET_CLS("basic"); + +static int __init init_basic(void) +{ + return register_tcf_proto_ops(&cls_basic_ops); +} + +static void __exit exit_basic(void) +{ + unregister_tcf_proto_ops(&cls_basic_ops); +} + +module_init(init_basic) +module_exit(exit_basic) +MODULE_DESCRIPTION("TC basic classifier"); +MODULE_LICENSE("GPL"); -- cgit v1.3.1