From 0d6526f82c3cdefcca47f73f5fc08dc6f335eac6 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Mon, 21 Sep 2026 17:37:22 -0700 Subject: sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling bug alb_break_llc() decides whether to break LLC preference during active load balance. It does so by testing that every runnable fair task on the source rq prefers its LLC: env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable But the two counters cover different sets. nr_pref_llc_running is updated in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued, so it follows queued tasks. h_nr_runnable is updated in set_delayed()/ clear_delayed() and drops delay-dequeued tasks. So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks, alb_break_llc() returns false, and active balance is free to pull a task off its preferred LLC. Active balance only moves runnable tasks, and this is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB skips the per-task test in can_migrate_task(). The runnable set is the one we want. Fix it on the counter side. A task should be counted in nr_pref_llc_running exactly while it is both queued on its preferred LLC (pref_llc_queued) and runnable (!sched_delayed). Define that membership once in task_pref_llc_runnable(), and adjust the counter only through pref_llc_running_inc()/pref_llc_running_dec() from the four sites that change either input: account_llc_enqueue(), account_llc_dequeue(), set_delayed() and clear_delayed(). Gating every update on the same predicate keeps the delay, wake and dequeue paths from double-counting or underflowing; see the comments at those sites for the ordering. nr_llc_running and sd->llc_counts are not touched and stay on queued semantics. Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC") Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/ Reported-by: Zhan Xusheng Suggested-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Kayra Cizmeci Cc: # v7.2.x Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com --- drivers/clk/clk-devres.c | 359 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 359 insertions(+) create mode 100644 drivers/clk/clk-devres.c (limited to 'drivers/clk/clk-devres.c') diff --git a/drivers/clk/clk-devres.c b/drivers/clk/clk-devres.c new file mode 100644 index 000000000..4cc3863bd --- /dev/null +++ b/drivers/clk/clk-devres.c @@ -0,0 +1,359 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include + +struct devm_clk_state { + struct clk *clk; + void (*exit)(struct clk *clk); +}; + +static void devm_clk_release(struct device *dev, void *res) +{ + struct devm_clk_state *state = res; + + if (state->exit) + state->exit(state->clk); + + clk_put(state->clk); +} + +static struct clk *__devm_clk_get(struct device *dev, const char *id, + struct clk *(*get)(struct device *dev, const char *id), + int (*init)(struct clk *clk), + void (*exit)(struct clk *clk)) +{ + struct devm_clk_state *state; + struct clk *clk; + int ret; + + state = devres_alloc(devm_clk_release, sizeof(*state), GFP_KERNEL); + if (!state) + return ERR_PTR(-ENOMEM); + + clk = get(dev, id); + if (IS_ERR(clk)) { + ret = PTR_ERR(clk); + goto err_clk_get; + } + + if (init) { + ret = init(clk); + if (ret) + goto err_clk_init; + } + + state->clk = clk; + state->exit = exit; + + devres_add(dev, state); + + return clk; + +err_clk_init: + + clk_put(clk); +err_clk_get: + + devres_free(state); + return ERR_PTR(ret); +} + +struct clk *devm_clk_get(struct device *dev, const char *id) +{ + return __devm_clk_get(dev, id, clk_get, NULL, NULL); +} +EXPORT_SYMBOL(devm_clk_get); + +struct clk *devm_clk_get_prepared(struct device *dev, const char *id) +{ + return __devm_clk_get(dev, id, clk_get, clk_prepare, clk_unprepare); +} +EXPORT_SYMBOL_GPL(devm_clk_get_prepared); + +struct clk *devm_clk_get_enabled(struct device *dev, const char *id) +{ + return __devm_clk_get(dev, id, clk_get, + clk_prepare_enable, clk_disable_unprepare); +} +EXPORT_SYMBOL_GPL(devm_clk_get_enabled); + +struct clk *devm_clk_get_optional(struct device *dev, const char *id) +{ + return __devm_clk_get(dev, id, clk_get_optional, NULL, NULL); +} +EXPORT_SYMBOL(devm_clk_get_optional); + +struct clk *devm_clk_get_optional_prepared(struct device *dev, const char *id) +{ + return __devm_clk_get(dev, id, clk_get_optional, + clk_prepare, clk_unprepare); +} +EXPORT_SYMBOL_GPL(devm_clk_get_optional_prepared); + +struct clk *devm_clk_get_optional_enabled(struct device *dev, const char *id) +{ + return __devm_clk_get(dev, id, clk_get_optional, + clk_prepare_enable, clk_disable_unprepare); +} +EXPORT_SYMBOL_GPL(devm_clk_get_optional_enabled); + +static void devm_clk_disable_unprepare(void *data) +{ + clk_disable_unprepare(data); +} + +struct clk *devm_clk_get_optional_enabled_with_rate(struct device *dev, + const char *id, + unsigned long rate) +{ + struct clk *clk; + int ret; + + clk = devm_clk_get_optional(dev, id); + if (IS_ERR(clk)) + return ERR_CAST(clk); + + ret = clk_set_rate(clk, rate); + if (ret) + goto out_put_clk; + + ret = clk_prepare_enable(clk); + if (ret) + goto out_put_clk; + + ret = devm_add_action_or_reset(dev, devm_clk_disable_unprepare, clk); + if (ret) + goto out_put_clk; + + return clk; + +out_put_clk: + devm_clk_put(dev, clk); + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(devm_clk_get_optional_enabled_with_rate); + +struct clk_bulk_devres { + struct clk_bulk_data *clks; + int num_clks; +}; + +static void devm_clk_bulk_release(struct device *dev, void *res) +{ + struct clk_bulk_devres *devres = res; + + clk_bulk_put(devres->num_clks, devres->clks); +} + +static int __devm_clk_bulk_get(struct device *dev, int num_clks, + struct clk_bulk_data *clks, bool optional) +{ + struct clk_bulk_devres *devres; + int ret; + + devres = devres_alloc(devm_clk_bulk_release, + sizeof(*devres), GFP_KERNEL); + if (!devres) + return -ENOMEM; + + if (optional) + ret = clk_bulk_get_optional(dev, num_clks, clks); + else + ret = clk_bulk_get(dev, num_clks, clks); + if (!ret) { + devres->clks = clks; + devres->num_clks = num_clks; + devres_add(dev, devres); + } else { + devres_free(devres); + } + + return ret; +} + +int __must_check devm_clk_bulk_get(struct device *dev, int num_clks, + struct clk_bulk_data *clks) +{ + return __devm_clk_bulk_get(dev, num_clks, clks, false); +} +EXPORT_SYMBOL_GPL(devm_clk_bulk_get); + +int __must_check devm_clk_bulk_get_optional(struct device *dev, int num_clks, + struct clk_bulk_data *clks) +{ + return __devm_clk_bulk_get(dev, num_clks, clks, true); +} +EXPORT_SYMBOL_GPL(devm_clk_bulk_get_optional); + +static void devm_clk_bulk_release_enable(struct device *dev, void *res) +{ + struct clk_bulk_devres *devres = res; + + clk_bulk_disable_unprepare(devres->num_clks, devres->clks); + clk_bulk_put(devres->num_clks, devres->clks); +} + +static int __devm_clk_bulk_get_enable(struct device *dev, int num_clks, + struct clk_bulk_data *clks, bool optional) +{ + struct clk_bulk_devres *devres; + int ret; + + devres = devres_alloc(devm_clk_bulk_release_enable, + sizeof(*devres), GFP_KERNEL); + if (!devres) + return -ENOMEM; + + if (optional) + ret = clk_bulk_get_optional(dev, num_clks, clks); + else + ret = clk_bulk_get(dev, num_clks, clks); + if (ret) + goto err_clk_get; + + ret = clk_bulk_prepare_enable(num_clks, clks); + if (ret) + goto err_clk_prepare; + + devres->clks = clks; + devres->num_clks = num_clks; + devres_add(dev, devres); + + return 0; + +err_clk_prepare: + clk_bulk_put(num_clks, clks); +err_clk_get: + devres_free(devres); + return ret; +} + +int __must_check devm_clk_bulk_get_enable(struct device *dev, int num_clks, + struct clk_bulk_data *clks) +{ + return __devm_clk_bulk_get_enable(dev, num_clks, clks, false); +} +EXPORT_SYMBOL_GPL(devm_clk_bulk_get_enable); + +int __must_check devm_clk_bulk_get_optional_enable(struct device *dev, int num_clks, + struct clk_bulk_data *clks) +{ + return __devm_clk_bulk_get_enable(dev, num_clks, clks, true); +} +EXPORT_SYMBOL_GPL(devm_clk_bulk_get_optional_enable); + +static void devm_clk_bulk_release_all(struct device *dev, void *res) +{ + struct clk_bulk_devres *devres = res; + + clk_bulk_put_all(devres->num_clks, devres->clks); +} + +int __must_check devm_clk_bulk_get_all(struct device *dev, + struct clk_bulk_data **clks) +{ + struct clk_bulk_devres *devres; + int ret; + + devres = devres_alloc(devm_clk_bulk_release_all, + sizeof(*devres), GFP_KERNEL); + if (!devres) + return -ENOMEM; + + ret = clk_bulk_get_all(dev, &devres->clks); + if (ret > 0) { + *clks = devres->clks; + devres->num_clks = ret; + devres_add(dev, devres); + } else { + devres_free(devres); + } + + return ret; +} +EXPORT_SYMBOL_GPL(devm_clk_bulk_get_all); + +static void devm_clk_bulk_release_all_enable(struct device *dev, void *res) +{ + struct clk_bulk_devres *devres = res; + + clk_bulk_disable_unprepare(devres->num_clks, devres->clks); + clk_bulk_put_all(devres->num_clks, devres->clks); +} + +int __must_check devm_clk_bulk_get_all_enabled(struct device *dev, + struct clk_bulk_data **clks) +{ + struct clk_bulk_devres *devres; + int ret; + + devres = devres_alloc(devm_clk_bulk_release_all_enable, + sizeof(*devres), GFP_KERNEL); + if (!devres) + return -ENOMEM; + + ret = clk_bulk_get_all(dev, &devres->clks); + if (ret > 0) { + *clks = devres->clks; + devres->num_clks = ret; + } else { + devres_free(devres); + return ret; + } + + ret = clk_bulk_prepare_enable(devres->num_clks, *clks); + if (!ret) { + devres_add(dev, devres); + } else { + clk_bulk_put_all(devres->num_clks, devres->clks); + devres_free(devres); + return ret; + } + + return devres->num_clks; +} +EXPORT_SYMBOL_GPL(devm_clk_bulk_get_all_enabled); + +static int devm_clk_match(struct device *dev, void *res, void *data) +{ + struct clk **c = res; + if (!c || !*c) { + WARN_ON(!c || !*c); + return 0; + } + return *c == data; +} + +void devm_clk_put(struct device *dev, struct clk *clk) +{ + int ret; + + ret = devres_release(dev, devm_clk_release, devm_clk_match, clk); + + WARN_ON(ret); +} +EXPORT_SYMBOL(devm_clk_put); + +struct clk *devm_get_clk_from_child(struct device *dev, + struct device_node *np, const char *con_id) +{ + struct devm_clk_state *state; + struct clk *clk; + + state = devres_alloc(devm_clk_release, sizeof(*state), GFP_KERNEL); + if (!state) + return ERR_PTR(-ENOMEM); + + clk = of_clk_get_by_name(np, con_id); + if (!IS_ERR(clk)) { + state->clk = clk; + devres_add(dev, state); + } else { + devres_free(state); + } + + return clk; +} +EXPORT_SYMBOL(devm_get_clk_from_child); -- cgit v1.3.1