From 0d6526f82c3cdefcca47f73f5fc08dc6f335eac6 Mon Sep 17 00:00:00 2001 From: Tim Chen Date: Mon, 21 Sep 2026 17:37:22 -0700 Subject: sched/cache: Keep nr_pref_llc_running in the runnable domain, to fix LLC mis-scheduling bug alb_break_llc() decides whether to break LLC preference during active load balance. It does so by testing that every runnable fair task on the source rq prefers its LLC: env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable But the two counters cover different sets. nr_pref_llc_running is updated in account_llc_enqueue()/account_llc_dequeue(), next to cfs_rq->nr_queued, so it follows queued tasks. h_nr_runnable is updated in set_delayed()/ clear_delayed() and drops delay-dequeued tasks. So under DELAY_DEQUEUE, a preferring task that goes to sleep stays counted in nr_pref_llc_running while h_nr_runnable falls. The equality then breaks, alb_break_llc() returns false, and active balance is free to pull a task off its preferred LLC. Active balance only moves runnable tasks, and this is the only LLC check it consults: once the stopper runs, LBF_ACTIVE_LB skips the per-task test in can_migrate_task(). The runnable set is the one we want. Fix it on the counter side. A task should be counted in nr_pref_llc_running exactly while it is both queued on its preferred LLC (pref_llc_queued) and runnable (!sched_delayed). Define that membership once in task_pref_llc_runnable(), and adjust the counter only through pref_llc_running_inc()/pref_llc_running_dec() from the four sites that change either input: account_llc_enqueue(), account_llc_dequeue(), set_delayed() and clear_delayed(). Gating every update on the same predicate keeps the delay, wake and dequeue paths from double-counting or underflowing; see the comments at those sites for the ordering. nr_llc_running and sd->llc_counts are not touched and stay on queued semantics. Fixes: 714059f79ff0 ("sched/cache: Handle moving single tasks to/from their preferred LLC") Closes: https://lore.kernel.org/lkml/20260827135000.735138-1-zhanxusheng@xiaomi.com/ Reported-by: Zhan Xusheng Suggested-by: Chen Yu Signed-off-by: Tim Chen Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Ingo Molnar Reviewed-by: Kayra Cizmeci Cc: # v7.2.x Link: https://patch.msgid.link/06af61afedac32e6477f57feb4d658f6c411c3af.1790035273.git.tim.c.chen@linux.intel.com --- include/net/switchdev.h | 529 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 529 insertions(+) create mode 100644 include/net/switchdev.h (limited to 'include/net/switchdev.h') diff --git a/include/net/switchdev.h b/include/net/switchdev.h new file mode 100644 index 000000000..ee5007064 --- /dev/null +++ b/include/net/switchdev.h @@ -0,0 +1,529 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * include/net/switchdev.h - Switch device API + * Copyright (c) 2014-2015 Jiri Pirko + * Copyright (c) 2014-2015 Scott Feldman + */ +#ifndef _LINUX_SWITCHDEV_H_ +#define _LINUX_SWITCHDEV_H_ + +#include +#include +#include +#include + +#define SWITCHDEV_F_NO_RECURSE BIT(0) +#define SWITCHDEV_F_SKIP_EOPNOTSUPP BIT(1) +#define SWITCHDEV_F_DEFER BIT(2) +#define SWITCHDEV_F_NO_FOREIGN BIT(3) + +enum switchdev_attr_id { + SWITCHDEV_ATTR_ID_UNDEFINED, + SWITCHDEV_ATTR_ID_PORT_STP_STATE, + SWITCHDEV_ATTR_ID_PORT_MST_STATE, + SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS, + SWITCHDEV_ATTR_ID_PORT_PRE_BRIDGE_FLAGS, + SWITCHDEV_ATTR_ID_PORT_MROUTER, + SWITCHDEV_ATTR_ID_BRIDGE_AGEING_TIME, + SWITCHDEV_ATTR_ID_BRIDGE_VLAN_FILTERING, + SWITCHDEV_ATTR_ID_BRIDGE_VLAN_PROTOCOL, + SWITCHDEV_ATTR_ID_BRIDGE_MC_DISABLED, + SWITCHDEV_ATTR_ID_BRIDGE_MROUTER, + SWITCHDEV_ATTR_ID_BRIDGE_MST, + SWITCHDEV_ATTR_ID_MRP_PORT_ROLE, + SWITCHDEV_ATTR_ID_VLAN_MSTI, +}; + +struct switchdev_mst_state { + u16 msti; + u8 state; +}; + +struct switchdev_brport_flags { + unsigned long val; + unsigned long mask; +}; + +struct switchdev_vlan_msti { + u16 vid; + u16 msti; +}; + +struct switchdev_attr { + struct net_device *orig_dev; + enum switchdev_attr_id id; + u32 flags; + void *complete_priv; + void (*complete)(struct net_device *dev, int err, void *priv); + union { + u8 stp_state; /* PORT_STP_STATE */ + struct switchdev_mst_state mst_state; /* PORT_MST_STATE */ + struct switchdev_brport_flags brport_flags; /* PORT_BRIDGE_FLAGS */ + bool mrouter; /* PORT_MROUTER */ + clock_t ageing_time; /* BRIDGE_AGEING_TIME */ + bool vlan_filtering; /* BRIDGE_VLAN_FILTERING */ + u16 vlan_protocol; /* BRIDGE_VLAN_PROTOCOL */ + bool mst; /* BRIDGE_MST */ + bool mc_disabled; /* MC_DISABLED */ + u8 mrp_port_role; /* MRP_PORT_ROLE */ + struct switchdev_vlan_msti vlan_msti; /* VLAN_MSTI */ + } u; +}; + +enum switchdev_obj_id { + SWITCHDEV_OBJ_ID_UNDEFINED, + SWITCHDEV_OBJ_ID_PORT_VLAN, + SWITCHDEV_OBJ_ID_PORT_MDB, + SWITCHDEV_OBJ_ID_HOST_MDB, + SWITCHDEV_OBJ_ID_MRP, + SWITCHDEV_OBJ_ID_RING_TEST_MRP, + SWITCHDEV_OBJ_ID_RING_ROLE_MRP, + SWITCHDEV_OBJ_ID_RING_STATE_MRP, + SWITCHDEV_OBJ_ID_IN_TEST_MRP, + SWITCHDEV_OBJ_ID_IN_ROLE_MRP, + SWITCHDEV_OBJ_ID_IN_STATE_MRP, +}; + +struct switchdev_obj { + struct list_head list; + struct net_device *orig_dev; + enum switchdev_obj_id id; + u32 flags; + void *complete_priv; + void (*complete)(struct net_device *dev, int err, void *priv); +}; + +/* SWITCHDEV_OBJ_ID_PORT_VLAN */ +struct switchdev_obj_port_vlan { + struct switchdev_obj obj; + u16 flags; + u16 vid; + /* If set, the notifier signifies a change of one of the following + * flags for a VLAN that already exists: + * - BRIDGE_VLAN_INFO_PVID + * - BRIDGE_VLAN_INFO_UNTAGGED + * Entries with BRIDGE_VLAN_INFO_BRENTRY unset are not notified at all. + */ + bool changed; +}; + +#define SWITCHDEV_OBJ_PORT_VLAN(OBJ) \ + container_of((OBJ), struct switchdev_obj_port_vlan, obj) + +/* SWITCHDEV_OBJ_ID_PORT_MDB */ +struct switchdev_obj_port_mdb { + struct switchdev_obj obj; + unsigned char addr[ETH_ALEN]; + u16 vid; +}; + +#define SWITCHDEV_OBJ_PORT_MDB(OBJ) \ + container_of((OBJ), struct switchdev_obj_port_mdb, obj) + + +/* SWITCHDEV_OBJ_ID_MRP */ +struct switchdev_obj_mrp { + struct switchdev_obj obj; + struct net_device *p_port; + struct net_device *s_port; + u32 ring_id; + u16 prio; +}; + +#define SWITCHDEV_OBJ_MRP(OBJ) \ + container_of((OBJ), struct switchdev_obj_mrp, obj) + +/* SWITCHDEV_OBJ_ID_RING_TEST_MRP */ +struct switchdev_obj_ring_test_mrp { + struct switchdev_obj obj; + /* The value is in us and a value of 0 represents to stop */ + u32 interval; + u8 max_miss; + u32 ring_id; + u32 period; + bool monitor; +}; + +#define SWITCHDEV_OBJ_RING_TEST_MRP(OBJ) \ + container_of((OBJ), struct switchdev_obj_ring_test_mrp, obj) + +/* SWICHDEV_OBJ_ID_RING_ROLE_MRP */ +struct switchdev_obj_ring_role_mrp { + struct switchdev_obj obj; + u8 ring_role; + u32 ring_id; + u8 sw_backup; +}; + +#define SWITCHDEV_OBJ_RING_ROLE_MRP(OBJ) \ + container_of((OBJ), struct switchdev_obj_ring_role_mrp, obj) + +struct switchdev_obj_ring_state_mrp { + struct switchdev_obj obj; + u8 ring_state; + u32 ring_id; +}; + +#define SWITCHDEV_OBJ_RING_STATE_MRP(OBJ) \ + container_of((OBJ), struct switchdev_obj_ring_state_mrp, obj) + +/* SWITCHDEV_OBJ_ID_IN_TEST_MRP */ +struct switchdev_obj_in_test_mrp { + struct switchdev_obj obj; + /* The value is in us and a value of 0 represents to stop */ + u32 interval; + u32 in_id; + u32 period; + u8 max_miss; +}; + +#define SWITCHDEV_OBJ_IN_TEST_MRP(OBJ) \ + container_of((OBJ), struct switchdev_obj_in_test_mrp, obj) + +/* SWICHDEV_OBJ_ID_IN_ROLE_MRP */ +struct switchdev_obj_in_role_mrp { + struct switchdev_obj obj; + struct net_device *i_port; + u32 ring_id; + u16 in_id; + u8 in_role; + u8 sw_backup; +}; + +#define SWITCHDEV_OBJ_IN_ROLE_MRP(OBJ) \ + container_of((OBJ), struct switchdev_obj_in_role_mrp, obj) + +struct switchdev_obj_in_state_mrp { + struct switchdev_obj obj; + u32 in_id; + u8 in_state; +}; + +#define SWITCHDEV_OBJ_IN_STATE_MRP(OBJ) \ + container_of((OBJ), struct switchdev_obj_in_state_mrp, obj) + +struct switchdev_brport { + struct net_device *dev; + const void *ctx; + struct notifier_block *atomic_nb; + struct notifier_block *blocking_nb; + bool tx_fwd_offload; +}; + +enum switchdev_notifier_type { + SWITCHDEV_FDB_ADD_TO_BRIDGE = 1, + SWITCHDEV_FDB_DEL_TO_BRIDGE, + SWITCHDEV_FDB_ADD_TO_DEVICE, + SWITCHDEV_FDB_DEL_TO_DEVICE, + SWITCHDEV_FDB_OFFLOADED, + SWITCHDEV_FDB_FLUSH_TO_BRIDGE, + + SWITCHDEV_PORT_OBJ_ADD, /* Blocking. */ + SWITCHDEV_PORT_OBJ_DEL, /* Blocking. */ + SWITCHDEV_PORT_ATTR_SET, /* May be blocking . */ + + SWITCHDEV_VXLAN_FDB_ADD_TO_BRIDGE, + SWITCHDEV_VXLAN_FDB_DEL_TO_BRIDGE, + SWITCHDEV_VXLAN_FDB_ADD_TO_DEVICE, + SWITCHDEV_VXLAN_FDB_DEL_TO_DEVICE, + SWITCHDEV_VXLAN_FDB_OFFLOADED, + + SWITCHDEV_BRPORT_OFFLOADED, + SWITCHDEV_BRPORT_UNOFFLOADED, + SWITCHDEV_BRPORT_REPLAY, +}; + +struct switchdev_notifier_info { + struct net_device *dev; + struct netlink_ext_ack *extack; + const void *ctx; +}; + +/* Remember to update br_switchdev_fdb_populate() when adding + * new members to this structure + */ +struct switchdev_notifier_fdb_info { + struct switchdev_notifier_info info; /* must be first */ + const unsigned char *addr; + u16 vid; + u8 added_by_user:1, + is_local:1, + locked:1, + offloaded:1; +}; + +struct switchdev_notifier_port_obj_info { + struct switchdev_notifier_info info; /* must be first */ + const struct switchdev_obj *obj; + bool handled; +}; + +struct switchdev_notifier_port_attr_info { + struct switchdev_notifier_info info; /* must be first */ + const struct switchdev_attr *attr; + bool handled; +}; + +struct switchdev_notifier_brport_info { + struct switchdev_notifier_info info; /* must be first */ + const struct switchdev_brport brport; +}; + +static inline struct net_device * +switchdev_notifier_info_to_dev(const struct switchdev_notifier_info *info) +{ + return info->dev; +} + +static inline struct netlink_ext_ack * +switchdev_notifier_info_to_extack(const struct switchdev_notifier_info *info) +{ + return info->extack; +} + +static inline bool +switchdev_fdb_is_dynamically_learned(const struct switchdev_notifier_fdb_info *fdb_info) +{ + return !fdb_info->added_by_user && !fdb_info->is_local; +} + +#ifdef CONFIG_NET_SWITCHDEV + +int switchdev_bridge_port_offload(struct net_device *brport_dev, + struct net_device *dev, const void *ctx, + struct notifier_block *atomic_nb, + struct notifier_block *blocking_nb, + bool tx_fwd_offload, + struct netlink_ext_ack *extack); +void switchdev_bridge_port_unoffload(struct net_device *brport_dev, + const void *ctx, + struct notifier_block *atomic_nb, + struct notifier_block *blocking_nb); +int switchdev_bridge_port_replay(struct net_device *brport_dev, + struct net_device *dev, const void *ctx, + struct notifier_block *atomic_nb, + struct notifier_block *blocking_nb, + struct netlink_ext_ack *extack); + +void switchdev_deferred_process(void); +int switchdev_port_attr_set(struct net_device *dev, + const struct switchdev_attr *attr, + struct netlink_ext_ack *extack); +bool switchdev_port_obj_act_is_deferred(struct net_device *dev, + enum switchdev_notifier_type nt, + const struct switchdev_obj *obj); +int switchdev_port_obj_add(struct net_device *dev, + const struct switchdev_obj *obj, + struct netlink_ext_ack *extack); +int switchdev_port_obj_del(struct net_device *dev, + const struct switchdev_obj *obj); + +int register_switchdev_notifier(struct notifier_block *nb); +int unregister_switchdev_notifier(struct notifier_block *nb); +int call_switchdev_notifiers(unsigned long val, struct net_device *dev, + struct switchdev_notifier_info *info, + struct netlink_ext_ack *extack); + +int register_switchdev_blocking_notifier(struct notifier_block *nb); +int unregister_switchdev_blocking_notifier(struct notifier_block *nb); +int call_switchdev_blocking_notifiers(unsigned long val, struct net_device *dev, + struct switchdev_notifier_info *info, + struct netlink_ext_ack *extack); + +int switchdev_handle_fdb_event_to_device(struct net_device *dev, unsigned long event, + const struct switchdev_notifier_fdb_info *fdb_info, + bool (*check_cb)(const struct net_device *dev), + bool (*foreign_dev_check_cb)(const struct net_device *dev, + const struct net_device *foreign_dev), + int (*mod_cb)(struct net_device *dev, struct net_device *orig_dev, + unsigned long event, const void *ctx, + const struct switchdev_notifier_fdb_info *fdb_info)); + +int switchdev_handle_port_obj_add(struct net_device *dev, + struct switchdev_notifier_port_obj_info *port_obj_info, + bool (*check_cb)(const struct net_device *dev), + int (*add_cb)(struct net_device *dev, const void *ctx, + const struct switchdev_obj *obj, + struct netlink_ext_ack *extack)); +int switchdev_handle_port_obj_add_foreign(struct net_device *dev, + struct switchdev_notifier_port_obj_info *port_obj_info, + bool (*check_cb)(const struct net_device *dev), + bool (*foreign_dev_check_cb)(const struct net_device *dev, + const struct net_device *foreign_dev), + int (*add_cb)(struct net_device *dev, const void *ctx, + const struct switchdev_obj *obj, + struct netlink_ext_ack *extack)); +int switchdev_handle_port_obj_del(struct net_device *dev, + struct switchdev_notifier_port_obj_info *port_obj_info, + bool (*check_cb)(const struct net_device *dev), + int (*del_cb)(struct net_device *dev, const void *ctx, + const struct switchdev_obj *obj)); +int switchdev_handle_port_obj_del_foreign(struct net_device *dev, + struct switchdev_notifier_port_obj_info *port_obj_info, + bool (*check_cb)(const struct net_device *dev), + bool (*foreign_dev_check_cb)(const struct net_device *dev, + const struct net_device *foreign_dev), + int (*del_cb)(struct net_device *dev, const void *ctx, + const struct switchdev_obj *obj)); + +int switchdev_handle_port_attr_set(struct net_device *dev, + struct switchdev_notifier_port_attr_info *port_attr_info, + bool (*check_cb)(const struct net_device *dev), + int (*set_cb)(struct net_device *dev, const void *ctx, + const struct switchdev_attr *attr, + struct netlink_ext_ack *extack)); +#else + +static inline int +switchdev_bridge_port_offload(struct net_device *brport_dev, + struct net_device *dev, const void *ctx, + struct notifier_block *atomic_nb, + struct notifier_block *blocking_nb, + bool tx_fwd_offload, + struct netlink_ext_ack *extack) +{ + return -EOPNOTSUPP; +} + +static inline void +switchdev_bridge_port_unoffload(struct net_device *brport_dev, + const void *ctx, + struct notifier_block *atomic_nb, + struct notifier_block *blocking_nb) +{ +} + +static inline void switchdev_deferred_process(void) +{ +} + +static inline int switchdev_port_attr_set(struct net_device *dev, + const struct switchdev_attr *attr, + struct netlink_ext_ack *extack) +{ + return -EOPNOTSUPP; +} + +static inline int switchdev_port_obj_add(struct net_device *dev, + const struct switchdev_obj *obj, + struct netlink_ext_ack *extack) +{ + return -EOPNOTSUPP; +} + +static inline int switchdev_port_obj_del(struct net_device *dev, + const struct switchdev_obj *obj) +{ + return -EOPNOTSUPP; +} + +static inline int register_switchdev_notifier(struct notifier_block *nb) +{ + return 0; +} + +static inline int unregister_switchdev_notifier(struct notifier_block *nb) +{ + return 0; +} + +static inline int call_switchdev_notifiers(unsigned long val, + struct net_device *dev, + struct switchdev_notifier_info *info, + struct netlink_ext_ack *extack) +{ + return NOTIFY_DONE; +} + +static inline int +register_switchdev_blocking_notifier(struct notifier_block *nb) +{ + return 0; +} + +static inline int +unregister_switchdev_blocking_notifier(struct notifier_block *nb) +{ + return 0; +} + +static inline int +call_switchdev_blocking_notifiers(unsigned long val, + struct net_device *dev, + struct switchdev_notifier_info *info, + struct netlink_ext_ack *extack) +{ + return NOTIFY_DONE; +} + +static inline int +switchdev_handle_fdb_event_to_device(struct net_device *dev, unsigned long event, + const struct switchdev_notifier_fdb_info *fdb_info, + bool (*check_cb)(const struct net_device *dev), + bool (*foreign_dev_check_cb)(const struct net_device *dev, + const struct net_device *foreign_dev), + int (*mod_cb)(struct net_device *dev, struct net_device *orig_dev, + unsigned long event, const void *ctx, + const struct switchdev_notifier_fdb_info *fdb_info)) +{ + return 0; +} + +static inline int +switchdev_handle_port_obj_add(struct net_device *dev, + struct switchdev_notifier_port_obj_info *port_obj_info, + bool (*check_cb)(const struct net_device *dev), + int (*add_cb)(struct net_device *dev, const void *ctx, + const struct switchdev_obj *obj, + struct netlink_ext_ack *extack)) +{ + return 0; +} + +static inline int switchdev_handle_port_obj_add_foreign(struct net_device *dev, + struct switchdev_notifier_port_obj_info *port_obj_info, + bool (*check_cb)(const struct net_device *dev), + bool (*foreign_dev_check_cb)(const struct net_device *dev, + const struct net_device *foreign_dev), + int (*add_cb)(struct net_device *dev, const void *ctx, + const struct switchdev_obj *obj, + struct netlink_ext_ack *extack)) +{ + return 0; +} + +static inline int +switchdev_handle_port_obj_del(struct net_device *dev, + struct switchdev_notifier_port_obj_info *port_obj_info, + bool (*check_cb)(const struct net_device *dev), + int (*del_cb)(struct net_device *dev, const void *ctx, + const struct switchdev_obj *obj)) +{ + return 0; +} + +static inline int +switchdev_handle_port_obj_del_foreign(struct net_device *dev, + struct switchdev_notifier_port_obj_info *port_obj_info, + bool (*check_cb)(const struct net_device *dev), + bool (*foreign_dev_check_cb)(const struct net_device *dev, + const struct net_device *foreign_dev), + int (*del_cb)(struct net_device *dev, const void *ctx, + const struct switchdev_obj *obj)) +{ + return 0; +} + +static inline int +switchdev_handle_port_attr_set(struct net_device *dev, + struct switchdev_notifier_port_attr_info *port_attr_info, + bool (*check_cb)(const struct net_device *dev), + int (*set_cb)(struct net_device *dev, const void *ctx, + const struct switchdev_attr *attr, + struct netlink_ext_ack *extack)) +{ + return 0; +} +#endif + +#endif /* _LINUX_SWITCHDEV_H_ */ -- cgit v1.3.1