sched_ext: Maintain an online cid mask in the scheduler arenagrafted

Schedulers on the default cid mapping treat [0, nr_online_cids) as the
online set and restart on hotplug. Schedulers that install their own mapping
with scx_bpf_cid_override() have no way to learn which cids are online: the
count no longer identifies members and the CPU-form cpumask is unusable from
cid programs. This is an obvious hole in the cid API.

Add scx_bpf_online_cmask(), a kernel-maintained cmask in the scheduler's
arena, allocated alongside the per-CPU scratch masks and populated after the
cid mapping is finalized and before ops.init(), for child schedulers too.
The pointer stays valid through ops.exit() with no reference to take. It is
the arena offset as a void pointer, the same form struct_ops arena arguments
arrive in. The verifier types the void return as a scalar for the program's
arena cast.

The mask follows the SCX hotplug notifications: seeded from cpu_active_mask
and updated before ops.cid_online/offline() runs, so it lags cpu_online_mask
only inside a hotplug transition. Updates walk the scheduler list under the
lock that also serializes unlinking. Reads are live, not atomic snapshots.
Root initialization excludes hotplug.

v2: Reworded the getter kerneldoc (Andrea Righi).

Signed-off-by: Tejun Heo <tj@kernel.org>
Reviewed-by: Andrea Righi <arighi@nvidia.com>

1 files changed, 426 insertions, 0 deletions

diff --git a/sound/soc/sof/pm.c b/sound/soc/sof/pm.c
new file mode 100644
index 000000000..dd7cd87f1
--- /dev/null
+++ b/sound/soc/sof/pm.c
@@ -0,0 +1,426 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018 Intel Corporation
+//
+// Author: Liam Girdwood <liam.r.girdwood@linux.intel.com>
+//
+
+#include <linux/module.h>
+#include "ops.h"
+#include "sof-priv.h"
+#include "sof-audio.h"
+
+static int override_on_demand_boot = -1;
+module_param_named(on_demand_boot, override_on_demand_boot, int, 0444);
+MODULE_PARM_DESC(on_demand_boot, "Force on-demand DSP boot: 0 - disabled, 1 - enabled");
+
+/*
+ * Helper function to determine the target DSP state during
+ * system suspend. This function only cares about the device
+ * D-states. Platform-specific substates, if any, should be
+ * handled by the platform-specific parts.
+ */
+static u32 snd_sof_dsp_power_target(struct snd_sof_dev *sdev)
+{
+	u32 target_dsp_state;
+
+	switch (sdev->system_suspend_target) {
+	case SOF_SUSPEND_S5:
+	case SOF_SUSPEND_S4:
+		/* DSP should be in D3 if the system is suspending to S3+ */
+	case SOF_SUSPEND_S3:
+		/* DSP should be in D3 if the system is suspending to S3 */
+		target_dsp_state = SOF_DSP_PM_D3;
+		break;
+	case SOF_SUSPEND_S0IX:
+		/*
+		 * Currently, the only criterion for retaining the DSP in D0
+		 * is that there are streams that ignored the suspend trigger.
+		 * Additional criteria such Soundwire clock-stop mode and
+		 * device suspend latency considerations will be added later.
+		 */
+		if (snd_sof_stream_suspend_ignored(sdev))
+			target_dsp_state = SOF_DSP_PM_D0;
+		else
+			target_dsp_state = SOF_DSP_PM_D3;
+		break;
+	default:
+		/* This case would be during runtime suspend */
+		target_dsp_state = SOF_DSP_PM_D3;
+		break;
+	}
+
+	return target_dsp_state;
+}
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_ENABLE_DEBUGFS_CACHE)
+static void sof_cache_debugfs(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_dfsentry *dfse;
+
+	list_for_each_entry(dfse, &sdev->dfsentry_list, list) {
+
+		/* nothing to do if debugfs buffer is not IO mem */
+		if (dfse->type == SOF_DFSENTRY_TYPE_BUF)
+			continue;
+
+		/* cache memory that is only accessible in D0 */
+		if (dfse->access_type == SOF_DEBUGFS_ACCESS_D0_ONLY)
+			memcpy_fromio(dfse->cache_buf, dfse->io_mem,
+				      dfse->size);
+	}
+}
+#endif
+
+int snd_sof_boot_dsp_firmware(struct snd_sof_dev *sdev)
+{
+	const struct sof_ipc_pm_ops *pm_ops = sof_ipc_get_ops(sdev, pm);
+	const struct sof_ipc_tplg_ops *tplg_ops = sof_ipc_get_ops(sdev, tplg);
+	int ret;
+
+	guard(mutex)(&sdev->dsp_fw_boot_mutex);
+
+	if (sdev->fw_state == SOF_FW_BOOT_COMPLETE) {
+		/* Firmware already booted, just return */
+		return 0;
+	}
+
+	dev_dbg(sdev->dev, "Booting DSP firmware\n");
+
+	sof_set_fw_state(sdev, SOF_FW_BOOT_PREPARE);
+
+	/* load the firmware */
+	ret = snd_sof_load_firmware(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "%s: failed to load DSP firmware: %d\n",
+			__func__, ret);
+		sof_set_fw_state(sdev, SOF_FW_BOOT_FAILED);
+		return ret;
+	}
+
+	sof_set_fw_state(sdev, SOF_FW_BOOT_IN_PROGRESS);
+
+	/*
+	 * Boot the firmware. The FW boot status will be modified
+	 * in snd_sof_run_firmware() depending on the outcome.
+	 */
+	ret = snd_sof_run_firmware(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev, "%s: failed to boot DSP firmware: %d\n",
+			__func__, ret);
+		sof_set_fw_state(sdev, SOF_FW_BOOT_FAILED);
+		return ret;
+	}
+
+	/* resume DMA trace */
+	ret = sof_fw_trace_resume(sdev);
+	if (ret < 0) {
+		/* non fatal */
+		dev_warn(sdev->dev, "%s: failed to resume trace: %d\n",
+			 __func__, ret);
+	}
+
+	/* restore pipelines */
+	if (tplg_ops && tplg_ops->set_up_all_pipelines) {
+		ret = tplg_ops->set_up_all_pipelines(sdev, false);
+		if (ret < 0) {
+			dev_err(sdev->dev, "%s: failed to restore pipeline: %d\n",
+				__func__, ret);
+			goto setup_fail;
+		}
+	}
+
+	/* Notify clients not managed by pm framework about core resume */
+	sof_resume_clients(sdev);
+
+	/* notify DSP of system resume */
+	if (pm_ops && pm_ops->ctx_restore) {
+		ret = pm_ops->ctx_restore(sdev);
+		if (ret < 0)
+			dev_err(sdev->dev, "%s: ctx_restore IPC failed: %d\n",
+				__func__, ret);
+	}
+
+setup_fail:
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_ENABLE_DEBUGFS_CACHE)
+	if (ret < 0) {
+		/*
+		 * Debugfs cannot be read in runtime suspend, so cache
+		 * the contents upon failure. This allows to capture
+		 * possible DSP coredump information.
+		 */
+		sof_cache_debugfs(sdev);
+	}
+#endif
+
+	return ret;
+}
+EXPORT_SYMBOL(snd_sof_boot_dsp_firmware);
+
+static int sof_resume(struct device *dev, bool runtime_resume)
+{
+	struct snd_sof_dev *sdev = dev_get_drvdata(dev);
+	u32 old_state = sdev->dsp_power_state.state;
+	bool on_demand_boot;
+	int ret;
+
+	/* do nothing if dsp resume callbacks are not set */
+	if (!runtime_resume && !sof_ops(sdev)->resume)
+		return 0;
+
+	if (runtime_resume && !sof_ops(sdev)->runtime_resume)
+		return 0;
+
+	/* DSP was never successfully started, nothing to resume */
+	if (sdev->first_boot)
+		return 0;
+
+	/*
+	 * if the runtime_resume flag is set, call the runtime_resume routine
+	 * or else call the system resume routine
+	 */
+	if (runtime_resume)
+		ret = snd_sof_dsp_runtime_resume(sdev);
+	else
+		ret = snd_sof_dsp_resume(sdev);
+	if (ret < 0) {
+		dev_err(sdev->dev,
+			"error: failed to power up DSP after resume\n");
+		return ret;
+	}
+
+	if (sdev->dspless_mode_selected) {
+		sof_set_fw_state(sdev, SOF_DSPLESS_MODE);
+		return 0;
+	}
+
+	/*
+	 * Nothing further to be done for platforms that support the low power
+	 * D0 substate. Resume trace and return when resuming from
+	 * low-power D0 substate
+	 */
+	if (!runtime_resume && sof_ops(sdev)->set_power_state &&
+	    old_state == SOF_DSP_PM_D0) {
+		ret = sof_fw_trace_resume(sdev);
+		if (ret < 0)
+			/* non fatal */
+			dev_warn(sdev->dev,
+				 "failed to enable trace after resume %d\n", ret);
+		return 0;
+	}
+
+	if (override_on_demand_boot > -1)
+		on_demand_boot = override_on_demand_boot ? true : false;
+	else
+		on_demand_boot = sdev->pdata->desc->on_demand_dsp_boot;
+
+	if (on_demand_boot) {
+		/* Only change the fw_state to PREPARE but skip booting */
+		sof_set_fw_state(sdev, SOF_FW_BOOT_PREPARE);
+		return 0;
+	}
+
+	return snd_sof_boot_dsp_firmware(sdev);
+}
+
+static int sof_suspend(struct device *dev, bool runtime_suspend)
+{
+	struct snd_sof_dev *sdev = dev_get_drvdata(dev);
+	const struct sof_ipc_pm_ops *pm_ops = sof_ipc_get_ops(sdev, pm);
+	const struct sof_ipc_tplg_ops *tplg_ops = sof_ipc_get_ops(sdev, tplg);
+	pm_message_t pm_state;
+	u32 target_state = snd_sof_dsp_power_target(sdev);
+	u32 old_state = sdev->dsp_power_state.state;
+	int ret;
+
+	/* do nothing if dsp suspend callback is not set */
+	if (!runtime_suspend && !sof_ops(sdev)->suspend)
+		return 0;
+
+	if (runtime_suspend && !sof_ops(sdev)->runtime_suspend)
+		return 0;
+
+	/* we need to tear down pipelines only if the DSP hardware is
+	 * active, which happens for PCI devices. if the device is
+	 * suspended, it is brought back to full power and then
+	 * suspended again
+	 */
+	if (tplg_ops && tplg_ops->tear_down_all_pipelines && (old_state == SOF_DSP_PM_D0))
+		tplg_ops->tear_down_all_pipelines(sdev, false);
+
+	if (sdev->fw_state != SOF_FW_BOOT_COMPLETE)
+		goto suspend;
+
+	/* prepare for streams to be resumed properly upon resume */
+	if (!runtime_suspend) {
+		ret = snd_sof_dsp_hw_params_upon_resume(sdev);
+		if (ret < 0) {
+			dev_err(sdev->dev,
+				"error: setting hw_params flag during suspend %d\n",
+				ret);
+			return ret;
+		}
+	}
+
+	pm_state.event = target_state;
+
+	/* suspend DMA trace */
+	sof_fw_trace_suspend(sdev, pm_state);
+
+	/* Notify clients not managed by pm framework about core suspend */
+	sof_suspend_clients(sdev, pm_state);
+
+	/* Skip to platform-specific suspend if DSP is entering D0 */
+	if (target_state == SOF_DSP_PM_D0)
+		goto suspend;
+
+#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_ENABLE_DEBUGFS_CACHE)
+	/* cache debugfs contents during runtime suspend */
+	if (runtime_suspend)
+		sof_cache_debugfs(sdev);
+#endif
+	/* notify DSP of upcoming power down */
+	if (pm_ops && pm_ops->ctx_save) {
+		ret = pm_ops->ctx_save(sdev);
+		if (ret == -EBUSY || ret == -EAGAIN) {
+			/*
+			 * runtime PM has logic to handle -EBUSY/-EAGAIN so
+			 * pass these errors up
+			 */
+			dev_err(sdev->dev, "ctx_save IPC error during suspend: %d\n", ret);
+			return ret;
+		} else if (ret < 0) {
+			/* FW in unexpected state, continue to power down */
+			dev_warn(sdev->dev, "ctx_save IPC error: %d, proceeding with suspend\n",
+				 ret);
+		}
+	}
+
+suspend:
+
+	/* return if the DSP was not probed successfully */
+	if (sdev->fw_state == SOF_FW_BOOT_NOT_STARTED)
+		return 0;
+
+	/* platform-specific suspend */
+	if (runtime_suspend)
+		ret = snd_sof_dsp_runtime_suspend(sdev);
+	else
+		ret = snd_sof_dsp_suspend(sdev, target_state);
+	if (ret < 0)
+		dev_err(sdev->dev,
+			"error: failed to power down DSP during suspend %d\n",
+			ret);
+
+	/* Do not reset FW state if DSP is in D0 */
+	if (target_state == SOF_DSP_PM_D0)
+		return ret;
+
+	/* reset FW state */
+	sof_set_fw_state(sdev, SOF_FW_BOOT_NOT_STARTED);
+	sdev->enabled_cores_mask = 0;
+
+	return ret;
+}
+
+int snd_sof_dsp_power_down_notify(struct snd_sof_dev *sdev)
+{
+	const struct sof_ipc_pm_ops *pm_ops = sof_ipc_get_ops(sdev, pm);
+
+	/*
+	 * Notify DSP of upcoming power down only if the firmware has been
+	 * booted up
+	 */
+	if (sdev->fw_state == SOF_FW_BOOT_COMPLETE && sof_ops(sdev)->remove &&
+	    pm_ops && pm_ops->ctx_save)
+		return pm_ops->ctx_save(sdev);
+
+	return 0;
+}
+
+int snd_sof_runtime_suspend(struct device *dev)
+{
+	return sof_suspend(dev, true);
+}
+EXPORT_SYMBOL(snd_sof_runtime_suspend);
+
+int snd_sof_runtime_idle(struct device *dev)
+{
+	struct snd_sof_dev *sdev = dev_get_drvdata(dev);
+
+	return snd_sof_dsp_runtime_idle(sdev);
+}
+EXPORT_SYMBOL(snd_sof_runtime_idle);
+
+int snd_sof_runtime_resume(struct device *dev)
+{
+	return sof_resume(dev, true);
+}
+EXPORT_SYMBOL(snd_sof_runtime_resume);
+
+int snd_sof_resume(struct device *dev)
+{
+	return sof_resume(dev, false);
+}
+EXPORT_SYMBOL(snd_sof_resume);
+
+int snd_sof_suspend(struct device *dev)
+{
+	return sof_suspend(dev, false);
+}
+EXPORT_SYMBOL(snd_sof_suspend);
+
+int snd_sof_prepare(struct device *dev)
+{
+	struct snd_sof_dev *sdev = dev_get_drvdata(dev);
+	const struct sof_dev_desc *desc = sdev->pdata->desc;
+
+	/* will suspend to S3 by default */
+	sdev->system_suspend_target = SOF_SUSPEND_S3;
+
+	/*
+	 * if the firmware is crashed or boot failed then we try to aim for S3
+	 * to reboot the firmware
+	 */
+	if (sdev->fw_state == SOF_FW_CRASHED ||
+	    sdev->fw_state == SOF_FW_BOOT_FAILED)
+		return 0;
+
+	if (!desc->use_acpi_target_states)
+		return 0;
+
+#if defined(CONFIG_ACPI)
+	switch (acpi_target_system_state()) {
+	case ACPI_STATE_S0:
+		sdev->system_suspend_target = SOF_SUSPEND_S0IX;
+		break;
+	case ACPI_STATE_S1:
+	case ACPI_STATE_S2:
+	case ACPI_STATE_S3:
+		sdev->system_suspend_target = SOF_SUSPEND_S3;
+		break;
+	case ACPI_STATE_S4:
+		sdev->system_suspend_target = SOF_SUSPEND_S4;
+		break;
+	case ACPI_STATE_S5:
+		sdev->system_suspend_target = SOF_SUSPEND_S5;
+		break;
+	default:
+		break;
+	}
+#endif
+
+	return 0;
+}
+EXPORT_SYMBOL(snd_sof_prepare);
+
+void snd_sof_complete(struct device *dev)
+{
+	struct snd_sof_dev *sdev = dev_get_drvdata(dev);
+
+	sdev->system_suspend_target = SOF_SUSPEND_NONE;
+}
+EXPORT_SYMBOL(snd_sof_complete);