x86/kprobes: Fix crash when probing CS CALL instructionsgrafted

When using eBPF to probe CS CALL instructions within a function,
a crash can be triggered.

The eBPF tool probes offset 257 of the __hrtimer_run_queues()
function:

<__hrtimer_run_queues+249>:  nopl   0x0(%rax,%rax,1)
<__hrtimer_run_queues+254>:  mov    %r14,%rdi
<__hrtimer_run_queues+257>:  cs call <__x86_indirect_thunk_r12>
<__hrtimer_run_queues+263>:  mov    %eax,%r12d
<__hrtimer_run_queues+266>:  xchg   %ax,%ax
<__hrtimer_run_queues+268>:  mov    %r13,%rdi

Which triggers this crash:

  BUG: unable to handle page fault for address: 00000000000f41c9
  #PF: supervisor write access in kernel mode
  #PF: error_code(0x0002) - not-present page
  PGD 0 P4D 0
  Oops: 0002 [#1] SMP NOPTI
  CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P
  RIP: 0010:__hrtimer_run_queues+0x106/0x230

Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is
at the 6th byte of the above CS CALL instruction. Since the CS CALL
instruction occupies 6 bytes, the exception occurred in the middle of that
call instruction.

The root cause is that when using eBPF tools to probe in the middle of a
function, a kprobe with INT3 is used as the underlying implementation.

During single-step emulation of the original CALL instruction,
int3_emulate_call() assumes that the probed CALL instruction is 5 bytes
long. However, the actual CS-prefixed CALL instruction occupies 6 bytes,
so it constructs an incorrect exception return address. When the CPU
returns from the kprobe handler, the next instruction to be executed is at
the address of the last byte of that CS CALL instruction. Coincidentally,
starting from that address, the CPU fetches and decodes a completely
different instruction, which ultimately triggers a kernel crash.

Fix the issue by using the actual instruction length obtained from
the instruction decoder when constructing the exception return
address, rather than relying on the hardcoded CALL_INSN_SIZE macro.

[ mingo: Refined the changelog ]

Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step")
Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Jinke Han <jinkehan@didiglobal.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Yafang Shao <laoar.shao@gmail.com>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000

3 files changed, 367 insertions, 0 deletions

diff --git a/drivers/dax/hmem/Makefile b/drivers/dax/hmem/Makefile
new file mode 100644
index 000000000..d4c4cd6bc
--- /dev/null
+++ b/drivers/dax/hmem/Makefile
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0
+# device_hmem.o deliberately precedes dax_hmem.o for initcall ordering
+obj-$(CONFIG_DEV_DAX_HMEM_DEVICES) += device_hmem.o
+obj-$(CONFIG_DEV_DAX_HMEM) += dax_hmem.o
+
+device_hmem-y := device.o
+dax_hmem-y := hmem.o
diff --git a/drivers/dax/hmem/device.c b/drivers/dax/hmem/device.c
new file mode 100644
index 000000000..d70359b43
--- /dev/null
+++ b/drivers/dax/hmem/device.c
@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/platform_device.h>
+#include <linux/memregion.h>
+#include <linux/module.h>
+#include <linux/dax.h>
+#include <linux/mm.h>
+#include "../bus.h"
+
+static bool nohmem;
+module_param_named(disable, nohmem, bool, 0444);
+
+static bool platform_initialized;
+static DEFINE_MUTEX(hmem_resource_lock);
+static struct resource hmem_active = {
+	.name = "HMEM devices",
+	.start = 0,
+	.end = -1,
+	.flags = IORESOURCE_MEM,
+};
+
+int walk_hmem_resources(struct device *host, walk_hmem_fn fn)
+{
+	struct resource *res;
+	int rc = 0;
+
+	mutex_lock(&hmem_resource_lock);
+	for (res = hmem_active.child; res; res = res->sibling) {
+		rc = fn(host, (int) res->desc, res);
+		if (rc)
+			break;
+	}
+	mutex_unlock(&hmem_resource_lock);
+	return rc;
+}
+EXPORT_SYMBOL_GPL(walk_hmem_resources);
+
+static void hmem_work(struct work_struct *work)
+{
+	/* place holder until dax_hmem driver attaches */
+}
+
+static struct hmem_platform_device hmem_platform = {
+	.pdev = {
+		.name = "hmem_platform",
+		.id = 0,
+	},
+	.work = __WORK_INITIALIZER(hmem_platform.work, hmem_work),
+};
+
+static void __hmem_register_resource(int target_nid, struct resource *res)
+{
+	struct resource *new;
+	int rc;
+
+	new = __request_region(&hmem_active, res->start, resource_size(res), "",
+			       0);
+	if (!new) {
+		pr_debug("hmem range %pr already active\n", res);
+		return;
+	}
+
+	new->desc = target_nid;
+
+	if (platform_initialized)
+		return;
+
+	rc = platform_device_register(&hmem_platform.pdev);
+	if (rc) {
+		pr_err_once("failed to register device-dax hmem_platform device\n");
+		return;
+	}
+
+	platform_initialized = true;
+}
+
+void hmem_register_resource(int target_nid, struct resource *res)
+{
+	if (nohmem)
+		return;
+
+	mutex_lock(&hmem_resource_lock);
+	__hmem_register_resource(target_nid, res);
+	mutex_unlock(&hmem_resource_lock);
+}
+
+static __init int hmem_register_one(struct resource *res, void *data)
+{
+	hmem_register_resource(phys_to_target_node(res->start), res);
+
+	return 0;
+}
+
+static __init int hmem_init(void)
+{
+	walk_soft_reserve_res(0, -1, NULL, hmem_register_one);
+	return 0;
+}
+
+/*
+ * As this is a fallback for address ranges unclaimed by the ACPI HMAT
+ * parsing it must be at an initcall level greater than hmat_init().
+ */
+device_initcall(hmem_init);
diff --git a/drivers/dax/hmem/hmem.c b/drivers/dax/hmem/hmem.c
new file mode 100644
index 000000000..af21f66bf
--- /dev/null
+++ b/drivers/dax/hmem/hmem.c
@@ -0,0 +1,257 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/platform_device.h>
+#include <linux/memregion.h>
+#include <linux/module.h>
+#include <linux/dax.h>
+#include "../../cxl/cxl.h"
+#include "../bus.h"
+
+static bool region_idle;
+module_param_named(region_idle, region_idle, bool, 0644);
+
+static int dax_hmem_probe(struct platform_device *pdev)
+{
+	unsigned long flags = IORESOURCE_DAX_KMEM;
+	struct device *dev = &pdev->dev;
+	struct dax_region *dax_region;
+	struct memregion_info *mri;
+	struct dev_dax_data data;
+
+	/*
+	 * @region_idle == true indicates that an administrative agent
+	 * wants to manipulate the range partitioning before the devices
+	 * are created, so do not send them to the dax_kmem driver by
+	 * default.
+	 */
+	if (region_idle)
+		flags = 0;
+
+	mri = dev->platform_data;
+	dax_region = alloc_dax_region(dev, pdev->id, &mri->range,
+				      mri->target_node, PMD_SIZE, flags);
+	if (!dax_region)
+		return -ENOMEM;
+
+	data = (struct dev_dax_data) {
+		.dax_region = dax_region,
+		.id = -1,
+		.size = region_idle ? 0 : range_len(&mri->range),
+		.memmap_on_memory = false,
+	};
+
+	return PTR_ERR_OR_ZERO(devm_create_dev_dax(&data));
+}
+
+static struct platform_driver dax_hmem_driver = {
+	.probe = dax_hmem_probe,
+	.driver = {
+		.name = "hmem",
+	},
+};
+
+static void release_memregion(void *data)
+{
+	memregion_free((long) data);
+}
+
+static void release_hmem(void *pdev)
+{
+	platform_device_unregister(pdev);
+}
+
+static struct workqueue_struct *dax_hmem_wq;
+
+void dax_hmem_flush_work(void)
+{
+	flush_workqueue(dax_hmem_wq);
+}
+EXPORT_SYMBOL_FOR_MODULES(dax_hmem_flush_work, "dax_cxl");
+
+static int __hmem_register_device(struct device *host, int target_nid,
+				  const struct resource *res)
+{
+	struct platform_device *pdev;
+	struct memregion_info info;
+	long id;
+	int rc;
+
+	rc = region_intersects_soft_reserve(res->start, resource_size(res));
+	if (rc != REGION_INTERSECTS)
+		return 0;
+
+	/* TODO: Add Soft-Reserved memory back to iomem */
+
+	id = memregion_alloc(GFP_KERNEL);
+	if (id < 0) {
+		dev_err(host, "memregion allocation failure for %pr\n", res);
+		return -ENOMEM;
+	}
+	rc = devm_add_action_or_reset(host, release_memregion, (void *) id);
+	if (rc)
+		return rc;
+
+	pdev = platform_device_alloc("hmem", id);
+	if (!pdev) {
+		dev_err(host, "device allocation failure for %pr\n", res);
+		return -ENOMEM;
+	}
+
+	pdev->dev.parent = host;
+	pdev->dev.numa_node = numa_map_to_online_node(target_nid);
+	info = (struct memregion_info) {
+		.target_node = target_nid,
+		.range = {
+			.start = res->start,
+			.end = res->end,
+		},
+	};
+	rc = platform_device_add_data(pdev, &info, sizeof(info));
+	if (rc < 0) {
+		dev_err(host, "memregion_info allocation failure for %pr\n",
+		       res);
+		goto out_put;
+	}
+
+	rc = platform_device_add(pdev);
+	if (rc < 0) {
+		dev_err(host, "%s add failed for %pr\n", dev_name(&pdev->dev),
+			res);
+		goto out_put;
+	}
+
+	return devm_add_action_or_reset(host, release_hmem, pdev);
+
+out_put:
+	platform_device_put(pdev);
+	return rc;
+}
+
+static int hmem_register_cxl_device(struct device *host, int target_nid,
+				    const struct resource *res)
+{
+	if (region_intersects(res->start, resource_size(res), IORESOURCE_MEM,
+			      IORES_DESC_CXL) == REGION_DISJOINT)
+		return 0;
+
+	if (cxl_region_contains_resource(res)) {
+		dev_dbg(host, "CXL claims resource, dropping: %pr\n", res);
+		return 0;
+	}
+
+	dev_dbg(host, "CXL did not claim resource, registering: %pr\n", res);
+	return __hmem_register_device(host, target_nid, res);
+}
+
+static void process_defer_work(struct work_struct *w)
+{
+	struct hmem_platform_device *hpdev = container_of(w, typeof(*hpdev), work);
+	struct device *dev = &hpdev->pdev.dev;
+
+	/* Relies on cxl_acpi and cxl_pci having had a chance to load */
+	wait_for_device_probe();
+
+	guard(device)(dev);
+	if (!dev->driver)
+		goto out;
+
+	if (!hpdev->did_probe) {
+		hpdev->did_probe = true;
+		walk_hmem_resources(dev, hmem_register_cxl_device);
+	}
+out:
+	put_device(dev);
+}
+
+static int hmem_register_device(struct device *host, int target_nid,
+				const struct resource *res)
+{
+	struct platform_device *pdev = to_platform_device(host);
+	struct hmem_platform_device *hpdev = to_hmem_platform_device(pdev);
+
+	if (IS_ENABLED(CONFIG_DEV_DAX_CXL) &&
+	    region_intersects(res->start, resource_size(res), IORESOURCE_MEM,
+			      IORES_DESC_CXL) != REGION_DISJOINT) {
+		if (!hpdev->did_probe) {
+			dev_dbg(host, "await CXL initial probe: %pr\n", res);
+			hpdev->work.func = process_defer_work;
+			get_device(host);
+			if (!queue_work(dax_hmem_wq, &hpdev->work))
+				put_device(host);
+			return 0;
+		}
+		dev_dbg(host, "deferring range to CXL: %pr\n", res);
+		return 0;
+	}
+
+	return __hmem_register_device(host, target_nid, res);
+}
+
+static int dax_hmem_platform_probe(struct platform_device *pdev)
+{
+	struct hmem_platform_device *hpdev = to_hmem_platform_device(pdev);
+
+	/* queue is only flushed on module unload, fail rebind with pending work */
+	if (work_pending(&hpdev->work))
+		return -EBUSY;
+
+	return walk_hmem_resources(&pdev->dev, hmem_register_device);
+}
+
+static struct platform_driver dax_hmem_platform_driver = {
+	.probe = dax_hmem_platform_probe,
+	.driver = {
+		.name = "hmem_platform",
+	},
+};
+
+static __init int dax_hmem_init(void)
+{
+	int rc;
+
+	/*
+	 * Ensure that cxl_acpi and cxl_pci have a chance to kick off
+	 * CXL topology discovery at least once before scanning the
+	 * iomem resource tree for IORES_DESC_CXL resources.
+	 */
+	if (IS_ENABLED(CONFIG_DEV_DAX_CXL)) {
+		request_module("cxl_acpi");
+		request_module("cxl_pci");
+	}
+
+	dax_hmem_wq = alloc_ordered_workqueue("dax_hmem_wq", 0);
+	if (!dax_hmem_wq)
+		return -ENOMEM;
+
+	rc = platform_driver_register(&dax_hmem_platform_driver);
+	if (rc)
+		goto err_platform_driver;
+
+	rc = platform_driver_register(&dax_hmem_driver);
+	if (rc)
+		goto err_driver;
+
+	return 0;
+
+err_driver:
+	platform_driver_unregister(&dax_hmem_platform_driver);
+err_platform_driver:
+	destroy_workqueue(dax_hmem_wq);
+
+	return rc;
+}
+
+static __exit void dax_hmem_exit(void)
+{
+	platform_driver_unregister(&dax_hmem_driver);
+	platform_driver_unregister(&dax_hmem_platform_driver);
+	destroy_workqueue(dax_hmem_wq);
+}
+
+module_init(dax_hmem_init);
+module_exit(dax_hmem_exit);
+
+MODULE_ALIAS("platform:hmem*");
+MODULE_ALIAS("platform:hmem_platform*");
+MODULE_DESCRIPTION("HMEM DAX: direct access to 'specific purpose' memory");
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Intel Corporation");