x86/kprobes: Fix crash when probing CS CALL instructionsgrafted

When using eBPF to probe CS CALL instructions within a function,
a crash can be triggered.

The eBPF tool probes offset 257 of the __hrtimer_run_queues()
function:

<__hrtimer_run_queues+249>:  nopl   0x0(%rax,%rax,1)
<__hrtimer_run_queues+254>:  mov    %r14,%rdi
<__hrtimer_run_queues+257>:  cs call <__x86_indirect_thunk_r12>
<__hrtimer_run_queues+263>:  mov    %eax,%r12d
<__hrtimer_run_queues+266>:  xchg   %ax,%ax
<__hrtimer_run_queues+268>:  mov    %r13,%rdi

Which triggers this crash:

  BUG: unable to handle page fault for address: 00000000000f41c9
  #PF: supervisor write access in kernel mode
  #PF: error_code(0x0002) - not-present page
  PGD 0 P4D 0
  Oops: 0002 [#1] SMP NOPTI
  CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P
  RIP: 0010:__hrtimer_run_queues+0x106/0x230

Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is
at the 6th byte of the above CS CALL instruction. Since the CS CALL
instruction occupies 6 bytes, the exception occurred in the middle of that
call instruction.

The root cause is that when using eBPF tools to probe in the middle of a
function, a kprobe with INT3 is used as the underlying implementation.

During single-step emulation of the original CALL instruction,
int3_emulate_call() assumes that the probed CALL instruction is 5 bytes
long. However, the actual CS-prefixed CALL instruction occupies 6 bytes,
so it constructs an incorrect exception return address. When the CPU
returns from the kprobe handler, the next instruction to be executed is at
the address of the last byte of that CS CALL instruction. Coincidentally,
starting from that address, the CPU fetches and decodes a completely
different instruction, which ultimately triggers a kernel crash.

Fix the issue by using the actual instruction length obtained from
the instruction decoder when constructing the exception return
address, rather than relying on the hardcoded CALL_INSN_SIZE macro.

[ mingo: Refined the changelog ]

Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step")
Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Jinke Han <jinkehan@didiglobal.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Yafang Shao <laoar.shao@gmail.com>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000

1 files changed, 505 insertions, 0 deletions

diff --git a/drivers/base/attribute_container.c b/drivers/base/attribute_container.c
new file mode 100644
index 000000000..3c4563742
--- /dev/null
+++ b/drivers/base/attribute_container.c
@@ -0,0 +1,505 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * attribute_container.c - implementation of a simple container for classes
+ *
+ * Copyright (c) 2005 - James Bottomley <James.Bottomley@steeleye.com>
+ *
+ * The basic idea here is to enable a device to be attached to an
+ * aritrary numer of classes without having to allocate storage for them.
+ * Instead, the contained classes select the devices they need to attach
+ * to via a matching function.
+ */
+
+#include <linux/attribute_container.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+
+#include "base.h"
+
+/* This is a private structure used to tie the classdev and the
+ * container .. it should never be visible outside this file */
+struct internal_container {
+	struct klist_node node;
+	struct attribute_container *cont;
+	struct device classdev;
+};
+
+static void internal_container_klist_get(struct klist_node *n)
+{
+	struct internal_container *ic =
+		container_of(n, struct internal_container, node);
+	get_device(&ic->classdev);
+}
+
+static void internal_container_klist_put(struct klist_node *n)
+{
+	struct internal_container *ic =
+		container_of(n, struct internal_container, node);
+	put_device(&ic->classdev);
+}
+
+
+/**
+ * attribute_container_classdev_to_container - given a classdev, return the container
+ *
+ * @classdev: the class device created by attribute_container_add_device.
+ *
+ * Returns the container associated with this classdev.
+ */
+struct attribute_container *
+attribute_container_classdev_to_container(struct device *classdev)
+{
+	struct internal_container *ic =
+		container_of(classdev, struct internal_container, classdev);
+	return ic->cont;
+}
+EXPORT_SYMBOL_GPL(attribute_container_classdev_to_container);
+
+static LIST_HEAD(attribute_container_list);
+
+static DEFINE_MUTEX(attribute_container_mutex);
+
+/**
+ * attribute_container_register - register an attribute container
+ *
+ * @cont: The container to register.  This must be allocated by the
+ *        callee and should also be zeroed by it.
+ */
+void
+attribute_container_register(struct attribute_container *cont)
+{
+	INIT_LIST_HEAD(&cont->node);
+	klist_init(&cont->containers, internal_container_klist_get,
+		   internal_container_klist_put);
+
+	mutex_lock(&attribute_container_mutex);
+	list_add_tail(&cont->node, &attribute_container_list);
+	mutex_unlock(&attribute_container_mutex);
+}
+EXPORT_SYMBOL_GPL(attribute_container_register);
+
+/**
+ * attribute_container_unregister - remove a container registration
+ *
+ * @cont: previously registered container to remove
+ */
+int
+attribute_container_unregister(struct attribute_container *cont)
+{
+	int retval = -EBUSY;
+
+	mutex_lock(&attribute_container_mutex);
+	spin_lock(&cont->containers.k_lock);
+	if (!list_empty(&cont->containers.k_list))
+		goto out;
+	retval = 0;
+	list_del(&cont->node);
+ out:
+	spin_unlock(&cont->containers.k_lock);
+	mutex_unlock(&attribute_container_mutex);
+	return retval;
+
+}
+EXPORT_SYMBOL_GPL(attribute_container_unregister);
+
+/* private function used as class release */
+static void attribute_container_release(struct device *classdev)
+{
+	struct internal_container *ic
+		= container_of(classdev, struct internal_container, classdev);
+	struct device *dev = classdev->parent;
+
+	kfree(ic);
+	put_device(dev);
+}
+
+/**
+ * attribute_container_add_device - see if any container is interested in dev
+ *
+ * @dev: device to add attributes to
+ * @fn:	 function to trigger addition of class device.
+ *
+ * This function allocates storage for the class device(s) to be
+ * attached to dev (one for each matching attribute_container).  If no
+ * fn is provided, the code will simply register the class device via
+ * device_add.  If a function is provided, it is expected to add
+ * the class device at the appropriate time.  One of the things that
+ * might be necessary is to allocate and initialise the classdev and
+ * then add it a later time.  To do this, call this routine for
+ * allocation and initialisation and then use
+ * attribute_container_device_trigger() to call device_add() on
+ * it.  Note: after this, the class device contains a reference to dev
+ * which is not relinquished until the release of the classdev.
+ */
+void
+attribute_container_add_device(struct device *dev,
+			       int (*fn)(struct attribute_container *,
+					 struct device *,
+					 struct device *))
+{
+	struct attribute_container *cont;
+
+	mutex_lock(&attribute_container_mutex);
+	list_for_each_entry(cont, &attribute_container_list, node) {
+		struct internal_container *ic;
+
+		if (attribute_container_no_classdevs(cont))
+			continue;
+
+		if (!cont->match(cont, dev))
+			continue;
+
+		ic = kzalloc_obj(*ic);
+		if (!ic) {
+			dev_err(dev, "failed to allocate class container\n");
+			continue;
+		}
+
+		ic->cont = cont;
+		device_initialize(&ic->classdev);
+		ic->classdev.parent = get_device(dev);
+		ic->classdev.class = cont->class;
+		cont->class->dev_release = attribute_container_release;
+		dev_set_name(&ic->classdev, "%s", dev_name(dev));
+		if (fn)
+			fn(cont, dev, &ic->classdev);
+		else
+			attribute_container_add_class_device(&ic->classdev);
+		klist_add_tail(&ic->node, &cont->containers);
+	}
+	mutex_unlock(&attribute_container_mutex);
+}
+
+/* FIXME: can't break out of this unless klist_iter_exit is also
+ * called before doing the break
+ */
+#define klist_for_each_entry(pos, head, member, iter) \
+	for (klist_iter_init(head, iter); (pos = ({ \
+		struct klist_node *n = klist_next(iter); \
+		n ? container_of(n, typeof(*pos), member) : \
+			({ klist_iter_exit(iter) ; NULL; }); \
+	})) != NULL;)
+
+
+/**
+ * attribute_container_remove_device - make device eligible for removal.
+ *
+ * @dev:  The generic device
+ * @fn:	  A function to call to remove the device
+ *
+ * This routine triggers device removal.  If fn is NULL, then it is
+ * simply done via device_unregister (note that if something
+ * still has a reference to the classdev, then the memory occupied
+ * will not be freed until the classdev is released).  If you want a
+ * two phase release: remove from visibility and then delete the
+ * device, then you should use this routine with a fn that calls
+ * device_del() and then use attribute_container_device_trigger()
+ * to do the final put on the classdev.
+ */
+void
+attribute_container_remove_device(struct device *dev,
+				  void (*fn)(struct attribute_container *,
+					     struct device *,
+					     struct device *))
+{
+	struct attribute_container *cont;
+
+	mutex_lock(&attribute_container_mutex);
+	list_for_each_entry(cont, &attribute_container_list, node) {
+		struct internal_container *ic;
+		struct klist_iter iter;
+
+		if (attribute_container_no_classdevs(cont))
+			continue;
+
+		if (!cont->match(cont, dev))
+			continue;
+
+		klist_for_each_entry(ic, &cont->containers, node, &iter) {
+			if (dev != ic->classdev.parent)
+				continue;
+			klist_del(&ic->node);
+			if (fn)
+				fn(cont, dev, &ic->classdev);
+			else {
+				attribute_container_remove_attrs(&ic->classdev);
+				device_unregister(&ic->classdev);
+			}
+		}
+	}
+	mutex_unlock(&attribute_container_mutex);
+}
+
+static int
+do_attribute_container_device_trigger_safe(struct device *dev,
+					   struct attribute_container *cont,
+					   int (*fn)(struct attribute_container *,
+						     struct device *, struct device *),
+					   int (*undo)(struct attribute_container *,
+						       struct device *, struct device *))
+{
+	int ret;
+	struct internal_container *ic, *failed;
+	struct klist_iter iter;
+
+	if (attribute_container_no_classdevs(cont))
+		return fn(cont, dev, NULL);
+
+	klist_for_each_entry(ic, &cont->containers, node, &iter) {
+		if (dev == ic->classdev.parent) {
+			ret = fn(cont, dev, &ic->classdev);
+			if (ret) {
+				failed = ic;
+				klist_iter_exit(&iter);
+				goto fail;
+			}
+		}
+	}
+	return 0;
+
+fail:
+	if (!undo)
+		return ret;
+
+	/* Attempt to undo the work partially done. */
+	klist_for_each_entry(ic, &cont->containers, node, &iter) {
+		if (ic == failed) {
+			klist_iter_exit(&iter);
+			break;
+		}
+		if (dev == ic->classdev.parent)
+			undo(cont, dev, &ic->classdev);
+	}
+	return ret;
+}
+
+/**
+ * attribute_container_device_trigger_safe - execute a trigger for each
+ * matching classdev or fail all of them.
+ *
+ * @dev:  The generic device to run the trigger for
+ * @fn:   the function to execute for each classdev.
+ * @undo: A function to undo the work previously done in case of error
+ *
+ * This function is a safe version of
+ * attribute_container_device_trigger. It stops on the first error and
+ * undo the partial work that has been done, on previous classdev.  It
+ * is guaranteed that either they all succeeded, or none of them
+ * succeeded.
+ */
+int
+attribute_container_device_trigger_safe(struct device *dev,
+					int (*fn)(struct attribute_container *,
+						  struct device *,
+						  struct device *),
+					int (*undo)(struct attribute_container *,
+						    struct device *,
+						    struct device *))
+{
+	struct attribute_container *cont, *failed = NULL;
+	int ret = 0;
+
+	mutex_lock(&attribute_container_mutex);
+
+	list_for_each_entry(cont, &attribute_container_list, node) {
+
+		if (!cont->match(cont, dev))
+			continue;
+
+		ret = do_attribute_container_device_trigger_safe(dev, cont,
+								 fn, undo);
+		if (ret) {
+			failed = cont;
+			break;
+		}
+	}
+
+	if (ret && !WARN_ON(!undo)) {
+		list_for_each_entry(cont, &attribute_container_list, node) {
+
+			if (failed == cont)
+				break;
+
+			if (!cont->match(cont, dev))
+				continue;
+
+			do_attribute_container_device_trigger_safe(dev, cont,
+								   undo, NULL);
+		}
+	}
+
+	mutex_unlock(&attribute_container_mutex);
+	return ret;
+
+}
+
+/**
+ * attribute_container_device_trigger - execute a trigger for each matching classdev
+ *
+ * @dev:  The generic device to run the trigger for
+ * @fn:   the function to execute for each classdev.
+ *
+ * This function is for executing a trigger when you need to know both
+ * the container and the classdev.
+ */
+void
+attribute_container_device_trigger(struct device *dev,
+				   int (*fn)(struct attribute_container *,
+					     struct device *,
+					     struct device *))
+{
+	struct attribute_container *cont;
+
+	mutex_lock(&attribute_container_mutex);
+	list_for_each_entry(cont, &attribute_container_list, node) {
+		struct internal_container *ic;
+		struct klist_iter iter;
+
+		if (!cont->match(cont, dev))
+			continue;
+
+		if (attribute_container_no_classdevs(cont)) {
+			fn(cont, dev, NULL);
+			continue;
+		}
+
+		klist_for_each_entry(ic, &cont->containers, node, &iter) {
+			if (dev == ic->classdev.parent)
+				fn(cont, dev, &ic->classdev);
+		}
+	}
+	mutex_unlock(&attribute_container_mutex);
+}
+
+/**
+ * attribute_container_add_attrs - add attributes
+ *
+ * @classdev: The class device
+ *
+ * This simply creates all the class device sysfs files from the
+ * attributes listed in the container
+ */
+int
+attribute_container_add_attrs(struct device *classdev)
+{
+	struct attribute_container *cont =
+		attribute_container_classdev_to_container(classdev);
+	struct device_attribute **attrs = cont->attrs;
+	int i, error;
+
+	BUG_ON(attrs && cont->grp);
+
+	if (!attrs && !cont->grp)
+		return 0;
+
+	if (cont->grp)
+		return sysfs_create_group(&classdev->kobj, cont->grp);
+
+	for (i = 0; attrs[i]; i++) {
+		sysfs_attr_init(&attrs[i]->attr);
+		error = device_create_file(classdev, attrs[i]);
+		if (error)
+			return error;
+	}
+
+	return 0;
+}
+
+/**
+ * attribute_container_add_class_device - same function as device_add
+ *
+ * @classdev:	the class device to add
+ *
+ * This performs essentially the same function as device_add except for
+ * attribute containers, namely add the classdev to the system and then
+ * create the attribute files
+ */
+int
+attribute_container_add_class_device(struct device *classdev)
+{
+	int error = device_add(classdev);
+
+	if (error)
+		return error;
+
+	error = attribute_container_add_attrs(classdev);
+	if (error)
+		device_del(classdev);
+
+	return error;
+}
+
+/**
+ * attribute_container_remove_attrs - remove any attribute files
+ *
+ * @classdev: The class device to remove the files from
+ *
+ */
+void
+attribute_container_remove_attrs(struct device *classdev)
+{
+	struct attribute_container *cont =
+		attribute_container_classdev_to_container(classdev);
+	struct device_attribute **attrs = cont->attrs;
+	int i;
+
+	if (!attrs && !cont->grp)
+		return;
+
+	if (cont->grp) {
+		sysfs_remove_group(&classdev->kobj, cont->grp);
+		return ;
+	}
+
+	for (i = 0; attrs[i]; i++)
+		device_remove_file(classdev, attrs[i]);
+}
+
+/**
+ * attribute_container_class_device_del - equivalent of class_device_del
+ *
+ * @classdev: the class device
+ *
+ * This function simply removes all the attribute files and then calls
+ * device_del.
+ */
+void
+attribute_container_class_device_del(struct device *classdev)
+{
+	attribute_container_remove_attrs(classdev);
+	device_del(classdev);
+}
+
+/**
+ * attribute_container_find_class_device - find the corresponding class_device
+ *
+ * @cont:	the container
+ * @dev:	the generic device
+ *
+ * Looks up the device in the container's list of class devices and returns
+ * the corresponding class_device.
+ */
+struct device *
+attribute_container_find_class_device(struct attribute_container *cont,
+				      struct device *dev)
+{
+	struct device *cdev = NULL;
+	struct internal_container *ic;
+	struct klist_iter iter;
+
+	klist_for_each_entry(ic, &cont->containers, node, &iter) {
+		if (ic->classdev.parent == dev) {
+			cdev = &ic->classdev;
+			/* FIXME: must exit iterator then break */
+			klist_iter_exit(&iter);
+			break;
+		}
+	}
+
+	return cdev;
+}
+EXPORT_SYMBOL_GPL(attribute_container_find_class_device);