x86/kprobes: Fix crash when probing CS CALL instructionsgrafted

When using eBPF to probe CS CALL instructions within a function,
a crash can be triggered.

The eBPF tool probes offset 257 of the __hrtimer_run_queues()
function:

<__hrtimer_run_queues+249>:  nopl   0x0(%rax,%rax,1)
<__hrtimer_run_queues+254>:  mov    %r14,%rdi
<__hrtimer_run_queues+257>:  cs call <__x86_indirect_thunk_r12>
<__hrtimer_run_queues+263>:  mov    %eax,%r12d
<__hrtimer_run_queues+266>:  xchg   %ax,%ax
<__hrtimer_run_queues+268>:  mov    %r13,%rdi

Which triggers this crash:

  BUG: unable to handle page fault for address: 00000000000f41c9
  #PF: supervisor write access in kernel mode
  #PF: error_code(0x0002) - not-present page
  PGD 0 P4D 0
  Oops: 0002 [#1] SMP NOPTI
  CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P
  RIP: 0010:__hrtimer_run_queues+0x106/0x230

Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is
at the 6th byte of the above CS CALL instruction. Since the CS CALL
instruction occupies 6 bytes, the exception occurred in the middle of that
call instruction.

The root cause is that when using eBPF tools to probe in the middle of a
function, a kprobe with INT3 is used as the underlying implementation.

During single-step emulation of the original CALL instruction,
int3_emulate_call() assumes that the probed CALL instruction is 5 bytes
long. However, the actual CS-prefixed CALL instruction occupies 6 bytes,
so it constructs an incorrect exception return address. When the CPU
returns from the kprobe handler, the next instruction to be executed is at
the address of the last byte of that CS CALL instruction. Coincidentally,
starting from that address, the CPU fetches and decodes a completely
different instruction, which ultimately triggers a kernel crash.

Fix the issue by using the actual instruction length obtained from
the instruction decoder when constructing the exception return
address, rather than relying on the hardcoded CALL_INSN_SIZE macro.

[ mingo: Refined the changelog ]

Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step")
Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Jinke Han <jinkehan@didiglobal.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Yafang Shao <laoar.shao@gmail.com>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000

1 files changed, 381 insertions, 0 deletions

diff --git a/drivers/gpio/gpio-pmic-eic-sprd.c b/drivers/gpio/gpio-pmic-eic-sprd.c
new file mode 100644
index 000000000..cb015fb5c
--- /dev/null
+++ b/drivers/gpio/gpio-pmic-eic-sprd.c
@@ -0,0 +1,381 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018 Spreadtrum Communications Inc.
+ * Copyright (C) 2018 Linaro Ltd.
+ */
+
+#include <linux/gpio/driver.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+/* EIC registers definition */
+#define SPRD_PMIC_EIC_DATA		0x0
+#define SPRD_PMIC_EIC_DMSK		0x4
+#define SPRD_PMIC_EIC_IEV		0x14
+#define SPRD_PMIC_EIC_IE		0x18
+#define SPRD_PMIC_EIC_RIS		0x1c
+#define SPRD_PMIC_EIC_MIS		0x20
+#define SPRD_PMIC_EIC_IC		0x24
+#define SPRD_PMIC_EIC_TRIG		0x28
+#define SPRD_PMIC_EIC_CTRL0		0x40
+
+/*
+ * The PMIC EIC controller only has one bank, and each bank now can contain
+ * 16 EICs.
+ */
+#define SPRD_PMIC_EIC_PER_BANK_NR	16
+#define SPRD_PMIC_EIC_NR		SPRD_PMIC_EIC_PER_BANK_NR
+#define SPRD_PMIC_EIC_DATA_MASK		GENMASK(15, 0)
+#define SPRD_PMIC_EIC_BIT(x)		((x) & (SPRD_PMIC_EIC_PER_BANK_NR - 1))
+#define SPRD_PMIC_EIC_DBNC_MASK		GENMASK(11, 0)
+
+/*
+ * These registers are modified under the irq bus lock and cached to avoid
+ * unnecessary writes in bus_sync_unlock.
+ */
+enum {
+	REG_IEV,
+	REG_IE,
+	REG_TRIG,
+	CACHE_NR_REGS
+};
+
+/**
+ * struct sprd_pmic_eic - PMIC EIC controller
+ * @chip: the gpio_chip structure.
+ * @map:  the regmap from the parent device.
+ * @offset: the EIC controller's offset address of the PMIC.
+ * @reg: the array to cache the EIC registers.
+ * @buslock: for bus lock/sync and unlock.
+ * @irq: the interrupt number of the PMIC EIC conteroller.
+ */
+struct sprd_pmic_eic {
+	struct gpio_chip chip;
+	struct regmap *map;
+	u32 offset;
+	u8 reg[CACHE_NR_REGS];
+	struct mutex buslock;
+	int irq;
+};
+
+static void sprd_pmic_eic_update(struct gpio_chip *chip, unsigned int offset,
+				 u16 reg, unsigned int val)
+{
+	struct sprd_pmic_eic *pmic_eic = gpiochip_get_data(chip);
+	u32 shift = SPRD_PMIC_EIC_BIT(offset);
+
+	regmap_update_bits(pmic_eic->map, pmic_eic->offset + reg,
+			   BIT(shift), val << shift);
+}
+
+static int sprd_pmic_eic_read(struct gpio_chip *chip, unsigned int offset,
+			      u16 reg)
+{
+	struct sprd_pmic_eic *pmic_eic = gpiochip_get_data(chip);
+	u32 value;
+	int ret;
+
+	ret = regmap_read(pmic_eic->map, pmic_eic->offset + reg, &value);
+	if (ret)
+		return ret;
+
+	return !!(value & BIT(SPRD_PMIC_EIC_BIT(offset)));
+}
+
+static int sprd_pmic_eic_request(struct gpio_chip *chip, unsigned int offset)
+{
+	sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_DMSK, 1);
+	return 0;
+}
+
+static void sprd_pmic_eic_free(struct gpio_chip *chip, unsigned int offset)
+{
+	sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_DMSK, 0);
+}
+
+static int sprd_pmic_eic_get(struct gpio_chip *chip, unsigned int offset)
+{
+	return sprd_pmic_eic_read(chip, offset, SPRD_PMIC_EIC_DATA);
+}
+
+static int sprd_pmic_eic_direction_input(struct gpio_chip *chip,
+					 unsigned int offset)
+{
+	/* EICs are always input, nothing need to do here. */
+	return 0;
+}
+
+static int sprd_pmic_eic_set_debounce(struct gpio_chip *chip,
+				      unsigned int offset,
+				      unsigned int debounce)
+{
+	struct sprd_pmic_eic *pmic_eic = gpiochip_get_data(chip);
+	u32 reg, value;
+	int ret;
+
+	reg = SPRD_PMIC_EIC_CTRL0 + SPRD_PMIC_EIC_BIT(offset) * 0x4;
+	ret = regmap_read(pmic_eic->map, pmic_eic->offset + reg, &value);
+	if (ret)
+		return ret;
+
+	value &= ~SPRD_PMIC_EIC_DBNC_MASK;
+	value |= (debounce / 1000) & SPRD_PMIC_EIC_DBNC_MASK;
+	return regmap_write(pmic_eic->map, pmic_eic->offset + reg, value);
+}
+
+static int sprd_pmic_eic_set_config(struct gpio_chip *chip, unsigned int offset,
+				    unsigned long config)
+{
+	unsigned long param = pinconf_to_config_param(config);
+	u32 arg = pinconf_to_config_argument(config);
+
+	if (param == PIN_CONFIG_INPUT_DEBOUNCE)
+		return sprd_pmic_eic_set_debounce(chip, offset, arg);
+
+	return -ENOTSUPP;
+}
+
+static void sprd_pmic_eic_irq_mask(struct irq_data *data)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct sprd_pmic_eic *pmic_eic = gpiochip_get_data(chip);
+	u32 offset = irqd_to_hwirq(data);
+
+	pmic_eic->reg[REG_IE] &= ~BIT(offset);
+	pmic_eic->reg[REG_TRIG] &= ~BIT(offset);
+
+	gpiochip_disable_irq(chip, offset);
+}
+
+static void sprd_pmic_eic_irq_unmask(struct irq_data *data)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct sprd_pmic_eic *pmic_eic = gpiochip_get_data(chip);
+	u32 offset = irqd_to_hwirq(data);
+
+	gpiochip_enable_irq(chip, offset);
+
+	pmic_eic->reg[REG_IE] |= BIT(offset);
+	pmic_eic->reg[REG_TRIG] |= BIT(offset);
+}
+
+static int sprd_pmic_eic_irq_set_type(struct irq_data *data,
+				      unsigned int flow_type)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct sprd_pmic_eic *pmic_eic = gpiochip_get_data(chip);
+	u32 offset = irqd_to_hwirq(data);
+
+	switch (flow_type) {
+	case IRQ_TYPE_LEVEL_HIGH:
+		pmic_eic->reg[REG_IEV] |= BIT(offset);
+		break;
+	case IRQ_TYPE_LEVEL_LOW:
+		pmic_eic->reg[REG_IEV] &= ~BIT(offset);
+		break;
+	case IRQ_TYPE_EDGE_RISING:
+	case IRQ_TYPE_EDGE_FALLING:
+	case IRQ_TYPE_EDGE_BOTH:
+		/*
+		 * Will set the trigger level according to current EIC level
+		 * in irq_bus_sync_unlock() interface, so here nothing to do.
+		 */
+		break;
+	default:
+		return -ENOTSUPP;
+	}
+
+	return 0;
+}
+
+static void sprd_pmic_eic_bus_lock(struct irq_data *data)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct sprd_pmic_eic *pmic_eic = gpiochip_get_data(chip);
+
+	mutex_lock(&pmic_eic->buslock);
+}
+
+static void sprd_pmic_eic_bus_sync_unlock(struct irq_data *data)
+{
+	struct gpio_chip *chip = irq_data_get_irq_chip_data(data);
+	struct sprd_pmic_eic *pmic_eic = gpiochip_get_data(chip);
+	u32 trigger = irqd_get_trigger_type(data);
+	u32 offset = irqd_to_hwirq(data);
+	int state;
+
+	/* Set irq type */
+	if (trigger & IRQ_TYPE_EDGE_BOTH) {
+		state = sprd_pmic_eic_get(chip, offset);
+		if (state)
+			sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_IEV, 0);
+		else
+			sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_IEV, 1);
+	} else {
+		sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_IEV,
+				     !!(pmic_eic->reg[REG_IEV] & BIT(offset)));
+	}
+
+	/* Set irq unmask */
+	sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_IE,
+			     !!(pmic_eic->reg[REG_IE] & BIT(offset)));
+	/* Generate trigger start pulse for debounce EIC */
+	sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_TRIG,
+			     !!(pmic_eic->reg[REG_TRIG] & BIT(offset)));
+
+	mutex_unlock(&pmic_eic->buslock);
+}
+
+static void sprd_pmic_eic_toggle_trigger(struct gpio_chip *chip,
+					 unsigned int irq, unsigned int offset)
+{
+	u32 trigger = irq_get_trigger_type(irq);
+	int state, post_state;
+
+	if (!(trigger & IRQ_TYPE_EDGE_BOTH))
+		return;
+
+	state = sprd_pmic_eic_get(chip, offset);
+retry:
+	if (state)
+		sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_IEV, 0);
+	else
+		sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_IEV, 1);
+
+	post_state = sprd_pmic_eic_get(chip, offset);
+	if (state != post_state) {
+		dev_warn(chip->parent, "PMIC EIC level was changed.\n");
+		state = post_state;
+		goto retry;
+	}
+
+	/* Set irq unmask */
+	sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_IE, 1);
+	/* Generate trigger start pulse for debounce EIC */
+	sprd_pmic_eic_update(chip, offset, SPRD_PMIC_EIC_TRIG, 1);
+}
+
+static irqreturn_t sprd_pmic_eic_irq_handler(int irq, void *data)
+{
+	struct sprd_pmic_eic *pmic_eic = data;
+	struct gpio_chip *chip = &pmic_eic->chip;
+	unsigned long status;
+	u32 n, girq, val;
+	int ret;
+
+	ret = regmap_read(pmic_eic->map, pmic_eic->offset + SPRD_PMIC_EIC_MIS,
+			  &val);
+	if (ret)
+		return IRQ_RETVAL(ret);
+
+	status = val & SPRD_PMIC_EIC_DATA_MASK;
+
+	for_each_set_bit(n, &status, chip->ngpio) {
+		/* Clear the interrupt */
+		sprd_pmic_eic_update(chip, n, SPRD_PMIC_EIC_IC, 1);
+
+		girq = irq_find_mapping(chip->irq.domain, n);
+		handle_nested_irq(girq);
+
+		/*
+		 * The PMIC EIC can only support level trigger, so we can
+		 * toggle the level trigger to emulate the edge trigger.
+		 */
+		sprd_pmic_eic_toggle_trigger(chip, girq, n);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static const struct irq_chip pmic_eic_irq_chip = {
+	.name			= "sprd-pmic-eic",
+	.irq_mask		= sprd_pmic_eic_irq_mask,
+	.irq_unmask		= sprd_pmic_eic_irq_unmask,
+	.irq_set_type		= sprd_pmic_eic_irq_set_type,
+	.irq_bus_lock		= sprd_pmic_eic_bus_lock,
+	.irq_bus_sync_unlock	= sprd_pmic_eic_bus_sync_unlock,
+	.flags			= IRQCHIP_SKIP_SET_WAKE | IRQCHIP_IMMUTABLE,
+	GPIOCHIP_IRQ_RESOURCE_HELPERS,
+};
+
+static int sprd_pmic_eic_probe(struct platform_device *pdev)
+{
+	struct gpio_irq_chip *irq;
+	struct sprd_pmic_eic *pmic_eic;
+	int ret;
+
+	pmic_eic = devm_kzalloc(&pdev->dev, sizeof(*pmic_eic), GFP_KERNEL);
+	if (!pmic_eic)
+		return -ENOMEM;
+
+	mutex_init(&pmic_eic->buslock);
+
+	pmic_eic->irq = platform_get_irq(pdev, 0);
+	if (pmic_eic->irq < 0)
+		return pmic_eic->irq;
+
+	pmic_eic->map = dev_get_regmap(pdev->dev.parent, NULL);
+	if (!pmic_eic->map)
+		return -ENODEV;
+
+	ret = of_property_read_u32(pdev->dev.of_node, "reg", &pmic_eic->offset);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to get PMIC EIC base address.\n");
+		return ret;
+	}
+
+	ret = devm_request_threaded_irq(&pdev->dev, pmic_eic->irq, NULL,
+					sprd_pmic_eic_irq_handler,
+					IRQF_ONESHOT | IRQF_NO_SUSPEND,
+					dev_name(&pdev->dev), pmic_eic);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to request PMIC EIC IRQ.\n");
+		return ret;
+	}
+
+	pmic_eic->chip.label = dev_name(&pdev->dev);
+	pmic_eic->chip.ngpio = SPRD_PMIC_EIC_NR;
+	pmic_eic->chip.base = -1;
+	pmic_eic->chip.parent = &pdev->dev;
+	pmic_eic->chip.direction_input = sprd_pmic_eic_direction_input;
+	pmic_eic->chip.request = sprd_pmic_eic_request;
+	pmic_eic->chip.free = sprd_pmic_eic_free;
+	pmic_eic->chip.set_config = sprd_pmic_eic_set_config;
+	pmic_eic->chip.get = sprd_pmic_eic_get;
+	pmic_eic->chip.can_sleep = true;
+
+	irq = &pmic_eic->chip.irq;
+	gpio_irq_chip_set_chip(irq, &pmic_eic_irq_chip);
+	irq->threaded = true;
+
+	ret = devm_gpiochip_add_data(&pdev->dev, &pmic_eic->chip, pmic_eic);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "Could not register gpiochip %d.\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct of_device_id sprd_pmic_eic_of_match[] = {
+	{ .compatible = "sprd,sc2731-eic", },
+	{ /* end of list */ }
+};
+MODULE_DEVICE_TABLE(of, sprd_pmic_eic_of_match);
+
+static struct platform_driver sprd_pmic_eic_driver = {
+	.probe = sprd_pmic_eic_probe,
+	.driver = {
+		.name = "sprd-pmic-eic",
+		.of_match_table	= sprd_pmic_eic_of_match,
+	},
+};
+
+module_platform_driver(sprd_pmic_eic_driver);
+
+MODULE_DESCRIPTION("Spreadtrum PMIC EIC driver");
+MODULE_LICENSE("GPL v2");