ftrace: Synchronize the initialization of ftrace_opsgrafted

There's some internal state that ftrace_ops needs to have set, but since
it can be declared outside of the ftrace.c code, it calls
ftrace_ops_init() on the ops in every global function. The issue is that
if two tasks call it on the same ops at the same time it is possible to
have the initialization of one corrupt the initialization of the other
call.

Create a ops_mutex to use to synchronize every initialization of the
ftrace_ops. The mutex is taken within checking the ftrace_ops flag that
states it was initializied but the flag is checked again after the mutex
has been taken. Checking first outside the mutex allows it to shortcut
having to take the mutex. But then the check needs to be done again after
the mute is taken in case of races.

Cc: stable@vger.kernel.org
Link: https://patch.msgid.link/20260902095501.6b59af20@gandalf.local.home
Fixes: f04f24fb7e48d ("ftrace, kprobes: Fix a deadlock on ftrace_regex_lock")
Reported-by: sashiko-bot@kernel.org
Close: https://lore.kernel.org/all/20260829025528.49A831F000E9@smtp.kernel.org/
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>

7 files changed, 979 insertions, 0 deletions

diff --git a/arch/arm/mach-hisi/Kconfig b/arch/arm/mach-hisi/Kconfig
new file mode 100644
index 000000000..7b3440687
--- /dev/null
+++ b/arch/arm/mach-hisi/Kconfig
@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config ARCH_HISI
+	bool "Hisilicon SoC Support"
+	depends on ARCH_MULTI_V7 || (ARCH_MULTI_V5 && CPU_LITTLE_ENDIAN)
+	select ARM_AMBA
+	select ARM_GIC if ARCH_MULTI_V7
+	select ARM_TIMER_SP804
+	select POWER_RESET
+	select POWER_RESET_HISI
+	select POWER_SUPPLY
+
+if ARCH_HISI
+
+menu "Hisilicon platform type"
+
+config ARCH_HI3xxx
+	bool "Hisilicon Hi36xx family"
+	depends on ARCH_MULTI_V7
+	select CACHE_L2X0
+	select HAVE_ARM_SCU if SMP
+	select HAVE_ARM_TWD if SMP
+	select PINCTRL
+	select PINCTRL_SINGLE
+	help
+	  Support for Hisilicon Hi36xx SoC family
+
+config ARCH_HIP01
+	bool "Hisilicon HIP01 family"
+	depends on ARCH_MULTI_V7
+	select HAVE_ARM_SCU if SMP
+	select HAVE_ARM_TWD if SMP
+	select ARM_GLOBAL_TIMER
+	help
+	  Support for Hisilicon HIP01 SoC family
+
+config ARCH_HIP04
+	bool "Hisilicon HiP04 Cortex A15 family"
+	depends on ARCH_MULTI_V7
+	select ARM_ERRATA_798181 if SMP
+	select HAVE_ARM_ARCH_TIMER
+	select MCPM if SMP
+	select MCPM_QUAD_CLUSTER if SMP
+	select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP
+	help
+	  Support for Hisilicon HiP04 SoC family
+
+config ARCH_HIX5HD2
+	bool "Hisilicon X5HD2 family"
+	depends on ARCH_MULTI_V7
+	select CACHE_L2X0
+	select HAVE_ARM_SCU if SMP
+	select HAVE_ARM_TWD if SMP
+	select PINCTRL
+	select PINCTRL_SINGLE
+	help
+	  Support for Hisilicon HIX5HD2 SoC family
+
+config ARCH_SD5203
+	bool "Hisilicon SD5203 family"
+	depends on ARCH_MULTI_V5
+	select DW_APB_ICTL
+	help
+	  Support for Hisilicon SD5203 SoC family
+
+endmenu
+
+endif
diff --git a/arch/arm/mach-hisi/Makefile b/arch/arm/mach-hisi/Makefile
new file mode 100644
index 000000000..39476355e
--- /dev/null
+++ b/arch/arm/mach-hisi/Makefile
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Makefile for Hisilicon processors family
+#
+
+CFLAGS_platmcpm.o	:= -march=armv7-a
+
+obj-y	+= hisilicon.o
+obj-$(CONFIG_MCPM)		+= platmcpm.o
+obj-$(CONFIG_SMP)		+= platsmp.o hotplug.o
diff --git a/arch/arm/mach-hisi/core.h b/arch/arm/mach-hisi/core.h
new file mode 100644
index 000000000..61245274f
--- /dev/null
+++ b/arch/arm/mach-hisi/core.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __HISILICON_CORE_H
+#define __HISILICON_CORE_H
+
+#include <linux/reboot.h>
+
+extern void hi3xxx_set_cpu_jump(int cpu, void *jump_addr);
+extern int hi3xxx_get_cpu_jump(int cpu);
+extern void secondary_startup(void);
+
+extern void hi3xxx_cpu_die(unsigned int cpu);
+extern int hi3xxx_cpu_kill(unsigned int cpu);
+extern void hi3xxx_set_cpu(int cpu, bool enable);
+
+extern void hix5hd2_set_cpu(int cpu, bool enable);
+extern void hix5hd2_cpu_die(unsigned int cpu);
+
+extern void hip01_set_cpu(int cpu, bool enable);
+#endif
diff --git a/arch/arm/mach-hisi/hisilicon.c b/arch/arm/mach-hisi/hisilicon.c
new file mode 100644
index 000000000..b8d14b369
--- /dev/null
+++ b/arch/arm/mach-hisi/hisilicon.c
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * (HiSilicon's SoC based) flattened device tree enabled machine
+ *
+ * Copyright (c) 2012-2013 HiSilicon Ltd.
+ * Copyright (c) 2012-2013 Linaro Ltd.
+ *
+ * Author: Haojian Zhuang <haojian.zhuang@linaro.org>
+*/
+
+#include <linux/clocksource.h>
+#include <linux/irqchip.h>
+
+#include <asm/mach/arch.h>
+#include <asm/mach/map.h>
+
+#define HI3620_SYSCTRL_PHYS_BASE		0xfc802000
+#define HI3620_SYSCTRL_VIRT_BASE		0xfe802000
+
+/*
+ * This table is only for optimization. Since ioremap() could always share
+ * the same mapping if it's defined as static IO mapping.
+ *
+ * Without this table, system could also work. The cost is some virtual address
+ * spaces wasted since ioremap() may be called multi times for the same
+ * IO space.
+ */
+static struct map_desc hi3620_io_desc[] __initdata = {
+	{
+		/* sysctrl */
+		.pfn		= __phys_to_pfn(HI3620_SYSCTRL_PHYS_BASE),
+		.virtual	= HI3620_SYSCTRL_VIRT_BASE,
+		.length		= 0x1000,
+		.type		= MT_DEVICE,
+	},
+};
+
+static void __init hi3620_map_io(void)
+{
+	debug_ll_io_init();
+	iotable_init(hi3620_io_desc, ARRAY_SIZE(hi3620_io_desc));
+}
+
+static const char *const hi3xxx_compat[] __initconst = {
+	"hisilicon,hi3620-hi4511",
+	NULL,
+};
+
+DT_MACHINE_START(HI3620, "Hisilicon Hi3620 (Flattened Device Tree)")
+	.map_io		= hi3620_map_io,
+	.dt_compat	= hi3xxx_compat,
+MACHINE_END
diff --git a/arch/arm/mach-hisi/hotplug.c b/arch/arm/mach-hisi/hotplug.c
new file mode 100644
index 000000000..f5655ad1c
--- /dev/null
+++ b/arch/arm/mach-hisi/hotplug.c
@@ -0,0 +1,298 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2013 Linaro Ltd.
+ * Copyright (c) 2013 HiSilicon Limited.
+ */
+
+#include <linux/cpu.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/of_address.h>
+#include <asm/cacheflush.h>
+#include <asm/smp_plat.h>
+#include "core.h"
+
+/* Sysctrl registers in Hi3620 SoC */
+#define SCISOEN				0xc0
+#define SCISODIS			0xc4
+#define SCPERPWREN			0xd0
+#define SCPERPWRDIS			0xd4
+#define SCCPUCOREEN			0xf4
+#define SCCPUCOREDIS			0xf8
+#define SCPERCTRL0			0x200
+#define SCCPURSTEN			0x410
+#define SCCPURSTDIS			0x414
+
+/*
+ * bit definition in SCISOEN/SCPERPWREN/...
+ *
+ * CPU2_ISO_CTRL	(1 << 5)
+ * CPU3_ISO_CTRL	(1 << 6)
+ * ...
+ */
+#define CPU2_ISO_CTRL			(1 << 5)
+
+/*
+ * bit definition in SCPERCTRL0
+ *
+ * CPU0_WFI_MASK_CFG	(1 << 28)
+ * CPU1_WFI_MASK_CFG	(1 << 29)
+ * ...
+ */
+#define CPU0_WFI_MASK_CFG		(1 << 28)
+
+/*
+ * bit definition in SCCPURSTEN/...
+ *
+ * CPU0_SRST_REQ_EN	(1 << 0)
+ * CPU1_SRST_REQ_EN	(1 << 1)
+ * ...
+ */
+#define CPU0_HPM_SRST_REQ_EN		(1 << 22)
+#define CPU0_DBG_SRST_REQ_EN		(1 << 12)
+#define CPU0_NEON_SRST_REQ_EN		(1 << 4)
+#define CPU0_SRST_REQ_EN		(1 << 0)
+
+#define HIX5HD2_PERI_CRG20		0x50
+#define CRG20_CPU1_RESET		(1 << 17)
+
+#define HIX5HD2_PERI_PMC0		0x1000
+#define PMC0_CPU1_WAIT_MTCOMS_ACK	(1 << 8)
+#define PMC0_CPU1_PMC_ENABLE		(1 << 7)
+#define PMC0_CPU1_POWERDOWN		(1 << 3)
+
+#define HIP01_PERI9                    0x50
+#define PERI9_CPU1_RESET               (1 << 1)
+
+enum {
+	HI3620_CTRL,
+	ERROR_CTRL,
+};
+
+static void __iomem *ctrl_base;
+static int id;
+
+static void set_cpu_hi3620(int cpu, bool enable)
+{
+	u32 val = 0;
+
+	if (enable) {
+		/* MTCMOS set */
+		if ((cpu == 2) || (cpu == 3))
+			writel_relaxed(CPU2_ISO_CTRL << (cpu - 2),
+				       ctrl_base + SCPERPWREN);
+		udelay(100);
+
+		/* Enable core */
+		writel_relaxed(0x01 << cpu, ctrl_base + SCCPUCOREEN);
+
+		/* unreset */
+		val = CPU0_DBG_SRST_REQ_EN | CPU0_NEON_SRST_REQ_EN
+			| CPU0_SRST_REQ_EN;
+		writel_relaxed(val << cpu, ctrl_base + SCCPURSTDIS);
+		/* reset */
+		val |= CPU0_HPM_SRST_REQ_EN;
+		writel_relaxed(val << cpu, ctrl_base + SCCPURSTEN);
+
+		/* ISO disable */
+		if ((cpu == 2) || (cpu == 3))
+			writel_relaxed(CPU2_ISO_CTRL << (cpu - 2),
+				       ctrl_base + SCISODIS);
+		udelay(1);
+
+		/* WFI Mask */
+		val = readl_relaxed(ctrl_base + SCPERCTRL0);
+		val &= ~(CPU0_WFI_MASK_CFG << cpu);
+		writel_relaxed(val, ctrl_base + SCPERCTRL0);
+
+		/* Unreset */
+		val = CPU0_DBG_SRST_REQ_EN | CPU0_NEON_SRST_REQ_EN
+			| CPU0_SRST_REQ_EN | CPU0_HPM_SRST_REQ_EN;
+		writel_relaxed(val << cpu, ctrl_base + SCCPURSTDIS);
+	} else {
+		/* wfi mask */
+		val = readl_relaxed(ctrl_base + SCPERCTRL0);
+		val |= (CPU0_WFI_MASK_CFG << cpu);
+		writel_relaxed(val, ctrl_base + SCPERCTRL0);
+
+		/* disable core*/
+		writel_relaxed(0x01 << cpu, ctrl_base + SCCPUCOREDIS);
+
+		if ((cpu == 2) || (cpu == 3)) {
+			/* iso enable */
+			writel_relaxed(CPU2_ISO_CTRL << (cpu - 2),
+				       ctrl_base + SCISOEN);
+			udelay(1);
+		}
+
+		/* reset */
+		val = CPU0_DBG_SRST_REQ_EN | CPU0_NEON_SRST_REQ_EN
+			| CPU0_SRST_REQ_EN | CPU0_HPM_SRST_REQ_EN;
+		writel_relaxed(val << cpu, ctrl_base + SCCPURSTEN);
+
+		if ((cpu == 2) || (cpu == 3)) {
+			/* MTCMOS unset */
+			writel_relaxed(CPU2_ISO_CTRL << (cpu - 2),
+				       ctrl_base + SCPERPWRDIS);
+			udelay(100);
+		}
+	}
+}
+
+static int hi3xxx_hotplug_init(void)
+{
+	struct device_node *node;
+
+	node = of_find_compatible_node(NULL, NULL, "hisilicon,sysctrl");
+	if (!node) {
+		id = ERROR_CTRL;
+		return -ENOENT;
+	}
+
+	ctrl_base = of_iomap(node, 0);
+	of_node_put(node);
+	if (!ctrl_base) {
+		id = ERROR_CTRL;
+		return -ENOMEM;
+	}
+
+	id = HI3620_CTRL;
+	return 0;
+}
+
+void hi3xxx_set_cpu(int cpu, bool enable)
+{
+	if (!ctrl_base) {
+		if (hi3xxx_hotplug_init() < 0)
+			return;
+	}
+
+	if (id == HI3620_CTRL)
+		set_cpu_hi3620(cpu, enable);
+}
+
+static bool hix5hd2_hotplug_init(void)
+{
+	struct device_node *np;
+
+	np = of_find_compatible_node(NULL, NULL, "hisilicon,cpuctrl");
+	if (!np)
+		return false;
+
+	ctrl_base = of_iomap(np, 0);
+	of_node_put(np);
+	if (!ctrl_base)
+		return false;
+
+	return true;
+}
+
+void hix5hd2_set_cpu(int cpu, bool enable)
+{
+	u32 val = 0;
+
+	if (!ctrl_base)
+		if (!hix5hd2_hotplug_init())
+			BUG();
+
+	if (enable) {
+		/* power on cpu1 */
+		val = readl_relaxed(ctrl_base + HIX5HD2_PERI_PMC0);
+		val &= ~(PMC0_CPU1_WAIT_MTCOMS_ACK | PMC0_CPU1_POWERDOWN);
+		val |= PMC0_CPU1_PMC_ENABLE;
+		writel_relaxed(val, ctrl_base + HIX5HD2_PERI_PMC0);
+		/* unreset */
+		val = readl_relaxed(ctrl_base + HIX5HD2_PERI_CRG20);
+		val &= ~CRG20_CPU1_RESET;
+		writel_relaxed(val, ctrl_base + HIX5HD2_PERI_CRG20);
+	} else {
+		/* power down cpu1 */
+		val = readl_relaxed(ctrl_base + HIX5HD2_PERI_PMC0);
+		val |= PMC0_CPU1_PMC_ENABLE | PMC0_CPU1_POWERDOWN;
+		val &= ~PMC0_CPU1_WAIT_MTCOMS_ACK;
+		writel_relaxed(val, ctrl_base + HIX5HD2_PERI_PMC0);
+
+		/* reset */
+		val = readl_relaxed(ctrl_base + HIX5HD2_PERI_CRG20);
+		val |= CRG20_CPU1_RESET;
+		writel_relaxed(val, ctrl_base + HIX5HD2_PERI_CRG20);
+	}
+}
+
+void hip01_set_cpu(int cpu, bool enable)
+{
+	unsigned int temp;
+	struct device_node *np;
+
+	if (!ctrl_base) {
+		np = of_find_compatible_node(NULL, NULL, "hisilicon,hip01-sysctrl");
+		BUG_ON(!np);
+		ctrl_base = of_iomap(np, 0);
+		of_node_put(np);
+		BUG_ON(!ctrl_base);
+	}
+
+	if (enable) {
+		/* reset on CPU1  */
+		temp = readl_relaxed(ctrl_base + HIP01_PERI9);
+		temp |= PERI9_CPU1_RESET;
+		writel_relaxed(temp, ctrl_base + HIP01_PERI9);
+
+		udelay(50);
+
+		/* unreset on CPU1 */
+		temp = readl_relaxed(ctrl_base + HIP01_PERI9);
+		temp &= ~PERI9_CPU1_RESET;
+		writel_relaxed(temp, ctrl_base + HIP01_PERI9);
+	}
+}
+
+static inline void cpu_enter_lowpower(void)
+{
+	unsigned int v;
+
+	flush_cache_all();
+
+	/*
+	 * Turn off coherency and L1 D-cache
+	 */
+	asm volatile(
+	"	mrc	p15, 0, %0, c1, c0, 1\n"
+	"	bic	%0, %0, #0x40\n"
+	"	mcr	p15, 0, %0, c1, c0, 1\n"
+	"	mrc	p15, 0, %0, c1, c0, 0\n"
+	"	bic	%0, %0, #0x04\n"
+	"	mcr	p15, 0, %0, c1, c0, 0\n"
+	  : "=&r" (v)
+	  : "r" (0)
+	  : "cc");
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+void hi3xxx_cpu_die(unsigned int cpu)
+{
+	cpu_enter_lowpower();
+	hi3xxx_set_cpu_jump(cpu, phys_to_virt(0));
+	cpu_do_idle();
+
+	/* We should have never returned from idle */
+	panic("cpu %d unexpectedly exit from shutdown\n", cpu);
+}
+
+int hi3xxx_cpu_kill(unsigned int cpu)
+{
+	unsigned long timeout = jiffies + msecs_to_jiffies(50);
+
+	while (hi3xxx_get_cpu_jump(cpu))
+		if (time_after(jiffies, timeout))
+			return 0;
+	hi3xxx_set_cpu(cpu, false);
+	return 1;
+}
+
+void hix5hd2_cpu_die(unsigned int cpu)
+{
+	flush_cache_all();
+	hix5hd2_set_cpu(cpu, false);
+}
+#endif
diff --git a/arch/arm/mach-hisi/platmcpm.c b/arch/arm/mach-hisi/platmcpm.c
new file mode 100644
index 000000000..258586e31
--- /dev/null
+++ b/arch/arm/mach-hisi/platmcpm.c
@@ -0,0 +1,346 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2013-2014 Linaro Ltd.
+ * Copyright (c) 2013-2014 HiSilicon Limited.
+ */
+#include <linux/init.h>
+#include <linux/smp.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/memblock.h>
+#include <linux/of_address.h>
+
+#include <asm/cputype.h>
+#include <asm/cp15.h>
+#include <asm/cacheflush.h>
+#include <asm/smp.h>
+#include <asm/smp_plat.h>
+
+#include "core.h"
+
+/* bits definition in SC_CPU_RESET_REQ[x]/SC_CPU_RESET_DREQ[x]
+ * 1 -- unreset; 0 -- reset
+ */
+#define CORE_RESET_BIT(x)		(1 << x)
+#define NEON_RESET_BIT(x)		(1 << (x + 4))
+#define CORE_DEBUG_RESET_BIT(x)		(1 << (x + 9))
+#define CLUSTER_L2_RESET_BIT		(1 << 8)
+#define CLUSTER_DEBUG_RESET_BIT		(1 << 13)
+
+/*
+ * bits definition in SC_CPU_RESET_STATUS[x]
+ * 1 -- reset status; 0 -- unreset status
+ */
+#define CORE_RESET_STATUS(x)		(1 << x)
+#define NEON_RESET_STATUS(x)		(1 << (x + 4))
+#define CORE_DEBUG_RESET_STATUS(x)	(1 << (x + 9))
+#define CLUSTER_L2_RESET_STATUS		(1 << 8)
+#define CLUSTER_DEBUG_RESET_STATUS	(1 << 13)
+#define CORE_WFI_STATUS(x)		(1 << (x + 16))
+#define CORE_WFE_STATUS(x)		(1 << (x + 20))
+#define CORE_DEBUG_ACK(x)		(1 << (x + 24))
+
+#define SC_CPU_RESET_REQ(x)		(0x520 + (x << 3))	/* reset */
+#define SC_CPU_RESET_DREQ(x)		(0x524 + (x << 3))	/* unreset */
+#define SC_CPU_RESET_STATUS(x)		(0x1520 + (x << 3))
+
+#define FAB_SF_MODE			0x0c
+#define FAB_SF_INVLD			0x10
+
+/* bits definition in FB_SF_INVLD */
+#define FB_SF_INVLD_START		(1 << 8)
+
+#define HIP04_MAX_CLUSTERS		4
+#define HIP04_MAX_CPUS_PER_CLUSTER	4
+
+#define POLL_MSEC	10
+#define TIMEOUT_MSEC	1000
+
+static void __iomem *sysctrl, *fabric;
+static int hip04_cpu_table[HIP04_MAX_CLUSTERS][HIP04_MAX_CPUS_PER_CLUSTER];
+static DEFINE_SPINLOCK(boot_lock);
+static u32 fabric_phys_addr;
+/*
+ * [0]: bootwrapper physical address
+ * [1]: bootwrapper size
+ * [2]: relocation address
+ * [3]: relocation size
+ */
+static u32 hip04_boot_method[4];
+
+static bool hip04_cluster_is_down(unsigned int cluster)
+{
+	int i;
+
+	for (i = 0; i < HIP04_MAX_CPUS_PER_CLUSTER; i++)
+		if (hip04_cpu_table[cluster][i])
+			return false;
+	return true;
+}
+
+static void hip04_set_snoop_filter(unsigned int cluster, unsigned int on)
+{
+	unsigned long data;
+
+	if (!fabric)
+		BUG();
+	data = readl_relaxed(fabric + FAB_SF_MODE);
+	if (on)
+		data |= 1 << cluster;
+	else
+		data &= ~(1 << cluster);
+	writel_relaxed(data, fabric + FAB_SF_MODE);
+	do {
+		cpu_relax();
+	} while (data != readl_relaxed(fabric + FAB_SF_MODE));
+}
+
+static int hip04_boot_secondary(unsigned int l_cpu, struct task_struct *idle)
+{
+	unsigned int mpidr, cpu, cluster;
+	unsigned long data;
+	void __iomem *sys_dreq, *sys_status;
+
+	mpidr = cpu_logical_map(l_cpu);
+	cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
+	cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
+
+	if (!sysctrl)
+		return -ENODEV;
+	if (cluster >= HIP04_MAX_CLUSTERS || cpu >= HIP04_MAX_CPUS_PER_CLUSTER)
+		return -EINVAL;
+
+	spin_lock_irq(&boot_lock);
+
+	if (hip04_cpu_table[cluster][cpu])
+		goto out;
+
+	sys_dreq = sysctrl + SC_CPU_RESET_DREQ(cluster);
+	sys_status = sysctrl + SC_CPU_RESET_STATUS(cluster);
+	if (hip04_cluster_is_down(cluster)) {
+		data = CLUSTER_DEBUG_RESET_BIT;
+		writel_relaxed(data, sys_dreq);
+		do {
+			cpu_relax();
+			data = readl_relaxed(sys_status);
+		} while (data & CLUSTER_DEBUG_RESET_STATUS);
+		hip04_set_snoop_filter(cluster, 1);
+	}
+
+	data = CORE_RESET_BIT(cpu) | NEON_RESET_BIT(cpu) | \
+	       CORE_DEBUG_RESET_BIT(cpu);
+	writel_relaxed(data, sys_dreq);
+	do {
+		cpu_relax();
+	} while (data == readl_relaxed(sys_status));
+
+	/*
+	 * We may fail to power up core again without this delay.
+	 * It's not mentioned in document. It's found by test.
+	 */
+	udelay(20);
+
+	arch_send_wakeup_ipi_mask(cpumask_of(l_cpu));
+
+out:
+	hip04_cpu_table[cluster][cpu]++;
+	spin_unlock_irq(&boot_lock);
+
+	return 0;
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+static void hip04_cpu_die(unsigned int l_cpu)
+{
+	unsigned int mpidr, cpu, cluster;
+	bool last_man;
+
+	mpidr = cpu_logical_map(l_cpu);
+	cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
+	cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
+
+	spin_lock(&boot_lock);
+	hip04_cpu_table[cluster][cpu]--;
+	if (hip04_cpu_table[cluster][cpu] == 1) {
+		/* A power_up request went ahead of us. */
+		spin_unlock(&boot_lock);
+		return;
+	} else if (hip04_cpu_table[cluster][cpu] > 1) {
+		pr_err("Cluster %d CPU%d boots multiple times\n", cluster, cpu);
+		BUG();
+	}
+
+	last_man = hip04_cluster_is_down(cluster);
+	spin_unlock(&boot_lock);
+	if (last_man) {
+		/* Since it's Cortex A15, disable L2 prefetching. */
+		asm volatile(
+		"mcr	p15, 1, %0, c15, c0, 3 \n\t"
+		"isb	\n\t"
+		"dsb	"
+		: : "r" (0x400) );
+		v7_exit_coherency_flush(all);
+	} else {
+		v7_exit_coherency_flush(louis);
+	}
+
+	for (;;)
+		wfi();
+}
+
+static int hip04_cpu_kill(unsigned int l_cpu)
+{
+	unsigned int mpidr, cpu, cluster;
+	unsigned int data, tries, count;
+
+	mpidr = cpu_logical_map(l_cpu);
+	cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
+	cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
+	BUG_ON(cluster >= HIP04_MAX_CLUSTERS ||
+	       cpu >= HIP04_MAX_CPUS_PER_CLUSTER);
+
+	count = TIMEOUT_MSEC / POLL_MSEC;
+	spin_lock_irq(&boot_lock);
+	for (tries = 0; tries < count; tries++) {
+		if (hip04_cpu_table[cluster][cpu])
+			goto err;
+		cpu_relax();
+		data = readl_relaxed(sysctrl + SC_CPU_RESET_STATUS(cluster));
+		if (data & CORE_WFI_STATUS(cpu))
+			break;
+		spin_unlock_irq(&boot_lock);
+		/* Wait for clean L2 when the whole cluster is down. */
+		msleep(POLL_MSEC);
+		spin_lock_irq(&boot_lock);
+	}
+	if (tries >= count)
+		goto err;
+	data = CORE_RESET_BIT(cpu) | NEON_RESET_BIT(cpu) | \
+	       CORE_DEBUG_RESET_BIT(cpu);
+	writel_relaxed(data, sysctrl + SC_CPU_RESET_REQ(cluster));
+	for (tries = 0; tries < count; tries++) {
+		cpu_relax();
+		data = readl_relaxed(sysctrl + SC_CPU_RESET_STATUS(cluster));
+		if (data & CORE_RESET_STATUS(cpu))
+			break;
+	}
+	if (tries >= count)
+		goto err;
+	if (hip04_cluster_is_down(cluster))
+		hip04_set_snoop_filter(cluster, 0);
+	spin_unlock_irq(&boot_lock);
+	return 1;
+err:
+	spin_unlock_irq(&boot_lock);
+	return 0;
+}
+#endif
+
+static const struct smp_operations hip04_smp_ops __initconst = {
+	.smp_boot_secondary	= hip04_boot_secondary,
+#ifdef CONFIG_HOTPLUG_CPU
+	.cpu_die		= hip04_cpu_die,
+	.cpu_kill		= hip04_cpu_kill,
+#endif
+};
+
+static bool __init hip04_cpu_table_init(void)
+{
+	unsigned int mpidr, cpu, cluster;
+
+	mpidr = read_cpuid_mpidr();
+	cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
+	cluster = MPIDR_AFFINITY_LEVEL(mpidr, 1);
+
+	if (cluster >= HIP04_MAX_CLUSTERS ||
+	    cpu >= HIP04_MAX_CPUS_PER_CLUSTER) {
+		pr_err("%s: boot CPU is out of bound!\n", __func__);
+		return false;
+	}
+	hip04_set_snoop_filter(cluster, 1);
+	hip04_cpu_table[cluster][cpu] = 1;
+	return true;
+}
+
+static int __init hip04_smp_init(void)
+{
+	struct device_node *np, *np_sctl, *np_fab;
+	struct resource fab_res;
+	void __iomem *relocation;
+	int ret = -ENODEV;
+
+	np = of_find_compatible_node(NULL, NULL, "hisilicon,hip04-bootwrapper");
+	if (!np)
+		goto err;
+	ret = of_property_read_u32_array(np, "boot-method",
+					 &hip04_boot_method[0], 4);
+	if (ret)
+		goto err;
+
+	ret = -ENODEV;
+	np_sctl = of_find_compatible_node(NULL, NULL, "hisilicon,sysctrl");
+	if (!np_sctl)
+		goto err;
+	np_fab = of_find_compatible_node(NULL, NULL, "hisilicon,hip04-fabric");
+	if (!np_fab)
+		goto err;
+
+	ret = memblock_reserve(hip04_boot_method[0], hip04_boot_method[1]);
+	if (ret)
+		goto err;
+
+	relocation = ioremap(hip04_boot_method[2], hip04_boot_method[3]);
+	if (!relocation) {
+		pr_err("failed to map relocation space\n");
+		ret = -ENOMEM;
+		goto err_reloc;
+	}
+	sysctrl = of_iomap(np_sctl, 0);
+	if (!sysctrl) {
+		pr_err("failed to get sysctrl base\n");
+		ret = -ENOMEM;
+		goto err_sysctrl;
+	}
+	ret = of_address_to_resource(np_fab, 0, &fab_res);
+	if (ret) {
+		pr_err("failed to get fabric base phys\n");
+		goto err_fabric;
+	}
+	fabric_phys_addr = fab_res.start;
+	sync_cache_w(&fabric_phys_addr);
+	fabric = of_iomap(np_fab, 0);
+	if (!fabric) {
+		pr_err("failed to get fabric base\n");
+		ret = -ENOMEM;
+		goto err_fabric;
+	}
+
+	if (!hip04_cpu_table_init()) {
+		ret = -EINVAL;
+		goto err_table;
+	}
+
+	/*
+	 * Fill the instruction address that is used after secondary core
+	 * out of reset.
+	 */
+	writel_relaxed(hip04_boot_method[0], relocation);
+	writel_relaxed(0xa5a5a5a5, relocation + 4);	/* magic number */
+	writel_relaxed(__pa_symbol(secondary_startup), relocation + 8);
+	writel_relaxed(0, relocation + 12);
+	iounmap(relocation);
+
+	smp_set_ops(&hip04_smp_ops);
+	return ret;
+err_table:
+	iounmap(fabric);
+err_fabric:
+	iounmap(sysctrl);
+err_sysctrl:
+	iounmap(relocation);
+err_reloc:
+	memblock_phys_free(hip04_boot_method[0], hip04_boot_method[1]);
+err:
+	return ret;
+}
+early_initcall(hip04_smp_init);
diff --git a/arch/arm/mach-hisi/platsmp.c b/arch/arm/mach-hisi/platsmp.c
new file mode 100644
index 000000000..9ce93e0b6
--- /dev/null
+++ b/arch/arm/mach-hisi/platsmp.c
@@ -0,0 +1,187 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2013 Linaro Ltd.
+ * Copyright (c) 2013 HiSilicon Limited.
+ * Based on arch/arm/mach-vexpress/platsmp.c, Copyright (C) 2002 ARM Ltd.
+ */
+#include <linux/smp.h>
+#include <linux/io.h>
+#include <linux/of_address.h>
+#include <linux/delay.h>
+
+#include <asm/cacheflush.h>
+#include <asm/smp_plat.h>
+#include <asm/smp_scu.h>
+#include <asm/mach/map.h>
+
+#include "core.h"
+
+#define HIX5HD2_BOOT_ADDRESS		0xffff0000
+
+static void __iomem *ctrl_base;
+
+void hi3xxx_set_cpu_jump(int cpu, void *jump_addr)
+{
+	cpu = cpu_logical_map(cpu);
+	if (!cpu || !ctrl_base)
+		return;
+	writel_relaxed(__pa_symbol(jump_addr), ctrl_base + ((cpu - 1) << 2));
+}
+
+int hi3xxx_get_cpu_jump(int cpu)
+{
+	cpu = cpu_logical_map(cpu);
+	if (!cpu || !ctrl_base)
+		return 0;
+	return readl_relaxed(ctrl_base + ((cpu - 1) << 2));
+}
+
+static void __init hisi_enable_scu_a9(void)
+{
+	unsigned long base = 0;
+	void __iomem *scu_base = NULL;
+
+	if (scu_a9_has_base()) {
+		base = scu_a9_get_base();
+		scu_base = ioremap(base, SZ_4K);
+		if (!scu_base) {
+			pr_err("ioremap(scu_base) failed\n");
+			return;
+		}
+		scu_enable(scu_base);
+		iounmap(scu_base);
+	}
+}
+
+static void __init hi3xxx_smp_prepare_cpus(unsigned int max_cpus)
+{
+	struct device_node *np = NULL;
+	u32 offset = 0;
+
+	hisi_enable_scu_a9();
+	if (!ctrl_base) {
+		np = of_find_compatible_node(NULL, NULL, "hisilicon,sysctrl");
+		if (!np) {
+			pr_err("failed to find hisilicon,sysctrl node\n");
+			return;
+		}
+		ctrl_base = of_iomap(np, 0);
+		if (!ctrl_base) {
+			of_node_put(np);
+			pr_err("failed to map address\n");
+			return;
+		}
+		if (of_property_read_u32(np, "smp-offset", &offset) < 0) {
+			of_node_put(np);
+			pr_err("failed to find smp-offset property\n");
+			return;
+		}
+		ctrl_base += offset;
+		of_node_put(np);
+	}
+}
+
+static int hi3xxx_boot_secondary(unsigned int cpu, struct task_struct *idle)
+{
+	hi3xxx_set_cpu(cpu, true);
+	hi3xxx_set_cpu_jump(cpu, secondary_startup);
+	arch_send_wakeup_ipi_mask(cpumask_of(cpu));
+	return 0;
+}
+
+static const struct smp_operations hi3xxx_smp_ops __initconst = {
+	.smp_prepare_cpus	= hi3xxx_smp_prepare_cpus,
+	.smp_boot_secondary	= hi3xxx_boot_secondary,
+#ifdef CONFIG_HOTPLUG_CPU
+	.cpu_die		= hi3xxx_cpu_die,
+	.cpu_kill		= hi3xxx_cpu_kill,
+#endif
+};
+
+static void __init hisi_common_smp_prepare_cpus(unsigned int max_cpus)
+{
+	hisi_enable_scu_a9();
+}
+
+static void hix5hd2_set_scu_boot_addr(phys_addr_t start_addr, phys_addr_t jump_addr)
+{
+	void __iomem *virt;
+
+	virt = ioremap(start_addr, PAGE_SIZE);
+
+	writel_relaxed(0xe51ff004, virt);	/* ldr pc, [pc, #-4] */
+	writel_relaxed(jump_addr, virt + 4);	/* pc jump phy address */
+	iounmap(virt);
+}
+
+static int hix5hd2_boot_secondary(unsigned int cpu, struct task_struct *idle)
+{
+	phys_addr_t jumpaddr;
+
+	jumpaddr = __pa_symbol(secondary_startup);
+	hix5hd2_set_scu_boot_addr(HIX5HD2_BOOT_ADDRESS, jumpaddr);
+	hix5hd2_set_cpu(cpu, true);
+	arch_send_wakeup_ipi_mask(cpumask_of(cpu));
+	return 0;
+}
+
+
+static const struct smp_operations hix5hd2_smp_ops __initconst = {
+	.smp_prepare_cpus	= hisi_common_smp_prepare_cpus,
+	.smp_boot_secondary	= hix5hd2_boot_secondary,
+#ifdef CONFIG_HOTPLUG_CPU
+	.cpu_die		= hix5hd2_cpu_die,
+#endif
+};
+
+
+#define SC_SCTL_REMAP_CLR      0x00000100
+#define HIP01_BOOT_ADDRESS     0x80000000
+#define REG_SC_CTRL            0x000
+
+static void hip01_set_boot_addr(phys_addr_t start_addr, phys_addr_t jump_addr)
+{
+	void __iomem *virt;
+
+	virt = phys_to_virt(start_addr);
+
+	writel_relaxed(0xe51ff004, virt);
+	writel_relaxed(jump_addr, virt + 4);
+}
+
+static int hip01_boot_secondary(unsigned int cpu, struct task_struct *idle)
+{
+	phys_addr_t jumpaddr;
+	unsigned int remap_reg_value = 0;
+	struct device_node *node;
+
+
+	jumpaddr = __pa_symbol(secondary_startup);
+	hip01_set_boot_addr(HIP01_BOOT_ADDRESS, jumpaddr);
+
+	node = of_find_compatible_node(NULL, NULL, "hisilicon,hip01-sysctrl");
+	if (WARN_ON(!node))
+		return -1;
+	ctrl_base = of_iomap(node, 0);
+	of_node_put(node);
+
+	/* set the secondary core boot from DDR */
+	remap_reg_value = readl_relaxed(ctrl_base + REG_SC_CTRL);
+	barrier();
+	remap_reg_value |= SC_SCTL_REMAP_CLR;
+	barrier();
+	writel_relaxed(remap_reg_value, ctrl_base + REG_SC_CTRL);
+
+	hip01_set_cpu(cpu, true);
+
+	return 0;
+}
+
+static const struct smp_operations hip01_smp_ops __initconst = {
+	.smp_prepare_cpus       = hisi_common_smp_prepare_cpus,
+	.smp_boot_secondary     = hip01_boot_secondary,
+};
+
+CPU_METHOD_OF_DECLARE(hi3xxx_smp, "hisilicon,hi3620-smp", &hi3xxx_smp_ops);
+CPU_METHOD_OF_DECLARE(hix5hd2_smp, "hisilicon,hix5hd2-smp", &hix5hd2_smp_ops);
+CPU_METHOD_OF_DECLARE(hip01_smp, "hisilicon,hip01-smp", &hip01_smp_ops);