Merge tag 'trace-v7.3-2' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-tracegrafted

Pull tracing fixes from Steven Rostedt:

 - Fix error output of boot instance creation failure

   Currently if a boot instance creation fails, instead of printing out
   the name of the instance that failed, it prints "(null)". That is
   because it prints "cur_str" that had already been processed by
   strsep(). Print the saved name instead.

   While at it, print the error code of the failure.

 - Fix use-after-free for same named historgrams

   Histograms can be named so that they can be used in multiple events.
   But if the named histogram has a variable attached, the second event
   that uses the named histogram which duplicates it and needs to free
   the original after duplication leaves the old variable in place and
   still visible. If another histogram uses than variable, it will use
   the stale one which will try to reference the freed duplicate
   histogram and crash the kernel.

   Free the duplicate variables along with the duplicated histogram
   data.

 - Check return value of kthread_run() in event self test

   The events self tests uses a kthread for testing but does not check
   if it succeeded in creating a kthread. If the kthread creation were
   to fail, the code will still try to call kthread_stop() on the error
   returned.

 - Fix race between reading trace_pipe and updating subbuffer size

   If a user is reading the trace_pipe file at the same time they update
   the ring buffer sub-buffer size, can cause the trace_pipe read to
   read stale data. Add trace_access_lock() around updating the ring
   buffer sub-buffer size.

 - Fix eventfs_inode on failure path in creation of the events directory

   In the creation of the "events" directory, if after allocating the
   eventfs_inode a failure is detected, it calls cleanup_ei() which
   calls free_ei(). The free_ei() will test if eventfs_inode being freed
   has no children. It is a bug if it does. But on the failure case of
   the creation of the "events" directory, the children lists have not
   yet been initialized and the free will trigger a warning because
   list_empty() on an uninitialized list returns false.

   Move the initialization into init_ei() where it makes more sense and
   makes sure that a created eventfs_inode has its lists initialized
   upon creation.

 - Check return value of kthread_run() in ftrace direct sample code

   The sample code that shows how to use the ftrace direct calls does
   not test the return of kthread_run() to see if it succeeds. Return a
   failure if the kthread_run() doesn't succeed.

 - Clear user events state on fork in case of alloc failure

   On fork, the child gets a pointer to the parent's user events state.
   It makes a copy of it then updates the child's pointer to it. But if
   the allocation fails, the duplication function leaves the child with
   a pointer to its parent's descriptor. When the child cleans up its
   data, it will free the parent's descriptor while the parent is still
   using it.

   In the duplication function, set the child's user_event_mm to NULL
   before testing if the allocation succeeded, and when it exits it will
   not free the parent's descriptor.

 - Fix retry exhaustion in simple ring buffer reader swap

   simple_ring_buffer_swap_reader_page() starts with retry set to 8 and
   post-decrements it only after a failed link replacement. On the final
   attempt, a successful replacement leaves retry at zero, while a
   failed replacement leaves it at -1.

   But the check for success expects the retry value to be non-zero and
   exits with an error on zero. This is the opposite result. Fix it.

 - Fail nicely when the remote swap_reader_page() returns an error

   Currently, if the swap_reader_page() of a remote buffer fails, it
   triggers a WARN_ON_ONCE() and continues normally. Instead, have it
   exit with an error and a pr_warn() print instead of a full WARNING.

* tag 'trace-v7.3-2' of git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace:
  ring-buffer: Stop remote reader update when page swap fails
  tracing: Fix retry exhaustion in simple ring buffer reader swap
  tracing/user_events: Clear copied tracing state before fork duplication
  samples/ftrace: Fix kthread_stop() on ERR_PTR in ftrace-direct-multi-modify
  samples/ftrace: Fix kthread_stop() on ERR_PTR in ftrace-direct-modify
  eventfs: Initialize ei->children and ei->list in init_ei()
  tracing: Fix use-after-free in trace_pipe read on sub-buffer order change
  tracing: Fix crash passing ERR_PTR to kthread_stop()
  tracing: Fix use-after-free with same-name named triggers
  tracing: Fix logged instance name on creation failure

1 files changed, 862 insertions, 0 deletions

diff --git a/drivers/perf/arm-ni.c b/drivers/perf/arm-ni.c
new file mode 100644
index 000000000..03a1c6bf9
--- /dev/null
+++ b/drivers/perf/arm-ni.c
@@ -0,0 +1,862 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (C) 2022-2024 Arm Limited
+// NI-700 Network-on-Chip PMU driver
+
+#include <linux/acpi.h>
+#include <linux/bitfield.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/perf_event.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+/* Common registers */
+#define NI_NODE_TYPE		0x000
+#define NI_NODE_TYPE_NODE_ID	GENMASK(31, 16)
+#define NI_NODE_TYPE_NODE_TYPE	GENMASK(15, 0)
+
+#define NI_CHILD_NODE_INFO	0x004
+#define NI_CHILD_PTR(n)		(0x008 + (n) * 4)
+#define NI_NUM_SUB_FEATURES	0x100
+#define NI_SUB_FEATURE_TYPE(n)	(0x108 + (n) * 8)
+#define NI_SUB_FEATURE_PTR(n)	(0x10c + (n) * 8)
+
+#define NI_SUB_FEATURE_TYPE_FCU	0x2
+
+#define NI700_PMUSELA		0x00c
+
+/* Config node */
+#define NI_PERIPHERAL_ID0	0xfe0
+#define NI_PIDR0_PART_7_0	GENMASK(7, 0)
+#define NI_PERIPHERAL_ID1	0xfe4
+#define NI_PIDR1_PART_11_8	GENMASK(3, 0)
+#define NI_PERIPHERAL_ID2	0xfe8
+#define NI_PIDR2_VERSION	GENMASK(7, 4)
+
+/* PMU node */
+#define NI700_PMEVCNTR(n)	(0x008 + (n) * 8)
+#define NI700_PMCCNTR_L		0x0f8
+#define NI_PMEVCNTR(n)		(0x200 + (n) * 8)
+#define NI_PMCCNTR_L		0x2f8
+#define NI_PMEVTYPER(n)		(0x400 + (n) * 4)
+#define NI_PMEVTYPER_NODE_TYPE	GENMASK(12, 9)
+#define NI_PMEVTYPER_NODE_ID	GENMASK(8, 0)
+#define NI_PMCNTENSET		0xc00
+#define NI_PMCNTENCLR		0xc20
+#define NI_PMINTENSET		0xc40
+#define NI_PMINTENCLR		0xc60
+#define NI_PMOVSCLR		0xc80
+#define NI_PMOVSSET		0xcc0
+#define NI_PMCFGR		0xe00
+#define NI_PMCR			0xe04
+#define NI_PMCR_RESET_CCNT	BIT(2)
+#define NI_PMCR_RESET_EVCNT	BIT(1)
+#define NI_PMCR_ENABLE		BIT(0)
+
+#define NI_NUM_COUNTERS		8
+#define NI_CCNT_IDX		31
+
+/* Event attributes */
+#define NI_CONFIG_TYPE		GENMASK_ULL(15, 0)
+#define NI_CONFIG_NODEID	GENMASK_ULL(31, 16)
+#define NI_CONFIG_EVENTID	GENMASK_ULL(47, 32)
+
+#define NI_EVENT_TYPE(event)	FIELD_GET(NI_CONFIG_TYPE, (event)->attr.config)
+#define NI_EVENT_NODEID(event)	FIELD_GET(NI_CONFIG_NODEID, (event)->attr.config)
+#define NI_EVENT_EVENTID(event)	FIELD_GET(NI_CONFIG_EVENTID, (event)->attr.config)
+
+enum ni_part {
+	PART_NI_700 = 0x43b,
+	PART_NI_710AE = 0x43d,
+	PART_NOC_S3 = 0x43f,
+	PART_SI_L1 = 0x455,
+};
+
+enum ni_node_type {
+	NI_GLOBAL,
+	NI_VOLTAGE,
+	NI_POWER,
+	NI_CLOCK,
+	NI_ASNI,
+	NI_AMNI,
+	NI_PMU,
+	NI_HSNI,
+	NI_HMNI,
+	NI_PMNI,
+	NI_TSNI,
+	NI_TMNI,
+	NI_CMNI = 0x0e,
+	NI_MCN = 0x63,
+};
+
+struct arm_ni_node {
+	void __iomem *base;
+	enum ni_node_type type;
+	u16 id;
+	u32 num_components;
+};
+
+struct arm_ni_unit {
+	void __iomem *pmusela;
+	enum ni_node_type type;
+	u16 id;
+	bool ns;
+	union {
+		__le64 pmusel;
+		u8 event[8];
+	};
+};
+
+struct arm_ni_cd {
+	void __iomem *pmu_base;
+	u16 id;
+	s8 irq_friend;
+	int num_units;
+	int irq;
+	struct pmu pmu;
+	struct arm_ni_unit *units;
+	struct perf_event *evcnt[NI_NUM_COUNTERS];
+	struct perf_event *ccnt;
+};
+
+struct arm_ni {
+	struct device *dev;
+	void __iomem *base;
+	enum ni_part part;
+	int id;
+	int cpu;
+	int num_cds;
+	struct hlist_node cpuhp_node;
+	struct arm_ni_cd cds[] __counted_by(num_cds);
+};
+
+#define cd_to_ni(cd) container_of((cd), struct arm_ni, cds[(cd)->id])
+#define pmu_to_cd(p) container_of((p), struct arm_ni_cd, pmu)
+
+#define ni_for_each_cd(n, c) \
+	for (struct arm_ni_cd *c = n->cds; c < n->cds + n->num_cds; c++) if (c->pmu_base)
+
+#define cd_for_each_unit(cd, u) \
+	for (struct arm_ni_unit *u = cd->units; u < cd->units + cd->num_units; u++)
+
+static int arm_ni_hp_state;
+
+struct arm_ni_event_attr {
+	struct device_attribute attr;
+	enum ni_node_type type;
+};
+
+#define NI_EVENT_ATTR(_name, _type)					\
+	(&((struct arm_ni_event_attr[]) {{				\
+		.attr = __ATTR(_name, 0444, arm_ni_event_show, NULL),	\
+		.type = _type,						\
+	}})[0].attr.attr)
+
+static ssize_t arm_ni_event_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	struct arm_ni_event_attr *eattr = container_of(attr, typeof(*eattr), attr);
+
+	if (eattr->type == NI_PMU)
+		return sysfs_emit(buf, "type=0x%x\n", eattr->type);
+
+	return sysfs_emit(buf, "type=0x%x,eventid=?,nodeid=?\n", eattr->type);
+}
+
+static umode_t arm_ni_event_attr_is_visible(struct kobject *kobj,
+					    struct attribute *attr, int unused)
+{
+	struct device *dev = kobj_to_dev(kobj);
+	struct arm_ni_cd *cd = pmu_to_cd(dev_get_drvdata(dev));
+	struct arm_ni_event_attr *eattr;
+
+	eattr = container_of(attr, typeof(*eattr), attr.attr);
+
+	cd_for_each_unit(cd, unit) {
+		if (unit->type == eattr->type && unit->ns)
+			return attr->mode;
+	}
+
+	return 0;
+}
+
+static struct attribute *arm_ni_event_attrs[] = {
+	NI_EVENT_ATTR(asni, NI_ASNI),
+	NI_EVENT_ATTR(amni, NI_AMNI),
+	NI_EVENT_ATTR(cycles, NI_PMU),
+	NI_EVENT_ATTR(hsni, NI_HSNI),
+	NI_EVENT_ATTR(hmni, NI_HMNI),
+	NI_EVENT_ATTR(pmni, NI_PMNI),
+	NI_EVENT_ATTR(tsni, NI_TSNI),
+	NI_EVENT_ATTR(tmni, NI_TMNI),
+	NI_EVENT_ATTR(cmni, NI_CMNI),
+	NULL
+};
+
+static const struct attribute_group arm_ni_event_attrs_group = {
+	.name = "events",
+	.attrs = arm_ni_event_attrs,
+	.is_visible = arm_ni_event_attr_is_visible,
+};
+
+struct arm_ni_format_attr {
+	struct device_attribute attr;
+	u64 field;
+};
+
+#define NI_FORMAT_ATTR(_name, _fld)					\
+	(&((struct arm_ni_format_attr[]) {{				\
+		.attr = __ATTR(_name, 0444, arm_ni_format_show, NULL),	\
+		.field = _fld,						\
+	}})[0].attr.attr)
+
+static ssize_t arm_ni_format_show(struct device *dev,
+				  struct device_attribute *attr, char *buf)
+{
+	struct arm_ni_format_attr *fmt = container_of(attr, typeof(*fmt), attr);
+
+	return sysfs_emit(buf, "config:%*pbl\n", 64, &fmt->field);
+}
+
+static struct attribute *arm_ni_format_attrs[] = {
+	NI_FORMAT_ATTR(type, NI_CONFIG_TYPE),
+	NI_FORMAT_ATTR(nodeid, NI_CONFIG_NODEID),
+	NI_FORMAT_ATTR(eventid, NI_CONFIG_EVENTID),
+	NULL
+};
+
+static const struct attribute_group arm_ni_format_attrs_group = {
+	.name = "format",
+	.attrs = arm_ni_format_attrs,
+};
+
+static ssize_t arm_ni_cpumask_show(struct device *dev,
+				   struct device_attribute *attr, char *buf)
+{
+	struct arm_ni *ni = cd_to_ni(pmu_to_cd(dev_get_drvdata(dev)));
+
+	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(cpumask_of(ni->cpu)));
+}
+
+static struct device_attribute arm_ni_cpumask_attr =
+		__ATTR(cpumask, 0444, arm_ni_cpumask_show, NULL);
+
+static ssize_t arm_ni_identifier_show(struct device *dev,
+				      struct device_attribute *attr, char *buf)
+{
+	struct arm_ni *ni = cd_to_ni(pmu_to_cd(dev_get_drvdata(dev)));
+	u32 reg = readl_relaxed(ni->base + NI_PERIPHERAL_ID2);
+	int version = FIELD_GET(NI_PIDR2_VERSION, reg);
+
+	return sysfs_emit(buf, "%03x%02x\n", ni->part, version);
+}
+
+static struct device_attribute arm_ni_identifier_attr =
+		__ATTR(identifier, 0444, arm_ni_identifier_show, NULL);
+
+static struct attribute *arm_ni_other_attrs[] = {
+	&arm_ni_cpumask_attr.attr,
+	&arm_ni_identifier_attr.attr,
+	NULL
+};
+
+static const struct attribute_group arm_ni_other_attr_group = {
+	.attrs = arm_ni_other_attrs,
+};
+
+static const struct attribute_group *arm_ni_attr_groups[] = {
+	&arm_ni_event_attrs_group,
+	&arm_ni_format_attrs_group,
+	&arm_ni_other_attr_group,
+	NULL
+};
+
+static void arm_ni_pmu_enable(struct pmu *pmu)
+{
+	writel_relaxed(NI_PMCR_ENABLE, pmu_to_cd(pmu)->pmu_base + NI_PMCR);
+}
+
+static void arm_ni_pmu_disable(struct pmu *pmu)
+{
+	writel_relaxed(0, pmu_to_cd(pmu)->pmu_base + NI_PMCR);
+}
+
+struct arm_ni_val {
+	unsigned int evcnt;
+	unsigned int ccnt;
+};
+
+static bool arm_ni_val_count_event(struct perf_event *evt, struct arm_ni_val *val)
+{
+	if (is_software_event(evt))
+		return true;
+
+	if (NI_EVENT_TYPE(evt) == NI_PMU) {
+		val->ccnt++;
+		return val->ccnt <= 1;
+	}
+
+	val->evcnt++;
+	return val->evcnt <= NI_NUM_COUNTERS;
+}
+
+static int arm_ni_validate_group(struct perf_event *event)
+{
+	struct perf_event *sibling, *leader = event->group_leader;
+	struct arm_ni_val val = { 0 };
+
+	if (leader == event)
+		return 0;
+
+	arm_ni_val_count_event(event, &val);
+	if (!arm_ni_val_count_event(leader, &val))
+		return -EINVAL;
+
+	for_each_sibling_event(sibling, leader) {
+		if (!arm_ni_val_count_event(sibling, &val))
+			return -EINVAL;
+	}
+	return 0;
+}
+
+static bool arm_ni_is_7xx(const struct arm_ni *ni)
+{
+	return ni->part == PART_NI_700 || ni->part == PART_NI_710AE;
+}
+
+static int arm_ni_event_init(struct perf_event *event)
+{
+	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
+	struct arm_ni *ni;
+
+	if (event->attr.type != event->pmu->type)
+		return -ENOENT;
+
+	if (is_sampling_event(event))
+		return -EINVAL;
+
+	ni = cd_to_ni(cd);
+	event->cpu = ni->cpu;
+	event->hw.flags = arm_ni_is_7xx(ni);
+
+	if (NI_EVENT_TYPE(event) == NI_PMU)
+		return arm_ni_validate_group(event);
+
+	cd_for_each_unit(cd, unit) {
+		if (unit->type == NI_EVENT_TYPE(event) &&
+		    unit->id == NI_EVENT_NODEID(event) && unit->ns) {
+			event->hw.config_base = (unsigned long)unit;
+			return arm_ni_validate_group(event);
+		}
+	}
+	return -EINVAL;
+}
+
+static u64 arm_ni_read_ccnt(void __iomem *pmccntr)
+{
+	u64 l, u_old, u_new;
+	int retries = 3; /* 1st time unlucky, 2nd improbable, 3rd just broken */
+
+	u_new = readl_relaxed(pmccntr + 4);
+	do {
+		u_old = u_new;
+		l = readl_relaxed(pmccntr);
+		u_new = readl_relaxed(pmccntr + 4);
+	} while (u_new != u_old && --retries);
+	WARN_ON(!retries);
+
+	return (u_new << 32) | l;
+}
+
+static void arm_ni_event_read(struct perf_event *event)
+{
+	struct hw_perf_event *hw = &event->hw;
+	u64 count, prev;
+	bool ccnt = hw->idx == NI_CCNT_IDX;
+
+	do {
+		prev = local64_read(&hw->prev_count);
+		if (ccnt)
+			count = arm_ni_read_ccnt((void __iomem *)event->hw.event_base);
+		else
+			count = readl_relaxed((void __iomem *)event->hw.event_base);
+	} while (local64_cmpxchg(&hw->prev_count, prev, count) != prev);
+
+	count -= prev;
+	if (!ccnt)
+		count = (u32)count;
+	local64_add(count, &event->count);
+}
+
+static void arm_ni_event_start(struct perf_event *event, int flags)
+{
+	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
+
+	writel_relaxed(1U << event->hw.idx, cd->pmu_base + NI_PMCNTENSET);
+}
+
+static void arm_ni_event_stop(struct perf_event *event, int flags)
+{
+	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
+
+	writel_relaxed(1U << event->hw.idx, cd->pmu_base + NI_PMCNTENCLR);
+	if (flags & PERF_EF_UPDATE)
+		arm_ni_event_read(event);
+}
+
+static void arm_ni_init_ccnt(struct hw_perf_event *hw)
+{
+	local64_set(&hw->prev_count, S64_MIN);
+	lo_hi_writeq_relaxed(S64_MIN, (void __iomem *)hw->event_base);
+}
+
+static void arm_ni_init_evcnt(struct hw_perf_event *hw)
+{
+	local64_set(&hw->prev_count, S32_MIN);
+	writel_relaxed(S32_MIN, (void __iomem *)hw->event_base);
+}
+
+static int arm_ni_event_add(struct perf_event *event, int flags)
+{
+	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
+	struct hw_perf_event *hw = &event->hw;
+	struct arm_ni_unit *unit;
+	enum ni_node_type type = NI_EVENT_TYPE(event);
+	u32 reg;
+
+	if (type == NI_PMU) {
+		if (cd->ccnt)
+			return -ENOSPC;
+		hw->idx = NI_CCNT_IDX;
+		hw->event_base = (unsigned long)cd->pmu_base +
+				 (hw->flags ? NI700_PMCCNTR_L : NI_PMCCNTR_L);
+		cd->ccnt = event;
+		arm_ni_init_ccnt(hw);
+	} else {
+		hw->idx = 0;
+		while (cd->evcnt[hw->idx]) {
+			if (++hw->idx == NI_NUM_COUNTERS)
+				return -ENOSPC;
+		}
+		cd->evcnt[hw->idx] = event;
+		unit = (void *)hw->config_base;
+		unit->event[hw->idx] = NI_EVENT_EVENTID(event);
+		hw->event_base = (unsigned long)cd->pmu_base +
+				 (hw->flags ? NI700_PMEVCNTR(hw->idx) : NI_PMEVCNTR(hw->idx));
+		arm_ni_init_evcnt(hw);
+		lo_hi_writeq_relaxed(le64_to_cpu(unit->pmusel), unit->pmusela);
+
+		reg = FIELD_PREP(NI_PMEVTYPER_NODE_TYPE, type) |
+		      FIELD_PREP(NI_PMEVTYPER_NODE_ID, NI_EVENT_NODEID(event));
+		writel_relaxed(reg, cd->pmu_base + NI_PMEVTYPER(hw->idx));
+	}
+	if (flags & PERF_EF_START)
+		arm_ni_event_start(event, 0);
+	return 0;
+}
+
+static void arm_ni_event_del(struct perf_event *event, int flags)
+{
+	struct arm_ni_cd *cd = pmu_to_cd(event->pmu);
+	struct hw_perf_event *hw = &event->hw;
+
+	arm_ni_event_stop(event, PERF_EF_UPDATE);
+
+	if (hw->idx == NI_CCNT_IDX)
+		cd->ccnt = NULL;
+	else
+		cd->evcnt[hw->idx] = NULL;
+}
+
+static irqreturn_t arm_ni_handle_irq(int irq, void *dev_id)
+{
+	struct arm_ni_cd *cd = dev_id;
+	irqreturn_t ret = IRQ_NONE;
+
+	for (;;) {
+		u32 reg = readl_relaxed(cd->pmu_base + NI_PMOVSCLR);
+
+		if (reg & (1U << NI_CCNT_IDX)) {
+			ret = IRQ_HANDLED;
+			if (!(WARN_ON(!cd->ccnt))) {
+				arm_ni_event_read(cd->ccnt);
+				arm_ni_init_ccnt(&cd->ccnt->hw);
+			}
+		}
+		for (int i = 0; i < NI_NUM_COUNTERS; i++) {
+			if (!(reg & (1U << i)))
+				continue;
+			ret = IRQ_HANDLED;
+			if (!(WARN_ON(!cd->evcnt[i]))) {
+				arm_ni_event_read(cd->evcnt[i]);
+				arm_ni_init_evcnt(&cd->evcnt[i]->hw);
+			}
+		}
+		writel_relaxed(reg, cd->pmu_base + NI_PMOVSCLR);
+		if (!cd->irq_friend)
+			return ret;
+		cd += cd->irq_friend;
+	}
+}
+
+static void __iomem *arm_ni_get_pmusel(struct arm_ni *ni, void __iomem *unit_base)
+{
+	u32 type, ptr, num;
+
+	if (arm_ni_is_7xx(ni))
+		return unit_base + NI700_PMUSELA;
+
+	num = readl_relaxed(unit_base + NI_NUM_SUB_FEATURES);
+	for (int i = 0; i < num; i++) {
+		type = readl_relaxed(unit_base + NI_SUB_FEATURE_TYPE(i));
+		if (type != NI_SUB_FEATURE_TYPE_FCU)
+			continue;
+		ptr = readl_relaxed(unit_base + NI_SUB_FEATURE_PTR(i));
+		return ni->base + ptr;
+	}
+	/* Should be impossible */
+	return NULL;
+}
+
+static int arm_ni_init_cd(struct arm_ni *ni, struct arm_ni_node *node, u64 res_start)
+{
+	struct arm_ni_cd *cd = ni->cds + node->id;
+	const char *name;
+
+	cd->id = node->id;
+	cd->num_units = node->num_components;
+	cd->units = devm_kcalloc(ni->dev, cd->num_units, sizeof(*(cd->units)), GFP_KERNEL);
+	if (!cd->units)
+		return -ENOMEM;
+
+	for (int i = 0; i < cd->num_units; i++) {
+		u32 reg = readl_relaxed(node->base + NI_CHILD_PTR(i));
+		void __iomem *unit_base = ni->base + reg;
+		struct arm_ni_unit *unit = cd->units + i;
+
+		reg = readl_relaxed(unit_base + NI_NODE_TYPE);
+		unit->type = FIELD_GET(NI_NODE_TYPE_NODE_TYPE, reg);
+		unit->id = FIELD_GET(NI_NODE_TYPE_NODE_ID, reg);
+
+		switch (unit->type) {
+		case NI_PMU:
+			reg = readl_relaxed(unit_base + NI_PMCFGR);
+			if (!reg) {
+				dev_info(ni->dev, "No access to PMU %d\n", cd->id);
+				devm_kfree(ni->dev, cd->units);
+				return 0;
+			}
+			unit->ns = true;
+			cd->pmu_base = unit_base;
+			break;
+		case NI_ASNI:
+		case NI_AMNI:
+		case NI_HSNI:
+		case NI_HMNI:
+		case NI_PMNI:
+		case NI_TSNI:
+		case NI_TMNI:
+		case NI_CMNI:
+			unit->pmusela = arm_ni_get_pmusel(ni, unit_base);
+			writel_relaxed(1, unit->pmusela);
+			if (readl_relaxed(unit->pmusela) != 1)
+				dev_info(ni->dev, "No access to node 0x%04x%04x\n", unit->id, unit->type);
+			else
+				unit->ns = true;
+			break;
+		case NI_MCN:
+			break;
+		default:
+			/*
+			 * e.g. FMU - thankfully bits 3:2 of FMU_ERR_FR0 are RES0 so
+			 * can't alias any of the leaf node types we're looking for.
+			 */
+			dev_dbg(ni->dev, "Mystery node 0x%04x%04x\n", unit->id, unit->type);
+			break;
+		}
+	}
+
+	res_start += cd->pmu_base - ni->base;
+	if (!devm_request_mem_region(ni->dev, res_start, SZ_4K, dev_name(ni->dev))) {
+		dev_err(ni->dev, "Failed to request PMU region 0x%llx\n", res_start);
+		return -EBUSY;
+	}
+
+	writel_relaxed(NI_PMCR_RESET_CCNT | NI_PMCR_RESET_EVCNT,
+		       cd->pmu_base + NI_PMCR);
+	writel_relaxed(U32_MAX, cd->pmu_base + NI_PMCNTENCLR);
+	writel_relaxed(U32_MAX, cd->pmu_base + NI_PMOVSCLR);
+
+	cd->irq = platform_get_irq(to_platform_device(ni->dev), cd->id);
+	if (cd->irq < 0)
+		return cd->irq;
+
+	cd->pmu = (struct pmu) {
+		.module = THIS_MODULE,
+		.parent = ni->dev,
+		.attr_groups = arm_ni_attr_groups,
+		.capabilities = PERF_PMU_CAP_NO_EXCLUDE,
+		.task_ctx_nr = perf_invalid_context,
+		.pmu_enable = arm_ni_pmu_enable,
+		.pmu_disable = arm_ni_pmu_disable,
+		.event_init = arm_ni_event_init,
+		.add = arm_ni_event_add,
+		.del = arm_ni_event_del,
+		.start = arm_ni_event_start,
+		.stop = arm_ni_event_stop,
+		.read = arm_ni_event_read,
+	};
+
+	name = devm_kasprintf(ni->dev, GFP_KERNEL, "arm_ni_%d_cd_%d", ni->id, cd->id);
+	if (!name)
+		return -ENOMEM;
+
+	return perf_pmu_register(&cd->pmu, name, -1);
+}
+
+static void arm_ni_remove(struct platform_device *pdev)
+{
+	struct arm_ni *ni = platform_get_drvdata(pdev);
+
+	ni_for_each_cd(ni, cd) {
+		writel_relaxed(0, cd->pmu_base + NI_PMCR);
+		writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENCLR);
+		perf_pmu_unregister(&cd->pmu);
+	}
+	cpuhp_state_remove_instance_nocalls(arm_ni_hp_state, &ni->cpuhp_node);
+}
+
+static void arm_ni_probe_domain(void __iomem *base, struct arm_ni_node *node)
+{
+	u32 reg = readl_relaxed(base + NI_NODE_TYPE);
+
+	node->base = base;
+	node->type = FIELD_GET(NI_NODE_TYPE_NODE_TYPE, reg);
+	node->id = FIELD_GET(NI_NODE_TYPE_NODE_ID, reg);
+	node->num_components = readl_relaxed(base + NI_CHILD_NODE_INFO);
+}
+
+static int arm_ni_init_irqs(struct arm_ni *ni)
+{
+	int err;
+
+	ni_for_each_cd(ni, cd) {
+		for (struct arm_ni_cd *prev = cd; prev-- > ni->cds; ) {
+			if (prev->irq == cd->irq) {
+				prev->irq_friend = cd - prev;
+				goto set_inten;
+			}
+		}
+		err = devm_request_irq(ni->dev, cd->irq, arm_ni_handle_irq,
+				       IRQF_NOBALANCING | IRQF_NO_THREAD | IRQF_NO_AUTOEN,
+				       dev_name(ni->dev), cd);
+		if (err)
+			return err;
+
+		irq_set_affinity(cd->irq, cpumask_of(ni->cpu));
+set_inten:
+		writel_relaxed(U32_MAX, cd->pmu_base + NI_PMINTENSET);
+	}
+
+	ni_for_each_cd(ni, cd)
+		if (!cd->irq_friend)
+			enable_irq(cd->irq);
+	return 0;
+}
+
+static int arm_ni_probe(struct platform_device *pdev)
+{
+	struct arm_ni_node cfg, vd, pd, cd;
+	struct arm_ni *ni;
+	struct resource *res;
+	void __iomem *base;
+	static atomic_t id;
+	int ret, num_cds;
+	u32 reg, part;
+
+	/*
+	 * We want to map the whole configuration space for ease of discovery,
+	 * but the PMU pages are the only ones for which we can honestly claim
+	 * exclusive ownership, so we'll request them explicitly once found.
+	 */
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap(&pdev->dev, res->start, resource_size(res));
+	if (!base)
+		return -ENOMEM;
+
+	arm_ni_probe_domain(base, &cfg);
+	if (cfg.type != NI_GLOBAL)
+		return -ENODEV;
+
+	reg = readl_relaxed(cfg.base + NI_PERIPHERAL_ID0);
+	part = FIELD_GET(NI_PIDR0_PART_7_0, reg);
+	reg = readl_relaxed(cfg.base + NI_PERIPHERAL_ID1);
+	part |= FIELD_GET(NI_PIDR1_PART_11_8, reg) << 8;
+
+	switch (part) {
+	case PART_NI_700:
+	case PART_NI_710AE:
+	case PART_NOC_S3:
+	case PART_SI_L1:
+		break;
+	default:
+		dev_WARN(&pdev->dev, "Unknown part number: 0x%03x, this may go badly\n", part);
+		break;
+	}
+
+	num_cds = 0;
+	for (int v = 0; v < cfg.num_components; v++) {
+		reg = readl_relaxed(cfg.base + NI_CHILD_PTR(v));
+		arm_ni_probe_domain(base + reg, &vd);
+		for (int p = 0; p < vd.num_components; p++) {
+			reg = readl_relaxed(vd.base + NI_CHILD_PTR(p));
+			arm_ni_probe_domain(base + reg, &pd);
+			num_cds += pd.num_components;
+		}
+	}
+
+	ni = devm_kzalloc(&pdev->dev, struct_size(ni, cds, num_cds), GFP_KERNEL);
+	if (!ni)
+		return -ENOMEM;
+
+	ni->dev = &pdev->dev;
+	ni->base = base;
+	ni->num_cds = num_cds;
+	ni->part = part;
+	ni->id = atomic_fetch_inc(&id);
+	ni->cpu = cpumask_local_spread(0, dev_to_node(ni->dev));
+	platform_set_drvdata(pdev, ni);
+
+	ret = cpuhp_state_add_instance_nocalls(arm_ni_hp_state, &ni->cpuhp_node);
+	if (ret)
+		return ret;
+
+	for (int v = 0; v < cfg.num_components; v++) {
+		reg = readl_relaxed(cfg.base + NI_CHILD_PTR(v));
+		arm_ni_probe_domain(base + reg, &vd);
+		for (int p = 0; p < vd.num_components; p++) {
+			reg = readl_relaxed(vd.base + NI_CHILD_PTR(p));
+			arm_ni_probe_domain(base + reg, &pd);
+			for (int c = 0; c < pd.num_components; c++) {
+				reg = readl_relaxed(pd.base + NI_CHILD_PTR(c));
+				arm_ni_probe_domain(base + reg, &cd);
+				ret = arm_ni_init_cd(ni, &cd, res->start);
+				if (ret) {
+					ni->cds[cd.id].pmu_base = NULL;
+					arm_ni_remove(pdev);
+					return ret;
+				}
+			}
+		}
+	}
+
+	ret = arm_ni_init_irqs(ni);
+	if (ret)
+		arm_ni_remove(pdev);
+
+	return ret;
+}
+
+#ifdef CONFIG_OF
+static const struct of_device_id arm_ni_of_match[] = {
+	{ .compatible = "arm,ni-700" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, arm_ni_of_match);
+#endif
+
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id arm_ni_acpi_match[] = {
+	{ "ARMHCB70" },
+	{}
+};
+MODULE_DEVICE_TABLE(acpi, arm_ni_acpi_match);
+#endif
+
+static struct platform_driver arm_ni_driver = {
+	.driver = {
+		.name = "arm-ni",
+		.of_match_table = of_match_ptr(arm_ni_of_match),
+		.acpi_match_table = ACPI_PTR(arm_ni_acpi_match),
+		.suppress_bind_attrs = true,
+	},
+	.probe = arm_ni_probe,
+	.remove = arm_ni_remove,
+};
+
+static void arm_ni_pmu_migrate(struct arm_ni *ni, unsigned int cpu)
+{
+	ni_for_each_cd(ni, cd) {
+		perf_pmu_migrate_context(&cd->pmu, ni->cpu, cpu);
+		irq_set_affinity(cd->irq, cpumask_of(cpu));
+	}
+	ni->cpu = cpu;
+}
+
+static int arm_ni_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
+{
+	struct arm_ni *ni;
+	int node;
+
+	ni = hlist_entry_safe(cpuhp_node, struct arm_ni, cpuhp_node);
+	node = dev_to_node(ni->dev);
+	if (cpu_to_node(ni->cpu) != node && cpu_to_node(cpu) == node)
+		arm_ni_pmu_migrate(ni, cpu);
+	return 0;
+}
+
+static int arm_ni_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node)
+{
+	struct arm_ni *ni;
+	unsigned int target;
+	int node;
+
+	ni = hlist_entry_safe(cpuhp_node, struct arm_ni, cpuhp_node);
+	if (cpu != ni->cpu)
+		return 0;
+
+	node = dev_to_node(ni->dev);
+	target = cpumask_any_and_but(cpumask_of_node(node), cpu_online_mask, cpu);
+	if (target >= nr_cpu_ids)
+		target = cpumask_any_but(cpu_online_mask, cpu);
+
+	if (target < nr_cpu_ids)
+		arm_ni_pmu_migrate(ni, target);
+	return 0;
+}
+
+static int __init arm_ni_init(void)
+{
+	int ret;
+
+	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
+				      "perf/arm/ni:online",
+				      arm_ni_pmu_online_cpu,
+				      arm_ni_pmu_offline_cpu);
+	if (ret < 0)
+		return ret;
+
+	arm_ni_hp_state = ret;
+
+	ret = platform_driver_register(&arm_ni_driver);
+	if (ret)
+		cpuhp_remove_multi_state(arm_ni_hp_state);
+	return ret;
+}
+
+static void __exit arm_ni_exit(void)
+{
+	platform_driver_unregister(&arm_ni_driver);
+	cpuhp_remove_multi_state(arm_ni_hp_state);
+}
+
+module_init(arm_ni_init);
+module_exit(arm_ni_exit);
+
+MODULE_AUTHOR("Robin Murphy <robin.murphy@arm.com>");
+MODULE_DESCRIPTION("Arm NI-700 PMU driver");
+MODULE_LICENSE("GPL v2");