x86/kprobes: Fix crash when probing CS CALL instructionsgrafted

When using eBPF to probe CS CALL instructions within a function,
a crash can be triggered.

The eBPF tool probes offset 257 of the __hrtimer_run_queues()
function:

<__hrtimer_run_queues+249>:  nopl   0x0(%rax,%rax,1)
<__hrtimer_run_queues+254>:  mov    %r14,%rdi
<__hrtimer_run_queues+257>:  cs call <__x86_indirect_thunk_r12>
<__hrtimer_run_queues+263>:  mov    %eax,%r12d
<__hrtimer_run_queues+266>:  xchg   %ax,%ax
<__hrtimer_run_queues+268>:  mov    %r13,%rdi

Which triggers this crash:

  BUG: unable to handle page fault for address: 00000000000f41c9
  #PF: supervisor write access in kernel mode
  #PF: error_code(0x0002) - not-present page
  PGD 0 P4D 0
  Oops: 0002 [#1] SMP NOPTI
  CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P
  RIP: 0010:__hrtimer_run_queues+0x106/0x230

Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is
at the 6th byte of the above CS CALL instruction. Since the CS CALL
instruction occupies 6 bytes, the exception occurred in the middle of that
call instruction.

The root cause is that when using eBPF tools to probe in the middle of a
function, a kprobe with INT3 is used as the underlying implementation.

During single-step emulation of the original CALL instruction,
int3_emulate_call() assumes that the probed CALL instruction is 5 bytes
long. However, the actual CS-prefixed CALL instruction occupies 6 bytes,
so it constructs an incorrect exception return address. When the CPU
returns from the kprobe handler, the next instruction to be executed is at
the address of the last byte of that CS CALL instruction. Coincidentally,
starting from that address, the CPU fetches and decodes a completely
different instruction, which ultimately triggers a kernel crash.

Fix the issue by using the actual instruction length obtained from
the instruction decoder when constructing the exception return
address, rather than relying on the hardcoded CALL_INSN_SIZE macro.

[ mingo: Refined the changelog ]

Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step")
Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Jinke Han <jinkehan@didiglobal.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Yafang Shao <laoar.shao@gmail.com>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000

1 files changed, 2017 insertions, 0 deletions

diff --git a/drivers/edac/igen6_edac.c b/drivers/edac/igen6_edac.c
new file mode 100644
index 000000000..776c5db2f
--- /dev/null
+++ b/drivers/edac/igen6_edac.c
@@ -0,0 +1,2017 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Driver for Intel client SoC with integrated memory controller using IBECC
+ *
+ * Copyright (C) 2020 Intel Corporation
+ *
+ * The In-Band ECC (IBECC) IP provides ECC protection to all or specific
+ * regions of the physical memory space. It's used for memory controllers
+ * that don't support the out-of-band ECC which often needs an additional
+ * storage device to each channel for storing ECC data.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+#include <linux/irq_work.h>
+#include <linux/llist.h>
+#include <linux/genalloc.h>
+#include <linux/edac.h>
+#include <linux/bits.h>
+#include <linux/bitfield.h>
+#include <linux/io.h>
+#include <asm/mach_traps.h>
+#include <asm/nmi.h>
+#include <asm/mce.h>
+
+#include "edac_mc.h"
+#include "edac_module.h"
+
+#define IGEN6_REVISION	"v2.5.1"
+
+#define EDAC_MOD_STR	"igen6_edac"
+#define IGEN6_NMI_NAME	"igen6_ibecc"
+
+/* Debug macros */
+#define igen6_printk(level, fmt, arg...)		\
+	edac_printk(level, "igen6", fmt, ##arg)
+
+#define igen6_mc_printk(mci, level, fmt, arg...)	\
+	edac_mc_chipset_printk(mci, level, "igen6", fmt, ##arg)
+
+#define GET_BITFIELD(v, lo, hi) (((v) & GENMASK_ULL(hi, lo)) >> (lo))
+
+/* Probing upper bound, not a hardware capability limit. */
+#define MAX_IMC_TO_PROBE		8
+#define NUM_CHANNELS			2 /* Max channels */
+#define NUM_DIMMS			2 /* Max DIMMs per channel */
+
+#define _4GB				BIT_ULL(32)
+
+/* Size of physical memory */
+#define TOM_OFFSET			0xa0
+/* Top of low usable DRAM */
+#define TOLUD_OFFSET			0xbc
+/* Capability register C */
+#define CAPID_C_OFFSET			0xec
+#define CAPID_C_IBECC			BIT(15)
+
+/* Capability register E */
+#define CAPID_E_OFFSET			0xf0
+#define CAPID_E_IBECC			BIT(12)
+#define CAPID_E_IBECC_BIT18		BIT(18)
+
+/* Error Status */
+#define ERRSTS_OFFSET			0xc8
+#define ERRSTS_CE			BIT_ULL(6)
+#define ERRSTS_UE			BIT_ULL(7)
+
+/* Error Command */
+#define ERRCMD_OFFSET			0xca
+#define ERRCMD_CE			BIT_ULL(6)
+#define ERRCMD_UE			BIT_ULL(7)
+
+/* IBECC MMIO base address */
+#define IBECC_BASE			(res_cfg->ibecc_base)
+#define IBECC_ACTIVATE_OFFSET		IBECC_BASE
+#define IBECC_ACTIVATE_EN		BIT(0)
+
+/* IBECC error log */
+#define ECC_ERROR_LOG_OFFSET		(IBECC_BASE + res_cfg->ibecc_error_log_offset)
+#define ECC_ERROR_LOG_CE		BIT_ULL(62)
+#define ECC_ERROR_LOG_UE		BIT_ULL(63)
+#define ECC_ERROR_LOG_SYND(v)		GET_BITFIELD(v, 46, 61)
+
+/* Host MMIO base address */
+#define MCHBAR_OFFSET			0x48
+#define MCHBAR_EN			BIT_ULL(0)
+#define MCHBAR_SIZE			0x10000
+
+/* Parameters for the channel decode stage */
+#define IMC_BASE			(res_cfg->imc_base)
+#define MAD_INTER_CHANNEL_OFFSET	IMC_BASE
+#define MAD_INTER_CHANNEL_DDR_TYPE(v)	GET_BITFIELD(v, 0, 2)
+#define MAD_INTER_CHANNEL_ECHM(v)	GET_BITFIELD(v, 3, 3)
+#define MAD_INTER_CHANNEL_CH_L_MAP(v)	GET_BITFIELD(v, 4, 4)
+#define MAD_INTER_CHANNEL_CH_S_SIZE(v)	((u64)GET_BITFIELD(v, 12, 19) << 29)
+
+/* Parameters for DRAM decode stage */
+#define MAD_INTRA_CH0_OFFSET		(IMC_BASE + 4)
+#define MAD_INTRA_CH_DIMM_L_MAP(v)	GET_BITFIELD(v, 0, 0)
+
+/* DIMM characteristics */
+#define MAD_DIMM_CH0_OFFSET		(IMC_BASE + 0xc)
+#define MAD_DIMM_CH_DIMM_L_SIZE(v)	((u64)GET_BITFIELD(v, 0, 6) << 29)
+#define MAD_DIMM_CH_DLW(v)		GET_BITFIELD(v, 7, 8)
+#define MAD_DIMM_CH_DIMM_S_SIZE(v)	((u64)GET_BITFIELD(v, 16, 22) << 29)
+#define MAD_DIMM_CH_DSW(v)		GET_BITFIELD(v, 24, 25)
+
+/* Hash for memory controller selection */
+#define MAD_MC_HASH_OFFSET		(IMC_BASE + 0x1b8)
+#define MAC_MC_HASH_LSB(v)		GET_BITFIELD(v, 1, 3)
+
+/* Hash for channel selection */
+#define CHANNEL_HASH_OFFSET		(IMC_BASE + 0x24)
+/* Hash for enhanced channel selection */
+#define CHANNEL_EHASH_OFFSET		(IMC_BASE + 0x28)
+#define CHANNEL_HASH_MASK(v)		(GET_BITFIELD(v, 6, 19) << 6)
+#define CHANNEL_HASH_LSB_MASK_BIT(v)	GET_BITFIELD(v, 24, 26)
+#define CHANNEL_HASH_MODE(v)		GET_BITFIELD(v, 28, 28)
+
+/* Parameters for memory slice decode stage */
+#define MEM_SLICE_HASH_MASK(v)		(GET_BITFIELD(v, 6, 19) << 6)
+#define MEM_SLICE_HASH_LSB_MASK_BIT(v)	GET_BITFIELD(v, 24, 26)
+
+/*
+ * A slice represents a portion of memory space participating in an
+ * interleave relationship within the memory hierarchy.
+ *
+ * It can represent in different levels such as:
+ *
+ *   - a pair of memory controllers
+ *   - a memory controller
+ *   - a memory channel
+ *   - a memory sub-channel / DIMM
+ *
+ * +--------+
+ * |        |
+ * | Zone 1 |
+ * |        |
+ * +--------+  +--------+
+ * |        |  |        |
+ * |        |  |        |
+ * | Zone 0 |  | Zone 0 |
+ * |        |  |        |
+ * |        |  |        |
+ * +--------+  +--------+
+ *
+ *  Slice L     Slice S
+ *
+ * Memory space is divided into:
+ *
+ *   - Zone 0 : Interleaved region
+ *   - Zone 1 : Non-interleaved region (upper part of the large slice).
+ */
+struct slice {
+	/* Slice address. */
+	u64 addr;
+	/* Slice that @addr belongs to. */
+	int id;
+};
+
+struct igen6_imc {
+	int mc;
+	struct mem_ctl_info *mci;
+	struct pci_dev *pdev;
+	struct device dev;
+	void __iomem *window;
+	u64 size;
+	u64 ch_s_size;
+	int ch_l_map;
+	u64 dimm_s_size[NUM_CHANNELS];
+	u64 dimm_l_size[NUM_CHANNELS];
+	int dimm_l_map[NUM_CHANNELS];
+};
+
+static struct res_config {
+	bool machine_check;
+	/* The number of present memory controllers. */
+	int num_imc;
+	/* Host MMIO configuration */
+	u64 reg_mchbar_mask;
+	/* Top of memory */
+	u64 reg_tom_mask;
+	/* Top of upper usable DRAM */
+	u64 reg_touud_mask;
+	/* IBECC error log */
+	u64 reg_eccerrlog_addr_mask;
+	/* MEMSS_PMA_CR registers. */
+	u32 reg_mem_config_offset;
+	u32 reg_mem_config_ddr_type_mask;
+	u32 reg_mem_config_ibecc_en_mask;
+	u32 reg_capabilities_misc_offset;
+	u32 reg_capabilities_misc_ibecc_dis;
+	/* Memory controller registers. */
+	u32 reg_mad_inter_size_mask[NUM_CHANNELS];
+	u64 reg_mad_inter_size_granularity;
+	u32 reg_mad_intra_rank_mask[NUM_DIMMS];
+	u32 reg_mad_intra_width_mask[NUM_DIMMS];
+	u32 reg_mad_intra_density_mask[NUM_DIMMS];
+	u32 imc_base;
+	u32 cmf_base;
+	u32 cmf_size;
+	u32 ms_hash_offset;
+	u32 ibecc_base;
+	u32 ibecc_error_log_offset;
+	/* Get memory type. */
+	enum mem_type (*get_mem_type)(struct igen6_imc *imc);
+	/* Get DRAM chip type. */
+	enum dev_type (*get_dev_type)(struct igen6_imc *imc, int chan, int dimm_l);
+	/* Set imc->ch_{s_size,l_map}. */
+	void (*set_chan_params)(struct igen6_imc *imc);
+	/* Set imc->dimm_{l_size,s_size,l_map}[chan]. */
+	void (*set_dimm_params)(struct igen6_imc *imc, int chan);
+	bool (*ibecc_available)(struct pci_dev *pdev);
+	/* Convert error address logged in IBECC to system physical address */
+	u64 (*err_addr_to_sys_addr)(u64 eaddr, int mc);
+	/* Convert error address logged in IBECC to integrated memory controller address */
+	u64 (*err_addr_to_imc_addr)(u64 eaddr, int mc);
+} *res_cfg;
+
+static struct igen6_pvt {
+	void __iomem *memss_pma_cr;
+	u64 ms_hash;
+	u64 ms_s_size;
+	int ms_l_map;
+	struct igen6_imc imc[];
+} *igen6_pvt;
+
+/* The top of low usable DRAM */
+static u32 igen6_tolud;
+/* The size of physical memory */
+static u64 igen6_tom;
+
+struct decoded_addr {
+	int mc;
+	u64 imc_addr;
+	u64 sys_addr;
+	int channel_idx;
+	u64 channel_addr;
+	int sub_channel_idx;
+	u64 sub_channel_addr;
+};
+
+struct ecclog_node {
+	struct llist_node llnode;
+	int mc;
+	u64 ecclog;
+};
+
+/*
+ * In the NMI handler, the driver uses the lock-less memory allocator
+ * to allocate memory to store the IBECC error logs and links the logs
+ * to the lock-less list. Delay printk() and the work of error reporting
+ * to EDAC core in a worker.
+ */
+#define ECCLOG_POOL_SIZE	PAGE_SIZE
+static LLIST_HEAD(ecclog_llist);
+static struct gen_pool *ecclog_pool;
+static char ecclog_buf[ECCLOG_POOL_SIZE];
+static struct irq_work ecclog_irq_work;
+static struct work_struct ecclog_work;
+
+/* SoC compute die IDs with IBECC capability. */
+/* Elkhart Lake */
+#define DID_EHL_SKU5	0x4514
+#define DID_EHL_SKU6	0x4528
+#define DID_EHL_SKU7	0x452a
+#define DID_EHL_SKU8	0x4516
+#define DID_EHL_SKU9	0x452c
+#define DID_EHL_SKU10	0x452e
+#define DID_EHL_SKU11	0x4532
+#define DID_EHL_SKU12	0x4518
+#define DID_EHL_SKU13	0x451a
+#define DID_EHL_SKU14	0x4534
+#define DID_EHL_SKU15	0x4536
+
+/* ICL-NNPI */
+#define DID_ICL_SKU8	0x4581
+#define DID_ICL_SKU10	0x4585
+#define DID_ICL_SKU11	0x4589
+#define DID_ICL_SKU12	0x458d
+
+/* Tiger Lake */
+#define DID_TGL_SKU	0x9a14
+
+/* Alder Lake */
+#define DID_ADL_SKU1	0x4601
+#define DID_ADL_SKU2	0x4602
+#define DID_ADL_SKU3	0x4621
+#define DID_ADL_SKU4	0x4641
+
+/* Alder Lake-N */
+#define DID_ADL_N_SKU1	0x4614
+#define DID_ADL_N_SKU2	0x4617
+#define DID_ADL_N_SKU3	0x461b
+#define DID_ADL_N_SKU4	0x461c
+#define DID_ADL_N_SKU5	0x4673
+#define DID_ADL_N_SKU6	0x4674
+#define DID_ADL_N_SKU7	0x4675
+#define DID_ADL_N_SKU8	0x4677
+#define DID_ADL_N_SKU9	0x4678
+#define DID_ADL_N_SKU10	0x4679
+#define DID_ADL_N_SKU11	0x467c
+#define DID_ADL_N_SKU12	0x4632
+
+/* Arizona Beach */
+#define DID_AZB_SKU1	0x4676
+
+/* Amston Lake */
+#define DID_ASL_SKU1	0x464a
+#define DID_ASL_SKU2	0x4646
+#define DID_ASL_SKU3	0x4652
+
+/* Raptor Lake-P */
+#define DID_RPL_P_SKU1	0xa706
+#define DID_RPL_P_SKU2	0xa707
+#define DID_RPL_P_SKU3	0xa708
+#define DID_RPL_P_SKU4	0xa716
+#define DID_RPL_P_SKU5	0xa718
+
+/* Meteor Lake-PS */
+#define DID_MTL_PS_SKU1	0x7d21
+#define DID_MTL_PS_SKU2	0x7d22
+#define DID_MTL_PS_SKU3	0x7d23
+#define DID_MTL_PS_SKU4	0x7d24
+
+/* Meteor Lake-P */
+#define DID_MTL_P_SKU1	0x7d01
+#define DID_MTL_P_SKU2	0x7d02
+#define DID_MTL_P_SKU3	0x7d14
+
+/* Arrow Lake-UH */
+#define DID_ARL_UH_SKU1	0x7d06
+#define DID_ARL_UH_SKU2	0x7d20
+#define DID_ARL_UH_SKU3	0x7d30
+
+/* Panther Lake-H */
+#define DID_PTL_H_SKU1	0xb000
+#define DID_PTL_H_SKU2	0xb001
+#define DID_PTL_H_SKU3	0xb002
+#define DID_PTL_H_SKU4	0xb003
+#define DID_PTL_H_SKU5	0xb004
+#define DID_PTL_H_SKU6	0xb005
+#define DID_PTL_H_SKU7	0xb008
+#define DID_PTL_H_SKU8	0xb011
+#define DID_PTL_H_SKU9	0xb014
+#define DID_PTL_H_SKU10	0xb015
+#define DID_PTL_H_SKU11	0xb028
+#define DID_PTL_H_SKU12	0xb029
+#define DID_PTL_H_SKU13	0xb02a
+#define DID_PTL_H_SKU14	0xb00a
+
+/* Starfire */
+#define DID_STF_SKU1	0xb02b
+
+/* Wildcat Lake */
+#define DID_WCL_SKU1	0xfd00
+
+/* Nova Lake-H/HX */
+#define DID_NVL_H_SKU1	0xd701
+#define DID_NVL_H_SKU2	0xd702
+#define DID_NVL_H_SKU3	0xd704
+#define DID_NVL_H_SKU4	0xd705
+
+/* Remove the interleave bit and shift upper part down to fill gap. */
+static u64 squeeze_addr(u64 addr, int intlv_bit)
+{
+	u64 slice_addr;
+
+	slice_addr  = GET_BITFIELD(addr, intlv_bit + 1, 63) << intlv_bit;
+	slice_addr |= GET_BITFIELD(addr, 0, intlv_bit - 1);
+
+	return slice_addr;
+}
+
+/* Shift the upper bits up and insert a zero at the @intlv_bit bit position. */
+static u64 inflate_addr(u64 addr, int intlv_bit)
+{
+	u64 inflated_addr;
+
+	/* Insert a zero at @intlv_bit position. */
+	inflated_addr  = GET_BITFIELD(addr, intlv_bit, 63) << (intlv_bit + 1);
+	inflated_addr |= GET_BITFIELD(addr, 0, intlv_bit - 1);
+
+	return inflated_addr;
+}
+
+static u64 compute_hash(u64 addr, u64 hash_mask, u64 hash_base, int intlv_bit)
+{
+	u64 hash_addr;
+	int i;
+
+	/*
+	 * In hash mode, @intlv_bit is the lowest selected bit of @addr
+	 * to be XORed. While @mask may or may not include this @intlv_bit,
+	 * we enforce that @mask includes @intlv_bit to ensure @intlv_bit is
+	 * XORed exactly once.
+	 */
+	hash_mask |= BIT_ULL(intlv_bit);
+	hash_addr  = addr & hash_mask;
+
+	for (i = 6; i < 20; i++)
+		hash_base ^= (hash_addr >> i) & 1;
+
+	return hash_base;
+}
+
+/*
+ * Converts a higher-level address (system / IMC / channel) into a lower-level
+ * slice address and identifier.
+ */
+static void translate_to_lower_level(u64 addr, u64 hash_mask, u64 hash_base,
+				     int intlv_bit, u64 s_size, int l_map,
+				     struct slice *slice)
+{
+	/* In non-interleave zone. */
+	if (addr >= 2 * s_size) {
+		slice->addr = addr - s_size;
+		slice->id  = l_map;
+		return;
+	}
+
+	/* In interleave zone. */
+	slice->addr = squeeze_addr(addr, intlv_bit);
+
+	/* Non-hash mode. */
+	if (!hash_mask) {
+		slice->id = GET_BITFIELD(addr, intlv_bit, intlv_bit);
+		return;
+	}
+
+	/* Hash mode. */
+	slice->id = compute_hash(addr, hash_mask, hash_base, intlv_bit);
+}
+
+/* Reconstruct address for upper memory hierarchy level. */
+static u64 translate_to_upper_level(u64 addr, u64 hash_mask, u64 hash_base,
+				    int intlv_bit, u64 s_size)
+{
+	u64 inflated_addr, hash_val;
+
+	/* In non-interleave zone. */
+	if (addr >= s_size)
+		return addr + s_size;
+
+	/*
+	 * In interleave zone.
+	 *
+	 * Insert a zero at @intlv_bit position.
+	 */
+	inflated_addr = inflate_addr(addr, intlv_bit);
+
+	/*
+	 * Reconstruct the removed interleave bit and use it to replace
+	 * the zero at @intlv_bit position.
+	 */
+	hash_val = compute_hash(inflated_addr, hash_mask, hash_base, intlv_bit);
+	return inflated_addr | (hash_val << intlv_bit);
+}
+
+static int get_mchbar(struct pci_dev *pdev, u64 *mchbar)
+{
+	union  {
+		u64 v;
+		struct {
+			u32 v_lo;
+			u32 v_hi;
+		};
+	} u;
+
+	if (pci_read_config_dword(pdev, MCHBAR_OFFSET, &u.v_lo)) {
+		igen6_printk(KERN_ERR, "Failed to read lower MCHBAR\n");
+		return -ENODEV;
+	}
+
+	if (pci_read_config_dword(pdev, MCHBAR_OFFSET + 4, &u.v_hi)) {
+		igen6_printk(KERN_ERR, "Failed to read upper MCHBAR\n");
+		return -ENODEV;
+	}
+
+	if (!(u.v & MCHBAR_EN)) {
+		igen6_printk(KERN_ERR, "MCHBAR is disabled\n");
+		return -ENODEV;
+	}
+
+	*mchbar = u.v & res_cfg->reg_mchbar_mask;
+	edac_dbg(2, "MCHBAR 0x%llx (reg 0x%llx)\n", *mchbar, u.v);
+
+	return 0;
+}
+
+/* Check whether the memory controller is absent. */
+static bool imc_absent(void __iomem *window)
+{
+	return readl(window + MAD_INTER_CHANNEL_OFFSET) == ~0;
+}
+
+/* Return MMIO base address of the memory controller if it's present, otherwise return NULL. */
+static void __iomem *map_imc_window(u64 mchbar, int pmc)
+{
+	void __iomem *window;
+
+	window = ioremap(mchbar + pmc * MCHBAR_SIZE, MCHBAR_SIZE);
+	if (!window)
+		return NULL;
+
+	if (imc_absent(window)) {
+		iounmap(window);
+		return NULL;
+	}
+
+	return window;
+}
+
+/* Return the number of present memory controllers. */
+static int get_imc_num(u64 mchbar)
+{
+	void __iomem *window;
+	int lmc, pmc;
+
+	for (lmc = 0, pmc = 0; pmc < MAX_IMC_TO_PROBE; pmc++) {
+		window = map_imc_window(mchbar, pmc);
+		if (window) {
+			iounmap(window);
+			lmc++;
+		}
+	}
+
+	return lmc;
+}
+
+static bool ehl_ibecc_available(struct pci_dev *pdev)
+{
+	u32 v;
+
+	if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v))
+		return false;
+
+	return !!(CAPID_C_IBECC & v);
+}
+
+static u64 ehl_err_addr_to_sys_addr(u64 eaddr, int mc)
+{
+	return eaddr;
+}
+
+static u64 ehl_err_addr_to_imc_addr(u64 eaddr, int mc)
+{
+	if (eaddr < igen6_tolud)
+		return eaddr;
+
+	if (igen6_tom <= _4GB)
+		return eaddr + igen6_tolud - _4GB;
+
+	if (eaddr >= igen6_tom)
+		return eaddr + igen6_tolud - igen6_tom;
+
+	return eaddr;
+}
+
+static bool icl_ibecc_available(struct pci_dev *pdev)
+{
+	u32 v;
+
+	if (pci_read_config_dword(pdev, CAPID_C_OFFSET, &v))
+		return false;
+
+	return !(CAPID_C_IBECC & v) &&
+		(boot_cpu_data.x86_stepping >= 1);
+}
+
+static bool tgl_ibecc_available(struct pci_dev *pdev)
+{
+	u32 v;
+
+	if (pci_read_config_dword(pdev, CAPID_E_OFFSET, &v))
+		return false;
+
+	return !(CAPID_E_IBECC & v);
+}
+
+static bool mtl_p_ibecc_available(struct pci_dev *pdev)
+{
+	u32 v;
+
+	if (pci_read_config_dword(pdev, CAPID_E_OFFSET, &v))
+		return false;
+
+	return !(CAPID_E_IBECC_BIT18 & v);
+}
+
+static bool generic_ibecc_available(struct pci_dev *pdev)
+{
+	void __iomem *base = igen6_pvt->memss_pma_cr;
+	bool present;
+	u32 val;
+
+	if (res_cfg->reg_capabilities_misc_offset) {
+		val = readl(base + res_cfg->reg_capabilities_misc_offset);
+		present = !(val & res_cfg->reg_capabilities_misc_ibecc_dis);
+		edac_dbg(2, "capabilities misc reg 0x%x\n", val);
+	} else if (res_cfg->reg_mem_config_offset) {
+		val = readl(base + res_cfg->reg_mem_config_offset);
+		present = !!(val & res_cfg->reg_mem_config_ibecc_en_mask);
+		edac_dbg(2, "mem config reg 0x%x\n", val);
+	} else {
+		igen6_printk(KERN_ERR, "No register for detecting IBECC presence.\n");
+		present = false;
+	}
+
+	return present;
+}
+
+static u64 mem_addr_to_sys_addr(u64 maddr)
+{
+	if (maddr < igen6_tolud)
+		return maddr;
+
+	if (igen6_tom <= _4GB)
+		return maddr - igen6_tolud + _4GB;
+
+	if (maddr < _4GB)
+		return maddr - igen6_tolud + igen6_tom;
+
+	return maddr;
+}
+
+static u64 tgl_err_addr_to_mem_addr(u64 eaddr, int mc)
+{
+	u64 mask, ms_s_size;
+	int intlv_bit;
+	u32 ms_hash;
+
+	ms_s_size = igen6_pvt->ms_s_size;
+	if (eaddr >= ms_s_size)
+		return eaddr + ms_s_size;
+
+	ms_hash = igen6_pvt->ms_hash;
+
+	mask = MEM_SLICE_HASH_MASK(ms_hash);
+	intlv_bit = MEM_SLICE_HASH_LSB_MASK_BIT(ms_hash) + 6;
+
+	return translate_to_upper_level(eaddr, mask, mc, intlv_bit, ms_s_size);
+}
+
+static u64 tgl_err_addr_to_sys_addr(u64 eaddr, int mc)
+{
+	u64 maddr = tgl_err_addr_to_mem_addr(eaddr, mc);
+
+	return mem_addr_to_sys_addr(maddr);
+}
+
+static u64 tgl_err_addr_to_imc_addr(u64 eaddr, int mc)
+{
+	return eaddr;
+}
+
+static u64 adl_err_addr_to_sys_addr(u64 eaddr, int mc)
+{
+	return mem_addr_to_sys_addr(eaddr);
+}
+
+static u64 adl_err_addr_to_imc_addr(u64 eaddr, int mc)
+{
+	u64 ms_s_size = igen6_pvt->ms_s_size;
+	struct igen6_imc *imc = &igen6_pvt->imc[mc];
+	struct slice slice;
+	int intlv_bit;
+	u32 mc_hash;
+
+	if (eaddr >= 2 * ms_s_size)
+		return eaddr - ms_s_size;
+
+	mc_hash = readl(imc->window + MAD_MC_HASH_OFFSET);
+
+	intlv_bit = MAC_MC_HASH_LSB(mc_hash) + 6;
+
+	translate_to_lower_level(eaddr, 0, 0, intlv_bit, ms_s_size, 0, &slice);
+	return slice.addr;
+}
+
+static enum mem_type ptl_h_get_mem_type(struct igen6_imc *imc)
+{
+	u32 mtype, val;
+
+	val = readl(igen6_pvt->memss_pma_cr + res_cfg->reg_mem_config_offset);
+	mtype = field_get(res_cfg->reg_mem_config_ddr_type_mask, val);
+
+	edac_dbg(2, "mtype %u (reg 0x%x)\n", mtype, val);
+
+	switch (mtype) {
+	case 1:
+		return MEM_DDR5;
+	case 2:
+		return MEM_LPDDR5;
+	case 3:
+		return MEM_LPDDR4;
+	default:
+		return MEM_UNKNOWN;
+	}
+}
+
+static enum dev_type ptl_h_get_dev_type(struct igen6_imc *imc, int chan, int dimm)
+{
+	u32 width, val;
+
+	val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4);
+	width = field_get(res_cfg->reg_mad_intra_width_mask[dimm], val);
+
+	switch (width) {
+	case 1:
+		return DEV_X8;
+	default:
+		return DEV_X16;
+	}
+}
+
+static u64 ptl_h_get_chan_size(struct igen6_imc *imc, int chan)
+{
+	u32 val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET);
+
+	return field_get(res_cfg->reg_mad_inter_size_mask[chan], val) *
+	       res_cfg->reg_mad_inter_size_granularity;
+}
+
+static u64 ptl_h_get_dimm_size(struct igen6_imc *imc, int chan, int dimm)
+{
+	u32 val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4);
+	u32 ranks = 1 << field_get(res_cfg->reg_mad_intra_rank_mask[dimm], val);
+	/* DRAM device density in Gb */
+	u64 density = field_get(res_cfg->reg_mad_intra_density_mask[dimm], val) * 4;
+
+	enum mem_type mtype = ptl_h_get_mem_type(imc);
+	enum dev_type dtype = ptl_h_get_dev_type(imc, chan, dimm);
+	u64 sub_ch_width, dev_num;
+
+	switch (mtype) {
+	case MEM_DDR5:
+		sub_ch_width = 32;
+		break;
+	case MEM_LPDDR5:
+	case MEM_LPDDR4:
+		sub_ch_width = 16;
+		break;
+	default:
+		sub_ch_width = 0;
+	}
+
+	switch (dtype) {
+	case DEV_X8:
+		dev_num = sub_ch_width / 8;
+		break;
+	case DEV_X16:
+		dev_num = sub_ch_width / 16;
+		break;
+	default:
+		dev_num = 0;
+	}
+
+	edac_dbg(2, "ranks %d, density %lluGb, sub_ch_width %llu, dev_num %llu (reg 0x%x)\n", ranks, density, sub_ch_width, dev_num, val);
+
+	return ((dev_num * density / 8) * ranks) << 30;
+}
+
+static void ptl_h_set_chan_params(struct igen6_imc *imc)
+{
+	u64 ch0_size = ptl_h_get_chan_size(imc, 0);
+	u64 ch1_size = ptl_h_get_chan_size(imc, 1);
+
+	if (ch0_size <= ch1_size) {
+		imc->ch_s_size = ch0_size;
+		imc->ch_l_map = 1;
+	} else {
+		imc->ch_s_size = ch1_size;
+		imc->ch_l_map = 0;
+	}
+}
+
+static void ptl_h_set_dimm_params(struct igen6_imc *imc, int chan)
+{
+	u64 dimm0_size = ptl_h_get_dimm_size(imc, chan, 0);
+	u64 dimm1_size = ptl_h_get_dimm_size(imc, chan, 1);
+
+	if (dimm0_size <= dimm1_size) {
+		imc->dimm_s_size[chan] = dimm0_size;
+		imc->dimm_l_size[chan] = dimm1_size;
+		imc->dimm_l_map[chan]  = 1;
+	} else {
+		imc->dimm_s_size[chan] = dimm1_size;
+		imc->dimm_l_size[chan] = dimm0_size;
+		imc->dimm_l_map[chan]  = 0;
+	}
+}
+
+static struct res_config ehl_cfg = {
+	.num_imc		= 1,
+	.reg_mchbar_mask	= GENMASK_ULL(38, 16),
+	.reg_tom_mask		= GENMASK_ULL(38, 20),
+	.reg_touud_mask		= GENMASK_ULL(38, 20),
+	.reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5),
+	.imc_base		= 0x5000,
+	.ibecc_base		= 0xdc00,
+	.ibecc_available	= ehl_ibecc_available,
+	.ibecc_error_log_offset	= 0x170,
+	.err_addr_to_sys_addr	= ehl_err_addr_to_sys_addr,
+	.err_addr_to_imc_addr	= ehl_err_addr_to_imc_addr,
+};
+
+static struct res_config icl_cfg = {
+	.num_imc		= 1,
+	.reg_mchbar_mask	= GENMASK_ULL(38, 16),
+	.reg_tom_mask		= GENMASK_ULL(38, 20),
+	.reg_touud_mask		= GENMASK_ULL(38, 20),
+	.reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5),
+	.imc_base		= 0x5000,
+	.ibecc_base		= 0xd800,
+	.ibecc_error_log_offset	= 0x170,
+	.ibecc_available	= icl_ibecc_available,
+	.err_addr_to_sys_addr	= ehl_err_addr_to_sys_addr,
+	.err_addr_to_imc_addr	= ehl_err_addr_to_imc_addr,
+};
+
+static struct res_config tgl_cfg = {
+	.machine_check		= true,
+	.num_imc		= 2,
+	.reg_mchbar_mask	= GENMASK_ULL(38, 17),
+	.reg_tom_mask		= GENMASK_ULL(38, 20),
+	.reg_touud_mask		= GENMASK_ULL(38, 20),
+	.reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5),
+	.imc_base		= 0x5000,
+	.cmf_base		= 0x11000,
+	.cmf_size		= 0x800,
+	.ms_hash_offset		= 0xac,
+	.ibecc_base		= 0xd400,
+	.ibecc_error_log_offset	= 0x170,
+	.ibecc_available	= tgl_ibecc_available,
+	.err_addr_to_sys_addr	= tgl_err_addr_to_sys_addr,
+	.err_addr_to_imc_addr	= tgl_err_addr_to_imc_addr,
+};
+
+/* Shared by Alder Lake, Alder Lake-N, Arizona Beach, Amston Lake, and Raptor Lake-P */
+static struct res_config adl_cfg = {
+	.machine_check		= true,
+	.num_imc		= 2,
+	.reg_mchbar_mask	= GENMASK_ULL(41, 17),
+	.reg_tom_mask		= GENMASK_ULL(41, 20),
+	.reg_touud_mask		= GENMASK_ULL(41, 20),
+	.reg_eccerrlog_addr_mask = GENMASK_ULL(45, 5),
+	.imc_base		= 0xd800,
+	.ibecc_base		= 0xd400,
+	.ibecc_error_log_offset	= 0x68,
+	.ibecc_available	= tgl_ibecc_available,
+	.err_addr_to_sys_addr	= adl_err_addr_to_sys_addr,
+	.err_addr_to_imc_addr	= adl_err_addr_to_imc_addr,
+};
+
+static struct res_config mtl_ps_cfg = {
+	.machine_check				= true,
+	.num_imc				= 2,
+	.reg_mchbar_mask			= GENMASK_ULL(41, 17),
+	.reg_tom_mask				= GENMASK_ULL(41, 20),
+	.reg_touud_mask				= GENMASK_ULL(41, 20),
+	.reg_eccerrlog_addr_mask		= GENMASK_ULL(38, 5),
+	.reg_capabilities_misc_offset		= 0x13c00,
+	.reg_capabilities_misc_ibecc_dis	= BIT(6),
+	.imc_base				= 0xd800,
+	.ibecc_base				= 0xd400,
+	.ibecc_error_log_offset			= 0x170,
+	.ibecc_available			= generic_ibecc_available,
+	.err_addr_to_sys_addr			= adl_err_addr_to_sys_addr,
+	.err_addr_to_imc_addr			= adl_err_addr_to_imc_addr,
+};
+
+/* Shared by Meteor Lake-P, Arrow Lake-UH, and Wildcat Lake */
+static struct res_config mtl_p_cfg = {
+	.machine_check		= true,
+	.num_imc		= 2,
+	.reg_mchbar_mask	= GENMASK_ULL(41, 17),
+	.reg_tom_mask		= GENMASK_ULL(41, 20),
+	.reg_touud_mask		= GENMASK_ULL(41, 20),
+	.reg_eccerrlog_addr_mask = GENMASK_ULL(38, 5),
+	.imc_base		= 0xd800,
+	.ibecc_base		= 0xd400,
+	.ibecc_error_log_offset	= 0x170,
+	.ibecc_available	= mtl_p_ibecc_available,
+	.err_addr_to_sys_addr	= adl_err_addr_to_sys_addr,
+	.err_addr_to_imc_addr	= adl_err_addr_to_imc_addr,
+};
+
+/* Shared by Panther Lake-H and Starfire */
+static struct res_config ptl_h_cfg = {
+	.machine_check			= true,
+	.num_imc			= 2,
+	.reg_mchbar_mask		= GENMASK_ULL(41, 17),
+	.reg_tom_mask			= GENMASK_ULL(41, 20),
+	.reg_touud_mask			= GENMASK_ULL(41, 20),
+	.reg_eccerrlog_addr_mask	= GENMASK_ULL(38, 5),
+	.reg_mem_config_offset		= 0x13d04,
+	.reg_mem_config_ddr_type_mask	= GENMASK(8, 6),
+	.reg_mad_inter_size_mask[0]	= GENMASK(15, 8),
+	.reg_mad_inter_size_mask[1]	= GENMASK(23, 16),
+	.reg_mad_inter_size_granularity	= BIT_ULL(29),
+	.reg_mad_intra_rank_mask[0]	= BIT(7),
+	.reg_mad_intra_rank_mask[1]	= BIT(15),
+	.reg_mad_intra_width_mask[0]	= BIT(6),
+	.reg_mad_intra_width_mask[1]	= BIT(14),
+	.reg_mad_intra_density_mask[0]	= GENMASK(3, 0),
+	.reg_mad_intra_density_mask[1]	= GENMASK(11, 8),
+	.imc_base			= 0xd800,
+	.ibecc_base			= 0xd400,
+	.ibecc_error_log_offset		= 0x170,
+	.get_mem_type			= ptl_h_get_mem_type,
+	.get_dev_type			= ptl_h_get_dev_type,
+	.set_chan_params		= ptl_h_set_chan_params,
+	.set_dimm_params		= ptl_h_set_dimm_params,
+	.ibecc_available		= mtl_p_ibecc_available,
+	.err_addr_to_sys_addr		= adl_err_addr_to_sys_addr,
+	.err_addr_to_imc_addr		= adl_err_addr_to_imc_addr,
+};
+
+static struct res_config nvl_h_cfg = {
+	.machine_check			= true,
+	.num_imc			= 2,
+	.reg_mchbar_mask		= GENMASK_ULL(41, 17),
+	.reg_tom_mask			= GENMASK_ULL(41, 20),
+	.reg_touud_mask			= GENMASK_ULL(41, 20),
+	.reg_eccerrlog_addr_mask	= GENMASK_ULL(38, 5),
+	.reg_mem_config_offset		= 0x12904,
+	.reg_mem_config_ddr_type_mask	= GENMASK(8, 6),
+	.reg_mem_config_ibecc_en_mask	= GENMASK(3, 2),
+	.reg_mad_inter_size_mask[0]	= GENMASK(15, 8),
+	.reg_mad_inter_size_mask[1]	= GENMASK(23, 16),
+	.reg_mad_inter_size_granularity	= BIT_ULL(29),
+	.reg_mad_intra_rank_mask[0]	= BIT(7),
+	.reg_mad_intra_rank_mask[1]	= BIT(15),
+	.reg_mad_intra_width_mask[0]	= BIT(6),
+	.reg_mad_intra_width_mask[1]	= BIT(14),
+	.reg_mad_intra_density_mask[0]	= GENMASK(3, 0),
+	.reg_mad_intra_density_mask[1]	= GENMASK(11, 8),
+	.imc_base			= 0xd800,
+	.ibecc_base			= 0xd400,
+	.ibecc_error_log_offset		= 0x170,
+	.get_mem_type			= ptl_h_get_mem_type,
+	.get_dev_type			= ptl_h_get_dev_type,
+	.set_chan_params		= ptl_h_set_chan_params,
+	.set_dimm_params		= ptl_h_set_dimm_params,
+	.ibecc_available		= generic_ibecc_available,
+	.err_addr_to_sys_addr		= adl_err_addr_to_sys_addr,
+	.err_addr_to_imc_addr		= adl_err_addr_to_imc_addr,
+};
+
+static struct pci_device_id igen6_pci_tbl[] = {
+	{ PCI_VDEVICE(INTEL, DID_EHL_SKU5), .driver_data = (kernel_ulong_t)&ehl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_EHL_SKU6), .driver_data = (kernel_ulong_t)&ehl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_EHL_SKU7), .driver_data = (kernel_ulong_t)&ehl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_EHL_SKU8), .driver_data = (kernel_ulong_t)&ehl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_EHL_SKU9), .driver_data = (kernel_ulong_t)&ehl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_EHL_SKU10), .driver_data = (kernel_ulong_t)&ehl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_EHL_SKU11), .driver_data = (kernel_ulong_t)&ehl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_EHL_SKU12), .driver_data = (kernel_ulong_t)&ehl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_EHL_SKU13), .driver_data = (kernel_ulong_t)&ehl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_EHL_SKU14), .driver_data = (kernel_ulong_t)&ehl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_EHL_SKU15), .driver_data = (kernel_ulong_t)&ehl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ICL_SKU8), .driver_data = (kernel_ulong_t)&icl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ICL_SKU10), .driver_data = (kernel_ulong_t)&icl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ICL_SKU11), .driver_data = (kernel_ulong_t)&icl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ICL_SKU12), .driver_data = (kernel_ulong_t)&icl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_TGL_SKU), .driver_data = (kernel_ulong_t)&tgl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ADL_SKU1), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ADL_SKU2), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ADL_SKU3), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ADL_SKU4), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU1), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU2), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU3), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU4), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU5), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU6), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU7), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU8), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU9), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU10), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU11), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ADL_N_SKU12), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_AZB_SKU1), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ASL_SKU1), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ASL_SKU2), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ASL_SKU3), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_RPL_P_SKU1), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_RPL_P_SKU2), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_RPL_P_SKU3), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_RPL_P_SKU4), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_RPL_P_SKU5), .driver_data = (kernel_ulong_t)&adl_cfg },
+	{ PCI_VDEVICE(INTEL, DID_MTL_PS_SKU1), .driver_data = (kernel_ulong_t)&mtl_ps_cfg },
+	{ PCI_VDEVICE(INTEL, DID_MTL_PS_SKU2), .driver_data = (kernel_ulong_t)&mtl_ps_cfg },
+	{ PCI_VDEVICE(INTEL, DID_MTL_PS_SKU3), .driver_data = (kernel_ulong_t)&mtl_ps_cfg },
+	{ PCI_VDEVICE(INTEL, DID_MTL_PS_SKU4), .driver_data = (kernel_ulong_t)&mtl_ps_cfg },
+	{ PCI_VDEVICE(INTEL, DID_MTL_P_SKU1), .driver_data = (kernel_ulong_t)&mtl_p_cfg },
+	{ PCI_VDEVICE(INTEL, DID_MTL_P_SKU2), .driver_data = (kernel_ulong_t)&mtl_p_cfg },
+	{ PCI_VDEVICE(INTEL, DID_MTL_P_SKU3), .driver_data = (kernel_ulong_t)&mtl_p_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ARL_UH_SKU1), .driver_data = (kernel_ulong_t)&mtl_p_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ARL_UH_SKU2), .driver_data = (kernel_ulong_t)&mtl_p_cfg },
+	{ PCI_VDEVICE(INTEL, DID_ARL_UH_SKU3), .driver_data = (kernel_ulong_t)&mtl_p_cfg },
+	{ PCI_VDEVICE(INTEL, DID_WCL_SKU1), .driver_data = (kernel_ulong_t)&mtl_p_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU1), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU2), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU3), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU4), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU5), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU6), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU7), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU8), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU9), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU10), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU11), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU12), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU13), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU14), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_STF_SKU1), .driver_data = (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_NVL_H_SKU1), .driver_data = (kernel_ulong_t)&nvl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_NVL_H_SKU2), .driver_data = (kernel_ulong_t)&nvl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_NVL_H_SKU3), .driver_data = (kernel_ulong_t)&nvl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_NVL_H_SKU4), .driver_data = (kernel_ulong_t)&nvl_h_cfg },
+	{ },
+};
+MODULE_DEVICE_TABLE(pci, igen6_pci_tbl);
+
+static enum mem_type get_mem_type(struct igen6_imc *imc)
+{
+	u32 val;
+
+	if (res_cfg->get_mem_type)
+		return res_cfg->get_mem_type(imc);
+
+	val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET);
+
+	switch (MAD_INTER_CHANNEL_DDR_TYPE(val)) {
+	case 0:
+		return MEM_DDR4;
+	case 1:
+		return MEM_DDR3;
+	case 2:
+		return MEM_LPDDR3;
+	case 3:
+		return MEM_LPDDR4;
+	case 4:
+		return MEM_WIO2;
+	default:
+		return MEM_UNKNOWN;
+	}
+}
+
+static bool large_dimm(struct igen6_imc *imc, int chan, int dimm)
+{
+	return dimm == imc->dimm_l_map[chan];
+}
+
+static enum dev_type get_dev_type(struct igen6_imc *imc, int chan, int dimm)
+{
+	u32 width, val;
+
+	if (res_cfg->get_dev_type)
+		return res_cfg->get_dev_type(imc, chan, dimm);
+
+	val = readl(imc->window + MAD_DIMM_CH0_OFFSET + chan * 4);
+	width = large_dimm(imc, chan, dimm) ? MAD_DIMM_CH_DLW(val) :
+					  MAD_DIMM_CH_DSW(val);
+
+	switch (width) {
+	case 0:
+		return DEV_X8;
+	case 1:
+		return DEV_X16;
+	case 2:
+		return DEV_X32;
+	default:
+		return DEV_UNKNOWN;
+	}
+}
+
+static u64 get_dimm_size(struct igen6_imc *imc, int chan, int dimm)
+{
+	if (large_dimm(imc, chan, dimm))
+		return imc->dimm_l_size[chan];
+
+	return imc->dimm_s_size[chan];
+}
+
+static void set_chan_params(struct igen6_imc *imc)
+{
+	u32 val;
+
+	if (res_cfg->set_chan_params) {
+		res_cfg->set_chan_params(imc);
+		return;
+	}
+
+	val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET);
+	imc->ch_s_size = MAD_INTER_CHANNEL_CH_S_SIZE(val);
+	imc->ch_l_map = MAD_INTER_CHANNEL_CH_L_MAP(val);
+}
+
+static void set_dimm_params(struct igen6_imc *imc, int chan)
+{
+	u32 val;
+
+	if (res_cfg->set_dimm_params) {
+		res_cfg->set_dimm_params(imc, chan);
+		return;
+	}
+
+	val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4);
+	imc->dimm_l_map[chan]  = MAD_INTRA_CH_DIMM_L_MAP(val);
+
+	val = readl(imc->window + MAD_DIMM_CH0_OFFSET + chan * 4);
+	imc->dimm_l_size[chan] = MAD_DIMM_CH_DIMM_L_SIZE(val);
+	imc->dimm_s_size[chan] = MAD_DIMM_CH_DIMM_S_SIZE(val);
+}
+
+static int igen6_decode(struct decoded_addr *res)
+{
+	struct igen6_imc *imc = &igen6_pvt->imc[res->mc];
+	u64 addr = res->imc_addr, s_size;
+	int intlv_bit, l_map;
+	u32 hash, hash_mask;
+	struct slice slice;
+
+	if (addr >= igen6_tom) {
+		edac_dbg(0, "Address 0x%llx out of range\n", addr);
+		return -EINVAL;
+	}
+
+	/* Decode channel */
+	hash   = readl(imc->window + CHANNEL_HASH_OFFSET);
+	s_size = imc->ch_s_size;
+	l_map  = imc->ch_l_map;
+	hash_mask = CHANNEL_HASH_MODE(hash) ? CHANNEL_HASH_MASK(hash) : 0;
+	intlv_bit = CHANNEL_HASH_LSB_MASK_BIT(hash) + 6;
+
+	translate_to_lower_level(addr, hash_mask, 0, intlv_bit, s_size, l_map, &slice);
+
+	res->channel_idx  = slice.id;
+	res->channel_addr = slice.addr;
+
+	/* Decode sub-channel/DIMM */
+	hash   = readl(imc->window + CHANNEL_EHASH_OFFSET);
+	s_size = imc->dimm_s_size[res->channel_idx];
+	l_map  = imc->dimm_l_map[res->channel_idx];
+	hash_mask = CHANNEL_HASH_MODE(hash) ? CHANNEL_HASH_MASK(hash) : 0;
+	intlv_bit = CHANNEL_HASH_LSB_MASK_BIT(hash) + 6;
+
+	translate_to_lower_level(res->channel_addr, hash_mask, 0, intlv_bit, s_size, l_map, &slice);
+
+	res->sub_channel_idx  = slice.id;
+	res->sub_channel_addr = slice.addr;
+
+	return 0;
+}
+
+static void igen6_output_error(struct decoded_addr *res,
+			       struct mem_ctl_info *mci, u64 ecclog)
+{
+	enum hw_event_mc_err_type type = ecclog & ECC_ERROR_LOG_UE ?
+					 HW_EVENT_ERR_UNCORRECTED :
+					 HW_EVENT_ERR_CORRECTED;
+
+	edac_mc_handle_error(type, mci, 1,
+			     res->sys_addr >> PAGE_SHIFT,
+			     res->sys_addr & ~PAGE_MASK,
+			     ECC_ERROR_LOG_SYND(ecclog),
+			     res->channel_idx, res->sub_channel_idx,
+			     -1, "", "");
+}
+
+static struct gen_pool *ecclog_gen_pool_create(void)
+{
+	struct gen_pool *pool;
+
+	pool = gen_pool_create(ilog2(sizeof(struct ecclog_node)), -1);
+	if (!pool)
+		return NULL;
+
+	if (gen_pool_add(pool, (unsigned long)ecclog_buf, ECCLOG_POOL_SIZE, -1)) {
+		gen_pool_destroy(pool);
+		return NULL;
+	}
+
+	return pool;
+}
+
+static int ecclog_gen_pool_add(int mc, u64 ecclog)
+{
+	struct ecclog_node *node;
+
+	node = (void *)gen_pool_alloc(ecclog_pool, sizeof(*node));
+	if (!node)
+		return -ENOMEM;
+
+	node->mc = mc;
+	node->ecclog = ecclog;
+	llist_add(&node->llnode, &ecclog_llist);
+
+	return 0;
+}
+
+/*
+ * Either the memory-mapped I/O status register ECC_ERROR_LOG or the PCI
+ * configuration space status register ERRSTS can indicate whether a
+ * correctable error or an uncorrectable error occurred. We only use the
+ * ECC_ERROR_LOG register to check error type, but need to clear both
+ * registers to enable future error events.
+ */
+static u64 ecclog_read_and_clear(struct igen6_imc *imc)
+{
+	u64 ecclog = readq(imc->window + ECC_ERROR_LOG_OFFSET);
+
+	/*
+	 * Quirk: The ECC_ERROR_LOG register of certain SoCs may contain
+	 *        the invalid value ~0. This will result in a flood of invalid
+	 *        error reports in polling mode. Skip it.
+	 */
+	if (ecclog == ~0)
+		return 0;
+
+	/* Neither a CE nor a UE. Skip it.*/
+	if (!(ecclog & (ECC_ERROR_LOG_CE | ECC_ERROR_LOG_UE)))
+		return 0;
+
+	/* Clear CE/UE bits by writing 1s */
+	writeq(ecclog, imc->window + ECC_ERROR_LOG_OFFSET);
+
+	return ecclog;
+}
+
+static void errsts_clear(struct igen6_imc *imc)
+{
+	u16 errsts;
+
+	if (pci_read_config_word(imc->pdev, ERRSTS_OFFSET, &errsts)) {
+		igen6_printk(KERN_ERR, "Failed to read ERRSTS\n");
+		return;
+	}
+
+	/* Clear CE/UE bits by writing 1s */
+	if (errsts & (ERRSTS_CE | ERRSTS_UE))
+		pci_write_config_word(imc->pdev, ERRSTS_OFFSET, errsts);
+}
+
+static int errcmd_enable_error_reporting(bool enable)
+{
+	struct igen6_imc *imc = &igen6_pvt->imc[0];
+	u16 errcmd;
+	int rc;
+
+	rc = pci_read_config_word(imc->pdev, ERRCMD_OFFSET, &errcmd);
+	if (rc)
+		return pcibios_err_to_errno(rc);
+
+	if (enable)
+		errcmd |= ERRCMD_CE | ERRSTS_UE;
+	else
+		errcmd &= ~(ERRCMD_CE | ERRSTS_UE);
+
+	rc = pci_write_config_word(imc->pdev, ERRCMD_OFFSET, errcmd);
+	if (rc)
+		return pcibios_err_to_errno(rc);
+
+	return 0;
+}
+
+static int ecclog_handler(void)
+{
+	struct igen6_imc *imc;
+	int i, n = 0;
+	u64 ecclog;
+
+	for (i = 0; i < res_cfg->num_imc; i++) {
+		imc = &igen6_pvt->imc[i];
+
+		/* errsts_clear() isn't NMI-safe. Delay it in the IRQ context */
+
+		ecclog = ecclog_read_and_clear(imc);
+		if (!ecclog)
+			continue;
+
+		if (!ecclog_gen_pool_add(i, ecclog))
+			irq_work_queue(&ecclog_irq_work);
+
+		n++;
+	}
+
+	return n;
+}
+
+static void ecclog_work_cb(struct work_struct *work)
+{
+	struct ecclog_node *node, *tmp;
+	struct mem_ctl_info *mci;
+	struct llist_node *head;
+	struct decoded_addr res;
+	u64 eaddr;
+
+	head = llist_del_all(&ecclog_llist);
+	if (!head)
+		return;
+
+	llist_for_each_entry_safe(node, tmp, head, llnode) {
+		memset(&res, 0, sizeof(res));
+		eaddr	     = node->ecclog & res_cfg->reg_eccerrlog_addr_mask;
+		res.mc	     = node->mc;
+		res.sys_addr = res_cfg->err_addr_to_sys_addr(eaddr, res.mc);
+		res.imc_addr = res_cfg->err_addr_to_imc_addr(eaddr, res.mc);
+
+		mci = igen6_pvt->imc[res.mc].mci;
+
+		edac_dbg(2, "MC %d, ecclog = 0x%llx\n", node->mc, node->ecclog);
+		igen6_mc_printk(mci, KERN_DEBUG, "HANDLING IBECC MEMORY ERROR\n");
+		igen6_mc_printk(mci, KERN_DEBUG, "ADDR 0x%llx ", res.sys_addr);
+
+		if (!igen6_decode(&res))
+			igen6_output_error(&res, mci, node->ecclog);
+
+		gen_pool_free(ecclog_pool, (unsigned long)node, sizeof(*node));
+	}
+}
+
+static void ecclog_irq_work_cb(struct irq_work *irq_work)
+{
+	int i;
+
+	for (i = 0; i < res_cfg->num_imc; i++)
+		errsts_clear(&igen6_pvt->imc[i]);
+
+	if (!llist_empty(&ecclog_llist))
+		schedule_work(&ecclog_work);
+}
+
+static int ecclog_nmi_handler(unsigned int cmd, struct pt_regs *regs)
+{
+	unsigned char reason;
+
+	if (!ecclog_handler())
+		return NMI_DONE;
+
+	/*
+	 * Both In-Band ECC correctable error and uncorrectable error are
+	 * reported by SERR# NMI. The NMI generic code (see pci_serr_error())
+	 * doesn't clear the bit NMI_REASON_CLEAR_SERR (in port 0x61) to
+	 * re-enable the SERR# NMI after NMI handling. So clear this bit here
+	 * to re-enable SERR# NMI for receiving future In-Band ECC errors.
+	 */
+	reason  = x86_platform.get_nmi_reason() & NMI_REASON_CLEAR_MASK;
+	reason |= NMI_REASON_CLEAR_SERR;
+	outb(reason, NMI_REASON_PORT);
+	reason &= ~NMI_REASON_CLEAR_SERR;
+	outb(reason, NMI_REASON_PORT);
+
+	return NMI_HANDLED;
+}
+
+static int ecclog_mce_handler(struct notifier_block *nb, unsigned long val,
+			      void *data)
+{
+	struct mce *mce = (struct mce *)data;
+	char *type;
+
+	if (mce->kflags & MCE_HANDLED_CEC)
+		return NOTIFY_DONE;
+
+	/*
+	 * Ignore unless this is a memory related error.
+	 * We don't check the bit MCI_STATUS_ADDRV of MCi_STATUS here,
+	 * since this bit isn't set on some CPU (e.g., Tiger Lake UP3).
+	 */
+	if ((mce->status & 0xefff) >> 7 != 1)
+		return NOTIFY_DONE;
+
+	if (mce->mcgstatus & MCG_STATUS_MCIP)
+		type = "Exception";
+	else
+		type = "Event";
+
+	edac_dbg(0, "CPU %d: Machine Check %s: 0x%llx Bank %d: 0x%llx\n",
+		 mce->extcpu, type, mce->mcgstatus,
+		 mce->bank, mce->status);
+	edac_dbg(0, "TSC 0x%llx\n", mce->tsc);
+	edac_dbg(0, "ADDR 0x%llx\n", mce->addr);
+	edac_dbg(0, "MISC 0x%llx\n", mce->misc);
+	edac_dbg(0, "PROCESSOR %u:0x%x TIME %llu SOCKET %u APIC 0x%x\n",
+		 mce->cpuvendor, mce->cpuid, mce->time,
+		 mce->socketid, mce->apicid);
+	/*
+	 * We just use the Machine Check for the memory error notification.
+	 * Each memory controller is associated with an IBECC instance.
+	 * Directly read and clear the error information(error address and
+	 * error type) on all the IBECC instances so that we know on which
+	 * memory controller the memory error(s) occurred.
+	 */
+	if (!ecclog_handler())
+		return NOTIFY_DONE;
+
+	mce->kflags |= MCE_HANDLED_EDAC;
+
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block ecclog_mce_dec = {
+	.notifier_call	= ecclog_mce_handler,
+	.priority	= MCE_PRIO_EDAC,
+};
+
+static bool igen6_check_ecc(struct igen6_imc *imc)
+{
+	u32 activate = readl(imc->window + IBECC_ACTIVATE_OFFSET);
+
+	return !!(activate & IBECC_ACTIVATE_EN);
+}
+
+static int igen6_get_dimm_config(struct mem_ctl_info *mci)
+{
+	struct igen6_imc *imc = mci->pvt_info;
+	int i, j, ndimms, mc = imc->mc;
+	struct dimm_info *dimm;
+	enum mem_type mtype;
+	enum dev_type dtype;
+	u64 dsize;
+	bool ecc;
+
+	edac_dbg(2, "\n");
+
+	mtype = get_mem_type(imc);
+	ecc = igen6_check_ecc(imc);
+	set_chan_params(imc);
+
+	for (i = 0; i < NUM_CHANNELS; i++) {
+		set_dimm_params(imc, i);
+		imc->size += imc->dimm_s_size[i];
+		imc->size += imc->dimm_l_size[i];
+		ndimms = 0;
+
+		for (j = 0; j < NUM_DIMMS; j++) {
+			dimm = edac_get_dimm(mci, i, j, 0);
+			dtype = get_dev_type(imc, i, j);
+			dsize = get_dimm_size(imc, i, j);
+
+			if (!dsize)
+				continue;
+
+			dimm->grain = 64;
+			dimm->mtype = mtype;
+			dimm->dtype = dtype;
+			dimm->nr_pages  = MiB_TO_PAGES(dsize >> 20);
+			dimm->edac_mode = EDAC_SECDED;
+			snprintf(dimm->label, sizeof(dimm->label),
+				 "MC#%d_Chan#%d_DIMM#%d", mc, i, j);
+			edac_dbg(0, "MC %d, Channel %d, DIMM %d, Size %llu MiB (%u pages)\n",
+				 mc, i, j, dsize >> 20, dimm->nr_pages);
+
+			ndimms++;
+		}
+
+		if (ndimms && !ecc) {
+			igen6_printk(KERN_ERR, "MC%d In-Band ECC is disabled\n", mc);
+			return -ENODEV;
+		}
+	}
+
+	edac_dbg(0, "MC %d, total size %llu MiB\n", mc, imc->size >> 20);
+
+	return 0;
+}
+
+#ifdef CONFIG_EDAC_DEBUG
+/* Top of upper usable DRAM */
+static u64 igen6_touud;
+#define TOUUD_OFFSET	0xa8
+
+static void igen6_reg_dump(struct igen6_imc *imc)
+{
+	int i;
+
+	edac_dbg(2, "CHANNEL_HASH     : 0x%x\n",
+		 readl(imc->window + CHANNEL_HASH_OFFSET));
+	edac_dbg(2, "CHANNEL_EHASH    : 0x%x\n",
+		 readl(imc->window + CHANNEL_EHASH_OFFSET));
+	edac_dbg(2, "MAD_INTER_CHANNEL: 0x%x\n",
+		 readl(imc->window + MAD_INTER_CHANNEL_OFFSET));
+	edac_dbg(2, "ECC_ERROR_LOG    : 0x%llx\n",
+		 readq(imc->window + ECC_ERROR_LOG_OFFSET));
+
+	for (i = 0; i < NUM_CHANNELS; i++) {
+		edac_dbg(2, "MAD_INTRA_CH%d    : 0x%x\n", i,
+			 readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4));
+		edac_dbg(2, "MAD_DIMM_CH%d     : 0x%x\n", i,
+			 readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4));
+	}
+	edac_dbg(2, "TOLUD            : 0x%x", igen6_tolud);
+	edac_dbg(2, "TOUUD            : 0x%llx", igen6_touud);
+	edac_dbg(2, "TOM              : 0x%llx", igen6_tom);
+}
+
+static struct dentry *igen6_test;
+
+static int debugfs_u64_set(void *data, u64 val)
+{
+	u64 ecclog;
+
+	if ((val >= igen6_tolud && val < _4GB) || val >= igen6_touud) {
+		edac_dbg(0, "Address 0x%llx out of range\n", val);
+		return 0;
+	}
+
+	pr_warn_once("Fake error to 0x%llx injected via debugfs\n", val);
+
+	ecclog = (val & res_cfg->reg_eccerrlog_addr_mask) | ECC_ERROR_LOG_CE;
+
+	if (!ecclog_gen_pool_add(0, ecclog))
+		irq_work_queue(&ecclog_irq_work);
+
+	return 0;
+}
+DEFINE_SIMPLE_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n");
+
+static void igen6_debug_setup(void)
+{
+	igen6_test = edac_debugfs_create_dir("igen6_test");
+	if (!igen6_test)
+		return;
+
+	if (!edac_debugfs_create_file("addr", 0200, igen6_test,
+				      NULL, &fops_u64_wo)) {
+		debugfs_remove(igen6_test);
+		igen6_test = NULL;
+	}
+}
+
+static void igen6_debug_teardown(void)
+{
+	debugfs_remove_recursive(igen6_test);
+}
+#else
+static void igen6_reg_dump(struct igen6_imc *imc) {}
+static void igen6_debug_setup(void) {}
+static void igen6_debug_teardown(void) {}
+#endif
+
+static struct igen6_pvt *igen6_pvt_setup(struct pci_dev *pdev)
+{
+	void __iomem *memss_pma_cr;
+	struct igen6_pvt *pvt;
+	int imc_num, rc;
+	u64 mchbar;
+
+	rc = get_mchbar(pdev, &mchbar);
+	if (rc)
+		return NULL;
+
+	imc_num = get_imc_num(mchbar);
+	if (!imc_num) {
+		igen6_printk(KERN_ERR, "No mc found.\n");
+		return NULL;
+	}
+	edac_dbg(2, "%d mcs found.\n", imc_num);
+
+	/* Use the runtime detected IMC count. */
+	if (res_cfg->num_imc != imc_num)
+		res_cfg->num_imc = imc_num;
+
+	pvt = kzalloc_flex(*pvt, imc, imc_num);
+	if (!pvt)
+		return NULL;
+
+	memss_pma_cr = ioremap(mchbar, MCHBAR_SIZE * 2);
+	if (!memss_pma_cr) {
+		kfree(pvt);
+		return NULL;
+	}
+	pvt->memss_pma_cr = memss_pma_cr;
+
+	return pvt;
+}
+
+static void igen6_pvt_release(struct igen6_pvt *pvt)
+{
+	iounmap(pvt->memss_pma_cr);
+	kfree(pvt);
+}
+
+static int igen6_pci_setup(struct pci_dev *pdev, u64 *mchbar)
+{
+	union  {
+		u64 v;
+		struct {
+			u32 v_lo;
+			u32 v_hi;
+		};
+	} u;
+
+	edac_dbg(2, "\n");
+
+	if (!res_cfg->ibecc_available(pdev)) {
+		edac_dbg(2, "No In-Band ECC IP\n");
+		goto fail;
+	}
+
+	if (pci_read_config_dword(pdev, TOLUD_OFFSET, &igen6_tolud)) {
+		igen6_printk(KERN_ERR, "Failed to read TOLUD\n");
+		goto fail;
+	}
+
+	igen6_tolud &= GENMASK(31, 20);
+
+	if (pci_read_config_dword(pdev, TOM_OFFSET, &u.v_lo)) {
+		igen6_printk(KERN_ERR, "Failed to read lower TOM\n");
+		goto fail;
+	}
+
+	if (pci_read_config_dword(pdev, TOM_OFFSET + 4, &u.v_hi)) {
+		igen6_printk(KERN_ERR, "Failed to read upper TOM\n");
+		goto fail;
+	}
+
+	igen6_tom = u.v & res_cfg->reg_tom_mask;
+
+	if (get_mchbar(pdev, mchbar))
+		goto fail;
+
+#ifdef CONFIG_EDAC_DEBUG
+	if (pci_read_config_dword(pdev, TOUUD_OFFSET, &u.v_lo))
+		edac_dbg(2, "Failed to read lower TOUUD\n");
+	else if (pci_read_config_dword(pdev, TOUUD_OFFSET + 4, &u.v_hi))
+		edac_dbg(2, "Failed to read upper TOUUD\n");
+	else
+		igen6_touud = u.v & res_cfg->reg_touud_mask;
+#endif
+
+	return 0;
+fail:
+	return -ENODEV;
+}
+
+static void igen6_check(struct mem_ctl_info *mci)
+{
+	struct igen6_imc *imc = mci->pvt_info;
+	u64 ecclog;
+
+	/* errsts_clear() isn't NMI-safe. Delay it in the IRQ context */
+	ecclog = ecclog_read_and_clear(imc);
+	if (!ecclog)
+		return;
+
+	if (!ecclog_gen_pool_add(imc->mc, ecclog))
+		irq_work_queue(&ecclog_irq_work);
+}
+
+static void imc_release(struct device *dev)
+{
+	/* Nothing to do, the 'imc' owns the 'dev' and will also release it. */
+}
+
+static int igen6_register_mci(int mc, void __iomem *window, struct pci_dev *pdev)
+{
+	struct edac_mc_layer layers[2];
+	struct mem_ctl_info *mci;
+	struct igen6_imc *imc;
+	int rc;
+
+	edac_dbg(2, "\n");
+
+	layers[0].type = EDAC_MC_LAYER_CHANNEL;
+	layers[0].size = NUM_CHANNELS;
+	layers[0].is_virt_csrow = false;
+	layers[1].type = EDAC_MC_LAYER_SLOT;
+	layers[1].size = NUM_DIMMS;
+	layers[1].is_virt_csrow = true;
+
+	mci = edac_mc_alloc(mc, ARRAY_SIZE(layers), layers, 0);
+	if (!mci) {
+		rc = -ENOMEM;
+		goto fail;
+	}
+
+	mci->ctl_name = kasprintf(GFP_KERNEL, "Intel_client_SoC MC#%d", mc);
+	if (!mci->ctl_name) {
+		rc = -ENOMEM;
+		goto fail2;
+	}
+
+	mci->mtype_cap = MEM_FLAG_LPDDR4 | MEM_FLAG_DDR4;
+	mci->edac_ctl_cap = EDAC_FLAG_SECDED;
+	mci->edac_cap = EDAC_FLAG_SECDED;
+	mci->mod_name = EDAC_MOD_STR;
+	mci->dev_name = pci_name(pdev);
+	if (edac_op_state == EDAC_OPSTATE_POLL)
+		mci->edac_check = igen6_check;
+	mci->pvt_info = &igen6_pvt->imc[mc];
+
+	imc = mci->pvt_info;
+	imc->dev.release = imc_release;
+	device_initialize(&imc->dev);
+	/*
+	 * EDAC core uses mci->pdev(pointer of structure device) as
+	 * memory controller ID. The client SoCs attach one or more
+	 * memory controllers to single pci_dev (single pci_dev->dev
+	 * can be for multiple memory controllers).
+	 *
+	 * To make mci->pdev unique, assign pci_dev->dev to mci->pdev
+	 * for the first memory controller and assign a unique imc->dev
+	 * to mci->pdev for each non-first memory controller.
+	 */
+	mci->pdev = mc ? &imc->dev : &pdev->dev;
+	imc->mc	= mc;
+	imc->pdev = pdev;
+	imc->window = window;
+
+	igen6_reg_dump(imc);
+
+	rc = igen6_get_dimm_config(mci);
+	if (rc)
+		goto fail3;
+
+	rc = edac_mc_add_mc(mci);
+	if (rc) {
+		igen6_printk(KERN_ERR, "Failed to register mci#%d\n", mc);
+		goto fail3;
+	}
+
+	imc->mci = mci;
+	return 0;
+fail3:
+	put_device(&imc->dev);
+	mci->pvt_info = NULL;
+	kfree(mci->ctl_name);
+fail2:
+	edac_mc_free(mci);
+fail:
+	return rc;
+}
+
+static void igen6_unregister_mcis(void)
+{
+	struct mem_ctl_info *mci;
+	struct igen6_imc *imc;
+	int i;
+
+	edac_dbg(2, "\n");
+
+	for (i = 0; i < res_cfg->num_imc; i++) {
+		imc = &igen6_pvt->imc[i];
+		mci = imc->mci;
+		if (!mci)
+			continue;
+
+		edac_mc_del_mc(mci->pdev);
+		kfree(mci->ctl_name);
+		mci->pvt_info = NULL;
+		edac_mc_free(mci);
+		put_device(&imc->dev);
+		iounmap(imc->window);
+	}
+}
+
+static int igen6_register_mcis(struct pci_dev *pdev, u64 mchbar)
+{
+	void __iomem *window;
+	int lmc, pmc, rc;
+
+	for (lmc = 0, pmc = 0; pmc < MAX_IMC_TO_PROBE; pmc++) {
+		window = map_imc_window(mchbar, pmc);
+		if (!window)
+			continue;
+
+		rc = igen6_register_mci(lmc, window, pdev);
+		if (rc)
+			goto err_unregister;
+
+		/* Done, if all present MCs are detected and registered. */
+		if (++lmc >= res_cfg->num_imc)
+			break;
+	}
+
+	if (!lmc) {
+		igen6_printk(KERN_ERR, "No mc found.\n");
+		return -ENODEV;
+	}
+
+	if (lmc < res_cfg->num_imc) {
+		igen6_printk(KERN_DEBUG, "Expected %d mcs, but only %d detected.",
+			     res_cfg->num_imc, lmc);
+		res_cfg->num_imc = lmc;
+	}
+
+	return 0;
+
+err_unregister:
+	iounmap(window);
+	igen6_unregister_mcis();
+
+	return rc;
+}
+
+static int igen6_mem_slice_setup(u64 mchbar)
+{
+	struct igen6_imc *imc = &igen6_pvt->imc[0];
+	u64 base = mchbar + res_cfg->cmf_base;
+	u32 offset = res_cfg->ms_hash_offset;
+	u32 size = res_cfg->cmf_size;
+	u64 ms_s_size, ms_hash;
+	void __iomem *cmf;
+	int ms_l_map;
+
+	edac_dbg(2, "\n");
+
+	if (imc[0].size < imc[1].size) {
+		ms_s_size = imc[0].size;
+		ms_l_map  = 1;
+	} else {
+		ms_s_size = imc[1].size;
+		ms_l_map  = 0;
+	}
+
+	igen6_pvt->ms_s_size = ms_s_size;
+	igen6_pvt->ms_l_map  = ms_l_map;
+
+	edac_dbg(0, "ms_s_size: %llu MiB, ms_l_map %d\n",
+		 ms_s_size >> 20, ms_l_map);
+
+	if (!size)
+		return 0;
+
+	cmf = ioremap(base, size);
+	if (!cmf) {
+		igen6_printk(KERN_ERR, "Failed to ioremap cmf 0x%llx\n", base);
+		return -ENODEV;
+	}
+
+	ms_hash = readq(cmf + offset);
+	igen6_pvt->ms_hash = ms_hash;
+
+	edac_dbg(0, "MEM_SLICE_HASH: 0x%llx\n", ms_hash);
+
+	iounmap(cmf);
+
+	return 0;
+}
+
+static int register_err_handler(void)
+{
+	int rc;
+
+	if (res_cfg->machine_check) {
+		mce_register_decode_chain(&ecclog_mce_dec);
+		return 0;
+	}
+
+	rc = register_nmi_handler(NMI_SERR, ecclog_nmi_handler,
+				  0, IGEN6_NMI_NAME);
+	if (rc) {
+		igen6_printk(KERN_ERR, "Failed to register NMI handler\n");
+		return rc;
+	}
+
+	return 0;
+}
+
+static void unregister_err_handler(void)
+{
+	if (res_cfg->machine_check) {
+		mce_unregister_decode_chain(&ecclog_mce_dec);
+		return;
+	}
+
+	unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME);
+}
+
+static void opstate_set(const struct res_config *cfg, const struct pci_device_id *ent)
+{
+	/*
+	 * Quirk: Certain SoCs' error reporting interrupts don't work.
+	 *        Force polling mode for them to ensure that memory error
+	 *        events can be handled.
+	 */
+	if (ent->device == DID_ADL_N_SKU4) {
+		edac_op_state = EDAC_OPSTATE_POLL;
+		return;
+	}
+
+	/* Set the mode according to the configuration data. */
+	if (cfg->machine_check)
+		edac_op_state = EDAC_OPSTATE_INT;
+	else
+		edac_op_state = EDAC_OPSTATE_NMI;
+}
+
+static int igen6_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+	u64 mchbar;
+	int rc;
+
+	edac_dbg(2, "\n");
+
+	res_cfg = (struct res_config *)ent->driver_data;
+
+	igen6_pvt = igen6_pvt_setup(pdev);
+	if (!igen6_pvt)
+		return -ENOMEM;
+
+	rc = igen6_pci_setup(pdev, &mchbar);
+	if (rc)
+		goto fail;
+
+	opstate_set(res_cfg, ent);
+
+	rc = igen6_register_mcis(pdev, mchbar);
+	if (rc)
+		goto fail;
+
+	if (res_cfg->num_imc > 1) {
+		rc = igen6_mem_slice_setup(mchbar);
+		if (rc)
+			goto fail2;
+	}
+
+	ecclog_pool = ecclog_gen_pool_create();
+	if (!ecclog_pool) {
+		rc = -ENOMEM;
+		goto fail2;
+	}
+
+	INIT_WORK(&ecclog_work, ecclog_work_cb);
+	init_irq_work(&ecclog_irq_work, ecclog_irq_work_cb);
+
+	rc = register_err_handler();
+	if (rc)
+		goto fail3;
+
+	/* Enable error reporting */
+	rc = errcmd_enable_error_reporting(true);
+	if (rc) {
+		igen6_printk(KERN_ERR, "Failed to enable error reporting\n");
+		goto fail4;
+	}
+
+	/* Check if any pending errors before/during the registration of the error handler */
+	ecclog_handler();
+
+	igen6_debug_setup();
+	return 0;
+fail4:
+	unregister_nmi_handler(NMI_SERR, IGEN6_NMI_NAME);
+fail3:
+	gen_pool_destroy(ecclog_pool);
+fail2:
+	igen6_unregister_mcis();
+fail:
+	igen6_pvt_release(igen6_pvt);
+	return rc;
+}
+
+static void igen6_remove(struct pci_dev *pdev)
+{
+	edac_dbg(2, "\n");
+
+	igen6_debug_teardown();
+	errcmd_enable_error_reporting(false);
+	unregister_err_handler();
+	irq_work_sync(&ecclog_irq_work);
+	flush_work(&ecclog_work);
+	gen_pool_destroy(ecclog_pool);
+	igen6_unregister_mcis();
+	igen6_pvt_release(igen6_pvt);
+}
+
+static struct pci_driver igen6_driver = {
+	.name     = EDAC_MOD_STR,
+	.probe    = igen6_probe,
+	.remove   = igen6_remove,
+	.id_table = igen6_pci_tbl,
+};
+
+static int __init igen6_init(void)
+{
+	const char *owner;
+	int rc;
+
+	edac_dbg(2, "\n");
+
+	if (ghes_get_devices())
+		return -EBUSY;
+
+	owner = edac_get_owner();
+	if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
+		return -EBUSY;
+
+	rc = pci_register_driver(&igen6_driver);
+	if (rc)
+		return rc;
+
+	igen6_printk(KERN_INFO, "%s\n", IGEN6_REVISION);
+
+	return 0;
+}
+
+static void __exit igen6_exit(void)
+{
+	edac_dbg(2, "\n");
+
+	pci_unregister_driver(&igen6_driver);
+}
+
+module_init(igen6_init);
+module_exit(igen6_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Qiuxu Zhuo");
+MODULE_DESCRIPTION("MC Driver for Intel client SoC using In-Band ECC");