diff options
| author | Jinke Han <jinkehan@didiglobal.com> | 2026-09-08 15:37:42 +0800 |
|---|---|---|
| committer | Ingo Molnar <mingo@kernel.org> | 2026-09-17 09:48:42 +0200 |
| commit | a5f7a5bb3b7f28ba7e4fa246775b29a0e5537255 (patch) | |
| tree | 73fef824f9a846cf935869d0dbc5070ef6e2f741 /drivers/pci/controller | |
| download | linux-stable-a5f7a5bb3b7f28ba7e4fa246775b29a0e5537255.tar.gz linux-stable-a5f7a5bb3b7f28ba7e4fa246775b29a0e5537255.zip | |
x86/kprobes: Fix crash when probing CS CALL instructionsgrafted
When using eBPF to probe CS CALL instructions within a function,
a crash can be triggered.
The eBPF tool probes offset 257 of the __hrtimer_run_queues()
function:
<__hrtimer_run_queues+249>: nopl 0x0(%rax,%rax,1)
<__hrtimer_run_queues+254>: mov %r14,%rdi
<__hrtimer_run_queues+257>: cs call <__x86_indirect_thunk_r12>
<__hrtimer_run_queues+263>: mov %eax,%r12d
<__hrtimer_run_queues+266>: xchg %ax,%ax
<__hrtimer_run_queues+268>: mov %r13,%rdi
Which triggers this crash:
BUG: unable to handle page fault for address: 00000000000f41c9
#PF: supervisor write access in kernel mode
#PF: error_code(0x0002) - not-present page
PGD 0 P4D 0
Oops: 0002 [#1] SMP NOPTI
CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P
RIP: 0010:__hrtimer_run_queues+0x106/0x230
Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is
at the 6th byte of the above CS CALL instruction. Since the CS CALL
instruction occupies 6 bytes, the exception occurred in the middle of that
call instruction.
The root cause is that when using eBPF tools to probe in the middle of a
function, a kprobe with INT3 is used as the underlying implementation.
During single-step emulation of the original CALL instruction,
int3_emulate_call() assumes that the probed CALL instruction is 5 bytes
long. However, the actual CS-prefixed CALL instruction occupies 6 bytes,
so it constructs an incorrect exception return address. When the CPU
returns from the kprobe handler, the next instruction to be executed is at
the address of the last byte of that CS CALL instruction. Coincidentally,
starting from that address, the CPU fetches and decodes a completely
different instruction, which ultimately triggers a kernel crash.
Fix the issue by using the actual instruction length obtained from
the instruction decoder when constructing the exception return
address, rather than relying on the hardcoded CALL_INSN_SIZE macro.
[ mingo: Refined the changelog ]
Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step")
Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Signed-off-by: Jinke Han <jinkehan@didiglobal.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org>
Acked-by: Yafang Shao <laoar.shao@gmail.com>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000
Diffstat (limited to 'drivers/pci/controller')
126 files changed, 79403 insertions, 0 deletions
diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig new file mode 100644 index 000000000..d246bbe37 --- /dev/null +++ b/drivers/pci/controller/Kconfig @@ -0,0 +1,400 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "PCI controller drivers" + depends on PCI + +config PCI_HOST_COMMON + tristate + select PCI_ECAM + +config PCI_AARDVARK + tristate "Aardvark PCIe controller" + depends on (ARCH_MVEBU && ARM64) || COMPILE_TEST + depends on OF + depends on PCI_MSI + select PCI_BRIDGE_EMUL + select IRQ_MSI_LIB + help + Add support for Aardvark 64bit PCIe Host Controller. This + controller is part of the South Bridge of the Marvel Armada + 3700 SoC. + +config PCIE_ALTERA + tristate "Altera PCIe controller" + depends on ARM || NIOS2 || ARM64 || COMPILE_TEST + help + Say Y here if you want to enable PCIe controller support on Altera + FPGA. + +config PCIE_ALTERA_MSI + tristate "Altera PCIe MSI feature" + depends on PCIE_ALTERA + depends on PCI_MSI + select IRQ_MSI_LIB + help + Say Y here if you want PCIe MSI support for the Altera FPGA. + This MSI driver supports Altera MSI to GIC controller IP. + +config PCIE_APPLE_MSI_DOORBELL_ADDR + hex + default 0xfffff000 + depends on PCIE_APPLE + +config PCIE_APPLE + tristate "Apple PCIe controller" + depends on ARCH_APPLE || COMPILE_TEST + depends on OF + depends on PCI_MSI + select PCI_HOST_COMMON + select IRQ_MSI_LIB + help + Say Y here if you want to enable PCIe controller support on Apple + system-on-chips, like the Apple M1. This is required for the USB + type-A ports, Ethernet, Wi-Fi, and Bluetooth. + + If unsure, say Y if you have an Apple Silicon system. + +config PCI_VERSATILE + bool "ARM Versatile PB PCI controller" + depends on ARCH_VERSATILE || COMPILE_TEST + +config PCIE_ASPEED + bool "ASPEED PCIe controller" + depends on ARCH_ASPEED || COMPILE_TEST + depends on OF + depends on PCI_MSI + select IRQ_MSI_LIB + help + Enable this option to support the PCIe controller found on ASPEED + SoCs. + + This driver provides initialization and management for PCIe + Root Complex functionality, including INTx and MSI support. + + Select Y if your platform uses an ASPEED SoC and requires PCIe + connectivity. + +config PCIE_BRCMSTB + tristate "Broadcom Brcmstb PCIe controller" + depends on ARCH_BRCMSTB || ARCH_BCM2835 || ARCH_BCMBCA || \ + BMIPS_GENERIC || COMPILE_TEST + depends on OF + depends on PCI_MSI + select IRQ_MSI_LIB + default ARCH_BRCMSTB || BMIPS_GENERIC + help + Say Y here to enable PCIe host controller support for + Broadcom STB based SoCs, like the Raspberry Pi 4. + +config PCIE_IPROC + tristate + help + This enables the iProc PCIe core controller support for Broadcom's + iProc family of SoCs. An appropriate bus interface driver needs + to be enabled to select this. + +config PCIE_IPROC_PLATFORM + tristate "Broadcom iProc PCIe platform bus driver" + depends on ARCH_BCM_IPROC || (ARM && COMPILE_TEST) + depends on OF + select PCIE_IPROC + default ARCH_BCM_IPROC + help + Say Y here if you want to use the Broadcom iProc PCIe controller + through the generic platform bus interface + +config PCIE_IPROC_BCMA + tristate "Broadcom iProc BCMA PCIe controller" + depends on ARM && (ARCH_BCM_IPROC || COMPILE_TEST) + select PCIE_IPROC + select BCMA + default ARCH_BCM_5301X + help + Say Y here if you want to use the Broadcom iProc PCIe controller + through the BCMA bus interface + +config PCIE_IPROC_MSI + bool "Broadcom iProc PCIe MSI support" + depends on PCIE_IPROC_PLATFORM || PCIE_IPROC_BCMA + depends on PCI_MSI + select IRQ_MSI_LIB + default ARCH_BCM_IPROC + help + Say Y here if you want to enable MSI support for Broadcom's iProc + PCIe controller + +config PCI_HOST_THUNDER_PEM + bool "Cavium Thunder PCIe controller to off-chip devices" + depends on ARM64 || COMPILE_TEST + depends on OF || (ACPI && PCI_QUIRKS) + select PCI_HOST_COMMON + help + Say Y here if you want PCIe support for CN88XX Cavium Thunder SoCs. + +config PCI_HOST_THUNDER_ECAM + bool "Cavium Thunder ECAM controller to on-chip devices on pass-1.x silicon" + depends on ARM64 || COMPILE_TEST + depends on OF || (ACPI && PCI_QUIRKS) + select PCI_HOST_COMMON + help + Say Y here if you want ECAM support for CN88XX-Pass-1.x Cavium Thunder SoCs. + +config PCI_FTPCI100 + bool "Faraday Technology FTPCI100 PCI controller" + depends on OF + default ARCH_GEMINI + +config PCI_HOST_GENERIC + tristate "Generic PCI host controller" + depends on OF + select PCI_HOST_COMMON + select IRQ_DOMAIN + help + Say Y here if you want to support a simple generic PCI host + controller, such as the one emulated by kvmtool. + +config PCIE_HISI_ERR + depends on ACPI_APEI_GHES && (ARM64 || COMPILE_TEST) + bool "HiSilicon HIP PCIe controller error handling driver" + help + Say Y here if you want error handling support + for the PCIe controller's errors on HiSilicon HIP SoCs + +config PCI_IXP4XX + bool "Intel IXP4xx PCI controller" + depends on OF + depends on ARCH_IXP4XX || COMPILE_TEST + default ARCH_IXP4XX + help + Say Y here if you want support for the PCI host controller found + in the Intel IXP4xx XScale-based network processor SoC. + +config VMD + depends on PCI_MSI && X86_64 && !UML + tristate "Intel Volume Management Device Driver" + select IRQ_MSI_LIB + help + Adds support for the Intel Volume Management Device (VMD). VMD is a + secondary PCI host bridge that allows PCI Express root ports, + and devices attached to them, to be removed from the default + PCI domain and placed within the VMD domain. This provides + more bus resources than are otherwise possible with a + single domain. If you know your system provides one of these and + has devices attached to it, say Y; if you are not sure, say N. + + To compile this driver as a module, choose M here: the + module will be called vmd. + +config PCI_LOONGSON + bool "LOONGSON PCIe controller" + depends on MACH_LOONGSON32 || MACH_LOONGSON64 || COMPILE_TEST + depends on OF || ACPI + depends on PCI_QUIRKS + default MACH_LOONGSON64 + help + Say Y here if you want to enable PCI controller support on + Loongson systems. + +config PCI_MVEBU + tristate "Marvell EBU PCIe controller" + depends on ARCH_MVEBU || ARCH_DOVE || COMPILE_TEST + depends on MVEBU_MBUS + depends on ARM + depends on OF + select PCI_BRIDGE_EMUL + help + Add support for Marvell EBU PCIe controller. This PCIe controller + is used on 32-bit Marvell ARM SoCs: Dove, Kirkwood, Armada 370, + Armada XP, Armada 375, Armada 38x and Armada 39x. + +config PCIE_MEDIATEK + tristate "MediaTek PCIe controller" + depends on ARCH_AIROHA || ARCH_MEDIATEK || ECONET || COMPILE_TEST + depends on OF + depends on PCI_MSI + select IRQ_MSI_LIB + help + Say Y here if you want to enable PCIe controller support on + MediaTek SoCs. + +config PCIE_MEDIATEK_GEN3 + tristate "MediaTek Gen3 PCIe controller" + depends on ARCH_AIROHA || ARCH_MEDIATEK || COMPILE_TEST + depends on PCI_MSI + select IRQ_MSI_LIB + select PCI_PWRCTRL_GENERIC + help + Adds support for PCIe Gen3 MAC controller for MediaTek SoCs. + This PCIe controller is compatible with Gen3, Gen2 and Gen1 speed, + and support up to 256 MSI interrupt numbers for + multi-function devices. + + Say Y here if you want to enable Gen3 PCIe controller support on + MediaTek SoCs. + +config PCIE_MT7621 + tristate "MediaTek MT7621 PCIe controller" + depends on SOC_MT7621 || COMPILE_TEST + select PHY_MT7621_PCI + default SOC_MT7621 + help + This selects a driver for the MediaTek MT7621 PCIe Controller. + +config PCI_HYPERV_INTERFACE + tristate "Microsoft Hyper-V PCI Interface" + depends on ((X86 && X86_64) || ARM64) && HYPERV && PCI_MSI + help + The Hyper-V PCI Interface is a helper driver that allows other + drivers to have a common interface with the Hyper-V PCI frontend + driver. + +config PCI_TEGRA + tristate "NVIDIA Tegra PCIe controller" + depends on ARCH_TEGRA || COMPILE_TEST + depends on PCI_MSI + select IRQ_MSI_LIB + help + Say Y here if you want support for the PCIe host controller found + on NVIDIA Tegra SoCs (Tegra20 through Tegra186). + +config PCIE_TEGRA264 + tristate "NVIDIA Tegra264 PCIe controller" + depends on ARCH_TEGRA || COMPILE_TEST + select PCI_ECAM + help + Say Y here if you want support for the PCIe Host Controller found + on NVIDIA Tegra264 SoCs. + +config PCIE_RCAR_HOST + bool "Renesas R-Car PCIe controller (host mode)" + depends on ARCH_RENESAS || COMPILE_TEST + depends on OF + depends on PCI_MSI + select IRQ_MSI_LIB + help + Say Y here if you want PCIe controller support on R-Car SoCs in host + mode. + +config PCIE_RCAR_EP + bool "Renesas R-Car PCIe controller (endpoint mode)" + depends on ARCH_RENESAS || COMPILE_TEST + depends on PCI_ENDPOINT + help + Say Y here if you want PCIe controller support on R-Car SoCs in + endpoint mode. + +config PCI_RCAR_GEN2 + bool "Renesas R-Car Gen2 Internal PCI controller" + depends on (ARCH_RENESAS && ARM) || COMPILE_TEST + help + Say Y here if you want internal PCI support on R-Car Gen2 SoC. + Each internal PCI controller contains a single built-in EHCI/OHCI + host controller. + +config PCIE_RENESAS_RZG3S_HOST + bool "Renesas RZ/G3S PCIe host controller" + depends on ARCH_RENESAS || COMPILE_TEST + select MFD_SYSCON + select IRQ_MSI_LIB + help + Say Y here if you want PCIe host controller support on Renesas RZ/G3S + SoC. + +config PCIE_ROCKCHIP + bool + depends on PCI + +config PCIE_ROCKCHIP_HOST + tristate "Rockchip PCIe controller (host mode)" + depends on ARCH_ROCKCHIP || COMPILE_TEST + depends on OF + depends on PCI_MSI + select MFD_SYSCON + select PCIE_ROCKCHIP + help + Say Y here if you want internal PCI support on Rockchip SoC. + There is 1 internal PCIe port available to support GEN2 with + 4 slots. + +config PCIE_ROCKCHIP_EP + bool "Rockchip PCIe controller (endpoint mode)" + depends on ARCH_ROCKCHIP || COMPILE_TEST + depends on OF + depends on PCI_ENDPOINT + select MFD_SYSCON + select PCIE_ROCKCHIP + help + Say Y here if you want to support Rockchip PCIe controller in + endpoint mode on Rockchip SoC. There is 1 internal PCIe port + available to support GEN2 with 4 slots. + +config PCI_V3_SEMI + bool "V3 Semiconductor PCI controller" + depends on OF + depends on ARM || COMPILE_TEST + default ARCH_INTEGRATOR_AP + +config PCI_XGENE + bool "X-Gene PCIe controller" + depends on ARM64 || COMPILE_TEST + depends on OF || (ACPI && PCI_QUIRKS) + help + Say Y here if you want internal PCI support on APM X-Gene SoC. + There are 5 internal PCIe ports available. Each port is GEN3 capable + and have varied lanes from x1 to x8. + +config PCI_XGENE_MSI + bool "X-Gene v1 PCIe MSI feature" + depends on PCI_XGENE + depends on PCI_MSI + select IRQ_MSI_LIB + default y + help + Say Y here if you want PCIe MSI support for the APM X-Gene v1 SoC. + This MSI driver supports 5 PCIe ports on the APM X-Gene v1 SoC. + +config PCIE_XILINX + bool "Xilinx AXI PCIe controller" + depends on OF + depends on PCI_MSI + select IRQ_MSI_LIB + help + Say 'Y' here if you want kernel to support the Xilinx AXI PCIe + Host Bridge driver. + +config PCIE_XILINX_DMA_PL + bool "Xilinx DMA PL PCIe host bridge support" + depends on ARCH_ZYNQMP || COMPILE_TEST + depends on PCI_MSI + select PCI_HOST_COMMON + select IRQ_MSI_LIB + help + Say 'Y' here if you want kernel support for the Xilinx PL DMA + PCIe host bridge. The controller is a Soft IP which can act as + Root Port. If your system provides Xilinx PCIe host controller + bridge DMA as Soft IP say 'Y'; if you are not sure, say 'N'. + +config PCIE_XILINX_NWL + bool "Xilinx NWL PCIe controller" + depends on ARCH_ZYNQMP || COMPILE_TEST + depends on PCI_MSI + select IRQ_MSI_LIB + help + Say 'Y' here if you want kernel support for Xilinx + NWL PCIe controller. The controller can act as Root Port + or End Point. The current option selection will only + support root port enabling. + +config PCIE_XILINX_CPM + bool "Xilinx Versal CPM PCI controller" + depends on ARCH_ZYNQMP || COMPILE_TEST + select PCI_HOST_COMMON + help + Say 'Y' here if you want kernel support for the + Xilinx Versal CPM host bridge. + +source "drivers/pci/controller/cadence/Kconfig" +source "drivers/pci/controller/dwc/Kconfig" +source "drivers/pci/controller/mobiveil/Kconfig" +source "drivers/pci/controller/plda/Kconfig" +endmenu diff --git a/drivers/pci/controller/Makefile b/drivers/pci/controller/Makefile new file mode 100644 index 000000000..d478743b5 --- /dev/null +++ b/drivers/pci/controller/Makefile @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_PCIE_CADENCE) += cadence/ +obj-$(CONFIG_PCI_FTPCI100) += pci-ftpci100.o +obj-$(CONFIG_PCI_IXP4XX) += pci-ixp4xx.o +obj-$(CONFIG_PCI_HYPERV) += pci-hyperv.o +obj-$(CONFIG_PCI_HYPERV_INTERFACE) += pci-hyperv-intf.o +obj-$(CONFIG_PCI_MVEBU) += pci-mvebu.o +obj-$(CONFIG_PCI_AARDVARK) += pci-aardvark.o +obj-$(CONFIG_PCI_TEGRA) += pci-tegra.o +obj-$(CONFIG_PCIE_TEGRA264) += pcie-tegra264.o +obj-$(CONFIG_PCI_RCAR_GEN2) += pci-rcar-gen2.o +obj-$(CONFIG_PCIE_RCAR_HOST) += pcie-rcar.o pcie-rcar-host.o +obj-$(CONFIG_PCIE_RCAR_EP) += pcie-rcar.o pcie-rcar-ep.o +obj-$(CONFIG_PCIE_RENESAS_RZG3S_HOST) += pcie-rzg3s-host.o +obj-$(CONFIG_PCI_HOST_COMMON) += pci-host-common.o +obj-$(CONFIG_PCI_HOST_GENERIC) += pci-host-generic.o +obj-$(CONFIG_PCI_HOST_THUNDER_ECAM) += pci-thunder-ecam.o +obj-$(CONFIG_PCI_HOST_THUNDER_PEM) += pci-thunder-pem.o +obj-$(CONFIG_PCIE_XILINX) += pcie-xilinx.o +obj-$(CONFIG_PCIE_XILINX_NWL) += pcie-xilinx-nwl.o +obj-$(CONFIG_PCIE_XILINX_CPM) += pcie-xilinx-cpm.o +obj-$(CONFIG_PCIE_XILINX_DMA_PL) += pcie-xilinx-dma-pl.o +obj-$(CONFIG_PCI_V3_SEMI) += pci-v3-semi.o +obj-$(CONFIG_PCI_XGENE) += pci-xgene.o +obj-$(CONFIG_PCI_XGENE_MSI) += pci-xgene-msi.o +obj-$(CONFIG_PCI_VERSATILE) += pci-versatile.o +obj-$(CONFIG_PCIE_IPROC) += pcie-iproc.o +obj-$(CONFIG_PCIE_IPROC_MSI) += pcie-iproc-msi.o +obj-$(CONFIG_PCIE_IPROC_PLATFORM) += pcie-iproc-platform.o +obj-$(CONFIG_PCIE_IPROC_BCMA) += pcie-iproc-bcma.o +obj-$(CONFIG_PCIE_ALTERA) += pcie-altera.o +obj-$(CONFIG_PCIE_ALTERA_MSI) += pcie-altera-msi.o +obj-$(CONFIG_PCIE_ROCKCHIP) += pcie-rockchip.o +obj-$(CONFIG_PCIE_ROCKCHIP_EP) += pcie-rockchip-ep.o +obj-$(CONFIG_PCIE_ROCKCHIP_HOST) += pcie-rockchip-host.o +obj-$(CONFIG_PCIE_MEDIATEK) += pcie-mediatek.o +obj-$(CONFIG_PCIE_MEDIATEK_GEN3) += pcie-mediatek-gen3.o +obj-$(CONFIG_VMD) += vmd.o +obj-$(CONFIG_PCIE_BRCMSTB) += pcie-brcmstb.o +obj-$(CONFIG_PCI_LOONGSON) += pci-loongson.o +obj-$(CONFIG_PCIE_HISI_ERR) += pcie-hisi-error.o +obj-$(CONFIG_PCIE_APPLE) += pcie-apple.o +obj-$(CONFIG_PCIE_MT7621) += pcie-mt7621.o +obj-$(CONFIG_PCIE_ASPEED) += pcie-aspeed.o + +# pcie-hisi.o quirks are needed even without CONFIG_PCIE_DW +obj-y += dwc/ +obj-y += mobiveil/ +obj-y += plda/ + + +# The following drivers are for devices that use the generic ACPI +# pci_root.c driver but don't support standard ECAM config access. +# They contain MCFG quirks to replace the generic ECAM accessors with +# device-specific ones that are shared with the DT driver. + +# The ACPI driver is generic and should not require driver-specific +# config options to be enabled, so we always build these drivers on +# ARM64 and use internal ifdefs to only build the pieces we need +# depending on whether ACPI, the DT driver, or both are enabled. + +ifdef CONFIG_ACPI +ifdef CONFIG_PCI_QUIRKS +obj-$(CONFIG_ARM64) += pci-thunder-ecam.o +obj-$(CONFIG_ARM64) += pci-thunder-pem.o +obj-$(CONFIG_ARM64) += pci-xgene.o +endif +endif diff --git a/drivers/pci/controller/cadence/Kconfig b/drivers/pci/controller/cadence/Kconfig new file mode 100644 index 000000000..69efd7582 --- /dev/null +++ b/drivers/pci/controller/cadence/Kconfig @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "Cadence-based PCIe controllers" + depends on PCI + +config PCIE_CADENCE + tristate + +config PCIE_CADENCE_DEBUGFS + bool "Cadence PCIe debugfs entries" + depends on DEBUG_FS + depends on PCIE_CADENCE_HOST || PCIE_CADENCE_EP + help + Say Y here to enable debugfs entries for the PCIe controller. These + entries provide various debug features related to the controller and + the LTSSM status of link can be displayed. + +config PCIE_CADENCE_HOST + tristate + depends on OF + select IRQ_DOMAIN + select PCIE_CADENCE + +config PCIE_CADENCE_EP + tristate + depends on OF + depends on PCI_ENDPOINT + select PCIE_CADENCE + +config PCIE_CADENCE_PLAT + tristate + select PCIE_CADENCE_HOST if PCIE_CADENCE_PLAT_HOST != n + select PCIE_CADENCE_EP if PCIE_CADENCE_PLAT_EP != n + +config PCIE_CADENCE_PLAT_HOST + tristate "Cadence platform PCIe controller (host mode)" + depends on OF + select PCIE_CADENCE_PLAT + help + Say Y here if you want to support the Cadence PCIe platform controller in + host mode. This PCIe controller may be embedded into many different + vendors SoCs. + +config PCIE_CADENCE_PLAT_EP + tristate "Cadence platform PCIe controller (endpoint mode)" + depends on OF + depends on PCI_ENDPOINT + select PCIE_CADENCE_PLAT + help + Say Y here if you want to support the Cadence PCIe platform controller in + endpoint mode. This PCIe controller may be embedded into many + different vendors SoCs. + +config PCI_SKY1_HOST + tristate "CIX SKY1 PCIe controller (host mode)" + depends on OF && (ARCH_CIX || COMPILE_TEST) + select PCIE_CADENCE_HOST + select PCI_ECAM + help + Say Y here if you want to support the CIX SKY1 PCIe platform + controller in host mode. CIX SKY1 PCIe controller uses Cadence + HPA (High Performance Architecture IP [Second generation of + Cadence PCIe IP]) + + This driver requires Cadence PCIe core infrastructure + (PCIE_CADENCE_HOST) and hardware platform adaptation layer + to function. + +config PCIE_SG2042_HOST + tristate "Sophgo SG2042 PCIe controller (host mode)" + depends on OF && (ARCH_SOPHGO || COMPILE_TEST) + select PCIE_CADENCE_HOST + help + Say Y here if you want to support the Sophgo SG2042 PCIe platform + controller in host mode. Sophgo SG2042 PCIe controller uses Cadence + PCIe core. + +config PCI_J721E + tristate + select PCIE_CADENCE_HOST if PCI_J721E_HOST != n + select PCIE_CADENCE_EP if PCI_J721E_EP != n + +config PCI_J721E_HOST + tristate "TI J721E PCIe controller (host mode)" + depends on ARCH_K3 || COMPILE_TEST + depends on OF + select PCI_J721E + help + Say Y here if you want to support the TI J721E PCIe platform + controller in host mode. TI J721E PCIe controller uses Cadence PCIe + core. + +config PCI_J721E_EP + tristate "TI J721E PCIe controller (endpoint mode)" + depends on ARCH_K3 || COMPILE_TEST + depends on OF + depends on PCI_ENDPOINT + select PCI_J721E + help + Say Y here if you want to support the TI J721E PCIe platform + controller in endpoint mode. TI J721E PCIe controller uses Cadence PCIe + core. + +endmenu diff --git a/drivers/pci/controller/cadence/Makefile b/drivers/pci/controller/cadence/Makefile new file mode 100644 index 000000000..80be0b885 --- /dev/null +++ b/drivers/pci/controller/cadence/Makefile @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: GPL-2.0 +pcie-cadence-mod-y := pcie-cadence-hpa.o pcie-cadence.o +pcie-cadence-mod-$(CONFIG_PCIE_CADENCE_DEBUGFS) += pcie-cadence-debugfs.o +pcie-cadence-host-mod-y := pcie-cadence-host-common.o pcie-cadence-host.o pcie-cadence-host-hpa.o +pcie-cadence-ep-mod-y := pcie-cadence-ep.o + +obj-$(CONFIG_PCIE_CADENCE) = pcie-cadence-mod.o +obj-$(CONFIG_PCIE_CADENCE_HOST) += pcie-cadence-host-mod.o +obj-$(CONFIG_PCIE_CADENCE_EP) += pcie-cadence-ep-mod.o +obj-$(CONFIG_PCIE_CADENCE_PLAT) += pcie-cadence-plat.o +obj-$(CONFIG_PCI_J721E) += pci-j721e.o +obj-$(CONFIG_PCIE_SG2042_HOST) += pcie-sg2042.o +obj-$(CONFIG_PCI_SKY1_HOST) += pci-sky1.o diff --git a/drivers/pci/controller/cadence/pci-j721e.c b/drivers/pci/controller/cadence/pci-j721e.c new file mode 100644 index 000000000..822602836 --- /dev/null +++ b/drivers/pci/controller/cadence/pci-j721e.c @@ -0,0 +1,783 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * pci-j721e - PCIe controller driver for TI's J721E SoCs + * + * Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com + * Author: Kishon Vijay Abraham I <kishon@ti.com> + */ + +#include <linux/clk.h> +#include <linux/clk-provider.h> +#include <linux/container_of.h> +#include <linux/delay.h> +#include <linux/gpio/consumer.h> +#include <linux/io.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqdomain.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/regmap.h> + +#include "../../pci.h" +#include "pcie-cadence.h" + +#define cdns_pcie_to_rc(p) container_of(p, struct cdns_pcie_rc, pcie) + +#define ENABLE_REG_SYS_2 0x108 +#define ENABLE_CLR_REG_SYS_2 0x308 +#define STATUS_REG_SYS_2 0x508 +#define STATUS_CLR_REG_SYS_2 0x708 +#define LINK_DOWN BIT(1) +#define J7200_LINK_DOWN BIT(10) + +#define J721E_PCIE_USER_CMD_STATUS 0x4 +#define LINK_TRAINING_ENABLE BIT(0) + +#define J721E_PCIE_USER_LINKSTATUS 0x14 +#define LINK_STATUS GENMASK(1, 0) + +enum link_status { + NO_RECEIVERS_DETECTED, + LINK_TRAINING_IN_PROGRESS, + LINK_UP_DL_IN_PROGRESS, + LINK_UP_DL_COMPLETED, +}; + +#define J721E_MODE_RC BIT(7) +#define LANE_COUNT(n) ((n) << 8) + +#define ACSPCIE_PAD_DISABLE_MASK GENMASK(1, 0) +#define GENERATION_SEL_MASK GENMASK(1, 0) + +struct j721e_pcie { + struct cdns_pcie *cdns_pcie; + struct clk *refclk; + u32 mode; + u32 num_lanes; + u32 max_lanes; + struct gpio_desc *reset_gpio; + void __iomem *user_cfg_base; + void __iomem *intd_cfg_base; + u32 linkdown_irq_regfield; +}; + +enum j721e_pcie_mode { + PCI_MODE_RC, + PCI_MODE_EP, +}; + +struct j721e_pcie_data { + enum j721e_pcie_mode mode; + unsigned int quirk_retrain_flag:1; + unsigned int quirk_detect_quiet_flag:1; + unsigned int quirk_disable_flr:1; + u32 linkdown_irq_regfield; + unsigned int byte_access_allowed:1; + unsigned int max_lanes; +}; + +static inline u32 j721e_pcie_user_readl(struct j721e_pcie *pcie, u32 offset) +{ + return readl(pcie->user_cfg_base + offset); +} + +static inline void j721e_pcie_user_writel(struct j721e_pcie *pcie, u32 offset, + u32 value) +{ + writel(value, pcie->user_cfg_base + offset); +} + +static inline u32 j721e_pcie_intd_readl(struct j721e_pcie *pcie, u32 offset) +{ + return readl(pcie->intd_cfg_base + offset); +} + +static inline void j721e_pcie_intd_writel(struct j721e_pcie *pcie, u32 offset, + u32 value) +{ + writel(value, pcie->intd_cfg_base + offset); +} + +static irqreturn_t j721e_pcie_link_irq_handler(int irq, void *priv) +{ + struct j721e_pcie *pcie = priv; + struct device *dev = pcie->cdns_pcie->dev; + u32 reg; + + reg = j721e_pcie_intd_readl(pcie, STATUS_REG_SYS_2); + if (!(reg & pcie->linkdown_irq_regfield)) + return IRQ_NONE; + + dev_err(dev, "LINK DOWN!\n"); + + j721e_pcie_intd_writel(pcie, STATUS_CLR_REG_SYS_2, pcie->linkdown_irq_regfield); + return IRQ_HANDLED; +} + +static void j721e_pcie_disable_link_irq(struct j721e_pcie *pcie) +{ + u32 reg; + + reg = j721e_pcie_intd_readl(pcie, ENABLE_CLR_REG_SYS_2); + reg |= pcie->linkdown_irq_regfield; + j721e_pcie_intd_writel(pcie, ENABLE_CLR_REG_SYS_2, reg); +} + +static void j721e_pcie_config_link_irq(struct j721e_pcie *pcie) +{ + u32 reg; + + reg = j721e_pcie_intd_readl(pcie, ENABLE_REG_SYS_2); + reg |= pcie->linkdown_irq_regfield; + j721e_pcie_intd_writel(pcie, ENABLE_REG_SYS_2, reg); +} + +static int j721e_pcie_start_link(struct cdns_pcie *cdns_pcie) +{ + struct j721e_pcie *pcie = dev_get_drvdata(cdns_pcie->dev); + u32 reg; + + reg = j721e_pcie_user_readl(pcie, J721E_PCIE_USER_CMD_STATUS); + reg |= LINK_TRAINING_ENABLE; + j721e_pcie_user_writel(pcie, J721E_PCIE_USER_CMD_STATUS, reg); + + return 0; +} + +static void j721e_pcie_stop_link(struct cdns_pcie *cdns_pcie) +{ + struct j721e_pcie *pcie = dev_get_drvdata(cdns_pcie->dev); + u32 reg; + + reg = j721e_pcie_user_readl(pcie, J721E_PCIE_USER_CMD_STATUS); + reg &= ~LINK_TRAINING_ENABLE; + j721e_pcie_user_writel(pcie, J721E_PCIE_USER_CMD_STATUS, reg); +} + +static bool j721e_pcie_link_up(struct cdns_pcie *cdns_pcie) +{ + struct j721e_pcie *pcie = dev_get_drvdata(cdns_pcie->dev); + u32 reg; + + reg = j721e_pcie_user_readl(pcie, J721E_PCIE_USER_LINKSTATUS); + return (reg & LINK_STATUS) == LINK_UP_DL_COMPLETED; +} + +static const struct cdns_pcie_ops j721e_pcie_ops = { + .start_link = j721e_pcie_start_link, + .stop_link = j721e_pcie_stop_link, + .link_up = j721e_pcie_link_up, +}; + +static int j721e_pcie_set_mode(struct j721e_pcie *pcie, struct regmap *syscon, + unsigned int offset) +{ + struct device *dev = pcie->cdns_pcie->dev; + u32 mask = J721E_MODE_RC; + u32 mode = pcie->mode; + u32 val = 0; + int ret = 0; + + if (mode == PCI_MODE_RC) + val = J721E_MODE_RC; + + ret = regmap_update_bits(syscon, offset, mask, val); + if (ret) + dev_err(dev, "failed to set pcie mode\n"); + + return ret; +} + +static int j721e_pcie_set_link_speed(struct j721e_pcie *pcie, + struct regmap *syscon, unsigned int offset) +{ + struct device *dev = pcie->cdns_pcie->dev; + struct device_node *np = dev->of_node; + int link_speed; + u32 val = 0; + int ret; + + link_speed = of_pci_get_max_link_speed(np); + if ((link_speed < 2) || + (pcie_get_link_speed(link_speed) == PCI_SPEED_UNKNOWN)) + link_speed = 2; + + pcie->cdns_pcie->max_link_speed = link_speed; + val = link_speed - 1; + ret = regmap_update_bits(syscon, offset, GENERATION_SEL_MASK, val); + if (ret) + dev_err(dev, "failed to set link speed\n"); + + return ret; +} + +static int j721e_pcie_set_lane_count(struct j721e_pcie *pcie, + struct regmap *syscon, unsigned int offset) +{ + struct device *dev = pcie->cdns_pcie->dev; + u32 lanes = pcie->num_lanes; + u32 mask = BIT(8); + u32 val = 0; + int ret; + + if (pcie->max_lanes == 4) + mask = GENMASK(9, 8); + + val = LANE_COUNT(lanes - 1); + ret = regmap_update_bits(syscon, offset, mask, val); + if (ret) + dev_err(dev, "failed to set link count\n"); + + return ret; +} + +static int j721e_enable_acspcie_refclk(struct j721e_pcie *pcie, + struct regmap *syscon) +{ + struct device *dev = pcie->cdns_pcie->dev; + struct device_node *node = dev->of_node; + u32 mask = ACSPCIE_PAD_DISABLE_MASK; + struct of_phandle_args args; + u32 val; + int ret; + + ret = of_parse_phandle_with_fixed_args(node, + "ti,syscon-acspcie-proxy-ctrl", + 1, 0, &args); + if (ret) { + dev_err(dev, + "ti,syscon-acspcie-proxy-ctrl has invalid arguments\n"); + return ret; + } + + /* Clear PAD IO disable bits to enable refclk output */ + val = ~(args.args[0]); + ret = regmap_update_bits(syscon, 0, mask, val); + if (ret) { + dev_err(dev, "failed to enable ACSPCIE refclk: %d\n", ret); + return ret; + } + + return 0; +} + +static int j721e_pcie_ctrl_init(struct j721e_pcie *pcie) +{ + struct device *dev = pcie->cdns_pcie->dev; + struct device_node *node = dev->of_node; + struct of_phandle_args args; + unsigned int offset = 0; + struct regmap *syscon; + int ret; + + syscon = syscon_regmap_lookup_by_phandle(node, "ti,syscon-pcie-ctrl"); + if (IS_ERR(syscon)) { + dev_err(dev, "Unable to get ti,syscon-pcie-ctrl regmap\n"); + return PTR_ERR(syscon); + } + + /* Do not error out to maintain old DT compatibility */ + ret = of_parse_phandle_with_fixed_args(node, "ti,syscon-pcie-ctrl", 1, + 0, &args); + if (!ret) + offset = args.args[0]; + + /* + * The PCIe Controller's registers have different "reset-values" + * depending on the "strap" settings programmed into the PCIEn_CTRL + * register within the CTRL_MMR memory-mapped register space. + * The registers latch onto a "reset-value" based on the "strap" + * settings sampled after the PCIe Controller is powered on. + * To ensure that the "reset-values" are sampled accurately, power + * off the PCIe Controller before programming the "strap" settings + * and power it on after that. The runtime PM APIs namely + * pm_runtime_put_sync() and pm_runtime_get_sync() will decrement and + * increment the usage counter respectively, causing GENPD to power off + * and power on the PCIe Controller. + */ + ret = pm_runtime_put_sync(dev); + if (ret < 0) { + dev_err(dev, "Failed to power off PCIe Controller\n"); + return ret; + } + + ret = j721e_pcie_set_mode(pcie, syscon, offset); + if (ret < 0) { + dev_err(dev, "Failed to set pci mode\n"); + return ret; + } + + ret = j721e_pcie_set_link_speed(pcie, syscon, offset); + if (ret < 0) { + dev_err(dev, "Failed to set link speed\n"); + return ret; + } + + ret = j721e_pcie_set_lane_count(pcie, syscon, offset); + if (ret < 0) { + dev_err(dev, "Failed to set num-lanes\n"); + return ret; + } + + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + dev_err(dev, "Failed to power on PCIe Controller\n"); + return ret; + } + + /* Enable ACSPCIE refclk output if the optional property exists */ + syscon = syscon_regmap_lookup_by_phandle_optional(node, + "ti,syscon-acspcie-proxy-ctrl"); + if (!syscon) + return 0; + + return j721e_enable_acspcie_refclk(pcie, syscon); +} + +static int cdns_ti_pcie_config_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *value) +{ + if (pci_is_root_bus(bus)) + return pci_generic_config_read32(bus, devfn, where, size, + value); + + return pci_generic_config_read(bus, devfn, where, size, value); +} + +static int cdns_ti_pcie_config_write(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 value) +{ + if (pci_is_root_bus(bus)) + return pci_generic_config_write32(bus, devfn, where, size, + value); + + return pci_generic_config_write(bus, devfn, where, size, value); +} + +static struct pci_ops cdns_ti_pcie_host_ops = { + .map_bus = cdns_pci_map_bus, + .read = cdns_ti_pcie_config_read, + .write = cdns_ti_pcie_config_write, +}; + +static const struct j721e_pcie_data j721e_pcie_rc_data = { + .mode = PCI_MODE_RC, + .quirk_retrain_flag = true, + .byte_access_allowed = false, + .linkdown_irq_regfield = LINK_DOWN, + .max_lanes = 2, +}; + +static const struct j721e_pcie_data j721e_pcie_ep_data = { + .mode = PCI_MODE_EP, + .linkdown_irq_regfield = LINK_DOWN, + .max_lanes = 2, +}; + +static const struct j721e_pcie_data j7200_pcie_rc_data = { + .mode = PCI_MODE_RC, + .quirk_detect_quiet_flag = true, + .linkdown_irq_regfield = J7200_LINK_DOWN, + .byte_access_allowed = true, + .max_lanes = 4, +}; + +static const struct j721e_pcie_data j7200_pcie_ep_data = { + .mode = PCI_MODE_EP, + .quirk_detect_quiet_flag = true, + .linkdown_irq_regfield = J7200_LINK_DOWN, + .quirk_disable_flr = true, + .max_lanes = 4, +}; + +static const struct j721e_pcie_data am64_pcie_rc_data = { + .mode = PCI_MODE_RC, + .linkdown_irq_regfield = J7200_LINK_DOWN, + .byte_access_allowed = true, + .max_lanes = 1, +}; + +static const struct j721e_pcie_data am64_pcie_ep_data = { + .mode = PCI_MODE_EP, + .linkdown_irq_regfield = J7200_LINK_DOWN, + .max_lanes = 1, +}; + +static const struct j721e_pcie_data j784s4_pcie_rc_data = { + .mode = PCI_MODE_RC, + .quirk_retrain_flag = true, + .byte_access_allowed = false, + .linkdown_irq_regfield = J7200_LINK_DOWN, + .max_lanes = 4, +}; + +static const struct j721e_pcie_data j784s4_pcie_ep_data = { + .mode = PCI_MODE_EP, + .linkdown_irq_regfield = J7200_LINK_DOWN, + .max_lanes = 4, +}; + +static const struct j721e_pcie_data j722s_pcie_rc_data = { + .mode = PCI_MODE_RC, + .linkdown_irq_regfield = J7200_LINK_DOWN, + .byte_access_allowed = true, + .max_lanes = 1, +}; + +static const struct of_device_id of_j721e_pcie_match[] = { + { + .compatible = "ti,j721e-pcie-host", + .data = &j721e_pcie_rc_data, + }, + { + .compatible = "ti,j721e-pcie-ep", + .data = &j721e_pcie_ep_data, + }, + { + .compatible = "ti,j7200-pcie-host", + .data = &j7200_pcie_rc_data, + }, + { + .compatible = "ti,j7200-pcie-ep", + .data = &j7200_pcie_ep_data, + }, + { + .compatible = "ti,am64-pcie-host", + .data = &am64_pcie_rc_data, + }, + { + .compatible = "ti,am64-pcie-ep", + .data = &am64_pcie_ep_data, + }, + { + .compatible = "ti,j784s4-pcie-host", + .data = &j784s4_pcie_rc_data, + }, + { + .compatible = "ti,j784s4-pcie-ep", + .data = &j784s4_pcie_ep_data, + }, + { + .compatible = "ti,j722s-pcie-host", + .data = &j722s_pcie_rc_data, + }, + {}, +}; +MODULE_DEVICE_TABLE(of, of_j721e_pcie_match); + +static int j721e_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *node = dev->of_node; + struct pci_host_bridge *bridge; + const struct j721e_pcie_data *data; + struct cdns_pcie *cdns_pcie; + struct j721e_pcie *pcie; + struct cdns_pcie_rc *rc = NULL; + struct cdns_pcie_ep *ep = NULL; + void __iomem *base; + u32 num_lanes; + u32 mode; + int ret; + int irq; + + data = of_device_get_match_data(dev); + if (!data) + return -EINVAL; + + mode = (u32)data->mode; + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + switch (mode) { + case PCI_MODE_RC: + if (!IS_ENABLED(CONFIG_PCI_J721E_HOST)) + return -ENODEV; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rc)); + if (!bridge) + return -ENOMEM; + + if (!data->byte_access_allowed) + bridge->ops = &cdns_ti_pcie_host_ops; + rc = pci_host_bridge_priv(bridge); + rc->quirk_retrain_flag = data->quirk_retrain_flag; + rc->quirk_detect_quiet_flag = data->quirk_detect_quiet_flag; + + cdns_pcie = &rc->pcie; + cdns_pcie->dev = dev; + cdns_pcie->ops = &j721e_pcie_ops; + pcie->cdns_pcie = cdns_pcie; + break; + case PCI_MODE_EP: + if (!IS_ENABLED(CONFIG_PCI_J721E_EP)) + return -ENODEV; + + ep = devm_kzalloc(dev, sizeof(*ep), GFP_KERNEL); + if (!ep) + return -ENOMEM; + + ep->quirk_detect_quiet_flag = data->quirk_detect_quiet_flag; + ep->quirk_disable_flr = data->quirk_disable_flr; + + cdns_pcie = &ep->pcie; + cdns_pcie->dev = dev; + cdns_pcie->ops = &j721e_pcie_ops; + pcie->cdns_pcie = cdns_pcie; + break; + default: + dev_err(dev, "INVALID device type %d\n", mode); + return 0; + } + + pcie->mode = mode; + pcie->linkdown_irq_regfield = data->linkdown_irq_regfield; + + base = devm_platform_ioremap_resource_byname(pdev, "intd_cfg"); + if (IS_ERR(base)) + return PTR_ERR(base); + pcie->intd_cfg_base = base; + + base = devm_platform_ioremap_resource_byname(pdev, "user_cfg"); + if (IS_ERR(base)) + return PTR_ERR(base); + pcie->user_cfg_base = base; + + ret = of_property_read_u32(node, "num-lanes", &num_lanes); + if (ret || num_lanes > data->max_lanes) { + dev_warn(dev, "num-lanes property not provided or invalid, setting num-lanes to 1\n"); + num_lanes = 1; + } + + pcie->num_lanes = num_lanes; + pcie->max_lanes = data->max_lanes; + + if (dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48))) + return -EINVAL; + + irq = platform_get_irq_byname(pdev, "link_state"); + if (irq < 0) + return irq; + + dev_set_drvdata(dev, pcie); + pm_runtime_enable(dev); + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + dev_err_probe(dev, ret, "pm_runtime_get_sync failed\n"); + goto err_get_sync; + } + + ret = j721e_pcie_ctrl_init(pcie); + if (ret < 0) { + dev_err_probe(dev, ret, "j721e_pcie_ctrl_init failed\n"); + goto err_get_sync; + } + + ret = devm_request_irq(dev, irq, j721e_pcie_link_irq_handler, 0, + "j721e-pcie-link-down-irq", pcie); + if (ret < 0) { + dev_err_probe(dev, ret, "failed to request link state IRQ %d\n", irq); + goto err_get_sync; + } + + j721e_pcie_config_link_irq(pcie); + + switch (mode) { + case PCI_MODE_RC: + pcie->reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW); + if (IS_ERR(pcie->reset_gpio)) { + ret = dev_err_probe(dev, PTR_ERR(pcie->reset_gpio), + "Failed to get reset GPIO\n"); + goto err_get_sync; + } + + ret = cdns_pcie_init_phy(dev, cdns_pcie); + if (ret) { + dev_err_probe(dev, ret, "Failed to init phy\n"); + goto err_get_sync; + } + + pcie->refclk = devm_clk_get_optional_enabled(dev, "pcie_refclk"); + if (IS_ERR(pcie->refclk)) { + ret = dev_err_probe(dev, PTR_ERR(pcie->refclk), + "failed to enable pcie_refclk\n"); + goto err_pcie_setup; + } + + /* + * Section 2.2 of the PCI Express Card Electromechanical + * Specification (Revision 5.1) mandates that the deassertion + * of the PERST# signal should be delayed by 100 ms (TPVPERL). + * This shall ensure that the power and the reference clock + * are stable. + */ + if (pcie->reset_gpio) { + msleep(PCIE_T_PVPERL_MS); + gpiod_set_value_cansleep(pcie->reset_gpio, 1); + } + + if (IS_ENABLED(CONFIG_PCI_J721E_HOST)) { + ret = cdns_pcie_host_setup(rc); + if (ret < 0) + goto err_pcie_setup; + } + + break; + case PCI_MODE_EP: + ret = cdns_pcie_init_phy(dev, cdns_pcie); + if (ret) { + dev_err_probe(dev, ret, "Failed to init phy\n"); + goto err_get_sync; + } + + if (IS_ENABLED(CONFIG_PCI_J721E_EP)) { + ret = cdns_pcie_ep_setup(ep); + if (ret < 0) + goto err_pcie_setup; + } + + break; + } + + return 0; + +err_pcie_setup: + cdns_pcie_disable_phy(cdns_pcie); + +err_get_sync: + pm_runtime_put(dev); + pm_runtime_disable(dev); + + return ret; +} + +static void j721e_pcie_remove(struct platform_device *pdev) +{ + struct j721e_pcie *pcie = platform_get_drvdata(pdev); + struct cdns_pcie *cdns_pcie = pcie->cdns_pcie; + struct device *dev = &pdev->dev; + struct cdns_pcie_ep *ep; + struct cdns_pcie_rc *rc; + + if (IS_ENABLED(CONFIG_PCI_J721E_HOST) && + pcie->mode == PCI_MODE_RC) { + rc = container_of(cdns_pcie, struct cdns_pcie_rc, pcie); + cdns_pcie_host_disable(rc); + } else if (IS_ENABLED(CONFIG_PCI_J721E_EP)) { + ep = container_of(cdns_pcie, struct cdns_pcie_ep, pcie); + cdns_pcie_ep_disable(ep); + } + + gpiod_set_value_cansleep(pcie->reset_gpio, 0); + + cdns_pcie_disable_phy(cdns_pcie); + j721e_pcie_disable_link_irq(pcie); + pm_runtime_put(dev); + pm_runtime_disable(dev); +} + +static int j721e_pcie_suspend_noirq(struct device *dev) +{ + struct j721e_pcie *pcie = dev_get_drvdata(dev); + + if (pcie->mode == PCI_MODE_RC) { + gpiod_set_value_cansleep(pcie->reset_gpio, 0); + clk_disable_unprepare(pcie->refclk); + } + + cdns_pcie_disable_phy(pcie->cdns_pcie); + + return 0; +} + +static int j721e_pcie_resume_noirq(struct device *dev) +{ + struct j721e_pcie *pcie = dev_get_drvdata(dev); + struct cdns_pcie *cdns_pcie = pcie->cdns_pcie; + int ret; + + ret = j721e_pcie_ctrl_init(pcie); + if (ret < 0) + return ret; + + j721e_pcie_config_link_irq(pcie); + + /* + * This is not called explicitly in the probe, it is called by + * cdns_pcie_init_phy(). + */ + ret = cdns_pcie_enable_phy(pcie->cdns_pcie); + if (ret < 0) + return ret; + + if (pcie->mode == PCI_MODE_RC) { + struct cdns_pcie_rc *rc = cdns_pcie_to_rc(cdns_pcie); + + ret = clk_prepare_enable(pcie->refclk); + if (ret < 0) + return ret; + + /* + * Section 2.2 of the PCI Express Card Electromechanical + * Specification (Revision 5.1) mandates that the deassertion + * of the PERST# signal should be delayed by 100 ms (TPVPERL). + * This shall ensure that the power and the reference clock + * are stable. + */ + if (pcie->reset_gpio) { + msleep(PCIE_T_PVPERL_MS); + gpiod_set_value_cansleep(pcie->reset_gpio, 1); + } + + if (IS_ENABLED(CONFIG_PCI_J721E_HOST)) { + ret = cdns_pcie_host_link_setup(rc); + if (ret < 0) { + clk_disable_unprepare(pcie->refclk); + return ret; + } + } + + /* + * Reset internal status of BARs to force reinitialization in + * cdns_pcie_host_init(). + */ + for (enum cdns_pcie_rp_bar bar = RP_BAR0; bar <= RP_NO_BAR; bar++) + rc->avail_ib_bar[bar] = true; + + if (IS_ENABLED(CONFIG_PCI_J721E_HOST)) { + ret = cdns_pcie_host_init(rc); + if (ret) { + clk_disable_unprepare(pcie->refclk); + return ret; + } + } + } + + return 0; +} + +static DEFINE_NOIRQ_DEV_PM_OPS(j721e_pcie_pm_ops, + j721e_pcie_suspend_noirq, + j721e_pcie_resume_noirq); + +static struct platform_driver j721e_pcie_driver = { + .probe = j721e_pcie_probe, + .remove = j721e_pcie_remove, + .driver = { + .name = "j721e-pcie", + .of_match_table = of_j721e_pcie_match, + .suppress_bind_attrs = true, + .pm = pm_sleep_ptr(&j721e_pcie_pm_ops), + }, +}; +module_platform_driver(j721e_pcie_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("PCIe controller driver for TI's J721E and related SoCs"); +MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>"); diff --git a/drivers/pci/controller/cadence/pci-sky1.c b/drivers/pci/controller/cadence/pci-sky1.c new file mode 100644 index 000000000..b8632f1d3 --- /dev/null +++ b/drivers/pci/controller/cadence/pci-sky1.c @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe controller driver for CIX's sky1 SoCs + * + * Copyright 2025 Cix Technology Group Co., Ltd. + * Author: Hans Zhang <hans.zhang@cixtech.com> + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/pci.h> +#include <linux/pci-ecam.h> +#include <linux/pci_ids.h> + +#include "pcie-cadence.h" +#include "pcie-cadence-host-common.h" + +#define PCI_VENDOR_ID_CIX 0x1f6c +#define PCI_DEVICE_ID_CIX_SKY1 0x0001 + +#define STRAP_REG(n) ((n) * 0x04) +#define STATUS_REG(n) ((n) * 0x04) +#define LINK_TRAINING_ENABLE BIT(0) +#define LINK_COMPLETE BIT(0) + +#define SKY1_IP_REG_BANK 0x1000 +#define SKY1_IP_CFG_CTRL_REG_BANK 0x4c00 +#define SKY1_IP_AXI_MASTER_COMMON 0xf000 +#define SKY1_AXI_SLAVE 0x9000 +#define SKY1_AXI_MASTER 0xb000 +#define SKY1_AXI_HLS_REGISTERS 0xc000 +#define SKY1_AXI_RAS_REGISTERS 0xe000 +#define SKY1_DTI_REGISTERS 0xd000 + +#define IP_REG_I_DBG_STS_0 0x420 + +struct sky1_pcie { + struct cdns_pcie *cdns_pcie; + struct cdns_pcie_rc *cdns_pcie_rc; + + struct resource *cfg_res; + struct resource *msg_res; + struct pci_config_window *cfg; + void __iomem *strap_base; + void __iomem *status_base; + void __iomem *reg_base; + void __iomem *cfg_base; + void __iomem *msg_base; +}; + +static int sky1_pcie_resource_get(struct platform_device *pdev, + struct sky1_pcie *pcie) +{ + struct device *dev = &pdev->dev; + struct resource *res; + void __iomem *base; + + base = devm_platform_ioremap_resource_byname(pdev, "reg"); + if (IS_ERR(base)) + return dev_err_probe(dev, PTR_ERR(base), + "unable to find \"reg\" registers\n"); + pcie->reg_base = base; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg"); + if (!res) + return dev_err_probe(dev, -ENODEV, "unable to get \"cfg\" resource\n"); + pcie->cfg_res = res; + + base = devm_platform_ioremap_resource_byname(pdev, "rcsu_strap"); + if (IS_ERR(base)) + return dev_err_probe(dev, PTR_ERR(base), + "unable to find \"rcsu_strap\" registers\n"); + pcie->strap_base = base; + + base = devm_platform_ioremap_resource_byname(pdev, "rcsu_status"); + if (IS_ERR(base)) + return dev_err_probe(dev, PTR_ERR(base), + "unable to find \"rcsu_status\" registers\n"); + pcie->status_base = base; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "msg"); + if (!res) + return dev_err_probe(dev, -ENODEV, "unable to get \"msg\" resource\n"); + pcie->msg_res = res; + pcie->msg_base = devm_ioremap_resource(dev, res); + if (IS_ERR(pcie->msg_base)) { + return dev_err_probe(dev, PTR_ERR(pcie->msg_base), + "unable to ioremap msg resource\n"); + } + + return 0; +} + +static int sky1_pcie_start_link(struct cdns_pcie *cdns_pcie) +{ + struct sky1_pcie *pcie = dev_get_drvdata(cdns_pcie->dev); + u32 val; + + val = readl(pcie->strap_base + STRAP_REG(1)); + val |= LINK_TRAINING_ENABLE; + writel(val, pcie->strap_base + STRAP_REG(1)); + + return 0; +} + +static void sky1_pcie_stop_link(struct cdns_pcie *cdns_pcie) +{ + struct sky1_pcie *pcie = dev_get_drvdata(cdns_pcie->dev); + u32 val; + + val = readl(pcie->strap_base + STRAP_REG(1)); + val &= ~LINK_TRAINING_ENABLE; + writel(val, pcie->strap_base + STRAP_REG(1)); +} + +static bool sky1_pcie_link_up(struct cdns_pcie *cdns_pcie) +{ + u32 val; + + val = cdns_pcie_hpa_readl(cdns_pcie, REG_BANK_IP_REG, + IP_REG_I_DBG_STS_0); + return val & LINK_COMPLETE; +} + +static const struct cdns_pcie_ops sky1_pcie_ops = { + .start_link = sky1_pcie_start_link, + .stop_link = sky1_pcie_stop_link, + .link_up = sky1_pcie_link_up, +}; + +static int sky1_pcie_probe(struct platform_device *pdev) +{ + struct cdns_plat_pcie_of_data *reg_off; + struct device *dev = &pdev->dev; + struct pci_host_bridge *bridge; + struct cdns_pcie *cdns_pcie; + struct resource_entry *bus; + struct cdns_pcie_rc *rc; + struct sky1_pcie *pcie; + int ret; + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rc)); + if (!bridge) + return -ENOMEM; + + ret = sky1_pcie_resource_get(pdev, pcie); + if (ret < 0) + return ret; + + bus = resource_list_first_type(&bridge->windows, IORESOURCE_BUS); + if (!bus) + return -ENODEV; + + pcie->cfg = pci_ecam_create(dev, pcie->cfg_res, bus->res, + &pci_generic_ecam_ops); + if (IS_ERR(pcie->cfg)) + return PTR_ERR(pcie->cfg); + + bridge->ops = (struct pci_ops *)&pci_generic_ecam_ops.pci_ops; + rc = pci_host_bridge_priv(bridge); + rc->ecam_supported = 1; + rc->cfg_base = pcie->cfg->win; + rc->cfg_res = &pcie->cfg->res; + + cdns_pcie = &rc->pcie; + cdns_pcie->dev = dev; + cdns_pcie->ops = &sky1_pcie_ops; + cdns_pcie->reg_base = pcie->reg_base; + cdns_pcie->msg_res = pcie->msg_res; + cdns_pcie->is_rc = true; + cdns_pcie->is_hpa = true; + + reg_off = devm_kzalloc(dev, sizeof(*reg_off), GFP_KERNEL); + if (!reg_off) { + pci_ecam_free(pcie->cfg); + return -ENOMEM; + } + + reg_off->ip_reg_bank_offset = SKY1_IP_REG_BANK; + reg_off->ip_cfg_ctrl_reg_offset = SKY1_IP_CFG_CTRL_REG_BANK; + reg_off->axi_mstr_common_offset = SKY1_IP_AXI_MASTER_COMMON; + reg_off->axi_slave_offset = SKY1_AXI_SLAVE; + reg_off->axi_master_offset = SKY1_AXI_MASTER; + reg_off->axi_hls_offset = SKY1_AXI_HLS_REGISTERS; + reg_off->axi_ras_offset = SKY1_AXI_RAS_REGISTERS; + reg_off->axi_dti_offset = SKY1_DTI_REGISTERS; + cdns_pcie->cdns_pcie_reg_offsets = reg_off; + + pcie->cdns_pcie = cdns_pcie; + pcie->cdns_pcie_rc = rc; + pcie->cfg_base = rc->cfg_base; + bridge->sysdata = pcie->cfg; + + rc->vendor_id = PCI_VENDOR_ID_CIX; + rc->device_id = PCI_DEVICE_ID_CIX_SKY1; + rc->no_inbound_map = 1; + + dev_set_drvdata(dev, pcie); + + ret = cdns_pcie_hpa_host_setup(rc); + if (ret < 0) { + pci_ecam_free(pcie->cfg); + return ret; + } + + return 0; +} + +static const struct of_device_id of_sky1_pcie_match[] = { + { .compatible = "cix,sky1-pcie-host", }, + {}, +}; +MODULE_DEVICE_TABLE(of, of_sky1_pcie_match); + +static void sky1_pcie_remove(struct platform_device *pdev) +{ + struct sky1_pcie *pcie = platform_get_drvdata(pdev); + struct cdns_pcie_rc *rc; + + rc = container_of(pcie->cdns_pcie, struct cdns_pcie_rc, pcie); + cdns_pcie_hpa_host_disable(rc); + + pci_ecam_free(pcie->cfg); +} + +static struct platform_driver sky1_pcie_driver = { + .probe = sky1_pcie_probe, + .remove = sky1_pcie_remove, + .driver = { + .name = "sky1-pcie", + .of_match_table = of_sky1_pcie_match, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, +}; +module_platform_driver(sky1_pcie_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("PCIe controller driver for CIX's sky1 SoCs"); +MODULE_AUTHOR("Hans Zhang <hans.zhang@cixtech.com>"); diff --git a/drivers/pci/controller/cadence/pcie-cadence-debugfs.c b/drivers/pci/controller/cadence/pcie-cadence-debugfs.c new file mode 100644 index 000000000..0a308f95e --- /dev/null +++ b/drivers/pci/controller/cadence/pcie-cadence-debugfs.c @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Cadence PCIe controller debugfs driver + * + * Copyright (C) 2026 Hans Zhang <18255117159@163.com> + */ + +#include <linux/bitfield.h> +#include <linux/debugfs.h> +#include <linux/seq_file.h> + +#include "pcie-cadence.h" + +#define CDNS_DEBUGFS_BUF_MAX 128 + +static const char *cdns_pcie_lga_ltssm_status_string(enum cdns_pcie_lga_ltssm ltssm) +{ + const char *str; + + switch (ltssm) { +#define CDNS_PCIE_LGA_LTSSM_NAME(n) case n: str = #n; break + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_DETECT_QUIET); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_DETECT_ACTIVE); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_POLLING_ACTIVE); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_POLLING_COMPLIANCE); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_POLLING_CONFIGURATION); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_CONFIGURATION_LINKWIDTH_START); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_CONFIGURATION_LINKWIDTH_ACCEPT); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_CONFIGURATION_LANENUM_ACCEPT); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_CONFIGURATION_LANENUM_WAIT); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_CONFIGURATION_COMPLETE); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_CONFIGURATION_IDLE); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_RECOVERY_RCVRLOCK); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_RECOVERY_SPEED); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_RECOVERY_RCVRCFG); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_RECOVERY_IDLE); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_L0); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_RX_L0S_ENTRY); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_RX_L0S_IDLE); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_RX_L0S_FTS); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_TX_L0S_ENTRY); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_TX_L0S_IDLE); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_TX_L0S_FTS); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_L1_ENTRY); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_L1_IDLE); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_L2_IDLE); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_L2_TRANSMITWAKE); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_DISABLED); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_LOOPBACK_ENTRY_MASTER); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_LOOPBACK_ACTIVE_MASTER); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_LOOPBACK_EXIT_MASTER); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_LOOPBACK_ENTRY_SLAVE); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_LOOPBACK_ACTIVE_SLAVE); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_LOOPBACK_EXIT_SLAVE); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_HOT_RESET); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_RECOVERY_EQUALIZATION_PHASE_0); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_RECOVERY_EQUALIZATION_PHASE_1); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_RECOVERY_EQUALIZATION_PHASE_2); + CDNS_PCIE_LGA_LTSSM_NAME(CDNS_PCIE_LGA_LTSSM_RECOVERY_EQUALIZATION_PHASE_3); + default: + str = "CDNS_PCIE_LGA_LTSSM_UNKNOWN"; + break; + } + + return str + strlen("CDNS_PCIE_LGA_LTSSM_"); +} + +static const char *cdns_pcie_hpa_ltssm_status_string(enum cdns_pcie_hpa_ltssm ltssm) +{ + const char *str; + + switch (ltssm) { +#define CDNS_PCIE_HPA_LTSSM_NAME(n) case n: str = #n; break + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_DETECT_QUIET); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_DETECT_QUIET_ENTRY); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_DETECT_ACTIVE); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_DETECT_ACTIVE_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_DETECT_ACTIVE_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_DETECT_ACTIVE_3); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_RCVR_DETECTED_ST); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_RCVR_DETECTED_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_POLLING_ACTIVE); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_POLLING_ACTIVE_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_POLLING_ACTIVE_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_POLLING_ACTIVE_3); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_POLLING_CONFIG); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_POLLING_CONFIG_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_POLLING_CONFIG_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_LW_START_RC); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_LW_START_RC_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_LW_START_RC_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_LW_ACC_RC); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_LANENUM_WAIT_RC); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_LANENUM_WAIT_RC_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_LANENUM_ACC_RC); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_LW_START_EP); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_LW_START_EP_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_LW_START_EP_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_LW_ACC_EP); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_LANENUM_WAIT_EP); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_LANENUM_WAIT_EP_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_LANENUM_ACC_EP); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_LANENUM_ACC_EP_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_DUMMY_STATE_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_COMPLETE); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_COMPLETE_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_COMPLETE_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_IDLE); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_CONFIG_IDLE_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_DUMMY_STATE_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_DUMMY_STATE_3); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_DUMMY_STATE_4); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L0_STATE); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_RECOVERY_RCVR_LOCK); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_RECOVERY_RCVR_LOCK_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_RECOVERY_RCVR_CFG); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_RECOVERY_RCVR_CFG_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_RECOVERY_IDLE); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_RECOVERY_IDLE_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_DISABLE_LINK); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_DISABLE_LINK_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_DISABLE_LINK_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_DISABLE_LINK_3); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_DISABLE_LINK_4); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_DISABLE_LINK_5); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_DISABLE_LINK_6); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_DISABLE_LINK_7); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_HOT_RESET); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_HOT_RESET_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_HOT_RESET_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_HOT_RESET_3); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L0S_ENTRY); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L0S_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L0S_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L0S_3); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L0S_4); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L0S_5); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_WAIT_FOR_LINK_TX); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_TX_FTS_ENTRY); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_TX_FTS_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_TX_FTS_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_TX_ELEC_IDLE_ST); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_TX_ELEC_IDLE_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_TX_ELEC_IDLE_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_TX_ELEC_IDLE_3); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_RECOVERY_SPEED); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_RECOVERY_SPEED_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_RECOVERY_SPEED_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_RECOVERY_SPEED_3); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_GEN23); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_GEN23_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_GEN23_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_GEN23_3); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_GEN23_4); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_GEN23_5); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_GEN23_6); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_GEN23_7); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_GEN23_8); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_ENTRY); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_ENTRY_FROM_RECOVERY); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_EXIT_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_EXIT); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_GEN2_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_GEN2_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_GEN2_3); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_GEN2_4); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_GEN2_5); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_ACTIVE); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L1_ENTRY); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L1_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L1_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L1_3); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L1_4); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L1_IDLE); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L1_EXIT); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L2_ENTRY); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L2_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L2_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L2_3); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L2_4); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L2_5); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_L2_IDLE); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_ENTRY); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_ENTRY_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_ENTRY_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_ENTRY_3); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_ENTRY_4); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_ENTRY_5); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_ENTRY_FROM_RECOVERY); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_ACTIVE); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_EXIT); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_EXIT_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_EXIT_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_RECOVERY_EQUALIZATION_PHASE0); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_RECOVERY_EQUALIZATION_PHASE1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_RECOVERY_EQUALIZATION_PHASE2_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_RECOVERY_EQUALIZATION_PHASE2_2); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_RECOVERY_EQUALIZATION_PHASE3_1); + CDNS_PCIE_HPA_LTSSM_NAME(CDNS_PCIE_HPA_LTSSM_RECOVERY_EQUALIZATION_PHASE3_2); + default: + str = "CDNS_PCIE_HPA_LTSSM_UNKNOWN"; + break; + } + + return str + strlen("CDNS_PCIE_HPA_LTSSM_"); +} + +static int ltssm_status_show(struct seq_file *s, void *v) +{ + struct cdns_pcie *pci = s->private; + enum cdns_pcie_lga_ltssm lga_ltssm; + enum cdns_pcie_hpa_ltssm hpa_ltssm; + const char *str_ltssm; + u32 val; + + if (pci->is_hpa) { + val = cdns_pcie_hpa_readl(pci, REG_BANK_IP_REG, + CDNS_PCIE_HPA_PHY_DBG_STS_REG0); + hpa_ltssm = FIELD_GET(CDNS_PCIE_HPA_LTSSM_STATUS_MASK, val); + str_ltssm = cdns_pcie_hpa_ltssm_status_string(hpa_ltssm); + } else { + val = cdns_pcie_readl(pci, CDNS_PCIE_LM_BASE); + lga_ltssm = FIELD_GET(CDNS_PCIE_LGA_LTSSM_STATUS_MASK, val); + str_ltssm = cdns_pcie_lga_ltssm_status_string(lga_ltssm); + } + + seq_printf(s, "%s (0x%02x)\n", str_ltssm, + pci->is_hpa ? hpa_ltssm : lga_ltssm); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(ltssm_status); + +static void cdns_pcie_ltssm_debugfs_init(struct cdns_pcie *pci, struct dentry *dir) +{ + debugfs_create_file("ltssm_status", 0444, dir, pci, + <ssm_status_fops); +} + +void cdns_pcie_debugfs_deinit(struct cdns_pcie *pci) +{ + if (!pci->debug_dir) + return; + + debugfs_remove_recursive(pci->debug_dir); +} +EXPORT_SYMBOL_GPL(cdns_pcie_debugfs_deinit); + +void cdns_pcie_debugfs_init(struct cdns_pcie *pci) +{ + char dirname[CDNS_DEBUGFS_BUF_MAX]; + struct device *dev = pci->dev; + + /* Create main directory for each platform driver. */ + snprintf(dirname, CDNS_DEBUGFS_BUF_MAX, "cdns_pcie_%s", dev_name(dev)); + pci->debug_dir = debugfs_create_dir(dirname, NULL); + + cdns_pcie_ltssm_debugfs_init(pci, pci->debug_dir); +} +EXPORT_SYMBOL_GPL(cdns_pcie_debugfs_init); diff --git a/drivers/pci/controller/cadence/pcie-cadence-ep.c b/drivers/pci/controller/cadence/pcie-cadence-ep.c new file mode 100644 index 000000000..370b19f4d --- /dev/null +++ b/drivers/pci/controller/cadence/pcie-cadence-ep.c @@ -0,0 +1,778 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2017 Cadence +// Cadence PCIe endpoint controller driver. +// Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com> + +#include <linux/bitfield.h> +#include <linux/delay.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/pci-epc.h> +#include <linux/platform_device.h> +#include <linux/sizes.h> + +#include "pcie-cadence.h" +#include "../../pci.h" + +#define CDNS_PCIE_EP_MIN_APERTURE 128 /* 128 bytes */ +#define CDNS_PCIE_EP_IRQ_PCI_ADDR_NONE 0x1 +#define CDNS_PCIE_EP_IRQ_PCI_ADDR_LEGACY 0x3 + +static u8 cdns_pcie_get_fn_from_vfn(struct cdns_pcie *pcie, u8 fn, u8 vfn) +{ + u32 first_vf_offset, stride; + u16 cap; + + if (vfn == 0) + return fn; + + cap = cdns_pcie_find_ext_capability(pcie, PCI_EXT_CAP_ID_SRIOV); + first_vf_offset = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_SRIOV_VF_OFFSET); + stride = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_SRIOV_VF_STRIDE); + fn = fn + first_vf_offset + ((vfn - 1) * stride); + + return fn; +} + +static int cdns_pcie_ep_write_header(struct pci_epc *epc, u8 fn, u8 vfn, + struct pci_epf_header *hdr) +{ + struct cdns_pcie_ep *ep = epc_get_drvdata(epc); + struct cdns_pcie *pcie = &ep->pcie; + u32 reg; + u16 cap; + + cap = cdns_pcie_find_ext_capability(pcie, PCI_EXT_CAP_ID_SRIOV); + if (vfn > 1) { + dev_err(&epc->dev, "Only Virtual Function #1 has deviceID\n"); + return -EINVAL; + } else if (vfn == 1) { + reg = cap + PCI_SRIOV_VF_DID; + cdns_pcie_ep_fn_writew(pcie, fn, reg, hdr->deviceid); + return 0; + } + + cdns_pcie_ep_fn_writew(pcie, fn, PCI_DEVICE_ID, hdr->deviceid); + cdns_pcie_ep_fn_writeb(pcie, fn, PCI_REVISION_ID, hdr->revid); + cdns_pcie_ep_fn_writeb(pcie, fn, PCI_CLASS_PROG, hdr->progif_code); + cdns_pcie_ep_fn_writew(pcie, fn, PCI_CLASS_DEVICE, + hdr->subclass_code | hdr->baseclass_code << 8); + cdns_pcie_ep_fn_writeb(pcie, fn, PCI_CACHE_LINE_SIZE, + hdr->cache_line_size); + cdns_pcie_ep_fn_writew(pcie, fn, PCI_SUBSYSTEM_ID, hdr->subsys_id); + cdns_pcie_ep_fn_writeb(pcie, fn, PCI_INTERRUPT_PIN, hdr->interrupt_pin); + + /* + * Vendor ID can only be modified from function 0, all other functions + * use the same vendor ID as function 0. + */ + if (fn == 0) { + /* Update the vendor IDs. */ + u32 id = CDNS_PCIE_LM_ID_VENDOR(hdr->vendorid) | + CDNS_PCIE_LM_ID_SUBSYS(hdr->subsys_vendor_id); + + cdns_pcie_writel(pcie, CDNS_PCIE_LM_ID, id); + } + + return 0; +} + +static int cdns_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, u8 vfn, + struct pci_epf_bar *epf_bar) +{ + struct cdns_pcie_ep *ep = epc_get_drvdata(epc); + struct cdns_pcie_epf *epf = &ep->epf[fn]; + struct cdns_pcie *pcie = &ep->pcie; + dma_addr_t bar_phys = epf_bar->phys_addr; + enum pci_barno bar = epf_bar->barno; + int flags = epf_bar->flags; + u32 addr0, addr1, reg, cfg, b, aperture, ctrl; + u64 sz; + + /* BAR size is 2^(aperture + 7) */ + sz = max_t(size_t, epf_bar->size, CDNS_PCIE_EP_MIN_APERTURE); + /* + * roundup_pow_of_two() returns an unsigned long, which is not suited + * for 64bit values. + */ + sz = 1ULL << fls64(sz - 1); + aperture = ilog2(sz) - 7; /* 128B -> 0, 256B -> 1, 512B -> 2, ... */ + + if ((flags & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_IO) { + ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_IO_32BITS; + } else { + bool is_prefetch = !!(flags & PCI_BASE_ADDRESS_MEM_PREFETCH); + bool is_64bits = !!(flags & PCI_BASE_ADDRESS_MEM_TYPE_64); + + if (is_64bits && (bar & 1)) + return -EINVAL; + + if (is_64bits && is_prefetch) + ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_PREFETCH_MEM_64BITS; + else if (is_prefetch) + ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_PREFETCH_MEM_32BITS; + else if (is_64bits) + ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_MEM_64BITS; + else + ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_MEM_32BITS; + } + + addr0 = lower_32_bits(bar_phys); + addr1 = upper_32_bits(bar_phys); + + if (vfn == 1) + reg = CDNS_PCIE_LM_EP_VFUNC_BAR_CFG(bar, fn); + else + reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn); + b = (bar < BAR_4) ? bar : bar - BAR_4; + + if (vfn == 0 || vfn == 1) { + cfg = cdns_pcie_readl(pcie, reg); + cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) | + CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b)); + cfg |= (CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE(b, aperture) | + CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl)); + cdns_pcie_writel(pcie, reg, cfg); + } + + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar), + addr0); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar), + addr1); + + if (vfn > 0) + epf = &epf->epf[vfn - 1]; + epf->epf_bar[bar] = epf_bar; + + return 0; +} + +static void cdns_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, u8 vfn, + struct pci_epf_bar *epf_bar) +{ + struct cdns_pcie_ep *ep = epc_get_drvdata(epc); + struct cdns_pcie_epf *epf = &ep->epf[fn]; + struct cdns_pcie *pcie = &ep->pcie; + enum pci_barno bar = epf_bar->barno; + u32 reg, cfg, b, ctrl; + + if (vfn == 1) + reg = CDNS_PCIE_LM_EP_VFUNC_BAR_CFG(bar, fn); + else + reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn); + b = (bar < BAR_4) ? bar : bar - BAR_4; + + if (vfn == 0 || vfn == 1) { + ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED; + cfg = cdns_pcie_readl(pcie, reg); + cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) | + CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b)); + cfg |= CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl); + cdns_pcie_writel(pcie, reg, cfg); + } + + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar), 0); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar), 0); + + if (vfn > 0) + epf = &epf->epf[vfn - 1]; + epf->epf_bar[bar] = NULL; +} + +static int cdns_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, u8 vfn, + phys_addr_t addr, u64 pci_addr, size_t size) +{ + struct cdns_pcie_ep *ep = epc_get_drvdata(epc); + struct cdns_pcie *pcie = &ep->pcie; + u32 r; + + r = find_first_zero_bit(&ep->ob_region_map, BITS_PER_LONG); + if (r >= ep->max_regions - 1) { + dev_err(&epc->dev, "no free outbound region\n"); + return -EINVAL; + } + + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn); + cdns_pcie_set_outbound_region(pcie, 0, fn, r, false, addr, pci_addr, size); + + set_bit(r, &ep->ob_region_map); + ep->ob_addr[r] = addr; + + return 0; +} + +static void cdns_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn, u8 vfn, + phys_addr_t addr) +{ + struct cdns_pcie_ep *ep = epc_get_drvdata(epc); + struct cdns_pcie *pcie = &ep->pcie; + u32 r; + + for (r = 0; r < ep->max_regions - 1; r++) + if (ep->ob_addr[r] == addr) + break; + + if (r == ep->max_regions - 1) + return; + + cdns_pcie_reset_outbound_region(pcie, r); + + ep->ob_addr[r] = 0; + clear_bit(r, &ep->ob_region_map); +} + +static int cdns_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, u8 vfn, u8 nr_irqs) +{ + struct cdns_pcie_ep *ep = epc_get_drvdata(epc); + struct cdns_pcie *pcie = &ep->pcie; + u8 mmc = order_base_2(nr_irqs); + u16 flags; + u8 cap; + + cap = cdns_pcie_find_capability(pcie, PCI_CAP_ID_MSI); + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn); + + /* + * Set the Multiple Message Capable bitfield into the Message Control + * register. + */ + flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_FLAGS); + flags = (flags & ~PCI_MSI_FLAGS_QMASK) | (mmc << 1); + flags |= PCI_MSI_FLAGS_64BIT; + flags &= ~PCI_MSI_FLAGS_MASKBIT; + cdns_pcie_ep_fn_writew(pcie, fn, cap + PCI_MSI_FLAGS, flags); + + return 0; +} + +static int cdns_pcie_ep_get_msi(struct pci_epc *epc, u8 fn, u8 vfn) +{ + struct cdns_pcie_ep *ep = epc_get_drvdata(epc); + struct cdns_pcie *pcie = &ep->pcie; + u16 flags, mme; + u8 cap; + + cap = cdns_pcie_find_capability(pcie, PCI_CAP_ID_MSI); + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn); + + /* Validate that the MSI feature is actually enabled. */ + flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_FLAGS); + if (!(flags & PCI_MSI_FLAGS_ENABLE)) + return -EINVAL; + + /* + * Get the Multiple Message Enable bitfield from the Message Control + * register. + */ + mme = FIELD_GET(PCI_MSI_FLAGS_QSIZE, flags); + + return 1 << mme; +} + +static int cdns_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no) +{ + struct cdns_pcie_ep *ep = epc_get_drvdata(epc); + struct cdns_pcie *pcie = &ep->pcie; + u32 val, reg; + u8 cap; + + cap = cdns_pcie_find_capability(pcie, PCI_CAP_ID_MSIX); + func_no = cdns_pcie_get_fn_from_vfn(pcie, func_no, vfunc_no); + + reg = cap + PCI_MSIX_FLAGS; + val = cdns_pcie_ep_fn_readw(pcie, func_no, reg); + if (!(val & PCI_MSIX_FLAGS_ENABLE)) + return -EINVAL; + + val &= PCI_MSIX_FLAGS_QSIZE; + + return val + 1; +} + +static int cdns_pcie_ep_set_msix(struct pci_epc *epc, u8 fn, u8 vfn, + u16 nr_irqs, enum pci_barno bir, u32 offset) +{ + struct cdns_pcie_ep *ep = epc_get_drvdata(epc); + struct cdns_pcie *pcie = &ep->pcie; + u32 val, reg; + u8 cap; + + cap = cdns_pcie_find_capability(pcie, PCI_CAP_ID_MSIX); + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn); + + reg = cap + PCI_MSIX_FLAGS; + val = cdns_pcie_ep_fn_readw(pcie, fn, reg); + val &= ~PCI_MSIX_FLAGS_QSIZE; + val |= nr_irqs - 1; /* encoded as N-1 */ + cdns_pcie_ep_fn_writew(pcie, fn, reg, val); + + /* Set MSI-X BAR and offset */ + reg = cap + PCI_MSIX_TABLE; + val = offset | bir; + cdns_pcie_ep_fn_writel(pcie, fn, reg, val); + + /* Set PBA BAR and offset. BAR must match MSI-X BAR */ + reg = cap + PCI_MSIX_PBA; + val = (offset + (nr_irqs * PCI_MSIX_ENTRY_SIZE)) | bir; + cdns_pcie_ep_fn_writel(pcie, fn, reg, val); + + return 0; +} + +static void cdns_pcie_ep_assert_intx(struct cdns_pcie_ep *ep, u8 fn, u8 intx, + bool is_asserted) +{ + struct cdns_pcie *pcie = &ep->pcie; + unsigned long flags; + u32 offset; + u16 status; + u8 msg_code; + + intx &= 3; + + /* Set the outbound region if needed. */ + if (unlikely(ep->irq_pci_addr != CDNS_PCIE_EP_IRQ_PCI_ADDR_LEGACY || + ep->irq_pci_fn != fn)) { + /* First region was reserved for IRQ writes. */ + cdns_pcie_set_outbound_region_for_normal_msg(pcie, 0, fn, 0, + ep->irq_phys_addr); + ep->irq_pci_addr = CDNS_PCIE_EP_IRQ_PCI_ADDR_LEGACY; + ep->irq_pci_fn = fn; + } + + if (is_asserted) { + ep->irq_pending |= BIT(intx); + msg_code = PCIE_MSG_CODE_ASSERT_INTA + intx; + } else { + ep->irq_pending &= ~BIT(intx); + msg_code = PCIE_MSG_CODE_DEASSERT_INTA + intx; + } + + spin_lock_irqsave(&ep->lock, flags); + status = cdns_pcie_ep_fn_readw(pcie, fn, PCI_STATUS); + if (((status & PCI_STATUS_INTERRUPT) != 0) ^ (ep->irq_pending != 0)) { + status ^= PCI_STATUS_INTERRUPT; + cdns_pcie_ep_fn_writew(pcie, fn, PCI_STATUS, status); + } + spin_unlock_irqrestore(&ep->lock, flags); + + offset = CDNS_PCIE_NORMAL_MSG_ROUTING(PCIE_MSG_TYPE_R_LOCAL) | + CDNS_PCIE_NORMAL_MSG_CODE(msg_code); + writel(0, ep->irq_cpu_addr + offset); +} + +static int cdns_pcie_ep_send_intx_irq(struct cdns_pcie_ep *ep, u8 fn, u8 vfn, + u8 intx) +{ + u16 cmd; + + cmd = cdns_pcie_ep_fn_readw(&ep->pcie, fn, PCI_COMMAND); + if (cmd & PCI_COMMAND_INTX_DISABLE) + return -EINVAL; + + cdns_pcie_ep_assert_intx(ep, fn, intx, true); + /* + * The mdelay() value was taken from dra7xx_pcie_raise_intx_irq() + */ + mdelay(1); + cdns_pcie_ep_assert_intx(ep, fn, intx, false); + return 0; +} + +static int cdns_pcie_ep_send_msi_irq(struct cdns_pcie_ep *ep, u8 fn, u8 vfn, + u8 interrupt_num) +{ + struct cdns_pcie *pcie = &ep->pcie; + u16 flags, mme, data, data_mask; + u64 pci_addr, pci_addr_mask = 0xff; + u8 msi_count, cap; + + cap = cdns_pcie_find_capability(pcie, PCI_CAP_ID_MSI); + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn); + + /* Check whether the MSI feature has been enabled by the PCI host. */ + flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_FLAGS); + if (!(flags & PCI_MSI_FLAGS_ENABLE)) + return -EINVAL; + + /* Get the number of enabled MSIs */ + mme = FIELD_GET(PCI_MSI_FLAGS_QSIZE, flags); + msi_count = 1 << mme; + if (!interrupt_num || interrupt_num > msi_count) + return -EINVAL; + + /* Compute the data value to be written. */ + data_mask = msi_count - 1; + data = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_DATA_64); + data = (data & ~data_mask) | ((interrupt_num - 1) & data_mask); + + /* Get the PCI address where to write the data into. */ + pci_addr = cdns_pcie_ep_fn_readl(pcie, fn, cap + PCI_MSI_ADDRESS_HI); + pci_addr <<= 32; + pci_addr |= cdns_pcie_ep_fn_readl(pcie, fn, cap + PCI_MSI_ADDRESS_LO); + pci_addr &= GENMASK_ULL(63, 2); + + /* Set the outbound region if needed. */ + if (unlikely(ep->irq_pci_addr != (pci_addr & ~pci_addr_mask) || + ep->irq_pci_fn != fn)) { + /* First region was reserved for IRQ writes. */ + cdns_pcie_set_outbound_region(pcie, 0, fn, 0, + false, + ep->irq_phys_addr, + pci_addr & ~pci_addr_mask, + pci_addr_mask + 1); + ep->irq_pci_addr = (pci_addr & ~pci_addr_mask); + ep->irq_pci_fn = fn; + } + writel(data, ep->irq_cpu_addr + (pci_addr & pci_addr_mask)); + + return 0; +} + +static int cdns_pcie_ep_map_msi_irq(struct pci_epc *epc, u8 fn, u8 vfn, + phys_addr_t addr, u8 interrupt_num, + u32 entry_size, u32 *msi_data, + u32 *msi_addr_offset) +{ + struct cdns_pcie_ep *ep = epc_get_drvdata(epc); + struct cdns_pcie *pcie = &ep->pcie; + u64 pci_addr, pci_addr_mask = 0xff; + u16 flags, mme, data, data_mask; + u8 msi_count, cap; + int ret; + int i; + + cap = cdns_pcie_find_capability(pcie, PCI_CAP_ID_MSI); + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn); + + /* Check whether the MSI feature has been enabled by the PCI host. */ + flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_FLAGS); + if (!(flags & PCI_MSI_FLAGS_ENABLE)) + return -EINVAL; + + /* Get the number of enabled MSIs */ + mme = FIELD_GET(PCI_MSI_FLAGS_QSIZE, flags); + msi_count = 1 << mme; + if (!interrupt_num || interrupt_num > msi_count) + return -EINVAL; + + /* Compute the data value to be written. */ + data_mask = msi_count - 1; + data = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_DATA_64); + data = data & ~data_mask; + + /* Get the PCI address where to write the data into. */ + pci_addr = cdns_pcie_ep_fn_readl(pcie, fn, cap + PCI_MSI_ADDRESS_HI); + pci_addr <<= 32; + pci_addr |= cdns_pcie_ep_fn_readl(pcie, fn, cap + PCI_MSI_ADDRESS_LO); + pci_addr &= GENMASK_ULL(63, 2); + + for (i = 0; i < interrupt_num; i++) { + ret = cdns_pcie_ep_map_addr(epc, fn, vfn, addr, + pci_addr & ~pci_addr_mask, + entry_size); + if (ret) + return ret; + addr = addr + entry_size; + } + + *msi_data = data; + *msi_addr_offset = pci_addr & pci_addr_mask; + + return 0; +} + +static int cdns_pcie_ep_send_msix_irq(struct cdns_pcie_ep *ep, u8 fn, u8 vfn, + u16 interrupt_num) +{ + u32 tbl_offset, msg_data, reg; + struct cdns_pcie *pcie = &ep->pcie; + struct pci_epf_msix_tbl *msix_tbl; + struct cdns_pcie_epf *epf; + u64 pci_addr_mask = 0xff; + u64 msg_addr; + u8 bir, cap; + u16 flags; + + cap = cdns_pcie_find_capability(pcie, PCI_CAP_ID_MSIX); + epf = &ep->epf[fn]; + if (vfn > 0) + epf = &epf->epf[vfn - 1]; + + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn); + + /* Check whether the MSI-X feature has been enabled by the PCI host. */ + flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSIX_FLAGS); + if (!(flags & PCI_MSIX_FLAGS_ENABLE)) + return -EINVAL; + + reg = cap + PCI_MSIX_TABLE; + tbl_offset = cdns_pcie_ep_fn_readl(pcie, fn, reg); + bir = FIELD_GET(PCI_MSIX_TABLE_BIR, tbl_offset); + tbl_offset &= PCI_MSIX_TABLE_OFFSET; + + msix_tbl = epf->epf_bar[bir]->addr + tbl_offset; + msg_addr = msix_tbl[(interrupt_num - 1)].msg_addr; + msg_data = msix_tbl[(interrupt_num - 1)].msg_data; + + /* Set the outbound region if needed. */ + if (ep->irq_pci_addr != (msg_addr & ~pci_addr_mask) || + ep->irq_pci_fn != fn) { + /* First region was reserved for IRQ writes. */ + cdns_pcie_set_outbound_region(pcie, 0, fn, 0, + false, + ep->irq_phys_addr, + msg_addr & ~pci_addr_mask, + pci_addr_mask + 1); + ep->irq_pci_addr = (msg_addr & ~pci_addr_mask); + ep->irq_pci_fn = fn; + } + writel(msg_data, ep->irq_cpu_addr + (msg_addr & pci_addr_mask)); + + return 0; +} + +static int cdns_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn, u8 vfn, + unsigned int type, u16 interrupt_num) +{ + struct cdns_pcie_ep *ep = epc_get_drvdata(epc); + struct cdns_pcie *pcie = &ep->pcie; + struct device *dev = pcie->dev; + + switch (type) { + case PCI_IRQ_INTX: + if (vfn > 0) { + dev_err(dev, "Cannot raise INTX interrupts for VF\n"); + return -EINVAL; + } + return cdns_pcie_ep_send_intx_irq(ep, fn, vfn, 0); + + case PCI_IRQ_MSI: + return cdns_pcie_ep_send_msi_irq(ep, fn, vfn, interrupt_num); + + case PCI_IRQ_MSIX: + return cdns_pcie_ep_send_msix_irq(ep, fn, vfn, interrupt_num); + + default: + break; + } + + return -EINVAL; +} + +static int cdns_pcie_ep_start(struct pci_epc *epc) +{ + struct cdns_pcie_ep *ep = epc_get_drvdata(epc); + struct cdns_pcie *pcie = &ep->pcie; + struct device *dev = pcie->dev; + int max_epfs = sizeof(epc->function_num_map) * 8; + int ret, epf, last_fn; + u32 reg, value; + u8 cap; + + cap = cdns_pcie_find_capability(pcie, PCI_CAP_ID_EXP); + /* + * BIT(0) is hardwired to 1, hence function 0 is always enabled + * and can't be disabled anyway. + */ + cdns_pcie_writel(pcie, CDNS_PCIE_LM_EP_FUNC_CFG, epc->function_num_map); + + /* + * Next function field in ARI_CAP_AND_CTR register for last function + * should be 0. Clear Next Function Number field for the last + * function used. + */ + last_fn = find_last_bit(&epc->function_num_map, BITS_PER_LONG); + reg = CDNS_PCIE_CORE_PF_I_ARI_CAP_AND_CTRL(last_fn); + value = cdns_pcie_readl(pcie, reg); + value &= ~CDNS_PCIE_ARI_CAP_NFN_MASK; + cdns_pcie_writel(pcie, reg, value); + + if (ep->quirk_disable_flr) { + for (epf = 0; epf < max_epfs; epf++) { + if (!(epc->function_num_map & BIT(epf))) + continue; + + value = cdns_pcie_ep_fn_readl(pcie, epf, + cap + PCI_EXP_DEVCAP); + value &= ~PCI_EXP_DEVCAP_FLR; + cdns_pcie_ep_fn_writel(pcie, epf, + cap + PCI_EXP_DEVCAP, value); + } + } + + ret = cdns_pcie_start_link(pcie); + if (ret) { + dev_err(dev, "Failed to start link\n"); + return ret; + } + + return 0; +} + +static const struct pci_epc_features cdns_pcie_epc_vf_features = { + .msi_capable = true, + .msix_capable = true, + .align = 65536, +}; + +static const struct pci_epc_features cdns_pcie_epc_features = { + .msi_capable = true, + .msix_capable = true, + .align = 256, +}; + +static const struct pci_epc_features* +cdns_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no) +{ + if (!vfunc_no) + return &cdns_pcie_epc_features; + + return &cdns_pcie_epc_vf_features; +} + +static const struct pci_epc_ops cdns_pcie_epc_ops = { + .write_header = cdns_pcie_ep_write_header, + .set_bar = cdns_pcie_ep_set_bar, + .clear_bar = cdns_pcie_ep_clear_bar, + .map_addr = cdns_pcie_ep_map_addr, + .unmap_addr = cdns_pcie_ep_unmap_addr, + .set_msi = cdns_pcie_ep_set_msi, + .get_msi = cdns_pcie_ep_get_msi, + .set_msix = cdns_pcie_ep_set_msix, + .get_msix = cdns_pcie_ep_get_msix, + .raise_irq = cdns_pcie_ep_raise_irq, + .map_msi_irq = cdns_pcie_ep_map_msi_irq, + .start = cdns_pcie_ep_start, + .get_features = cdns_pcie_ep_get_features, +}; + +void cdns_pcie_ep_disable(struct cdns_pcie_ep *ep) +{ + struct device *dev = ep->pcie.dev; + struct pci_epc *epc = to_pci_epc(dev); + + cdns_pcie_debugfs_deinit(&ep->pcie); + pci_epc_deinit_notify(epc); + pci_epc_mem_free_addr(epc, ep->irq_phys_addr, ep->irq_cpu_addr, + SZ_128K); + pci_epc_mem_exit(epc); +} +EXPORT_SYMBOL_GPL(cdns_pcie_ep_disable); + +int cdns_pcie_ep_setup(struct cdns_pcie_ep *ep) +{ + struct device *dev = ep->pcie.dev; + struct platform_device *pdev = to_platform_device(dev); + struct device_node *np = dev->of_node; + struct cdns_pcie *pcie = &ep->pcie; + struct cdns_pcie_epf *epf; + struct resource *res; + struct pci_epc *epc; + int ret; + int i; + + pcie->is_rc = false; + + pcie->reg_base = devm_platform_ioremap_resource_byname(pdev, "reg"); + if (IS_ERR(pcie->reg_base)) { + dev_err(dev, "missing \"reg\"\n"); + return PTR_ERR(pcie->reg_base); + } + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mem"); + if (!res) { + dev_err(dev, "missing \"mem\"\n"); + return -EINVAL; + } + pcie->mem_res = res; + + ep->max_regions = CDNS_PCIE_MAX_OB; + of_property_read_u32(np, "cdns,max-outbound-regions", &ep->max_regions); + + ep->ob_addr = devm_kcalloc(dev, + ep->max_regions, sizeof(*ep->ob_addr), + GFP_KERNEL); + if (!ep->ob_addr) + return -ENOMEM; + + /* Disable all but function 0 (anyway BIT(0) is hardwired to 1). */ + cdns_pcie_writel(pcie, CDNS_PCIE_LM_EP_FUNC_CFG, BIT(0)); + + epc = devm_pci_epc_create(dev, &cdns_pcie_epc_ops); + if (IS_ERR(epc)) { + dev_err(dev, "failed to create epc device\n"); + return PTR_ERR(epc); + } + + epc_set_drvdata(epc, ep); + + if (of_property_read_u8(np, "max-functions", &epc->max_functions) < 0) + epc->max_functions = 1; + + ep->epf = devm_kcalloc(dev, epc->max_functions, sizeof(*ep->epf), + GFP_KERNEL); + if (!ep->epf) + return -ENOMEM; + + epc->max_vfs = devm_kcalloc(dev, epc->max_functions, + sizeof(*epc->max_vfs), GFP_KERNEL); + if (!epc->max_vfs) + return -ENOMEM; + + ret = of_property_read_u8_array(np, "max-virtual-functions", + epc->max_vfs, epc->max_functions); + if (ret == 0) { + for (i = 0; i < epc->max_functions; i++) { + epf = &ep->epf[i]; + if (epc->max_vfs[i] == 0) + continue; + epf->epf = devm_kcalloc(dev, epc->max_vfs[i], + sizeof(*ep->epf), GFP_KERNEL); + if (!epf->epf) + return -ENOMEM; + } + } + + ret = pci_epc_mem_init(epc, pcie->mem_res->start, + resource_size(pcie->mem_res), PAGE_SIZE); + if (ret < 0) { + dev_err(dev, "failed to initialize the memory space\n"); + return ret; + } + + ep->irq_cpu_addr = pci_epc_mem_alloc_addr(epc, &ep->irq_phys_addr, + SZ_128K); + if (!ep->irq_cpu_addr) { + dev_err(dev, "failed to reserve memory space for MSI\n"); + ret = -ENOMEM; + goto free_epc_mem; + } + ep->irq_pci_addr = CDNS_PCIE_EP_IRQ_PCI_ADDR_NONE; + /* Reserve region 0 for IRQs */ + set_bit(0, &ep->ob_region_map); + + if (ep->quirk_detect_quiet_flag) + cdns_pcie_detect_quiet_min_delay_set(&ep->pcie); + + spin_lock_init(&ep->lock); + + pci_epc_init_notify(epc); + + cdns_pcie_debugfs_init(pcie); + + return 0; + + free_epc_mem: + pci_epc_mem_exit(epc); + + return ret; +} +EXPORT_SYMBOL_GPL(cdns_pcie_ep_setup); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Cadence PCIe endpoint controller driver"); +MODULE_AUTHOR("Cyrille Pitchen <cyrille.pitchen@free-electrons.com>"); diff --git a/drivers/pci/controller/cadence/pcie-cadence-host-common.c b/drivers/pci/controller/cadence/pcie-cadence-host-common.c new file mode 100644 index 000000000..0ef439615 --- /dev/null +++ b/drivers/pci/controller/cadence/pcie-cadence-host-common.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Cadence PCIe host controller library. + * + * Copyright (c) 2017 Cadence + * Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com> + */ +#include <linux/delay.h> +#include <linux/kernel.h> +#include <linux/list_sort.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/platform_device.h> + +#include "pcie-cadence.h" +#include "pcie-cadence-host-common.h" +#include "../pci-host-common.h" + +#include "../../pci.h" + +#define LINK_RETRAIN_TIMEOUT HZ + +u64 bar_max_size[] = { + [RP_BAR0] = _ULL(128 * SZ_2G), + [RP_BAR1] = SZ_2G, + [RP_NO_BAR] = _BITULL(63), +}; +EXPORT_SYMBOL_GPL(bar_max_size); + +int cdns_pcie_host_training_complete(struct cdns_pcie *pcie) +{ + u32 pcie_cap_off = CDNS_PCIE_RP_CAP_OFFSET; + unsigned long end_jiffies; + u16 lnk_stat; + + /* Wait for link training to complete. Exit after timeout. */ + end_jiffies = jiffies + LINK_RETRAIN_TIMEOUT; + do { + lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA); + if (!(lnk_stat & PCI_EXP_LNKSTA_LT)) + break; + usleep_range(0, 1000); + } while (time_before(jiffies, end_jiffies)); + + if (!(lnk_stat & PCI_EXP_LNKSTA_LT)) + return 0; + + return -ETIMEDOUT; +} +EXPORT_SYMBOL_GPL(cdns_pcie_host_training_complete); + +int cdns_pcie_host_wait_for_link(struct cdns_pcie *pcie, + cdns_pcie_linkup_func pcie_link_up) +{ + struct device *dev = pcie->dev; + int retries; + + /* Check if the link is up or not */ + for (retries = 0; retries < PCIE_LINK_WAIT_MAX_RETRIES; retries++) { + if (pcie_link_up(pcie)) { + dev_info(dev, "Link up\n"); + return 0; + } + msleep(PCIE_LINK_WAIT_SLEEP_MS); + } + + return -ETIMEDOUT; +} +EXPORT_SYMBOL_GPL(cdns_pcie_host_wait_for_link); + +int cdns_pcie_retrain(struct cdns_pcie *pcie, + cdns_pcie_linkup_func pcie_link_up) +{ + u32 lnk_cap_sls, pcie_cap_off = CDNS_PCIE_RP_CAP_OFFSET; + u16 lnk_stat, lnk_ctl; + int ret = 0; + + /* + * Set retrain bit if current speed is 2.5 GB/s, + * but the PCIe root port support is > 2.5 GB/s. + */ + + lnk_cap_sls = cdns_pcie_readl(pcie, (CDNS_PCIE_RP_BASE + pcie_cap_off + + PCI_EXP_LNKCAP)); + if ((lnk_cap_sls & PCI_EXP_LNKCAP_SLS) <= PCI_EXP_LNKCAP_SLS_2_5GB) + return ret; + + lnk_stat = cdns_pcie_rp_readw(pcie, pcie_cap_off + PCI_EXP_LNKSTA); + if ((lnk_stat & PCI_EXP_LNKSTA_CLS) == PCI_EXP_LNKSTA_CLS_2_5GB) { + lnk_ctl = cdns_pcie_rp_readw(pcie, + pcie_cap_off + PCI_EXP_LNKCTL); + lnk_ctl |= PCI_EXP_LNKCTL_RL; + cdns_pcie_rp_writew(pcie, pcie_cap_off + PCI_EXP_LNKCTL, + lnk_ctl); + + ret = cdns_pcie_host_training_complete(pcie); + if (ret) + return ret; + + ret = cdns_pcie_host_wait_for_link(pcie, pcie_link_up); + } + return ret; +} +EXPORT_SYMBOL_GPL(cdns_pcie_retrain); + +int cdns_pcie_host_start_link(struct cdns_pcie_rc *rc, + cdns_pcie_linkup_func pcie_link_up) +{ + struct cdns_pcie *pcie = &rc->pcie; + int ret; + + ret = cdns_pcie_host_wait_for_link(pcie, pcie_link_up); + + /* + * Retrain link for Gen2 training defect + * if quirk flag is set. + */ + if (!ret && rc->quirk_retrain_flag) + ret = cdns_pcie_retrain(pcie, pcie_link_up); + + if (!ret) + pci_host_common_link_train_delay(pcie->max_link_speed); + + return ret; +} +EXPORT_SYMBOL_GPL(cdns_pcie_host_start_link); + +enum cdns_pcie_rp_bar +cdns_pcie_host_find_min_bar(struct cdns_pcie_rc *rc, u64 size) +{ + enum cdns_pcie_rp_bar bar, sel_bar; + + sel_bar = RP_BAR_UNDEFINED; + for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) { + if (!rc->avail_ib_bar[bar]) + continue; + + if (size <= bar_max_size[bar]) { + if (sel_bar == RP_BAR_UNDEFINED) { + sel_bar = bar; + continue; + } + + if (bar_max_size[bar] < bar_max_size[sel_bar]) + sel_bar = bar; + } + } + + return sel_bar; +} +EXPORT_SYMBOL_GPL(cdns_pcie_host_find_min_bar); + +enum cdns_pcie_rp_bar +cdns_pcie_host_find_max_bar(struct cdns_pcie_rc *rc, u64 size) +{ + enum cdns_pcie_rp_bar bar, sel_bar; + + sel_bar = RP_BAR_UNDEFINED; + for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) { + if (!rc->avail_ib_bar[bar]) + continue; + + if (size >= bar_max_size[bar]) { + if (sel_bar == RP_BAR_UNDEFINED) { + sel_bar = bar; + continue; + } + + if (bar_max_size[bar] > bar_max_size[sel_bar]) + sel_bar = bar; + } + } + + return sel_bar; +} +EXPORT_SYMBOL_GPL(cdns_pcie_host_find_max_bar); + +int cdns_pcie_host_dma_ranges_cmp(void *priv, const struct list_head *a, + const struct list_head *b) +{ + struct resource_entry *entry1, *entry2; + u64 size1, size2; + + entry1 = container_of(a, struct resource_entry, node); + entry2 = container_of(b, struct resource_entry, node); + + size1 = resource_size(entry1->res); + size2 = resource_size(entry2->res); + + if (size1 > size2) + return -1; + + if (size1 < size2) + return 1; + + return 0; +} +EXPORT_SYMBOL_GPL(cdns_pcie_host_dma_ranges_cmp); + +int cdns_pcie_host_bar_config(struct cdns_pcie_rc *rc, + struct resource_entry *entry, + cdns_pcie_host_bar_ib_cfg pci_host_ib_config) +{ + struct cdns_pcie *pcie = &rc->pcie; + struct device *dev = pcie->dev; + u64 cpu_addr, size, winsize; + enum cdns_pcie_rp_bar bar; + unsigned long flags; + int ret; + + cpu_addr = entry->res->start; + flags = entry->res->flags; + size = resource_size(entry->res); + + while (size > 0) { + /* + * Try to find a minimum BAR whose size is greater than + * or equal to the remaining resource_entry size. This will + * fail if the size of each of the available BARs is less than + * the remaining resource_entry size. + * + * If a minimum BAR is found, IB ATU will be configured and + * exited. + */ + bar = cdns_pcie_host_find_min_bar(rc, size); + if (bar != RP_BAR_UNDEFINED) { + ret = pci_host_ib_config(rc, bar, cpu_addr, size, flags); + if (ret) + dev_err(dev, "IB BAR: %d config failed\n", bar); + return ret; + } + + /* + * If the control reaches here, it would mean the remaining + * resource_entry size cannot be fitted in a single BAR. So we + * find a maximum BAR whose size is less than or equal to the + * remaining resource_entry size and split the resource entry + * so that part of resource entry is fitted inside the maximum + * BAR. The remaining size would be fitted during the next + * iteration of the loop. + * + * If a maximum BAR is not found, there is no way we can fit + * this resource_entry, so we error out. + */ + bar = cdns_pcie_host_find_max_bar(rc, size); + if (bar == RP_BAR_UNDEFINED) { + dev_err(dev, "No free BAR to map cpu_addr %llx\n", + cpu_addr); + return -EINVAL; + } + + winsize = bar_max_size[bar]; + ret = pci_host_ib_config(rc, bar, cpu_addr, winsize, flags); + if (ret) { + dev_err(dev, "IB BAR: %d config failed\n", bar); + return ret; + } + + size -= winsize; + cpu_addr += winsize; + } + + return 0; +} + +int cdns_pcie_host_map_dma_ranges(struct cdns_pcie_rc *rc, + cdns_pcie_host_bar_ib_cfg pci_host_ib_config) +{ + struct cdns_pcie *pcie = &rc->pcie; + struct device *dev = pcie->dev; + struct device_node *np = dev->of_node; + struct pci_host_bridge *bridge; + struct resource_entry *entry; + u32 no_bar_nbits = 32; + int err; + + bridge = pci_host_bridge_from_priv(rc); + if (!bridge) + return -ENOMEM; + + if (list_empty(&bridge->dma_ranges)) { + of_property_read_u32(np, "cdns,no-bar-match-nbits", + &no_bar_nbits); + err = pci_host_ib_config(rc, RP_NO_BAR, 0x0, (u64)1 << no_bar_nbits, 0); + if (err) + dev_err(dev, "IB BAR: %d config failed\n", RP_NO_BAR); + return err; + } + + list_sort(NULL, &bridge->dma_ranges, cdns_pcie_host_dma_ranges_cmp); + + resource_list_for_each_entry(entry, &bridge->dma_ranges) { + err = cdns_pcie_host_bar_config(rc, entry, pci_host_ib_config); + if (err) { + dev_err(dev, "Fail to configure IB using dma-ranges\n"); + return err; + } + } + + return 0; +} + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Cadence PCIe host controller driver"); diff --git a/drivers/pci/controller/cadence/pcie-cadence-host-common.h b/drivers/pci/controller/cadence/pcie-cadence-host-common.h new file mode 100644 index 000000000..fe7d4202a --- /dev/null +++ b/drivers/pci/controller/cadence/pcie-cadence-host-common.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Cadence PCIe Host controller driver. + * + * Copyright (c) 2017 Cadence + * Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com> + */ +#ifndef _PCIE_CADENCE_HOST_COMMON_H +#define _PCIE_CADENCE_HOST_COMMON_H + +#include <linux/kernel.h> +#include <linux/pci.h> + +extern u64 bar_max_size[]; + +typedef int (*cdns_pcie_host_bar_ib_cfg)(struct cdns_pcie_rc *, + enum cdns_pcie_rp_bar, + u64, + u64, + unsigned long); +typedef bool (*cdns_pcie_linkup_func)(struct cdns_pcie *); + +int cdns_pcie_host_training_complete(struct cdns_pcie *pcie); +int cdns_pcie_host_wait_for_link(struct cdns_pcie *pcie, + cdns_pcie_linkup_func pcie_link_up); +int cdns_pcie_retrain(struct cdns_pcie *pcie, cdns_pcie_linkup_func pcie_linkup_func); +int cdns_pcie_host_start_link(struct cdns_pcie_rc *rc, + cdns_pcie_linkup_func pcie_link_up); +enum cdns_pcie_rp_bar +cdns_pcie_host_find_min_bar(struct cdns_pcie_rc *rc, u64 size); +enum cdns_pcie_rp_bar +cdns_pcie_host_find_max_bar(struct cdns_pcie_rc *rc, u64 size); +int cdns_pcie_host_dma_ranges_cmp(void *priv, const struct list_head *a, + const struct list_head *b); +int cdns_pcie_host_bar_ib_config(struct cdns_pcie_rc *rc, + enum cdns_pcie_rp_bar bar, + u64 cpu_addr, + u64 size, + unsigned long flags); +int cdns_pcie_host_bar_config(struct cdns_pcie_rc *rc, + struct resource_entry *entry, + cdns_pcie_host_bar_ib_cfg pci_host_ib_config); +int cdns_pcie_host_map_dma_ranges(struct cdns_pcie_rc *rc, + cdns_pcie_host_bar_ib_cfg pci_host_ib_config); + +#endif /* _PCIE_CADENCE_HOST_COMMON_H */ diff --git a/drivers/pci/controller/cadence/pcie-cadence-host-hpa.c b/drivers/pci/controller/cadence/pcie-cadence-host-hpa.c new file mode 100644 index 000000000..54d7221f7 --- /dev/null +++ b/drivers/pci/controller/cadence/pcie-cadence-host-hpa.c @@ -0,0 +1,395 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Cadence PCIe host controller driver. + * + * Copyright (c) 2024, Cadence Design Systems + * Author: Manikandan K Pillai <mpillai@cadence.com> + */ +#include <linux/delay.h> +#include <linux/kernel.h> +#include <linux/list_sort.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/of_irq.h> +#include <linux/platform_device.h> + +#include "pcie-cadence.h" +#include "pcie-cadence-host-common.h" +#include "../pci-host-common.h" +#include "../../pci.h" + +static u8 bar_aperture_mask[] = { + [RP_BAR0] = 0x3F, + [RP_BAR1] = 0x3F, +}; + +void __iomem *cdns_pci_hpa_map_bus(struct pci_bus *bus, unsigned int devfn, + int where) +{ + struct pci_host_bridge *bridge = pci_find_host_bridge(bus); + struct cdns_pcie_rc *rc = pci_host_bridge_priv(bridge); + struct cdns_pcie *pcie = &rc->pcie; + unsigned int busn = bus->number; + u32 addr0, desc0, desc1, ctrl0; + u32 regval; + + if (pci_is_root_bus(bus)) { + /* + * Only the root port (devfn == 0) is connected to this bus. + * All other PCI devices are behind some bridge hence on another + * bus. + */ + if (devfn) + return NULL; + + return pcie->reg_base + (where & 0xfff); + } + + /* Clear AXI link-down status */ + regval = cdns_pcie_hpa_readl(pcie, REG_BANK_AXI_SLAVE, CDNS_PCIE_HPA_AT_LINKDOWN); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, CDNS_PCIE_HPA_AT_LINKDOWN, + (regval & ~GENMASK(0, 0))); + + /* Update Output registers for AXI region 0 */ + addr0 = CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR0_NBITS(12) | + CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR0_DEVFN(devfn) | + CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR0_BUS(busn); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR0(0), addr0); + + desc1 = cdns_pcie_hpa_readl(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_DESC1(0)); + desc1 &= ~CDNS_PCIE_HPA_AT_OB_REGION_DESC1_DEVFN_MASK; + desc1 |= CDNS_PCIE_HPA_AT_OB_REGION_DESC1_DEVFN(0); + ctrl0 = CDNS_PCIE_HPA_AT_OB_REGION_CTRL0_SUPPLY_BUS | + CDNS_PCIE_HPA_AT_OB_REGION_CTRL0_SUPPLY_DEV_FN; + + if (busn == bridge->busnr + 1) + desc0 = CDNS_PCIE_HPA_AT_OB_REGION_DESC0_TYPE_CONF_TYPE0; + else + desc0 = CDNS_PCIE_HPA_AT_OB_REGION_DESC0_TYPE_CONF_TYPE1; + + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_DESC0(0), desc0); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_DESC1(0), desc1); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_CTRL0(0), ctrl0); + + return rc->cfg_base + (where & 0xfff); +} + +static struct pci_ops cdns_pcie_hpa_host_ops = { + .map_bus = cdns_pci_hpa_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; + +static void cdns_pcie_hpa_host_enable_ptm_response(struct cdns_pcie *pcie) +{ + u32 val; + + val = cdns_pcie_hpa_readl(pcie, REG_BANK_IP_REG, CDNS_PCIE_HPA_LM_PTM_CTRL); + cdns_pcie_hpa_writel(pcie, REG_BANK_IP_REG, CDNS_PCIE_HPA_LM_PTM_CTRL, + val | CDNS_PCIE_HPA_LM_PTM_CTRL_PTMRSEN); +} + +static int cdns_pcie_hpa_host_bar_ib_config(struct cdns_pcie_rc *rc, + enum cdns_pcie_rp_bar bar, + u64 cpu_addr, u64 size, + unsigned long flags) +{ + struct cdns_pcie *pcie = &rc->pcie; + u32 addr0, addr1, aperture, value; + + if (!rc->avail_ib_bar[bar]) + return -ENODEV; + + rc->avail_ib_bar[bar] = false; + + aperture = ilog2(size); + if (bar == RP_NO_BAR) { + addr0 = CDNS_PCIE_HPA_AT_IB_RP_BAR_ADDR0_NBITS(aperture) | + (lower_32_bits(cpu_addr) & GENMASK(31, 8)); + addr1 = upper_32_bits(cpu_addr); + } else { + addr0 = lower_32_bits(cpu_addr); + addr1 = upper_32_bits(cpu_addr); + } + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_MASTER, + CDNS_PCIE_HPA_AT_IB_RP_BAR_ADDR0(bar), addr0); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_MASTER, + CDNS_PCIE_HPA_AT_IB_RP_BAR_ADDR1(bar), addr1); + + if (bar == RP_NO_BAR) + bar = (enum cdns_pcie_rp_bar)BAR_0; + + value = cdns_pcie_hpa_readl(pcie, REG_BANK_IP_CFG_CTRL_REG, CDNS_PCIE_HPA_LM_RC_BAR_CFG); + value &= ~(HPA_LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar) | + HPA_LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar) | + HPA_LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar) | + HPA_LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar) | + HPA_LM_RC_BAR_CFG_APERTURE(bar, bar_aperture_mask[bar] + 7)); + if (size + cpu_addr >= SZ_4G) { + value |= HPA_LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar); + if ((flags & IORESOURCE_PREFETCH)) + value |= HPA_LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar); + } else { + value |= HPA_LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar); + if ((flags & IORESOURCE_PREFETCH)) + value |= HPA_LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar); + } + + value |= HPA_LM_RC_BAR_CFG_APERTURE(bar, aperture); + cdns_pcie_hpa_writel(pcie, REG_BANK_IP_CFG_CTRL_REG, CDNS_PCIE_HPA_LM_RC_BAR_CFG, value); + + return 0; +} + +static int cdns_pcie_hpa_host_init_root_port(struct cdns_pcie_rc *rc) +{ + struct cdns_pcie *pcie = &rc->pcie; + u32 value, ctrl; + + /* + * Set the root port BAR configuration register: + * - disable both BAR0 and BAR1 + * - enable Prefetchable Memory Base and Limit registers in type 1 + * config space (64 bits) + * - enable IO Base and Limit registers in type 1 config + * space (32 bits) + */ + + ctrl = CDNS_PCIE_HPA_LM_BAR_CFG_CTRL_DISABLED; + value = CDNS_PCIE_HPA_LM_RC_BAR_CFG_BAR0_CTRL(ctrl) | + CDNS_PCIE_HPA_LM_RC_BAR_CFG_BAR1_CTRL(ctrl) | + CDNS_PCIE_HPA_LM_RC_BAR_CFG_PREFETCH_MEM_ENABLE | + CDNS_PCIE_HPA_LM_RC_BAR_CFG_PREFETCH_MEM_64BITS | + CDNS_PCIE_HPA_LM_RC_BAR_CFG_IO_ENABLE | + CDNS_PCIE_HPA_LM_RC_BAR_CFG_IO_32BITS; + cdns_pcie_hpa_writel(pcie, REG_BANK_IP_CFG_CTRL_REG, + CDNS_PCIE_HPA_LM_RC_BAR_CFG, value); + + if (rc->vendor_id != 0xffff) + cdns_pcie_hpa_rp_writew(pcie, PCI_VENDOR_ID, rc->vendor_id); + + if (rc->device_id != 0xffff) + cdns_pcie_hpa_rp_writew(pcie, PCI_DEVICE_ID, rc->device_id); + + cdns_pcie_hpa_rp_writeb(pcie, PCI_CLASS_REVISION, 0); + cdns_pcie_hpa_rp_writeb(pcie, PCI_CLASS_PROG, 0); + cdns_pcie_hpa_rp_writew(pcie, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI); + + /* Enable bus mastering */ + value = cdns_pcie_hpa_readl(pcie, REG_BANK_RP, PCI_COMMAND); + value |= (PCI_COMMAND_MEMORY | PCI_COMMAND_IO | PCI_COMMAND_MASTER); + cdns_pcie_hpa_writel(pcie, REG_BANK_RP, PCI_COMMAND, value); + return 0; +} + +static void cdns_pcie_hpa_create_region_for_cfg(struct cdns_pcie_rc *rc) +{ + struct cdns_pcie *pcie = &rc->pcie; + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rc); + struct resource *cfg_res = rc->cfg_res; + struct resource_entry *entry; + u64 cpu_addr = cfg_res->start; + u32 addr0, addr1, desc1; + int busnr = 0; + + entry = resource_list_first_type(&bridge->windows, IORESOURCE_BUS); + if (entry) + busnr = entry->res->start; + + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_TAG_MANAGEMENT, 0x01000000); + /* + * Reserve region 0 for PCI configure space accesses: + * OB_REGION_PCI_ADDR0 and OB_REGION_DESC0 are updated dynamically by + * cdns_pci_map_bus(), other region registers are set here once for all + */ + desc1 = CDNS_PCIE_HPA_AT_OB_REGION_DESC1_BUS(busnr); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR1(0), 0x0); + /* Type-1 CFG */ + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_DESC0(0), 0x05000000); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_DESC1(0), desc1); + + addr0 = CDNS_PCIE_HPA_AT_OB_REGION_CPU_ADDR0_NBITS(12) | + (lower_32_bits(cpu_addr) & GENMASK(31, 8)); + addr1 = upper_32_bits(cpu_addr); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_CPU_ADDR0(0), addr0); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_CPU_ADDR1(0), addr1); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_CTRL0(0), 0x06000000); +} + +static int cdns_pcie_hpa_host_init_address_translation(struct cdns_pcie_rc *rc) +{ + struct cdns_pcie *pcie = &rc->pcie; + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rc); + struct resource_entry *entry; + int r = 0, busnr = 0; + + if (!rc->ecam_supported) + cdns_pcie_hpa_create_region_for_cfg(rc); + + entry = resource_list_first_type(&bridge->windows, IORESOURCE_BUS); + if (entry) + busnr = entry->res->start; + + r++; + if (pcie->msg_res) { + cdns_pcie_hpa_set_outbound_region_for_normal_msg(pcie, busnr, 0, r, + pcie->msg_res->start); + + r++; + } + resource_list_for_each_entry(entry, &bridge->windows) { + struct resource *res = entry->res; + u64 pci_addr = res->start - entry->offset; + + if (resource_type(res) == IORESOURCE_IO) + cdns_pcie_hpa_set_outbound_region(pcie, busnr, 0, r, + true, + pci_pio_to_address(res->start), + pci_addr, + resource_size(res)); + else + cdns_pcie_hpa_set_outbound_region(pcie, busnr, 0, r, + false, + res->start, + pci_addr, + resource_size(res)); + + r++; + } + + if (rc->no_inbound_map) + return 0; + else + return cdns_pcie_host_map_dma_ranges(rc, cdns_pcie_hpa_host_bar_ib_config); +} + +static int cdns_pcie_hpa_host_init(struct cdns_pcie_rc *rc) +{ + int err; + + err = cdns_pcie_hpa_host_init_root_port(rc); + if (err) + return err; + + return cdns_pcie_hpa_host_init_address_translation(rc); +} + +int cdns_pcie_hpa_host_link_setup(struct cdns_pcie_rc *rc) +{ + struct cdns_pcie *pcie = &rc->pcie; + struct device *dev = rc->pcie.dev; + int ret; + + if (rc->quirk_detect_quiet_flag) + cdns_pcie_hpa_detect_quiet_min_delay_set(&rc->pcie); + + cdns_pcie_hpa_host_enable_ptm_response(pcie); + + ret = cdns_pcie_start_link(pcie); + if (ret) { + dev_err(dev, "Failed to start link\n"); + return ret; + } + + ret = cdns_pcie_host_wait_for_link(pcie, cdns_pcie_hpa_link_up); + if (ret) + dev_dbg(dev, "PCIe link never came up\n"); + else + pci_host_common_link_train_delay(pcie->max_link_speed); + + return ret; +} +EXPORT_SYMBOL_GPL(cdns_pcie_hpa_host_link_setup); + +void cdns_pcie_hpa_host_disable(struct cdns_pcie_rc *rc) +{ + struct pci_host_bridge *bridge; + + cdns_pcie_debugfs_deinit(&rc->pcie); + bridge = pci_host_bridge_from_priv(rc); + pci_lock_rescan_remove(); + pci_stop_root_bus(bridge->bus); + pci_remove_root_bus(bridge->bus); + pci_unlock_rescan_remove(); +} +EXPORT_SYMBOL_GPL(cdns_pcie_hpa_host_disable); + +int cdns_pcie_hpa_host_setup(struct cdns_pcie_rc *rc) +{ + struct device *dev = rc->pcie.dev; + struct platform_device *pdev = to_platform_device(dev); + struct device_node *np = dev->of_node; + struct pci_host_bridge *bridge; + enum cdns_pcie_rp_bar bar; + struct cdns_pcie *pcie; + struct resource *res; + int ret; + + bridge = pci_host_bridge_from_priv(rc); + if (!bridge) + return -ENOMEM; + + pcie = &rc->pcie; + pcie->is_rc = true; + + if (!pcie->reg_base) { + pcie->reg_base = devm_platform_ioremap_resource_byname(pdev, "reg"); + if (IS_ERR(pcie->reg_base)) { + dev_err(dev, "missing \"reg\"\n"); + return PTR_ERR(pcie->reg_base); + } + } + + /* ECAM config space is remapped at glue layer */ + if (!rc->cfg_base) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg"); + rc->cfg_base = devm_pci_remap_cfg_resource(dev, res); + if (IS_ERR(rc->cfg_base)) + return PTR_ERR(rc->cfg_base); + rc->cfg_res = res; + } + + if (pcie->max_link_speed < 1) + pcie->max_link_speed = of_pci_get_max_link_speed(np); + + /* Put EROM Bar aperture to 0 */ + cdns_pcie_hpa_writel(pcie, REG_BANK_IP_CFG_CTRL_REG, CDNS_PCIE_EROM, 0x0); + + ret = cdns_pcie_hpa_host_link_setup(rc); + if (ret) + return ret; + + for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) + rc->avail_ib_bar[bar] = true; + + ret = cdns_pcie_hpa_host_init(rc); + if (ret) + return ret; + + if (!bridge->ops) + bridge->ops = &cdns_pcie_hpa_host_ops; + + ret = pci_host_probe(bridge); + if (ret) + return ret; + + cdns_pcie_debugfs_init(pcie); + + return 0; +} +EXPORT_SYMBOL_GPL(cdns_pcie_hpa_host_setup); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Cadence PCIe host controller driver"); diff --git a/drivers/pci/controller/cadence/pcie-cadence-host.c b/drivers/pci/controller/cadence/pcie-cadence-host.c new file mode 100644 index 000000000..15aabaac3 --- /dev/null +++ b/drivers/pci/controller/cadence/pcie-cadence-host.c @@ -0,0 +1,445 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2017 Cadence +// Cadence PCIe host controller driver. +// Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com> + +#include <linux/delay.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/list_sort.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/platform_device.h> + +#include "pcie-cadence.h" +#include "pcie-cadence-host-common.h" +#include "../../pci.h" + +static u8 bar_aperture_mask[] = { + [RP_BAR0] = 0x1F, + [RP_BAR1] = 0xF, +}; + +void __iomem *cdns_pci_map_bus(struct pci_bus *bus, unsigned int devfn, + int where) +{ + struct pci_host_bridge *bridge = pci_find_host_bridge(bus); + struct cdns_pcie_rc *rc = pci_host_bridge_priv(bridge); + struct cdns_pcie *pcie = &rc->pcie; + unsigned int busn = bus->number; + u32 addr0, desc0; + + if (pci_is_root_bus(bus)) { + /* + * Only the root port (devfn == 0) is connected to this bus. + * All other PCI devices are behind some bridge hence on another + * bus. + */ + if (devfn) + return NULL; + + return pcie->reg_base + (where & 0xfff); + } + /* Check that the link is up */ + if (!(cdns_pcie_readl(pcie, CDNS_PCIE_LM_BASE) & 0x1)) + return NULL; + /* Clear AXI link-down status */ + cdns_pcie_writel(pcie, CDNS_PCIE_AT_LINKDOWN, 0x0); + + /* Update Output registers for AXI region 0. */ + addr0 = CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_NBITS(12) | + CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN(devfn) | + CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_BUS(busn); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR0(0), addr0); + + /* Configuration Type 0 or Type 1 access. */ + desc0 = CDNS_PCIE_AT_OB_REGION_DESC0_HARDCODED_RID | + CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN(0); + /* + * The bus number was already set once for all in desc1 by + * cdns_pcie_host_init_address_translation(). + */ + if (busn == bridge->busnr + 1) + desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE0; + else + desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE1; + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC0(0), desc0); + + return rc->cfg_base + (where & 0xfff); +} +EXPORT_SYMBOL_GPL(cdns_pci_map_bus); + +static struct pci_ops cdns_pcie_host_ops = { + .map_bus = cdns_pci_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; + +static void cdns_pcie_host_disable_ptm_response(struct cdns_pcie *pcie) +{ + u32 val; + + val = cdns_pcie_readl(pcie, CDNS_PCIE_LM_PTM_CTRL); + cdns_pcie_writel(pcie, CDNS_PCIE_LM_PTM_CTRL, val & ~CDNS_PCIE_LM_TPM_CTRL_PTMRSEN); +} + +static void cdns_pcie_host_enable_ptm_response(struct cdns_pcie *pcie) +{ + u32 val; + + val = cdns_pcie_readl(pcie, CDNS_PCIE_LM_PTM_CTRL); + cdns_pcie_writel(pcie, CDNS_PCIE_LM_PTM_CTRL, val | CDNS_PCIE_LM_TPM_CTRL_PTMRSEN); +} + +static void cdns_pcie_host_deinit_root_port(struct cdns_pcie_rc *rc) +{ + struct cdns_pcie *pcie = &rc->pcie; + u32 value, ctrl; + + cdns_pcie_rp_writew(pcie, PCI_CLASS_DEVICE, 0xffff); + cdns_pcie_rp_writeb(pcie, PCI_CLASS_PROG, 0xff); + cdns_pcie_rp_writeb(pcie, PCI_CLASS_REVISION, 0xff); + cdns_pcie_writel(pcie, CDNS_PCIE_LM_ID, 0xffffffff); + cdns_pcie_rp_writew(pcie, PCI_DEVICE_ID, 0xffff); + ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED; + value = ~(CDNS_PCIE_LM_RC_BAR_CFG_BAR0_CTRL(ctrl) | + CDNS_PCIE_LM_RC_BAR_CFG_BAR1_CTRL(ctrl) | + CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_ENABLE | + CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_64BITS | + CDNS_PCIE_LM_RC_BAR_CFG_IO_ENABLE | + CDNS_PCIE_LM_RC_BAR_CFG_IO_32BITS); + cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value); +} + +static int cdns_pcie_host_init_root_port(struct cdns_pcie_rc *rc) +{ + struct cdns_pcie *pcie = &rc->pcie; + u32 value, ctrl; + u32 id; + + /* + * Set the root complex BAR configuration register: + * - disable both BAR0 and BAR1. + * - enable Prefetchable Memory Base and Limit registers in type 1 + * config space (64 bits). + * - enable IO Base and Limit registers in type 1 config + * space (32 bits). + */ + ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED; + value = CDNS_PCIE_LM_RC_BAR_CFG_BAR0_CTRL(ctrl) | + CDNS_PCIE_LM_RC_BAR_CFG_BAR1_CTRL(ctrl) | + CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_ENABLE | + CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_64BITS | + CDNS_PCIE_LM_RC_BAR_CFG_IO_ENABLE | + CDNS_PCIE_LM_RC_BAR_CFG_IO_32BITS; + cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value); + + /* Set root port configuration space */ + if (rc->vendor_id != 0xffff) { + id = CDNS_PCIE_LM_ID_VENDOR(rc->vendor_id) | + CDNS_PCIE_LM_ID_SUBSYS(rc->vendor_id); + cdns_pcie_writel(pcie, CDNS_PCIE_LM_ID, id); + } + + if (rc->device_id != 0xffff) + cdns_pcie_rp_writew(pcie, PCI_DEVICE_ID, rc->device_id); + + cdns_pcie_rp_writeb(pcie, PCI_CLASS_REVISION, 0); + cdns_pcie_rp_writeb(pcie, PCI_CLASS_PROG, 0); + cdns_pcie_rp_writew(pcie, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI); + + value = cdns_pcie_rp_readl(pcie, CDNS_PCIE_RP_CAP_OFFSET + PCI_EXP_LNKCAP); + if (rc->quirk_broken_aspm_l0s) + value &= ~PCI_EXP_LNKCAP_ASPM_L0S; + if (rc->quirk_broken_aspm_l1) + value &= ~PCI_EXP_LNKCAP_ASPM_L1; + cdns_pcie_rp_writel(pcie, CDNS_PCIE_RP_CAP_OFFSET + PCI_EXP_LNKCAP, value); + + return 0; +} + +int cdns_pcie_host_bar_ib_config(struct cdns_pcie_rc *rc, + enum cdns_pcie_rp_bar bar, + u64 cpu_addr, + u64 size, + unsigned long flags) +{ + struct cdns_pcie *pcie = &rc->pcie; + u32 addr0, addr1, aperture, value; + + if (!rc->avail_ib_bar[bar]) + return -EBUSY; + + rc->avail_ib_bar[bar] = false; + + aperture = ilog2(size); + addr0 = CDNS_PCIE_AT_IB_RP_BAR_ADDR0_NBITS(aperture) | + (lower_32_bits(cpu_addr) & GENMASK(31, 8)); + addr1 = upper_32_bits(cpu_addr); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR0(bar), addr0); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR1(bar), addr1); + + if (bar == RP_NO_BAR) + return 0; + + value = cdns_pcie_readl(pcie, CDNS_PCIE_LM_RC_BAR_CFG); + value &= ~(LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar) | + LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar) | + LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar) | + LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar) | + LM_RC_BAR_CFG_APERTURE(bar, bar_aperture_mask[bar] + 2)); + if (size + cpu_addr >= SZ_4G) { + if (!(flags & IORESOURCE_PREFETCH)) + value |= LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar); + value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar); + } else { + if (!(flags & IORESOURCE_PREFETCH)) + value |= LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar); + value |= LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar); + } + + value |= LM_RC_BAR_CFG_APERTURE(bar, aperture); + cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value); + + return 0; +} + +static void cdns_pcie_host_unmap_dma_ranges(struct cdns_pcie_rc *rc) +{ + struct cdns_pcie *pcie = &rc->pcie; + enum cdns_pcie_rp_bar bar; + u32 value; + + /* Reset inbound configuration for all BARs which were being used */ + for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) { + if (rc->avail_ib_bar[bar]) + continue; + + cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR0(bar), 0); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_RP_BAR_ADDR1(bar), 0); + + if (bar == RP_NO_BAR) + continue; + + value = ~(LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar) | + LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar) | + LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar) | + LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar) | + LM_RC_BAR_CFG_APERTURE(bar, bar_aperture_mask[bar] + 2)); + cdns_pcie_writel(pcie, CDNS_PCIE_LM_RC_BAR_CFG, value); + } +} + +static void cdns_pcie_host_deinit_address_translation(struct cdns_pcie_rc *rc) +{ + struct cdns_pcie *pcie = &rc->pcie; + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rc); + struct resource_entry *entry; + int r; + + cdns_pcie_host_unmap_dma_ranges(rc); + + /* + * Reset outbound region 0 which was reserved for configuration space + * accesses. + */ + cdns_pcie_reset_outbound_region(pcie, 0); + + /* Reset rest of the outbound regions */ + r = 1; + resource_list_for_each_entry(entry, &bridge->windows) { + cdns_pcie_reset_outbound_region(pcie, r); + r++; + } +} + +static int cdns_pcie_host_init_address_translation(struct cdns_pcie_rc *rc) +{ + struct cdns_pcie *pcie = &rc->pcie; + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rc); + struct resource *cfg_res = rc->cfg_res; + struct resource_entry *entry; + u64 cpu_addr = cfg_res->start; + u32 addr0, addr1, desc1; + int r, busnr = 0; + + entry = resource_list_first_type(&bridge->windows, IORESOURCE_BUS); + if (entry) + busnr = entry->res->start; + + /* + * Reserve region 0 for PCI configure space accesses: + * OB_REGION_PCI_ADDR0 and OB_REGION_DESC0 are updated dynamically by + * cdns_pci_map_bus(), other region registers are set here once for all. + */ + addr1 = 0; /* Should be programmed to zero. */ + desc1 = CDNS_PCIE_AT_OB_REGION_DESC1_BUS(busnr); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR1(0), addr1); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(0), desc1); + + if (pcie->ops && pcie->ops->cpu_addr_fixup) + cpu_addr = pcie->ops->cpu_addr_fixup(pcie, cpu_addr); + + addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(12) | + (lower_32_bits(cpu_addr) & GENMASK(31, 8)); + addr1 = upper_32_bits(cpu_addr); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR0(0), addr0); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR1(0), addr1); + + r = 1; + resource_list_for_each_entry(entry, &bridge->windows) { + struct resource *res = entry->res; + u64 pci_addr = res->start - entry->offset; + + if (resource_type(res) == IORESOURCE_IO) + cdns_pcie_set_outbound_region(pcie, busnr, 0, r, + true, + pci_pio_to_address(res->start), + pci_addr, + resource_size(res)); + else + cdns_pcie_set_outbound_region(pcie, busnr, 0, r, + false, + res->start, + pci_addr, + resource_size(res)); + + r++; + } + + return cdns_pcie_host_map_dma_ranges(rc, cdns_pcie_host_bar_ib_config); +} + +static void cdns_pcie_host_deinit(struct cdns_pcie_rc *rc) +{ + cdns_pcie_host_deinit_address_translation(rc); + cdns_pcie_host_deinit_root_port(rc); +} + +int cdns_pcie_host_init(struct cdns_pcie_rc *rc) +{ + int err; + + err = cdns_pcie_host_init_root_port(rc); + if (err) + return err; + + return cdns_pcie_host_init_address_translation(rc); +} +EXPORT_SYMBOL_GPL(cdns_pcie_host_init); + +static void cdns_pcie_host_link_disable(struct cdns_pcie_rc *rc) +{ + struct cdns_pcie *pcie = &rc->pcie; + + cdns_pcie_stop_link(pcie); + cdns_pcie_host_disable_ptm_response(pcie); +} + +int cdns_pcie_host_link_setup(struct cdns_pcie_rc *rc) +{ + struct cdns_pcie *pcie = &rc->pcie; + struct device *dev = rc->pcie.dev; + int ret; + + if (rc->quirk_detect_quiet_flag) + cdns_pcie_detect_quiet_min_delay_set(&rc->pcie); + + cdns_pcie_host_enable_ptm_response(pcie); + + ret = cdns_pcie_start_link(pcie); + if (ret) { + dev_err(dev, "Failed to start link\n"); + return ret; + } + + ret = cdns_pcie_host_start_link(rc, cdns_pcie_link_up); + if (ret) + dev_dbg(dev, "PCIe link never came up\n"); + + return 0; +} +EXPORT_SYMBOL_GPL(cdns_pcie_host_link_setup); + +void cdns_pcie_host_disable(struct cdns_pcie_rc *rc) +{ + struct pci_host_bridge *bridge; + + cdns_pcie_debugfs_deinit(&rc->pcie); + bridge = pci_host_bridge_from_priv(rc); + pci_lock_rescan_remove(); + pci_stop_root_bus(bridge->bus); + pci_remove_root_bus(bridge->bus); + pci_unlock_rescan_remove(); + + cdns_pcie_host_deinit(rc); + cdns_pcie_host_link_disable(rc); +} +EXPORT_SYMBOL_GPL(cdns_pcie_host_disable); + +int cdns_pcie_host_setup(struct cdns_pcie_rc *rc) +{ + struct device *dev = rc->pcie.dev; + struct platform_device *pdev = to_platform_device(dev); + struct device_node *np = dev->of_node; + struct pci_host_bridge *bridge; + enum cdns_pcie_rp_bar bar; + struct cdns_pcie *pcie; + struct resource *res; + int ret; + + bridge = pci_host_bridge_from_priv(rc); + if (!bridge) + return -ENOMEM; + + pcie = &rc->pcie; + pcie->is_rc = true; + + rc->vendor_id = 0xffff; + of_property_read_u32(np, "vendor-id", &rc->vendor_id); + + rc->device_id = 0xffff; + of_property_read_u32(np, "device-id", &rc->device_id); + + if (pcie->max_link_speed < 1) + pcie->max_link_speed = of_pci_get_max_link_speed(np); + + pcie->reg_base = devm_platform_ioremap_resource_byname(pdev, "reg"); + if (IS_ERR(pcie->reg_base)) { + dev_err(dev, "missing \"reg\"\n"); + return PTR_ERR(pcie->reg_base); + } + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg"); + rc->cfg_base = devm_pci_remap_cfg_resource(dev, res); + if (IS_ERR(rc->cfg_base)) + return PTR_ERR(rc->cfg_base); + rc->cfg_res = res; + + ret = cdns_pcie_host_link_setup(rc); + if (ret) + return ret; + + for (bar = RP_BAR0; bar <= RP_NO_BAR; bar++) + rc->avail_ib_bar[bar] = true; + + ret = cdns_pcie_host_init(rc); + if (ret) + return ret; + + if (!bridge->ops) + bridge->ops = &cdns_pcie_host_ops; + + ret = pci_host_probe(bridge); + if (ret) + return ret; + + cdns_pcie_debugfs_init(pcie); + + return 0; +} +EXPORT_SYMBOL_GPL(cdns_pcie_host_setup); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Cadence PCIe host controller driver"); +MODULE_AUTHOR("Cyrille Pitchen <cyrille.pitchen@free-electrons.com>"); diff --git a/drivers/pci/controller/cadence/pcie-cadence-hpa-regs.h b/drivers/pci/controller/cadence/pcie-cadence-hpa-regs.h new file mode 100644 index 000000000..72299121f --- /dev/null +++ b/drivers/pci/controller/cadence/pcie-cadence-hpa-regs.h @@ -0,0 +1,195 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Cadence PCIe controller driver. + * + * Copyright (c) 2024, Cadence Design Systems + * Author: Manikandan K Pillai <mpillai@cadence.com> + */ +#ifndef _PCIE_CADENCE_HPA_REGS_H +#define _PCIE_CADENCE_HPA_REGS_H + +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/pci-epf.h> +#include <linux/phy/phy.h> +#include <linux/bitfield.h> + +#include "../../pci.h" + +/* High Performance Architecture (HPA) PCIe controller registers */ +#define CDNS_PCIE_HPA_IP_REG_BANK 0x01000000 +#define CDNS_PCIE_HPA_IP_CFG_CTRL_REG_BANK 0x01003C00 +#define CDNS_PCIE_HPA_IP_AXI_MASTER_COMMON 0x02020000 + +/* Address Translation Registers */ +#define CDNS_PCIE_HPA_AXI_SLAVE 0x03000000 +#define CDNS_PCIE_HPA_AXI_MASTER 0x03002000 + +/* Root Port register base address */ +#define CDNS_PCIE_HPA_RP_BASE 0x0 + +#define CDNS_PCIE_HPA_LM_ID 0x1420 + +/* Endpoint Function BARs */ +#define CDNS_PCIE_HPA_LM_EP_FUNC_BAR_CFG(bar, fn) \ + (((bar) < BAR_3) ? CDNS_PCIE_HPA_LM_EP_FUNC_BAR_CFG0(fn) : \ + CDNS_PCIE_HPA_LM_EP_FUNC_BAR_CFG1(fn)) +#define CDNS_PCIE_HPA_LM_EP_FUNC_BAR_CFG0(pfn) (0x4000 * (pfn)) +#define CDNS_PCIE_HPA_LM_EP_FUNC_BAR_CFG1(pfn) ((0x4000 * (pfn)) + 0x04) +#define CDNS_PCIE_HPA_LM_EP_VFUNC_BAR_CFG(bar, fn) \ + (((bar) < BAR_3) ? CDNS_PCIE_HPA_LM_EP_VFUNC_BAR_CFG0(fn) : \ + CDNS_PCIE_HPA_LM_EP_VFUNC_BAR_CFG1(fn)) +#define CDNS_PCIE_HPA_LM_EP_VFUNC_BAR_CFG0(vfn) ((0x4000 * (vfn)) + 0x08) +#define CDNS_PCIE_HPA_LM_EP_VFUNC_BAR_CFG1(vfn) ((0x4000 * (vfn)) + 0x0C) +#define CDNS_PCIE_HPA_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(f) \ + (GENMASK(5, 0) << (0x4 + (f) * 10)) +#define CDNS_PCIE_HPA_LM_EP_FUNC_BAR_CFG_BAR_APERTURE(b, a) \ + (((a) << (4 + ((b) * 10))) & (CDNS_PCIE_HPA_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b))) +#define CDNS_PCIE_HPA_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(f) \ + (GENMASK(3, 0) << ((f) * 10)) +#define CDNS_PCIE_HPA_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, c) \ + (((c) << ((b) * 10)) & (CDNS_PCIE_HPA_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b))) + +/* Endpoint Function Configuration Register */ +#define CDNS_PCIE_HPA_LM_EP_FUNC_CFG 0x02C0 + +/* Root Complex BAR Configuration Register */ +#define CDNS_PCIE_HPA_LM_RC_BAR_CFG 0x14 +#define CDNS_PCIE_HPA_LM_RC_BAR_CFG_BAR0_APERTURE_MASK GENMASK(9, 4) +#define CDNS_PCIE_HPA_LM_RC_BAR_CFG_BAR0_APERTURE(a) \ + FIELD_PREP(CDNS_PCIE_HPA_LM_RC_BAR_CFG_BAR0_APERTURE_MASK, a) +#define CDNS_PCIE_HPA_LM_RC_BAR_CFG_BAR0_CTRL_MASK GENMASK(3, 0) +#define CDNS_PCIE_HPA_LM_RC_BAR_CFG_BAR0_CTRL(c) \ + FIELD_PREP(CDNS_PCIE_HPA_LM_RC_BAR_CFG_BAR0_CTRL_MASK, c) +#define CDNS_PCIE_HPA_LM_RC_BAR_CFG_BAR1_APERTURE_MASK GENMASK(19, 14) +#define CDNS_PCIE_HPA_LM_RC_BAR_CFG_BAR1_APERTURE(a) \ + FIELD_PREP(CDNS_PCIE_HPA_LM_RC_BAR_CFG_BAR1_APERTURE_MASK, a) +#define CDNS_PCIE_HPA_LM_RC_BAR_CFG_BAR1_CTRL_MASK GENMASK(13, 10) +#define CDNS_PCIE_HPA_LM_RC_BAR_CFG_BAR1_CTRL(c) \ + FIELD_PREP(CDNS_PCIE_HPA_LM_RC_BAR_CFG_BAR1_CTRL_MASK, c) + +#define CDNS_PCIE_HPA_LM_RC_BAR_CFG_PREFETCH_MEM_ENABLE BIT(20) +#define CDNS_PCIE_HPA_LM_RC_BAR_CFG_PREFETCH_MEM_64BITS BIT(21) +#define CDNS_PCIE_HPA_LM_RC_BAR_CFG_IO_ENABLE BIT(22) +#define CDNS_PCIE_HPA_LM_RC_BAR_CFG_IO_32BITS BIT(23) + +/* BAR control values applicable to both Endpoint Function and Root Complex */ +#define CDNS_PCIE_HPA_LM_BAR_CFG_CTRL_DISABLED 0x0 +#define CDNS_PCIE_HPA_LM_BAR_CFG_CTRL_IO_32BITS 0x3 +#define CDNS_PCIE_HPA_LM_BAR_CFG_CTRL_MEM_32BITS 0x1 +#define CDNS_PCIE_HPA_LM_BAR_CFG_CTRL_PREFETCH_MEM_32BITS 0x9 +#define CDNS_PCIE_HPA_LM_BAR_CFG_CTRL_MEM_64BITS 0x5 +#define CDNS_PCIE_HPA_LM_BAR_CFG_CTRL_PREFETCH_MEM_64BITS 0xD + +#define HPA_LM_RC_BAR_CFG_CTRL_DISABLED(bar) \ + (CDNS_PCIE_HPA_LM_BAR_CFG_CTRL_DISABLED << ((bar) * 10)) +#define HPA_LM_RC_BAR_CFG_CTRL_IO_32BITS(bar) \ + (CDNS_PCIE_HPA_LM_BAR_CFG_CTRL_IO_32BITS << ((bar) * 10)) +#define HPA_LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar) \ + (CDNS_PCIE_HPA_LM_BAR_CFG_CTRL_MEM_32BITS << ((bar) * 10)) +#define HPA_LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar) \ + (CDNS_PCIE_HPA_LM_BAR_CFG_CTRL_PREFETCH_MEM_32BITS << ((bar) * 10)) +#define HPA_LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar) \ + (CDNS_PCIE_HPA_LM_BAR_CFG_CTRL_MEM_64BITS << ((bar) * 10)) +#define HPA_LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar) \ + (CDNS_PCIE_HPA_LM_BAR_CFG_CTRL_PREFETCH_MEM_64BITS << ((bar) * 10)) +#define HPA_LM_RC_BAR_CFG_APERTURE(bar, aperture) \ + (((aperture) - 7) << (((bar) * 10) + 4)) + +#define CDNS_PCIE_HPA_LM_PTM_CTRL 0x0520 +#define CDNS_PCIE_HPA_LM_PTM_CTRL_PTMRSEN BIT(17) + +/* Root Port Registers PCI config space for root port function */ +#define CDNS_PCIE_HPA_RP_CAP_OFFSET 0xC0 + +/* Region r Outbound AXI to PCIe Address Translation Register 0 */ +#define CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR0(r) (0x1010 + ((r) & 0x1F) * 0x0080) +#define CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR0_NBITS_MASK GENMASK(5, 0) +#define CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR0_NBITS(nbits) \ + (((nbits) - 1) & CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR0_NBITS_MASK) +#define CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR0_DEVFN_MASK GENMASK(23, 16) +#define CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR0_DEVFN(devfn) \ + FIELD_PREP(CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR0_DEVFN_MASK, devfn) +#define CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR0_BUS_MASK GENMASK(31, 24) +#define CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR0_BUS(bus) \ + FIELD_PREP(CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR0_BUS_MASK, bus) + +/* Region r Outbound AXI to PCIe Address Translation Register 1 */ +#define CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR1(r) (0x1014 + ((r) & 0x1F) * 0x0080) + +/* Region r Outbound PCIe Descriptor Register */ +#define CDNS_PCIE_HPA_AT_OB_REGION_DESC0(r) (0x1008 + ((r) & 0x1F) * 0x0080) +#define CDNS_PCIE_HPA_AT_OB_REGION_DESC0_TYPE_MASK GENMASK(28, 24) +#define CDNS_PCIE_HPA_AT_OB_REGION_DESC0_TYPE_MEM \ + FIELD_PREP(CDNS_PCIE_HPA_AT_OB_REGION_DESC0_TYPE_MASK, PCIE_TLP_TYPE_MEM_RDWR) +#define CDNS_PCIE_HPA_AT_OB_REGION_DESC0_TYPE_IO \ + FIELD_PREP(CDNS_PCIE_HPA_AT_OB_REGION_DESC0_TYPE_MASK, PCIE_TLP_TYPE_IO_RDWR) +#define CDNS_PCIE_HPA_AT_OB_REGION_DESC0_TYPE_CONF_TYPE0 \ + FIELD_PREP(CDNS_PCIE_HPA_AT_OB_REGION_DESC0_TYPE_MASK, PCIE_TLP_TYPE_CFG0_RDWR) +#define CDNS_PCIE_HPA_AT_OB_REGION_DESC0_TYPE_CONF_TYPE1 \ + FIELD_PREP(CDNS_PCIE_HPA_AT_OB_REGION_DESC0_TYPE_MASK, PCIE_TLP_TYPE_CFG1_RDWR) +#define CDNS_PCIE_HPA_AT_OB_REGION_DESC0_TYPE_NORMAL_MSG \ + FIELD_PREP(CDNS_PCIE_HPA_AT_OB_REGION_DESC0_TYPE_MASK, PCIE_TLP_TYPE_MSG) + +/* Region r Outbound PCIe Descriptor Register */ +#define CDNS_PCIE_HPA_AT_OB_REGION_DESC1(r) (0x100C + ((r) & 0x1F) * 0x0080) +#define CDNS_PCIE_HPA_AT_OB_REGION_DESC1_BUS_MASK GENMASK(31, 24) +#define CDNS_PCIE_HPA_AT_OB_REGION_DESC1_BUS(bus) \ + FIELD_PREP(CDNS_PCIE_HPA_AT_OB_REGION_DESC1_BUS_MASK, bus) +#define CDNS_PCIE_HPA_AT_OB_REGION_DESC1_DEVFN_MASK GENMASK(23, 16) +#define CDNS_PCIE_HPA_AT_OB_REGION_DESC1_DEVFN(devfn) \ + FIELD_PREP(CDNS_PCIE_HPA_AT_OB_REGION_DESC1_DEVFN_MASK, devfn) + +#define CDNS_PCIE_HPA_AT_OB_REGION_CTRL0(r) (0x1018 + ((r) & 0x1F) * 0x0080) +#define CDNS_PCIE_HPA_AT_OB_REGION_CTRL0_SUPPLY_BUS BIT(26) +#define CDNS_PCIE_HPA_AT_OB_REGION_CTRL0_SUPPLY_DEV_FN BIT(25) + +/* Region r AXI Region Base Address Register 0 */ +#define CDNS_PCIE_HPA_AT_OB_REGION_CPU_ADDR0(r) (0x1000 + ((r) & 0x1F) * 0x0080) +#define CDNS_PCIE_HPA_AT_OB_REGION_CPU_ADDR0_NBITS_MASK GENMASK(5, 0) +#define CDNS_PCIE_HPA_AT_OB_REGION_CPU_ADDR0_NBITS(nbits) \ + (((nbits) - 1) & CDNS_PCIE_HPA_AT_OB_REGION_CPU_ADDR0_NBITS_MASK) + +/* Region r AXI Region Base Address Register 1 */ +#define CDNS_PCIE_HPA_AT_OB_REGION_CPU_ADDR1(r) (0x1004 + ((r) & 0x1F) * 0x0080) + +/* Root Port BAR Inbound PCIe to AXI Address Translation Register */ +#define CDNS_PCIE_HPA_AT_IB_RP_BAR_ADDR0(bar) (((bar) * 0x0008)) +#define CDNS_PCIE_HPA_AT_IB_RP_BAR_ADDR0_NBITS_MASK GENMASK(5, 0) +#define CDNS_PCIE_HPA_AT_IB_RP_BAR_ADDR0_NBITS(nbits) \ + (((nbits) - 1) & CDNS_PCIE_HPA_AT_IB_RP_BAR_ADDR0_NBITS_MASK) +#define CDNS_PCIE_HPA_AT_IB_RP_BAR_ADDR1(bar) (0x04 + ((bar) * 0x0008)) + +/* AXI link down register */ +#define CDNS_PCIE_HPA_AT_LINKDOWN 0x04 + +/* + * Physical Layer Configuration Register 0 + * This register contains the parameters required for functional setup + * of Physical Layer. + */ +#define CDNS_PCIE_HPA_PHY_LAYER_CFG0 0x0400 +#define CDNS_PCIE_HPA_DETECT_QUIET_MIN_DELAY_MASK GENMASK(26, 24) +#define CDNS_PCIE_HPA_DETECT_QUIET_MIN_DELAY(delay) \ + FIELD_PREP(CDNS_PCIE_HPA_DETECT_QUIET_MIN_DELAY_MASK, delay) +#define CDNS_PCIE_HPA_LINK_TRNG_EN_MASK GENMASK(27, 27) + +#define CDNS_PCIE_HPA_PHY_DBG_STS_REG0 0x0420 + +#define CDNS_PCIE_HPA_RP_MAX_IB 0x3 +#define CDNS_PCIE_HPA_MAX_OB 15 + +/* Endpoint Function BAR Inbound PCIe to AXI Address Translation Register */ +#define CDNS_PCIE_HPA_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar) (((fn) * 0x0080) + ((bar) * 0x0008)) +#define CDNS_PCIE_HPA_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar) (0x4 + ((fn) * 0x0080) + ((bar) * 0x0008)) + +/* Miscellaneous offsets definitions */ +#define CDNS_PCIE_HPA_TAG_MANAGEMENT 0x0 +#define CDNS_PCIE_HPA_SLAVE_RESP 0x100 + +#define I_ROOT_PORT_REQ_ID_REG 0x141c +#define LM_HAL_SBSA_CTRL 0x1170 + +#define I_PCIE_BUS_NUMBERS (CDNS_PCIE_HPA_RP_BASE + 0x18) +#define CDNS_PCIE_EROM 0x18 +#endif /* _PCIE_CADENCE_HPA_REGS_H */ diff --git a/drivers/pci/controller/cadence/pcie-cadence-hpa.c b/drivers/pci/controller/cadence/pcie-cadence-hpa.c new file mode 100644 index 000000000..f60a16938 --- /dev/null +++ b/drivers/pci/controller/cadence/pcie-cadence-hpa.c @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Cadence PCIe controller driver. + * + * Copyright (c) 2024, Cadence Design Systems + * Author: Manikandan K Pillai <mpillai@cadence.com> + */ +#include <linux/kernel.h> +#include <linux/of.h> + +#include "pcie-cadence.h" + +bool cdns_pcie_hpa_link_up(struct cdns_pcie *pcie) +{ + u32 pl_reg_val; + + pl_reg_val = cdns_pcie_hpa_readl(pcie, REG_BANK_IP_REG, CDNS_PCIE_HPA_PHY_DBG_STS_REG0); + if (pl_reg_val & GENMASK(0, 0)) + return true; + return false; +} +EXPORT_SYMBOL_GPL(cdns_pcie_hpa_link_up); + +void cdns_pcie_hpa_detect_quiet_min_delay_set(struct cdns_pcie *pcie) +{ + u32 delay = 0x3; + u32 ltssm_control_cap; + + /* Set the LTSSM Detect Quiet state min. delay to 2ms */ + ltssm_control_cap = cdns_pcie_hpa_readl(pcie, REG_BANK_IP_REG, + CDNS_PCIE_HPA_PHY_LAYER_CFG0); + ltssm_control_cap = ((ltssm_control_cap & + ~CDNS_PCIE_HPA_DETECT_QUIET_MIN_DELAY_MASK) | + CDNS_PCIE_HPA_DETECT_QUIET_MIN_DELAY(delay)); + + cdns_pcie_hpa_writel(pcie, REG_BANK_IP_REG, + CDNS_PCIE_HPA_PHY_LAYER_CFG0, ltssm_control_cap); +} +EXPORT_SYMBOL_GPL(cdns_pcie_hpa_detect_quiet_min_delay_set); + +void cdns_pcie_hpa_set_outbound_region(struct cdns_pcie *pcie, u8 busnr, u8 fn, + u32 r, bool is_io, + u64 cpu_addr, u64 pci_addr, size_t size) +{ + /* + * roundup_pow_of_two() returns an unsigned long, which is not suited + * for 64bit values + */ + u64 sz = 1ULL << fls64(size - 1); + int nbits = ilog2(sz); + u32 addr0, addr1, desc0, desc1, ctrl0; + + if (nbits < 8) + nbits = 8; + + /* Set the PCI address */ + addr0 = CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR0_NBITS(nbits) | + (lower_32_bits(pci_addr) & GENMASK(31, 8)); + addr1 = upper_32_bits(pci_addr); + + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR0(r), addr0); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR1(r), addr1); + + /* Set the PCIe header descriptor */ + if (is_io) + desc0 = CDNS_PCIE_HPA_AT_OB_REGION_DESC0_TYPE_IO; + else + desc0 = CDNS_PCIE_HPA_AT_OB_REGION_DESC0_TYPE_MEM; + desc1 = 0; + ctrl0 = 0; + + /* + * Whether Bit [26] is set or not inside DESC0 register of the outbound + * PCIe descriptor, the PCI function number must be set into + * Bits [31:24] of DESC1 anyway. + * + * In Root Complex mode, the function number is always 0 but in Endpoint + * mode, the PCIe controller may support more than one function. This + * function number needs to be set properly into the outbound PCIe + * descriptor. + * + * Besides, setting Bit [26] is mandatory when in Root Complex mode: + * then the driver must provide the bus, resp. device, number in + * Bits [31:24] of DESC1, resp. Bits[23:16] of DESC0. Like the function + * number, the device number is always 0 in Root Complex mode. + * + * However when in Endpoint mode, we can clear Bit [26] of DESC0, hence + * the PCIe controller will use the captured values for the bus and + * device numbers. + */ + if (pcie->is_rc) { + /* The device and function numbers are always 0 */ + desc1 = CDNS_PCIE_HPA_AT_OB_REGION_DESC1_BUS(busnr) | + CDNS_PCIE_HPA_AT_OB_REGION_DESC1_DEVFN(0); + ctrl0 = CDNS_PCIE_HPA_AT_OB_REGION_CTRL0_SUPPLY_BUS | + CDNS_PCIE_HPA_AT_OB_REGION_CTRL0_SUPPLY_DEV_FN; + } else { + /* + * Use captured values for bus and device numbers but still + * need to set the function number + */ + desc1 |= CDNS_PCIE_HPA_AT_OB_REGION_DESC1_DEVFN(fn); + } + + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_DESC0(r), desc0); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_DESC1(r), desc1); + + addr0 = CDNS_PCIE_HPA_AT_OB_REGION_CPU_ADDR0_NBITS(nbits) | + (lower_32_bits(cpu_addr) & GENMASK(31, 8)); + addr1 = upper_32_bits(cpu_addr); + + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_CPU_ADDR0(r), addr0); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_CPU_ADDR1(r), addr1); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_CTRL0(r), ctrl0); +} +EXPORT_SYMBOL_GPL(cdns_pcie_hpa_set_outbound_region); + +void cdns_pcie_hpa_set_outbound_region_for_normal_msg(struct cdns_pcie *pcie, + u8 busnr, u8 fn, + u32 r, u64 cpu_addr) +{ + u32 addr0, addr1, desc0, desc1, ctrl0; + + desc0 = CDNS_PCIE_HPA_AT_OB_REGION_DESC0_TYPE_NORMAL_MSG; + desc1 = 0; + ctrl0 = 0; + + /* See cdns_pcie_set_outbound_region() comments above */ + if (pcie->is_rc) { + desc1 = CDNS_PCIE_HPA_AT_OB_REGION_DESC1_BUS(busnr) | + CDNS_PCIE_HPA_AT_OB_REGION_DESC1_DEVFN(0); + ctrl0 = CDNS_PCIE_HPA_AT_OB_REGION_CTRL0_SUPPLY_BUS | + CDNS_PCIE_HPA_AT_OB_REGION_CTRL0_SUPPLY_DEV_FN; + } else { + desc1 |= CDNS_PCIE_HPA_AT_OB_REGION_DESC1_DEVFN(fn); + } + + addr0 = CDNS_PCIE_HPA_AT_OB_REGION_CPU_ADDR0_NBITS(17) | + (lower_32_bits(cpu_addr) & GENMASK(31, 8)); + addr1 = upper_32_bits(cpu_addr); + + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR0(r), 0); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_PCI_ADDR1(r), 0); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_DESC0(r), desc0); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_DESC1(r), desc1); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_CPU_ADDR0(r), addr0); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_CPU_ADDR1(r), addr1); + cdns_pcie_hpa_writel(pcie, REG_BANK_AXI_SLAVE, + CDNS_PCIE_HPA_AT_OB_REGION_CTRL0(r), ctrl0); +} +EXPORT_SYMBOL_GPL(cdns_pcie_hpa_set_outbound_region_for_normal_msg); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Cadence PCIe controller driver"); diff --git a/drivers/pci/controller/cadence/pcie-cadence-lga-regs.h b/drivers/pci/controller/cadence/pcie-cadence-lga-regs.h new file mode 100644 index 000000000..15dc4fcaf --- /dev/null +++ b/drivers/pci/controller/cadence/pcie-cadence-lga-regs.h @@ -0,0 +1,225 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Cadence PCIe controller driver. + * + * Copyright (c) 2017 Cadence + * Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com> + */ +#ifndef _PCIE_CADENCE_LGA_REGS_H +#define _PCIE_CADENCE_LGA_REGS_H + +#include <linux/bitfield.h> + +/* Local Management Registers */ +#define CDNS_PCIE_LM_BASE 0x00100000 + +/* Vendor ID Register */ +#define CDNS_PCIE_LM_ID (CDNS_PCIE_LM_BASE + 0x0044) +#define CDNS_PCIE_LM_ID_VENDOR_MASK GENMASK(15, 0) +#define CDNS_PCIE_LM_ID_VENDOR_SHIFT 0 +#define CDNS_PCIE_LM_ID_VENDOR(vid) \ + (((vid) << CDNS_PCIE_LM_ID_VENDOR_SHIFT) & CDNS_PCIE_LM_ID_VENDOR_MASK) +#define CDNS_PCIE_LM_ID_SUBSYS_MASK GENMASK(31, 16) +#define CDNS_PCIE_LM_ID_SUBSYS_SHIFT 16 +#define CDNS_PCIE_LM_ID_SUBSYS(sub) \ + (((sub) << CDNS_PCIE_LM_ID_SUBSYS_SHIFT) & CDNS_PCIE_LM_ID_SUBSYS_MASK) + +/* Root Port Requester ID Register */ +#define CDNS_PCIE_LM_RP_RID (CDNS_PCIE_LM_BASE + 0x0228) +#define CDNS_PCIE_LM_RP_RID_MASK GENMASK(15, 0) +#define CDNS_PCIE_LM_RP_RID_SHIFT 0 +#define CDNS_PCIE_LM_RP_RID_(rid) \ + (((rid) << CDNS_PCIE_LM_RP_RID_SHIFT) & CDNS_PCIE_LM_RP_RID_MASK) + +/* Endpoint Bus and Device Number Register */ +#define CDNS_PCIE_LM_EP_ID (CDNS_PCIE_LM_BASE + 0x022C) +#define CDNS_PCIE_LM_EP_ID_DEV_MASK GENMASK(4, 0) +#define CDNS_PCIE_LM_EP_ID_DEV_SHIFT 0 +#define CDNS_PCIE_LM_EP_ID_BUS_MASK GENMASK(15, 8) +#define CDNS_PCIE_LM_EP_ID_BUS_SHIFT 8 + +/* Endpoint Function f BAR b Configuration Registers */ +#define CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn) \ + (((bar) < BAR_4) ? CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn) : CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn)) +#define CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn) \ + (CDNS_PCIE_LM_BASE + 0x0240 + (fn) * 0x0008) +#define CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn) \ + (CDNS_PCIE_LM_BASE + 0x0244 + (fn) * 0x0008) +#define CDNS_PCIE_LM_EP_VFUNC_BAR_CFG(bar, fn) \ + (((bar) < BAR_4) ? CDNS_PCIE_LM_EP_VFUNC_BAR_CFG0(fn) : CDNS_PCIE_LM_EP_VFUNC_BAR_CFG1(fn)) +#define CDNS_PCIE_LM_EP_VFUNC_BAR_CFG0(fn) \ + (CDNS_PCIE_LM_BASE + 0x0280 + (fn) * 0x0008) +#define CDNS_PCIE_LM_EP_VFUNC_BAR_CFG1(fn) \ + (CDNS_PCIE_LM_BASE + 0x0284 + (fn) * 0x0008) +#define CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) \ + (GENMASK(4, 0) << ((b) * 8)) +#define CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE(b, a) \ + (((a) << ((b) * 8)) & CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b)) +#define CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b) \ + (GENMASK(7, 5) << ((b) * 8)) +#define CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, c) \ + (((c) << ((b) * 8 + 5)) & CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b)) + +/* Endpoint Function Configuration Register */ +#define CDNS_PCIE_LM_EP_FUNC_CFG (CDNS_PCIE_LM_BASE + 0x02C0) + +/* Root Complex BAR Configuration Register */ +#define CDNS_PCIE_LM_RC_BAR_CFG (CDNS_PCIE_LM_BASE + 0x0300) +#define CDNS_PCIE_LM_RC_BAR_CFG_BAR0_APERTURE_MASK GENMASK(5, 0) +#define CDNS_PCIE_LM_RC_BAR_CFG_BAR0_APERTURE(a) \ + (((a) << 0) & CDNS_PCIE_LM_RC_BAR_CFG_BAR0_APERTURE_MASK) +#define CDNS_PCIE_LM_RC_BAR_CFG_BAR0_CTRL_MASK GENMASK(8, 6) +#define CDNS_PCIE_LM_RC_BAR_CFG_BAR0_CTRL(c) \ + (((c) << 6) & CDNS_PCIE_LM_RC_BAR_CFG_BAR0_CTRL_MASK) +#define CDNS_PCIE_LM_RC_BAR_CFG_BAR1_APERTURE_MASK GENMASK(13, 9) +#define CDNS_PCIE_LM_RC_BAR_CFG_BAR1_APERTURE(a) \ + (((a) << 9) & CDNS_PCIE_LM_RC_BAR_CFG_BAR1_APERTURE_MASK) +#define CDNS_PCIE_LM_RC_BAR_CFG_BAR1_CTRL_MASK GENMASK(16, 14) +#define CDNS_PCIE_LM_RC_BAR_CFG_BAR1_CTRL(c) \ + (((c) << 14) & CDNS_PCIE_LM_RC_BAR_CFG_BAR1_CTRL_MASK) +#define CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_ENABLE BIT(17) +#define CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_32BITS 0 +#define CDNS_PCIE_LM_RC_BAR_CFG_PREFETCH_MEM_64BITS BIT(18) +#define CDNS_PCIE_LM_RC_BAR_CFG_IO_ENABLE BIT(19) +#define CDNS_PCIE_LM_RC_BAR_CFG_IO_16BITS 0 +#define CDNS_PCIE_LM_RC_BAR_CFG_IO_32BITS BIT(20) +#define CDNS_PCIE_LM_RC_BAR_CFG_CHECK_ENABLE BIT(31) + +/* BAR control values applicable to both Endpoint Function and Root Complex */ +#define CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED 0x0 +#define CDNS_PCIE_LM_BAR_CFG_CTRL_IO_32BITS 0x1 +#define CDNS_PCIE_LM_BAR_CFG_CTRL_MEM_32BITS 0x4 +#define CDNS_PCIE_LM_BAR_CFG_CTRL_PREFETCH_MEM_32BITS 0x5 +#define CDNS_PCIE_LM_BAR_CFG_CTRL_MEM_64BITS 0x6 +#define CDNS_PCIE_LM_BAR_CFG_CTRL_PREFETCH_MEM_64BITS 0x7 + +#define LM_RC_BAR_CFG_CTRL_DISABLED(bar) \ + (CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED << (((bar) * 8) + 6)) +#define LM_RC_BAR_CFG_CTRL_IO_32BITS(bar) \ + (CDNS_PCIE_LM_BAR_CFG_CTRL_IO_32BITS << (((bar) * 8) + 6)) +#define LM_RC_BAR_CFG_CTRL_MEM_32BITS(bar) \ + (CDNS_PCIE_LM_BAR_CFG_CTRL_MEM_32BITS << (((bar) * 8) + 6)) +#define LM_RC_BAR_CFG_CTRL_PREF_MEM_32BITS(bar) \ + (CDNS_PCIE_LM_BAR_CFG_CTRL_PREFETCH_MEM_32BITS << (((bar) * 8) + 6)) +#define LM_RC_BAR_CFG_CTRL_MEM_64BITS(bar) \ + (CDNS_PCIE_LM_BAR_CFG_CTRL_MEM_64BITS << (((bar) * 8) + 6)) +#define LM_RC_BAR_CFG_CTRL_PREF_MEM_64BITS(bar) \ + (CDNS_PCIE_LM_BAR_CFG_CTRL_PREFETCH_MEM_64BITS << (((bar) * 8) + 6)) +#define LM_RC_BAR_CFG_APERTURE(bar, aperture) \ + (((aperture) - 2) << ((bar) * 8)) + +/* PTM Control Register */ +#define CDNS_PCIE_LM_PTM_CTRL (CDNS_PCIE_LM_BASE + 0x0DA8) +#define CDNS_PCIE_LM_TPM_CTRL_PTMRSEN BIT(17) + +/* + * Endpoint Function Registers (PCI configuration space for endpoint functions) + */ +#define CDNS_PCIE_EP_FUNC_BASE(fn) (((fn) << 12) & GENMASK(19, 12)) + +#define CDNS_PCIE_EP_FUNC_MSI_CAP_OFFSET 0x90 +#define CDNS_PCIE_EP_FUNC_MSIX_CAP_OFFSET 0xB0 +#define CDNS_PCIE_EP_FUNC_DEV_CAP_OFFSET 0xC0 +#define CDNS_PCIE_EP_FUNC_SRIOV_CAP_OFFSET 0x200 + +/* Endpoint PF Registers */ +#define CDNS_PCIE_CORE_PF_I_ARI_CAP_AND_CTRL(fn) (0x144 + (fn) * 0x1000) +#define CDNS_PCIE_ARI_CAP_NFN_MASK GENMASK(15, 8) + +/* Root Port Registers (PCI configuration space for the root port function) */ +#define CDNS_PCIE_RP_BASE 0x00200000 +#define CDNS_PCIE_RP_CAP_OFFSET 0xC0 + +/* Address Translation Registers */ +#define CDNS_PCIE_AT_BASE 0x00400000 + +/* Region r Outbound AXI to PCIe Address Translation Register 0 */ +#define CDNS_PCIE_AT_OB_REGION_PCI_ADDR0(r) \ + (CDNS_PCIE_AT_BASE + 0x0000 + ((r) & 0x1F) * 0x0020) +#define CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_NBITS_MASK GENMASK(5, 0) +#define CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_NBITS(nbits) \ + (((nbits) - 1) & CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_NBITS_MASK) +#define CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN_MASK GENMASK(19, 12) +#define CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN(devfn) \ + (((devfn) << 12) & CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN_MASK) +#define CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_BUS_MASK GENMASK(27, 20) +#define CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_BUS(bus) \ + (((bus) << 20) & CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_BUS_MASK) + +/* Region r Outbound AXI to PCIe Address Translation Register 1 */ +#define CDNS_PCIE_AT_OB_REGION_PCI_ADDR1(r) \ + (CDNS_PCIE_AT_BASE + 0x0004 + ((r) & 0x1F) * 0x0020) + +/* Region r Outbound PCIe Descriptor Register 0 */ +#define CDNS_PCIE_AT_OB_REGION_DESC0(r) \ + (CDNS_PCIE_AT_BASE + 0x0008 + ((r) & 0x1F) * 0x0020) +#define CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_MASK GENMASK(3, 0) +#define CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_MEM 0x2 +#define CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_IO 0x6 +#define CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE0 0xA +#define CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_CONF_TYPE1 0xB +#define CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_NORMAL_MSG 0xC +#define CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_VENDOR_MSG 0xD +/* Bit 23 MUST be set in RC mode. */ +#define CDNS_PCIE_AT_OB_REGION_DESC0_HARDCODED_RID BIT(23) +#define CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN_MASK GENMASK(31, 24) +#define CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN(devfn) \ + (((devfn) << 24) & CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN_MASK) + +/* Region r Outbound PCIe Descriptor Register 1 */ +#define CDNS_PCIE_AT_OB_REGION_DESC1(r) \ + (CDNS_PCIE_AT_BASE + 0x000C + ((r) & 0x1F) * 0x0020) +#define CDNS_PCIE_AT_OB_REGION_DESC1_BUS_MASK GENMASK(7, 0) +#define CDNS_PCIE_AT_OB_REGION_DESC1_BUS(bus) \ + ((bus) & CDNS_PCIE_AT_OB_REGION_DESC1_BUS_MASK) + +/* Region r AXI Region Base Address Register 0 */ +#define CDNS_PCIE_AT_OB_REGION_CPU_ADDR0(r) \ + (CDNS_PCIE_AT_BASE + 0x0018 + ((r) & 0x1F) * 0x0020) +#define CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS_MASK GENMASK(5, 0) +#define CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(nbits) \ + (((nbits) - 1) & CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS_MASK) + +/* Region r AXI Region Base Address Register 1 */ +#define CDNS_PCIE_AT_OB_REGION_CPU_ADDR1(r) \ + (CDNS_PCIE_AT_BASE + 0x001C + ((r) & 0x1F) * 0x0020) + +/* Root Port BAR Inbound PCIe to AXI Address Translation Register */ +#define CDNS_PCIE_AT_IB_RP_BAR_ADDR0(bar) \ + (CDNS_PCIE_AT_BASE + 0x0800 + (bar) * 0x0008) +#define CDNS_PCIE_AT_IB_RP_BAR_ADDR0_NBITS_MASK GENMASK(5, 0) +#define CDNS_PCIE_AT_IB_RP_BAR_ADDR0_NBITS(nbits) \ + (((nbits) - 1) & CDNS_PCIE_AT_IB_RP_BAR_ADDR0_NBITS_MASK) +#define CDNS_PCIE_AT_IB_RP_BAR_ADDR1(bar) \ + (CDNS_PCIE_AT_BASE + 0x0804 + (bar) * 0x0008) + +/* AXI link down register */ +#define CDNS_PCIE_AT_LINKDOWN (CDNS_PCIE_AT_BASE + 0x0824) + +/* LTSSM Capabilities register */ +#define CDNS_PCIE_LTSSM_CONTROL_CAP (CDNS_PCIE_LM_BASE + 0x0054) +#define CDNS_PCIE_DETECT_QUIET_MIN_DELAY_MASK GENMASK(2, 1) +#define CDNS_PCIE_DETECT_QUIET_MIN_DELAY_SHIFT 1 +#define CDNS_PCIE_DETECT_QUIET_MIN_DELAY(delay) \ + (((delay) << CDNS_PCIE_DETECT_QUIET_MIN_DELAY_SHIFT) & \ + CDNS_PCIE_DETECT_QUIET_MIN_DELAY_MASK) + +#define CDNS_PCIE_RP_MAX_IB 0x3 +#define CDNS_PCIE_MAX_OB 32 + +/* Endpoint Function BAR Inbound PCIe to AXI Address Translation Register */ +#define CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar) \ + (CDNS_PCIE_AT_BASE + 0x0840 + (fn) * 0x0040 + (bar) * 0x0008) +#define CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar) \ + (CDNS_PCIE_AT_BASE + 0x0844 + (fn) * 0x0040 + (bar) * 0x0008) + +/* Normal/Vendor specific message access: offset inside some outbound region */ +#define CDNS_PCIE_NORMAL_MSG_ROUTING_MASK GENMASK(7, 5) +#define CDNS_PCIE_NORMAL_MSG_ROUTING(route) \ + (((route) << 5) & CDNS_PCIE_NORMAL_MSG_ROUTING_MASK) +#define CDNS_PCIE_NORMAL_MSG_CODE_MASK GENMASK(15, 8) +#define CDNS_PCIE_NORMAL_MSG_CODE(code) \ + (((code) << 8) & CDNS_PCIE_NORMAL_MSG_CODE_MASK) +#define CDNS_PCIE_MSG_NO_DATA BIT(16) + +#endif /* _PCIE_CADENCE_LGA_REGS_H */ diff --git a/drivers/pci/controller/cadence/pcie-cadence-plat.c b/drivers/pci/controller/cadence/pcie-cadence-plat.c new file mode 100644 index 000000000..a1ea24fc3 --- /dev/null +++ b/drivers/pci/controller/cadence/pcie-cadence-plat.c @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Cadence PCIe platform driver. + * + * Copyright (c) 2019, Cadence Design Systems + * Author: Tom Joseph <tjoseph@cadence.com> + */ +#include <linux/kernel.h> +#include <linux/of.h> +#include <linux/of_pci.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include "pcie-cadence.h" + +#define CDNS_PLAT_CPU_TO_BUS_ADDR 0x0FFFFFFF + +/** + * struct cdns_plat_pcie - private data for this PCIe platform driver + * @pcie: Cadence PCIe controller + */ +struct cdns_plat_pcie { + struct cdns_pcie *pcie; +}; + +static const struct of_device_id cdns_plat_pcie_of_match[]; + +static u64 cdns_plat_cpu_addr_fixup(struct cdns_pcie *pcie, u64 cpu_addr) +{ + return cpu_addr & CDNS_PLAT_CPU_TO_BUS_ADDR; +} + +static const struct cdns_pcie_ops cdns_plat_ops = { + .cpu_addr_fixup = cdns_plat_cpu_addr_fixup, +}; + +static int cdns_plat_pcie_probe(struct platform_device *pdev) +{ + const struct cdns_plat_pcie_of_data *data; + struct cdns_plat_pcie *cdns_plat_pcie; + struct device *dev = &pdev->dev; + struct pci_host_bridge *bridge; + struct cdns_pcie_ep *ep; + struct cdns_pcie_rc *rc; + int phy_count; + bool is_rc; + int ret; + + data = of_device_get_match_data(dev); + if (!data) + return -EINVAL; + + is_rc = data->is_rc; + + pr_debug(" Started %s with is_rc: %d\n", __func__, is_rc); + cdns_plat_pcie = devm_kzalloc(dev, sizeof(*cdns_plat_pcie), GFP_KERNEL); + if (!cdns_plat_pcie) + return -ENOMEM; + + platform_set_drvdata(pdev, cdns_plat_pcie); + if (is_rc) { + if (!IS_ENABLED(CONFIG_PCIE_CADENCE_PLAT_HOST)) + return -ENODEV; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rc)); + if (!bridge) + return -ENOMEM; + + rc = pci_host_bridge_priv(bridge); + rc->pcie.dev = dev; + rc->pcie.ops = &cdns_plat_ops; + cdns_plat_pcie->pcie = &rc->pcie; + + ret = cdns_pcie_init_phy(dev, cdns_plat_pcie->pcie); + if (ret) { + dev_err(dev, "failed to init phy\n"); + return ret; + } + pm_runtime_enable(dev); + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + dev_err(dev, "pm_runtime_get_sync() failed\n"); + goto err_get_sync; + } + + ret = cdns_pcie_host_setup(rc); + if (ret) + goto err_init; + } else { + if (!IS_ENABLED(CONFIG_PCIE_CADENCE_PLAT_EP)) + return -ENODEV; + + ep = devm_kzalloc(dev, sizeof(*ep), GFP_KERNEL); + if (!ep) + return -ENOMEM; + + ep->pcie.dev = dev; + ep->pcie.ops = &cdns_plat_ops; + cdns_plat_pcie->pcie = &ep->pcie; + + ret = cdns_pcie_init_phy(dev, cdns_plat_pcie->pcie); + if (ret) { + dev_err(dev, "failed to init phy\n"); + return ret; + } + + pm_runtime_enable(dev); + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + dev_err(dev, "pm_runtime_get_sync() failed\n"); + goto err_get_sync; + } + + ret = cdns_pcie_ep_setup(ep); + if (ret) + goto err_init; + } + + return 0; + + err_init: + err_get_sync: + pm_runtime_put_sync(dev); + pm_runtime_disable(dev); + cdns_pcie_disable_phy(cdns_plat_pcie->pcie); + phy_count = cdns_plat_pcie->pcie->phy_count; + while (phy_count--) + device_link_del(cdns_plat_pcie->pcie->link[phy_count]); + + return 0; +} + +static void cdns_plat_pcie_shutdown(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct cdns_pcie *pcie = dev_get_drvdata(dev); + int ret; + + ret = pm_runtime_put_sync(dev); + if (ret < 0) + dev_dbg(dev, "pm_runtime_put_sync failed\n"); + + pm_runtime_disable(dev); + + cdns_pcie_disable_phy(pcie); +} + +static const struct cdns_plat_pcie_of_data cdns_plat_pcie_host_of_data = { + .is_rc = true, +}; + +static const struct cdns_plat_pcie_of_data cdns_plat_pcie_ep_of_data = { + .is_rc = false, +}; + +static const struct of_device_id cdns_plat_pcie_of_match[] = { + { + .compatible = "cdns,cdns-pcie-host", + .data = &cdns_plat_pcie_host_of_data, + }, + { + .compatible = "cdns,cdns-pcie-ep", + .data = &cdns_plat_pcie_ep_of_data, + }, + {}, +}; +MODULE_DEVICE_TABLE(of, cdns_plat_pcie_of_match); + +static struct platform_driver cdns_plat_pcie_driver = { + .driver = { + .name = "cdns-pcie", + .of_match_table = cdns_plat_pcie_of_match, + .pm = &cdns_pcie_pm_ops, + }, + .probe = cdns_plat_pcie_probe, + .shutdown = cdns_plat_pcie_shutdown, +}; +module_platform_driver(cdns_plat_pcie_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Cadence PCIe controller platform driver"); diff --git a/drivers/pci/controller/cadence/pcie-cadence.c b/drivers/pci/controller/cadence/pcie-cadence.c new file mode 100644 index 000000000..a1eada56e --- /dev/null +++ b/drivers/pci/controller/cadence/pcie-cadence.c @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2017 Cadence +// Cadence PCIe controller driver. +// Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com> + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> + +#include "pcie-cadence.h" +#include "../../pci.h" + +u8 cdns_pcie_find_capability(struct cdns_pcie *pcie, u8 cap) +{ + return PCI_FIND_NEXT_CAP(cdns_pcie_read_cfg, PCI_CAPABILITY_LIST, + cap, NULL, pcie); +} +EXPORT_SYMBOL_GPL(cdns_pcie_find_capability); + +u16 cdns_pcie_find_ext_capability(struct cdns_pcie *pcie, u8 cap) +{ + return PCI_FIND_NEXT_EXT_CAP(cdns_pcie_read_cfg, 0, cap, NULL, pcie); +} +EXPORT_SYMBOL_GPL(cdns_pcie_find_ext_capability); + +bool cdns_pcie_linkup(struct cdns_pcie *pcie) +{ + u32 pl_reg_val; + + pl_reg_val = cdns_pcie_readl(pcie, CDNS_PCIE_LM_BASE); + if (pl_reg_val & GENMASK(0, 0)) + return true; + return false; +} +EXPORT_SYMBOL_GPL(cdns_pcie_linkup); + +void cdns_pcie_detect_quiet_min_delay_set(struct cdns_pcie *pcie) +{ + u32 delay = 0x3; + u32 ltssm_control_cap; + + /* + * Set the LTSSM Detect Quiet state min. delay to 2ms. + */ + ltssm_control_cap = cdns_pcie_readl(pcie, CDNS_PCIE_LTSSM_CONTROL_CAP); + ltssm_control_cap = ((ltssm_control_cap & + ~CDNS_PCIE_DETECT_QUIET_MIN_DELAY_MASK) | + CDNS_PCIE_DETECT_QUIET_MIN_DELAY(delay)); + + cdns_pcie_writel(pcie, CDNS_PCIE_LTSSM_CONTROL_CAP, ltssm_control_cap); +} +EXPORT_SYMBOL_GPL(cdns_pcie_detect_quiet_min_delay_set); + +void cdns_pcie_set_outbound_region(struct cdns_pcie *pcie, u8 busnr, u8 fn, + u32 r, bool is_io, + u64 cpu_addr, u64 pci_addr, size_t size) +{ + /* + * roundup_pow_of_two() returns an unsigned long, which is not suited + * for 64bit values. + */ + u64 sz = 1ULL << fls64(size - 1); + int nbits = ilog2(sz); + u32 addr0, addr1, desc0, desc1; + + if (nbits < 8) + nbits = 8; + + /* Set the PCI address */ + addr0 = CDNS_PCIE_AT_OB_REGION_PCI_ADDR0_NBITS(nbits) | + (lower_32_bits(pci_addr) & GENMASK(31, 8)); + addr1 = upper_32_bits(pci_addr); + + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR0(r), addr0); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR1(r), addr1); + + /* Set the PCIe header descriptor */ + if (is_io) + desc0 = CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_IO; + else + desc0 = CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_MEM; + desc1 = 0; + + /* + * Whatever Bit [23] is set or not inside DESC0 register of the outbound + * PCIe descriptor, the PCI function number must be set into + * Bits [26:24] of DESC0 anyway. + * + * In Root Complex mode, the function number is always 0 but in Endpoint + * mode, the PCIe controller may support more than one function. This + * function number needs to be set properly into the outbound PCIe + * descriptor. + * + * Besides, setting Bit [23] is mandatory when in Root Complex mode: + * then the driver must provide the bus, resp. device, number in + * Bits [7:0] of DESC1, resp. Bits[31:27] of DESC0. Like the function + * number, the device number is always 0 in Root Complex mode. + * + * However when in Endpoint mode, we can clear Bit [23] of DESC0, hence + * the PCIe controller will use the captured values for the bus and + * device numbers. + */ + if (pcie->is_rc) { + /* The device and function numbers are always 0. */ + desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_HARDCODED_RID | + CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN(0); + desc1 |= CDNS_PCIE_AT_OB_REGION_DESC1_BUS(busnr); + } else { + /* + * Use captured values for bus and device numbers but still + * need to set the function number. + */ + desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN(fn); + } + + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC0(r), desc0); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(r), desc1); + + /* Set the CPU address */ + if (pcie->ops && pcie->ops->cpu_addr_fixup) + cpu_addr = pcie->ops->cpu_addr_fixup(pcie, cpu_addr); + + addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(nbits) | + (lower_32_bits(cpu_addr) & GENMASK(31, 8)); + addr1 = upper_32_bits(cpu_addr); + + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR0(r), addr0); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR1(r), addr1); +} +EXPORT_SYMBOL_GPL(cdns_pcie_set_outbound_region); + +void cdns_pcie_set_outbound_region_for_normal_msg(struct cdns_pcie *pcie, + u8 busnr, u8 fn, + u32 r, u64 cpu_addr) +{ + u32 addr0, addr1, desc0, desc1; + + desc0 = CDNS_PCIE_AT_OB_REGION_DESC0_TYPE_NORMAL_MSG; + desc1 = 0; + + /* See cdns_pcie_set_outbound_region() comments above. */ + if (pcie->is_rc) { + desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_HARDCODED_RID | + CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN(0); + desc1 |= CDNS_PCIE_AT_OB_REGION_DESC1_BUS(busnr); + } else { + desc0 |= CDNS_PCIE_AT_OB_REGION_DESC0_DEVFN(fn); + } + + /* Set the CPU address */ + if (pcie->ops && pcie->ops->cpu_addr_fixup) + cpu_addr = pcie->ops->cpu_addr_fixup(pcie, cpu_addr); + + addr0 = CDNS_PCIE_AT_OB_REGION_CPU_ADDR0_NBITS(17) | + (lower_32_bits(cpu_addr) & GENMASK(31, 8)); + addr1 = upper_32_bits(cpu_addr); + + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR0(r), 0); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR1(r), 0); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC0(r), desc0); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(r), desc1); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR0(r), addr0); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR1(r), addr1); +} +EXPORT_SYMBOL_GPL(cdns_pcie_set_outbound_region_for_normal_msg); + +void cdns_pcie_reset_outbound_region(struct cdns_pcie *pcie, u32 r) +{ + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR0(r), 0); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_PCI_ADDR1(r), 0); + + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC0(r), 0); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_DESC1(r), 0); + + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR0(r), 0); + cdns_pcie_writel(pcie, CDNS_PCIE_AT_OB_REGION_CPU_ADDR1(r), 0); +} +EXPORT_SYMBOL_GPL(cdns_pcie_reset_outbound_region); + +void cdns_pcie_disable_phy(struct cdns_pcie *pcie) +{ + int i = pcie->phy_count; + + while (i--) { + phy_power_off(pcie->phy[i]); + phy_exit(pcie->phy[i]); + } +} +EXPORT_SYMBOL_GPL(cdns_pcie_disable_phy); + +int cdns_pcie_enable_phy(struct cdns_pcie *pcie) +{ + int ret; + int i; + + for (i = 0; i < pcie->phy_count; i++) { + ret = phy_init(pcie->phy[i]); + if (ret < 0) + goto err_phy; + + ret = phy_power_on(pcie->phy[i]); + if (ret < 0) { + phy_exit(pcie->phy[i]); + goto err_phy; + } + } + + return 0; + +err_phy: + while (--i >= 0) { + phy_power_off(pcie->phy[i]); + phy_exit(pcie->phy[i]); + } + + return ret; +} +EXPORT_SYMBOL_GPL(cdns_pcie_enable_phy); + +int cdns_pcie_init_phy(struct device *dev, struct cdns_pcie *pcie) +{ + struct device_node *np = dev->of_node; + int phy_count; + struct phy **phy; + struct device_link **link; + int i; + int ret; + const char *name; + + phy_count = of_property_count_strings(np, "phy-names"); + if (phy_count < 1) { + dev_info(dev, "no \"phy-names\" property found; PHY will not be initialized\n"); + pcie->phy_count = 0; + return 0; + } + + phy = devm_kcalloc(dev, phy_count, sizeof(*phy), GFP_KERNEL); + if (!phy) + return -ENOMEM; + + link = devm_kcalloc(dev, phy_count, sizeof(*link), GFP_KERNEL); + if (!link) + return -ENOMEM; + + for (i = 0; i < phy_count; i++) { + of_property_read_string_index(np, "phy-names", i, &name); + phy[i] = devm_phy_get(dev, name); + if (IS_ERR(phy[i])) { + ret = PTR_ERR(phy[i]); + goto err_phy; + } + link[i] = device_link_add(dev, &phy[i]->dev, DL_FLAG_STATELESS); + if (!link[i]) { + devm_phy_put(dev, phy[i]); + ret = -EINVAL; + goto err_phy; + } + } + + pcie->phy_count = phy_count; + pcie->phy = phy; + pcie->link = link; + + ret = cdns_pcie_enable_phy(pcie); + if (ret) + goto err_phy; + + return 0; + +err_phy: + while (--i >= 0) { + device_link_del(link[i]); + devm_phy_put(dev, phy[i]); + } + + return ret; +} +EXPORT_SYMBOL_GPL(cdns_pcie_init_phy); + +static int cdns_pcie_suspend_noirq(struct device *dev) +{ + struct cdns_pcie *pcie = dev_get_drvdata(dev); + + cdns_pcie_disable_phy(pcie); + + return 0; +} + +static int cdns_pcie_resume_noirq(struct device *dev) +{ + struct cdns_pcie *pcie = dev_get_drvdata(dev); + int ret; + + ret = cdns_pcie_enable_phy(pcie); + if (ret) { + dev_err(dev, "failed to enable PHY\n"); + return ret; + } + + return 0; +} + +const struct dev_pm_ops cdns_pcie_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(cdns_pcie_suspend_noirq, + cdns_pcie_resume_noirq) +}; +EXPORT_SYMBOL_GPL(cdns_pcie_pm_ops); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Cadence PCIe controller driver"); +MODULE_AUTHOR("Cyrille Pitchen <cyrille.pitchen@free-electrons.com>"); diff --git a/drivers/pci/controller/cadence/pcie-cadence.h b/drivers/pci/controller/cadence/pcie-cadence.h new file mode 100644 index 000000000..b4d51adef --- /dev/null +++ b/drivers/pci/controller/cadence/pcie-cadence.h @@ -0,0 +1,736 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +// Copyright (c) 2017 Cadence +// Cadence PCIe controller driver. +// Author: Cyrille Pitchen <cyrille.pitchen@free-electrons.com> + +#ifndef _PCIE_CADENCE_H +#define _PCIE_CADENCE_H + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/pci-epf.h> +#include <linux/phy/phy.h> +#include "pcie-cadence-lga-regs.h" +#include "pcie-cadence-hpa-regs.h" + +#define CDNS_PCIE_LGA_LTSSM_STATUS_MASK GENMASK(29, 24) +#define CDNS_PCIE_HPA_LTSSM_STATUS_MASK GENMASK(27, 20) + +enum cdns_pcie_rp_bar { + RP_BAR_UNDEFINED = -1, + RP_BAR0, + RP_BAR1, + RP_NO_BAR +}; + +struct cdns_pcie_rp_ib_bar { + u64 size; + bool free; +}; + +struct cdns_pcie; +struct cdns_pcie_rc; + +enum cdns_pcie_reg_bank { + REG_BANK_RP, + REG_BANK_IP_REG, + REG_BANK_IP_CFG_CTRL_REG, + REG_BANK_AXI_MASTER_COMMON, + REG_BANK_AXI_MASTER, + REG_BANK_AXI_SLAVE, + REG_BANK_AXI_HLS, + REG_BANK_AXI_RAS, + REG_BANK_AXI_DTI, + REG_BANKS_MAX, +}; + +enum cdns_pcie_lga_ltssm { + CDNS_PCIE_LGA_LTSSM_DETECT_QUIET = 0x00, + CDNS_PCIE_LGA_LTSSM_DETECT_ACTIVE = 0x01, + CDNS_PCIE_LGA_LTSSM_POLLING_ACTIVE = 0x02, + CDNS_PCIE_LGA_LTSSM_POLLING_COMPLIANCE = 0x03, + CDNS_PCIE_LGA_LTSSM_POLLING_CONFIGURATION = 0x04, + CDNS_PCIE_LGA_LTSSM_CONFIGURATION_LINKWIDTH_START = 0x05, + CDNS_PCIE_LGA_LTSSM_CONFIGURATION_LINKWIDTH_ACCEPT = 0x06, + CDNS_PCIE_LGA_LTSSM_CONFIGURATION_LANENUM_ACCEPT = 0x07, + CDNS_PCIE_LGA_LTSSM_CONFIGURATION_LANENUM_WAIT = 0x08, + CDNS_PCIE_LGA_LTSSM_CONFIGURATION_COMPLETE = 0x09, + CDNS_PCIE_LGA_LTSSM_CONFIGURATION_IDLE = 0x0A, + CDNS_PCIE_LGA_LTSSM_RECOVERY_RCVRLOCK = 0x0B, + CDNS_PCIE_LGA_LTSSM_RECOVERY_SPEED = 0x0C, + CDNS_PCIE_LGA_LTSSM_RECOVERY_RCVRCFG = 0x0D, + CDNS_PCIE_LGA_LTSSM_RECOVERY_IDLE = 0x0E, + CDNS_PCIE_LGA_LTSSM_L0 = 0x10, + CDNS_PCIE_LGA_LTSSM_RX_L0S_ENTRY = 0x11, + CDNS_PCIE_LGA_LTSSM_RX_L0S_IDLE = 0x12, + CDNS_PCIE_LGA_LTSSM_RX_L0S_FTS = 0x13, + CDNS_PCIE_LGA_LTSSM_TX_L0S_ENTRY = 0x14, + CDNS_PCIE_LGA_LTSSM_TX_L0S_IDLE = 0x15, + CDNS_PCIE_LGA_LTSSM_TX_L0S_FTS = 0x16, + CDNS_PCIE_LGA_LTSSM_L1_ENTRY = 0x17, + CDNS_PCIE_LGA_LTSSM_L1_IDLE = 0x18, + CDNS_PCIE_LGA_LTSSM_L2_IDLE = 0x19, + CDNS_PCIE_LGA_LTSSM_L2_TRANSMITWAKE = 0x1A, + CDNS_PCIE_LGA_LTSSM_DISABLED = 0x20, + CDNS_PCIE_LGA_LTSSM_LOOPBACK_ENTRY_MASTER = 0x21, + CDNS_PCIE_LGA_LTSSM_LOOPBACK_ACTIVE_MASTER = 0x22, + CDNS_PCIE_LGA_LTSSM_LOOPBACK_EXIT_MASTER = 0x23, + CDNS_PCIE_LGA_LTSSM_LOOPBACK_ENTRY_SLAVE = 0x24, + CDNS_PCIE_LGA_LTSSM_LOOPBACK_ACTIVE_SLAVE = 0x25, + CDNS_PCIE_LGA_LTSSM_LOOPBACK_EXIT_SLAVE = 0x26, + CDNS_PCIE_LGA_LTSSM_HOT_RESET = 0x27, + CDNS_PCIE_LGA_LTSSM_RECOVERY_EQUALIZATION_PHASE_0 = 0x28, + CDNS_PCIE_LGA_LTSSM_RECOVERY_EQUALIZATION_PHASE_1 = 0x29, + CDNS_PCIE_LGA_LTSSM_RECOVERY_EQUALIZATION_PHASE_2 = 0x2A, + CDNS_PCIE_LGA_LTSSM_RECOVERY_EQUALIZATION_PHASE_3 = 0x2B, + CDNS_PCIE_LGA_LTSSM_UNKNOWN = 0xFFFFFFFF, +}; + +enum cdns_pcie_hpa_ltssm { + CDNS_PCIE_HPA_LTSSM_DETECT_QUIET = 0, + CDNS_PCIE_HPA_LTSSM_DETECT_QUIET_ENTRY = 1, + CDNS_PCIE_HPA_LTSSM_DETECT_ACTIVE = 2, + CDNS_PCIE_HPA_LTSSM_DETECT_ACTIVE_1 = 3, + CDNS_PCIE_HPA_LTSSM_DETECT_ACTIVE_2 = 4, + CDNS_PCIE_HPA_LTSSM_DETECT_ACTIVE_3 = 5, + CDNS_PCIE_HPA_LTSSM_RCVR_DETECTED_ST = 6, + CDNS_PCIE_HPA_LTSSM_RCVR_DETECTED_1 = 7, + CDNS_PCIE_HPA_LTSSM_POLLING_ACTIVE = 8, + CDNS_PCIE_HPA_LTSSM_POLLING_ACTIVE_1 = 9, + CDNS_PCIE_HPA_LTSSM_POLLING_ACTIVE_2 = 10, + CDNS_PCIE_HPA_LTSSM_POLLING_ACTIVE_3 = 11, + CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE = 12, + CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_1 = 13, + CDNS_PCIE_HPA_LTSSM_POLLING_CONFIG = 14, + CDNS_PCIE_HPA_LTSSM_POLLING_CONFIG_1 = 15, + CDNS_PCIE_HPA_LTSSM_POLLING_CONFIG_2 = 16, + CDNS_PCIE_HPA_LTSSM_CONFIG_LW_START_RC = 17, + CDNS_PCIE_HPA_LTSSM_CONFIG_LW_START_RC_1 = 18, + CDNS_PCIE_HPA_LTSSM_CONFIG_LW_START_RC_2 = 19, + CDNS_PCIE_HPA_LTSSM_CONFIG_LW_ACC_RC = 20, + CDNS_PCIE_HPA_LTSSM_CONFIG_LANENUM_WAIT_RC = 21, + CDNS_PCIE_HPA_LTSSM_CONFIG_LANENUM_WAIT_RC_1 = 22, + CDNS_PCIE_HPA_LTSSM_CONFIG_LANENUM_ACC_RC = 23, + CDNS_PCIE_HPA_LTSSM_CONFIG_LW_START_EP = 24, + CDNS_PCIE_HPA_LTSSM_CONFIG_LW_START_EP_1 = 25, + CDNS_PCIE_HPA_LTSSM_CONFIG_LW_START_EP_2 = 26, + CDNS_PCIE_HPA_LTSSM_CONFIG_LW_ACC_EP = 27, + CDNS_PCIE_HPA_LTSSM_CONFIG_LANENUM_WAIT_EP = 28, + CDNS_PCIE_HPA_LTSSM_CONFIG_LANENUM_WAIT_EP_1 = 29, + CDNS_PCIE_HPA_LTSSM_CONFIG_LANENUM_ACC_EP = 30, + CDNS_PCIE_HPA_LTSSM_CONFIG_LANENUM_ACC_EP_1 = 31, + CDNS_PCIE_HPA_LTSSM_DUMMY_STATE_1 = 32, + CDNS_PCIE_HPA_LTSSM_CONFIG_COMPLETE = 33, + CDNS_PCIE_HPA_LTSSM_CONFIG_COMPLETE_1 = 34, + CDNS_PCIE_HPA_LTSSM_CONFIG_COMPLETE_2 = 35, + CDNS_PCIE_HPA_LTSSM_CONFIG_IDLE = 36, + CDNS_PCIE_HPA_LTSSM_CONFIG_IDLE_1 = 37, + CDNS_PCIE_HPA_LTSSM_DUMMY_STATE_2 = 38, + CDNS_PCIE_HPA_LTSSM_DUMMY_STATE_3 = 39, + CDNS_PCIE_HPA_LTSSM_DUMMY_STATE_4 = 40, + CDNS_PCIE_HPA_LTSSM_L0_STATE = 41, + CDNS_PCIE_HPA_LTSSM_RECOVERY_RCVR_LOCK = 42, + CDNS_PCIE_HPA_LTSSM_RECOVERY_RCVR_LOCK_1 = 43, + CDNS_PCIE_HPA_LTSSM_RECOVERY_RCVR_CFG = 44, + CDNS_PCIE_HPA_LTSSM_RECOVERY_RCVR_CFG_1 = 45, + CDNS_PCIE_HPA_LTSSM_RECOVERY_IDLE = 46, + CDNS_PCIE_HPA_LTSSM_RECOVERY_IDLE_1 = 47, + CDNS_PCIE_HPA_LTSSM_DISABLE_LINK = 48, + CDNS_PCIE_HPA_LTSSM_DISABLE_LINK_1 = 49, + CDNS_PCIE_HPA_LTSSM_DISABLE_LINK_2 = 50, + CDNS_PCIE_HPA_LTSSM_DISABLE_LINK_3 = 51, + CDNS_PCIE_HPA_LTSSM_DISABLE_LINK_4 = 52, + CDNS_PCIE_HPA_LTSSM_DISABLE_LINK_5 = 53, + CDNS_PCIE_HPA_LTSSM_DISABLE_LINK_6 = 54, + CDNS_PCIE_HPA_LTSSM_DISABLE_LINK_7 = 55, + CDNS_PCIE_HPA_LTSSM_HOT_RESET = 56, + CDNS_PCIE_HPA_LTSSM_HOT_RESET_1 = 57, + CDNS_PCIE_HPA_LTSSM_HOT_RESET_2 = 58, + CDNS_PCIE_HPA_LTSSM_HOT_RESET_3 = 59, + CDNS_PCIE_HPA_LTSSM_L0S_ENTRY = 60, + CDNS_PCIE_HPA_LTSSM_L0S_1 = 61, + CDNS_PCIE_HPA_LTSSM_L0S_2 = 62, + CDNS_PCIE_HPA_LTSSM_L0S_3 = 63, + CDNS_PCIE_HPA_LTSSM_L0S_4 = 64, + CDNS_PCIE_HPA_LTSSM_L0S_5 = 65, + CDNS_PCIE_HPA_LTSSM_WAIT_FOR_LINK_TX = 66, + CDNS_PCIE_HPA_LTSSM_TX_FTS_ENTRY = 67, + CDNS_PCIE_HPA_LTSSM_TX_FTS_1 = 68, + CDNS_PCIE_HPA_LTSSM_TX_FTS_2 = 69, + CDNS_PCIE_HPA_LTSSM_TX_ELEC_IDLE_ST = 70, + CDNS_PCIE_HPA_LTSSM_TX_ELEC_IDLE_1 = 71, + CDNS_PCIE_HPA_LTSSM_TX_ELEC_IDLE_2 = 72, + CDNS_PCIE_HPA_LTSSM_TX_ELEC_IDLE_3 = 73, + CDNS_PCIE_HPA_LTSSM_RECOVERY_SPEED = 74, + CDNS_PCIE_HPA_LTSSM_RECOVERY_SPEED_1 = 75, + CDNS_PCIE_HPA_LTSSM_RECOVERY_SPEED_2 = 76, + CDNS_PCIE_HPA_LTSSM_RECOVERY_SPEED_3 = 77, + CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_GEN23 = 78, + CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_GEN23_1 = 79, + CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_GEN23_2 = 80, + CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_GEN23_3 = 81, + CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_GEN23_4 = 82, + CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_GEN23_5 = 83, + CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_GEN23_6 = 84, + CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_GEN23_7 = 85, + CDNS_PCIE_HPA_LTSSM_POLLING_COMPLIANCE_GEN23_8 = 86, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_ENTRY = 87, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_ENTRY_FROM_RECOVERY = 88, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_EXIT_1 = 89, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_EXIT = 90, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_GEN2_1 = 91, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_GEN2_2 = 92, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_GEN2_3 = 93, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_GEN2_4 = 94, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_GEN2_5 = 95, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_SLAVE_ACTIVE = 96, + CDNS_PCIE_HPA_LTSSM_L1_ENTRY = 97, + CDNS_PCIE_HPA_LTSSM_L1_1 = 98, + CDNS_PCIE_HPA_LTSSM_L1_2 = 99, + CDNS_PCIE_HPA_LTSSM_L1_3 = 100, + CDNS_PCIE_HPA_LTSSM_L1_4 = 101, + CDNS_PCIE_HPA_LTSSM_L1_IDLE = 102, + CDNS_PCIE_HPA_LTSSM_L1_EXIT = 103, + CDNS_PCIE_HPA_LTSSM_L2_ENTRY = 104, + CDNS_PCIE_HPA_LTSSM_L2_1 = 105, + CDNS_PCIE_HPA_LTSSM_L2_2 = 106, + CDNS_PCIE_HPA_LTSSM_L2_3 = 107, + CDNS_PCIE_HPA_LTSSM_L2_4 = 108, + CDNS_PCIE_HPA_LTSSM_L2_5 = 109, + CDNS_PCIE_HPA_LTSSM_L2_IDLE = 110, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_ENTRY = 111, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_ENTRY_1 = 112, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_ENTRY_2 = 113, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_ENTRY_3 = 114, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_ENTRY_4 = 115, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_ENTRY_5 = 116, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_ENTRY_FROM_RECOVERY = 117, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_ACTIVE = 118, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_EXIT = 119, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_EXIT_1 = 120, + CDNS_PCIE_HPA_LTSSM_LOOPBACK_MASTER_EXIT_2 = 121, + CDNS_PCIE_HPA_LTSSM_RECOVERY_EQUALIZATION_PHASE0 = 122, + CDNS_PCIE_HPA_LTSSM_RECOVERY_EQUALIZATION_PHASE1 = 123, + CDNS_PCIE_HPA_LTSSM_RECOVERY_EQUALIZATION_PHASE2_1 = 124, + CDNS_PCIE_HPA_LTSSM_RECOVERY_EQUALIZATION_PHASE2_2 = 125, + CDNS_PCIE_HPA_LTSSM_RECOVERY_EQUALIZATION_PHASE3_1 = 126, + CDNS_PCIE_HPA_LTSSM_RECOVERY_EQUALIZATION_PHASE3_2 = 127, + CDNS_PCIE_HPA_LTSSM_UNKNOWN = 0xFFFFFFFF, +}; + +struct cdns_pcie_ops { + int (*start_link)(struct cdns_pcie *pcie); + void (*stop_link)(struct cdns_pcie *pcie); + bool (*link_up)(struct cdns_pcie *pcie); + u64 (*cpu_addr_fixup)(struct cdns_pcie *pcie, u64 cpu_addr); +}; + +/** + * struct cdns_plat_pcie_of_data - Register bank offset for a platform + * @is_rc: controller is a RC + * @ip_reg_bank_offset: ip register bank start offset + * @ip_cfg_ctrl_reg_offset: ip config control register start offset + * @axi_mstr_common_offset: AXI master common register start offset + * @axi_slave_offset: AXI slave start offset + * @axi_master_offset: AXI master start offset + * @axi_hls_offset: AXI HLS offset start + * @axi_ras_offset: AXI RAS offset + * @axi_dti_offset: AXI DTI offset + */ +struct cdns_plat_pcie_of_data { + u32 is_rc:1; + u32 ip_reg_bank_offset; + u32 ip_cfg_ctrl_reg_offset; + u32 axi_mstr_common_offset; + u32 axi_slave_offset; + u32 axi_master_offset; + u32 axi_hls_offset; + u32 axi_ras_offset; + u32 axi_dti_offset; +}; + +/** + * struct cdns_pcie - private data for Cadence PCIe controller drivers + * @reg_base: IO mapped register base + * @mem_res: start/end offsets in the physical system memory to map PCI accesses + * @msg_res: Region for send message to map PCI accesses + * @dev: PCIe controller + * @is_rc: tell whether the PCIe controller mode is Root Complex or Endpoint. + * @is_hpa: indicates if the architecture is HPA + * @phy_count: number of supported PHY devices + * @phy: list of pointers to specific PHY control blocks + * @link: list of pointers to corresponding device link representations + * @ops: Platform-specific ops to control various inputs from Cadence PCIe + * wrapper + * @cdns_pcie_reg_offsets: Register bank offsets for different SoC + * @max_link_speed: Maximum supported link speed + * @debug_dir: debugfs node + */ +struct cdns_pcie { + void __iomem *reg_base; + struct resource *mem_res; + struct resource *msg_res; + struct device *dev; + bool is_rc; + bool is_hpa; + int phy_count; + struct phy **phy; + struct device_link **link; + const struct cdns_pcie_ops *ops; + const struct cdns_plat_pcie_of_data *cdns_pcie_reg_offsets; + int max_link_speed; + struct dentry *debug_dir; +}; + +/** + * struct cdns_pcie_rc - private data for this PCIe Root Complex driver + * @pcie: Cadence PCIe controller + * @cfg_res: start/end offsets in the physical system memory to map PCI + * configuration space accesses + * @cfg_base: IO mapped window to access the PCI configuration space of a + * single function at a time + * @vendor_id: PCI vendor ID + * @device_id: PCI device ID + * @avail_ib_bar: Status of RP_BAR0, RP_BAR1 and RP_NO_BAR if it's free or + * available + * @quirk_retrain_flag: Retrain link as quirk for PCIe Gen2 + * @quirk_detect_quiet_flag: LTSSM Detect Quiet min delay set as quirk + * @ecam_supported: Whether the ECAM is supported + * @no_inbound_map: Whether inbound mapping is supported + * @quirk_broken_aspm_l0s: Disable ASPM L0s support as quirk + * @quirk_broken_aspm_l1: Disable ASPM L1 support as quirk + */ +struct cdns_pcie_rc { + struct cdns_pcie pcie; + struct resource *cfg_res; + void __iomem *cfg_base; + u32 vendor_id; + u32 device_id; + bool avail_ib_bar[CDNS_PCIE_RP_MAX_IB]; + unsigned int quirk_retrain_flag:1; + unsigned int quirk_detect_quiet_flag:1; + unsigned int ecam_supported:1; + unsigned int no_inbound_map:1; + unsigned int quirk_broken_aspm_l0s:1; + unsigned int quirk_broken_aspm_l1:1; +}; + +/** + * struct cdns_pcie_epf - Structure to hold info about endpoint function + * @epf: Info about virtual functions attached to the physical function + * @epf_bar: reference to the pci_epf_bar for the six Base Address Registers + */ +struct cdns_pcie_epf { + struct cdns_pcie_epf *epf; + struct pci_epf_bar *epf_bar[PCI_STD_NUM_BARS]; +}; + +/** + * struct cdns_pcie_ep - private data for this PCIe endpoint controller driver + * @pcie: Cadence PCIe controller + * @max_regions: maximum number of regions supported by hardware + * @ob_region_map: bitmask of mapped outbound regions + * @ob_addr: base addresses in the AXI bus where the outbound regions start + * @irq_phys_addr: base address on the AXI bus where the MSI/INTX IRQ + * dedicated outbound regions is mapped. + * @irq_cpu_addr: base address in the CPU space where a write access triggers + * the sending of a memory write (MSI) / normal message (INTX + * IRQ) TLP through the PCIe bus. + * @irq_pci_addr: used to save the current mapping of the MSI/INTX IRQ + * dedicated outbound region. + * @irq_pci_fn: the latest PCI function that has updated the mapping of + * the MSI/INTX IRQ dedicated outbound region. + * @irq_pending: bitmask of asserted INTX IRQs. + * @lock: spin lock to disable interrupts while modifying PCIe controller + * registers fields (RMW) accessible by both remote RC and EP to + * minimize time between read and write + * @epf: Structure to hold info about endpoint function + * @quirk_detect_quiet_flag: LTSSM Detect Quiet min delay set as quirk + * @quirk_disable_flr: Disable FLR (Function Level Reset) quirk flag + */ +struct cdns_pcie_ep { + struct cdns_pcie pcie; + u32 max_regions; + unsigned long ob_region_map; + phys_addr_t *ob_addr; + phys_addr_t irq_phys_addr; + void __iomem *irq_cpu_addr; + u64 irq_pci_addr; + u8 irq_pci_fn; + u8 irq_pending; + /* protect writing to PCI_STATUS while raising INTX interrupts */ + spinlock_t lock; + struct cdns_pcie_epf *epf; + unsigned int quirk_detect_quiet_flag:1; + unsigned int quirk_disable_flr:1; +}; + +static inline u32 cdns_reg_bank_to_off(struct cdns_pcie *pcie, enum cdns_pcie_reg_bank bank) +{ + u32 offset = 0x0; + + switch (bank) { + case REG_BANK_RP: + offset = 0; + break; + case REG_BANK_IP_REG: + offset = pcie->cdns_pcie_reg_offsets->ip_reg_bank_offset; + break; + case REG_BANK_IP_CFG_CTRL_REG: + offset = pcie->cdns_pcie_reg_offsets->ip_cfg_ctrl_reg_offset; + break; + case REG_BANK_AXI_MASTER_COMMON: + offset = pcie->cdns_pcie_reg_offsets->axi_mstr_common_offset; + break; + case REG_BANK_AXI_MASTER: + offset = pcie->cdns_pcie_reg_offsets->axi_master_offset; + break; + case REG_BANK_AXI_SLAVE: + offset = pcie->cdns_pcie_reg_offsets->axi_slave_offset; + break; + case REG_BANK_AXI_HLS: + offset = pcie->cdns_pcie_reg_offsets->axi_hls_offset; + break; + case REG_BANK_AXI_RAS: + offset = pcie->cdns_pcie_reg_offsets->axi_ras_offset; + break; + case REG_BANK_AXI_DTI: + offset = pcie->cdns_pcie_reg_offsets->axi_dti_offset; + break; + default: + break; + } + return offset; +} + +/* Register access */ +static inline void cdns_pcie_writel(struct cdns_pcie *pcie, u32 reg, u32 value) +{ + writel(value, pcie->reg_base + reg); +} + +static inline u32 cdns_pcie_readl(struct cdns_pcie *pcie, u32 reg) +{ + return readl(pcie->reg_base + reg); +} + +static inline void cdns_pcie_hpa_writel(struct cdns_pcie *pcie, + enum cdns_pcie_reg_bank bank, + u32 reg, + u32 value) +{ + u32 offset = cdns_reg_bank_to_off(pcie, bank); + + reg += offset; + writel(value, pcie->reg_base + reg); +} + +static inline u32 cdns_pcie_hpa_readl(struct cdns_pcie *pcie, + enum cdns_pcie_reg_bank bank, + u32 reg) +{ + u32 offset = cdns_reg_bank_to_off(pcie, bank); + + reg += offset; + return readl(pcie->reg_base + reg); +} + +static inline u32 cdns_pcie_read_sz(void __iomem *addr, int size) +{ + void __iomem *aligned_addr = PTR_ALIGN_DOWN(addr, 0x4); + unsigned int offset = (unsigned long)addr & 0x3; + u32 val = readl(aligned_addr); + + if (!IS_ALIGNED((uintptr_t)addr, size)) { + pr_warn("Address %p and size %d are not aligned\n", addr, size); + return 0; + } + + if (size > 2) + return val; + + return (val >> (8 * offset)) & ((1 << (size * 8)) - 1); +} + +static inline void cdns_pcie_write_sz(void __iomem *addr, int size, u32 value) +{ + void __iomem *aligned_addr = PTR_ALIGN_DOWN(addr, 0x4); + unsigned int offset = (unsigned long)addr & 0x3; + u32 mask; + u32 val; + + if (!IS_ALIGNED((uintptr_t)addr, size)) { + pr_warn("Address %p and size %d are not aligned\n", addr, size); + return; + } + + if (size > 2) { + writel(value, addr); + return; + } + + mask = ~(((1 << (size * 8)) - 1) << (offset * 8)); + val = readl(aligned_addr) & mask; + val |= value << (offset * 8); + writel(val, aligned_addr); +} + +static inline int cdns_pcie_read_cfg_byte(struct cdns_pcie *pcie, int where, + u8 *val) +{ + void __iomem *addr = pcie->reg_base + where; + + *val = cdns_pcie_read_sz(addr, 0x1); + return PCIBIOS_SUCCESSFUL; +} + +static inline int cdns_pcie_read_cfg_word(struct cdns_pcie *pcie, int where, + u16 *val) +{ + void __iomem *addr = pcie->reg_base + where; + + *val = cdns_pcie_read_sz(addr, 0x2); + return PCIBIOS_SUCCESSFUL; +} + +static inline int cdns_pcie_read_cfg_dword(struct cdns_pcie *pcie, int where, + u32 *val) +{ + *val = cdns_pcie_readl(pcie, where); + return PCIBIOS_SUCCESSFUL; +} + +/* Root Port register access */ +static inline void cdns_pcie_rp_writeb(struct cdns_pcie *pcie, + u32 reg, u8 value) +{ + void __iomem *addr = pcie->reg_base + CDNS_PCIE_RP_BASE + reg; + + cdns_pcie_write_sz(addr, 0x1, value); +} + +static inline void cdns_pcie_rp_writew(struct cdns_pcie *pcie, + u32 reg, u16 value) +{ + void __iomem *addr = pcie->reg_base + CDNS_PCIE_RP_BASE + reg; + + cdns_pcie_write_sz(addr, 0x2, value); +} + +static inline u16 cdns_pcie_rp_readw(struct cdns_pcie *pcie, u32 reg) +{ + void __iomem *addr = pcie->reg_base + CDNS_PCIE_RP_BASE + reg; + + return cdns_pcie_read_sz(addr, 0x2); +} + +static inline void cdns_pcie_rp_writel(struct cdns_pcie *pcie, + u32 reg, u32 value) +{ + void __iomem *addr = pcie->reg_base + CDNS_PCIE_RP_BASE + reg; + + cdns_pcie_write_sz(addr, 0x4, value); +} + +static inline u32 cdns_pcie_rp_readl(struct cdns_pcie *pcie, u32 reg) +{ + void __iomem *addr = pcie->reg_base + CDNS_PCIE_RP_BASE + reg; + + return cdns_pcie_read_sz(addr, 0x4); +} + +static inline void cdns_pcie_hpa_rp_writeb(struct cdns_pcie *pcie, + u32 reg, u8 value) +{ + void __iomem *addr = pcie->reg_base + CDNS_PCIE_HPA_RP_BASE + reg; + + cdns_pcie_write_sz(addr, 0x1, value); +} + +static inline void cdns_pcie_hpa_rp_writew(struct cdns_pcie *pcie, + u32 reg, u16 value) +{ + void __iomem *addr = pcie->reg_base + CDNS_PCIE_HPA_RP_BASE + reg; + + cdns_pcie_write_sz(addr, 0x2, value); +} + +static inline u16 cdns_pcie_hpa_rp_readw(struct cdns_pcie *pcie, u32 reg) +{ + void __iomem *addr = pcie->reg_base + CDNS_PCIE_HPA_RP_BASE + reg; + + return cdns_pcie_read_sz(addr, 0x2); +} + +/* Endpoint Function register access */ +static inline void cdns_pcie_ep_fn_writeb(struct cdns_pcie *pcie, u8 fn, + u32 reg, u8 value) +{ + void __iomem *addr = pcie->reg_base + CDNS_PCIE_EP_FUNC_BASE(fn) + reg; + + cdns_pcie_write_sz(addr, 0x1, value); +} + +static inline void cdns_pcie_ep_fn_writew(struct cdns_pcie *pcie, u8 fn, + u32 reg, u16 value) +{ + void __iomem *addr = pcie->reg_base + CDNS_PCIE_EP_FUNC_BASE(fn) + reg; + + cdns_pcie_write_sz(addr, 0x2, value); +} + +static inline void cdns_pcie_ep_fn_writel(struct cdns_pcie *pcie, u8 fn, + u32 reg, u32 value) +{ + writel(value, pcie->reg_base + CDNS_PCIE_EP_FUNC_BASE(fn) + reg); +} + +static inline u16 cdns_pcie_ep_fn_readw(struct cdns_pcie *pcie, u8 fn, u32 reg) +{ + void __iomem *addr = pcie->reg_base + CDNS_PCIE_EP_FUNC_BASE(fn) + reg; + + return cdns_pcie_read_sz(addr, 0x2); +} + +static inline u32 cdns_pcie_ep_fn_readl(struct cdns_pcie *pcie, u8 fn, u32 reg) +{ + return readl(pcie->reg_base + CDNS_PCIE_EP_FUNC_BASE(fn) + reg); +} + +static inline int cdns_pcie_start_link(struct cdns_pcie *pcie) +{ + if (pcie->ops && pcie->ops->start_link) + return pcie->ops->start_link(pcie); + + return 0; +} + +static inline void cdns_pcie_stop_link(struct cdns_pcie *pcie) +{ + if (pcie->ops && pcie->ops->stop_link) + pcie->ops->stop_link(pcie); +} + +static inline bool cdns_pcie_link_up(struct cdns_pcie *pcie) +{ + if (pcie->ops && pcie->ops->link_up) + return pcie->ops->link_up(pcie); + + return true; +} + +#if IS_ENABLED(CONFIG_PCIE_CADENCE_HOST) +int cdns_pcie_host_link_setup(struct cdns_pcie_rc *rc); +int cdns_pcie_host_init(struct cdns_pcie_rc *rc); +int cdns_pcie_host_setup(struct cdns_pcie_rc *rc); +void cdns_pcie_host_disable(struct cdns_pcie_rc *rc); +void __iomem *cdns_pci_map_bus(struct pci_bus *bus, unsigned int devfn, + int where); +int cdns_pcie_hpa_host_setup(struct cdns_pcie_rc *rc); +void cdns_pcie_hpa_host_disable(struct cdns_pcie_rc *rc); +#else +static inline int cdns_pcie_host_link_setup(struct cdns_pcie_rc *rc) +{ + return 0; +} + +static inline int cdns_pcie_host_init(struct cdns_pcie_rc *rc) +{ + return 0; +} + +static inline int cdns_pcie_host_setup(struct cdns_pcie_rc *rc) +{ + return 0; +} + +static inline int cdns_pcie_hpa_host_setup(struct cdns_pcie_rc *rc) +{ + return 0; +} + +static inline void cdns_pcie_host_disable(struct cdns_pcie_rc *rc) +{ +} + +static inline void cdns_pcie_hpa_host_disable(struct cdns_pcie_rc *rc) +{ +} + +static inline void __iomem *cdns_pci_map_bus(struct pci_bus *bus, unsigned int devfn, + int where) +{ + return NULL; +} +#endif + +#if IS_ENABLED(CONFIG_PCIE_CADENCE_EP) +int cdns_pcie_ep_setup(struct cdns_pcie_ep *ep); +void cdns_pcie_ep_disable(struct cdns_pcie_ep *ep); +int cdns_pcie_hpa_ep_setup(struct cdns_pcie_ep *ep); +#else +static inline int cdns_pcie_ep_setup(struct cdns_pcie_ep *ep) +{ + return 0; +} + +static inline void cdns_pcie_ep_disable(struct cdns_pcie_ep *ep) +{ +} + +static inline int cdns_pcie_hpa_ep_setup(struct cdns_pcie_ep *ep) +{ + return 0; +} + +#endif + +u8 cdns_pcie_find_capability(struct cdns_pcie *pcie, u8 cap); +u16 cdns_pcie_find_ext_capability(struct cdns_pcie *pcie, u8 cap); +bool cdns_pcie_linkup(struct cdns_pcie *pcie); + +void cdns_pcie_detect_quiet_min_delay_set(struct cdns_pcie *pcie); + +void cdns_pcie_set_outbound_region(struct cdns_pcie *pcie, u8 busnr, u8 fn, + u32 r, bool is_io, + u64 cpu_addr, u64 pci_addr, size_t size); + +void cdns_pcie_set_outbound_region_for_normal_msg(struct cdns_pcie *pcie, + u8 busnr, u8 fn, + u32 r, u64 cpu_addr); + +void cdns_pcie_reset_outbound_region(struct cdns_pcie *pcie, u32 r); +void cdns_pcie_disable_phy(struct cdns_pcie *pcie); +int cdns_pcie_enable_phy(struct cdns_pcie *pcie); +int cdns_pcie_init_phy(struct device *dev, struct cdns_pcie *pcie); +void cdns_pcie_hpa_detect_quiet_min_delay_set(struct cdns_pcie *pcie); +void cdns_pcie_hpa_set_outbound_region(struct cdns_pcie *pcie, u8 busnr, u8 fn, + u32 r, bool is_io, + u64 cpu_addr, u64 pci_addr, size_t size); +void cdns_pcie_hpa_set_outbound_region_for_normal_msg(struct cdns_pcie *pcie, + u8 busnr, u8 fn, + u32 r, u64 cpu_addr); +int cdns_pcie_hpa_host_link_setup(struct cdns_pcie_rc *rc); +void __iomem *cdns_pci_hpa_map_bus(struct pci_bus *bus, unsigned int devfn, + int where); +int cdns_pcie_hpa_host_start_link(struct cdns_pcie_rc *rc); +int cdns_pcie_hpa_start_link(struct cdns_pcie *pcie); +void cdns_pcie_hpa_stop_link(struct cdns_pcie *pcie); +bool cdns_pcie_hpa_link_up(struct cdns_pcie *pcie); + +extern const struct dev_pm_ops cdns_pcie_pm_ops; + +#if IS_ENABLED(CONFIG_PCIE_CADENCE_DEBUGFS) +void cdns_pcie_debugfs_deinit(struct cdns_pcie *pci); +void cdns_pcie_debugfs_init(struct cdns_pcie *pci); +#else +static inline void cdns_pcie_debugfs_deinit(struct cdns_pcie *pci) +{ +} +static inline void cdns_pcie_debugfs_init(struct cdns_pcie *pci) +{ +} +#endif + +#endif /* _PCIE_CADENCE_H */ diff --git a/drivers/pci/controller/cadence/pcie-sg2042.c b/drivers/pci/controller/cadence/pcie-sg2042.c new file mode 100644 index 000000000..265246aa1 --- /dev/null +++ b/drivers/pci/controller/cadence/pcie-sg2042.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * pcie-sg2042 - PCIe controller driver for Sophgo SG2042 SoC + * + * Copyright (C) 2025 Sophgo Technology Inc. + * Copyright (C) 2025 Chen Wang <unicorn_wang@outlook.com> + */ + +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> + +#include "pcie-cadence.h" + +/* + * SG2042 only supports 4-byte aligned access, so for the rootbus (i.e. to + * read/write the Root Port itself, read32/write32 is required. For + * non-rootbus (i.e. to read/write the PCIe peripheral registers, supports + * 1/2/4 byte aligned access, so directly using read/write should be fine. + */ + +static struct pci_ops sg2042_pcie_root_ops = { + .map_bus = cdns_pci_map_bus, + .read = pci_generic_config_read32, + .write = pci_generic_config_write32, +}; + +static struct pci_ops sg2042_pcie_child_ops = { + .map_bus = cdns_pci_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; + +static int sg2042_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct pci_host_bridge *bridge; + struct cdns_pcie *pcie; + struct cdns_pcie_rc *rc; + int ret; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rc)); + if (!bridge) + return dev_err_probe(dev, -ENOMEM, "Failed to alloc host bridge!\n"); + + bridge->ops = &sg2042_pcie_root_ops; + bridge->child_ops = &sg2042_pcie_child_ops; + + rc = pci_host_bridge_priv(bridge); + rc->quirk_broken_aspm_l0s = 1; + rc->quirk_broken_aspm_l1 = 1; + pcie = &rc->pcie; + pcie->dev = dev; + + platform_set_drvdata(pdev, pcie); + + pm_runtime_set_active(dev); + pm_runtime_no_callbacks(dev); + devm_pm_runtime_enable(dev); + + ret = cdns_pcie_init_phy(dev, pcie); + if (ret) + return dev_err_probe(dev, ret, "Failed to init phy!\n"); + + ret = cdns_pcie_host_setup(rc); + if (ret) { + dev_err_probe(dev, ret, "Failed to setup host!\n"); + cdns_pcie_disable_phy(pcie); + return ret; + } + + return 0; +} + +static void sg2042_pcie_remove(struct platform_device *pdev) +{ + struct cdns_pcie *pcie = platform_get_drvdata(pdev); + struct cdns_pcie_rc *rc; + + rc = container_of(pcie, struct cdns_pcie_rc, pcie); + cdns_pcie_host_disable(rc); + + cdns_pcie_disable_phy(pcie); +} + +static int sg2042_pcie_suspend_noirq(struct device *dev) +{ + struct cdns_pcie *pcie = dev_get_drvdata(dev); + + cdns_pcie_disable_phy(pcie); + + return 0; +} + +static int sg2042_pcie_resume_noirq(struct device *dev) +{ + struct cdns_pcie *pcie = dev_get_drvdata(dev); + int ret; + + ret = cdns_pcie_enable_phy(pcie); + if (ret) { + dev_err(dev, "failed to enable PHY\n"); + return ret; + } + + return 0; +} + +static DEFINE_NOIRQ_DEV_PM_OPS(sg2042_pcie_pm_ops, + sg2042_pcie_suspend_noirq, + sg2042_pcie_resume_noirq); + +static const struct of_device_id sg2042_pcie_of_match[] = { + { .compatible = "sophgo,sg2042-pcie-host" }, + {}, +}; +MODULE_DEVICE_TABLE(of, sg2042_pcie_of_match); + +static struct platform_driver sg2042_pcie_driver = { + .driver = { + .name = "sg2042-pcie", + .of_match_table = sg2042_pcie_of_match, + .pm = pm_sleep_ptr(&sg2042_pcie_pm_ops), + }, + .probe = sg2042_pcie_probe, + .remove = sg2042_pcie_remove, +}; +module_platform_driver(sg2042_pcie_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("PCIe controller driver for SG2042 SoCs"); +MODULE_AUTHOR("Chen Wang <unicorn_wang@outlook.com>"); diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig new file mode 100644 index 000000000..dcfbe7e22 --- /dev/null +++ b/drivers/pci/controller/dwc/Kconfig @@ -0,0 +1,580 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "DesignWare-based PCIe controllers" + depends on PCI + +config PCIE_DW + bool + +config PCIE_DW_DEBUGFS + bool "DesignWare PCIe debugfs entries" + depends on DEBUG_FS + depends on PCIE_DW_HOST || PCIE_DW_EP + help + Say Y here to enable debugfs entries for the PCIe controller. These + entries provide various debug features related to the controller and + expose the RAS DES capabilities such as Silicon Debug, Error Injection + and Statistical Counters. + +config PCIE_DW_HOST + bool + select PCIE_DW + select IRQ_MSI_LIB + select PCI_HOST_COMMON + +config PCIE_DW_EP + bool + select PCIE_DW + +config PCIE_AL + bool "Amazon Annapurna Labs PCIe controller" + depends on OF && (ARM64 || COMPILE_TEST) + depends on PCI_MSI + select PCIE_DW_HOST + select PCI_ECAM + help + Say Y here to enable support of the Amazon's Annapurna Labs PCIe + controller IP on Amazon SoCs. The PCIe controller uses the DesignWare + core plus Annapurna Labs proprietary hardware wrappers. This is + required only for DT-based platforms. ACPI platforms with the + Annapurna Labs PCIe controller don't need to enable this. + +config PCIE_AMD_MDB + bool "AMD MDB Versal2 PCIe controller" + depends on OF && (ARM64 || COMPILE_TEST) + depends on PCI_MSI + select PCIE_DW_HOST + help + Say Y here if you want to enable PCIe controller support on AMD + Versal2 SoCs. The AMD MDB Versal2 PCIe controller is based on + DesignWare IP and therefore the driver re-uses the DesignWare + core functions to implement the driver. + +config PCI_MESON + tristate "Amlogic Meson PCIe controller" + default m if ARCH_MESON + depends on PCI_MSI + select PCIE_DW_HOST + help + Say Y here if you want to enable PCI controller support on Amlogic + SoCs. The PCI controller on Amlogic is based on DesignWare hardware + and therefore the driver re-uses the DesignWare core functions to + implement the driver. + +config PCIE_ANDES_QILAI + tristate "Andes QiLai PCIe controller" + depends on ARCH_ANDES || COMPILE_TEST + depends on PCI_MSI + select PCIE_DW_HOST + help + Say Y here to enable PCIe controller support on Andes QiLai SoCs, + which operate in Root Complex mode. The Andes QiLai SoC PCIe + controller is based on DesignWare IP and therefore the driver + re-uses the DesignWare core functions to implement the driver. + +config PCIE_ARTPEC6 + bool + +config PCIE_ARTPEC6_HOST + bool "Axis ARTPEC-6 PCIe controller (host mode)" + depends on MACH_ARTPEC6 || COMPILE_TEST + depends on PCI_MSI + select PCIE_DW_HOST + select PCIE_ARTPEC6 + help + Enables support for the PCIe controller in the ARTPEC-6 SoC to work in + host mode. This uses the DesignWare core. + +config PCIE_ARTPEC6_EP + bool "Axis ARTPEC-6 PCIe controller (endpoint mode)" + depends on MACH_ARTPEC6 || COMPILE_TEST + depends on PCI_ENDPOINT + select PCIE_DW_EP + select PCIE_ARTPEC6 + help + Enables support for the PCIe controller in the ARTPEC-6 SoC to work in + endpoint mode. This uses the DesignWare core. + +config PCIE_ESWIN + tristate "ESWIN PCIe controller" + depends on ARCH_ESWIN || COMPILE_TEST + depends on PCI_MSI + select PCIE_DW_HOST + help + Say Y here if you want PCIe controller support for the ESWIN SoCs. + The PCIe controller in ESWIN SoCs is based on DesignWare hardware, and + works only in host mode. + +config PCI_IMX6 + bool + +config PCI_IMX6_HOST + bool "Freescale i.MX6/7/8 PCIe controller (host mode)" + depends on ARCH_MXC || COMPILE_TEST + depends on PCI_MSI + select PCIE_DW_HOST + select PCI_IMX6 + select PCI_PWRCTRL_GENERIC + help + Enables support for the PCIe controller in the i.MX SoCs to + work in Root Complex mode. The PCI controller on i.MX is based + on DesignWare hardware and therefore the driver re-uses the + DesignWare core functions to implement the driver. + +config PCI_IMX6_EP + bool "Freescale i.MX6/7/8 PCIe controller (endpoint mode)" + depends on ARCH_MXC || COMPILE_TEST + depends on PCI_ENDPOINT + select PCIE_DW_EP + select PCI_HOST_COMMON + select PCI_IMX6 + select PCI_PWRCTRL_GENERIC + help + Enables support for the PCIe controller in the i.MX SoCs to + work in endpoint mode. The PCI controller on i.MX is based + on DesignWare hardware and therefore the driver re-uses the + DesignWare core functions to implement the driver. + +config PCI_LAYERSCAPE + tristate "Freescale Layerscape PCIe controller (host mode)" + depends on OF && (ARM || ARCH_LAYERSCAPE || COMPILE_TEST) + depends on PCI_MSI + select PCIE_DW_HOST + select MFD_SYSCON + help + Say Y here if you want to enable PCIe controller support on Layerscape + SoCs to work in Host mode. + This controller can work either as EP or RC. The RCW[HOST_AGT_PEX] + determines which PCIe controller works in EP mode and which PCIe + controller works in RC mode. + +config PCI_LAYERSCAPE_EP + bool "Freescale Layerscape PCIe controller (endpoint mode)" + depends on OF && (ARM || ARCH_LAYERSCAPE || COMPILE_TEST) + depends on PCI_ENDPOINT + select PCIE_DW_EP + help + Say Y here if you want to enable PCIe controller support on Layerscape + SoCs to work in Endpoint mode. + This controller can work either as EP or RC. The RCW[HOST_AGT_PEX] + determines which PCIe controller works in EP mode and which PCIe + controller works in RC mode. + +config PCI_HISI + depends on OF && (ARM64 || COMPILE_TEST) + bool "HiSilicon Hip05 and Hip06 SoCs PCIe controller" + depends on PCI_MSI + select PCIE_DW_HOST + select PCI_HOST_COMMON + help + Say Y here if you want PCIe controller support on HiSilicon + Hip05 and Hip06 SoCs + +config PCIE_KIRIN + depends on OF && (ARM64 || COMPILE_TEST) + tristate "HiSilicon Kirin PCIe controller" + depends on PCI_MSI + select PCIE_DW_HOST + select REGMAP_MMIO + help + Say Y here if you want PCIe controller support + on HiSilicon Kirin series SoCs. + +config PCIE_HISI_STB + bool "HiSilicon STB PCIe controller" + depends on ARCH_HISI || COMPILE_TEST + depends on PCI_MSI + select PCIE_DW_HOST + help + Say Y here if you want PCIe controller support on HiSilicon STB SoCs + +config PCIE_INTEL_GW + bool "Intel Gateway PCIe controller " + depends on OF && (X86 || COMPILE_TEST) + depends on PCI_MSI + select PCIE_DW_HOST + help + Say 'Y' here to enable PCIe Host controller support on Intel + Gateway SoCs. + The PCIe controller uses the DesignWare core plus Intel-specific + hardware wrappers. + +config PCIE_KEEMBAY + bool + +config PCIE_KEEMBAY_HOST + bool "Intel Keem Bay PCIe controller (host mode)" + depends on ARCH_KEEMBAY || COMPILE_TEST + depends on PCI_MSI + select PCIE_DW_HOST + select PCIE_KEEMBAY + help + Say 'Y' here to enable support for the PCIe controller in Keem Bay + to work in host mode. + The PCIe controller is based on DesignWare Hardware and uses + DesignWare core functions. + +config PCIE_KEEMBAY_EP + bool "Intel Keem Bay PCIe controller (endpoint mode)" + depends on ARCH_KEEMBAY || COMPILE_TEST + depends on PCI_MSI + depends on PCI_ENDPOINT + select PCIE_DW_EP + select PCIE_KEEMBAY + help + Say 'Y' here to enable support for the PCIe controller in Keem Bay + to work in endpoint mode. + The PCIe controller is based on DesignWare Hardware and uses + DesignWare core functions. + +config PCIE_ARMADA_8K + bool "Marvell Armada-8K PCIe controller" + depends on ARCH_MVEBU || COMPILE_TEST + depends on PCI_MSI + select PCIE_DW_HOST + help + Say Y here if you want to enable PCIe controller support on + Armada-8K SoCs. The PCIe controller on Armada-8K is based on + DesignWare hardware and therefore the driver re-uses the + DesignWare core functions to implement the driver. + +config PCIE_TEGRA194 + tristate + +config PCIE_TEGRA194_HOST + tristate "NVIDIA Tegra194 (and later) PCIe controller (host mode)" + depends on (ARCH_TEGRA && ARM64) || COMPILE_TEST + depends on PCI_MSI + select PCIE_DW_HOST + select PHY_TEGRA194_P2U + select PCIE_TEGRA194 + help + Enables support for the PCIe controller in the NVIDIA Tegra194 SoC to + work in host mode. There are two instances of PCIe controllers in + Tegra194. This controller can work either as EP or RC. In order to + enable host-specific features PCIE_TEGRA194_HOST must be selected and + in order to enable device-specific features PCIE_TEGRA194_EP must be + selected. This uses the DesignWare core. + +config PCIE_TEGRA194_EP + tristate "NVIDIA Tegra194 (and later) PCIe controller (endpoint mode)" + depends on (ARCH_TEGRA && ARM64) || COMPILE_TEST + depends on PCI_ENDPOINT + select PCIE_DW_EP + select PHY_TEGRA194_P2U + select PCIE_TEGRA194 + help + Enables support for the PCIe controller in the NVIDIA Tegra194 SoC to + work in endpoint mode. There are two instances of PCIe controllers in + Tegra194. This controller can work either as EP or RC. In order to + enable host-specific features PCIE_TEGRA194_HOST must be selected and + in order to enable device-specific features PCIE_TEGRA194_EP must be + selected. This uses the DesignWare core. + +config PCIE_NXP_S32G + bool "NXP S32G PCIe controller (host mode)" + depends on ARCH_S32 || COMPILE_TEST + select PCIE_DW_HOST + help + Enable support for the PCIe controller in NXP S32G based boards to + work in Host mode. The controller is based on DesignWare IP and + can work either as RC or EP. In order to enable host-specific + features PCIE_NXP_S32G must be selected. + +config PCIE_DW_PLAT + bool + +config PCIE_DW_PLAT_HOST + bool "Platform bus based DesignWare PCIe controller (host mode)" + depends on PCI_MSI + select PCIE_DW_HOST + select PCIE_DW_PLAT + help + Enables support for the PCIe controller in the Designware IP to + work in host mode. There are two instances of PCIe controller in + Designware IP. + This controller can work either as EP or RC. In order to enable + host-specific features PCIE_DW_PLAT_HOST must be selected and in + order to enable device-specific features PCI_DW_PLAT_EP must be + selected. + +config PCIE_DW_PLAT_EP + bool "Platform bus based DesignWare PCIe controller (endpoint mode)" + depends on PCI && PCI_MSI + depends on PCI_ENDPOINT + select PCIE_DW_EP + select PCIE_DW_PLAT + help + Enables support for the PCIe controller in the Designware IP to + work in endpoint mode. There are two instances of PCIe controller + in Designware IP. + This controller can work either as EP or RC. In order to enable + host-specific features PCIE_DW_PLAT_HOST must be selected and in + order to enable device-specific features PCI_DW_PLAT_EP must be + selected. + +config PCIE_QCOM_COMMON + bool + +config PCIE_QCOM + bool "Qualcomm PCIe controller (host mode)" + depends on OF && (ARCH_QCOM || COMPILE_TEST) + depends on PCI_MSI + select PCIE_DW_HOST + select CRC8 + select PCIE_QCOM_COMMON + select PCI_HOST_COMMON + select PCI_PWRCTRL_GENERIC + help + Say Y here to enable PCIe controller support on Qualcomm SoCs. The + PCIe controller uses the DesignWare core plus Qualcomm-specific + hardware wrappers. + +config PCIE_QCOM_EP + tristate "Qualcomm PCIe controller (endpoint mode)" + depends on OF && (ARCH_QCOM || COMPILE_TEST) + depends on PCI_ENDPOINT + select PCIE_DW_EP + select PCIE_QCOM_COMMON + help + Say Y here to enable support for the PCIe controllers on Qualcomm SoCs + to work in endpoint mode. The PCIe controller uses the DesignWare core + plus Qualcomm-specific hardware wrappers. + +config PCIE_RCAR_GEN4 + tristate + +config PCIE_RCAR_GEN4_HOST + tristate "Renesas R-Car Gen4 PCIe controller (host mode)" + depends on ARM64 && (ARCH_RENESAS || COMPILE_TEST) + depends on PCI_MSI + select PCIE_DW_HOST + select PCIE_RCAR_GEN4 + help + Say Y here if you want PCIe controller (host mode) on R-Car Gen4 SoCs. + To compile this driver as a module, choose M here: the module will be + called pcie-rcar-gen4.ko. This uses the DesignWare core. + +config PCIE_RCAR_GEN4_EP + tristate "Renesas R-Car Gen4 PCIe controller (endpoint mode)" + depends on ARM64 && (ARCH_RENESAS || COMPILE_TEST) + depends on PCI_ENDPOINT + select PCIE_DW_EP + select PCIE_RCAR_GEN4 + help + Say Y here if you want PCIe controller (endpoint mode) on R-Car Gen4 + SoCs. To compile this driver as a module, choose M here: the module + will be called pcie-rcar-gen4.ko. This uses the DesignWare core. + +config PCIE_ROCKCHIP_DW + bool + +config PCIE_ROCKCHIP_DW_HOST + bool "Rockchip DesignWare PCIe controller (host mode)" + depends on PCI_MSI + depends on ARCH_ROCKCHIP || COMPILE_TEST + depends on OF + select PCIE_DW_HOST + select PCIE_ROCKCHIP_DW + select PCI_HOST_COMMON + help + Enables support for the DesignWare PCIe controller in the + Rockchip SoC (except RK3399) to work in host mode. + +config PCIE_ROCKCHIP_DW_EP + bool "Rockchip DesignWare PCIe controller (endpoint mode)" + depends on ARCH_ROCKCHIP || COMPILE_TEST + depends on OF + depends on PCI_ENDPOINT + select PCIE_DW_EP + select PCIE_ROCKCHIP_DW + help + Enables support for the DesignWare PCIe controller in the + Rockchip SoC (except RK3399) to work in endpoint mode. + +config PCI_EXYNOS + tristate "Samsung Exynos PCIe controller" + depends on ARCH_EXYNOS || COMPILE_TEST + depends on PCI_MSI + select PCIE_DW_HOST + help + Enables support for the PCIe controller in the Samsung Exynos SoCs + to work in host mode. The PCI controller is based on the DesignWare + hardware and therefore the driver re-uses the DesignWare core + functions to implement the driver. + +config PCIE_FU740 + bool "SiFive FU740 PCIe controller" + depends on PCI_MSI + depends on ARCH_SIFIVE || COMPILE_TEST + select PCIE_DW_HOST + help + Say Y here if you want PCIe controller support for the SiFive + FU740. + +config PCIE_UNIPHIER + bool "Socionext UniPhier PCIe controller (host mode)" + depends on ARCH_UNIPHIER || COMPILE_TEST + depends on OF && HAS_IOMEM + depends on PCI_MSI + select PCIE_DW_HOST + help + Say Y here if you want PCIe host controller support on UniPhier SoCs. + This driver supports LD20 and PXs3 SoCs. + +config PCIE_UNIPHIER_EP + bool "Socionext UniPhier PCIe controller (endpoint mode)" + depends on ARCH_UNIPHIER || COMPILE_TEST + depends on OF && HAS_IOMEM + depends on PCI_ENDPOINT + select PCIE_DW_EP + help + Say Y here if you want PCIe endpoint controller support on + UniPhier SoCs. This driver supports Pro5 SoC. + +config PCIE_SOPHGO_DW + bool "Sophgo DesignWare PCIe controller (host mode)" + depends on ARCH_SOPHGO || COMPILE_TEST + depends on PCI_MSI + depends on OF + select PCIE_DW_HOST + help + Say Y here if you want PCIe host controller support on + Sophgo SoCs. + +config PCIE_SPACEMIT_K1 + tristate "SpacemiT K1 PCIe controller (host mode)" + depends on ARCH_SPACEMIT || COMPILE_TEST + depends on HAS_IOMEM + select PCIE_DW_HOST + select PCI_PWRCTRL_GENERIC + default ARCH_SPACEMIT + help + Enables support for the DesignWare based PCIe controller in + the SpacemiT K1 SoC operating in host mode. Three controllers + are available on the K1 SoC; the first of these shares a PHY + with a USB 3.0 host controller (one or the other can be used). + +config PCIE_SPEAR13XX + bool "STMicroelectronics SPEAr PCIe controller" + depends on ARCH_SPEAR13XX || COMPILE_TEST + depends on PCI_MSI + select PCIE_DW_HOST + help + Say Y here if you want PCIe support on SPEAr13XX SoCs. + +config PCIE_STM32_HOST + tristate "STMicroelectronics STM32MP25 PCIe Controller (host mode)" + depends on ARCH_STM32 || COMPILE_TEST + depends on PCI_MSI + select PCIE_DW_HOST + help + Enables Root Complex (RC) support for the DesignWare core based PCIe + controller found in STM32MP25 SoC. + + This driver can also be built as a module. If so, the module + will be called pcie-stm32. + +config PCIE_STM32_EP + tristate "STMicroelectronics STM32MP25 PCIe Controller (endpoint mode)" + depends on ARCH_STM32 || COMPILE_TEST + depends on PCI_ENDPOINT + select PCIE_DW_EP + help + Enables Endpoint (EP) support for the DesignWare core based PCIe + controller found in STM32MP25 SoC. + + This driver can also be built as a module. If so, the module + will be called pcie-stm32-ep. + +config PCI_DRA7XX + tristate + +config PCI_DRA7XX_HOST + tristate "TI DRA7xx PCIe controller (host mode)" + depends on SOC_DRA7XX || COMPILE_TEST + depends on OF && HAS_IOMEM && TI_PIPE3 + depends on PCI_MSI + select PCIE_DW_HOST + select PCI_DRA7XX + default y if SOC_DRA7XX + help + Enables support for the PCIe controller in the DRA7xx SoC to work in + host mode. There are two instances of PCIe controller in DRA7xx. + This controller can work either as EP or RC. In order to enable + host-specific features PCI_DRA7XX_HOST must be selected and in order + to enable device-specific features PCI_DRA7XX_EP must be selected. + This uses the DesignWare core. + +config PCI_DRA7XX_EP + tristate "TI DRA7xx PCIe controller (endpoint mode)" + depends on SOC_DRA7XX || COMPILE_TEST + depends on OF && HAS_IOMEM && TI_PIPE3 + depends on PCI_ENDPOINT + select PCIE_DW_EP + select PCI_DRA7XX + help + Enables support for the PCIe controller in the DRA7xx SoC to work in + endpoint mode. There are two instances of PCIe controller in DRA7xx. + This controller can work either as EP or RC. In order to enable + host-specific features PCI_DRA7XX_HOST must be selected and in order + to enable device-specific features PCI_DRA7XX_EP must be selected. + This uses the DesignWare core. + +# ARM32 platforms use hook_fault_code() and cannot support loadable module. +config PCI_KEYSTONE + bool + +# On non-ARM32 platforms, loadable module can be supported. +config PCI_KEYSTONE_TRISTATE + tristate + +config PCI_KEYSTONE_HOST + tristate "TI Keystone PCIe controller (host mode)" + depends on ARCH_KEYSTONE || ARCH_K3 || COMPILE_TEST + depends on PCI_MSI + select PCIE_DW_HOST + select PCI_KEYSTONE if ARM + select PCI_KEYSTONE_TRISTATE if !ARM + help + Enables support for the PCIe controller in the Keystone SoC to + work in host mode. The PCI controller on Keystone is based on + DesignWare hardware and therefore the driver re-uses the + DesignWare core functions to implement the driver. + +config PCI_KEYSTONE_EP + tristate "TI Keystone PCIe controller (endpoint mode)" + depends on ARCH_KEYSTONE || ARCH_K3 || COMPILE_TEST + depends on PCI_ENDPOINT + select PCIE_DW_EP + select PCI_KEYSTONE if ARM + select PCI_KEYSTONE_TRISTATE if !ARM + help + Enables support for the PCIe controller in the Keystone SoC to + work in endpoint mode. The PCI controller on Keystone is based + on DesignWare hardware and therefore the driver re-uses the + DesignWare core functions to implement the driver. + +config PCIE_VISCONTI_HOST + bool "Toshiba Visconti PCIe controller" + depends on ARCH_VISCONTI || COMPILE_TEST + depends on PCI_MSI + select PCIE_DW_HOST + help + Say Y here if you want PCIe controller support on Toshiba Visconti SoC. + This driver supports TMPV7708 SoC. + +config PCIE_ULTRARISC + tristate "UltraRISC PCIe host controller" + depends on ARCH_ULTRARISC || COMPILE_TEST + select PCIE_DW_HOST + select PCI_MSI + default y if ARCH_ULTRARISC + help + Enables support for the PCIe controller in the UltraRISC SoC. + This driver supports UR-DP1000 SoC. + + By default, this symbol is enabled when ARCH_ULTRARISC is active, + requiring no further configuration on that platform. + +endmenu diff --git a/drivers/pci/controller/dwc/Makefile b/drivers/pci/controller/dwc/Makefile new file mode 100644 index 000000000..d4f88cfc6 --- /dev/null +++ b/drivers/pci/controller/dwc/Makefile @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_PCIE_DW) += pcie-designware.o +obj-$(CONFIG_PCIE_DW_DEBUGFS) += pcie-designware-debugfs.o +obj-$(CONFIG_PCIE_DW_HOST) += pcie-designware-host.o +obj-$(CONFIG_PCIE_DW_EP) += pcie-designware-ep.o +obj-$(CONFIG_PCIE_DW_PLAT) += pcie-designware-plat.o +obj-$(CONFIG_PCIE_AMD_MDB) += pcie-amd-mdb.o +obj-$(CONFIG_PCIE_ANDES_QILAI) += pcie-andes-qilai.o +obj-$(CONFIG_PCIE_ESWIN) += pcie-eswin.o +obj-$(CONFIG_PCI_DRA7XX) += pci-dra7xx.o +obj-$(CONFIG_PCI_EXYNOS) += pci-exynos.o +obj-$(CONFIG_PCIE_FU740) += pcie-fu740.o +obj-$(CONFIG_PCI_IMX6) += pci-imx6.o +obj-$(CONFIG_PCIE_NXP_S32G) += pcie-nxp-s32g.o +obj-$(CONFIG_PCIE_SPEAR13XX) += pcie-spear13xx.o +# ARM32 platforms use hook_fault_code() and cannot support loadable module. +obj-$(CONFIG_PCI_KEYSTONE) += pci-keystone.o +# On non-ARM32 platforms, loadable module can be supported. +obj-$(CONFIG_PCI_KEYSTONE_TRISTATE) += pci-keystone.o +obj-$(CONFIG_PCI_LAYERSCAPE) += pci-layerscape.o +obj-$(CONFIG_PCI_LAYERSCAPE_EP) += pci-layerscape-ep.o +obj-$(CONFIG_PCIE_QCOM_COMMON) += pcie-qcom-common.o +obj-$(CONFIG_PCIE_QCOM) += pcie-qcom.o +obj-$(CONFIG_PCIE_QCOM_EP) += pcie-qcom-ep.o +obj-$(CONFIG_PCIE_ARMADA_8K) += pcie-armada8k.o +obj-$(CONFIG_PCIE_ARTPEC6) += pcie-artpec6.o +obj-$(CONFIG_PCIE_ROCKCHIP_DW) += pcie-dw-rockchip.o +obj-$(CONFIG_PCIE_SOPHGO_DW) += pcie-sophgo.o +obj-$(CONFIG_PCIE_INTEL_GW) += pcie-intel-gw.o +obj-$(CONFIG_PCIE_KEEMBAY) += pcie-keembay.o +obj-$(CONFIG_PCIE_KIRIN) += pcie-kirin.o +obj-$(CONFIG_PCIE_HISI_STB) += pcie-histb.o +obj-$(CONFIG_PCI_MESON) += pci-meson.o +obj-$(CONFIG_PCIE_TEGRA194) += pcie-tegra194.o +obj-$(CONFIG_PCIE_UNIPHIER) += pcie-uniphier.o +obj-$(CONFIG_PCIE_UNIPHIER_EP) += pcie-uniphier-ep.o +obj-$(CONFIG_PCIE_VISCONTI_HOST) += pcie-visconti.o +obj-$(CONFIG_PCIE_RCAR_GEN4) += pcie-rcar-gen4.o +obj-$(CONFIG_PCIE_SPACEMIT_K1) += pcie-spacemit-k1.o +obj-$(CONFIG_PCIE_STM32_HOST) += pcie-stm32.o +obj-$(CONFIG_PCIE_STM32_EP) += pcie-stm32-ep.o +obj-$(CONFIG_PCIE_ULTRARISC) += pcie-ultrarisc.o + +# The following drivers are for devices that use the generic ACPI +# pci_root.c driver but don't support standard ECAM config access. +# They contain MCFG quirks to replace the generic ECAM accessors with +# device-specific ones that are shared with the DT driver. + +# The ACPI driver is generic and should not require driver-specific +# config options to be enabled, so we always build these drivers on +# ARM64 and use internal ifdefs to only build the pieces we need +# depending on whether ACPI, the DT driver, or both are enabled. + +obj-$(CONFIG_PCIE_AL) += pcie-al.o +obj-$(CONFIG_PCI_HISI) += pcie-hisi.o + +ifdef CONFIG_ACPI +ifdef CONFIG_PCI_QUIRKS +obj-$(CONFIG_ARM64) += pcie-al.o +obj-$(CONFIG_ARM64) += pcie-hisi.o +obj-$(CONFIG_ARM64) += pcie-tegra194-acpi.o +endif +endif diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c new file mode 100644 index 000000000..6ae5b27e2 --- /dev/null +++ b/drivers/pci/controller/dwc/pci-dra7xx.c @@ -0,0 +1,958 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * pcie-dra7xx - PCIe controller driver for TI DRA7xx SoCs + * + * Copyright (C) 2013-2014 Texas Instruments Incorporated - https://www.ti.com + * + * Authors: Kishon Vijay Abraham I <kishon@ti.com> + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqdomain.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/resource.h> +#include <linux/types.h> +#include <linux/mfd/syscon.h> +#include <linux/regmap.h> +#include <linux/gpio/consumer.h> + +#include "../../pci.h" +#include "pcie-designware.h" + +/* PCIe controller wrapper DRA7XX configuration registers */ + +#define PCIECTRL_DRA7XX_CONF_IRQSTATUS_MAIN 0x0024 +#define PCIECTRL_DRA7XX_CONF_IRQENABLE_SET_MAIN 0x0028 +#define ERR_SYS BIT(0) +#define ERR_FATAL BIT(1) +#define ERR_NONFATAL BIT(2) +#define ERR_COR BIT(3) +#define ERR_AXI BIT(4) +#define ERR_ECRC BIT(5) +#define PME_TURN_OFF BIT(8) +#define PME_TO_ACK BIT(9) +#define PM_PME BIT(10) +#define LINK_REQ_RST BIT(11) +#define LINK_UP_EVT BIT(12) +#define CFG_BME_EVT BIT(13) +#define CFG_MSE_EVT BIT(14) +#define INTERRUPTS (ERR_SYS | ERR_FATAL | ERR_NONFATAL | ERR_COR | ERR_AXI | \ + ERR_ECRC | PME_TURN_OFF | PME_TO_ACK | PM_PME | \ + LINK_REQ_RST | LINK_UP_EVT | CFG_BME_EVT | CFG_MSE_EVT) + +#define PCIECTRL_DRA7XX_CONF_IRQSTATUS_MSI 0x0034 +#define PCIECTRL_DRA7XX_CONF_IRQENABLE_SET_MSI 0x0038 +#define INTA BIT(0) +#define INTB BIT(1) +#define INTC BIT(2) +#define INTD BIT(3) +#define MSI BIT(4) +#define LEG_EP_INTERRUPTS (INTA | INTB | INTC | INTD) + +#define PCIECTRL_TI_CONF_DEVICE_TYPE 0x0100 +#define DEVICE_TYPE_EP 0x0 +#define DEVICE_TYPE_LEG_EP 0x1 +#define DEVICE_TYPE_RC 0x4 + +#define PCIECTRL_DRA7XX_CONF_DEVICE_CMD 0x0104 +#define LTSSM_EN 0x1 + +#define PCIECTRL_DRA7XX_CONF_PHY_CS 0x010C +#define LINK_UP BIT(16) +#define DRA7XX_CPU_TO_BUS_ADDR 0x0FFFFFFF + +#define PCIECTRL_TI_CONF_INTX_ASSERT 0x0124 +#define PCIECTRL_TI_CONF_INTX_DEASSERT 0x0128 + +#define PCIECTRL_TI_CONF_MSI_XMT 0x012c +#define MSI_REQ_GRANT BIT(0) +#define MSI_VECTOR_SHIFT 7 + +#define PCIE_1LANE_2LANE_SELECTION BIT(13) +#define PCIE_B1C0_MODE_SEL BIT(2) +#define PCIE_B0_B1_TSYNCEN BIT(0) + +struct dra7xx_pcie { + struct dw_pcie *pci; + void __iomem *base; /* DT ti_conf */ + int phy_count; /* DT phy-names count */ + struct phy **phy; + struct irq_domain *irq_domain; + struct clk *clk; +}; + +struct dra7xx_pcie_of_data { + enum dw_pcie_device_mode mode; + u32 b1co_mode_sel_mask; +}; + +#define to_dra7xx_pcie(x) dev_get_drvdata((x)->dev) + +static inline u32 dra7xx_pcie_readl(struct dra7xx_pcie *pcie, u32 offset) +{ + return readl(pcie->base + offset); +} + +static inline void dra7xx_pcie_writel(struct dra7xx_pcie *pcie, u32 offset, + u32 value) +{ + writel(value, pcie->base + offset); +} + +static u64 dra7xx_pcie_cpu_addr_fixup(struct dw_pcie *pci, u64 cpu_addr) +{ + return cpu_addr & DRA7XX_CPU_TO_BUS_ADDR; +} + +static bool dra7xx_pcie_link_up(struct dw_pcie *pci) +{ + struct dra7xx_pcie *dra7xx = to_dra7xx_pcie(pci); + u32 reg = dra7xx_pcie_readl(dra7xx, PCIECTRL_DRA7XX_CONF_PHY_CS); + + return reg & LINK_UP; +} + +static void dra7xx_pcie_stop_link(struct dw_pcie *pci) +{ + struct dra7xx_pcie *dra7xx = to_dra7xx_pcie(pci); + u32 reg; + + reg = dra7xx_pcie_readl(dra7xx, PCIECTRL_DRA7XX_CONF_DEVICE_CMD); + reg &= ~LTSSM_EN; + dra7xx_pcie_writel(dra7xx, PCIECTRL_DRA7XX_CONF_DEVICE_CMD, reg); +} + +static int dra7xx_pcie_establish_link(struct dw_pcie *pci) +{ + struct dra7xx_pcie *dra7xx = to_dra7xx_pcie(pci); + struct device *dev = pci->dev; + u32 reg; + + if (dw_pcie_link_up(pci)) { + dev_err(dev, "link is already up\n"); + return 0; + } + + reg = dra7xx_pcie_readl(dra7xx, PCIECTRL_DRA7XX_CONF_DEVICE_CMD); + reg |= LTSSM_EN; + dra7xx_pcie_writel(dra7xx, PCIECTRL_DRA7XX_CONF_DEVICE_CMD, reg); + + return 0; +} + +static void dra7xx_pcie_enable_msi_interrupts(struct dra7xx_pcie *dra7xx) +{ + dra7xx_pcie_writel(dra7xx, PCIECTRL_DRA7XX_CONF_IRQSTATUS_MSI, + LEG_EP_INTERRUPTS | MSI); + + dra7xx_pcie_writel(dra7xx, + PCIECTRL_DRA7XX_CONF_IRQENABLE_SET_MSI, + MSI | LEG_EP_INTERRUPTS); +} + +static void dra7xx_pcie_enable_wrapper_interrupts(struct dra7xx_pcie *dra7xx) +{ + dra7xx_pcie_writel(dra7xx, PCIECTRL_DRA7XX_CONF_IRQSTATUS_MAIN, + INTERRUPTS); + dra7xx_pcie_writel(dra7xx, PCIECTRL_DRA7XX_CONF_IRQENABLE_SET_MAIN, + INTERRUPTS); +} + +static void dra7xx_pcie_enable_interrupts(struct dra7xx_pcie *dra7xx) +{ + dra7xx_pcie_enable_wrapper_interrupts(dra7xx); + dra7xx_pcie_enable_msi_interrupts(dra7xx); +} + +static int dra7xx_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct dra7xx_pcie *dra7xx = to_dra7xx_pcie(pci); + + dra7xx_pcie_enable_interrupts(dra7xx); + + return 0; +} + +static int dra7xx_pcie_intx_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq); + irq_set_chip_data(irq, domain->host_data); + + return 0; +} + +static const struct irq_domain_ops intx_domain_ops = { + .map = dra7xx_pcie_intx_map, + .xlate = pci_irqd_intx_xlate, +}; + +static int dra7xx_pcie_handle_msi(struct dw_pcie_rp *pp, int index) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + unsigned long val; + int pos; + + val = dw_pcie_readl_dbi(pci, PCIE_MSI_INTR0_STATUS + + (index * MSI_REG_CTRL_BLOCK_SIZE)); + if (!val) + return 0; + + pos = find_first_bit(&val, MAX_MSI_IRQS_PER_CTRL); + while (pos != MAX_MSI_IRQS_PER_CTRL) { + generic_handle_domain_irq(pp->irq_domain, + (index * MAX_MSI_IRQS_PER_CTRL) + pos); + pos++; + pos = find_next_bit(&val, MAX_MSI_IRQS_PER_CTRL, pos); + } + + return 1; +} + +static void dra7xx_pcie_handle_msi_irq(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + int ret, i, count, num_ctrls; + + num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL; + + /** + * Need to make sure all MSI status bits read 0 before exiting. + * Else, new MSI IRQs are not registered by the wrapper. Have an + * upperbound for the loop and exit the IRQ in case of IRQ flood + * to avoid locking up system in interrupt context. + */ + count = 0; + do { + ret = 0; + + for (i = 0; i < num_ctrls; i++) + ret |= dra7xx_pcie_handle_msi(pp, i); + count++; + } while (ret && count <= 1000); + + if (count > 1000) + dev_warn_ratelimited(pci->dev, + "Too many MSI IRQs to handle\n"); +} + +static void dra7xx_pcie_msi_irq_handler(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + struct dra7xx_pcie *dra7xx; + struct dw_pcie_rp *pp; + struct dw_pcie *pci; + unsigned long reg; + u32 bit; + + chained_irq_enter(chip, desc); + + pp = irq_desc_get_handler_data(desc); + pci = to_dw_pcie_from_pp(pp); + dra7xx = to_dra7xx_pcie(pci); + + reg = dra7xx_pcie_readl(dra7xx, PCIECTRL_DRA7XX_CONF_IRQSTATUS_MSI); + dra7xx_pcie_writel(dra7xx, PCIECTRL_DRA7XX_CONF_IRQSTATUS_MSI, reg); + + switch (reg) { + case MSI: + dra7xx_pcie_handle_msi_irq(pp); + break; + case INTA: + case INTB: + case INTC: + case INTD: + for_each_set_bit(bit, ®, PCI_NUM_INTX) + generic_handle_domain_irq(dra7xx->irq_domain, bit); + break; + } + + chained_irq_exit(chip, desc); +} + +static irqreturn_t dra7xx_pcie_irq_handler(int irq, void *arg) +{ + struct dra7xx_pcie *dra7xx = arg; + struct dw_pcie *pci = dra7xx->pci; + struct device *dev = pci->dev; + struct dw_pcie_ep *ep = &pci->ep; + u32 reg; + + reg = dra7xx_pcie_readl(dra7xx, PCIECTRL_DRA7XX_CONF_IRQSTATUS_MAIN); + + if (reg & ERR_SYS) + dev_dbg(dev, "System Error\n"); + + if (reg & ERR_FATAL) + dev_dbg(dev, "Fatal Error\n"); + + if (reg & ERR_NONFATAL) + dev_dbg(dev, "Non Fatal Error\n"); + + if (reg & ERR_COR) + dev_dbg(dev, "Correctable Error\n"); + + if (reg & ERR_AXI) + dev_dbg(dev, "AXI tag lookup fatal Error\n"); + + if (reg & ERR_ECRC) + dev_dbg(dev, "ECRC Error\n"); + + if (reg & PME_TURN_OFF) + dev_dbg(dev, + "Power Management Event Turn-Off message received\n"); + + if (reg & PME_TO_ACK) + dev_dbg(dev, + "Power Management Turn-Off Ack message received\n"); + + if (reg & PM_PME) + dev_dbg(dev, "PM Power Management Event message received\n"); + + if (reg & LINK_REQ_RST) + dev_dbg(dev, "Link Request Reset\n"); + + if (reg & LINK_UP_EVT) { + if (dra7xx->pci->mode == DW_PCIE_EP_TYPE) + dw_pcie_ep_linkup(ep); + dev_dbg(dev, "Link-up state change\n"); + } + + if (reg & CFG_BME_EVT) + dev_dbg(dev, "CFG 'Bus Master Enable' change\n"); + + if (reg & CFG_MSE_EVT) + dev_dbg(dev, "CFG 'Memory Space Enable' change\n"); + + dra7xx_pcie_writel(dra7xx, PCIECTRL_DRA7XX_CONF_IRQSTATUS_MAIN, reg); + + return IRQ_HANDLED; +} + +static int dra7xx_pcie_init_irq_domain(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct device *dev = pci->dev; + struct dra7xx_pcie *dra7xx = to_dra7xx_pcie(pci); + struct device_node *node = dev->of_node; + struct device_node *pcie_intc_node = of_get_next_child(node, NULL); + + if (!pcie_intc_node) { + dev_err(dev, "No PCIe Intc node found\n"); + return -ENODEV; + } + + irq_set_chained_handler_and_data(pp->irq, dra7xx_pcie_msi_irq_handler, + pp); + dra7xx->irq_domain = irq_domain_create_linear(of_fwnode_handle(pcie_intc_node), + PCI_NUM_INTX, &intx_domain_ops, pp); + of_node_put(pcie_intc_node); + if (!dra7xx->irq_domain) { + dev_err(dev, "Failed to get a INTx IRQ domain\n"); + return -ENODEV; + } + + return 0; +} + +static const struct dw_pcie_host_ops dra7xx_pcie_host_ops = { + .init = dra7xx_pcie_host_init, +}; + +static int dra7xx_pcie_ep_init(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct dra7xx_pcie *dra7xx = to_dra7xx_pcie(pci); + + dra7xx_pcie_enable_wrapper_interrupts(dra7xx); + + return 0; +} + +static void dra7xx_pcie_raise_intx_irq(struct dra7xx_pcie *dra7xx) +{ + dra7xx_pcie_writel(dra7xx, PCIECTRL_TI_CONF_INTX_ASSERT, 0x1); + mdelay(1); + dra7xx_pcie_writel(dra7xx, PCIECTRL_TI_CONF_INTX_DEASSERT, 0x1); +} + +static void dra7xx_pcie_raise_msi_irq(struct dra7xx_pcie *dra7xx, + u8 interrupt_num) +{ + u32 reg; + + reg = (interrupt_num - 1) << MSI_VECTOR_SHIFT; + reg |= MSI_REQ_GRANT; + dra7xx_pcie_writel(dra7xx, PCIECTRL_TI_CONF_MSI_XMT, reg); +} + +static int dra7xx_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no, + unsigned int type, u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct dra7xx_pcie *dra7xx = to_dra7xx_pcie(pci); + + switch (type) { + case PCI_IRQ_INTX: + dra7xx_pcie_raise_intx_irq(dra7xx); + break; + case PCI_IRQ_MSI: + dra7xx_pcie_raise_msi_irq(dra7xx, interrupt_num); + break; + default: + dev_err(pci->dev, "UNKNOWN IRQ type\n"); + } + + return 0; +} + +static const struct pci_epc_features dra7xx_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, + .linkup_notifier = true, + .msi_capable = true, +}; + +static const struct pci_epc_features* +dra7xx_pcie_get_features(struct dw_pcie_ep *ep) +{ + return &dra7xx_pcie_epc_features; +} + +static const struct dw_pcie_ep_ops pcie_ep_ops = { + .init = dra7xx_pcie_ep_init, + .raise_irq = dra7xx_pcie_raise_irq, + .get_features = dra7xx_pcie_get_features, +}; + +static int dra7xx_add_pcie_ep(struct dra7xx_pcie *dra7xx, + struct platform_device *pdev) +{ + int ret; + struct dw_pcie_ep *ep; + struct device *dev = &pdev->dev; + struct dw_pcie *pci = dra7xx->pci; + + ep = &pci->ep; + ep->ops = &pcie_ep_ops; + + pci->dbi_base = devm_platform_ioremap_resource_byname(pdev, "ep_dbics"); + if (IS_ERR(pci->dbi_base)) + return PTR_ERR(pci->dbi_base); + + pci->dbi_base2 = + devm_platform_ioremap_resource_byname(pdev, "ep_dbics2"); + if (IS_ERR(pci->dbi_base2)) + return PTR_ERR(pci->dbi_base2); + + ret = dw_pcie_ep_init(ep); + if (ret) { + dev_err(dev, "failed to initialize endpoint\n"); + return ret; + } + + ret = dw_pcie_ep_init_registers(ep); + if (ret) { + dev_err(dev, "Failed to initialize DWC endpoint registers\n"); + dw_pcie_ep_deinit(ep); + return ret; + } + + pci_epc_init_notify(ep->epc); + + return 0; +} + +static int dra7xx_add_pcie_port(struct dra7xx_pcie *dra7xx, + struct platform_device *pdev) +{ + int ret; + struct dw_pcie *pci = dra7xx->pci; + struct dw_pcie_rp *pp = &pci->pp; + struct device *dev = pci->dev; + + pp->irq = platform_get_irq(pdev, 1); + if (pp->irq < 0) + return pp->irq; + + /* MSI IRQ is muxed */ + pp->msi_irq[0] = -ENODEV; + + ret = dra7xx_pcie_init_irq_domain(pp); + if (ret < 0) + return ret; + + pci->dbi_base = devm_platform_ioremap_resource_byname(pdev, "rc_dbics"); + if (IS_ERR(pci->dbi_base)) + return PTR_ERR(pci->dbi_base); + + pp->ops = &dra7xx_pcie_host_ops; + + ret = dw_pcie_host_init(pp); + if (ret) { + dev_err(dev, "failed to initialize host\n"); + return ret; + } + + return 0; +} + +static const struct dw_pcie_ops dw_pcie_ops = { + .cpu_addr_fixup = dra7xx_pcie_cpu_addr_fixup, + .start_link = dra7xx_pcie_establish_link, + .stop_link = dra7xx_pcie_stop_link, + .link_up = dra7xx_pcie_link_up, +}; + +static void dra7xx_pcie_disable_phy(struct dra7xx_pcie *dra7xx) +{ + int phy_count = dra7xx->phy_count; + + while (phy_count--) { + phy_power_off(dra7xx->phy[phy_count]); + phy_exit(dra7xx->phy[phy_count]); + } +} + +static int dra7xx_pcie_enable_phy(struct dra7xx_pcie *dra7xx) +{ + int phy_count = dra7xx->phy_count; + int ret; + int i; + + for (i = 0; i < phy_count; i++) { + ret = phy_set_mode(dra7xx->phy[i], PHY_MODE_PCIE); + if (ret < 0) + goto err_phy; + + ret = phy_init(dra7xx->phy[i]); + if (ret < 0) + goto err_phy; + + ret = phy_power_on(dra7xx->phy[i]); + if (ret < 0) { + phy_exit(dra7xx->phy[i]); + goto err_phy; + } + } + + return 0; + +err_phy: + while (--i >= 0) { + phy_power_off(dra7xx->phy[i]); + phy_exit(dra7xx->phy[i]); + } + + return ret; +} + +static const struct dra7xx_pcie_of_data dra7xx_pcie_rc_of_data = { + .mode = DW_PCIE_RC_TYPE, +}; + +static const struct dra7xx_pcie_of_data dra7xx_pcie_ep_of_data = { + .mode = DW_PCIE_EP_TYPE, +}; + +static const struct dra7xx_pcie_of_data dra746_pcie_rc_of_data = { + .b1co_mode_sel_mask = BIT(2), + .mode = DW_PCIE_RC_TYPE, +}; + +static const struct dra7xx_pcie_of_data dra726_pcie_rc_of_data = { + .b1co_mode_sel_mask = GENMASK(3, 2), + .mode = DW_PCIE_RC_TYPE, +}; + +static const struct dra7xx_pcie_of_data dra746_pcie_ep_of_data = { + .b1co_mode_sel_mask = BIT(2), + .mode = DW_PCIE_EP_TYPE, +}; + +static const struct dra7xx_pcie_of_data dra726_pcie_ep_of_data = { + .b1co_mode_sel_mask = GENMASK(3, 2), + .mode = DW_PCIE_EP_TYPE, +}; + +static const struct of_device_id of_dra7xx_pcie_match[] = { + { + .compatible = "ti,dra7-pcie", + .data = &dra7xx_pcie_rc_of_data, + }, + { + .compatible = "ti,dra7-pcie-ep", + .data = &dra7xx_pcie_ep_of_data, + }, + { + .compatible = "ti,dra746-pcie-rc", + .data = &dra746_pcie_rc_of_data, + }, + { + .compatible = "ti,dra726-pcie-rc", + .data = &dra726_pcie_rc_of_data, + }, + { + .compatible = "ti,dra746-pcie-ep", + .data = &dra746_pcie_ep_of_data, + }, + { + .compatible = "ti,dra726-pcie-ep", + .data = &dra726_pcie_ep_of_data, + }, + {}, +}; +MODULE_DEVICE_TABLE(of, of_dra7xx_pcie_match); + +/* + * dra7xx_pcie_unaligned_memaccess: workaround for AM572x/AM571x Errata i870 + * @dra7xx: the dra7xx device where the workaround should be applied + * + * Access to the PCIe slave port that are not 32-bit aligned will result + * in incorrect mapping to TLP Address and Byte enable fields. Therefore, + * byte and half-word accesses are not possible to byte offset 0x1, 0x2, or + * 0x3. + * + * To avoid this issue set PCIE_SS1_AXI2OCP_LEGACY_MODE_ENABLE to 1. + */ +static int dra7xx_pcie_unaligned_memaccess(struct device *dev) +{ + int ret; + struct device_node *np = dev->of_node; + unsigned int args[2]; + struct regmap *regmap; + + regmap = syscon_regmap_lookup_by_phandle_args(np, "ti,syscon-unaligned-access", + 2, args); + if (IS_ERR(regmap)) { + dev_dbg(dev, "can't get ti,syscon-unaligned-access\n"); + return -EINVAL; + } + + ret = regmap_update_bits(regmap, args[0], args[1], args[1]); + if (ret) + dev_err(dev, "failed to enable unaligned access\n"); + + return ret; +} + +static int dra7xx_pcie_configure_two_lane(struct device *dev, + u32 b1co_mode_sel_mask) +{ + struct device_node *np = dev->of_node; + struct regmap *pcie_syscon; + unsigned int pcie_reg; + u32 mask; + u32 val; + + pcie_syscon = syscon_regmap_lookup_by_phandle_args(np, "ti,syscon-lane-sel", + 1, &pcie_reg); + if (IS_ERR(pcie_syscon)) { + dev_err(dev, "unable to get ti,syscon-lane-sel\n"); + return -EINVAL; + } + + mask = b1co_mode_sel_mask | PCIE_B0_B1_TSYNCEN; + val = PCIE_B1C0_MODE_SEL | PCIE_B0_B1_TSYNCEN; + regmap_update_bits(pcie_syscon, pcie_reg, mask, val); + + return 0; +} + +static int dra7xx_pcie_probe(struct platform_device *pdev) +{ + u32 reg; + int ret; + int irq; + int i; + int phy_count; + struct phy **phy; + struct device_link **link; + void __iomem *base; + struct dw_pcie *pci; + struct dra7xx_pcie *dra7xx; + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + char name[10]; + struct gpio_desc *reset; + const struct dra7xx_pcie_of_data *data; + enum dw_pcie_device_mode mode; + u32 b1co_mode_sel_mask; + + data = of_device_get_match_data(dev); + if (!data) + return -EINVAL; + + mode = (enum dw_pcie_device_mode)data->mode; + b1co_mode_sel_mask = data->b1co_mode_sel_mask; + + dra7xx = devm_kzalloc(dev, sizeof(*dra7xx), GFP_KERNEL); + if (!dra7xx) + return -ENOMEM; + + pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); + if (!pci) + return -ENOMEM; + + pci->dev = dev; + pci->ops = &dw_pcie_ops; + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + base = devm_platform_ioremap_resource_byname(pdev, "ti_conf"); + if (IS_ERR(base)) + return PTR_ERR(base); + + phy_count = of_property_count_strings(np, "phy-names"); + if (phy_count < 0) { + dev_err(dev, "unable to find the strings\n"); + return phy_count; + } + + phy = devm_kcalloc(dev, phy_count, sizeof(*phy), GFP_KERNEL); + if (!phy) + return -ENOMEM; + + link = devm_kcalloc(dev, phy_count, sizeof(*link), GFP_KERNEL); + if (!link) + return -ENOMEM; + + dra7xx->clk = devm_clk_get_optional(dev, NULL); + if (IS_ERR(dra7xx->clk)) + return dev_err_probe(dev, PTR_ERR(dra7xx->clk), + "clock request failed"); + + ret = clk_prepare_enable(dra7xx->clk); + if (ret) + return ret; + + for (i = 0; i < phy_count; i++) { + snprintf(name, sizeof(name), "pcie-phy%d", i); + phy[i] = devm_phy_get(dev, name); + if (IS_ERR(phy[i])) + return PTR_ERR(phy[i]); + + link[i] = device_link_add(dev, &phy[i]->dev, DL_FLAG_STATELESS); + if (!link[i]) { + ret = -EINVAL; + goto err_link; + } + } + + dra7xx->base = base; + dra7xx->phy = phy; + dra7xx->pci = pci; + dra7xx->phy_count = phy_count; + + if (phy_count == 2) { + ret = dra7xx_pcie_configure_two_lane(dev, b1co_mode_sel_mask); + if (ret < 0) + dra7xx->phy_count = 1; /* Fallback to x1 lane mode */ + } + + ret = dra7xx_pcie_enable_phy(dra7xx); + if (ret) { + dev_err(dev, "failed to enable phy\n"); + return ret; + } + + platform_set_drvdata(pdev, dra7xx); + + pm_runtime_enable(dev); + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + dev_err(dev, "pm_runtime_get_sync failed\n"); + goto err_get_sync; + } + + reset = devm_gpiod_get_optional(dev, NULL, GPIOD_OUT_HIGH); + if (IS_ERR(reset)) { + ret = PTR_ERR(reset); + dev_err(&pdev->dev, "gpio request failed, ret %d\n", ret); + goto err_gpio; + } + + reg = dra7xx_pcie_readl(dra7xx, PCIECTRL_DRA7XX_CONF_DEVICE_CMD); + reg &= ~LTSSM_EN; + dra7xx_pcie_writel(dra7xx, PCIECTRL_DRA7XX_CONF_DEVICE_CMD, reg); + + switch (mode) { + case DW_PCIE_RC_TYPE: + if (!IS_ENABLED(CONFIG_PCI_DRA7XX_HOST)) { + ret = -ENODEV; + goto err_gpio; + } + + dra7xx_pcie_writel(dra7xx, PCIECTRL_TI_CONF_DEVICE_TYPE, + DEVICE_TYPE_RC); + + ret = dra7xx_pcie_unaligned_memaccess(dev); + if (ret) + dev_err(dev, "WA for Errata i870 not applied\n"); + + ret = dra7xx_add_pcie_port(dra7xx, pdev); + if (ret < 0) + goto err_gpio; + break; + case DW_PCIE_EP_TYPE: + if (!IS_ENABLED(CONFIG_PCI_DRA7XX_EP)) { + ret = -ENODEV; + goto err_gpio; + } + + dra7xx_pcie_writel(dra7xx, PCIECTRL_TI_CONF_DEVICE_TYPE, + DEVICE_TYPE_EP); + + ret = dra7xx_pcie_unaligned_memaccess(dev); + if (ret) + goto err_gpio; + + ret = dra7xx_add_pcie_ep(dra7xx, pdev); + if (ret < 0) + goto err_gpio; + break; + default: + dev_err(dev, "INVALID device type %d\n", mode); + } + dra7xx->pci->mode = mode; + + ret = devm_request_threaded_irq(dev, irq, NULL, dra7xx_pcie_irq_handler, + IRQF_SHARED | IRQF_ONESHOT, + "dra7xx-pcie-main", dra7xx); + if (ret) { + dev_err(dev, "failed to request irq\n"); + goto err_deinit; + } + + return 0; + +err_deinit: + if (dra7xx->pci->mode == DW_PCIE_RC_TYPE) + dw_pcie_host_deinit(&dra7xx->pci->pp); + else + dw_pcie_ep_deinit(&dra7xx->pci->ep); + +err_gpio: +err_get_sync: + pm_runtime_put(dev); + pm_runtime_disable(dev); + dra7xx_pcie_disable_phy(dra7xx); + +err_link: + while (--i >= 0) + device_link_del(link[i]); + + return ret; +} + +static int dra7xx_pcie_suspend(struct device *dev) +{ + struct dra7xx_pcie *dra7xx = dev_get_drvdata(dev); + struct dw_pcie *pci = dra7xx->pci; + u32 val; + + if (pci->mode != DW_PCIE_RC_TYPE) + return 0; + + /* clear MSE */ + val = dw_pcie_readl_dbi(pci, PCI_COMMAND); + val &= ~PCI_COMMAND_MEMORY; + dw_pcie_writel_dbi(pci, PCI_COMMAND, val); + + return 0; +} + +static int dra7xx_pcie_resume(struct device *dev) +{ + struct dra7xx_pcie *dra7xx = dev_get_drvdata(dev); + struct dw_pcie *pci = dra7xx->pci; + u32 val; + + if (pci->mode != DW_PCIE_RC_TYPE) + return 0; + + /* set MSE */ + val = dw_pcie_readl_dbi(pci, PCI_COMMAND); + val |= PCI_COMMAND_MEMORY; + dw_pcie_writel_dbi(pci, PCI_COMMAND, val); + + return 0; +} + +static int dra7xx_pcie_suspend_noirq(struct device *dev) +{ + struct dra7xx_pcie *dra7xx = dev_get_drvdata(dev); + + dra7xx_pcie_disable_phy(dra7xx); + + return 0; +} + +static int dra7xx_pcie_resume_noirq(struct device *dev) +{ + struct dra7xx_pcie *dra7xx = dev_get_drvdata(dev); + int ret; + + ret = dra7xx_pcie_enable_phy(dra7xx); + if (ret) { + dev_err(dev, "failed to enable phy\n"); + return ret; + } + + return 0; +} + +static void dra7xx_pcie_shutdown(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dra7xx_pcie *dra7xx = dev_get_drvdata(dev); + int ret; + + dra7xx_pcie_stop_link(dra7xx->pci); + + ret = pm_runtime_put_sync(dev); + if (ret < 0) + dev_dbg(dev, "pm_runtime_put_sync failed\n"); + + pm_runtime_disable(dev); + dra7xx_pcie_disable_phy(dra7xx); + + clk_disable_unprepare(dra7xx->clk); +} + +static const struct dev_pm_ops dra7xx_pcie_pm_ops = { + SYSTEM_SLEEP_PM_OPS(dra7xx_pcie_suspend, dra7xx_pcie_resume) + NOIRQ_SYSTEM_SLEEP_PM_OPS(dra7xx_pcie_suspend_noirq, + dra7xx_pcie_resume_noirq) +}; + +static struct platform_driver dra7xx_pcie_driver = { + .probe = dra7xx_pcie_probe, + .driver = { + .name = "dra7-pcie", + .of_match_table = of_dra7xx_pcie_match, + .suppress_bind_attrs = true, + .pm = &dra7xx_pcie_pm_ops, + }, + .shutdown = dra7xx_pcie_shutdown, +}; +module_platform_driver(dra7xx_pcie_driver); + +MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>"); +MODULE_DESCRIPTION("PCIe controller driver for TI DRA7xx SoCs"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/dwc/pci-exynos.c b/drivers/pci/controller/dwc/pci-exynos.c new file mode 100644 index 000000000..e3c05ade3 --- /dev/null +++ b/drivers/pci/controller/dwc/pci-exynos.c @@ -0,0 +1,395 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for Samsung Exynos SoCs + * + * Copyright (C) 2013-2020 Samsung Electronics Co., Ltd. + * https://www.samsung.com + * + * Author: Jingoo Han <jg1.han@samsung.com> + * Jaehoon Chung <jh80.chung@samsung.com> + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/phy/phy.h> +#include <linux/regulator/consumer.h> +#include <linux/module.h> + +#include "pcie-designware.h" + +#define to_exynos_pcie(x) dev_get_drvdata((x)->dev) + +/* PCIe ELBI registers */ +#define PCIE_IRQ_PULSE 0x000 +#define IRQ_INTA_ASSERT BIT(0) +#define IRQ_INTB_ASSERT BIT(2) +#define IRQ_INTC_ASSERT BIT(4) +#define IRQ_INTD_ASSERT BIT(6) +#define PCIE_IRQ_LEVEL 0x004 +#define PCIE_IRQ_SPECIAL 0x008 +#define PCIE_IRQ_EN_PULSE 0x00c +#define PCIE_IRQ_EN_LEVEL 0x010 +#define PCIE_IRQ_EN_SPECIAL 0x014 +#define PCIE_SW_WAKE 0x018 +#define PCIE_BUS_EN BIT(1) +#define PCIE_CORE_RESET 0x01c +#define PCIE_CORE_RESET_ENABLE BIT(0) +#define PCIE_STICKY_RESET 0x020 +#define PCIE_NONSTICKY_RESET 0x024 +#define PCIE_APP_INIT_RESET 0x028 +#define PCIE_APP_LTSSM_ENABLE 0x02c +#define PCIE_ELBI_RDLH_LINKUP 0x074 +#define PCIE_ELBI_XMLH_LINKUP BIT(4) +#define PCIE_ELBI_LTSSM_ENABLE 0x1 +#define PCIE_ELBI_SLV_AWMISC 0x11c +#define PCIE_ELBI_SLV_ARMISC 0x120 +#define PCIE_ELBI_SLV_DBI_ENABLE BIT(21) + +struct exynos_pcie { + struct dw_pcie pci; + struct clk_bulk_data *clks; + struct phy *phy; + struct regulator_bulk_data supplies[2]; +}; + +static void exynos_pcie_writel(void __iomem *base, u32 val, u32 reg) +{ + writel(val, base + reg); +} + +static u32 exynos_pcie_readl(void __iomem *base, u32 reg) +{ + return readl(base + reg); +} + +static void exynos_pcie_sideband_dbi_w_mode(struct exynos_pcie *ep, bool on) +{ + struct dw_pcie *pci = &ep->pci; + u32 val; + + val = exynos_pcie_readl(pci->elbi_base, PCIE_ELBI_SLV_AWMISC); + if (on) + val |= PCIE_ELBI_SLV_DBI_ENABLE; + else + val &= ~PCIE_ELBI_SLV_DBI_ENABLE; + exynos_pcie_writel(pci->elbi_base, val, PCIE_ELBI_SLV_AWMISC); +} + +static void exynos_pcie_sideband_dbi_r_mode(struct exynos_pcie *ep, bool on) +{ + struct dw_pcie *pci = &ep->pci; + u32 val; + + val = exynos_pcie_readl(pci->elbi_base, PCIE_ELBI_SLV_ARMISC); + if (on) + val |= PCIE_ELBI_SLV_DBI_ENABLE; + else + val &= ~PCIE_ELBI_SLV_DBI_ENABLE; + exynos_pcie_writel(pci->elbi_base, val, PCIE_ELBI_SLV_ARMISC); +} + +static void exynos_pcie_assert_core_reset(struct exynos_pcie *ep) +{ + struct dw_pcie *pci = &ep->pci; + u32 val; + + val = exynos_pcie_readl(pci->elbi_base, PCIE_CORE_RESET); + val &= ~PCIE_CORE_RESET_ENABLE; + exynos_pcie_writel(pci->elbi_base, val, PCIE_CORE_RESET); + exynos_pcie_writel(pci->elbi_base, 0, PCIE_STICKY_RESET); + exynos_pcie_writel(pci->elbi_base, 0, PCIE_NONSTICKY_RESET); +} + +static void exynos_pcie_deassert_core_reset(struct exynos_pcie *ep) +{ + struct dw_pcie *pci = &ep->pci; + u32 val; + + val = exynos_pcie_readl(pci->elbi_base, PCIE_CORE_RESET); + val |= PCIE_CORE_RESET_ENABLE; + + exynos_pcie_writel(pci->elbi_base, val, PCIE_CORE_RESET); + exynos_pcie_writel(pci->elbi_base, 1, PCIE_STICKY_RESET); + exynos_pcie_writel(pci->elbi_base, 1, PCIE_NONSTICKY_RESET); + exynos_pcie_writel(pci->elbi_base, 1, PCIE_APP_INIT_RESET); + exynos_pcie_writel(pci->elbi_base, 0, PCIE_APP_INIT_RESET); +} + +static int exynos_pcie_start_link(struct dw_pcie *pci) +{ + u32 val; + + val = exynos_pcie_readl(pci->elbi_base, PCIE_SW_WAKE); + val &= ~PCIE_BUS_EN; + exynos_pcie_writel(pci->elbi_base, val, PCIE_SW_WAKE); + + /* assert LTSSM enable */ + exynos_pcie_writel(pci->elbi_base, PCIE_ELBI_LTSSM_ENABLE, + PCIE_APP_LTSSM_ENABLE); + return 0; +} + +static void exynos_pcie_clear_irq_pulse(struct exynos_pcie *ep) +{ + struct dw_pcie *pci = &ep->pci; + + u32 val = exynos_pcie_readl(pci->elbi_base, PCIE_IRQ_PULSE); + + exynos_pcie_writel(pci->elbi_base, val, PCIE_IRQ_PULSE); +} + +static irqreturn_t exynos_pcie_irq_handler(int irq, void *arg) +{ + struct exynos_pcie *ep = arg; + + exynos_pcie_clear_irq_pulse(ep); + return IRQ_HANDLED; +} + +static void exynos_pcie_enable_irq_pulse(struct exynos_pcie *ep) +{ + struct dw_pcie *pci = &ep->pci; + + u32 val = IRQ_INTA_ASSERT | IRQ_INTB_ASSERT | + IRQ_INTC_ASSERT | IRQ_INTD_ASSERT; + + exynos_pcie_writel(pci->elbi_base, val, PCIE_IRQ_EN_PULSE); + exynos_pcie_writel(pci->elbi_base, 0, PCIE_IRQ_EN_LEVEL); + exynos_pcie_writel(pci->elbi_base, 0, PCIE_IRQ_EN_SPECIAL); +} + +static u32 exynos_pcie_read_dbi(struct dw_pcie *pci, void __iomem *base, + u32 reg, size_t size) +{ + struct exynos_pcie *ep = to_exynos_pcie(pci); + u32 val; + + exynos_pcie_sideband_dbi_r_mode(ep, true); + dw_pcie_read(base + reg, size, &val); + exynos_pcie_sideband_dbi_r_mode(ep, false); + return val; +} + +static void exynos_pcie_write_dbi(struct dw_pcie *pci, void __iomem *base, + u32 reg, size_t size, u32 val) +{ + struct exynos_pcie *ep = to_exynos_pcie(pci); + + exynos_pcie_sideband_dbi_w_mode(ep, true); + dw_pcie_write(base + reg, size, val); + exynos_pcie_sideband_dbi_w_mode(ep, false); +} + +static int exynos_pcie_rd_own_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(bus->sysdata); + + if (PCI_SLOT(devfn)) + return PCIBIOS_DEVICE_NOT_FOUND; + + *val = dw_pcie_read_dbi(pci, where, size); + return PCIBIOS_SUCCESSFUL; +} + +static int exynos_pcie_wr_own_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(bus->sysdata); + + if (PCI_SLOT(devfn)) + return PCIBIOS_DEVICE_NOT_FOUND; + + dw_pcie_write_dbi(pci, where, size, val); + return PCIBIOS_SUCCESSFUL; +} + +static struct pci_ops exynos_pci_ops = { + .read = exynos_pcie_rd_own_conf, + .write = exynos_pcie_wr_own_conf, +}; + +static bool exynos_pcie_link_up(struct dw_pcie *pci) +{ + u32 val = exynos_pcie_readl(pci->elbi_base, PCIE_ELBI_RDLH_LINKUP); + + return val & PCIE_ELBI_XMLH_LINKUP; +} + +static int exynos_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct exynos_pcie *ep = to_exynos_pcie(pci); + + pp->bridge->ops = &exynos_pci_ops; + + exynos_pcie_assert_core_reset(ep); + + phy_init(ep->phy); + phy_power_on(ep->phy); + + exynos_pcie_deassert_core_reset(ep); + exynos_pcie_enable_irq_pulse(ep); + + return 0; +} + +static const struct dw_pcie_host_ops exynos_pcie_host_ops = { + .init = exynos_pcie_host_init, +}; + +static int exynos_add_pcie_port(struct exynos_pcie *ep, + struct platform_device *pdev) +{ + struct dw_pcie *pci = &ep->pci; + struct dw_pcie_rp *pp = &pci->pp; + struct device *dev = &pdev->dev; + int ret; + + pp->irq = platform_get_irq(pdev, 0); + if (pp->irq < 0) + return pp->irq; + + ret = devm_request_irq(dev, pp->irq, exynos_pcie_irq_handler, + IRQF_SHARED, "exynos-pcie", ep); + if (ret) { + dev_err(dev, "failed to request irq\n"); + return ret; + } + + pp->ops = &exynos_pcie_host_ops; + pp->msi_irq[0] = -ENODEV; + + ret = dw_pcie_host_init(pp); + if (ret) { + dev_err(dev, "failed to initialize host\n"); + return ret; + } + + return 0; +} + +static const struct dw_pcie_ops dw_pcie_ops = { + .read_dbi = exynos_pcie_read_dbi, + .write_dbi = exynos_pcie_write_dbi, + .link_up = exynos_pcie_link_up, + .start_link = exynos_pcie_start_link, +}; + +static int exynos_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct exynos_pcie *ep; + struct device_node *np = dev->of_node; + int ret; + + ep = devm_kzalloc(dev, sizeof(*ep), GFP_KERNEL); + if (!ep) + return -ENOMEM; + + ep->pci.dev = dev; + ep->pci.ops = &dw_pcie_ops; + + ep->phy = devm_of_phy_get(dev, np, NULL); + if (IS_ERR(ep->phy)) + return PTR_ERR(ep->phy); + + ret = devm_clk_bulk_get_all_enabled(dev, &ep->clks); + if (ret < 0) + return ret; + + ep->supplies[0].supply = "vdd18"; + ep->supplies[1].supply = "vdd10"; + ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(ep->supplies), + ep->supplies); + if (ret) + return ret; + + ret = regulator_bulk_enable(ARRAY_SIZE(ep->supplies), ep->supplies); + if (ret) + return ret; + + platform_set_drvdata(pdev, ep); + + ret = exynos_add_pcie_port(ep, pdev); + if (ret < 0) + goto fail_probe; + + return 0; + +fail_probe: + phy_exit(ep->phy); + regulator_bulk_disable(ARRAY_SIZE(ep->supplies), ep->supplies); + + return ret; +} + +static void exynos_pcie_remove(struct platform_device *pdev) +{ + struct exynos_pcie *ep = platform_get_drvdata(pdev); + + dw_pcie_host_deinit(&ep->pci.pp); + exynos_pcie_assert_core_reset(ep); + phy_power_off(ep->phy); + phy_exit(ep->phy); + regulator_bulk_disable(ARRAY_SIZE(ep->supplies), ep->supplies); +} + +static int exynos_pcie_suspend_noirq(struct device *dev) +{ + struct exynos_pcie *ep = dev_get_drvdata(dev); + + exynos_pcie_assert_core_reset(ep); + phy_power_off(ep->phy); + phy_exit(ep->phy); + regulator_bulk_disable(ARRAY_SIZE(ep->supplies), ep->supplies); + + return 0; +} + +static int exynos_pcie_resume_noirq(struct device *dev) +{ + struct exynos_pcie *ep = dev_get_drvdata(dev); + struct dw_pcie *pci = &ep->pci; + struct dw_pcie_rp *pp = &pci->pp; + int ret; + + ret = regulator_bulk_enable(ARRAY_SIZE(ep->supplies), ep->supplies); + if (ret) + return ret; + + /* exynos_pcie_host_init controls ep->phy */ + exynos_pcie_host_init(pp); + dw_pcie_setup_rc(pp); + exynos_pcie_start_link(pci); + return dw_pcie_wait_for_link(pci); +} + +static const struct dev_pm_ops exynos_pcie_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(exynos_pcie_suspend_noirq, + exynos_pcie_resume_noirq) +}; + +static const struct of_device_id exynos_pcie_of_match[] = { + { .compatible = "samsung,exynos5433-pcie", }, + { }, +}; + +static struct platform_driver exynos_pcie_driver = { + .probe = exynos_pcie_probe, + .remove = exynos_pcie_remove, + .driver = { + .name = "exynos-pcie", + .of_match_table = exynos_pcie_of_match, + .pm = &exynos_pcie_pm_ops, + }, +}; +module_platform_driver(exynos_pcie_driver); +MODULE_DESCRIPTION("Samsung Exynos PCIe host controller driver"); +MODULE_LICENSE("GPL v2"); +MODULE_DEVICE_TABLE(of, exynos_pcie_of_match); diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c new file mode 100644 index 000000000..39790e66b --- /dev/null +++ b/drivers/pci/controller/dwc/pci-imx6.c @@ -0,0 +1,2290 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for Freescale i.MX6 SoCs + * + * Copyright (C) 2013 Kosagi + * https://www.kosagi.com + * + * Author: Sean Cross <xobs@kosagi.com> + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/gpio/consumer.h> +#include <linux/kernel.h> +#include <linux/mfd/syscon.h> +#include <linux/mfd/syscon/imx6q-iomuxc-gpr.h> +#include <linux/mfd/syscon/imx7-iomuxc-gpr.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/pci.h> +#include <linux/pci-pwrctrl.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/regulator/consumer.h> +#include <linux/resource.h> +#include <linux/signal.h> +#include <linux/types.h> +#include <linux/interrupt.h> +#include <linux/reset.h> +#include <linux/phy/pcie.h> +#include <linux/phy/phy.h> +#include <linux/pm_domain.h> +#include <linux/pm_runtime.h> + +#include "../../pci.h" +#include "../pci-host-common.h" +#include "pcie-designware.h" + +#define IMX8MQ_GPR_PCIE_REF_USE_PAD BIT(9) +#define IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN BIT(10) +#define IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE BIT(11) +#define IMX8MQ_GPR_PCIE_VREG_BYPASS BIT(12) +#define IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE GENMASK(11, 8) + +#define IMX95_PCIE_PHY_GEN_CTRL 0x0 +#define IMX95_PCIE_REF_USE_PAD BIT(17) + +#define IMX95_PCIE_PHY_MPLLA_CTRL 0x10 +#define IMX95_PCIE_PHY_MPLL_STATE BIT(30) + +#define IMX95_PCIE_SS_RW_REG_0 0xf0 +#define IMX95_PCIE_REF_CLKEN BIT(23) +#define IMX95_PCIE_PHY_CR_PARA_SEL BIT(9) +#define IMX95_PCIE_SS_RW_REG_1 0xf4 +#define IMX95_PCIE_CLKREQ_OVERRIDE_EN BIT(8) +#define IMX95_PCIE_CLKREQ_OVERRIDE_VAL BIT(9) +#define IMX95_PCIE_SYS_AUX_PWR_DET BIT(31) + +#define IMX95_PE0_GEN_CTRL_1 0x1050 +#define IMX95_PCIE_DEVICE_TYPE GENMASK(3, 0) + +#define IMX95_PE0_GEN_CTRL_3 0x1058 +#define IMX95_PCIE_LTSSM_EN BIT(0) + +#define IMX95_PE0_LUT_ACSCTRL 0x1008 +#define IMX95_PEO_LUT_RWA BIT(16) +#define IMX95_PE0_LUT_ENLOC GENMASK(4, 0) + +#define IMX95_PE0_LUT_DATA1 0x100c +#define IMX95_PE0_LUT_VLD BIT(31) +#define IMX95_PE0_LUT_DAC_ID GENMASK(10, 8) +#define IMX95_PE0_LUT_STREAM_ID GENMASK(5, 0) + +#define IMX95_PE0_LUT_DATA2 0x1010 +#define IMX95_PE0_LUT_REQID GENMASK(31, 16) +#define IMX95_PE0_LUT_MASK GENMASK(15, 0) + +#define IMX95_SID_MASK GENMASK(5, 0) +#define IMX95_MAX_LUT 32 + +#define IMX95_PCIE_RST_CTRL 0x3010 +#define IMX95_PCIE_COLD_RST BIT(0) + +#define to_imx_pcie(x) dev_get_drvdata((x)->dev) + +enum imx_pcie_variants { + IMX6Q, + IMX6SX, + IMX6QP, + IMX7D, + IMX8MQ, + IMX8MM, + IMX8MP, + IMX8Q, + IMX95, + IMX8MQ_EP, + IMX8MM_EP, + IMX8MP_EP, + IMX8Q_EP, + IMX95_EP, +}; + +#define IMX_PCIE_FLAG_IMX_PHY BIT(0) +#define IMX_PCIE_FLAG_SPEED_CHANGE_WORKAROUND BIT(1) +#define IMX_PCIE_FLAG_SUPPORTS_SUSPEND BIT(2) +#define IMX_PCIE_FLAG_HAS_PHYDRV BIT(3) +#define IMX_PCIE_FLAG_HAS_APP_RESET BIT(4) +#define IMX_PCIE_FLAG_HAS_PHY_RESET BIT(5) +#define IMX_PCIE_FLAG_HAS_SERDES BIT(6) +#define IMX_PCIE_FLAG_SUPPORT_64BIT BIT(7) +#define IMX_PCIE_FLAG_CPU_ADDR_FIXUP BIT(8) +/* + * Because of ERR005723 (PCIe does not support L2 power down) we need to + * workaround suspend resume on some devices which are affected by this errata. + */ +#define IMX_PCIE_FLAG_BROKEN_SUSPEND BIT(9) +#define IMX_PCIE_FLAG_HAS_LUT BIT(10) +#define IMX_PCIE_FLAG_8GT_ECN_ERR051586 BIT(11) +#define IMX_PCIE_FLAG_SKIP_L23_READY BIT(12) +/* Preserve MSI capability for platforms that require it */ +#define IMX_PCIE_FLAG_KEEP_MSI_CAP BIT(13) +#define IMX_PCIE_FLAG_PM_RUNTIME BIT(14) + +#define imx_check_flag(pci, val) (pci->drvdata->flags & val) + +#define IMX_PCIE_MAX_INSTANCES 2 + +struct imx_pcie; + +struct imx_pcie_drvdata { + enum imx_pcie_variants variant; + enum dw_pcie_device_mode mode; + u32 flags; + int dbi_length; + const char *gpr; + const u32 ltssm_off; + const u32 ltssm_mask; + const u32 mode_off[IMX_PCIE_MAX_INSTANCES]; + const u32 mode_mask[IMX_PCIE_MAX_INSTANCES]; + const struct pci_epc_features *epc_features; + int (*select_ref_clk_src)(struct imx_pcie *pcie); + int (*init_phy)(struct imx_pcie *pcie); + int (*enable_ref_clk)(struct imx_pcie *pcie, bool enable); + int (*core_reset)(struct imx_pcie *pcie, bool assert); + int (*wait_pll_lock)(struct imx_pcie *pcie); + void (*clr_clkreq_override)(struct imx_pcie *pcie); + const struct dw_pcie_host_ops *ops; +}; + +struct imx_lut_data { + u32 data1; + u32 data2; +}; + +struct imx_pcie { + struct dw_pcie *pci; + struct clk_bulk_data *clks; + int num_clks; + bool supports_clkreq; + bool enable_ext_refclk; + struct regmap *iomuxc_gpr; + u16 msi_ctrl; + u32 controller_id; + struct reset_control *pciephy_reset; + struct reset_control *apps_reset; + u32 tx_deemph_gen1; + u32 tx_deemph_gen2_3p5db; + u32 tx_deemph_gen2_6db; + u32 tx_swing_full; + u32 tx_swing_low; + struct regulator *vpcie; + struct regulator *vpcie_aux; + bool vpcie_aux_enabled; + struct regulator *vph; + void __iomem *phy_base; + + /* LUT data for pcie */ + struct imx_lut_data luts[IMX95_MAX_LUT]; + /* power domain for pcie */ + struct device *pd_pcie; + /* power domain for pcie phy */ + struct device *pd_pcie_phy; + struct phy *phy; + const struct imx_pcie_drvdata *drvdata; + + /* Ensure that only one device's LUT is configured at any given time */ + struct mutex lock; +}; + +/* Parameters for the waiting for PCIe PHY PLL to lock on i.MX7 */ +#define PHY_PLL_LOCK_WAIT_USLEEP_MAX 200 +#define PHY_PLL_LOCK_WAIT_TIMEOUT (2000 * PHY_PLL_LOCK_WAIT_USLEEP_MAX) + +/* PCIe Port Logic registers (memory-mapped) */ +#define PL_OFFSET 0x700 + +#define PCIE_PHY_CTRL (PL_OFFSET + 0x114) +#define PCIE_PHY_CTRL_DATA(x) FIELD_PREP(GENMASK(15, 0), (x)) +#define PCIE_PHY_CTRL_CAP_ADR BIT(16) +#define PCIE_PHY_CTRL_CAP_DAT BIT(17) +#define PCIE_PHY_CTRL_WR BIT(18) +#define PCIE_PHY_CTRL_RD BIT(19) + +#define PCIE_PHY_STAT (PL_OFFSET + 0x110) +#define PCIE_PHY_STAT_ACK BIT(16) + +/* PHY registers (not memory-mapped) */ +#define PCIE_PHY_ATEOVRD 0x10 +#define PCIE_PHY_ATEOVRD_EN BIT(2) +#define PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT 0 +#define PCIE_PHY_ATEOVRD_REF_CLKDIV_MASK 0x1 + +#define PCIE_PHY_MPLL_OVRD_IN_LO 0x11 +#define PCIE_PHY_MPLL_MULTIPLIER_SHIFT 2 +#define PCIE_PHY_MPLL_MULTIPLIER_MASK 0x7f +#define PCIE_PHY_MPLL_MULTIPLIER_OVRD BIT(9) + +#define PCIE_PHY_RX_ASIC_OUT 0x100D +#define PCIE_PHY_RX_ASIC_OUT_VALID (1 << 0) + +/* iMX7 PCIe PHY registers */ +#define PCIE_PHY_CMN_REG4 0x14 +/* These are probably the bits that *aren't* DCC_FB_EN */ +#define PCIE_PHY_CMN_REG4_DCC_FB_EN 0x29 + +#define PCIE_PHY_CMN_REG15 0x54 +#define PCIE_PHY_CMN_REG15_DLY_4 BIT(2) +#define PCIE_PHY_CMN_REG15_PLL_PD BIT(5) +#define PCIE_PHY_CMN_REG15_OVRD_PLL_PD BIT(7) + +#define PCIE_PHY_CMN_REG24 0x90 +#define PCIE_PHY_CMN_REG24_RX_EQ BIT(6) +#define PCIE_PHY_CMN_REG24_RX_EQ_SEL BIT(3) + +#define PCIE_PHY_CMN_REG26 0x98 +#define PCIE_PHY_CMN_REG26_ATT_MODE 0xBC + +#define PHY_RX_OVRD_IN_LO 0x1005 +#define PHY_RX_OVRD_IN_LO_RX_DATA_EN BIT(5) +#define PHY_RX_OVRD_IN_LO_RX_PLL_EN BIT(3) + +static unsigned int imx_pcie_grp_offset(const struct imx_pcie *imx_pcie) +{ + WARN_ON(imx_pcie->drvdata->variant != IMX8MQ && + imx_pcie->drvdata->variant != IMX8MQ_EP && + imx_pcie->drvdata->variant != IMX8MM && + imx_pcie->drvdata->variant != IMX8MM_EP && + imx_pcie->drvdata->variant != IMX8MP && + imx_pcie->drvdata->variant != IMX8MP_EP); + return imx_pcie->controller_id == 1 ? IOMUXC_GPR16 : IOMUXC_GPR14; +} + +static int imx95_pcie_select_ref_clk_src(struct imx_pcie *imx_pcie) +{ + bool ext = imx_pcie->enable_ext_refclk; + + /* + * Regarding the Signal Descriptions of i.MX95 PCIe PHY, ref_use_pad is + * used to select reference clock connected to a pair of pads. + * + * Any change in this input must be followed by phy_reset assertion. + */ + + regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_PHY_GEN_CTRL, + IMX95_PCIE_REF_USE_PAD, + ext ? IMX95_PCIE_REF_USE_PAD : 0); + + return 0; +} + +static int imx95_pcie_init_phy(struct imx_pcie *imx_pcie) +{ + /* + * ERR051624: The Controller Without Vaux Cannot Exit L23 Ready + * Through Beacon or PERST# De-assertion + * + * When the auxiliary power is not available, the controller + * cannot exit from L23 Ready with beacon or PERST# de-assertion + * when main power is not removed. + * + * Workaround: Set SS_RW_REG_1[SYS_AUX_PWR_DET] to 1. + */ + regmap_set_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_1, + IMX95_PCIE_SYS_AUX_PWR_DET); + + regmap_update_bits(imx_pcie->iomuxc_gpr, + IMX95_PCIE_SS_RW_REG_0, + IMX95_PCIE_PHY_CR_PARA_SEL, + IMX95_PCIE_PHY_CR_PARA_SEL); + + return 0; +} + +static void imx_pcie_configure_type(struct imx_pcie *imx_pcie) +{ + const struct imx_pcie_drvdata *drvdata = imx_pcie->drvdata; + unsigned int mask, val, mode, id; + + if (drvdata->mode == DW_PCIE_EP_TYPE) + mode = PCI_EXP_TYPE_ENDPOINT; + else + mode = PCI_EXP_TYPE_ROOT_PORT; + + id = imx_pcie->controller_id; + + /* If mode_mask is 0, generic PHY driver is used to set the mode */ + if (!drvdata->mode_mask[0]) + return; + + /* If mode_mask[id] is 0, each controller has its individual GPR */ + if (!drvdata->mode_mask[id]) + id = 0; + + mask = drvdata->mode_mask[id]; + val = mode << (ffs(mask) - 1); + + regmap_update_bits(imx_pcie->iomuxc_gpr, drvdata->mode_off[id], mask, val); +} + +static int pcie_phy_poll_ack(struct imx_pcie *imx_pcie, bool exp_val) +{ + struct dw_pcie *pci = imx_pcie->pci; + bool val; + u32 max_iterations = 10; + u32 wait_counter = 0; + + do { + val = dw_pcie_readl_dbi(pci, PCIE_PHY_STAT) & + PCIE_PHY_STAT_ACK; + wait_counter++; + + if (val == exp_val) + return 0; + + udelay(1); + } while (wait_counter < max_iterations); + + return -ETIMEDOUT; +} + +static int pcie_phy_wait_ack(struct imx_pcie *imx_pcie, int addr) +{ + struct dw_pcie *pci = imx_pcie->pci; + u32 val; + int ret; + + val = PCIE_PHY_CTRL_DATA(addr); + dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val); + + val |= PCIE_PHY_CTRL_CAP_ADR; + dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val); + + ret = pcie_phy_poll_ack(imx_pcie, true); + if (ret) + return ret; + + val = PCIE_PHY_CTRL_DATA(addr); + dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, val); + + return pcie_phy_poll_ack(imx_pcie, false); +} + +/* Read from the 16-bit PCIe PHY control registers (not memory-mapped) */ +static int pcie_phy_read(struct imx_pcie *imx_pcie, int addr, u16 *data) +{ + struct dw_pcie *pci = imx_pcie->pci; + u32 phy_ctl; + int ret; + + ret = pcie_phy_wait_ack(imx_pcie, addr); + if (ret) + return ret; + + /* assert Read signal */ + phy_ctl = PCIE_PHY_CTRL_RD; + dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, phy_ctl); + + ret = pcie_phy_poll_ack(imx_pcie, true); + if (ret) + return ret; + + *data = dw_pcie_readl_dbi(pci, PCIE_PHY_STAT); + + /* deassert Read signal */ + dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, 0x00); + + return pcie_phy_poll_ack(imx_pcie, false); +} + +static int pcie_phy_write(struct imx_pcie *imx_pcie, int addr, u16 data) +{ + struct dw_pcie *pci = imx_pcie->pci; + u32 var; + int ret; + + /* write addr */ + /* cap addr */ + ret = pcie_phy_wait_ack(imx_pcie, addr); + if (ret) + return ret; + + var = PCIE_PHY_CTRL_DATA(data); + dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var); + + /* capture data */ + var |= PCIE_PHY_CTRL_CAP_DAT; + dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var); + + ret = pcie_phy_poll_ack(imx_pcie, true); + if (ret) + return ret; + + /* deassert cap data */ + var = PCIE_PHY_CTRL_DATA(data); + dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var); + + /* wait for ack de-assertion */ + ret = pcie_phy_poll_ack(imx_pcie, false); + if (ret) + return ret; + + /* assert wr signal */ + var = PCIE_PHY_CTRL_WR; + dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var); + + /* wait for ack */ + ret = pcie_phy_poll_ack(imx_pcie, true); + if (ret) + return ret; + + /* deassert wr signal */ + var = PCIE_PHY_CTRL_DATA(data); + dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, var); + + /* wait for ack de-assertion */ + ret = pcie_phy_poll_ack(imx_pcie, false); + if (ret) + return ret; + + dw_pcie_writel_dbi(pci, PCIE_PHY_CTRL, 0x0); + + return 0; +} + +static int imx8mq_pcie_init_phy(struct imx_pcie *imx_pcie) +{ + /* TODO: This code assumes external oscillator is being used */ + regmap_update_bits(imx_pcie->iomuxc_gpr, + imx_pcie_grp_offset(imx_pcie), + IMX8MQ_GPR_PCIE_REF_USE_PAD, + IMX8MQ_GPR_PCIE_REF_USE_PAD); + /* + * Per the datasheet, the PCIE_VPH is suggested to be 1.8V. If the + * PCIE_VPH is supplied by 3.3V, the VREG_BYPASS should be cleared + * to zero. + */ + if (imx_pcie->vph && regulator_get_voltage(imx_pcie->vph) > 3000000) + regmap_update_bits(imx_pcie->iomuxc_gpr, + imx_pcie_grp_offset(imx_pcie), + IMX8MQ_GPR_PCIE_VREG_BYPASS, + 0); + + return 0; +} + +static int imx_pcie_init_phy(struct imx_pcie *imx_pcie) +{ + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, + IMX6Q_GPR12_PCIE_CTL_2, 0 << 10); + + /* configure constant input signal to the pcie ctrl and phy */ + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, + IMX6Q_GPR12_LOS_LEVEL, 9 << 4); + + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8, + IMX6Q_GPR8_TX_DEEMPH_GEN1, + imx_pcie->tx_deemph_gen1 << 0); + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8, + IMX6Q_GPR8_TX_DEEMPH_GEN2_3P5DB, + imx_pcie->tx_deemph_gen2_3p5db << 6); + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8, + IMX6Q_GPR8_TX_DEEMPH_GEN2_6DB, + imx_pcie->tx_deemph_gen2_6db << 12); + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8, + IMX6Q_GPR8_TX_SWING_FULL, + imx_pcie->tx_swing_full << 18); + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR8, + IMX6Q_GPR8_TX_SWING_LOW, + imx_pcie->tx_swing_low << 25); + return 0; +} + +static int imx6sx_pcie_init_phy(struct imx_pcie *imx_pcie) +{ + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, + IMX6SX_GPR12_PCIE_RX_EQ_MASK, IMX6SX_GPR12_PCIE_RX_EQ_2); + + return imx_pcie_init_phy(imx_pcie); +} + +static void imx7d_pcie_wait_for_phy_pll_lock(struct imx_pcie *imx_pcie) +{ + u32 val; + struct device *dev = imx_pcie->pci->dev; + + if (regmap_read_poll_timeout(imx_pcie->iomuxc_gpr, + IOMUXC_GPR22, val, + val & IMX7D_GPR22_PCIE_PHY_PLL_LOCKED, + PHY_PLL_LOCK_WAIT_USLEEP_MAX, + PHY_PLL_LOCK_WAIT_TIMEOUT)) + dev_err(dev, "PCIe PLL lock timeout\n"); +} + +static int imx95_pcie_wait_for_phy_pll_lock(struct imx_pcie *imx_pcie) +{ + u32 val; + struct device *dev = imx_pcie->pci->dev; + + if (regmap_read_poll_timeout(imx_pcie->iomuxc_gpr, + IMX95_PCIE_PHY_MPLLA_CTRL, val, + val & IMX95_PCIE_PHY_MPLL_STATE, + PHY_PLL_LOCK_WAIT_USLEEP_MAX, + PHY_PLL_LOCK_WAIT_TIMEOUT)) { + dev_err(dev, "PCIe PLL lock timeout\n"); + return -ETIMEDOUT; + } + + return 0; +} + +static int imx_setup_phy_mpll(struct imx_pcie *imx_pcie) +{ + unsigned long phy_rate = 0; + int mult, div; + u16 val; + int i; + struct clk_bulk_data *clks = imx_pcie->clks; + + if (!(imx_pcie->drvdata->flags & IMX_PCIE_FLAG_IMX_PHY)) + return 0; + + for (i = 0; i < imx_pcie->num_clks; i++) + if (strncmp(clks[i].id, "pcie_phy", 8) == 0) + phy_rate = clk_get_rate(clks[i].clk); + + switch (phy_rate) { + case 125000000: + /* + * The default settings of the MPLL are for a 125MHz input + * clock, so no need to reconfigure anything in that case. + */ + return 0; + case 100000000: + mult = 25; + div = 0; + break; + case 200000000: + mult = 25; + div = 1; + break; + default: + dev_err(imx_pcie->pci->dev, + "Unsupported PHY reference clock rate %lu\n", phy_rate); + return -EINVAL; + } + + pcie_phy_read(imx_pcie, PCIE_PHY_MPLL_OVRD_IN_LO, &val); + val &= ~(PCIE_PHY_MPLL_MULTIPLIER_MASK << + PCIE_PHY_MPLL_MULTIPLIER_SHIFT); + val |= mult << PCIE_PHY_MPLL_MULTIPLIER_SHIFT; + val |= PCIE_PHY_MPLL_MULTIPLIER_OVRD; + pcie_phy_write(imx_pcie, PCIE_PHY_MPLL_OVRD_IN_LO, val); + + pcie_phy_read(imx_pcie, PCIE_PHY_ATEOVRD, &val); + val &= ~(PCIE_PHY_ATEOVRD_REF_CLKDIV_MASK << + PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT); + val |= div << PCIE_PHY_ATEOVRD_REF_CLKDIV_SHIFT; + val |= PCIE_PHY_ATEOVRD_EN; + pcie_phy_write(imx_pcie, PCIE_PHY_ATEOVRD, val); + + return 0; +} + +static void imx_pcie_reset_phy(struct imx_pcie *imx_pcie) +{ + u16 tmp; + + if (!(imx_pcie->drvdata->flags & IMX_PCIE_FLAG_IMX_PHY)) + return; + + pcie_phy_read(imx_pcie, PHY_RX_OVRD_IN_LO, &tmp); + tmp |= (PHY_RX_OVRD_IN_LO_RX_DATA_EN | + PHY_RX_OVRD_IN_LO_RX_PLL_EN); + pcie_phy_write(imx_pcie, PHY_RX_OVRD_IN_LO, tmp); + + usleep_range(2000, 3000); + + pcie_phy_read(imx_pcie, PHY_RX_OVRD_IN_LO, &tmp); + tmp &= ~(PHY_RX_OVRD_IN_LO_RX_DATA_EN | + PHY_RX_OVRD_IN_LO_RX_PLL_EN); + pcie_phy_write(imx_pcie, PHY_RX_OVRD_IN_LO, tmp); +} + +#ifdef CONFIG_ARM +/* Added for PCI abort handling */ +static int imx6q_pcie_abort_handler(unsigned long addr, + unsigned int fsr, struct pt_regs *regs) +{ + unsigned long pc = instruction_pointer(regs); + unsigned long instr = *(unsigned long *)pc; + int reg = (instr >> 12) & 15; + + /* + * If the instruction being executed was a read, + * make it look like it read all-ones. + */ + if ((instr & 0x0c100000) == 0x04100000) { + unsigned long val; + + if (instr & 0x00400000) + val = 255; + else + val = -1; + + regs->uregs[reg] = val; + regs->ARM_pc += 4; + return 0; + } + + if ((instr & 0x0e100090) == 0x00100090) { + regs->uregs[reg] = -1; + regs->ARM_pc += 4; + return 0; + } + + return 1; +} +#endif + +static int imx_pcie_attach_pd(struct device *dev) +{ + struct imx_pcie *imx_pcie = dev_get_drvdata(dev); + struct device_link *link; + + /* Do nothing when in a single power domain */ + if (dev->pm_domain) + return 0; + + imx_pcie->pd_pcie = dev_pm_domain_attach_by_name(dev, "pcie"); + if (IS_ERR(imx_pcie->pd_pcie)) + return PTR_ERR(imx_pcie->pd_pcie); + /* Do nothing when power domain missing */ + if (!imx_pcie->pd_pcie) + return 0; + link = device_link_add(dev, imx_pcie->pd_pcie, + DL_FLAG_STATELESS | + DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE); + if (!link) { + dev_err(dev, "Failed to add device_link to pcie pd\n"); + return -EINVAL; + } + + imx_pcie->pd_pcie_phy = dev_pm_domain_attach_by_name(dev, "pcie_phy"); + if (IS_ERR(imx_pcie->pd_pcie_phy)) + return PTR_ERR(imx_pcie->pd_pcie_phy); + + link = device_link_add(dev, imx_pcie->pd_pcie_phy, + DL_FLAG_STATELESS | + DL_FLAG_PM_RUNTIME | + DL_FLAG_RPM_ACTIVE); + if (!link) { + dev_err(dev, "Failed to add device_link to pcie_phy pd\n"); + return -EINVAL; + } + + return 0; +} + +static int imx6q_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable) +{ + if (enable) + regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, + IMX6Q_GPR1_PCIE_REF_CLK_EN); + else + regmap_clear_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, + IMX6Q_GPR1_PCIE_REF_CLK_EN); + + return 0; +} + +static void imx8mm_pcie_clkreq_override(struct imx_pcie *imx_pcie, bool enable) +{ + int offset = imx_pcie_grp_offset(imx_pcie); + + regmap_update_bits(imx_pcie->iomuxc_gpr, offset, + IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE, + enable ? 0 : IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE); + regmap_update_bits(imx_pcie->iomuxc_gpr, offset, + IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN, + enable ? IMX8MQ_GPR_PCIE_CLK_REQ_OVERRIDE_EN : 0); +} + +static int imx8mm_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable) +{ + imx8mm_pcie_clkreq_override(imx_pcie, enable); + return 0; +} + +static int imx7d_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable) +{ + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, + IMX7D_GPR12_PCIE_PHY_REFCLK_SEL, + enable ? 0 : IMX7D_GPR12_PCIE_PHY_REFCLK_SEL); + return 0; +} + +static void imx95_pcie_clkreq_override(struct imx_pcie *imx_pcie, bool enable) +{ + regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_1, + IMX95_PCIE_CLKREQ_OVERRIDE_EN, + enable ? IMX95_PCIE_CLKREQ_OVERRIDE_EN : 0); + regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_1, + IMX95_PCIE_CLKREQ_OVERRIDE_VAL, + enable ? IMX95_PCIE_CLKREQ_OVERRIDE_VAL : 0); +} + +static int imx95_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable) +{ + bool ext = imx_pcie->enable_ext_refclk; + + imx95_pcie_clkreq_override(imx_pcie, enable); + /* + * The ref_clk_en signal must remain de-asserted until the + * reference clock is running at appropriate frequency, at which + * point this bit can be asserted. For lower power states where + * the reference clock to the PHY is disabled, it may also be + * de-asserted. + * +------------------- -+--------+----------------+ + * | External clock mode | Enable | PCIE_REF_CLKEN | + * +---------------------+--------+----------------+ + * | TRUE | X | 1b'0 | + * +---------------------+--------+----------------+ + * | FALSE | TRUE | 1b'1 | + * +---------------------+--------+----------------+ + * | FALSE | FALSE | 1b'0 | + * +---------------------+--------+----------------+ + */ + regmap_update_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_SS_RW_REG_0, + IMX95_PCIE_REF_CLKEN, + ext || !enable ? 0 : IMX95_PCIE_REF_CLKEN); + + return 0; +} + +static void imx8mm_pcie_clr_clkreq_override(struct imx_pcie *imx_pcie) +{ + imx8mm_pcie_clkreq_override(imx_pcie, false); +} + +static void imx95_pcie_clr_clkreq_override(struct imx_pcie *imx_pcie) +{ + imx95_pcie_clkreq_override(imx_pcie, false); +} + +static int imx_pcie_clk_enable(struct imx_pcie *imx_pcie) +{ + struct dw_pcie *pci = imx_pcie->pci; + struct device *dev = pci->dev; + int ret; + + ret = clk_bulk_prepare_enable(imx_pcie->num_clks, imx_pcie->clks); + if (ret) + return ret; + + if (imx_pcie->drvdata->enable_ref_clk) { + ret = imx_pcie->drvdata->enable_ref_clk(imx_pcie, true); + if (ret) { + dev_err(dev, "Failed to enable PCIe REFCLK\n"); + goto err_ref_clk; + } + } + + /* allow the clocks to stabilize */ + usleep_range(200, 500); + return 0; + +err_ref_clk: + clk_bulk_disable_unprepare(imx_pcie->num_clks, imx_pcie->clks); + + return ret; +} + +static void imx_pcie_clk_disable(struct imx_pcie *imx_pcie) +{ + if (imx_pcie->drvdata->enable_ref_clk) + imx_pcie->drvdata->enable_ref_clk(imx_pcie, false); + clk_bulk_disable_unprepare(imx_pcie->num_clks, imx_pcie->clks); +} + +static int imx6sx_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert) +{ + if (assert) + regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, + IMX6SX_GPR12_PCIE_TEST_POWERDOWN); + else + regmap_clear_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, + IMX6SX_GPR12_PCIE_TEST_POWERDOWN); + + /* Force PCIe PHY reset */ + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR5, IMX6SX_GPR5_PCIE_BTNRST_RESET, + assert ? IMX6SX_GPR5_PCIE_BTNRST_RESET : 0); + return 0; +} + +static int imx6qp_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert) +{ + if (assert) + regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, + IMX6Q_GPR1_PCIE_TEST_PD); + else + regmap_clear_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, + IMX6Q_GPR1_PCIE_TEST_PD); + + regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, IMX6Q_GPR1_PCIE_SW_RST, + assert ? IMX6Q_GPR1_PCIE_SW_RST : 0); + + if (!assert) + usleep_range(200, 500); + + return 0; +} + +static int imx6q_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert) +{ + if (assert) + regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, + IMX6Q_GPR1_PCIE_TEST_PD); + else + regmap_clear_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR1, + IMX6Q_GPR1_PCIE_TEST_PD); + + if (!assert) + usleep_range(200, 500); + + return 0; +} + +static int imx7d_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert) +{ + struct dw_pcie *pci = imx_pcie->pci; + struct device *dev = pci->dev; + + if (assert) + return 0; + + /* + * Workaround for ERR010728 (IMX7DS_2N09P, Rev. 1.1, 4/2023): + * + * PCIe: PLL may fail to lock under corner conditions. + * + * Initial VCO oscillation may fail under corner conditions such as + * cold temperature which will cause the PCIe PLL fail to lock in the + * initialization phase. + * + * The Duty-cycle Corrector calibration must be disabled. + * + * 1. De-assert the G_RST signal by clearing + * SRC_PCIEPHY_RCR[PCIEPHY_G_RST]. + * 2. De-assert DCC_FB_EN by writing data “0x29” to the register + * address 0x306d0014 (PCIE_PHY_CMN_REG4). + * 3. Assert RX_EQS, RX_EQ_SEL by writing data “0x48” to the register + * address 0x306d0090 (PCIE_PHY_CMN_REG24). + * 4. Assert ATT_MODE by writing data “0xbc” to the register + * address 0x306d0098 (PCIE_PHY_CMN_REG26). + * 5. De-assert the CMN_RST signal by clearing register bit + * SRC_PCIEPHY_RCR[PCIEPHY_BTN] + */ + + if (likely(imx_pcie->phy_base)) { + /* De-assert DCC_FB_EN */ + writel(PCIE_PHY_CMN_REG4_DCC_FB_EN, imx_pcie->phy_base + PCIE_PHY_CMN_REG4); + /* Assert RX_EQS and RX_EQS_SEL */ + writel(PCIE_PHY_CMN_REG24_RX_EQ_SEL | PCIE_PHY_CMN_REG24_RX_EQ, + imx_pcie->phy_base + PCIE_PHY_CMN_REG24); + /* Assert ATT_MODE */ + writel(PCIE_PHY_CMN_REG26_ATT_MODE, imx_pcie->phy_base + PCIE_PHY_CMN_REG26); + } else { + dev_warn(dev, "Unable to apply ERR010728 workaround. DT missing fsl,imx7d-pcie-phy phandle ?\n"); + } + imx7d_pcie_wait_for_phy_pll_lock(imx_pcie); + return 0; +} + +static int imx95_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert) +{ + u32 val; + + if (assert) { + /* + * From i.MX95 PCIe PHY perspective, the COLD reset toggle + * should be complete after power-up by the following sequence. + * > 10us(at power-up) + * > 10ns(warm reset) + * |<------------>| + * ______________ + * phy_reset ____/ \________________ + * ____________ + * ref_clk_en_______________________/ + * Toggle COLD reset aligned with this sequence for i.MX95 PCIe. + */ + regmap_set_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_RST_CTRL, + IMX95_PCIE_COLD_RST); + /* + * Make sure the write to IMX95_PCIE_RST_CTRL is flushed to the + * hardware by doing a read. Otherwise, there is no guarantee + * that the write has reached the hardware before udelay(). + */ + regmap_read_bypassed(imx_pcie->iomuxc_gpr, IMX95_PCIE_RST_CTRL, + &val); + udelay(15); + regmap_clear_bits(imx_pcie->iomuxc_gpr, IMX95_PCIE_RST_CTRL, + IMX95_PCIE_COLD_RST); + regmap_read_bypassed(imx_pcie->iomuxc_gpr, IMX95_PCIE_RST_CTRL, + &val); + udelay(10); + } + + return 0; +} + +static void imx_pcie_assert_core_reset(struct imx_pcie *imx_pcie) +{ + reset_control_assert(imx_pcie->pciephy_reset); + + if (imx_pcie->drvdata->core_reset) + imx_pcie->drvdata->core_reset(imx_pcie, true); +} + +static void imx_pcie_deassert_core_reset(struct imx_pcie *imx_pcie) +{ + reset_control_deassert(imx_pcie->pciephy_reset); + + if (imx_pcie->drvdata->core_reset) + imx_pcie->drvdata->core_reset(imx_pcie, false); +} + +static int imx_pcie_wait_for_speed_change(struct imx_pcie *imx_pcie) +{ + struct dw_pcie *pci = imx_pcie->pci; + struct device *dev = pci->dev; + u32 tmp; + unsigned int retries; + + for (retries = 0; retries < 200; retries++) { + tmp = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); + /* Test if the speed change finished. */ + if (!(tmp & PORT_LOGIC_SPEED_CHANGE)) + return 0; + usleep_range(100, 1000); + } + + dev_err(dev, "Speed change timeout\n"); + return -ETIMEDOUT; +} + +static void imx_pcie_ltssm_enable(struct device *dev) +{ + struct imx_pcie *imx_pcie = dev_get_drvdata(dev); + const struct imx_pcie_drvdata *drvdata = imx_pcie->drvdata; + u8 offset = dw_pcie_find_capability(imx_pcie->pci, PCI_CAP_ID_EXP); + u32 tmp; + + tmp = dw_pcie_readl_dbi(imx_pcie->pci, offset + PCI_EXP_LNKCAP); + phy_set_speed(imx_pcie->phy, FIELD_GET(PCI_EXP_LNKCAP_SLS, tmp)); + if (drvdata->ltssm_mask) + regmap_update_bits(imx_pcie->iomuxc_gpr, drvdata->ltssm_off, drvdata->ltssm_mask, + drvdata->ltssm_mask); + + reset_control_deassert(imx_pcie->apps_reset); +} + +static void imx_pcie_ltssm_disable(struct device *dev) +{ + struct imx_pcie *imx_pcie = dev_get_drvdata(dev); + const struct imx_pcie_drvdata *drvdata = imx_pcie->drvdata; + + phy_set_speed(imx_pcie->phy, 0); + if (drvdata->ltssm_mask) + regmap_update_bits(imx_pcie->iomuxc_gpr, drvdata->ltssm_off, + drvdata->ltssm_mask, 0); + + reset_control_assert(imx_pcie->apps_reset); +} + +static int imx_pcie_start_link(struct dw_pcie *pci) +{ + struct imx_pcie *imx_pcie = to_imx_pcie(pci); + struct device *dev = pci->dev; + u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + u32 tmp; + int ret; + + if (!(imx_pcie->drvdata->flags & + IMX_PCIE_FLAG_SPEED_CHANGE_WORKAROUND)) { + imx_pcie_ltssm_enable(dev); + return 0; + } + + /* + * Force Gen1 operation when starting the link. In case the link is + * started in Gen2 mode, there is a possibility the devices on the + * bus will not be detected at all. This happens with PCIe switches. + */ + dw_pcie_dbi_ro_wr_en(pci); + tmp = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP); + tmp &= ~PCI_EXP_LNKCAP_SLS; + tmp |= PCI_EXP_LNKCAP_SLS_2_5GB; + dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, tmp); + dw_pcie_dbi_ro_wr_dis(pci); + + /* Start LTSSM. */ + imx_pcie_ltssm_enable(dev); + + if (pci->max_link_speed > 1) { + ret = dw_pcie_wait_for_link(pci); + if (ret) + goto err_reset_phy; + + /* Allow faster modes after the link is up */ + dw_pcie_dbi_ro_wr_en(pci); + tmp = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP); + tmp &= ~PCI_EXP_LNKCAP_SLS; + tmp |= pci->max_link_speed; + dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, tmp); + + /* + * Start Directed Speed Change so the best possible + * speed both link partners support can be negotiated. + */ + tmp = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); + tmp |= PORT_LOGIC_SPEED_CHANGE; + dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, tmp); + dw_pcie_dbi_ro_wr_dis(pci); + + ret = imx_pcie_wait_for_speed_change(imx_pcie); + if (ret) { + dev_err(dev, "Failed to bring link up!\n"); + goto err_reset_phy; + } + } else { + dev_info(dev, "Link: Only Gen1 is enabled\n"); + } + + return 0; + +err_reset_phy: + dev_dbg(dev, "PHY DEBUG_R0=0x%08x DEBUG_R1=0x%08x\n", + dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG0), + dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG1)); + imx_pcie_reset_phy(imx_pcie); + return 0; +} + +static void imx_pcie_stop_link(struct dw_pcie *pci) +{ + struct device *dev = pci->dev; + + /* Turn off PCIe LTSSM */ + imx_pcie_ltssm_disable(dev); +} + +static int imx_pcie_add_lut(struct imx_pcie *imx_pcie, u16 rid, u8 sid) +{ + struct dw_pcie *pci = imx_pcie->pci; + struct device *dev = pci->dev; + u32 data1, data2; + int free = -1; + int i; + + if (sid >= 64) { + dev_err(dev, "Invalid SID for index %d\n", sid); + return -EINVAL; + } + + guard(mutex)(&imx_pcie->lock); + + /* + * Iterate through all LUT entries to check for duplicate RID and + * identify the first available entry. Configure this available entry + * immediately after verification to avoid rescanning it. + */ + for (i = 0; i < IMX95_MAX_LUT; i++) { + regmap_write(imx_pcie->iomuxc_gpr, + IMX95_PE0_LUT_ACSCTRL, IMX95_PEO_LUT_RWA | i); + regmap_read(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA1, &data1); + + if (!(data1 & IMX95_PE0_LUT_VLD)) { + if (free < 0) + free = i; + continue; + } + + regmap_read(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA2, &data2); + + /* Do not add duplicate RID */ + if (rid == FIELD_GET(IMX95_PE0_LUT_REQID, data2)) { + dev_warn(dev, "Existing LUT entry available for RID (%d)", rid); + return 0; + } + } + + if (free < 0) { + dev_err(dev, "LUT entry is not available\n"); + return -ENOSPC; + } + + data1 = FIELD_PREP(IMX95_PE0_LUT_DAC_ID, 0); + data1 |= FIELD_PREP(IMX95_PE0_LUT_STREAM_ID, sid); + data1 |= IMX95_PE0_LUT_VLD; + regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA1, data1); + + if (imx_pcie->drvdata->mode == DW_PCIE_EP_TYPE) + data2 = 0x7; /* In the EP mode, only 'Device ID' is required */ + else + data2 = IMX95_PE0_LUT_MASK; /* Match all bits of RID */ + data2 |= FIELD_PREP(IMX95_PE0_LUT_REQID, rid); + regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA2, data2); + + regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_ACSCTRL, free); + + return 0; +} + +static void imx_pcie_remove_lut(struct imx_pcie *imx_pcie, u16 rid) +{ + u32 data2; + int i; + + guard(mutex)(&imx_pcie->lock); + + for (i = 0; i < IMX95_MAX_LUT; i++) { + regmap_write(imx_pcie->iomuxc_gpr, + IMX95_PE0_LUT_ACSCTRL, IMX95_PEO_LUT_RWA | i); + regmap_read(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA2, &data2); + if (FIELD_GET(IMX95_PE0_LUT_REQID, data2) == rid) { + regmap_write(imx_pcie->iomuxc_gpr, + IMX95_PE0_LUT_DATA1, 0); + regmap_write(imx_pcie->iomuxc_gpr, + IMX95_PE0_LUT_DATA2, 0); + regmap_write(imx_pcie->iomuxc_gpr, + IMX95_PE0_LUT_ACSCTRL, i); + + break; + } + } +} + +static int imx_pcie_add_lut_by_rid(struct imx_pcie *imx_pcie, u32 rid) +{ + struct of_phandle_args iommu_spec = {}; + struct of_phandle_args msi_spec = {}; + struct device *dev = imx_pcie->pci->dev; + struct device_node *msi_filter = NULL; + u32 sid_i, sid_m; + int err_i, err_m; + u32 sid = 0; + + err_i = of_map_iommu_id(dev->of_node, rid, &iommu_spec); + if (!err_i) { + if (!iommu_spec.np) + /* + * "iommu_spec.np == NULL && err_i == 0" means RID out of map + * range. Use 1:1 map RID to streamID. Hardware can't support + * this because the streamID is only 6 bits. + */ + err_i = -EINVAL; + else + sid_i = iommu_spec.args[0]; + } + of_node_put(iommu_spec.np); + + err_m = of_map_msi_id(dev->of_node, rid, &msi_filter, &msi_spec); + /* + * err_m msi_spec.np + * 0 != NULL Got correct streamID from RID via msi-map + * 0 NULL msi-map present but RID out of range + * -ENODEV NULL msi-map absent, use built-in MSI controller + */ + if (!err_m) { + if (!msi_spec.np) + /* msi-map present but RID out of range */ + return -EINVAL; + sid_m = msi_spec.args[0]; + } + of_node_put(msi_spec.np); + + /* + * msi-map iommu-map + * N N DWC MSI Ctrl + * Y Y ITS + SMMU, require the same SID + * Y N ITS + * N Y DWC MSI Ctrl + SMMU + */ + if (err_i && err_m) + return 0; + + if (!err_i && !err_m) { + /* + * Glue Layer + * <==========> + * ┌─────┐ ┌──────────┐ + * │ LUT │ 6-bit streamID │ │ + * │ │─────────────────►│ MSI │ + * └─────┘ 2-bit ctrl ID │ │ + * ┌───────────►│ │ + * (i.MX95) │ │ │ + * 00 PCIe0 │ │ │ + * 01 ENETC │ │ │ + * 10 PCIe1 │ │ │ + * │ └──────────┘ + * The MSI glue layer auto adds 2 bits controller ID ahead of + * streamID, so mask these 2 bits to get streamID. The + * IOMMU glue layer doesn't do that. + */ + if (sid_i != (sid_m & IMX95_SID_MASK)) { + dev_err(dev, "iommu-map and msi-map entries mismatch!\n"); + return -EINVAL; + } + } + + if (!err_i) + sid = sid_i; + else if (!err_m) + sid = sid_m & IMX95_SID_MASK; + + return imx_pcie_add_lut(imx_pcie, rid, sid); +} + +static int imx_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_dev *pdev) +{ + struct imx_pcie *imx_pcie = to_imx_pcie(to_dw_pcie_from_pp(bridge->sysdata)); + + return imx_pcie_add_lut_by_rid(imx_pcie, pci_dev_id(pdev)); +} + +static void imx_pcie_disable_device(struct pci_host_bridge *bridge, + struct pci_dev *pdev) +{ + struct imx_pcie *imx_pcie; + + imx_pcie = to_imx_pcie(to_dw_pcie_from_pp(bridge->sysdata)); + imx_pcie_remove_lut(imx_pcie, pci_dev_id(pdev)); +} + +static int imx_pcie_parse_legacy_binding(struct imx_pcie *pcie) +{ + struct device *dev = pcie->pci->dev; + struct pci_host_bridge *bridge = pcie->pci->pp.bridge; + struct pci_host_port *port; + struct pci_host_perst *perst; + struct gpio_desc *reset; + + reset = devm_gpiod_get_optional(dev, "reset", GPIOD_ASIS); + if (IS_ERR(reset)) + return PTR_ERR(reset); + + if (!reset) + return 0; + + port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); + if (!port) + return -ENOMEM; + + perst = devm_kzalloc(dev, sizeof(*perst), GFP_KERNEL); + if (!perst) + return -ENOMEM; + + INIT_LIST_HEAD(&port->perst); + perst->desc = reset; + INIT_LIST_HEAD(&perst->list); + list_add_tail(&perst->list, &port->perst); + + INIT_LIST_HEAD(&port->list); + list_add_tail(&port->list, &bridge->ports); + + return devm_add_action_or_reset(dev, pci_host_common_delete_ports, + &bridge->ports); +} + +static void imx_pcie_vpcie_aux_disable(void *data) +{ + struct regulator *vpcie_aux = data; + + regulator_disable(vpcie_aux); +} + +static void imx_pcie_assert_perst(struct imx_pcie *imx_pcie, bool assert) +{ + struct dw_pcie *pci = imx_pcie->pci; + struct pci_host_bridge *bridge = pci->pp.bridge; + struct pci_host_perst *perst; + struct pci_host_port *port; + + if (!bridge || list_empty(&bridge->ports)) + return; + + if (assert) { + list_for_each_entry(port, &bridge->ports, list) { + list_for_each_entry(perst, &port->perst, list) + gpiod_direction_output(perst->desc, 1); + } + } else { + mdelay(PCIE_T_PVPERL_MS); + list_for_each_entry(port, &bridge->ports, list) { + list_for_each_entry(perst, &port->perst, list) + gpiod_direction_output(perst->desc, 0); + } + mdelay(PCIE_RESET_CONFIG_WAIT_MS); + } +} + +static bool imx_pcie_perst_found(struct pci_host_bridge *bridge) +{ + struct pci_host_port *port; + + list_for_each_entry(port, &bridge->ports, list) { + if (!list_empty(&port->perst)) + return true; + } + + return false; +} + +static int imx_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct device *dev = pci->dev; + struct imx_pcie *imx_pcie = to_imx_pcie(pci); + struct pci_host_bridge *bridge = pp->bridge; + int ret; + + if (bridge && list_empty(&bridge->ports)) { + /* Parse Root Port nodes if present */ + ret = pci_host_common_parse_ports(dev, bridge); + if (ret) { + dev_err(dev, "Failed to parse Root Port nodes: %d\n", ret); + return ret; + } + + /* Fall back to legacy binding for DT backwards compatibility */ + if (!imx_pcie_perst_found(bridge)) { + ret = imx_pcie_parse_legacy_binding(imx_pcie); + if (ret) + return ret; + } + } + + imx_pcie_assert_perst(imx_pcie, true); + + /* Keep 3.3Vaux supply enabled for entire PCIe controller lifecycle */ + if (imx_pcie->vpcie_aux && !imx_pcie->vpcie_aux_enabled) { + ret = regulator_enable(imx_pcie->vpcie_aux); + if (ret) { + dev_err(dev, "failed to enable vpcie_aux regulator: %d\n", + ret); + return ret; + } + imx_pcie->vpcie_aux_enabled = true; + + ret = devm_add_action_or_reset(dev, imx_pcie_vpcie_aux_disable, + imx_pcie->vpcie_aux); + if (ret) + return ret; + } + + /* Legacy regulator handling for DT backward compatibility. */ + if (imx_pcie->vpcie) { + ret = regulator_enable(imx_pcie->vpcie); + if (ret) { + dev_err(dev, "failed to enable vpcie regulator: %d\n", + ret); + return ret; + } + } + + if (!pp->skip_pwrctrl_off) { + ret = pci_pwrctrl_power_on_devices(dev); + if (ret) { + dev_err(dev, "failed to power on pwrctrl devices\n"); + goto err_reg_disable; + } + } + + ret = imx_pcie_clk_enable(imx_pcie); + if (ret) { + dev_err(dev, "unable to enable pcie clocks: %d\n", ret); + goto err_pwrctrl_power_off; + } + + if (pp->bridge && imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_LUT)) { + pp->bridge->enable_device = imx_pcie_enable_device; + pp->bridge->disable_device = imx_pcie_disable_device; + } + + if (imx_pcie->drvdata->select_ref_clk_src) + imx_pcie->drvdata->select_ref_clk_src(imx_pcie); + + imx_pcie_assert_core_reset(imx_pcie); + + if (imx_pcie->drvdata->init_phy) + imx_pcie->drvdata->init_phy(imx_pcie); + + imx_pcie_configure_type(imx_pcie); + + if (imx_pcie->phy) { + ret = phy_init(imx_pcie->phy); + if (ret) { + dev_err(dev, "pcie PHY power up failed\n"); + goto err_clk_disable; + } + + ret = phy_set_mode_ext(imx_pcie->phy, PHY_MODE_PCIE, + imx_pcie->drvdata->mode == DW_PCIE_EP_TYPE ? + PHY_MODE_PCIE_EP : PHY_MODE_PCIE_RC); + if (ret) { + dev_err(dev, "unable to set PCIe PHY mode\n"); + goto err_phy_exit; + } + + ret = phy_power_on(imx_pcie->phy); + if (ret) { + dev_err(dev, "waiting for PHY ready timeout!\n"); + goto err_phy_exit; + } + } + + /* Make sure that PCIe LTSSM is cleared */ + imx_pcie_ltssm_disable(dev); + + imx_pcie_deassert_core_reset(imx_pcie); + imx_pcie_assert_perst(imx_pcie, false); + + if (imx_pcie->drvdata->wait_pll_lock) { + ret = imx_pcie->drvdata->wait_pll_lock(imx_pcie); + if (ret < 0) + goto err_phy_off; + } + + imx_setup_phy_mpll(imx_pcie); + + return 0; + +err_phy_off: + imx_pcie_assert_core_reset(imx_pcie); + phy_power_off(imx_pcie->phy); +err_phy_exit: + phy_exit(imx_pcie->phy); +err_clk_disable: + imx_pcie_clk_disable(imx_pcie); +err_pwrctrl_power_off: + if (!pp->skip_pwrctrl_off) + pci_pwrctrl_power_off_devices(dev); +err_reg_disable: + if (imx_pcie->vpcie) + regulator_disable(imx_pcie->vpcie); + return ret; +} + +static void imx_pcie_host_exit(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct imx_pcie *imx_pcie = to_imx_pcie(pci); + + imx_pcie_assert_core_reset(imx_pcie); + if (imx_pcie->phy) { + if (phy_power_off(imx_pcie->phy)) + dev_err(pci->dev, "unable to power off PHY\n"); + phy_exit(imx_pcie->phy); + } + imx_pcie_clk_disable(imx_pcie); + + if (!pci->pp.skip_pwrctrl_off) + pci_pwrctrl_power_off_devices(pci->dev); + if (imx_pcie->vpcie) + regulator_disable(imx_pcie->vpcie); +} + +static void imx_pcie_host_post_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct imx_pcie *imx_pcie = to_imx_pcie(pci); + u32 val; + + if (imx_pcie->drvdata->flags & IMX_PCIE_FLAG_8GT_ECN_ERR051586) { + /* + * ERR051586: Compliance with 8GT/s Receiver Impedance ECN + * + * The default value of GEN3_RELATED_OFF[GEN3_ZRXDC_NONCOMPL] + * is 1 which makes receiver non-compliant with the ZRX-DC + * parameter for 2.5 GT/s when operating at 8 GT/s or higher. + * It causes unnecessary timeout in L1. + * + * Workaround: Program GEN3_RELATED_OFF[GEN3_ZRXDC_NONCOMPL] + * to 0. + */ + dw_pcie_dbi_ro_wr_en(pci); + val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); + val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; + dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); + dw_pcie_dbi_ro_wr_dis(pci); + } + + /* Clear CLKREQ# override if supports_clkreq is true and link is up */ + if (dw_pcie_link_up(pci) && imx_pcie->supports_clkreq) { + if (imx_pcie->drvdata->clr_clkreq_override) + imx_pcie->drvdata->clr_clkreq_override(imx_pcie); + } +} + +/* + * In old DWC implementations, PCIE_ATU_INHIBIT_PAYLOAD in iATU Ctrl2 + * register is reserved, so the generic DWC implementation of sending the + * PME_Turn_Off message using a dummy MMIO write cannot be used. + */ +static void imx_pcie_pme_turn_off(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct imx_pcie *imx_pcie = to_imx_pcie(pci); + + regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, IMX6SX_GPR12_PCIE_PM_TURN_OFF); + regmap_clear_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12, IMX6SX_GPR12_PCIE_PM_TURN_OFF); + + usleep_range(PCIE_PME_TO_L2_TIMEOUT_US/10, PCIE_PME_TO_L2_TIMEOUT_US); +} + +static const struct dw_pcie_host_ops imx_pcie_host_ops = { + .init = imx_pcie_host_init, + .deinit = imx_pcie_host_exit, + .pme_turn_off = imx_pcie_pme_turn_off, +}; + +static const struct dw_pcie_host_ops imx_pcie_host_dw_pme_ops = { + .init = imx_pcie_host_init, + .deinit = imx_pcie_host_exit, + .post_init = imx_pcie_host_post_init, +}; + +static const struct dw_pcie_ops dw_pcie_ops = { + .start_link = imx_pcie_start_link, + .stop_link = imx_pcie_stop_link, +}; + +static int imx_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, + unsigned int type, u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + switch (type) { + case PCI_IRQ_INTX: + return dw_pcie_ep_raise_intx_irq(ep, func_no); + case PCI_IRQ_MSI: + return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); + case PCI_IRQ_MSIX: + return dw_pcie_ep_raise_msix_irq(ep, func_no, interrupt_num); + default: + dev_err(pci->dev, "UNKNOWN IRQ type\n"); + return -EINVAL; + } + + return 0; +} + +static const struct pci_epc_features imx8m_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, + .msi_capable = true, + .bar[BAR_1] = { .type = BAR_DISABLED, }, + .bar[BAR_3] = { .type = BAR_DISABLED, }, + .bar[BAR_4] = { .type = BAR_FIXED, .fixed_size = SZ_256, }, + .bar[BAR_5] = { .type = BAR_DISABLED, }, + .align = SZ_64K, +}; + +static const struct pci_epc_features imx8q_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, + .msi_capable = true, + .bar[BAR_1] = { .type = BAR_DISABLED, }, + .bar[BAR_3] = { .type = BAR_DISABLED, }, + .bar[BAR_5] = { .type = BAR_DISABLED, }, + .align = SZ_64K, +}; + +/* + * | Default | Default | Default | BAR Sizing + * BAR# | Enable? | Type | Size | Scheme + * ======================================================= + * BAR0 | Enable | 64-bit | 1 MB | Programmable Size + * BAR1 | Disable | 32-bit | 64 KB | Fixed Size + * (BAR1 should be disabled if BAR0 is 64-bit) + * BAR2 | Enable | 32-bit | 1 MB | Programmable Size + * BAR3 | Enable | 32-bit | 64 KB | Programmable Size + * BAR4 | Enable | 32-bit | 1 MB | Programmable Size + * BAR5 | Enable | 32-bit | 64 KB | Programmable Size + */ +static const struct pci_epc_features imx95_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, + .msi_capable = true, + .bar[BAR_1] = { .type = BAR_FIXED, .fixed_size = SZ_64K, }, + .align = SZ_4K, +}; + +static const struct pci_epc_features* +imx_pcie_ep_get_features(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct imx_pcie *imx_pcie = to_imx_pcie(pci); + + return imx_pcie->drvdata->epc_features; +} + +static const struct dw_pcie_ep_ops pcie_ep_ops = { + .raise_irq = imx_pcie_ep_raise_irq, + .get_features = imx_pcie_ep_get_features, +}; + +static int imx_add_pcie_ep(struct imx_pcie *imx_pcie, + struct platform_device *pdev) +{ + int ret; + struct dw_pcie_ep *ep; + struct dw_pcie *pci = imx_pcie->pci; + struct dw_pcie_rp *pp = &pci->pp; + struct device *dev = pci->dev; + + imx_pcie_host_init(pp); + ep = &pci->ep; + ep->ops = &pcie_ep_ops; + + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_SUPPORT_64BIT)) + dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + + ep->page_size = imx_pcie->drvdata->epc_features->align; + + ret = dw_pcie_ep_init(ep); + if (ret) { + dev_err(dev, "failed to initialize endpoint\n"); + return ret; + } + imx_pcie_host_post_init(pp); + + ret = dw_pcie_ep_init_registers(ep); + if (ret) { + dev_err(dev, "Failed to initialize DWC endpoint registers\n"); + dw_pcie_ep_deinit(ep); + return ret; + } + + pci_epc_init_notify(ep->epc); + + return 0; +} + +static void imx_pcie_msi_save_restore(struct imx_pcie *imx_pcie, bool save) +{ + u8 offset; + u16 val; + struct dw_pcie *pci = imx_pcie->pci; + + if (pci_msi_enabled()) { + offset = dw_pcie_find_capability(pci, PCI_CAP_ID_MSI); + if (save) { + val = dw_pcie_readw_dbi(pci, offset + PCI_MSI_FLAGS); + imx_pcie->msi_ctrl = val; + } else { + dw_pcie_dbi_ro_wr_en(pci); + val = imx_pcie->msi_ctrl; + dw_pcie_writew_dbi(pci, offset + PCI_MSI_FLAGS, val); + dw_pcie_dbi_ro_wr_dis(pci); + } + } +} + +static void imx_pcie_lut_save(struct imx_pcie *imx_pcie) +{ + u32 data1, data2; + int i; + + for (i = 0; i < IMX95_MAX_LUT; i++) { + regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_ACSCTRL, + IMX95_PEO_LUT_RWA | i); + regmap_read(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA1, &data1); + regmap_read(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA2, &data2); + if (data1 & IMX95_PE0_LUT_VLD) { + imx_pcie->luts[i].data1 = data1; + imx_pcie->luts[i].data2 = data2; + } else { + imx_pcie->luts[i].data1 = 0; + imx_pcie->luts[i].data2 = 0; + } + } +} + +static void imx_pcie_lut_restore(struct imx_pcie *imx_pcie) +{ + int i; + + for (i = 0; i < IMX95_MAX_LUT; i++) { + if ((imx_pcie->luts[i].data1 & IMX95_PE0_LUT_VLD) == 0) + continue; + + regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA1, + imx_pcie->luts[i].data1); + regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_DATA2, + imx_pcie->luts[i].data2); + regmap_write(imx_pcie->iomuxc_gpr, IMX95_PE0_LUT_ACSCTRL, i); + } +} + +static int imx_pcie_suspend_noirq(struct device *dev) +{ + struct imx_pcie *imx_pcie = dev_get_drvdata(dev); + + if (!(imx_pcie->drvdata->flags & IMX_PCIE_FLAG_SUPPORTS_SUSPEND)) + return 0; + + imx_pcie_msi_save_restore(imx_pcie, true); + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_LUT)) + imx_pcie_lut_save(imx_pcie); + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_BROKEN_SUSPEND)) { + /* + * The minimum for a workaround would be to set PERST# and to + * set the PCIE_TEST_PD flag. However, we can also disable the + * clock which saves some power. + */ + imx_pcie_assert_core_reset(imx_pcie); + imx_pcie_assert_perst(imx_pcie, true); + imx_pcie->drvdata->enable_ref_clk(imx_pcie, false); + } else { + return dw_pcie_suspend_noirq(imx_pcie->pci); + } + + return 0; +} + +static int imx_pcie_resume_noirq(struct device *dev) +{ + int ret; + struct imx_pcie *imx_pcie = dev_get_drvdata(dev); + + if (!(imx_pcie->drvdata->flags & IMX_PCIE_FLAG_SUPPORTS_SUSPEND)) + return 0; + + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_BROKEN_SUSPEND)) { + ret = imx_pcie->drvdata->enable_ref_clk(imx_pcie, true); + if (ret) + return ret; + imx_pcie_deassert_core_reset(imx_pcie); + imx_pcie_assert_perst(imx_pcie, false); + + /* + * Using PCIE_TEST_PD seems to disable MSI and powers down the + * root complex. This is why we have to setup the rc again and + * why we have to restore the MSI register. + */ + ret = dw_pcie_setup_rc(&imx_pcie->pci->pp); + if (ret) + return ret; + } else { + ret = dw_pcie_resume_noirq(imx_pcie->pci); + if (ret) + return ret; + } + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_LUT)) + imx_pcie_lut_restore(imx_pcie); + imx_pcie_msi_save_restore(imx_pcie, false); + + return 0; +} + +static const struct dev_pm_ops imx_pcie_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(imx_pcie_suspend_noirq, + imx_pcie_resume_noirq) +}; + +static int imx_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dw_pcie *pci; + struct imx_pcie *imx_pcie; + struct device_node *node = dev->of_node; + int i, ret, domain; + u16 val; + + imx_pcie = devm_kzalloc(dev, sizeof(*imx_pcie), GFP_KERNEL); + if (!imx_pcie) + return -ENOMEM; + + pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); + if (!pci) + return -ENOMEM; + + pci->dev = dev; + pci->ops = &dw_pcie_ops; + + imx_pcie->pci = pci; + imx_pcie->drvdata = of_device_get_match_data(dev); + + mutex_init(&imx_pcie->lock); + + if (imx_pcie->drvdata->ops) + pci->pp.ops = imx_pcie->drvdata->ops; + else + pci->pp.ops = &imx_pcie_host_dw_pme_ops; + + /* Find the PHY if one is defined, only imx7d uses it */ + struct device_node *np __free(device_node) = + of_parse_phandle(node, "fsl,imx7d-pcie-phy", 0); + if (np) { + struct resource res; + + ret = of_address_to_resource(np, 0, &res); + if (ret) { + dev_err(dev, "Unable to map PCIe PHY\n"); + return ret; + } + imx_pcie->phy_base = devm_ioremap_resource(dev, &res); + if (IS_ERR(imx_pcie->phy_base)) + return PTR_ERR(imx_pcie->phy_base); + } + + /* Fetch clocks */ + imx_pcie->num_clks = devm_clk_bulk_get_all(dev, &imx_pcie->clks); + if (imx_pcie->num_clks < 0) + return dev_err_probe(dev, imx_pcie->num_clks, + "failed to get clocks\n"); + for (i = 0; i < imx_pcie->num_clks; i++) + if (strncmp(imx_pcie->clks[i].id, "extref", 6) == 0) + imx_pcie->enable_ext_refclk = true; + + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_PHYDRV)) { + imx_pcie->phy = devm_phy_get(dev, "pcie-phy"); + if (IS_ERR(imx_pcie->phy)) + return dev_err_probe(dev, PTR_ERR(imx_pcie->phy), + "failed to get pcie phy\n"); + } + + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_APP_RESET)) { + imx_pcie->apps_reset = devm_reset_control_get_exclusive(dev, "apps"); + if (IS_ERR(imx_pcie->apps_reset)) + return dev_err_probe(dev, PTR_ERR(imx_pcie->apps_reset), + "failed to get pcie apps reset control\n"); + } + + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_PHY_RESET)) { + imx_pcie->pciephy_reset = devm_reset_control_get_exclusive(dev, "pciephy"); + if (IS_ERR(imx_pcie->pciephy_reset)) + return dev_err_probe(dev, PTR_ERR(imx_pcie->pciephy_reset), + "Failed to get PCIEPHY reset control\n"); + } + + switch (imx_pcie->drvdata->variant) { + case IMX8MQ: + case IMX8MQ_EP: + domain = of_get_pci_domain_nr(node); + if (domain < 0 || domain > 1) + return dev_err_probe(dev, -ENODEV, "no \"linux,pci-domain\" property in devicetree\n"); + + imx_pcie->controller_id = domain; + break; + default: + break; + } + + if (imx_pcie->drvdata->gpr) { + /* Grab GPR config register range */ + imx_pcie->iomuxc_gpr = + syscon_regmap_lookup_by_compatible(imx_pcie->drvdata->gpr); + if (IS_ERR(imx_pcie->iomuxc_gpr)) + return dev_err_probe(dev, PTR_ERR(imx_pcie->iomuxc_gpr), + "unable to find iomuxc registers\n"); + } + + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_SERDES)) { + void __iomem *off = devm_platform_ioremap_resource_byname(pdev, "app"); + + if (IS_ERR(off)) + return dev_err_probe(dev, PTR_ERR(off), + "unable to find serdes registers\n"); + + static const struct regmap_config regmap_config = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, + }; + + imx_pcie->iomuxc_gpr = devm_regmap_init_mmio(dev, off, ®map_config); + if (IS_ERR(imx_pcie->iomuxc_gpr)) + return dev_err_probe(dev, PTR_ERR(imx_pcie->iomuxc_gpr), + "unable to find iomuxc registers\n"); + } + + /* Grab PCIe PHY Tx Settings */ + if (of_property_read_u32(node, "fsl,tx-deemph-gen1", + &imx_pcie->tx_deemph_gen1)) + imx_pcie->tx_deemph_gen1 = 0; + + if (of_property_read_u32(node, "fsl,tx-deemph-gen2-3p5db", + &imx_pcie->tx_deemph_gen2_3p5db)) + imx_pcie->tx_deemph_gen2_3p5db = 0; + + if (of_property_read_u32(node, "fsl,tx-deemph-gen2-6db", + &imx_pcie->tx_deemph_gen2_6db)) + imx_pcie->tx_deemph_gen2_6db = 20; + + if (of_property_read_u32(node, "fsl,tx-swing-full", + &imx_pcie->tx_swing_full)) + imx_pcie->tx_swing_full = 127; + + if (of_property_read_u32(node, "fsl,tx-swing-low", + &imx_pcie->tx_swing_low)) + imx_pcie->tx_swing_low = 127; + + /* Limit link speed */ + pci->max_link_speed = 1; + of_property_read_u32(node, "fsl,max-link-speed", &pci->max_link_speed); + imx_pcie->supports_clkreq = of_property_read_bool(node, "supports-clkreq"); + + imx_pcie->vpcie_aux = devm_regulator_get_optional(&pdev->dev, + "vpcie3v3aux"); + if (IS_ERR(imx_pcie->vpcie_aux)) { + if (PTR_ERR(imx_pcie->vpcie_aux) != -ENODEV) + return PTR_ERR(imx_pcie->vpcie_aux); + imx_pcie->vpcie_aux = NULL; + } + + imx_pcie->vpcie = devm_regulator_get_optional(&pdev->dev, "vpcie"); + if (IS_ERR(imx_pcie->vpcie)) { + if (PTR_ERR(imx_pcie->vpcie) != -ENODEV) + return PTR_ERR(imx_pcie->vpcie); + imx_pcie->vpcie = NULL; + } + + imx_pcie->vph = devm_regulator_get_optional(&pdev->dev, "vph"); + if (IS_ERR(imx_pcie->vph)) { + if (PTR_ERR(imx_pcie->vph) != -ENODEV) + return PTR_ERR(imx_pcie->vph); + imx_pcie->vph = NULL; + } + + platform_set_drvdata(pdev, imx_pcie); + + ret = imx_pcie_attach_pd(dev); + if (ret) + return ret; + + ret = pci_pwrctrl_create_devices(dev); + if (ret) + return dev_err_probe(dev, ret, "failed to create pwrctrl devices\n"); + + pci->use_parent_dt_ranges = true; + if (imx_pcie->drvdata->mode == DW_PCIE_EP_TYPE) { + ret = imx_add_pcie_ep(imx_pcie, pdev); + if (ret < 0) + goto err_pwrctrl_destroy; + + /* + * FIXME: Only single Device (EPF) is supported due to the + * Endpoint framework limitation. + */ + imx_pcie_add_lut_by_rid(imx_pcie, 0); + } else { + if (imx_pcie->drvdata->flags & IMX_PCIE_FLAG_PM_RUNTIME) { + pm_runtime_no_callbacks(dev); + ret = devm_pm_runtime_set_active_enabled(dev); + if (ret < 0) + return ret; + } + + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_SKIP_L23_READY)) + pci->pp.skip_l23_ready = true; + if (imx_check_flag(imx_pcie, IMX_PCIE_FLAG_KEEP_MSI_CAP)) + pci->pp.keep_rp_msi_en = true; + pci->pp.use_atu_msg = true; + ret = dw_pcie_host_init(&pci->pp); + if (ret < 0) + goto err_pwrctrl_destroy; + + if (pci_msi_enabled()) { + u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_MSI); + + val = dw_pcie_readw_dbi(pci, offset + PCI_MSI_FLAGS); + val |= PCI_MSI_FLAGS_ENABLE; + dw_pcie_writew_dbi(pci, offset + PCI_MSI_FLAGS, val); + } + } + + return 0; + +err_pwrctrl_destroy: + if (ret != -EPROBE_DEFER) + pci_pwrctrl_destroy_devices(dev); + return ret; +} + +static void imx_pcie_shutdown(struct platform_device *pdev) +{ + struct imx_pcie *imx_pcie = platform_get_drvdata(pdev); + + /* bring down link, so bootloader gets clean state in case of reboot */ + imx_pcie_assert_core_reset(imx_pcie); + imx_pcie_assert_perst(imx_pcie, true); + pci_pwrctrl_power_off_devices(&pdev->dev); + pci_pwrctrl_destroy_devices(&pdev->dev); +} + +static const struct imx_pcie_drvdata drvdata[] = { + [IMX6Q] = { + .variant = IMX6Q, + .flags = IMX_PCIE_FLAG_IMX_PHY | + IMX_PCIE_FLAG_SPEED_CHANGE_WORKAROUND | + IMX_PCIE_FLAG_BROKEN_SUSPEND | + IMX_PCIE_FLAG_SUPPORTS_SUSPEND | + IMX_PCIE_FLAG_KEEP_MSI_CAP, + .dbi_length = 0x200, + .gpr = "fsl,imx6q-iomuxc-gpr", + .ltssm_off = IOMUXC_GPR12, + .ltssm_mask = IMX6Q_GPR12_PCIE_CTL_2, + .mode_off[0] = IOMUXC_GPR12, + .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, + .init_phy = imx_pcie_init_phy, + .enable_ref_clk = imx6q_pcie_enable_ref_clk, + .core_reset = imx6q_pcie_core_reset, + }, + [IMX6SX] = { + .variant = IMX6SX, + .flags = IMX_PCIE_FLAG_IMX_PHY | + IMX_PCIE_FLAG_SPEED_CHANGE_WORKAROUND | + IMX_PCIE_FLAG_SKIP_L23_READY | + IMX_PCIE_FLAG_SUPPORTS_SUSPEND | + IMX_PCIE_FLAG_KEEP_MSI_CAP, + .gpr = "fsl,imx6q-iomuxc-gpr", + .ltssm_off = IOMUXC_GPR12, + .ltssm_mask = IMX6Q_GPR12_PCIE_CTL_2, + .mode_off[0] = IOMUXC_GPR12, + .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, + .init_phy = imx6sx_pcie_init_phy, + .core_reset = imx6sx_pcie_core_reset, + .ops = &imx_pcie_host_ops, + }, + [IMX6QP] = { + .variant = IMX6QP, + .flags = IMX_PCIE_FLAG_IMX_PHY | + IMX_PCIE_FLAG_SPEED_CHANGE_WORKAROUND | + IMX_PCIE_FLAG_SKIP_L23_READY | + IMX_PCIE_FLAG_SUPPORTS_SUSPEND | + IMX_PCIE_FLAG_KEEP_MSI_CAP, + .dbi_length = 0x200, + .gpr = "fsl,imx6q-iomuxc-gpr", + .ltssm_off = IOMUXC_GPR12, + .ltssm_mask = IMX6Q_GPR12_PCIE_CTL_2, + .mode_off[0] = IOMUXC_GPR12, + .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, + .init_phy = imx_pcie_init_phy, + .enable_ref_clk = imx6q_pcie_enable_ref_clk, + .core_reset = imx6qp_pcie_core_reset, + .ops = &imx_pcie_host_ops, + }, + [IMX7D] = { + .variant = IMX7D, + .flags = IMX_PCIE_FLAG_SUPPORTS_SUSPEND | + IMX_PCIE_FLAG_KEEP_MSI_CAP | + IMX_PCIE_FLAG_HAS_APP_RESET | + IMX_PCIE_FLAG_SKIP_L23_READY | + IMX_PCIE_FLAG_HAS_PHY_RESET, + .gpr = "fsl,imx7d-iomuxc-gpr", + .mode_off[0] = IOMUXC_GPR12, + .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, + .enable_ref_clk = imx7d_pcie_enable_ref_clk, + .core_reset = imx7d_pcie_core_reset, + }, + [IMX8MQ] = { + .variant = IMX8MQ, + .flags = IMX_PCIE_FLAG_HAS_APP_RESET | + IMX_PCIE_FLAG_KEEP_MSI_CAP | + IMX_PCIE_FLAG_HAS_PHY_RESET | + IMX_PCIE_FLAG_SUPPORTS_SUSPEND, + .gpr = "fsl,imx8mq-iomuxc-gpr", + .mode_off[0] = IOMUXC_GPR12, + .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, + .mode_off[1] = IOMUXC_GPR12, + .mode_mask[1] = IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE, + .init_phy = imx8mq_pcie_init_phy, + .enable_ref_clk = imx8mm_pcie_enable_ref_clk, + .clr_clkreq_override = imx8mm_pcie_clr_clkreq_override, + }, + [IMX8MM] = { + .variant = IMX8MM, + .flags = IMX_PCIE_FLAG_SUPPORTS_SUSPEND | + IMX_PCIE_FLAG_KEEP_MSI_CAP | + IMX_PCIE_FLAG_HAS_PHYDRV | + IMX_PCIE_FLAG_HAS_APP_RESET, + .gpr = "fsl,imx8mm-iomuxc-gpr", + .mode_off[0] = IOMUXC_GPR12, + .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, + .enable_ref_clk = imx8mm_pcie_enable_ref_clk, + .clr_clkreq_override = imx8mm_pcie_clr_clkreq_override, + }, + [IMX8MP] = { + .variant = IMX8MP, + .flags = IMX_PCIE_FLAG_SUPPORTS_SUSPEND | + IMX_PCIE_FLAG_HAS_PHYDRV | + IMX_PCIE_FLAG_HAS_APP_RESET, + .gpr = "fsl,imx8mp-iomuxc-gpr", + .mode_off[0] = IOMUXC_GPR12, + .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, + .enable_ref_clk = imx8mm_pcie_enable_ref_clk, + .clr_clkreq_override = imx8mm_pcie_clr_clkreq_override, + }, + [IMX8Q] = { + .variant = IMX8Q, + .flags = IMX_PCIE_FLAG_HAS_PHYDRV | + IMX_PCIE_FLAG_CPU_ADDR_FIXUP | + IMX_PCIE_FLAG_SUPPORTS_SUSPEND, + }, + [IMX95] = { + .variant = IMX95, + .flags = IMX_PCIE_FLAG_HAS_SERDES | + IMX_PCIE_FLAG_HAS_LUT | + IMX_PCIE_FLAG_8GT_ECN_ERR051586 | + IMX_PCIE_FLAG_PM_RUNTIME | + IMX_PCIE_FLAG_SUPPORTS_SUSPEND, + .ltssm_off = IMX95_PE0_GEN_CTRL_3, + .ltssm_mask = IMX95_PCIE_LTSSM_EN, + .mode_off[0] = IMX95_PE0_GEN_CTRL_1, + .mode_mask[0] = IMX95_PCIE_DEVICE_TYPE, + .core_reset = imx95_pcie_core_reset, + .init_phy = imx95_pcie_init_phy, + .select_ref_clk_src = imx95_pcie_select_ref_clk_src, + .wait_pll_lock = imx95_pcie_wait_for_phy_pll_lock, + .enable_ref_clk = imx95_pcie_enable_ref_clk, + .clr_clkreq_override = imx95_pcie_clr_clkreq_override, + }, + [IMX8MQ_EP] = { + .variant = IMX8MQ_EP, + .flags = IMX_PCIE_FLAG_HAS_APP_RESET | + IMX_PCIE_FLAG_HAS_PHY_RESET, + .mode = DW_PCIE_EP_TYPE, + .gpr = "fsl,imx8mq-iomuxc-gpr", + .mode_off[0] = IOMUXC_GPR12, + .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, + .mode_off[1] = IOMUXC_GPR12, + .mode_mask[1] = IMX8MQ_GPR12_PCIE2_CTRL_DEVICE_TYPE, + .epc_features = &imx8q_pcie_epc_features, + .init_phy = imx8mq_pcie_init_phy, + .enable_ref_clk = imx8mm_pcie_enable_ref_clk, + }, + [IMX8MM_EP] = { + .variant = IMX8MM_EP, + .flags = IMX_PCIE_FLAG_HAS_APP_RESET | + IMX_PCIE_FLAG_HAS_PHYDRV, + .mode = DW_PCIE_EP_TYPE, + .gpr = "fsl,imx8mm-iomuxc-gpr", + .mode_off[0] = IOMUXC_GPR12, + .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, + .epc_features = &imx8m_pcie_epc_features, + .enable_ref_clk = imx8mm_pcie_enable_ref_clk, + }, + [IMX8MP_EP] = { + .variant = IMX8MP_EP, + .flags = IMX_PCIE_FLAG_HAS_APP_RESET | + IMX_PCIE_FLAG_HAS_PHYDRV, + .mode = DW_PCIE_EP_TYPE, + .gpr = "fsl,imx8mp-iomuxc-gpr", + .mode_off[0] = IOMUXC_GPR12, + .mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE, + .epc_features = &imx8m_pcie_epc_features, + .enable_ref_clk = imx8mm_pcie_enable_ref_clk, + }, + [IMX8Q_EP] = { + .variant = IMX8Q_EP, + .flags = IMX_PCIE_FLAG_HAS_PHYDRV, + .mode = DW_PCIE_EP_TYPE, + .epc_features = &imx8q_pcie_epc_features, + }, + [IMX95_EP] = { + .variant = IMX95_EP, + .flags = IMX_PCIE_FLAG_HAS_SERDES | + IMX_PCIE_FLAG_8GT_ECN_ERR051586 | + IMX_PCIE_FLAG_SUPPORT_64BIT, + .ltssm_off = IMX95_PE0_GEN_CTRL_3, + .ltssm_mask = IMX95_PCIE_LTSSM_EN, + .mode_off[0] = IMX95_PE0_GEN_CTRL_1, + .mode_mask[0] = IMX95_PCIE_DEVICE_TYPE, + .select_ref_clk_src = imx95_pcie_select_ref_clk_src, + .init_phy = imx95_pcie_init_phy, + .core_reset = imx95_pcie_core_reset, + .wait_pll_lock = imx95_pcie_wait_for_phy_pll_lock, + .epc_features = &imx95_pcie_epc_features, + .enable_ref_clk = imx95_pcie_enable_ref_clk, + .mode = DW_PCIE_EP_TYPE, + }, +}; + +static const struct of_device_id imx_pcie_of_match[] = { + { .compatible = "fsl,imx6q-pcie", .data = &drvdata[IMX6Q], }, + { .compatible = "fsl,imx6sx-pcie", .data = &drvdata[IMX6SX], }, + { .compatible = "fsl,imx6qp-pcie", .data = &drvdata[IMX6QP], }, + { .compatible = "fsl,imx7d-pcie", .data = &drvdata[IMX7D], }, + { .compatible = "fsl,imx8mq-pcie", .data = &drvdata[IMX8MQ], }, + { .compatible = "fsl,imx8mm-pcie", .data = &drvdata[IMX8MM], }, + { .compatible = "fsl,imx8mp-pcie", .data = &drvdata[IMX8MP], }, + { .compatible = "fsl,imx8q-pcie", .data = &drvdata[IMX8Q], }, + { .compatible = "fsl,imx95-pcie", .data = &drvdata[IMX95], }, + { .compatible = "fsl,imx8mq-pcie-ep", .data = &drvdata[IMX8MQ_EP], }, + { .compatible = "fsl,imx8mm-pcie-ep", .data = &drvdata[IMX8MM_EP], }, + { .compatible = "fsl,imx8mp-pcie-ep", .data = &drvdata[IMX8MP_EP], }, + { .compatible = "fsl,imx8q-pcie-ep", .data = &drvdata[IMX8Q_EP], }, + { .compatible = "fsl,imx95-pcie-ep", .data = &drvdata[IMX95_EP], }, + {}, +}; + +static struct platform_driver imx_pcie_driver = { + .driver = { + .name = "imx6q-pcie", + .of_match_table = imx_pcie_of_match, + .suppress_bind_attrs = true, + .pm = &imx_pcie_pm_ops, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, + .probe = imx_pcie_probe, + .shutdown = imx_pcie_shutdown, +}; + +static void imx_pcie_quirk(struct pci_dev *dev) +{ + struct pci_bus *bus = dev->bus; + struct dw_pcie_rp *pp = bus->sysdata; + + /* Bus parent is the PCI bridge, its parent is this platform driver */ + if (!bus->dev.parent || !bus->dev.parent->parent) + return; + + /* Make sure we only quirk devices associated with this driver */ + if (bus->dev.parent->parent->driver != &imx_pcie_driver.driver) + return; + + if (pci_is_root_bus(bus)) { + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct imx_pcie *imx_pcie = to_imx_pcie(pci); + + /* + * Limit config length to avoid the kernel reading beyond + * the register set and causing an abort on i.MX 6Quad + */ + if (imx_pcie->drvdata->dbi_length) { + dev->cfg_size = imx_pcie->drvdata->dbi_length; + dev_info(&dev->dev, "Limiting cfg_size to %d\n", + dev->cfg_size); + } + } +} +DECLARE_PCI_FIXUP_CLASS_HEADER(PCI_VENDOR_ID_SYNOPSYS, 0xabcd, + PCI_CLASS_BRIDGE_PCI, 8, imx_pcie_quirk); + +static int __init imx_pcie_init(void) +{ +#ifdef CONFIG_ARM + struct device_node *np; + + np = of_find_matching_node(NULL, imx_pcie_of_match); + if (!np) + return -ENODEV; + of_node_put(np); + + /* + * Since probe() can be deferred we need to make sure that + * hook_fault_code is not called after __init memory is freed + * by kernel and since imx6q_pcie_abort_handler() is a no-op, + * we can install the handler here without risking it + * accessing some uninitialized driver state. + */ + hook_fault_code(8, imx6q_pcie_abort_handler, SIGBUS, 0, + "external abort on non-linefetch"); +#endif + + return platform_driver_register(&imx_pcie_driver); +} +device_initcall(imx_pcie_init); diff --git a/drivers/pci/controller/dwc/pci-keystone.c b/drivers/pci/controller/dwc/pci-keystone.c new file mode 100644 index 000000000..602516239 --- /dev/null +++ b/drivers/pci/controller/dwc/pci-keystone.c @@ -0,0 +1,1415 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for Texas Instruments Keystone SoCs + * + * Copyright (C) 2013-2014 Texas Instruments., Ltd. + * https://www.ti.com + * + * Author: Murali Karicheri <m-karicheri2@ti.com> + * Implementation based on pci-exynos.c and pcie-designware.c + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/gpio/consumer.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqdomain.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/of.h> +#include <linux/of_irq.h> +#include <linux/of_pci.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/resource.h> +#include <linux/signal.h> + +#include "../../pci.h" +#include "pcie-designware.h" + +#define PCIE_VENDORID_MASK 0xffff +#define PCIE_DEVICEID_SHIFT 16 + +/* Application registers */ +#define PID 0x000 +#define RTL GENMASK(15, 11) +#define RTL_SHIFT 11 +#define AM6_PCI_PG1_RTL_VER 0x15 + +#define CMD_STATUS 0x004 +#define LTSSM_EN_VAL BIT(0) +#define OB_XLAT_EN_VAL BIT(1) +#define DBI_CS2 BIT(5) + +#define CFG_SETUP 0x008 +#define CFG_BUS(x) (((x) & 0xff) << 16) +#define CFG_DEVICE(x) (((x) & 0x1f) << 8) +#define CFG_FUNC(x) ((x) & 0x7) +#define CFG_TYPE1 BIT(24) + +#define OB_SIZE 0x030 +#define OB_OFFSET_INDEX(n) (0x200 + (8 * (n))) +#define OB_OFFSET_HI(n) (0x204 + (8 * (n))) +#define OB_ENABLEN BIT(0) +#define OB_WIN_SIZE 8 /* 8MB */ + +#define PCIE_LEGACY_IRQ_ENABLE_SET(n) (0x188 + (0x10 * ((n) - 1))) +#define PCIE_LEGACY_IRQ_ENABLE_CLR(n) (0x18c + (0x10 * ((n) - 1))) +#define PCIE_EP_IRQ_SET 0x64 +#define PCIE_EP_IRQ_CLR 0x68 +#define INT_ENABLE BIT(0) + +/* IRQ register defines */ +#define IRQ_EOI 0x050 + +#define MSI_IRQ 0x054 +#define MSI_IRQ_STATUS(n) (0x104 + ((n) << 4)) +#define MSI_IRQ_ENABLE_SET(n) (0x108 + ((n) << 4)) +#define MSI_IRQ_ENABLE_CLR(n) (0x10c + ((n) << 4)) +#define MSI_IRQ_OFFSET 4 + +#define IRQ_STATUS(n) (0x184 + ((n) << 4)) +#define IRQ_ENABLE_SET(n) (0x188 + ((n) << 4)) +#define INTx_EN BIT(0) + +#define ERR_IRQ_STATUS 0x1c4 +#define ERR_IRQ_ENABLE_SET 0x1c8 +#define ERR_AER BIT(5) /* ECRC error */ +#define AM6_ERR_AER BIT(4) /* AM6 ECRC error */ +#define ERR_AXI BIT(4) /* AXI tag lookup fatal error */ +#define ERR_CORR BIT(3) /* Correctable error */ +#define ERR_NONFATAL BIT(2) /* Non-fatal error */ +#define ERR_FATAL BIT(1) /* Fatal error */ +#define ERR_SYS BIT(0) /* System error */ +#define ERR_IRQ_ALL (ERR_AER | ERR_AXI | ERR_CORR | \ + ERR_NONFATAL | ERR_FATAL | ERR_SYS) + +/* PCIE controller device IDs */ +#define PCIE_RC_K2HK 0xb008 +#define PCIE_RC_K2E 0xb009 +#define PCIE_RC_K2L 0xb00a +#define PCIE_RC_K2G 0xb00b + +#define KS_PCIE_DEV_TYPE_MASK (0x3 << 1) +#define KS_PCIE_DEV_TYPE(mode) ((mode) << 1) + +#define EP 0x0 +#define LEG_EP 0x1 +#define RC 0x2 + +#define KS_PCIE_SYSCLOCKOUTEN BIT(0) + +#define AM654_PCIE_DEV_TYPE_MASK 0x3 +#define AM654_WIN_SIZE SZ_64K + +#define APP_ADDR_SPACE_0 (16 * SZ_1K) + +#define to_keystone_pcie(x) dev_get_drvdata((x)->dev) + +#define PCI_DEVICE_ID_TI_AM654X 0xb00c + +struct ks_pcie_of_data { + enum dw_pcie_device_mode mode; + const struct dw_pcie_host_ops *host_ops; + const struct dw_pcie_ep_ops *ep_ops; + u32 version; +}; + +struct keystone_pcie { + struct dw_pcie *pci; + /* PCI Device ID */ + u32 device_id; + int intx_host_irqs[PCI_NUM_INTX]; + + int msi_host_irq; + int num_lanes; + u32 num_viewport; + struct phy **phy; + struct device_link **link; + struct device_node *msi_intc_np; + struct irq_domain *intx_irq_domain; + struct device_node *np; + + /* Application register space */ + void __iomem *va_app_base; /* DT 1st resource */ + struct resource app; + bool is_am6; +}; + +static u32 ks_pcie_app_readl(struct keystone_pcie *ks_pcie, u32 offset) +{ + return readl(ks_pcie->va_app_base + offset); +} + +static void ks_pcie_app_writel(struct keystone_pcie *ks_pcie, u32 offset, + u32 val) +{ + writel(val, ks_pcie->va_app_base + offset); +} + +static void ks_pcie_msi_irq_ack(struct irq_data *data) +{ + struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(data); + struct keystone_pcie *ks_pcie; + u32 irq = data->hwirq; + struct dw_pcie *pci; + u32 reg_offset; + u32 bit_pos; + + pci = to_dw_pcie_from_pp(pp); + ks_pcie = to_keystone_pcie(pci); + + reg_offset = irq % 8; + bit_pos = irq >> 3; + + ks_pcie_app_writel(ks_pcie, MSI_IRQ_STATUS(reg_offset), + BIT(bit_pos)); + ks_pcie_app_writel(ks_pcie, IRQ_EOI, reg_offset + MSI_IRQ_OFFSET); +} + +static void ks_pcie_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(data); + struct keystone_pcie *ks_pcie; + struct dw_pcie *pci; + u64 msi_target; + + pci = to_dw_pcie_from_pp(pp); + ks_pcie = to_keystone_pcie(pci); + + msi_target = ks_pcie->app.start + MSI_IRQ; + msg->address_lo = lower_32_bits(msi_target); + msg->address_hi = upper_32_bits(msi_target); + msg->data = data->hwirq; + + dev_dbg(pci->dev, "msi#%d address_hi %#x address_lo %#x\n", + (int)data->hwirq, msg->address_hi, msg->address_lo); +} + +static void ks_pcie_msi_mask(struct irq_data *data) +{ + struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(data); + struct keystone_pcie *ks_pcie; + u32 irq = data->hwirq; + struct dw_pcie *pci; + unsigned long flags; + u32 reg_offset; + u32 bit_pos; + + raw_spin_lock_irqsave(&pp->lock, flags); + + pci = to_dw_pcie_from_pp(pp); + ks_pcie = to_keystone_pcie(pci); + + reg_offset = irq % 8; + bit_pos = irq >> 3; + + ks_pcie_app_writel(ks_pcie, MSI_IRQ_ENABLE_CLR(reg_offset), + BIT(bit_pos)); + + raw_spin_unlock_irqrestore(&pp->lock, flags); +} + +static void ks_pcie_msi_unmask(struct irq_data *data) +{ + struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(data); + struct keystone_pcie *ks_pcie; + u32 irq = data->hwirq; + struct dw_pcie *pci; + unsigned long flags; + u32 reg_offset; + u32 bit_pos; + + raw_spin_lock_irqsave(&pp->lock, flags); + + pci = to_dw_pcie_from_pp(pp); + ks_pcie = to_keystone_pcie(pci); + + reg_offset = irq % 8; + bit_pos = irq >> 3; + + ks_pcie_app_writel(ks_pcie, MSI_IRQ_ENABLE_SET(reg_offset), + BIT(bit_pos)); + + raw_spin_unlock_irqrestore(&pp->lock, flags); +} + +static struct irq_chip ks_pcie_msi_irq_chip = { + .name = "KEYSTONE-PCI-MSI", + .irq_ack = ks_pcie_msi_irq_ack, + .irq_compose_msi_msg = ks_pcie_compose_msi_msg, + .irq_mask = ks_pcie_msi_mask, + .irq_unmask = ks_pcie_msi_unmask, +}; + +/** + * ks_pcie_set_dbi_mode() - Set DBI mode to access overlaid BAR mask registers + * @ks_pcie: A pointer to the keystone_pcie structure which holds the KeyStone + * PCIe host controller driver information. + * + * Since modification of dbi_cs2 involves different clock domain, read the + * status back to ensure the transition is complete. + */ +static void ks_pcie_set_dbi_mode(struct keystone_pcie *ks_pcie) +{ + u32 val; + + val = ks_pcie_app_readl(ks_pcie, CMD_STATUS); + val |= DBI_CS2; + ks_pcie_app_writel(ks_pcie, CMD_STATUS, val); + + do { + val = ks_pcie_app_readl(ks_pcie, CMD_STATUS); + } while (!(val & DBI_CS2)); +} + +/** + * ks_pcie_clear_dbi_mode() - Disable DBI mode + * @ks_pcie: A pointer to the keystone_pcie structure which holds the KeyStone + * PCIe host controller driver information. + * + * Since modification of dbi_cs2 involves different clock domain, read the + * status back to ensure the transition is complete. + */ +static void ks_pcie_clear_dbi_mode(struct keystone_pcie *ks_pcie) +{ + u32 val; + + val = ks_pcie_app_readl(ks_pcie, CMD_STATUS); + val &= ~DBI_CS2; + ks_pcie_app_writel(ks_pcie, CMD_STATUS, val); + + do { + val = ks_pcie_app_readl(ks_pcie, CMD_STATUS); + } while (val & DBI_CS2); +} + +static int ks_pcie_msi_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); + + /* Configure and set up BAR0 */ + ks_pcie_set_dbi_mode(ks_pcie); + + /* Enable BAR0 */ + dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 1); + dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, SZ_4K - 1); + + ks_pcie_clear_dbi_mode(ks_pcie); + + /* + * For BAR0, just setting bus address for inbound writes (MSI) should + * be sufficient. Use physical address to avoid any conflicts. + */ + dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, ks_pcie->app.start); + + pp->msi_irq_chip = &ks_pcie_msi_irq_chip; + return dw_pcie_allocate_domains(pp); +} + +static void ks_pcie_handle_intx_irq(struct keystone_pcie *ks_pcie, + int offset) +{ + struct dw_pcie *pci = ks_pcie->pci; + struct device *dev = pci->dev; + u32 pending; + + pending = ks_pcie_app_readl(ks_pcie, IRQ_STATUS(offset)); + + if (BIT(0) & pending) { + dev_dbg(dev, ": irq: irq_offset %d", offset); + generic_handle_domain_irq(ks_pcie->intx_irq_domain, offset); + } + + /* EOI the INTx interrupt */ + ks_pcie_app_writel(ks_pcie, IRQ_EOI, offset); +} + +static void ks_pcie_enable_error_irq(struct keystone_pcie *ks_pcie) +{ + ks_pcie_app_writel(ks_pcie, ERR_IRQ_ENABLE_SET, ERR_IRQ_ALL); +} + +static irqreturn_t ks_pcie_handle_error_irq(struct keystone_pcie *ks_pcie) +{ + u32 reg; + struct device *dev = ks_pcie->pci->dev; + + reg = ks_pcie_app_readl(ks_pcie, ERR_IRQ_STATUS); + if (!reg) + return IRQ_NONE; + + if (reg & ERR_SYS) + dev_err(dev, "System Error\n"); + + if (reg & ERR_FATAL) + dev_err(dev, "Fatal Error\n"); + + if (reg & ERR_NONFATAL) + dev_dbg(dev, "Non Fatal Error\n"); + + if (reg & ERR_CORR) + dev_dbg(dev, "Correctable Error\n"); + + if (!ks_pcie->is_am6 && (reg & ERR_AXI)) + dev_err(dev, "AXI tag lookup fatal Error\n"); + + if (reg & ERR_AER || (ks_pcie->is_am6 && (reg & AM6_ERR_AER))) + dev_err(dev, "ECRC Error\n"); + + ks_pcie_app_writel(ks_pcie, ERR_IRQ_STATUS, reg); + + return IRQ_HANDLED; +} + +static void ks_pcie_ack_intx_irq(struct irq_data *d) +{ +} + +static void ks_pcie_mask_intx_irq(struct irq_data *d) +{ +} + +static void ks_pcie_unmask_intx_irq(struct irq_data *d) +{ +} + +static struct irq_chip ks_pcie_intx_irq_chip = { + .name = "Keystone-PCI-INTX-IRQ", + .irq_ack = ks_pcie_ack_intx_irq, + .irq_mask = ks_pcie_mask_intx_irq, + .irq_unmask = ks_pcie_unmask_intx_irq, +}; + +static int ks_pcie_init_intx_irq_map(struct irq_domain *d, + unsigned int irq, irq_hw_number_t hw_irq) +{ + irq_set_chip_and_handler(irq, &ks_pcie_intx_irq_chip, + handle_level_irq); + irq_set_chip_data(irq, d->host_data); + + return 0; +} + +static const struct irq_domain_ops ks_pcie_intx_irq_domain_ops = { + .map = ks_pcie_init_intx_irq_map, + .xlate = irq_domain_xlate_onetwocell, +}; + +static int ks_pcie_setup_rc_app_regs(struct keystone_pcie *ks_pcie) +{ + u32 val; + u32 num_viewport = ks_pcie->num_viewport; + struct dw_pcie *pci = ks_pcie->pci; + struct dw_pcie_rp *pp = &pci->pp; + struct resource_entry *entry; + struct resource *mem; + u64 start, end; + int i; + + entry = resource_list_first_type(&pp->bridge->windows, IORESOURCE_MEM); + if (!entry) + return -ENODEV; + + mem = entry->res; + start = mem->start; + end = mem->end; + + /* Disable BARs for inbound access */ + ks_pcie_set_dbi_mode(ks_pcie); + dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0); + dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_1, 0); + ks_pcie_clear_dbi_mode(ks_pcie); + + if (ks_pcie->is_am6) + return 0; + + val = ilog2(OB_WIN_SIZE); + ks_pcie_app_writel(ks_pcie, OB_SIZE, val); + + /* Using Direct 1:1 mapping of RC <-> PCI memory space */ + for (i = 0; i < num_viewport && (start < end); i++) { + ks_pcie_app_writel(ks_pcie, OB_OFFSET_INDEX(i), + lower_32_bits(start) | OB_ENABLEN); + ks_pcie_app_writel(ks_pcie, OB_OFFSET_HI(i), + upper_32_bits(start)); + start += OB_WIN_SIZE * SZ_1M; + } + + val = ks_pcie_app_readl(ks_pcie, CMD_STATUS); + val |= OB_XLAT_EN_VAL; + ks_pcie_app_writel(ks_pcie, CMD_STATUS, val); + + return 0; +} + +static void __iomem *ks_pcie_other_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + struct dw_pcie_rp *pp = bus->sysdata; + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); + u32 reg; + + /* + * Checking whether the link is up here is a last line of defense + * against platforms that forward errors on the system bus as + * SError upon PCI configuration transactions issued when the link + * is down. This check is racy by definition and does not stop + * the system from triggering an SError if the link goes down + * after this check is performed. + */ + if (!dw_pcie_link_up(pci)) + return NULL; + + reg = CFG_BUS(bus->number) | CFG_DEVICE(PCI_SLOT(devfn)) | + CFG_FUNC(PCI_FUNC(devfn)); + if (!pci_is_root_bus(bus->parent)) + reg |= CFG_TYPE1; + ks_pcie_app_writel(ks_pcie, CFG_SETUP, reg); + + return pp->va_cfg0_base + where; +} + +static struct pci_ops ks_child_pcie_ops = { + .map_bus = ks_pcie_other_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; + +static struct pci_ops ks_pcie_ops = { + .map_bus = dw_pcie_own_conf_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; + +/** + * ks_pcie_link_up() - Check if link up + * @pci: A pointer to the dw_pcie structure which holds the DesignWare PCIe host + * controller driver information. + */ +static bool ks_pcie_link_up(struct dw_pcie *pci) +{ + u32 val; + + val = dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG0); + return (val & PORT_LOGIC_LTSSM_STATE_MASK) == PORT_LOGIC_LTSSM_STATE_L0; +} + +static void ks_pcie_stop_link(struct dw_pcie *pci) +{ + struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); + u32 val; + + /* Disable Link training */ + val = ks_pcie_app_readl(ks_pcie, CMD_STATUS); + val &= ~LTSSM_EN_VAL; + ks_pcie_app_writel(ks_pcie, CMD_STATUS, val); +} + +static int ks_pcie_start_link(struct dw_pcie *pci) +{ + struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); + u32 val; + + /* Initiate Link Training */ + val = ks_pcie_app_readl(ks_pcie, CMD_STATUS); + ks_pcie_app_writel(ks_pcie, CMD_STATUS, LTSSM_EN_VAL | val); + + return 0; +} + +static void ks_pcie_quirk(struct pci_dev *dev) +{ + struct pci_bus *bus = dev->bus; + struct keystone_pcie *ks_pcie; + struct device *bridge_dev; + struct pci_dev *bridge; + u32 val; + + static const struct pci_device_id rc_pci_devids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_TI, PCIE_RC_K2HK), + .class = PCI_CLASS_BRIDGE_PCI_NORMAL, .class_mask = ~0, }, + { PCI_DEVICE(PCI_VENDOR_ID_TI, PCIE_RC_K2E), + .class = PCI_CLASS_BRIDGE_PCI_NORMAL, .class_mask = ~0, }, + { PCI_DEVICE(PCI_VENDOR_ID_TI, PCIE_RC_K2L), + .class = PCI_CLASS_BRIDGE_PCI_NORMAL, .class_mask = ~0, }, + { PCI_DEVICE(PCI_VENDOR_ID_TI, PCIE_RC_K2G), + .class = PCI_CLASS_BRIDGE_PCI_NORMAL, .class_mask = ~0, }, + { 0, }, + }; + static const struct pci_device_id am6_pci_devids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_AM654X), + .class = PCI_CLASS_BRIDGE_PCI << 8, .class_mask = ~0, }, + { 0, }, + }; + + if (pci_is_root_bus(bus)) + bridge = dev; + + /* look for the host bridge */ + while (!pci_is_root_bus(bus)) { + bridge = bus->self; + bus = bus->parent; + } + + if (!bridge) + return; + + /* + * Keystone PCI controller has a h/w limitation of + * 256 bytes maximum read request size. It can't handle + * anything higher than this. So force this limit on + * all downstream devices. + */ + if (pci_match_id(rc_pci_devids, bridge)) { + if (pcie_get_readrq(dev) > 256) { + dev_info(&dev->dev, "limiting MRRS to 256 bytes\n"); + pcie_set_readrq(dev, 256); + } + } + + /* + * Memory transactions fail with PCI controller in AM654 PG1.0 + * when MRRS is set to more than 128 bytes. Force the MRRS to + * 128 bytes in all downstream devices. + */ + if (pci_match_id(am6_pci_devids, bridge)) { + bridge_dev = pci_get_host_bridge_device(dev); + if (!bridge_dev || !bridge_dev->parent) + return; + + ks_pcie = dev_get_drvdata(bridge_dev->parent); + if (!ks_pcie) + return; + + val = ks_pcie_app_readl(ks_pcie, PID); + val &= RTL; + val >>= RTL_SHIFT; + if (val != AM6_PCI_PG1_RTL_VER) + return; + + if (pcie_get_readrq(dev) > 128) { + dev_info(&dev->dev, "limiting MRRS to 128 bytes\n"); + pcie_set_readrq(dev, 128); + } + } +} +DECLARE_PCI_FIXUP_ENABLE(PCI_ANY_ID, PCI_ANY_ID, ks_pcie_quirk); + +static void ks_pcie_msi_irq_handler(struct irq_desc *desc) +{ + unsigned int irq = desc->irq_data.hwirq; + struct keystone_pcie *ks_pcie = irq_desc_get_handler_data(desc); + u32 offset = irq - ks_pcie->msi_host_irq; + struct dw_pcie *pci = ks_pcie->pci; + struct dw_pcie_rp *pp = &pci->pp; + struct device *dev = pci->dev; + struct irq_chip *chip = irq_desc_get_chip(desc); + u32 vector, reg, pos; + + dev_dbg(dev, "%s, irq %d\n", __func__, irq); + + /* + * The chained irq handler installation would have replaced normal + * interrupt driver handler so we need to take care of mask/unmask and + * ack operation. + */ + chained_irq_enter(chip, desc); + + reg = ks_pcie_app_readl(ks_pcie, MSI_IRQ_STATUS(offset)); + /* + * MSI0 status bit 0-3 shows vectors 0, 8, 16, 24, MSI1 status bit + * shows 1, 9, 17, 25 and so forth + */ + for (pos = 0; pos < 4; pos++) { + if (!(reg & BIT(pos))) + continue; + + vector = offset + (pos << 3); + dev_dbg(dev, "irq: bit %d, vector %d\n", pos, vector); + generic_handle_domain_irq(pp->irq_domain, vector); + } + + chained_irq_exit(chip, desc); +} + +/** + * ks_pcie_intx_irq_handler() - Handle INTX interrupt + * @desc: Pointer to irq descriptor + * + * Traverse through pending INTX interrupts and invoke handler for each. Also + * takes care of interrupt controller level mask/ack operation. + */ +static void ks_pcie_intx_irq_handler(struct irq_desc *desc) +{ + unsigned int irq = irq_desc_get_irq(desc); + struct keystone_pcie *ks_pcie = irq_desc_get_handler_data(desc); + struct dw_pcie *pci = ks_pcie->pci; + struct device *dev = pci->dev; + u32 irq_offset = irq - ks_pcie->intx_host_irqs[0]; + struct irq_chip *chip = irq_desc_get_chip(desc); + + dev_dbg(dev, ": Handling INTX irq %d\n", irq); + + /* + * The chained irq handler installation would have replaced normal + * interrupt driver handler so we need to take care of mask/unmask and + * ack operation. + */ + chained_irq_enter(chip, desc); + ks_pcie_handle_intx_irq(ks_pcie, irq_offset); + chained_irq_exit(chip, desc); +} + +static int ks_pcie_config_msi_irq(struct keystone_pcie *ks_pcie) +{ + struct device *dev = ks_pcie->pci->dev; + struct device_node *np = ks_pcie->np; + struct device_node *intc_np; + struct irq_data *irq_data; + int irq_count, irq, ret, i; + + if (!IS_ENABLED(CONFIG_PCI_MSI)) + return 0; + + intc_np = of_get_child_by_name(np, "msi-interrupt-controller"); + if (!intc_np) { + if (ks_pcie->is_am6) + return 0; + dev_warn(dev, "msi-interrupt-controller node is absent\n"); + return -EINVAL; + } + + irq_count = of_irq_count(intc_np); + if (!irq_count) { + dev_err(dev, "No IRQ entries in msi-interrupt-controller\n"); + ret = -EINVAL; + goto err; + } + + for (i = 0; i < irq_count; i++) { + irq = irq_of_parse_and_map(intc_np, i); + if (!irq) { + ret = -EINVAL; + goto err; + } + + if (!ks_pcie->msi_host_irq) { + irq_data = irq_get_irq_data(irq); + if (!irq_data) { + ret = -EINVAL; + goto err; + } + ks_pcie->msi_host_irq = irq_data->hwirq; + } + + irq_set_chained_handler_and_data(irq, ks_pcie_msi_irq_handler, + ks_pcie); + } + + of_node_put(intc_np); + return 0; + +err: + of_node_put(intc_np); + return ret; +} + +static int ks_pcie_config_intx_irq(struct keystone_pcie *ks_pcie) +{ + struct device *dev = ks_pcie->pci->dev; + struct irq_domain *intx_irq_domain; + struct device_node *np = ks_pcie->np; + struct device_node *intc_np; + int irq_count, irq, ret = 0, i; + + intc_np = of_get_child_by_name(np, "legacy-interrupt-controller"); + if (!intc_np) { + /* + * Since INTX interrupts are modeled as edge-interrupts in + * AM6, keep it disabled for now. + */ + if (ks_pcie->is_am6) + return 0; + dev_warn(dev, "legacy-interrupt-controller node is absent\n"); + return -EINVAL; + } + + irq_count = of_irq_count(intc_np); + if (!irq_count) { + dev_err(dev, "No IRQ entries in legacy-interrupt-controller\n"); + ret = -EINVAL; + goto err; + } + + for (i = 0; i < irq_count; i++) { + irq = irq_of_parse_and_map(intc_np, i); + if (!irq) { + ret = -EINVAL; + goto err; + } + ks_pcie->intx_host_irqs[i] = irq; + + irq_set_chained_handler_and_data(irq, + ks_pcie_intx_irq_handler, + ks_pcie); + } + + intx_irq_domain = irq_domain_create_linear(of_fwnode_handle(intc_np), PCI_NUM_INTX, + &ks_pcie_intx_irq_domain_ops, NULL); + if (!intx_irq_domain) { + dev_err(dev, "Failed to add irq domain for INTX irqs\n"); + ret = -EINVAL; + goto err; + } + ks_pcie->intx_irq_domain = intx_irq_domain; + + for (i = 0; i < PCI_NUM_INTX; i++) + ks_pcie_app_writel(ks_pcie, IRQ_ENABLE_SET(i), INTx_EN); + +err: + of_node_put(intc_np); + return ret; +} + +static int ks_pcie_init_id(struct keystone_pcie *ks_pcie) +{ + int ret; + unsigned int id; + struct regmap *devctrl_regs; + struct dw_pcie *pci = ks_pcie->pci; + struct device *dev = pci->dev; + struct device_node *np = dev->of_node; + struct of_phandle_args args; + unsigned int offset = 0; + + devctrl_regs = syscon_regmap_lookup_by_phandle(np, "ti,syscon-pcie-id"); + if (IS_ERR(devctrl_regs)) + return PTR_ERR(devctrl_regs); + + /* Do not error out to maintain old DT compatibility */ + ret = of_parse_phandle_with_fixed_args(np, "ti,syscon-pcie-id", 1, 0, &args); + if (!ret) + offset = args.args[0]; + + ret = regmap_read(devctrl_regs, offset, &id); + if (ret) + return ret; + + dw_pcie_dbi_ro_wr_en(pci); + dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, id & PCIE_VENDORID_MASK); + dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, id >> PCIE_DEVICEID_SHIFT); + dw_pcie_dbi_ro_wr_dis(pci); + + return 0; +} + +static int ks_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); + int ret; + + pp->bridge->ops = &ks_pcie_ops; + if (!ks_pcie->is_am6) + pp->bridge->child_ops = &ks_child_pcie_ops; + + ret = ks_pcie_config_intx_irq(ks_pcie); + if (ret) + return ret; + + ret = ks_pcie_config_msi_irq(ks_pcie); + if (ret) + return ret; + + ks_pcie_stop_link(pci); + ret = ks_pcie_setup_rc_app_regs(ks_pcie); + if (ret) + return ret; + + writew(PCI_IO_RANGE_TYPE_32 | (PCI_IO_RANGE_TYPE_32 << 8), + pci->dbi_base + PCI_IO_BASE); + + ret = ks_pcie_init_id(ks_pcie); + if (ret < 0) + return ret; + + return 0; +} + +static const struct dw_pcie_host_ops ks_pcie_host_ops = { + .init = ks_pcie_host_init, + .msi_init = ks_pcie_msi_host_init, +}; + +static const struct dw_pcie_host_ops ks_pcie_am654_host_ops = { + .init = ks_pcie_host_init, +}; + +static irqreturn_t ks_pcie_err_irq_handler(int irq, void *priv) +{ + struct keystone_pcie *ks_pcie = priv; + + return ks_pcie_handle_error_irq(ks_pcie); +} + +static void ks_pcie_am654_write_dbi2(struct dw_pcie *pci, void __iomem *base, + u32 reg, size_t size, u32 val) +{ + struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); + + ks_pcie_set_dbi_mode(ks_pcie); + dw_pcie_write(base + reg, size, val); + ks_pcie_clear_dbi_mode(ks_pcie); +} + +static const struct dw_pcie_ops ks_pcie_dw_pcie_ops = { + .start_link = ks_pcie_start_link, + .stop_link = ks_pcie_stop_link, + .link_up = ks_pcie_link_up, + .write_dbi2 = ks_pcie_am654_write_dbi2, +}; + +static int ks_pcie_am654_ep_init(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + int flags; + + ep->page_size = AM654_WIN_SIZE; + flags = PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_32; + dw_pcie_writel_dbi2(pci, PCI_BASE_ADDRESS_0, APP_ADDR_SPACE_0 - 1); + dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, flags); + + return 0; +} + +static void ks_pcie_am654_raise_intx_irq(struct keystone_pcie *ks_pcie) +{ + struct dw_pcie *pci = ks_pcie->pci; + u8 int_pin; + + int_pin = dw_pcie_readb_dbi(pci, PCI_INTERRUPT_PIN); + if (int_pin == 0 || int_pin > 4) + return; + + ks_pcie_app_writel(ks_pcie, PCIE_LEGACY_IRQ_ENABLE_SET(int_pin), + INT_ENABLE); + ks_pcie_app_writel(ks_pcie, PCIE_EP_IRQ_SET, INT_ENABLE); + mdelay(1); + ks_pcie_app_writel(ks_pcie, PCIE_EP_IRQ_CLR, INT_ENABLE); + ks_pcie_app_writel(ks_pcie, PCIE_LEGACY_IRQ_ENABLE_CLR(int_pin), + INT_ENABLE); +} + +static int ks_pcie_am654_raise_irq(struct dw_pcie_ep *ep, u8 func_no, + unsigned int type, u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct keystone_pcie *ks_pcie = to_keystone_pcie(pci); + + switch (type) { + case PCI_IRQ_INTX: + ks_pcie_am654_raise_intx_irq(ks_pcie); + break; + case PCI_IRQ_MSI: + dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); + break; + case PCI_IRQ_MSIX: + dw_pcie_ep_raise_msix_irq(ep, func_no, interrupt_num); + break; + default: + dev_err(pci->dev, "UNKNOWN IRQ type\n"); + return -EINVAL; + } + + return 0; +} + +static const struct pci_epc_features ks_pcie_am654_epc_features = { + DWC_EPC_COMMON_FEATURES, + .msi_capable = true, + .msix_capable = true, + /* + * TODO: This driver is the only DWC glue driver that had BAR_RESERVED + * BARs, but did not call dw_pcie_ep_reset_bar() for the reserved BARs. + * + * To not change the existing behavior, these BARs were not migrated to + * BAR_DISABLED. If this driver wants the BAR_RESERVED BARs to be + * disabled, it should migrate them to BAR_DISABLED. + * + * If they actually should be enabled, then the driver must also define + * what is behind these reserved BARs, see the definition of struct + * pci_epc_bar_rsvd_region. + */ + .bar[BAR_0] = { .type = BAR_RESERVED, }, + .bar[BAR_1] = { .type = BAR_RESERVED, }, + .bar[BAR_2] = { .type = BAR_RESIZABLE, }, + .bar[BAR_3] = { .type = BAR_FIXED, .fixed_size = SZ_64K, }, + .bar[BAR_4] = { .type = BAR_FIXED, .fixed_size = 256, }, + .bar[BAR_5] = { .type = BAR_RESIZABLE, }, + .align = SZ_64K, +}; + +static const struct pci_epc_features* +ks_pcie_am654_get_features(struct dw_pcie_ep *ep) +{ + return &ks_pcie_am654_epc_features; +} + +static const struct dw_pcie_ep_ops ks_pcie_am654_ep_ops = { + .init = ks_pcie_am654_ep_init, + .raise_irq = ks_pcie_am654_raise_irq, + .get_features = &ks_pcie_am654_get_features, +}; + +static void ks_pcie_disable_phy(struct keystone_pcie *ks_pcie) +{ + int num_lanes = ks_pcie->num_lanes; + + while (num_lanes--) { + phy_power_off(ks_pcie->phy[num_lanes]); + phy_exit(ks_pcie->phy[num_lanes]); + } +} + +static int ks_pcie_enable_phy(struct keystone_pcie *ks_pcie) +{ + int i; + int ret; + int num_lanes = ks_pcie->num_lanes; + + for (i = 0; i < num_lanes; i++) { + ret = phy_reset(ks_pcie->phy[i]); + if (ret < 0) + goto err_phy; + + ret = phy_init(ks_pcie->phy[i]); + if (ret < 0) + goto err_phy; + + ret = phy_power_on(ks_pcie->phy[i]); + if (ret < 0) { + phy_exit(ks_pcie->phy[i]); + goto err_phy; + } + } + + return 0; + +err_phy: + while (--i >= 0) { + phy_power_off(ks_pcie->phy[i]); + phy_exit(ks_pcie->phy[i]); + } + + return ret; +} + +static int ks_pcie_set_mode(struct device *dev) +{ + struct device_node *np = dev->of_node; + struct of_phandle_args args; + unsigned int offset = 0; + struct regmap *syscon; + u32 val; + u32 mask; + int ret = 0; + + syscon = syscon_regmap_lookup_by_phandle(np, "ti,syscon-pcie-mode"); + if (IS_ERR(syscon)) + return 0; + + /* Do not error out to maintain old DT compatibility */ + ret = of_parse_phandle_with_fixed_args(np, "ti,syscon-pcie-mode", 1, 0, &args); + if (!ret) + offset = args.args[0]; + + mask = KS_PCIE_DEV_TYPE_MASK | KS_PCIE_SYSCLOCKOUTEN; + val = KS_PCIE_DEV_TYPE(RC) | KS_PCIE_SYSCLOCKOUTEN; + + ret = regmap_update_bits(syscon, offset, mask, val); + if (ret) { + dev_err(dev, "failed to set pcie mode\n"); + return ret; + } + + return 0; +} + +static int ks_pcie_am654_set_mode(struct device *dev, + enum dw_pcie_device_mode mode) +{ + struct device_node *np = dev->of_node; + struct of_phandle_args args; + unsigned int offset = 0; + struct regmap *syscon; + u32 val; + u32 mask; + int ret = 0; + + syscon = syscon_regmap_lookup_by_phandle(np, "ti,syscon-pcie-mode"); + if (IS_ERR(syscon)) + return 0; + + /* Do not error out to maintain old DT compatibility */ + ret = of_parse_phandle_with_fixed_args(np, "ti,syscon-pcie-mode", 1, 0, &args); + if (!ret) + offset = args.args[0]; + + mask = AM654_PCIE_DEV_TYPE_MASK; + + switch (mode) { + case DW_PCIE_RC_TYPE: + val = RC; + break; + case DW_PCIE_EP_TYPE: + val = EP; + break; + default: + dev_err(dev, "INVALID device type %d\n", mode); + return -EINVAL; + } + + ret = regmap_update_bits(syscon, offset, mask, val); + if (ret) { + dev_err(dev, "failed to set pcie mode\n"); + return ret; + } + + return 0; +} + +static const struct ks_pcie_of_data ks_pcie_rc_of_data = { + .host_ops = &ks_pcie_host_ops, + .mode = DW_PCIE_RC_TYPE, + .version = DW_PCIE_VER_365A, +}; + +static const struct ks_pcie_of_data ks_pcie_am654_rc_of_data = { + .host_ops = &ks_pcie_am654_host_ops, + .mode = DW_PCIE_RC_TYPE, + .version = DW_PCIE_VER_490A, +}; + +static const struct ks_pcie_of_data ks_pcie_am654_ep_of_data = { + .ep_ops = &ks_pcie_am654_ep_ops, + .mode = DW_PCIE_EP_TYPE, + .version = DW_PCIE_VER_490A, +}; + +static const struct of_device_id ks_pcie_of_match[] = { + { + .type = "pci", + .data = &ks_pcie_rc_of_data, + .compatible = "ti,keystone-pcie", + }, + { + .data = &ks_pcie_am654_rc_of_data, + .compatible = "ti,am654-pcie-rc", + }, + { + .data = &ks_pcie_am654_ep_of_data, + .compatible = "ti,am654-pcie-ep", + }, + { }, +}; +MODULE_DEVICE_TABLE(of, ks_pcie_of_match); + +static int ks_pcie_probe(struct platform_device *pdev) +{ + const struct dw_pcie_host_ops *host_ops; + const struct dw_pcie_ep_ops *ep_ops; + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + const struct ks_pcie_of_data *data; + enum dw_pcie_device_mode mode; + struct dw_pcie *pci; + struct keystone_pcie *ks_pcie; + struct device_link **link; + struct gpio_desc *gpiod; + struct resource *res; + void __iomem *base; + u32 num_viewport; + struct phy **phy; + u32 num_lanes; + char name[10]; + u32 version; + int ret; + int irq; + int i; + + data = of_device_get_match_data(dev); + if (!data) + return -EINVAL; + + version = data->version; + host_ops = data->host_ops; + ep_ops = data->ep_ops; + mode = data->mode; + + ks_pcie = devm_kzalloc(dev, sizeof(*ks_pcie), GFP_KERNEL); + if (!ks_pcie) + return -ENOMEM; + + pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); + if (!pci) + return -ENOMEM; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "app"); + ks_pcie->va_app_base = devm_ioremap_resource(dev, res); + if (IS_ERR(ks_pcie->va_app_base)) + return PTR_ERR(ks_pcie->va_app_base); + + ks_pcie->app = *res; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbics"); + base = devm_pci_remap_cfg_resource(dev, res); + if (IS_ERR(base)) + return PTR_ERR(base); + + if (of_device_is_compatible(np, "ti,am654-pcie-rc")) + ks_pcie->is_am6 = true; + + pci->dbi_base = base; + pci->dbi_base2 = base; + pci->dev = dev; + pci->ops = &ks_pcie_dw_pcie_ops; + pci->version = version; + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + ret = devm_request_irq(dev, irq, ks_pcie_err_irq_handler, IRQF_SHARED, + "ks-pcie-error-irq", ks_pcie); + if (ret < 0) { + dev_err(dev, "failed to request error IRQ %d\n", + irq); + return ret; + } + + ret = of_property_read_u32(np, "num-lanes", &num_lanes); + if (ret) + num_lanes = 1; + + phy = devm_kcalloc(dev, num_lanes, sizeof(*phy), GFP_KERNEL); + if (!phy) + return -ENOMEM; + + link = devm_kcalloc(dev, num_lanes, sizeof(*link), GFP_KERNEL); + if (!link) + return -ENOMEM; + + for (i = 0; i < num_lanes; i++) { + snprintf(name, sizeof(name), "pcie-phy%d", i); + phy[i] = devm_phy_optional_get(dev, name); + if (IS_ERR(phy[i])) { + ret = PTR_ERR(phy[i]); + goto err_link; + } + + if (!phy[i]) + continue; + + link[i] = device_link_add(dev, &phy[i]->dev, DL_FLAG_STATELESS); + if (!link[i]) { + ret = -EINVAL; + goto err_link; + } + } + + ks_pcie->np = np; + ks_pcie->pci = pci; + ks_pcie->link = link; + ks_pcie->num_lanes = num_lanes; + ks_pcie->phy = phy; + + gpiod = devm_gpiod_get_optional(dev, "reset", + GPIOD_OUT_LOW); + if (IS_ERR(gpiod)) { + ret = PTR_ERR(gpiod); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to get reset GPIO\n"); + goto err_link; + } + + /* Obtain references to the PHYs */ + for (i = 0; i < num_lanes; i++) + phy_pm_runtime_get_sync(ks_pcie->phy[i]); + + ret = ks_pcie_enable_phy(ks_pcie); + + /* Release references to the PHYs */ + for (i = 0; i < num_lanes; i++) + phy_pm_runtime_put_sync(ks_pcie->phy[i]); + + if (ret) { + dev_err(dev, "failed to enable phy\n"); + goto err_link; + } + + platform_set_drvdata(pdev, ks_pcie); + pm_runtime_enable(dev); + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + dev_err(dev, "pm_runtime_get_sync failed\n"); + goto err_get_sync; + } + + if (dw_pcie_ver_is_ge(pci, 480A)) + ret = ks_pcie_am654_set_mode(dev, mode); + else + ret = ks_pcie_set_mode(dev); + if (ret < 0) + goto err_get_sync; + + switch (mode) { + case DW_PCIE_RC_TYPE: + if (!IS_ENABLED(CONFIG_PCI_KEYSTONE_HOST)) { + ret = -ENODEV; + goto err_get_sync; + } + + ret = of_property_read_u32(np, "num-viewport", &num_viewport); + if (ret < 0) { + dev_err(dev, "unable to read *num-viewport* property\n"); + goto err_get_sync; + } + + /* + * "Power Sequencing and Reset Signal Timings" table in + * PCI EXPRESS CARD ELECTROMECHANICAL SPECIFICATION, REV. 2.0 + * indicates PERST# should be deasserted after minimum of 100us + * once REFCLK is stable. The REFCLK to the connector in RC + * mode is selected while enabling the PHY. So deassert PERST# + * after 100 us. + */ + if (gpiod) { + usleep_range(100, 200); + gpiod_set_value_cansleep(gpiod, 1); + } + + ks_pcie->num_viewport = num_viewport; + pci->pp.ops = host_ops; + ret = dw_pcie_host_init(&pci->pp); + if (ret < 0) + goto err_get_sync; + break; + case DW_PCIE_EP_TYPE: + if (!IS_ENABLED(CONFIG_PCI_KEYSTONE_EP)) { + ret = -ENODEV; + goto err_get_sync; + } + + pci->ep.ops = ep_ops; + ret = dw_pcie_ep_init(&pci->ep); + if (ret < 0) + goto err_get_sync; + + ret = dw_pcie_ep_init_registers(&pci->ep); + if (ret) { + dev_err(dev, "Failed to initialize DWC endpoint registers\n"); + goto err_ep_init; + } + + pci_epc_init_notify(pci->ep.epc); + + break; + default: + dev_err(dev, "INVALID device type %d\n", mode); + ret = -EINVAL; + goto err_get_sync; + } + + ks_pcie_enable_error_irq(ks_pcie); + + return 0; + +err_ep_init: + dw_pcie_ep_deinit(&pci->ep); +err_get_sync: + pm_runtime_put(dev); + pm_runtime_disable(dev); + ks_pcie_disable_phy(ks_pcie); + +err_link: + while (--i >= 0 && link[i]) + device_link_del(link[i]); + + return ret; +} + +static void ks_pcie_remove(struct platform_device *pdev) +{ + struct keystone_pcie *ks_pcie = platform_get_drvdata(pdev); + struct device_link **link = ks_pcie->link; + int num_lanes = ks_pcie->num_lanes; + struct device *dev = &pdev->dev; + + pm_runtime_put(dev); + pm_runtime_disable(dev); + ks_pcie_disable_phy(ks_pcie); + while (num_lanes--) + device_link_del(link[num_lanes]); +} + +static struct platform_driver ks_pcie_driver = { + .probe = ks_pcie_probe, + .remove = ks_pcie_remove, + .driver = { + .name = "keystone-pcie", + .of_match_table = ks_pcie_of_match, + }, +}; + +#ifdef CONFIG_ARM +/* + * When a PCI device does not exist during config cycles, keystone host + * gets a bus error instead of returning 0xffffffff (PCI_ERROR_RESPONSE). + * This handler always returns 0 for this kind of fault. + */ +static int ks_pcie_fault(unsigned long addr, unsigned int fsr, + struct pt_regs *regs) +{ + unsigned long instr = *(unsigned long *)instruction_pointer(regs); + + if ((instr & 0x0e100090) == 0x00100090) { + int reg = (instr >> 12) & 15; + + regs->uregs[reg] = -1; + regs->ARM_pc += 4; + } + + return 0; +} + +static int __init ks_pcie_init(void) +{ + struct device_node *np; + /* + * PCIe access errors that result into OCP errors are caught by ARM as + * "External aborts" + */ + np = of_find_matching_node(NULL, ks_pcie_of_match); + if (np) { + of_node_put(np); + hook_fault_code(17, ks_pcie_fault, SIGBUS, 0, + "Asynchronous external abort"); + } + + return platform_driver_register(&ks_pcie_driver); +} +device_initcall(ks_pcie_init); +#else +builtin_platform_driver(ks_pcie_driver); +#endif + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("PCIe controller driver for Texas Instruments Keystone SoCs"); +MODULE_AUTHOR("Murali Karicheri <m-karicheri2@ti.com>"); diff --git a/drivers/pci/controller/dwc/pci-layerscape-ep.c b/drivers/pci/controller/dwc/pci-layerscape-ep.c new file mode 100644 index 000000000..527c57114 --- /dev/null +++ b/drivers/pci/controller/dwc/pci-layerscape-ep.c @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe controller EP driver for Freescale Layerscape SoCs + * + * Copyright (C) 2018 NXP Semiconductor. + * + * Author: Xiaowei Bao <xiaowei.bao@nxp.com> + */ + +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/of_address.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/resource.h> + +#include "pcie-designware.h" + +#define PEX_PF0_CONFIG 0xC0014 +#define PEX_PF0_CFG_READY BIT(0) + +/* PEX PFa PCIE PME and message interrupt registers*/ +#define PEX_PF0_PME_MES_DR 0xC0020 +#define PEX_PF0_PME_MES_DR_LUD BIT(7) +#define PEX_PF0_PME_MES_DR_LDD BIT(9) +#define PEX_PF0_PME_MES_DR_HRD BIT(10) + +#define PEX_PF0_PME_MES_IER 0xC0028 +#define PEX_PF0_PME_MES_IER_LUDIE BIT(7) +#define PEX_PF0_PME_MES_IER_LDDIE BIT(9) +#define PEX_PF0_PME_MES_IER_HRDIE BIT(10) + +#define to_ls_pcie_ep(x) dev_get_drvdata((x)->dev) + +struct ls_pcie_ep_drvdata { + u32 func_offset; + const struct dw_pcie_ep_ops *ops; + const struct dw_pcie_ops *dw_pcie_ops; +}; + +struct ls_pcie_ep { + struct dw_pcie *pci; + struct pci_epc_features *ls_epc; + const struct ls_pcie_ep_drvdata *drvdata; + int irq; + u32 lnkcap; + bool big_endian; +}; + +static u32 ls_pcie_pf_lut_readl(struct ls_pcie_ep *pcie, u32 offset) +{ + struct dw_pcie *pci = pcie->pci; + + if (pcie->big_endian) + return ioread32be(pci->dbi_base + offset); + else + return ioread32(pci->dbi_base + offset); +} + +static void ls_pcie_pf_lut_writel(struct ls_pcie_ep *pcie, u32 offset, u32 value) +{ + struct dw_pcie *pci = pcie->pci; + + if (pcie->big_endian) + iowrite32be(value, pci->dbi_base + offset); + else + iowrite32(value, pci->dbi_base + offset); +} + +static irqreturn_t ls_pcie_ep_event_handler(int irq, void *dev_id) +{ + struct ls_pcie_ep *pcie = dev_id; + struct dw_pcie *pci = pcie->pci; + u32 val, cfg; + u8 offset; + + val = ls_pcie_pf_lut_readl(pcie, PEX_PF0_PME_MES_DR); + ls_pcie_pf_lut_writel(pcie, PEX_PF0_PME_MES_DR, val); + + if (!val) + return IRQ_NONE; + + if (val & PEX_PF0_PME_MES_DR_LUD) { + + offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + + /* + * The values of the Maximum Link Width and Supported Link + * Speed from the Link Capabilities Register will be lost + * during link down or hot reset. Restore initial value + * that configured by the Reset Configuration Word (RCW). + */ + dw_pcie_dbi_ro_wr_en(pci); + dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, pcie->lnkcap); + dw_pcie_dbi_ro_wr_dis(pci); + + cfg = ls_pcie_pf_lut_readl(pcie, PEX_PF0_CONFIG); + cfg |= PEX_PF0_CFG_READY; + ls_pcie_pf_lut_writel(pcie, PEX_PF0_CONFIG, cfg); + dw_pcie_ep_linkup(&pci->ep); + + dev_dbg(pci->dev, "Link up\n"); + } else if (val & PEX_PF0_PME_MES_DR_LDD) { + dev_dbg(pci->dev, "Link down\n"); + dw_pcie_ep_linkdown(&pci->ep); + } else if (val & PEX_PF0_PME_MES_DR_HRD) { + dev_dbg(pci->dev, "Hot reset\n"); + } + + return IRQ_HANDLED; +} + +static int ls_pcie_ep_interrupt_init(struct ls_pcie_ep *pcie, + struct platform_device *pdev) +{ + u32 val; + int ret; + + pcie->irq = platform_get_irq_byname(pdev, "pme"); + if (pcie->irq < 0) + return pcie->irq; + + ret = devm_request_irq(&pdev->dev, pcie->irq, ls_pcie_ep_event_handler, + IRQF_SHARED, pdev->name, pcie); + if (ret) { + dev_err(&pdev->dev, "Can't register PCIe IRQ\n"); + return ret; + } + + /* Enable interrupts */ + val = ls_pcie_pf_lut_readl(pcie, PEX_PF0_PME_MES_IER); + val |= PEX_PF0_PME_MES_IER_LDDIE | PEX_PF0_PME_MES_IER_HRDIE | + PEX_PF0_PME_MES_IER_LUDIE; + ls_pcie_pf_lut_writel(pcie, PEX_PF0_PME_MES_IER, val); + + return 0; +} + +static const struct pci_epc_features* +ls_pcie_ep_get_features(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); + + return pcie->ls_epc; +} + +static int ls_pcie_ep_init(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); + struct dw_pcie_ep_func *ep_func; + + ep_func = dw_pcie_ep_get_func_from_ep(ep, 0); + if (!ep_func) + return -ENODEV; + + pcie->ls_epc->msi_capable = ep_func->msi_cap ? true : false; + pcie->ls_epc->msix_capable = ep_func->msix_cap ? true : false; + + return 0; +} + +static int ls_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, + unsigned int type, u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + switch (type) { + case PCI_IRQ_INTX: + return dw_pcie_ep_raise_intx_irq(ep, func_no); + case PCI_IRQ_MSI: + return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); + case PCI_IRQ_MSIX: + return dw_pcie_ep_raise_msix_irq_doorbell(ep, func_no, + interrupt_num); + default: + dev_err(pci->dev, "UNKNOWN IRQ type\n"); + return -EINVAL; + } +} + +static unsigned int ls_pcie_ep_get_dbi_offset(struct dw_pcie_ep *ep, u8 func_no) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct ls_pcie_ep *pcie = to_ls_pcie_ep(pci); + + WARN_ON(func_no && !pcie->drvdata->func_offset); + return pcie->drvdata->func_offset * func_no; +} + +static const struct dw_pcie_ep_ops ls_pcie_ep_ops = { + .init = ls_pcie_ep_init, + .raise_irq = ls_pcie_ep_raise_irq, + .get_features = ls_pcie_ep_get_features, + .get_dbi_offset = ls_pcie_ep_get_dbi_offset, +}; + +static const struct ls_pcie_ep_drvdata ls1_ep_drvdata = { + .ops = &ls_pcie_ep_ops, +}; + +static const struct ls_pcie_ep_drvdata ls2_ep_drvdata = { + .func_offset = 0x20000, + .ops = &ls_pcie_ep_ops, +}; + +static const struct ls_pcie_ep_drvdata lx2_ep_drvdata = { + .func_offset = 0x8000, + .ops = &ls_pcie_ep_ops, +}; + +static const struct of_device_id ls_pcie_ep_of_match[] = { + { .compatible = "fsl,ls1028a-pcie-ep", .data = &ls1_ep_drvdata }, + { .compatible = "fsl,ls1046a-pcie-ep", .data = &ls1_ep_drvdata }, + { .compatible = "fsl,ls1088a-pcie-ep", .data = &ls2_ep_drvdata }, + { .compatible = "fsl,ls2088a-pcie-ep", .data = &ls2_ep_drvdata }, + { .compatible = "fsl,lx2160ar2-pcie-ep", .data = &lx2_ep_drvdata }, + { }, +}; + +static int __init ls_pcie_ep_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dw_pcie *pci; + struct ls_pcie_ep *pcie; + struct pci_epc_features *ls_epc; + struct resource *dbi_base; + u8 offset; + int ret; + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); + if (!pci) + return -ENOMEM; + + ls_epc = devm_kzalloc(dev, sizeof(*ls_epc), GFP_KERNEL); + if (!ls_epc) + return -ENOMEM; + + pcie->drvdata = of_device_get_match_data(dev); + + pci->dev = dev; + pci->ops = pcie->drvdata->dw_pcie_ops; + + ls_epc->bar[BAR_2].only_64bit = true; + ls_epc->bar[BAR_4].only_64bit = true; + ls_epc->linkup_notifier = true; + + pcie->pci = pci; + pcie->ls_epc = ls_epc; + + dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs"); + pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); + if (IS_ERR(pci->dbi_base)) + return PTR_ERR(pci->dbi_base); + + pci->ep.ops = &ls_pcie_ep_ops; + + pcie->big_endian = of_property_read_bool(dev->of_node, "big-endian"); + + dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + + platform_set_drvdata(pdev, pcie); + + offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + pcie->lnkcap = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP); + + ret = dw_pcie_ep_init(&pci->ep); + if (ret) + return ret; + + ret = dw_pcie_ep_init_registers(&pci->ep); + if (ret) { + dev_err(dev, "Failed to initialize DWC endpoint registers\n"); + dw_pcie_ep_deinit(&pci->ep); + return ret; + } + + pci_epc_init_notify(pci->ep.epc); + + return ls_pcie_ep_interrupt_init(pcie, pdev); +} + +static struct platform_driver ls_pcie_ep_driver = { + .driver = { + .name = "layerscape-pcie-ep", + .of_match_table = ls_pcie_ep_of_match, + .suppress_bind_attrs = true, + }, +}; +builtin_platform_driver_probe(ls_pcie_ep_driver, ls_pcie_ep_probe); diff --git a/drivers/pci/controller/dwc/pci-layerscape.c b/drivers/pci/controller/dwc/pci-layerscape.c new file mode 100644 index 000000000..14d6ac4fc --- /dev/null +++ b/drivers/pci/controller/dwc/pci-layerscape.c @@ -0,0 +1,429 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for Freescale Layerscape SoCs + * + * Copyright (C) 2014 Freescale Semiconductor. + * Copyright 2021 NXP + * + * Author: Minghuan Lian <Minghuan.Lian@freescale.com> + */ + +#include <linux/delay.h> +#include <linux/kernel.h> +#include <linux/interrupt.h> +#include <linux/init.h> +#include <linux/iopoll.h> +#include <linux/module.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/of_address.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/resource.h> +#include <linux/mfd/syscon.h> +#include <linux/regmap.h> + +#include "../../pci.h" +#include "pcie-designware.h" + +/* PEX Internal Configuration Registers */ +#define PCIE_STRFMR1 0x71c /* Symbol Timer & Filter Mask Register1 */ +#define PCIE_ABSERR 0x8d0 /* Bridge Slave Error Response Register */ +#define PCIE_ABSERR_SETTING 0x9401 /* Forward error of non-posted request */ + +/* PF Message Command Register */ +#define LS_PCIE_PF_MCR 0x2c +#define PF_MCR_PTOMR BIT(0) +#define PF_MCR_EXL2S BIT(1) + +/* LS1021A PEXn PM Write Control Register */ +#define SCFG_PEXPMWRCR(idx) (0x5c + (idx) * 0x64) +#define PMXMTTURNOFF BIT(31) +#define SCFG_PEXSFTRSTCR 0x190 +#define PEXSR(idx) BIT(idx) + +/* LS1043A PEX PME control register */ +#define SCFG_PEXPMECR 0x144 +#define PEXPME(idx) BIT(31 - (idx) * 4) + +/* LS1043A PEX LUT debug register */ +#define LS_PCIE_LDBG 0x7fc +#define LDBG_SR BIT(30) +#define LDBG_WE BIT(31) + +#define PCIE_IATU_NUM 6 + +struct ls_pcie_drvdata { + const u32 pf_lut_off; + const struct dw_pcie_host_ops *ops; + int (*exit_from_l2)(struct dw_pcie_rp *pp); + bool scfg_support; + bool pm_support; +}; + +struct ls_pcie { + struct dw_pcie *pci; + const struct ls_pcie_drvdata *drvdata; + void __iomem *pf_lut_base; + struct regmap *scfg; + int index; + bool big_endian; +}; + +#define ls_pcie_pf_lut_readl_addr(addr) ls_pcie_pf_lut_readl(pcie, addr) +#define to_ls_pcie(x) dev_get_drvdata((x)->dev) + +static bool ls_pcie_is_bridge(struct ls_pcie *pcie) +{ + struct dw_pcie *pci = pcie->pci; + u32 header_type; + + header_type = ioread8(pci->dbi_base + PCI_HEADER_TYPE); + header_type &= PCI_HEADER_TYPE_MASK; + + return header_type == PCI_HEADER_TYPE_BRIDGE; +} + +/* Clear multi-function bit */ +static void ls_pcie_clear_multifunction(struct ls_pcie *pcie) +{ + struct dw_pcie *pci = pcie->pci; + + iowrite8(PCI_HEADER_TYPE_BRIDGE, pci->dbi_base + PCI_HEADER_TYPE); +} + +/* Drop MSG TLP except for Vendor MSG */ +static void ls_pcie_drop_msg_tlp(struct ls_pcie *pcie) +{ + u32 val; + struct dw_pcie *pci = pcie->pci; + + val = ioread32(pci->dbi_base + PCIE_STRFMR1); + val &= 0xDFFFFFFF; + iowrite32(val, pci->dbi_base + PCIE_STRFMR1); +} + +/* Forward error response of outbound non-posted requests */ +static void ls_pcie_fix_error_response(struct ls_pcie *pcie) +{ + struct dw_pcie *pci = pcie->pci; + + iowrite32(PCIE_ABSERR_SETTING, pci->dbi_base + PCIE_ABSERR); +} + +static u32 ls_pcie_pf_lut_readl(struct ls_pcie *pcie, u32 off) +{ + if (pcie->big_endian) + return ioread32be(pcie->pf_lut_base + off); + + return ioread32(pcie->pf_lut_base + off); +} + +static void ls_pcie_pf_lut_writel(struct ls_pcie *pcie, u32 off, u32 val) +{ + if (pcie->big_endian) + iowrite32be(val, pcie->pf_lut_base + off); + else + iowrite32(val, pcie->pf_lut_base + off); +} + +static void ls_pcie_send_turnoff_msg(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct ls_pcie *pcie = to_ls_pcie(pci); + u32 val; + int ret; + + val = ls_pcie_pf_lut_readl(pcie, LS_PCIE_PF_MCR); + val |= PF_MCR_PTOMR; + ls_pcie_pf_lut_writel(pcie, LS_PCIE_PF_MCR, val); + + ret = readx_poll_timeout(ls_pcie_pf_lut_readl_addr, LS_PCIE_PF_MCR, + val, !(val & PF_MCR_PTOMR), + PCIE_PME_TO_L2_TIMEOUT_US/10, + PCIE_PME_TO_L2_TIMEOUT_US); + if (ret) + dev_err(pcie->pci->dev, "PME_Turn_off timeout\n"); +} + +static int ls_pcie_exit_from_l2(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct ls_pcie *pcie = to_ls_pcie(pci); + u32 val; + int ret; + + /* + * Set PF_MCR_EXL2S bit in LS_PCIE_PF_MCR register for the link + * to exit L2 state. + */ + val = ls_pcie_pf_lut_readl(pcie, LS_PCIE_PF_MCR); + val |= PF_MCR_EXL2S; + ls_pcie_pf_lut_writel(pcie, LS_PCIE_PF_MCR, val); + + /* + * L2 exit timeout of 10ms is not defined in the specifications, + * it was chosen based on empirical observations. + */ + ret = readx_poll_timeout(ls_pcie_pf_lut_readl_addr, LS_PCIE_PF_MCR, + val, !(val & PF_MCR_EXL2S), + 1000, + 10000); + if (ret) + dev_err(pcie->pci->dev, "L2 exit timeout\n"); + + return ret; +} + +static int ls_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct ls_pcie *pcie = to_ls_pcie(pci); + + ls_pcie_fix_error_response(pcie); + + dw_pcie_dbi_ro_wr_en(pci); + ls_pcie_clear_multifunction(pcie); + dw_pcie_dbi_ro_wr_dis(pci); + + ls_pcie_drop_msg_tlp(pcie); + + return 0; +} + +static void scfg_pcie_send_turnoff_msg(struct regmap *scfg, u32 reg, u32 mask) +{ + /* Send PME_Turn_Off message */ + regmap_write_bits(scfg, reg, mask, mask); + + /* + * There is no specific register to check for PME_To_Ack from endpoint. + * So on the safe side, wait for PCIE_PME_TO_L2_TIMEOUT_US. + */ + mdelay(PCIE_PME_TO_L2_TIMEOUT_US/1000); + + /* + * Layerscape hardware reference manual recommends clearing the PMXMTTURNOFF bit + * to complete the PME_Turn_Off handshake. + */ + regmap_write_bits(scfg, reg, mask, 0); +} + +static void ls1021a_pcie_send_turnoff_msg(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct ls_pcie *pcie = to_ls_pcie(pci); + + scfg_pcie_send_turnoff_msg(pcie->scfg, SCFG_PEXPMWRCR(pcie->index), PMXMTTURNOFF); +} + +static int scfg_pcie_exit_from_l2(struct regmap *scfg, u32 reg, u32 mask) +{ + /* Reset the PEX wrapper to bring the link out of L2 */ + regmap_write_bits(scfg, reg, mask, mask); + regmap_write_bits(scfg, reg, mask, 0); + + return 0; +} + +static int ls1021a_pcie_exit_from_l2(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct ls_pcie *pcie = to_ls_pcie(pci); + + return scfg_pcie_exit_from_l2(pcie->scfg, SCFG_PEXSFTRSTCR, PEXSR(pcie->index)); +} + +static void ls1043a_pcie_send_turnoff_msg(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct ls_pcie *pcie = to_ls_pcie(pci); + + scfg_pcie_send_turnoff_msg(pcie->scfg, SCFG_PEXPMECR, PEXPME(pcie->index)); +} + +static int ls1043a_pcie_exit_from_l2(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct ls_pcie *pcie = to_ls_pcie(pci); + u32 val; + + /* + * Reset the PEX wrapper to bring the link out of L2. + * LDBG_WE: allows the user to have write access to the PEXDBG[SR] for both setting and + * clearing the soft reset on the PEX module. + * LDBG_SR: When SR is set to 1, the PEX module enters soft reset. + */ + val = ls_pcie_pf_lut_readl(pcie, LS_PCIE_LDBG); + val |= LDBG_WE; + ls_pcie_pf_lut_writel(pcie, LS_PCIE_LDBG, val); + + val = ls_pcie_pf_lut_readl(pcie, LS_PCIE_LDBG); + val |= LDBG_SR; + ls_pcie_pf_lut_writel(pcie, LS_PCIE_LDBG, val); + + val = ls_pcie_pf_lut_readl(pcie, LS_PCIE_LDBG); + val &= ~LDBG_SR; + ls_pcie_pf_lut_writel(pcie, LS_PCIE_LDBG, val); + + val = ls_pcie_pf_lut_readl(pcie, LS_PCIE_LDBG); + val &= ~LDBG_WE; + ls_pcie_pf_lut_writel(pcie, LS_PCIE_LDBG, val); + + return 0; +} + +static const struct dw_pcie_host_ops ls_pcie_host_ops = { + .init = ls_pcie_host_init, + .pme_turn_off = ls_pcie_send_turnoff_msg, +}; + +static const struct dw_pcie_host_ops ls1021a_pcie_host_ops = { + .init = ls_pcie_host_init, + .pme_turn_off = ls1021a_pcie_send_turnoff_msg, +}; + +static const struct ls_pcie_drvdata ls1021a_drvdata = { + .pm_support = true, + .scfg_support = true, + .ops = &ls1021a_pcie_host_ops, + .exit_from_l2 = ls1021a_pcie_exit_from_l2, +}; + +static const struct dw_pcie_host_ops ls1043a_pcie_host_ops = { + .init = ls_pcie_host_init, + .pme_turn_off = ls1043a_pcie_send_turnoff_msg, +}; + +static const struct ls_pcie_drvdata ls1043a_drvdata = { + .pf_lut_off = 0x10000, + .pm_support = true, + .scfg_support = true, + .ops = &ls1043a_pcie_host_ops, + .exit_from_l2 = ls1043a_pcie_exit_from_l2, +}; + +static const struct ls_pcie_drvdata layerscape_drvdata = { + .pf_lut_off = 0xc0000, + .pm_support = true, + .ops = &ls_pcie_host_ops, + .exit_from_l2 = ls_pcie_exit_from_l2, +}; + +static const struct of_device_id ls_pcie_of_match[] = { + { .compatible = "fsl,ls1012a-pcie", .data = &layerscape_drvdata }, + { .compatible = "fsl,ls1021a-pcie", .data = &ls1021a_drvdata }, + { .compatible = "fsl,ls1028a-pcie", .data = &layerscape_drvdata }, + { .compatible = "fsl,ls1043a-pcie", .data = &ls1043a_drvdata }, + { .compatible = "fsl,ls1046a-pcie", .data = &layerscape_drvdata }, + { .compatible = "fsl,ls2080a-pcie", .data = &layerscape_drvdata }, + { .compatible = "fsl,ls2085a-pcie", .data = &layerscape_drvdata }, + { .compatible = "fsl,ls2088a-pcie", .data = &layerscape_drvdata }, + { .compatible = "fsl,ls1088a-pcie", .data = &layerscape_drvdata }, + { }, +}; + +static int ls_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dw_pcie *pci; + struct ls_pcie *pcie; + struct resource *dbi_base; + u32 index[2]; + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); + if (!pci) + return -ENOMEM; + + pcie->drvdata = of_device_get_match_data(dev); + + pci->dev = dev; + pcie->pci = pci; + pci->pp.ops = pcie->drvdata->ops; + + dbi_base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs"); + pci->dbi_base = devm_pci_remap_cfg_resource(dev, dbi_base); + if (IS_ERR(pci->dbi_base)) + return PTR_ERR(pci->dbi_base); + + pcie->big_endian = of_property_read_bool(dev->of_node, "big-endian"); + + pcie->pf_lut_base = pci->dbi_base + pcie->drvdata->pf_lut_off; + + if (pcie->drvdata->scfg_support) { + pcie->scfg = + syscon_regmap_lookup_by_phandle_args(dev->of_node, + "fsl,pcie-scfg", 1, + index); + if (IS_ERR(pcie->scfg)) { + dev_err(dev, "No syscfg phandle specified\n"); + return PTR_ERR(pcie->scfg); + } + + pcie->index = index[1]; + } + + if (!ls_pcie_is_bridge(pcie)) + return -ENODEV; + + platform_set_drvdata(pdev, pcie); + + return dw_pcie_host_init(&pci->pp); +} + +static int ls_pcie_suspend_noirq(struct device *dev) +{ + struct ls_pcie *pcie = dev_get_drvdata(dev); + + if (!pcie->drvdata->pm_support) + return 0; + + return dw_pcie_suspend_noirq(pcie->pci); +} + +static int ls_pcie_resume_noirq(struct device *dev) +{ + struct ls_pcie *pcie = dev_get_drvdata(dev); + int ret; + + if (!pcie->drvdata->pm_support) + return 0; + + ret = pcie->drvdata->exit_from_l2(&pcie->pci->pp); + if (ret) + return ret; + + return dw_pcie_resume_noirq(pcie->pci); +} + +static const struct dev_pm_ops ls_pcie_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(ls_pcie_suspend_noirq, ls_pcie_resume_noirq) +}; + +static void ls_pcie_remove(struct platform_device *pdev) +{ + struct ls_pcie *pcie = platform_get_drvdata(pdev); + + dw_pcie_host_deinit(&pcie->pci->pp); +} + +static struct platform_driver ls_pcie_driver = { + .probe = ls_pcie_probe, + .remove = ls_pcie_remove, + .driver = { + .name = "layerscape-pcie", + .of_match_table = ls_pcie_of_match, + .suppress_bind_attrs = true, + .pm = &ls_pcie_pm_ops, + }, +}; +module_platform_driver(ls_pcie_driver); + +MODULE_AUTHOR("Minghuan Lian <Minghuan.Lian@freescale.com>"); +MODULE_DESCRIPTION("Layerscape PCIe host controller driver"); +MODULE_LICENSE("GPL"); +MODULE_DEVICE_TABLE(of, ls_pcie_of_match); diff --git a/drivers/pci/controller/dwc/pci-meson.c b/drivers/pci/controller/dwc/pci-meson.c new file mode 100644 index 000000000..8559d132d --- /dev/null +++ b/drivers/pci/controller/dwc/pci-meson.c @@ -0,0 +1,487 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for Amlogic MESON SoCs + * + * Copyright (c) 2018 Amlogic, inc. + * Author: Yue Wang <yue.wang@amlogic.com> + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/gpio/consumer.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/reset.h> +#include <linux/resource.h> +#include <linux/types.h> +#include <linux/phy/phy.h> +#include <linux/module.h> + +#include "pcie-designware.h" + +#define to_meson_pcie(x) dev_get_drvdata((x)->dev) + +#define PCIE_CAP_MAX_PAYLOAD_SIZE(x) ((x) << 5) +#define PCIE_CAP_MAX_READ_REQ_SIZE(x) ((x) << 12) + +/* PCIe specific config registers */ +#define PCIE_CFG0 0x0 +#define APP_LTSSM_ENABLE BIT(7) + +#define PCIE_CFG_STATUS12 0x30 +#define IS_SMLH_LINK_UP(x) ((x) & (1 << 6)) +#define IS_RDLH_LINK_UP(x) ((x) & (1 << 16)) +#define IS_LTSSM_UP(x) ((((x) >> 10) & 0x1f) == 0x11) + +#define PCIE_CFG_STATUS17 0x44 +#define PM_CURRENT_STATE(x) (((x) >> 7) & 0x1) + +#define PORT_CLK_RATE 100000000UL +#define MAX_PAYLOAD_SIZE 256 +#define MAX_READ_REQ_SIZE 256 +#define PCIE_RESET_DELAY 500 +#define PCIE_SHARED_RESET 1 +#define PCIE_NORMAL_RESET 0 + +enum pcie_data_rate { + PCIE_GEN1, + PCIE_GEN2, + PCIE_GEN3, + PCIE_GEN4 +}; + +struct meson_pcie_clk_res { + struct clk *clk; + struct clk *port_clk; + struct clk *general_clk; +}; + +struct meson_pcie_rc_reset { + struct reset_control *port; + struct reset_control *apb; +}; + +struct meson_pcie { + struct dw_pcie pci; + void __iomem *cfg_base; + struct meson_pcie_clk_res clk_res; + struct meson_pcie_rc_reset mrst; + struct gpio_desc *reset_gpio; + struct phy *phy; +}; + +static struct reset_control *meson_pcie_get_reset(struct meson_pcie *mp, + const char *id, + u32 reset_type) +{ + struct device *dev = mp->pci.dev; + struct reset_control *reset; + + if (reset_type == PCIE_SHARED_RESET) + reset = devm_reset_control_get_shared(dev, id); + else + reset = devm_reset_control_get(dev, id); + + return reset; +} + +static int meson_pcie_get_resets(struct meson_pcie *mp) +{ + struct meson_pcie_rc_reset *mrst = &mp->mrst; + + mrst->port = meson_pcie_get_reset(mp, "port", PCIE_NORMAL_RESET); + if (IS_ERR(mrst->port)) + return PTR_ERR(mrst->port); + reset_control_deassert(mrst->port); + + mrst->apb = meson_pcie_get_reset(mp, "apb", PCIE_SHARED_RESET); + if (IS_ERR(mrst->apb)) + return PTR_ERR(mrst->apb); + reset_control_deassert(mrst->apb); + + return 0; +} + +static int meson_pcie_get_mems(struct platform_device *pdev, + struct meson_pcie *mp) +{ + struct dw_pcie *pci = &mp->pci; + struct resource *res; + + /* + * For the broken DTs that supply 'dbi' as 'elbi', parse the 'elbi' + * region and assign it to both 'pci->elbi_base' and 'pci->dbi_space' so + * that the DWC core can skip parsing both regions. + */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "elbi"); + if (res) { + pci->elbi_base = devm_pci_remap_cfg_resource(pci->dev, res); + if (IS_ERR(pci->elbi_base)) + return PTR_ERR(pci->elbi_base); + + pci->dbi_base = pci->elbi_base; + pci->dbi_phys_addr = res->start; + } + + mp->cfg_base = devm_platform_ioremap_resource_byname(pdev, "cfg"); + if (IS_ERR(mp->cfg_base)) + return PTR_ERR(mp->cfg_base); + + return 0; +} + +static int meson_pcie_power_on(struct meson_pcie *mp) +{ + int ret = 0; + + ret = phy_init(mp->phy); + if (ret) + return ret; + + ret = phy_power_on(mp->phy); + if (ret) { + phy_exit(mp->phy); + return ret; + } + + return 0; +} + +static void meson_pcie_power_off(struct meson_pcie *mp) +{ + phy_power_off(mp->phy); + phy_exit(mp->phy); +} + +static int meson_pcie_reset(struct meson_pcie *mp) +{ + struct meson_pcie_rc_reset *mrst = &mp->mrst; + int ret = 0; + + ret = phy_reset(mp->phy); + if (ret) + return ret; + + reset_control_assert(mrst->port); + reset_control_assert(mrst->apb); + udelay(PCIE_RESET_DELAY); + reset_control_deassert(mrst->port); + reset_control_deassert(mrst->apb); + udelay(PCIE_RESET_DELAY); + + return 0; +} + +static inline void meson_pcie_disable_clock(void *data) +{ + struct clk *clk = data; + + clk_disable_unprepare(clk); +} + +static inline struct clk *meson_pcie_probe_clock(struct device *dev, + const char *id, u64 rate) +{ + struct clk *clk; + int ret; + + clk = devm_clk_get(dev, id); + if (IS_ERR(clk)) + return clk; + + if (rate) { + ret = clk_set_rate(clk, rate); + if (ret) { + dev_err(dev, "set clk rate failed, ret = %d\n", ret); + return ERR_PTR(ret); + } + } + + ret = clk_prepare_enable(clk); + if (ret) { + dev_err(dev, "couldn't enable clk\n"); + return ERR_PTR(ret); + } + + ret = devm_add_action_or_reset(dev, meson_pcie_disable_clock, clk); + if (ret) + return ERR_PTR(ret); + + return clk; +} + +static int meson_pcie_probe_clocks(struct meson_pcie *mp) +{ + struct device *dev = mp->pci.dev; + struct meson_pcie_clk_res *res = &mp->clk_res; + + res->port_clk = meson_pcie_probe_clock(dev, "port", PORT_CLK_RATE); + if (IS_ERR(res->port_clk)) + return PTR_ERR(res->port_clk); + + res->general_clk = meson_pcie_probe_clock(dev, "general", 0); + if (IS_ERR(res->general_clk)) + return PTR_ERR(res->general_clk); + + res->clk = meson_pcie_probe_clock(dev, "pclk", 0); + if (IS_ERR(res->clk)) + return PTR_ERR(res->clk); + + return 0; +} + +static inline u32 meson_cfg_readl(struct meson_pcie *mp, u32 reg) +{ + return readl(mp->cfg_base + reg); +} + +static inline void meson_cfg_writel(struct meson_pcie *mp, u32 val, u32 reg) +{ + writel(val, mp->cfg_base + reg); +} + +static void meson_pcie_assert_reset(struct meson_pcie *mp) +{ + gpiod_set_value_cansleep(mp->reset_gpio, 1); + udelay(500); + gpiod_set_value_cansleep(mp->reset_gpio, 0); +} + +static void meson_pcie_ltssm_enable(struct meson_pcie *mp) +{ + u32 val; + + val = meson_cfg_readl(mp, PCIE_CFG0); + val |= APP_LTSSM_ENABLE; + meson_cfg_writel(mp, val, PCIE_CFG0); +} + +static int meson_size_to_payload(struct meson_pcie *mp, int size) +{ + struct device *dev = mp->pci.dev; + + /* + * dwc supports 2^(val+7) payload size, which val is 0~5 default to 1. + * So if input size is not 2^order alignment or less than 2^7 or bigger + * than 2^12, just set to default size 2^(1+7). + */ + if (!is_power_of_2(size) || size < 128 || size > 4096) { + dev_warn(dev, "payload size %d, set to default 256\n", size); + return 1; + } + + return fls(size) - 8; +} + +static void meson_set_max_payload(struct meson_pcie *mp, int size) +{ + struct dw_pcie *pci = &mp->pci; + u32 val; + u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + int max_payload_size = meson_size_to_payload(mp, size); + + val = dw_pcie_readl_dbi(pci, offset + PCI_EXP_DEVCTL); + val &= ~PCI_EXP_DEVCTL_PAYLOAD; + dw_pcie_writel_dbi(pci, offset + PCI_EXP_DEVCTL, val); + + val = dw_pcie_readl_dbi(pci, offset + PCI_EXP_DEVCTL); + val |= PCIE_CAP_MAX_PAYLOAD_SIZE(max_payload_size); + dw_pcie_writel_dbi(pci, offset + PCI_EXP_DEVCTL, val); +} + +static void meson_set_max_rd_req_size(struct meson_pcie *mp, int size) +{ + struct dw_pcie *pci = &mp->pci; + u32 val; + u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + int max_rd_req_size = meson_size_to_payload(mp, size); + + val = dw_pcie_readl_dbi(pci, offset + PCI_EXP_DEVCTL); + val &= ~PCI_EXP_DEVCTL_READRQ; + dw_pcie_writel_dbi(pci, offset + PCI_EXP_DEVCTL, val); + + val = dw_pcie_readl_dbi(pci, offset + PCI_EXP_DEVCTL); + val |= PCIE_CAP_MAX_READ_REQ_SIZE(max_rd_req_size); + dw_pcie_writel_dbi(pci, offset + PCI_EXP_DEVCTL, val); +} + +static int meson_pcie_start_link(struct dw_pcie *pci) +{ + struct meson_pcie *mp = to_meson_pcie(pci); + + meson_pcie_ltssm_enable(mp); + meson_pcie_assert_reset(mp); + + return 0; +} + +static int meson_pcie_rd_own_conf(struct pci_bus *bus, u32 devfn, + int where, int size, u32 *val) +{ + int ret; + + ret = pci_generic_config_read(bus, devfn, where, size, val); + if (ret != PCIBIOS_SUCCESSFUL) + return ret; + + /* + * There is a bug in the MESON AXG PCIe controller whereby software + * cannot program the PCI_CLASS_DEVICE register, so we must fabricate + * the return value in the config accessors. + */ + if ((where & ~3) == PCI_CLASS_REVISION) { + if (size <= 2) + *val = (*val & ((1 << (size * 8)) - 1)) << (8 * (where & 3)); + *val &= ~0xffffff00; + *val |= PCI_CLASS_BRIDGE_PCI_NORMAL << 8; + if (size <= 2) + *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); + } + + return PCIBIOS_SUCCESSFUL; +} + +static struct pci_ops meson_pci_ops = { + .map_bus = dw_pcie_own_conf_map_bus, + .read = meson_pcie_rd_own_conf, + .write = pci_generic_config_write, +}; + +static bool meson_pcie_link_up(struct dw_pcie *pci) +{ + struct meson_pcie *mp = to_meson_pcie(pci); + u32 state12; + + state12 = meson_cfg_readl(mp, PCIE_CFG_STATUS12); + return IS_SMLH_LINK_UP(state12) && IS_RDLH_LINK_UP(state12); +} + +static int meson_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct meson_pcie *mp = to_meson_pcie(pci); + + pp->bridge->ops = &meson_pci_ops; + + meson_set_max_payload(mp, MAX_PAYLOAD_SIZE); + meson_set_max_rd_req_size(mp, MAX_READ_REQ_SIZE); + + return 0; +} + +static const struct dw_pcie_host_ops meson_pcie_host_ops = { + .init = meson_pcie_host_init, +}; + +static const struct dw_pcie_ops dw_pcie_ops = { + .link_up = meson_pcie_link_up, + .start_link = meson_pcie_start_link, +}; + +static int meson_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dw_pcie *pci; + struct meson_pcie *mp; + int ret; + + mp = devm_kzalloc(dev, sizeof(*mp), GFP_KERNEL); + if (!mp) + return -ENOMEM; + + pci = &mp->pci; + pci->dev = dev; + pci->ops = &dw_pcie_ops; + pci->pp.ops = &meson_pcie_host_ops; + pci->num_lanes = 1; + + mp->phy = devm_phy_get(dev, "pcie"); + if (IS_ERR(mp->phy)) { + dev_err(dev, "get phy failed, %pe\n", mp->phy); + return PTR_ERR(mp->phy); + } + + mp->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH); + if (IS_ERR(mp->reset_gpio)) { + dev_err(dev, "get reset gpio failed\n"); + return PTR_ERR(mp->reset_gpio); + } + + ret = meson_pcie_get_resets(mp); + if (ret) { + dev_err(dev, "get reset resource failed, %d\n", ret); + return ret; + } + + ret = meson_pcie_get_mems(pdev, mp); + if (ret) { + dev_err(dev, "get memory resource failed, %d\n", ret); + return ret; + } + + ret = meson_pcie_power_on(mp); + if (ret) { + dev_err(dev, "phy power on failed, %d\n", ret); + return ret; + } + + ret = meson_pcie_reset(mp); + if (ret) { + dev_err(dev, "reset failed, %d\n", ret); + goto err_phy; + } + + ret = meson_pcie_probe_clocks(mp); + if (ret) { + dev_err(dev, "init clock resources failed, %d\n", ret); + goto err_phy; + } + + platform_set_drvdata(pdev, mp); + + ret = dw_pcie_host_init(&pci->pp); + if (ret < 0) { + dev_err(dev, "Add PCIe port failed, %d\n", ret); + goto err_phy; + } + + return 0; + +err_phy: + meson_pcie_power_off(mp); + return ret; +} + +static void meson_pcie_remove(struct platform_device *pdev) +{ + struct meson_pcie *mp = platform_get_drvdata(pdev); + + dw_pcie_host_deinit(&mp->pci.pp); + meson_pcie_power_off(mp); +} + +static const struct of_device_id meson_pcie_of_match[] = { + { + .compatible = "amlogic,axg-pcie", + }, + { + .compatible = "amlogic,g12a-pcie", + }, + {}, +}; +MODULE_DEVICE_TABLE(of, meson_pcie_of_match); + +static struct platform_driver meson_pcie_driver = { + .probe = meson_pcie_probe, + .remove = meson_pcie_remove, + .driver = { + .name = "meson-pcie", + .of_match_table = meson_pcie_of_match, + }, +}; + +module_platform_driver(meson_pcie_driver); + +MODULE_AUTHOR("Yue Wang <yue.wang@amlogic.com>"); +MODULE_DESCRIPTION("Amlogic PCIe Controller driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/dwc/pcie-al.c b/drivers/pci/controller/dwc/pcie-al.c new file mode 100644 index 000000000..ef8c8ce7c --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-al.c @@ -0,0 +1,385 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for Amazon's Annapurna Labs IP (used in chips + * such as Graviton and Alpine) + * + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Author: Jonathan Chocron <jonnyc@amazon.com> + */ + +#include <linux/pci.h> +#include <linux/pci-ecam.h> +#include <linux/pci-acpi.h> +#include "../../pci.h" + +#if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS) + +struct al_pcie_acpi { + void __iomem *dbi_base; +}; + +static void __iomem *al_pcie_map_bus(struct pci_bus *bus, unsigned int devfn, + int where) +{ + struct pci_config_window *cfg = bus->sysdata; + struct al_pcie_acpi *pcie = cfg->priv; + void __iomem *dbi_base = pcie->dbi_base; + + if (bus->number == cfg->busr.start) { + /* + * The DW PCIe core doesn't filter out transactions to other + * devices/functions on the root bus num, so we do this here. + */ + if (PCI_SLOT(devfn) > 0) + return NULL; + else + return dbi_base + where; + } + + return pci_ecam_map_bus(bus, devfn, where); +} + +static int al_pcie_init(struct pci_config_window *cfg) +{ + struct device *dev = cfg->parent; + struct acpi_device *adev = to_acpi_device(dev); + struct acpi_pci_root *root = acpi_driver_data(adev); + struct al_pcie_acpi *al_pcie; + struct resource *res; + int ret; + + al_pcie = devm_kzalloc(dev, sizeof(*al_pcie), GFP_KERNEL); + if (!al_pcie) + return -ENOMEM; + + res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL); + if (!res) + return -ENOMEM; + + ret = acpi_get_rc_resources(dev, "AMZN0001", root->segment, res); + if (ret) { + dev_err(dev, "can't get rc dbi base address for SEG %d\n", + root->segment); + return ret; + } + + dev_dbg(dev, "Root port dbi res: %pR\n", res); + + al_pcie->dbi_base = devm_pci_remap_cfg_resource(dev, res); + if (IS_ERR(al_pcie->dbi_base)) + return PTR_ERR(al_pcie->dbi_base); + + cfg->priv = al_pcie; + + return 0; +} + +const struct pci_ecam_ops al_pcie_ops = { + .init = al_pcie_init, + .pci_ops = { + .map_bus = al_pcie_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, + } +}; + +#endif /* defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS) */ + +#ifdef CONFIG_PCIE_AL + +#include <linux/of_pci.h> +#include "pcie-designware.h" + +#define AL_PCIE_REV_ID_2 2 +#define AL_PCIE_REV_ID_3 3 +#define AL_PCIE_REV_ID_4 4 + +#define AXI_BASE_OFFSET 0x0 + +#define DEVICE_ID_OFFSET 0x16c + +#define DEVICE_REV_ID 0x0 +#define DEVICE_REV_ID_DEV_ID_MASK GENMASK(31, 16) + +#define DEVICE_REV_ID_DEV_ID_X4 0 +#define DEVICE_REV_ID_DEV_ID_X8 2 +#define DEVICE_REV_ID_DEV_ID_X16 4 + +#define OB_CTRL_REV1_2_OFFSET 0x0040 +#define OB_CTRL_REV3_5_OFFSET 0x0030 + +#define CFG_TARGET_BUS 0x0 +#define CFG_TARGET_BUS_MASK_MASK GENMASK(7, 0) +#define CFG_TARGET_BUS_BUSNUM_MASK GENMASK(15, 8) + +#define CFG_CONTROL 0x4 +#define CFG_CONTROL_SUBBUS_MASK GENMASK(15, 8) +#define CFG_CONTROL_SEC_BUS_MASK GENMASK(23, 16) + +struct al_pcie_reg_offsets { + unsigned int ob_ctrl; +}; + +struct al_pcie_target_bus_cfg { + u8 reg_val; + u8 reg_mask; + u8 ecam_mask; +}; + +struct al_pcie { + struct dw_pcie *pci; + void __iomem *controller_base; /* base of PCIe unit (not DW core) */ + struct device *dev; + resource_size_t ecam_size; + unsigned int controller_rev_id; + struct al_pcie_reg_offsets reg_offsets; + struct al_pcie_target_bus_cfg target_bus_cfg; +}; + +#define to_al_pcie(x) dev_get_drvdata((x)->dev) + +static inline u32 al_pcie_controller_readl(struct al_pcie *pcie, u32 offset) +{ + return readl_relaxed(pcie->controller_base + offset); +} + +static inline void al_pcie_controller_writel(struct al_pcie *pcie, u32 offset, + u32 val) +{ + writel_relaxed(val, pcie->controller_base + offset); +} + +static int al_pcie_rev_id_get(struct al_pcie *pcie, unsigned int *rev_id) +{ + u32 dev_rev_id_val; + u32 dev_id_val; + + dev_rev_id_val = al_pcie_controller_readl(pcie, AXI_BASE_OFFSET + + DEVICE_ID_OFFSET + + DEVICE_REV_ID); + dev_id_val = FIELD_GET(DEVICE_REV_ID_DEV_ID_MASK, dev_rev_id_val); + + switch (dev_id_val) { + case DEVICE_REV_ID_DEV_ID_X4: + *rev_id = AL_PCIE_REV_ID_2; + break; + case DEVICE_REV_ID_DEV_ID_X8: + *rev_id = AL_PCIE_REV_ID_3; + break; + case DEVICE_REV_ID_DEV_ID_X16: + *rev_id = AL_PCIE_REV_ID_4; + break; + default: + dev_err(pcie->dev, "Unsupported dev_id_val (0x%x)\n", + dev_id_val); + return -EINVAL; + } + + dev_dbg(pcie->dev, "dev_id_val: 0x%x\n", dev_id_val); + + return 0; +} + +static int al_pcie_reg_offsets_set(struct al_pcie *pcie) +{ + switch (pcie->controller_rev_id) { + case AL_PCIE_REV_ID_2: + pcie->reg_offsets.ob_ctrl = OB_CTRL_REV1_2_OFFSET; + break; + case AL_PCIE_REV_ID_3: + case AL_PCIE_REV_ID_4: + pcie->reg_offsets.ob_ctrl = OB_CTRL_REV3_5_OFFSET; + break; + default: + dev_err(pcie->dev, "Unsupported controller rev_id: 0x%x\n", + pcie->controller_rev_id); + return -EINVAL; + } + + return 0; +} + +static inline void al_pcie_target_bus_set(struct al_pcie *pcie, + u8 target_bus, + u8 mask_target_bus) +{ + u32 reg; + + reg = FIELD_PREP(CFG_TARGET_BUS_MASK_MASK, mask_target_bus) | + FIELD_PREP(CFG_TARGET_BUS_BUSNUM_MASK, target_bus); + + al_pcie_controller_writel(pcie, AXI_BASE_OFFSET + + pcie->reg_offsets.ob_ctrl + CFG_TARGET_BUS, + reg); +} + +static void __iomem *al_pcie_conf_addr_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + struct dw_pcie_rp *pp = bus->sysdata; + struct al_pcie *pcie = to_al_pcie(to_dw_pcie_from_pp(pp)); + unsigned int busnr = bus->number; + struct al_pcie_target_bus_cfg *target_bus_cfg = &pcie->target_bus_cfg; + unsigned int busnr_ecam = busnr & target_bus_cfg->ecam_mask; + unsigned int busnr_reg = busnr & target_bus_cfg->reg_mask; + + if (busnr_reg != target_bus_cfg->reg_val) { + dev_dbg(pcie->pci->dev, "Changing target bus busnum val from 0x%x to 0x%x\n", + target_bus_cfg->reg_val, busnr_reg); + target_bus_cfg->reg_val = busnr_reg; + al_pcie_target_bus_set(pcie, + target_bus_cfg->reg_val, + target_bus_cfg->reg_mask); + } + + return pp->va_cfg0_base + PCIE_ECAM_OFFSET(busnr_ecam, devfn, where); +} + +static struct pci_ops al_child_pci_ops = { + .map_bus = al_pcie_conf_addr_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; + +static int al_pcie_config_prepare(struct al_pcie *pcie) +{ + struct al_pcie_target_bus_cfg *target_bus_cfg; + struct dw_pcie_rp *pp = &pcie->pci->pp; + unsigned int ecam_bus_mask; + struct resource_entry *ft; + u32 cfg_control_offset; + struct resource *bus; + u8 subordinate_bus; + u8 secondary_bus; + u32 cfg_control; + + ft = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS); + if (!ft) + return -ENODEV; + + bus = ft->res; + target_bus_cfg = &pcie->target_bus_cfg; + + ecam_bus_mask = (pcie->ecam_size >> PCIE_ECAM_BUS_SHIFT) - 1; + if (ecam_bus_mask > 255) { + dev_warn(pcie->dev, "ECAM window size is larger than 256MB. Cutting off at 256\n"); + ecam_bus_mask = 255; + } + + /* This portion is taken from the transaction address */ + target_bus_cfg->ecam_mask = ecam_bus_mask; + /* This portion is taken from the cfg_target_bus reg */ + target_bus_cfg->reg_mask = ~target_bus_cfg->ecam_mask; + target_bus_cfg->reg_val = bus->start & target_bus_cfg->reg_mask; + + al_pcie_target_bus_set(pcie, target_bus_cfg->reg_val, + target_bus_cfg->reg_mask); + + secondary_bus = bus->start + 1; + subordinate_bus = bus->end; + + /* Set the valid values of secondary and subordinate buses */ + cfg_control_offset = AXI_BASE_OFFSET + pcie->reg_offsets.ob_ctrl + + CFG_CONTROL; + + cfg_control = al_pcie_controller_readl(pcie, cfg_control_offset); + FIELD_MODIFY(CFG_CONTROL_SUBBUS_MASK, &cfg_control, subordinate_bus); + FIELD_MODIFY(CFG_CONTROL_SEC_BUS_MASK, &cfg_control, secondary_bus); + al_pcie_controller_writel(pcie, cfg_control_offset, cfg_control); + + return 0; +} + +static int al_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct al_pcie *pcie = to_al_pcie(pci); + int rc; + + pp->bridge->child_ops = &al_child_pci_ops; + + rc = al_pcie_rev_id_get(pcie, &pcie->controller_rev_id); + if (rc) + return rc; + + rc = al_pcie_reg_offsets_set(pcie); + if (rc) + return rc; + + rc = al_pcie_config_prepare(pcie); + if (rc) + return rc; + + return 0; +} + +static const struct dw_pcie_host_ops al_pcie_host_ops = { + .init = al_pcie_host_init, +}; + +static int al_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct resource *controller_res; + struct resource *ecam_res; + struct al_pcie *al_pcie; + struct dw_pcie *pci; + + al_pcie = devm_kzalloc(dev, sizeof(*al_pcie), GFP_KERNEL); + if (!al_pcie) + return -ENOMEM; + + pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); + if (!pci) + return -ENOMEM; + + pci->dev = dev; + pci->pp.ops = &al_pcie_host_ops; + + al_pcie->pci = pci; + al_pcie->dev = dev; + + ecam_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config"); + if (!ecam_res) { + dev_err(dev, "couldn't find 'config' reg in DT\n"); + return -ENOENT; + } + al_pcie->ecam_size = resource_size(ecam_res); + pci->pp.native_ecam = true; + + controller_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "controller"); + al_pcie->controller_base = devm_ioremap_resource(dev, controller_res); + if (IS_ERR(al_pcie->controller_base)) { + dev_err(dev, "couldn't remap controller base %pR\n", + controller_res); + return PTR_ERR(al_pcie->controller_base); + } + + dev_dbg(dev, "From DT: controller_base: %pR\n", controller_res); + + platform_set_drvdata(pdev, al_pcie); + + return dw_pcie_host_init(&pci->pp); +} + +static const struct of_device_id al_pcie_of_match[] = { + { .compatible = "amazon,al-alpine-v2-pcie", + }, + { .compatible = "amazon,al-alpine-v3-pcie", + }, + {}, +}; + +static struct platform_driver al_pcie_driver = { + .driver = { + .name = "al-pcie", + .of_match_table = al_pcie_of_match, + .suppress_bind_attrs = true, + }, + .probe = al_pcie_probe, +}; +builtin_platform_driver(al_pcie_driver); + +#endif /* CONFIG_PCIE_AL*/ diff --git a/drivers/pci/controller/dwc/pcie-amd-mdb.c b/drivers/pci/controller/dwc/pcie-amd-mdb.c new file mode 100644 index 000000000..dee295f28 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-amd-mdb.c @@ -0,0 +1,534 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for AMD MDB PCIe Bridge + * + * Copyright (C) 2024-2025, Advanced Micro Devices, Inc. + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/gpio/consumer.h> +#include <linux/interrupt.h> +#include <linux/irqdomain.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/of_device.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/resource.h> +#include <linux/types.h> + +#include "../../pci.h" +#include "pcie-designware.h" + +#define AMD_MDB_TLP_IR_STATUS_MISC 0x4C0 +#define AMD_MDB_TLP_IR_MASK_MISC 0x4C4 +#define AMD_MDB_TLP_IR_ENABLE_MISC 0x4C8 +#define AMD_MDB_TLP_IR_DISABLE_MISC 0x4CC + +#define AMD_MDB_TLP_PCIE_INTX_MASK GENMASK(23, 16) + +#define AMD_MDB_PCIE_INTR_INTX_ASSERT(x) BIT((x) * 2) + +/* Interrupt registers definitions. */ +#define AMD_MDB_PCIE_INTR_CMPL_TIMEOUT 15 +#define AMD_MDB_PCIE_INTR_INTX 16 +#define AMD_MDB_PCIE_INTR_PM_PME_RCVD 24 +#define AMD_MDB_PCIE_INTR_PME_TO_ACK_RCVD 25 +#define AMD_MDB_PCIE_INTR_MISC_CORRECTABLE 26 +#define AMD_MDB_PCIE_INTR_NONFATAL 27 +#define AMD_MDB_PCIE_INTR_FATAL 28 + +#define IMR(x) BIT(AMD_MDB_PCIE_INTR_ ##x) +#define AMD_MDB_PCIE_IMR_ALL_MASK \ + ( \ + IMR(CMPL_TIMEOUT) | \ + IMR(PM_PME_RCVD) | \ + IMR(PME_TO_ACK_RCVD) | \ + IMR(MISC_CORRECTABLE) | \ + IMR(NONFATAL) | \ + IMR(FATAL) | \ + AMD_MDB_TLP_PCIE_INTX_MASK \ + ) + +/** + * struct amd_mdb_pcie - PCIe port information + * @pci: DesignWare PCIe controller structure + * @slcr: MDB System Level Control and Status Register (SLCR) base + * @intx_domain: INTx IRQ domain pointer + * @mdb_domain: MDB IRQ domain pointer + * @perst_gpio: GPIO descriptor for PERST# signal handling + * @intx_irq: INTx IRQ interrupt number + */ +struct amd_mdb_pcie { + struct dw_pcie pci; + void __iomem *slcr; + struct irq_domain *intx_domain; + struct irq_domain *mdb_domain; + struct gpio_desc *perst_gpio; + int intx_irq; +}; + +static const struct dw_pcie_host_ops amd_mdb_pcie_host_ops = { +}; + +static void amd_mdb_intx_irq_mask(struct irq_data *data) +{ + struct amd_mdb_pcie *pcie = irq_data_get_irq_chip_data(data); + struct dw_pcie *pci = &pcie->pci; + struct dw_pcie_rp *port = &pci->pp; + unsigned long flags; + u32 val; + + raw_spin_lock_irqsave(&port->lock, flags); + val = FIELD_PREP(AMD_MDB_TLP_PCIE_INTX_MASK, + AMD_MDB_PCIE_INTR_INTX_ASSERT(data->hwirq)); + + /* + * Writing '1' to a bit in AMD_MDB_TLP_IR_DISABLE_MISC disables that + * interrupt, writing '0' has no effect. + */ + writel_relaxed(val, pcie->slcr + AMD_MDB_TLP_IR_DISABLE_MISC); + raw_spin_unlock_irqrestore(&port->lock, flags); +} + +static void amd_mdb_intx_irq_unmask(struct irq_data *data) +{ + struct amd_mdb_pcie *pcie = irq_data_get_irq_chip_data(data); + struct dw_pcie *pci = &pcie->pci; + struct dw_pcie_rp *port = &pci->pp; + unsigned long flags; + u32 val; + + raw_spin_lock_irqsave(&port->lock, flags); + val = FIELD_PREP(AMD_MDB_TLP_PCIE_INTX_MASK, + AMD_MDB_PCIE_INTR_INTX_ASSERT(data->hwirq)); + + /* + * Writing '1' to a bit in AMD_MDB_TLP_IR_ENABLE_MISC enables that + * interrupt, writing '0' has no effect. + */ + writel_relaxed(val, pcie->slcr + AMD_MDB_TLP_IR_ENABLE_MISC); + raw_spin_unlock_irqrestore(&port->lock, flags); +} + +static struct irq_chip amd_mdb_intx_irq_chip = { + .name = "AMD MDB INTx", + .irq_mask = amd_mdb_intx_irq_mask, + .irq_unmask = amd_mdb_intx_irq_unmask, +}; + +/** + * amd_mdb_pcie_intx_map - Set the handler for the INTx and mark IRQ as valid + * @domain: IRQ domain + * @irq: Virtual IRQ number + * @hwirq: Hardware interrupt number + * + * Return: Always returns '0'. + */ +static int amd_mdb_pcie_intx_map(struct irq_domain *domain, + unsigned int irq, irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &amd_mdb_intx_irq_chip, + handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + irq_set_status_flags(irq, IRQ_LEVEL); + + return 0; +} + +/* INTx IRQ domain operations. */ +static const struct irq_domain_ops amd_intx_domain_ops = { + .map = amd_mdb_pcie_intx_map, +}; + +static irqreturn_t dw_pcie_rp_intx(int irq, void *args) +{ + struct amd_mdb_pcie *pcie = args; + unsigned long val; + int i, int_status; + + val = readl_relaxed(pcie->slcr + AMD_MDB_TLP_IR_STATUS_MISC); + int_status = FIELD_GET(AMD_MDB_TLP_PCIE_INTX_MASK, val); + + for (i = 0; i < PCI_NUM_INTX; i++) { + if (int_status & AMD_MDB_PCIE_INTR_INTX_ASSERT(i)) + generic_handle_domain_irq(pcie->intx_domain, i); + } + + return IRQ_HANDLED; +} + +#define _IC(x, s)[AMD_MDB_PCIE_INTR_ ## x] = { __stringify(x), s } + +static const struct { + const char *sym; + const char *str; +} intr_cause[32] = { + _IC(CMPL_TIMEOUT, "Completion timeout"), + _IC(PM_PME_RCVD, "PM_PME message received"), + _IC(PME_TO_ACK_RCVD, "PME_TO_ACK message received"), + _IC(MISC_CORRECTABLE, "Correctable error message"), + _IC(NONFATAL, "Non fatal error message"), + _IC(FATAL, "Fatal error message"), +}; + +static void amd_mdb_event_irq_mask(struct irq_data *d) +{ + struct amd_mdb_pcie *pcie = irq_data_get_irq_chip_data(d); + struct dw_pcie *pci = &pcie->pci; + struct dw_pcie_rp *port = &pci->pp; + unsigned long flags; + u32 val; + + raw_spin_lock_irqsave(&port->lock, flags); + val = BIT(d->hwirq); + writel_relaxed(val, pcie->slcr + AMD_MDB_TLP_IR_DISABLE_MISC); + raw_spin_unlock_irqrestore(&port->lock, flags); +} + +static void amd_mdb_event_irq_unmask(struct irq_data *d) +{ + struct amd_mdb_pcie *pcie = irq_data_get_irq_chip_data(d); + struct dw_pcie *pci = &pcie->pci; + struct dw_pcie_rp *port = &pci->pp; + unsigned long flags; + u32 val; + + raw_spin_lock_irqsave(&port->lock, flags); + val = BIT(d->hwirq); + writel_relaxed(val, pcie->slcr + AMD_MDB_TLP_IR_ENABLE_MISC); + raw_spin_unlock_irqrestore(&port->lock, flags); +} + +static struct irq_chip amd_mdb_event_irq_chip = { + .name = "AMD MDB RC-Event", + .irq_mask = amd_mdb_event_irq_mask, + .irq_unmask = amd_mdb_event_irq_unmask, +}; + +static int amd_mdb_pcie_event_map(struct irq_domain *domain, + unsigned int irq, irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &amd_mdb_event_irq_chip, + handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + irq_set_status_flags(irq, IRQ_LEVEL); + + return 0; +} + +static const struct irq_domain_ops event_domain_ops = { + .map = amd_mdb_pcie_event_map, +}; + +static irqreturn_t amd_mdb_pcie_event(int irq, void *args) +{ + struct amd_mdb_pcie *pcie = args; + unsigned long val; + int i; + + val = readl_relaxed(pcie->slcr + AMD_MDB_TLP_IR_STATUS_MISC); + val &= ~readl_relaxed(pcie->slcr + AMD_MDB_TLP_IR_MASK_MISC); + for_each_set_bit(i, &val, 32) + generic_handle_domain_irq(pcie->mdb_domain, i); + writel_relaxed(val, pcie->slcr + AMD_MDB_TLP_IR_STATUS_MISC); + + return IRQ_HANDLED; +} + +static void amd_mdb_pcie_free_irq_domains(struct amd_mdb_pcie *pcie) +{ + if (pcie->intx_domain) { + irq_domain_remove(pcie->intx_domain); + pcie->intx_domain = NULL; + } + + if (pcie->mdb_domain) { + irq_domain_remove(pcie->mdb_domain); + pcie->mdb_domain = NULL; + } +} + +static int amd_mdb_pcie_init_port(struct amd_mdb_pcie *pcie) +{ + unsigned long val; + + /* Disable all TLP interrupts. */ + writel_relaxed(AMD_MDB_PCIE_IMR_ALL_MASK, + pcie->slcr + AMD_MDB_TLP_IR_DISABLE_MISC); + + /* Clear pending TLP interrupts. */ + val = readl_relaxed(pcie->slcr + AMD_MDB_TLP_IR_STATUS_MISC); + val &= AMD_MDB_PCIE_IMR_ALL_MASK; + writel_relaxed(val, pcie->slcr + AMD_MDB_TLP_IR_STATUS_MISC); + + /* Enable all TLP interrupts. */ + writel_relaxed(AMD_MDB_PCIE_IMR_ALL_MASK, + pcie->slcr + AMD_MDB_TLP_IR_ENABLE_MISC); + + return 0; +} + +/** + * amd_mdb_pcie_init_irq_domains - Initialize IRQ domain + * @pcie: PCIe port information + * @pdev: Platform device + * + * Return: Returns '0' on success and error value on failure. + */ +static int amd_mdb_pcie_init_irq_domains(struct amd_mdb_pcie *pcie, + struct platform_device *pdev) +{ + struct dw_pcie *pci = &pcie->pci; + struct dw_pcie_rp *pp = &pci->pp; + struct device *dev = &pdev->dev; + struct device_node *node = dev->of_node; + struct device_node *pcie_intc_node; + int err; + + pcie_intc_node = of_get_child_by_name(node, "interrupt-controller"); + if (!pcie_intc_node) { + dev_err(dev, "No PCIe Intc node found\n"); + return -ENODEV; + } + + pcie->mdb_domain = irq_domain_create_linear(of_fwnode_handle(pcie_intc_node), 32, + &event_domain_ops, pcie); + if (!pcie->mdb_domain) { + err = -ENOMEM; + dev_err(dev, "Failed to add MDB domain\n"); + goto out; + } + + irq_domain_update_bus_token(pcie->mdb_domain, DOMAIN_BUS_NEXUS); + + pcie->intx_domain = irq_domain_create_linear(of_fwnode_handle(pcie_intc_node), + PCI_NUM_INTX, &amd_intx_domain_ops, pcie); + if (!pcie->intx_domain) { + err = -ENOMEM; + dev_err(dev, "Failed to add INTx domain\n"); + goto mdb_out; + } + + of_node_put(pcie_intc_node); + irq_domain_update_bus_token(pcie->intx_domain, DOMAIN_BUS_WIRED); + + raw_spin_lock_init(&pp->lock); + + return 0; +mdb_out: + amd_mdb_pcie_free_irq_domains(pcie); +out: + of_node_put(pcie_intc_node); + return err; +} + +static irqreturn_t amd_mdb_pcie_intr_handler(int irq, void *args) +{ + struct amd_mdb_pcie *pcie = args; + struct device *dev; + struct irq_data *d; + + dev = pcie->pci.dev; + + /* + * In the future, error reporting will be hooked to the AER subsystem. + * Currently, the driver prints a warning message to the user. + */ + d = irq_domain_get_irq_data(pcie->mdb_domain, irq); + if (intr_cause[d->hwirq].str) + dev_warn(dev, "%s\n", intr_cause[d->hwirq].str); + else + dev_warn_once(dev, "Unknown IRQ %ld\n", d->hwirq); + + return IRQ_HANDLED; +} + +static int amd_mdb_setup_irq(struct amd_mdb_pcie *pcie, + struct platform_device *pdev) +{ + struct dw_pcie *pci = &pcie->pci; + struct dw_pcie_rp *pp = &pci->pp; + struct device *dev = &pdev->dev; + int i, irq, err; + + amd_mdb_pcie_init_port(pcie); + + pp->irq = platform_get_irq(pdev, 0); + if (pp->irq < 0) + return pp->irq; + + for (i = 0; i < ARRAY_SIZE(intr_cause); i++) { + if (!intr_cause[i].str) + continue; + + irq = irq_create_mapping(pcie->mdb_domain, i); + if (!irq) { + dev_err(dev, "Failed to map MDB domain interrupt\n"); + return -ENOMEM; + } + + err = devm_request_irq(dev, irq, amd_mdb_pcie_intr_handler, + IRQF_NO_THREAD, intr_cause[i].sym, pcie); + if (err) { + dev_err(dev, "Failed to request IRQ %d, err=%d\n", + irq, err); + return err; + } + } + + pcie->intx_irq = irq_create_mapping(pcie->mdb_domain, + AMD_MDB_PCIE_INTR_INTX); + if (!pcie->intx_irq) { + dev_err(dev, "Failed to map INTx interrupt\n"); + return -ENXIO; + } + + err = devm_request_irq(dev, pcie->intx_irq, dw_pcie_rp_intx, + IRQF_NO_THREAD, NULL, pcie); + if (err) { + dev_err(dev, "Failed to request INTx IRQ %d, err=%d\n", + pcie->intx_irq, err); + return err; + } + + /* Plug the main event handler. */ + err = devm_request_irq(dev, pp->irq, amd_mdb_pcie_event, IRQF_NO_THREAD, + "amd_mdb pcie_irq", pcie); + if (err) { + dev_err(dev, "Failed to request event IRQ %d, err=%d\n", + pp->irq, err); + return err; + } + + return 0; +} + +static int amd_mdb_parse_pcie_port(struct amd_mdb_pcie *pcie) +{ + struct device *dev = pcie->pci.dev; + struct device_node *pcie_port_node __maybe_unused; + + /* + * This platform currently supports only one Root Port, so the loop + * will execute only once. + * TODO: Enhance the driver to handle multiple Root Ports in the future. + */ + for_each_child_of_node_with_prefix(dev->of_node, pcie_port_node, "pcie") { + pcie->perst_gpio = devm_fwnode_gpiod_get(dev, of_fwnode_handle(pcie_port_node), + "reset", GPIOD_OUT_HIGH, NULL); + if (IS_ERR(pcie->perst_gpio)) + return dev_err_probe(dev, PTR_ERR(pcie->perst_gpio), + "Failed to request reset GPIO\n"); + return 0; + } + + return -ENODEV; +} + +static int amd_mdb_add_pcie_port(struct amd_mdb_pcie *pcie, + struct platform_device *pdev) +{ + struct dw_pcie *pci = &pcie->pci; + struct dw_pcie_rp *pp = &pci->pp; + struct device *dev = &pdev->dev; + int err; + + pcie->slcr = devm_platform_ioremap_resource_byname(pdev, "slcr"); + if (IS_ERR(pcie->slcr)) + return PTR_ERR(pcie->slcr); + + err = amd_mdb_pcie_init_irq_domains(pcie, pdev); + if (err) + return err; + + err = amd_mdb_setup_irq(pcie, pdev); + if (err) { + dev_err(dev, "Failed to set up interrupts, err=%d\n", err); + goto out; + } + + pp->ops = &amd_mdb_pcie_host_ops; + + if (pcie->perst_gpio) { + mdelay(PCIE_T_PVPERL_MS); + gpiod_set_value_cansleep(pcie->perst_gpio, 0); + mdelay(PCIE_RESET_CONFIG_WAIT_MS); + } + + err = dw_pcie_host_init(pp); + if (err) { + dev_err(dev, "Failed to initialize host, err=%d\n", err); + goto out; + } + + return 0; + +out: + amd_mdb_pcie_free_irq_domains(pcie); + return err; +} + +static int amd_mdb_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct amd_mdb_pcie *pcie; + struct dw_pcie *pci; + int ret; + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + pci = &pcie->pci; + pci->dev = dev; + + platform_set_drvdata(pdev, pcie); + + ret = amd_mdb_parse_pcie_port(pcie); + /* + * If amd_mdb_parse_pcie_port returns -ENODEV, it indicates that the + * PCIe Bridge node was not found in the device tree. This is not + * considered a fatal error and will trigger a fallback where the + * reset GPIO is acquired directly from the PCIe Host Bridge node. + */ + if (ret) { + if (ret != -ENODEV) + return ret; + + pcie->perst_gpio = devm_gpiod_get_optional(dev, "reset", + GPIOD_OUT_HIGH); + if (IS_ERR(pcie->perst_gpio)) + return dev_err_probe(dev, PTR_ERR(pcie->perst_gpio), + "Failed to request reset GPIO\n"); + } + + return amd_mdb_add_pcie_port(pcie, pdev); +} + +static void amd_mdb_pcie_shutdown(struct platform_device *pdev) +{ + struct amd_mdb_pcie *pcie = platform_get_drvdata(pdev); + + gpiod_set_value_cansleep(pcie->perst_gpio, 1); +} + +static const struct of_device_id amd_mdb_pcie_of_match[] = { + { + .compatible = "amd,versal2-mdb-host", + }, + {}, +}; + +static struct platform_driver amd_mdb_pcie_driver = { + .driver = { + .name = "amd-mdb-pcie", + .of_match_table = amd_mdb_pcie_of_match, + .suppress_bind_attrs = true, + }, + .probe = amd_mdb_pcie_probe, + .shutdown = amd_mdb_pcie_shutdown, +}; + +builtin_platform_driver(amd_mdb_pcie_driver); diff --git a/drivers/pci/controller/dwc/pcie-andes-qilai.c b/drivers/pci/controller/dwc/pcie-andes-qilai.c new file mode 100644 index 000000000..bd1588be4 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-andes-qilai.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Driver for the PCIe Controller in QiLai from Andes + * + * Copyright (C) 2026 Andes Technology Corporation + */ + +#include <linux/bitfield.h> +#include <linux/bits.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/types.h> + +#include "pcie-designware.h" + +#define PCIE_INTR_CONTROL1 0x15c +#define PCIE_MSI_CTRL_INT_EN BIT(28) + +#define PCIE_LOGIC_COHERENCY_CONTROL3 0x8e8 + +/* + * Refer to Table A4-5 (Memory type encoding) in the + * AMBA AXI and ACE Protocol Specification. + * + * The selected value corresponds to the Memory type field: + * "Write-back, Read and Write-allocate". + * + * The last three rows in the table A4-5 in + * AMBA AXI and ACE Protocol Specification: + * ARCACHE AWCACHE Memory type + * ------------------------------------------------------------------ + * 1111 (0111) 0111 Write-back Read-allocate + * 1011 1111 (1011) Write-back Write-allocate + * 1111 1111 Write-back Read and Write-allocate (selected) + */ +#define IOCP_ARCACHE 0b1111 +#define IOCP_AWCACHE 0b1111 + +#define PCIE_CFG_MSTR_ARCACHE_MODE GENMASK(6, 3) +#define PCIE_CFG_MSTR_AWCACHE_MODE GENMASK(14, 11) +#define PCIE_CFG_MSTR_ARCACHE_VALUE GENMASK(22, 19) +#define PCIE_CFG_MSTR_AWCACHE_VALUE GENMASK(30, 27) + +#define PCIE_GEN_CONTROL2 0x54 +#define PCIE_CFG_LTSSM_EN BIT(0) + +#define PCIE_REGS_PCIE_SII_PM_STATE 0xc0 +#define SMLH_LINK_UP BIT(6) +#define RDLH_LINK_UP BIT(7) + +struct qilai_pcie { + struct dw_pcie pci; + void __iomem *apb_base; +}; + +#define to_qilai_pcie(_pci) container_of(_pci, struct qilai_pcie, pci) + +static bool qilai_pcie_link_up(struct dw_pcie *pci) +{ + struct qilai_pcie *pcie = to_qilai_pcie(pci); + u32 val; + + val = readl(pcie->apb_base + PCIE_REGS_PCIE_SII_PM_STATE); + + return FIELD_GET(SMLH_LINK_UP, val) && FIELD_GET(RDLH_LINK_UP, val); +} + +static int qilai_pcie_start_link(struct dw_pcie *pci) +{ + struct qilai_pcie *pcie = to_qilai_pcie(pci); + u32 val; + + val = readl(pcie->apb_base + PCIE_GEN_CONTROL2); + val |= PCIE_CFG_LTSSM_EN; + writel(val, pcie->apb_base + PCIE_GEN_CONTROL2); + + return 0; +} + +static const struct dw_pcie_ops qilai_pcie_ops = { + .link_up = qilai_pcie_link_up, + .start_link = qilai_pcie_start_link, +}; + +/* + * Set up the QiLai PCIe IOCP (IO Coherence Port) Read/Write Behaviors to the + * Write-Back, Read and Write Allocate mode. + * + * The IOCP HW target is SoC last-level cache (L2 Cache), which serves as the + * system cache. The IOCP HW helps maintain cache monitoring, ensuring that + * the device can snoop data from/to the cache. + */ +static void qilai_pcie_iocp_cache_setup(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + u32 val; + + dw_pcie_dbi_ro_wr_en(pci); + + val = dw_pcie_readl_dbi(pci, PCIE_LOGIC_COHERENCY_CONTROL3); + FIELD_MODIFY(PCIE_CFG_MSTR_ARCACHE_MODE, &val, IOCP_ARCACHE); + FIELD_MODIFY(PCIE_CFG_MSTR_AWCACHE_MODE, &val, IOCP_AWCACHE); + FIELD_MODIFY(PCIE_CFG_MSTR_ARCACHE_VALUE, &val, IOCP_ARCACHE); + FIELD_MODIFY(PCIE_CFG_MSTR_AWCACHE_VALUE, &val, IOCP_AWCACHE); + dw_pcie_writel_dbi(pci, PCIE_LOGIC_COHERENCY_CONTROL3, val); + + dw_pcie_dbi_ro_wr_dis(pci); +} + +static void qilai_pcie_enable_msi(struct qilai_pcie *pcie) +{ + u32 val; + + val = readl(pcie->apb_base + PCIE_INTR_CONTROL1); + val |= PCIE_MSI_CTRL_INT_EN; + writel(val, pcie->apb_base + PCIE_INTR_CONTROL1); +} + +static int qilai_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct qilai_pcie *pcie = to_qilai_pcie(pci); + + qilai_pcie_enable_msi(pcie); + + return 0; +} + +static void qilai_pcie_host_post_init(struct dw_pcie_rp *pp) +{ + qilai_pcie_iocp_cache_setup(pp); +} + +static const struct dw_pcie_host_ops qilai_pcie_host_ops = { + .init = qilai_pcie_host_init, + .post_init = qilai_pcie_host_post_init, +}; + +static int qilai_pcie_probe(struct platform_device *pdev) +{ + struct qilai_pcie *pcie; + struct dw_pcie *pci; + struct device *dev = &pdev->dev; + int ret; + + pcie = devm_kzalloc(&pdev->dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + platform_set_drvdata(pdev, pcie); + + pci = &pcie->pci; + pcie->pci.dev = dev; + pcie->pci.ops = &qilai_pcie_ops; + pcie->pci.pp.ops = &qilai_pcie_host_ops; + pci->use_parent_dt_ranges = true; + + dw_pcie_cap_set(&pcie->pci, REQ_RES); + + pcie->apb_base = devm_platform_ioremap_resource_byname(pdev, "apb"); + if (IS_ERR(pcie->apb_base)) + return PTR_ERR(pcie->apb_base); + + pm_runtime_set_active(dev); + pm_runtime_no_callbacks(dev); + devm_pm_runtime_enable(dev); + + ret = dw_pcie_host_init(&pcie->pci.pp); + if (ret) + return dev_err_probe(dev, ret, "Failed to initialize PCIe host\n"); + + return 0; +} + +static const struct of_device_id qilai_pcie_of_match[] = { + { .compatible = "andestech,qilai-pcie" }, + {}, +}; +MODULE_DEVICE_TABLE(of, qilai_pcie_of_match); + +static struct platform_driver qilai_pcie_driver = { + .probe = qilai_pcie_probe, + .driver = { + .name = "qilai-pcie", + .of_match_table = qilai_pcie_of_match, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, +}; + +builtin_platform_driver(qilai_pcie_driver); + +MODULE_AUTHOR("Randolph Lin <randolph@andestech.com>"); +MODULE_DESCRIPTION("Andes QiLai PCIe driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pci/controller/dwc/pcie-armada8k.c b/drivers/pci/controller/dwc/pcie-armada8k.c new file mode 100644 index 000000000..c2650fd0d --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-armada8k.c @@ -0,0 +1,349 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for Marvell Armada-8K SoCs + * + * Armada-8K PCIe Glue Layer Source Code + * + * Copyright (C) 2016 Marvell Technology Group Ltd. + * + * Author: Yehuda Yitshak <yehuday@marvell.com> + * Author: Shadi Ammouri <shadi@marvell.com> + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/of.h> +#include <linux/pci.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/resource.h> +#include <linux/of_pci.h> + +#include "pcie-designware.h" + +#define ARMADA8K_PCIE_MAX_LANES PCIE_LNK_X4 + +struct armada8k_pcie { + struct dw_pcie *pci; + struct clk *clk; + struct clk *clk_reg; + struct phy *phy[ARMADA8K_PCIE_MAX_LANES]; + unsigned int phy_count; +}; + +#define PCIE_VENDOR_REGS_OFFSET 0x8000 + +#define PCIE_GLOBAL_CONTROL_REG (PCIE_VENDOR_REGS_OFFSET + 0x0) +#define PCIE_APP_LTSSM_EN BIT(2) +#define PCIE_DEVICE_TYPE_SHIFT 4 +#define PCIE_DEVICE_TYPE_MASK 0xF +#define PCIE_DEVICE_TYPE_RC 0x4 /* Root complex */ + +#define PCIE_GLOBAL_STATUS_REG (PCIE_VENDOR_REGS_OFFSET + 0x8) +#define PCIE_GLB_STS_RDLH_LINK_UP BIT(1) +#define PCIE_GLB_STS_PHY_LINK_UP BIT(9) + +#define PCIE_GLOBAL_INT_CAUSE1_REG (PCIE_VENDOR_REGS_OFFSET + 0x1C) +#define PCIE_GLOBAL_INT_MASK1_REG (PCIE_VENDOR_REGS_OFFSET + 0x20) +#define PCIE_INT_A_ASSERT_MASK BIT(9) +#define PCIE_INT_B_ASSERT_MASK BIT(10) +#define PCIE_INT_C_ASSERT_MASK BIT(11) +#define PCIE_INT_D_ASSERT_MASK BIT(12) + +#define PCIE_ARCACHE_TRC_REG (PCIE_VENDOR_REGS_OFFSET + 0x50) +#define PCIE_AWCACHE_TRC_REG (PCIE_VENDOR_REGS_OFFSET + 0x54) +#define PCIE_ARUSER_REG (PCIE_VENDOR_REGS_OFFSET + 0x5C) +#define PCIE_AWUSER_REG (PCIE_VENDOR_REGS_OFFSET + 0x60) +/* + * AR/AW Cache defaults: Normal memory, Write-Back, Read / Write + * allocate + */ +#define ARCACHE_DEFAULT_VALUE 0x3511 +#define AWCACHE_DEFAULT_VALUE 0x5311 + +#define DOMAIN_OUTER_SHAREABLE 0x2 +#define AX_USER_DOMAIN_MASK 0x3 +#define AX_USER_DOMAIN_SHIFT 4 + +#define to_armada8k_pcie(x) dev_get_drvdata((x)->dev) + +static void armada8k_pcie_disable_phys(struct armada8k_pcie *pcie) +{ + int i; + + for (i = 0; i < ARMADA8K_PCIE_MAX_LANES; i++) { + phy_power_off(pcie->phy[i]); + phy_exit(pcie->phy[i]); + } +} + +static int armada8k_pcie_enable_phys(struct armada8k_pcie *pcie) +{ + int ret; + int i; + + for (i = 0; i < ARMADA8K_PCIE_MAX_LANES; i++) { + ret = phy_init(pcie->phy[i]); + if (ret) + return ret; + + ret = phy_set_mode_ext(pcie->phy[i], PHY_MODE_PCIE, + pcie->phy_count); + if (ret) { + phy_exit(pcie->phy[i]); + return ret; + } + + ret = phy_power_on(pcie->phy[i]); + if (ret) { + phy_exit(pcie->phy[i]); + return ret; + } + } + + return 0; +} + +static int armada8k_pcie_setup_phys(struct armada8k_pcie *pcie) +{ + struct dw_pcie *pci = pcie->pci; + struct device *dev = pci->dev; + struct device_node *node = dev->of_node; + int ret = 0; + int i; + + for (i = 0; i < ARMADA8K_PCIE_MAX_LANES; i++) { + pcie->phy[i] = devm_of_phy_get_by_index(dev, node, i); + if (IS_ERR(pcie->phy[i])) { + if (PTR_ERR(pcie->phy[i]) != -ENODEV) + return PTR_ERR(pcie->phy[i]); + + pcie->phy[i] = NULL; + continue; + } + + pcie->phy_count++; + } + + /* Old bindings miss the PHY handle, so just warn if there is no PHY */ + if (!pcie->phy_count) + dev_warn(dev, "No available PHY\n"); + + ret = armada8k_pcie_enable_phys(pcie); + if (ret) + dev_err(dev, "Failed to initialize PHY(s) (%d)\n", ret); + + return ret; +} + +static bool armada8k_pcie_link_up(struct dw_pcie *pci) +{ + u32 reg; + u32 mask = PCIE_GLB_STS_RDLH_LINK_UP | PCIE_GLB_STS_PHY_LINK_UP; + + reg = dw_pcie_readl_dbi(pci, PCIE_GLOBAL_STATUS_REG); + + if ((reg & mask) == mask) + return true; + + dev_dbg(pci->dev, "No link detected (Global-Status: 0x%08x).\n", reg); + return false; +} + +static int armada8k_pcie_start_link(struct dw_pcie *pci) +{ + u32 reg; + + /* Start LTSSM */ + reg = dw_pcie_readl_dbi(pci, PCIE_GLOBAL_CONTROL_REG); + reg |= PCIE_APP_LTSSM_EN; + dw_pcie_writel_dbi(pci, PCIE_GLOBAL_CONTROL_REG, reg); + + return 0; +} + +static int armada8k_pcie_host_init(struct dw_pcie_rp *pp) +{ + u32 reg; + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + + if (!dw_pcie_link_up(pci)) { + /* Disable LTSSM state machine to enable configuration */ + reg = dw_pcie_readl_dbi(pci, PCIE_GLOBAL_CONTROL_REG); + reg &= ~(PCIE_APP_LTSSM_EN); + dw_pcie_writel_dbi(pci, PCIE_GLOBAL_CONTROL_REG, reg); + } + + /* Set the device to root complex mode */ + reg = dw_pcie_readl_dbi(pci, PCIE_GLOBAL_CONTROL_REG); + reg &= ~(PCIE_DEVICE_TYPE_MASK << PCIE_DEVICE_TYPE_SHIFT); + reg |= PCIE_DEVICE_TYPE_RC << PCIE_DEVICE_TYPE_SHIFT; + dw_pcie_writel_dbi(pci, PCIE_GLOBAL_CONTROL_REG, reg); + + /* Set the PCIe master AxCache attributes */ + dw_pcie_writel_dbi(pci, PCIE_ARCACHE_TRC_REG, ARCACHE_DEFAULT_VALUE); + dw_pcie_writel_dbi(pci, PCIE_AWCACHE_TRC_REG, AWCACHE_DEFAULT_VALUE); + + /* Set the PCIe master AxDomain attributes */ + reg = dw_pcie_readl_dbi(pci, PCIE_ARUSER_REG); + reg &= ~(AX_USER_DOMAIN_MASK << AX_USER_DOMAIN_SHIFT); + reg |= DOMAIN_OUTER_SHAREABLE << AX_USER_DOMAIN_SHIFT; + dw_pcie_writel_dbi(pci, PCIE_ARUSER_REG, reg); + + reg = dw_pcie_readl_dbi(pci, PCIE_AWUSER_REG); + reg &= ~(AX_USER_DOMAIN_MASK << AX_USER_DOMAIN_SHIFT); + reg |= DOMAIN_OUTER_SHAREABLE << AX_USER_DOMAIN_SHIFT; + dw_pcie_writel_dbi(pci, PCIE_AWUSER_REG, reg); + + /* Enable INT A-D interrupts */ + reg = dw_pcie_readl_dbi(pci, PCIE_GLOBAL_INT_MASK1_REG); + reg |= PCIE_INT_A_ASSERT_MASK | PCIE_INT_B_ASSERT_MASK | + PCIE_INT_C_ASSERT_MASK | PCIE_INT_D_ASSERT_MASK; + dw_pcie_writel_dbi(pci, PCIE_GLOBAL_INT_MASK1_REG, reg); + + return 0; +} + +static irqreturn_t armada8k_pcie_irq_handler(int irq, void *arg) +{ + struct armada8k_pcie *pcie = arg; + struct dw_pcie *pci = pcie->pci; + u32 val; + + /* + * Interrupts are directly handled by the device driver of the + * PCI device. However, they are also latched into the PCIe + * controller, so we simply discard them. + */ + val = dw_pcie_readl_dbi(pci, PCIE_GLOBAL_INT_CAUSE1_REG); + dw_pcie_writel_dbi(pci, PCIE_GLOBAL_INT_CAUSE1_REG, val); + + return IRQ_HANDLED; +} + +static const struct dw_pcie_host_ops armada8k_pcie_host_ops = { + .init = armada8k_pcie_host_init, +}; + +static int armada8k_add_pcie_port(struct armada8k_pcie *pcie, + struct platform_device *pdev) +{ + struct dw_pcie *pci = pcie->pci; + struct dw_pcie_rp *pp = &pci->pp; + struct device *dev = &pdev->dev; + int ret; + + pp->ops = &armada8k_pcie_host_ops; + + pp->irq = platform_get_irq(pdev, 0); + if (pp->irq < 0) + return pp->irq; + + ret = devm_request_irq(dev, pp->irq, armada8k_pcie_irq_handler, + IRQF_SHARED, "armada8k-pcie", pcie); + if (ret) { + dev_err(dev, "failed to request irq %d\n", pp->irq); + return ret; + } + + ret = dw_pcie_host_init(pp); + if (ret) { + dev_err(dev, "failed to initialize host: %d\n", ret); + return ret; + } + + return 0; +} + +static const struct dw_pcie_ops dw_pcie_ops = { + .link_up = armada8k_pcie_link_up, + .start_link = armada8k_pcie_start_link, +}; + +static int armada8k_pcie_probe(struct platform_device *pdev) +{ + struct dw_pcie *pci; + struct armada8k_pcie *pcie; + struct device *dev = &pdev->dev; + struct resource *base; + int ret; + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); + if (!pci) + return -ENOMEM; + + pci->dev = dev; + pci->ops = &dw_pcie_ops; + + pcie->pci = pci; + + pcie->clk = devm_clk_get(dev, NULL); + if (IS_ERR(pcie->clk)) + return PTR_ERR(pcie->clk); + + ret = clk_prepare_enable(pcie->clk); + if (ret) + return ret; + + pcie->clk_reg = devm_clk_get(dev, "reg"); + if (pcie->clk_reg == ERR_PTR(-EPROBE_DEFER)) { + ret = -EPROBE_DEFER; + goto fail; + } + if (!IS_ERR(pcie->clk_reg)) { + ret = clk_prepare_enable(pcie->clk_reg); + if (ret) + goto fail_clkreg; + } + + /* Get the dw-pcie unit configuration/control registers base. */ + base = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ctrl"); + pci->dbi_base = devm_pci_remap_cfg_resource(dev, base); + if (IS_ERR(pci->dbi_base)) { + ret = PTR_ERR(pci->dbi_base); + goto fail_clkreg; + } + + ret = armada8k_pcie_setup_phys(pcie); + if (ret) + goto fail_clkreg; + + platform_set_drvdata(pdev, pcie); + + ret = armada8k_add_pcie_port(pcie, pdev); + if (ret) + goto disable_phy; + + return 0; + +disable_phy: + armada8k_pcie_disable_phys(pcie); +fail_clkreg: + clk_disable_unprepare(pcie->clk_reg); +fail: + clk_disable_unprepare(pcie->clk); + + return ret; +} + +static const struct of_device_id armada8k_pcie_of_match[] = { + { .compatible = "marvell,armada8k-pcie", }, + {}, +}; + +static struct platform_driver armada8k_pcie_driver = { + .probe = armada8k_pcie_probe, + .driver = { + .name = "armada8k-pcie", + .of_match_table = armada8k_pcie_of_match, + .suppress_bind_attrs = true, + }, +}; +builtin_platform_driver(armada8k_pcie_driver); diff --git a/drivers/pci/controller/dwc/pcie-artpec6.c b/drivers/pci/controller/dwc/pcie-artpec6.c new file mode 100644 index 000000000..7f0ba506d --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-artpec6.c @@ -0,0 +1,522 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for Axis ARTPEC-6 SoC + * + * Author: Niklas Cassel <niklas.cassel@axis.com> + * + * Based on work done by Phil Edworthy <phil@edworthys.org> + */ + +#include <linux/delay.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/of.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/resource.h> +#include <linux/signal.h> +#include <linux/types.h> +#include <linux/interrupt.h> +#include <linux/mfd/syscon.h> +#include <linux/regmap.h> + +#include "pcie-designware.h" + +#define to_artpec6_pcie(x) dev_get_drvdata((x)->dev) + +enum artpec_pcie_variants { + ARTPEC6, + ARTPEC7, +}; + +struct artpec6_pcie { + struct dw_pcie *pci; + struct regmap *regmap; /* DT axis,syscon-pcie */ + void __iomem *phy_base; /* DT phy */ + enum artpec_pcie_variants variant; +}; + +struct artpec_pcie_of_data { + enum artpec_pcie_variants variant; + enum dw_pcie_device_mode mode; +}; + +static const struct of_device_id artpec6_pcie_of_match[]; + +/* ARTPEC-6 specific registers */ +#define PCIECFG 0x18 +#define PCIECFG_DBG_OEN BIT(24) +#define PCIECFG_CORE_RESET_REQ BIT(21) +#define PCIECFG_LTSSM_ENABLE BIT(20) +#define PCIECFG_DEVICE_TYPE_MASK GENMASK(19, 16) +#define PCIECFG_CLKREQ_B BIT(11) +#define PCIECFG_REFCLK_ENABLE BIT(10) +#define PCIECFG_PLL_ENABLE BIT(9) +#define PCIECFG_PCLK_ENABLE BIT(8) +#define PCIECFG_RISRCREN BIT(4) +#define PCIECFG_MODE_TX_DRV_EN BIT(3) +#define PCIECFG_CISRREN BIT(2) +#define PCIECFG_MACRO_ENABLE BIT(0) +/* ARTPEC-7 specific fields */ +#define PCIECFG_REFCLKSEL BIT(23) +#define PCIECFG_NOC_RESET BIT(3) + +#define PCIESTAT 0x1c +/* ARTPEC-7 specific fields */ +#define PCIESTAT_EXTREFCLK BIT(3) + +#define NOCCFG 0x40 +#define NOCCFG_ENABLE_CLK_PCIE BIT(4) +#define NOCCFG_POWER_PCIE_IDLEACK BIT(3) +#define NOCCFG_POWER_PCIE_IDLE BIT(2) +#define NOCCFG_POWER_PCIE_IDLEREQ BIT(1) + +#define PHY_STATUS 0x118 +#define PHY_COSPLLLOCK BIT(0) + +#define PHY_TX_ASIC_OUT 0x4040 +#define PHY_TX_ASIC_OUT_TX_ACK BIT(0) + +#define PHY_RX_ASIC_OUT 0x405c +#define PHY_RX_ASIC_OUT_ACK BIT(0) + +static u32 artpec6_pcie_readl(struct artpec6_pcie *artpec6_pcie, u32 offset) +{ + u32 val; + + regmap_read(artpec6_pcie->regmap, offset, &val); + return val; +} + +static void artpec6_pcie_writel(struct artpec6_pcie *artpec6_pcie, u32 offset, u32 val) +{ + regmap_write(artpec6_pcie->regmap, offset, val); +} + +static u64 artpec6_pcie_cpu_addr_fixup(struct dw_pcie *pci, u64 cpu_addr) +{ + struct artpec6_pcie *artpec6_pcie = to_artpec6_pcie(pci); + struct dw_pcie_rp *pp = &pci->pp; + struct dw_pcie_ep *ep = &pci->ep; + + switch (artpec6_pcie->pci->mode) { + case DW_PCIE_RC_TYPE: + return cpu_addr - pp->cfg0_base; + case DW_PCIE_EP_TYPE: + return cpu_addr - ep->phys_base; + default: + dev_err(pci->dev, "UNKNOWN device type\n"); + } + return cpu_addr; +} + +static int artpec6_pcie_establish_link(struct dw_pcie *pci) +{ + struct artpec6_pcie *artpec6_pcie = to_artpec6_pcie(pci); + u32 val; + + val = artpec6_pcie_readl(artpec6_pcie, PCIECFG); + val |= PCIECFG_LTSSM_ENABLE; + artpec6_pcie_writel(artpec6_pcie, PCIECFG, val); + + return 0; +} + +static void artpec6_pcie_stop_link(struct dw_pcie *pci) +{ + struct artpec6_pcie *artpec6_pcie = to_artpec6_pcie(pci); + u32 val; + + val = artpec6_pcie_readl(artpec6_pcie, PCIECFG); + val &= ~PCIECFG_LTSSM_ENABLE; + artpec6_pcie_writel(artpec6_pcie, PCIECFG, val); +} + +static const struct dw_pcie_ops dw_pcie_ops = { + .cpu_addr_fixup = artpec6_pcie_cpu_addr_fixup, + .start_link = artpec6_pcie_establish_link, + .stop_link = artpec6_pcie_stop_link, +}; + +static void artpec6_pcie_wait_for_phy_a6(struct artpec6_pcie *artpec6_pcie) +{ + struct dw_pcie *pci = artpec6_pcie->pci; + struct device *dev = pci->dev; + u32 val; + unsigned int retries; + + retries = 50; + do { + usleep_range(1000, 2000); + val = artpec6_pcie_readl(artpec6_pcie, NOCCFG); + retries--; + } while (retries && + (val & (NOCCFG_POWER_PCIE_IDLEACK | NOCCFG_POWER_PCIE_IDLE))); + if (!retries) + dev_err(dev, "PCIe clock manager did not leave idle state\n"); + + retries = 50; + do { + usleep_range(1000, 2000); + val = readl(artpec6_pcie->phy_base + PHY_STATUS); + retries--; + } while (retries && !(val & PHY_COSPLLLOCK)); + if (!retries) + dev_err(dev, "PHY PLL did not lock\n"); +} + +static void artpec6_pcie_wait_for_phy_a7(struct artpec6_pcie *artpec6_pcie) +{ + struct dw_pcie *pci = artpec6_pcie->pci; + struct device *dev = pci->dev; + u32 val; + u16 phy_status_tx, phy_status_rx; + unsigned int retries; + + retries = 50; + do { + usleep_range(1000, 2000); + val = artpec6_pcie_readl(artpec6_pcie, NOCCFG); + retries--; + } while (retries && + (val & (NOCCFG_POWER_PCIE_IDLEACK | NOCCFG_POWER_PCIE_IDLE))); + if (!retries) + dev_err(dev, "PCIe clock manager did not leave idle state\n"); + + retries = 50; + do { + usleep_range(1000, 2000); + phy_status_tx = readw(artpec6_pcie->phy_base + PHY_TX_ASIC_OUT); + phy_status_rx = readw(artpec6_pcie->phy_base + PHY_RX_ASIC_OUT); + retries--; + } while (retries && ((phy_status_tx & PHY_TX_ASIC_OUT_TX_ACK) || + (phy_status_rx & PHY_RX_ASIC_OUT_ACK))); + if (!retries) + dev_err(dev, "PHY did not enter Pn state\n"); +} + +static void artpec6_pcie_wait_for_phy(struct artpec6_pcie *artpec6_pcie) +{ + switch (artpec6_pcie->variant) { + case ARTPEC6: + artpec6_pcie_wait_for_phy_a6(artpec6_pcie); + break; + case ARTPEC7: + artpec6_pcie_wait_for_phy_a7(artpec6_pcie); + break; + } +} + +static void artpec6_pcie_init_phy_a6(struct artpec6_pcie *artpec6_pcie) +{ + u32 val; + + val = artpec6_pcie_readl(artpec6_pcie, PCIECFG); + val |= PCIECFG_RISRCREN | /* Receiver term. 50 Ohm */ + PCIECFG_MODE_TX_DRV_EN | + PCIECFG_CISRREN | /* Reference clock term. 100 Ohm */ + PCIECFG_MACRO_ENABLE; + val |= PCIECFG_REFCLK_ENABLE; + val &= ~PCIECFG_DBG_OEN; + val &= ~PCIECFG_CLKREQ_B; + artpec6_pcie_writel(artpec6_pcie, PCIECFG, val); + usleep_range(5000, 6000); + + val = artpec6_pcie_readl(artpec6_pcie, NOCCFG); + val |= NOCCFG_ENABLE_CLK_PCIE; + artpec6_pcie_writel(artpec6_pcie, NOCCFG, val); + usleep_range(20, 30); + + val = artpec6_pcie_readl(artpec6_pcie, PCIECFG); + val |= PCIECFG_PCLK_ENABLE | PCIECFG_PLL_ENABLE; + artpec6_pcie_writel(artpec6_pcie, PCIECFG, val); + usleep_range(6000, 7000); + + val = artpec6_pcie_readl(artpec6_pcie, NOCCFG); + val &= ~NOCCFG_POWER_PCIE_IDLEREQ; + artpec6_pcie_writel(artpec6_pcie, NOCCFG, val); +} + +static void artpec6_pcie_init_phy_a7(struct artpec6_pcie *artpec6_pcie) +{ + struct dw_pcie *pci = artpec6_pcie->pci; + u32 val; + bool extrefclk; + + /* Check if external reference clock is connected */ + val = artpec6_pcie_readl(artpec6_pcie, PCIESTAT); + extrefclk = !!(val & PCIESTAT_EXTREFCLK); + dev_dbg(pci->dev, "Using reference clock: %s\n", + extrefclk ? "external" : "internal"); + + val = artpec6_pcie_readl(artpec6_pcie, PCIECFG); + val |= PCIECFG_RISRCREN | /* Receiver term. 50 Ohm */ + PCIECFG_PCLK_ENABLE; + if (extrefclk) + val |= PCIECFG_REFCLKSEL; + else + val &= ~PCIECFG_REFCLKSEL; + artpec6_pcie_writel(artpec6_pcie, PCIECFG, val); + usleep_range(10, 20); + + val = artpec6_pcie_readl(artpec6_pcie, NOCCFG); + val |= NOCCFG_ENABLE_CLK_PCIE; + artpec6_pcie_writel(artpec6_pcie, NOCCFG, val); + usleep_range(20, 30); + + val = artpec6_pcie_readl(artpec6_pcie, NOCCFG); + val &= ~NOCCFG_POWER_PCIE_IDLEREQ; + artpec6_pcie_writel(artpec6_pcie, NOCCFG, val); +} + +static void artpec6_pcie_init_phy(struct artpec6_pcie *artpec6_pcie) +{ + switch (artpec6_pcie->variant) { + case ARTPEC6: + artpec6_pcie_init_phy_a6(artpec6_pcie); + break; + case ARTPEC7: + artpec6_pcie_init_phy_a7(artpec6_pcie); + break; + } +} + +static void artpec6_pcie_assert_core_reset(struct artpec6_pcie *artpec6_pcie) +{ + u32 val; + + val = artpec6_pcie_readl(artpec6_pcie, PCIECFG); + switch (artpec6_pcie->variant) { + case ARTPEC6: + val |= PCIECFG_CORE_RESET_REQ; + break; + case ARTPEC7: + val &= ~PCIECFG_NOC_RESET; + break; + } + artpec6_pcie_writel(artpec6_pcie, PCIECFG, val); +} + +static void artpec6_pcie_deassert_core_reset(struct artpec6_pcie *artpec6_pcie) +{ + u32 val; + + val = artpec6_pcie_readl(artpec6_pcie, PCIECFG); + switch (artpec6_pcie->variant) { + case ARTPEC6: + val &= ~PCIECFG_CORE_RESET_REQ; + break; + case ARTPEC7: + val |= PCIECFG_NOC_RESET; + break; + } + artpec6_pcie_writel(artpec6_pcie, PCIECFG, val); + usleep_range(100, 200); +} + +static int artpec6_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct artpec6_pcie *artpec6_pcie = to_artpec6_pcie(pci); + + if (artpec6_pcie->variant == ARTPEC7) { + pci->n_fts[0] = 180; + pci->n_fts[1] = 180; + } + artpec6_pcie_assert_core_reset(artpec6_pcie); + artpec6_pcie_init_phy(artpec6_pcie); + artpec6_pcie_deassert_core_reset(artpec6_pcie); + artpec6_pcie_wait_for_phy(artpec6_pcie); + + return 0; +} + +static const struct dw_pcie_host_ops artpec6_pcie_host_ops = { + .init = artpec6_pcie_host_init, +}; + +static int artpec6_pcie_ep_init(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct artpec6_pcie *artpec6_pcie = to_artpec6_pcie(pci); + + artpec6_pcie_assert_core_reset(artpec6_pcie); + artpec6_pcie_init_phy(artpec6_pcie); + artpec6_pcie_deassert_core_reset(artpec6_pcie); + artpec6_pcie_wait_for_phy(artpec6_pcie); + + return 0; +} + +static int artpec6_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no, + unsigned int type, u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + switch (type) { + case PCI_IRQ_INTX: + dev_err(pci->dev, "EP cannot trigger INTx IRQs\n"); + return -EINVAL; + case PCI_IRQ_MSI: + return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); + default: + dev_err(pci->dev, "UNKNOWN IRQ type\n"); + } + + return 0; +} + +static const struct pci_epc_features artpec6_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, + .msi_capable = true, +}; + +static const struct pci_epc_features * +artpec6_pcie_get_features(struct dw_pcie_ep *ep) +{ + return &artpec6_pcie_epc_features; +} + +static const struct dw_pcie_ep_ops pcie_ep_ops = { + .init = artpec6_pcie_ep_init, + .raise_irq = artpec6_pcie_raise_irq, + .get_features = artpec6_pcie_get_features, +}; + +static int artpec6_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dw_pcie *pci; + struct artpec6_pcie *artpec6_pcie; + int ret; + const struct artpec_pcie_of_data *data; + enum artpec_pcie_variants variant; + enum dw_pcie_device_mode mode; + u32 val; + + data = of_device_get_match_data(dev); + if (!data) + return -EINVAL; + + variant = (enum artpec_pcie_variants)data->variant; + mode = (enum dw_pcie_device_mode)data->mode; + + artpec6_pcie = devm_kzalloc(dev, sizeof(*artpec6_pcie), GFP_KERNEL); + if (!artpec6_pcie) + return -ENOMEM; + + pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); + if (!pci) + return -ENOMEM; + + pci->dev = dev; + pci->ops = &dw_pcie_ops; + + artpec6_pcie->pci = pci; + artpec6_pcie->variant = variant; + artpec6_pcie->pci->mode = mode; + + artpec6_pcie->phy_base = + devm_platform_ioremap_resource_byname(pdev, "phy"); + if (IS_ERR(artpec6_pcie->phy_base)) + return PTR_ERR(artpec6_pcie->phy_base); + + artpec6_pcie->regmap = + syscon_regmap_lookup_by_phandle(dev->of_node, + "axis,syscon-pcie"); + if (IS_ERR(artpec6_pcie->regmap)) + return PTR_ERR(artpec6_pcie->regmap); + + platform_set_drvdata(pdev, artpec6_pcie); + + switch (artpec6_pcie->pci->mode) { + case DW_PCIE_RC_TYPE: + if (!IS_ENABLED(CONFIG_PCIE_ARTPEC6_HOST)) + return -ENODEV; + + pci->pp.ops = &artpec6_pcie_host_ops; + + ret = dw_pcie_host_init(&pci->pp); + if (ret < 0) + return ret; + break; + case DW_PCIE_EP_TYPE: + if (!IS_ENABLED(CONFIG_PCIE_ARTPEC6_EP)) + return -ENODEV; + + val = artpec6_pcie_readl(artpec6_pcie, PCIECFG); + val &= ~PCIECFG_DEVICE_TYPE_MASK; + artpec6_pcie_writel(artpec6_pcie, PCIECFG, val); + + pci->ep.ops = &pcie_ep_ops; + + ret = dw_pcie_ep_init(&pci->ep); + if (ret) + return ret; + + ret = dw_pcie_ep_init_registers(&pci->ep); + if (ret) { + dev_err(dev, "Failed to initialize DWC endpoint registers\n"); + dw_pcie_ep_deinit(&pci->ep); + return ret; + } + + pci_epc_init_notify(pci->ep.epc); + + break; + default: + dev_err(dev, "INVALID device type %d\n", artpec6_pcie->pci->mode); + } + + return 0; +} + +static const struct artpec_pcie_of_data artpec6_pcie_rc_of_data = { + .variant = ARTPEC6, + .mode = DW_PCIE_RC_TYPE, +}; + +static const struct artpec_pcie_of_data artpec6_pcie_ep_of_data = { + .variant = ARTPEC6, + .mode = DW_PCIE_EP_TYPE, +}; + +static const struct artpec_pcie_of_data artpec7_pcie_rc_of_data = { + .variant = ARTPEC7, + .mode = DW_PCIE_RC_TYPE, +}; + +static const struct artpec_pcie_of_data artpec7_pcie_ep_of_data = { + .variant = ARTPEC7, + .mode = DW_PCIE_EP_TYPE, +}; + +static const struct of_device_id artpec6_pcie_of_match[] = { + { + .compatible = "axis,artpec6-pcie", + .data = &artpec6_pcie_rc_of_data, + }, + { + .compatible = "axis,artpec6-pcie-ep", + .data = &artpec6_pcie_ep_of_data, + }, + { + .compatible = "axis,artpec7-pcie", + .data = &artpec7_pcie_rc_of_data, + }, + { + .compatible = "axis,artpec7-pcie-ep", + .data = &artpec7_pcie_ep_of_data, + }, + {}, +}; + +static struct platform_driver artpec6_pcie_driver = { + .probe = artpec6_pcie_probe, + .driver = { + .name = "artpec6-pcie", + .of_match_table = artpec6_pcie_of_match, + .suppress_bind_attrs = true, + }, +}; +builtin_platform_driver(artpec6_pcie_driver); diff --git a/drivers/pci/controller/dwc/pcie-designware-debugfs.c b/drivers/pci/controller/dwc/pcie-designware-debugfs.c new file mode 100644 index 000000000..be0b84665 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-designware-debugfs.c @@ -0,0 +1,919 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Synopsys DesignWare PCIe controller debugfs driver + * + * Copyright (C) 2025 Samsung Electronics Co., Ltd. + * http://www.samsung.com + * + * Author: Shradha Todi <shradha.t@samsung.com> + */ + +#include <linux/debugfs.h> + +#include "pcie-designware.h" + +#define SD_STATUS_L1LANE_REG 0xb0 +#define PIPE_RXVALID BIT(18) +#define PIPE_DETECT_LANE BIT(17) +#define LANE_SELECT GENMASK(3, 0) + +#define ERR_INJ0_OFF 0x34 +#define EINJ_VAL_DIFF GENMASK(28, 16) +#define EINJ_VC_NUM GENMASK(14, 12) +#define EINJ_TYPE_SHIFT 8 +#define EINJ0_TYPE GENMASK(11, 8) +#define EINJ1_TYPE BIT(8) +#define EINJ2_TYPE GENMASK(9, 8) +#define EINJ3_TYPE GENMASK(10, 8) +#define EINJ4_TYPE GENMASK(10, 8) +#define EINJ5_TYPE BIT(8) +#define EINJ_COUNT GENMASK(7, 0) + +#define ERR_INJ_ENABLE_REG 0x30 + +#define RAS_DES_EVENT_COUNTER_DATA_REG 0xc + +#define RAS_DES_EVENT_COUNTER_CTRL_REG 0x8 +#define EVENT_COUNTER_GROUP_SELECT GENMASK(27, 24) +#define EVENT_COUNTER_EVENT_SELECT GENMASK(23, 16) +#define EVENT_COUNTER_LANE_SELECT GENMASK(11, 8) +#define EVENT_COUNTER_STATUS BIT(7) +#define EVENT_COUNTER_ENABLE GENMASK(4, 2) +#define PER_EVENT_ON 0x3 +#define PER_EVENT_OFF 0x1 + +#define DWC_DEBUGFS_BUF_MAX 128 + +/** + * struct dwc_pcie_rasdes_info - Stores controller common information + * @ras_cap_offset: RAS DES vendor specific extended capability offset + * @reg_event_lock: Mutex used for RAS DES shadow event registers + * + * Any parameter constant to all files of the debugfs hierarchy for a single + * controller will be stored in this struct. It is allocated and assigned to + * controller specific struct dw_pcie during initialization. + */ +struct dwc_pcie_rasdes_info { + u32 ras_cap_offset; + struct mutex reg_event_lock; +}; + +/** + * struct dwc_pcie_rasdes_priv - Stores file specific private data information + * @pci: Reference to the dw_pcie structure + * @idx: Index of specific file related information in array of structs + * + * All debugfs files will have this struct as its private data. + */ +struct dwc_pcie_rasdes_priv { + struct dw_pcie *pci; + int idx; +}; + +/** + * struct dwc_pcie_err_inj - Store details about each error injection + * supported by DWC RAS DES + * @name: Name of the error that can be injected + * @err_inj_group: Group number to which the error belongs. The value + * can range from 0 to 5 + * @err_inj_type: Each group can have multiple types of error + */ +struct dwc_pcie_err_inj { + const char *name; + u32 err_inj_group; + u32 err_inj_type; +}; + +static const struct dwc_pcie_err_inj err_inj_list[] = { + {"tx_lcrc", 0x0, 0x0}, + {"b16_crc_dllp", 0x0, 0x1}, + {"b16_crc_upd_fc", 0x0, 0x2}, + {"tx_ecrc", 0x0, 0x3}, + {"fcrc_tlp", 0x0, 0x4}, + {"parity_tsos", 0x0, 0x5}, + {"parity_skpos", 0x0, 0x6}, + {"rx_lcrc", 0x0, 0x8}, + {"rx_ecrc", 0x0, 0xb}, + {"tlp_err_seq", 0x1, 0x0}, + {"ack_nak_dllp_seq", 0x1, 0x1}, + {"ack_nak_dllp", 0x2, 0x0}, + {"upd_fc_dllp", 0x2, 0x1}, + {"nak_dllp", 0x2, 0x2}, + {"inv_sync_hdr_sym", 0x3, 0x0}, + {"com_pad_ts1", 0x3, 0x1}, + {"com_pad_ts2", 0x3, 0x2}, + {"com_fts", 0x3, 0x3}, + {"com_idl", 0x3, 0x4}, + {"end_edb", 0x3, 0x5}, + {"stp_sdp", 0x3, 0x6}, + {"com_skp", 0x3, 0x7}, + {"posted_tlp_hdr", 0x4, 0x0}, + {"non_post_tlp_hdr", 0x4, 0x1}, + {"cmpl_tlp_hdr", 0x4, 0x2}, + {"posted_tlp_data", 0x4, 0x4}, + {"non_post_tlp_data", 0x4, 0x5}, + {"cmpl_tlp_data", 0x4, 0x6}, + {"duplicate_tlp", 0x5, 0x0}, + {"nullified_tlp", 0x5, 0x1}, +}; + +static const u32 err_inj_type_mask[] = { + EINJ0_TYPE, + EINJ1_TYPE, + EINJ2_TYPE, + EINJ3_TYPE, + EINJ4_TYPE, + EINJ5_TYPE, +}; + +/** + * struct dwc_pcie_event_counter - Store details about each event counter + * supported in DWC RAS DES + * @name: Name of the error counter + * @group_no: Group number that the event belongs to. The value can range + * from 0 to 7 + * @event_no: Event number of the particular event. The value ranges are: + * Group 0: 0 - 10 + * Group 1: 5 - 13 + * Group 2: 0 - 7 + * Group 3: 0 - 5 + * Group 4: 0 - 1 + * Group 5: 0 - 13 + * Group 6: 0 - 6 + * Group 7: 0 - 25 + */ +struct dwc_pcie_event_counter { + const char *name; + u32 group_no; + u32 event_no; +}; + +static const struct dwc_pcie_event_counter event_list[] = { + {"ebuf_overflow", 0x0, 0x0}, + {"ebuf_underrun", 0x0, 0x1}, + {"decode_err", 0x0, 0x2}, + {"running_disparity_err", 0x0, 0x3}, + {"skp_os_parity_err", 0x0, 0x4}, + {"sync_header_err", 0x0, 0x5}, + {"rx_valid_deassertion", 0x0, 0x6}, + {"ctl_skp_os_parity_err", 0x0, 0x7}, + {"retimer_parity_err_1st", 0x0, 0x8}, + {"retimer_parity_err_2nd", 0x0, 0x9}, + {"margin_crc_parity_err", 0x0, 0xA}, + {"detect_ei_infer", 0x1, 0x5}, + {"receiver_err", 0x1, 0x6}, + {"rx_recovery_req", 0x1, 0x7}, + {"n_fts_timeout", 0x1, 0x8}, + {"framing_err", 0x1, 0x9}, + {"deskew_err", 0x1, 0xa}, + {"framing_err_in_l0", 0x1, 0xc}, + {"deskew_uncompleted_err", 0x1, 0xd}, + {"bad_tlp", 0x2, 0x0}, + {"lcrc_err", 0x2, 0x1}, + {"bad_dllp", 0x2, 0x2}, + {"replay_num_rollover", 0x2, 0x3}, + {"replay_timeout", 0x2, 0x4}, + {"rx_nak_dllp", 0x2, 0x5}, + {"tx_nak_dllp", 0x2, 0x6}, + {"retry_tlp", 0x2, 0x7}, + {"fc_timeout", 0x3, 0x0}, + {"poisoned_tlp", 0x3, 0x1}, + {"ecrc_error", 0x3, 0x2}, + {"unsupported_request", 0x3, 0x3}, + {"completer_abort", 0x3, 0x4}, + {"completion_timeout", 0x3, 0x5}, + {"ebuf_skp_add", 0x4, 0x0}, + {"ebuf_skp_del", 0x4, 0x1}, + {"l0_to_recovery_entry", 0x5, 0x0}, + {"l1_to_recovery_entry", 0x5, 0x1}, + {"tx_l0s_entry", 0x5, 0x2}, + {"rx_l0s_entry", 0x5, 0x3}, + {"aspm_l1_reject", 0x5, 0x4}, + {"l1_entry", 0x5, 0x5}, + {"l1_cpm", 0x5, 0x6}, + {"l1.1_entry", 0x5, 0x7}, + {"l1.2_entry", 0x5, 0x8}, + {"l1_short_duration", 0x5, 0x9}, + {"l1.2_abort", 0x5, 0xa}, + {"l2_entry", 0x5, 0xb}, + {"speed_change", 0x5, 0xc}, + {"link_width_change", 0x5, 0xd}, + {"tx_ack_dllp", 0x6, 0x0}, + {"tx_update_fc_dllp", 0x6, 0x1}, + {"rx_ack_dllp", 0x6, 0x2}, + {"rx_update_fc_dllp", 0x6, 0x3}, + {"rx_nullified_tlp", 0x6, 0x4}, + {"tx_nullified_tlp", 0x6, 0x5}, + {"rx_duplicate_tlp", 0x6, 0x6}, + {"tx_memory_write", 0x7, 0x0}, + {"tx_memory_read", 0x7, 0x1}, + {"tx_configuration_write", 0x7, 0x2}, + {"tx_configuration_read", 0x7, 0x3}, + {"tx_io_write", 0x7, 0x4}, + {"tx_io_read", 0x7, 0x5}, + {"tx_completion_without_data", 0x7, 0x6}, + {"tx_completion_w_data", 0x7, 0x7}, + {"tx_message_tlp_pcie_vc_only", 0x7, 0x8}, + {"tx_atomic", 0x7, 0x9}, + {"tx_tlp_with_prefix", 0x7, 0xa}, + {"rx_memory_write", 0x7, 0xb}, + {"rx_memory_read", 0x7, 0xc}, + {"rx_configuration_write", 0x7, 0xd}, + {"rx_configuration_read", 0x7, 0xe}, + {"rx_io_write", 0x7, 0xf}, + {"rx_io_read", 0x7, 0x10}, + {"rx_completion_without_data", 0x7, 0x11}, + {"rx_completion_w_data", 0x7, 0x12}, + {"rx_message_tlp_pcie_vc_only", 0x7, 0x13}, + {"rx_atomic", 0x7, 0x14}, + {"rx_tlp_with_prefix", 0x7, 0x15}, + {"tx_ccix_tlp", 0x7, 0x16}, + {"rx_ccix_tlp", 0x7, 0x17}, + {"tx_deferrable_memory_write_tlp", 0x7, 0x18}, + {"rx_deferrable_memory_write_tlp", 0x7, 0x19}, +}; + +static ssize_t lane_detect_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct dw_pcie *pci = file->private_data; + struct dwc_pcie_rasdes_info *rinfo = pci->debugfs->rasdes_info; + char debugfs_buf[DWC_DEBUGFS_BUF_MAX]; + ssize_t pos; + u32 val; + + val = dw_pcie_readl_dbi(pci, rinfo->ras_cap_offset + SD_STATUS_L1LANE_REG); + val = FIELD_GET(PIPE_DETECT_LANE, val); + if (val) + pos = scnprintf(debugfs_buf, DWC_DEBUGFS_BUF_MAX, "Lane Detected\n"); + else + pos = scnprintf(debugfs_buf, DWC_DEBUGFS_BUF_MAX, "Lane Undetected\n"); + + return simple_read_from_buffer(buf, count, ppos, debugfs_buf, pos); +} + +static ssize_t lane_detect_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct dw_pcie *pci = file->private_data; + struct dwc_pcie_rasdes_info *rinfo = pci->debugfs->rasdes_info; + u32 lane, val; + int ret; + + ret = kstrtou32_from_user(buf, count, 0, &lane); + if (ret) + return ret; + + val = dw_pcie_readl_dbi(pci, rinfo->ras_cap_offset + SD_STATUS_L1LANE_REG); + FIELD_MODIFY(LANE_SELECT, &val, lane); + dw_pcie_writel_dbi(pci, rinfo->ras_cap_offset + SD_STATUS_L1LANE_REG, val); + + return count; +} + +static ssize_t rx_valid_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct dw_pcie *pci = file->private_data; + struct dwc_pcie_rasdes_info *rinfo = pci->debugfs->rasdes_info; + char debugfs_buf[DWC_DEBUGFS_BUF_MAX]; + ssize_t pos; + u32 val; + + val = dw_pcie_readl_dbi(pci, rinfo->ras_cap_offset + SD_STATUS_L1LANE_REG); + val = FIELD_GET(PIPE_RXVALID, val); + if (val) + pos = scnprintf(debugfs_buf, DWC_DEBUGFS_BUF_MAX, "RX Valid\n"); + else + pos = scnprintf(debugfs_buf, DWC_DEBUGFS_BUF_MAX, "RX Invalid\n"); + + return simple_read_from_buffer(buf, count, ppos, debugfs_buf, pos); +} + +static ssize_t rx_valid_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + return lane_detect_write(file, buf, count, ppos); +} + +static ssize_t err_inj_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct dwc_pcie_rasdes_priv *pdata = file->private_data; + struct dw_pcie *pci = pdata->pci; + struct dwc_pcie_rasdes_info *rinfo = pci->debugfs->rasdes_info; + u32 val, counter, vc_num, err_group, type_mask; + int val_diff = 0; + char *kern_buf; + int ret; + + err_group = err_inj_list[pdata->idx].err_inj_group; + type_mask = err_inj_type_mask[err_group]; + + kern_buf = memdup_user_nul(buf, count); + if (IS_ERR(kern_buf)) + return PTR_ERR(kern_buf); + + if (err_group == 4) { + val = sscanf(kern_buf, "%u %d %u", &counter, &val_diff, &vc_num); + if ((val != 3) || (val_diff < -4095 || val_diff > 4095)) { + kfree(kern_buf); + return -EINVAL; + } + } else if (err_group == 1) { + val = sscanf(kern_buf, "%u %d", &counter, &val_diff); + if ((val != 2) || (val_diff < -4095 || val_diff > 4095)) { + kfree(kern_buf); + return -EINVAL; + } + } else { + ret = kstrtou32(kern_buf, 0, &counter); + if (ret) { + kfree(kern_buf); + return ret; + } + } + + val = dw_pcie_readl_dbi(pci, rinfo->ras_cap_offset + ERR_INJ0_OFF + (0x4 * err_group)); + val &= ~(type_mask | EINJ_COUNT); + val |= ((err_inj_list[pdata->idx].err_inj_type << EINJ_TYPE_SHIFT) & type_mask); + val |= FIELD_PREP(EINJ_COUNT, counter); + + if (err_group == 1 || err_group == 4) + FIELD_MODIFY(EINJ_VAL_DIFF, &val, val_diff); + if (err_group == 4) + FIELD_MODIFY(EINJ_VC_NUM, &val, vc_num); + + dw_pcie_writel_dbi(pci, rinfo->ras_cap_offset + ERR_INJ0_OFF + (0x4 * err_group), val); + dw_pcie_writel_dbi(pci, rinfo->ras_cap_offset + ERR_INJ_ENABLE_REG, (0x1 << err_group)); + + kfree(kern_buf); + return count; +} + +static void set_event_number(struct dwc_pcie_rasdes_priv *pdata, + struct dw_pcie *pci, struct dwc_pcie_rasdes_info *rinfo) +{ + u32 val; + + val = dw_pcie_readl_dbi(pci, rinfo->ras_cap_offset + RAS_DES_EVENT_COUNTER_CTRL_REG); + val &= ~EVENT_COUNTER_ENABLE; + FIELD_MODIFY(EVENT_COUNTER_GROUP_SELECT, &val, event_list[pdata->idx].group_no); + FIELD_MODIFY(EVENT_COUNTER_EVENT_SELECT, &val, event_list[pdata->idx].event_no); + dw_pcie_writel_dbi(pci, rinfo->ras_cap_offset + RAS_DES_EVENT_COUNTER_CTRL_REG, val); +} + +static ssize_t counter_enable_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct dwc_pcie_rasdes_priv *pdata = file->private_data; + struct dw_pcie *pci = pdata->pci; + struct dwc_pcie_rasdes_info *rinfo = pci->debugfs->rasdes_info; + char debugfs_buf[DWC_DEBUGFS_BUF_MAX]; + ssize_t pos; + u32 val; + + mutex_lock(&rinfo->reg_event_lock); + set_event_number(pdata, pci, rinfo); + val = dw_pcie_readl_dbi(pci, rinfo->ras_cap_offset + RAS_DES_EVENT_COUNTER_CTRL_REG); + mutex_unlock(&rinfo->reg_event_lock); + val = FIELD_GET(EVENT_COUNTER_STATUS, val); + if (val) + pos = scnprintf(debugfs_buf, DWC_DEBUGFS_BUF_MAX, "Counter Enabled\n"); + else + pos = scnprintf(debugfs_buf, DWC_DEBUGFS_BUF_MAX, "Counter Disabled\n"); + + return simple_read_from_buffer(buf, count, ppos, debugfs_buf, pos); +} + +static ssize_t counter_enable_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct dwc_pcie_rasdes_priv *pdata = file->private_data; + struct dw_pcie *pci = pdata->pci; + struct dwc_pcie_rasdes_info *rinfo = pci->debugfs->rasdes_info; + u32 val, enable; + int ret; + + ret = kstrtou32_from_user(buf, count, 0, &enable); + if (ret) + return ret; + + mutex_lock(&rinfo->reg_event_lock); + set_event_number(pdata, pci, rinfo); + val = dw_pcie_readl_dbi(pci, rinfo->ras_cap_offset + RAS_DES_EVENT_COUNTER_CTRL_REG); + if (enable) + val |= FIELD_PREP(EVENT_COUNTER_ENABLE, PER_EVENT_ON); + else + val |= FIELD_PREP(EVENT_COUNTER_ENABLE, PER_EVENT_OFF); + + dw_pcie_writel_dbi(pci, rinfo->ras_cap_offset + RAS_DES_EVENT_COUNTER_CTRL_REG, val); + + /* + * While enabling the counter, always read the status back to check if + * it is enabled or not. Return error if it is not enabled to let the + * users know that the counter is not supported on the platform. + */ + if (enable) { + val = dw_pcie_readl_dbi(pci, rinfo->ras_cap_offset + + RAS_DES_EVENT_COUNTER_CTRL_REG); + if (!FIELD_GET(EVENT_COUNTER_STATUS, val)) { + mutex_unlock(&rinfo->reg_event_lock); + return -EOPNOTSUPP; + } + } + + mutex_unlock(&rinfo->reg_event_lock); + + return count; +} + +static ssize_t counter_lane_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct dwc_pcie_rasdes_priv *pdata = file->private_data; + struct dw_pcie *pci = pdata->pci; + struct dwc_pcie_rasdes_info *rinfo = pci->debugfs->rasdes_info; + char debugfs_buf[DWC_DEBUGFS_BUF_MAX]; + ssize_t pos; + u32 val; + + mutex_lock(&rinfo->reg_event_lock); + set_event_number(pdata, pci, rinfo); + val = dw_pcie_readl_dbi(pci, rinfo->ras_cap_offset + RAS_DES_EVENT_COUNTER_CTRL_REG); + mutex_unlock(&rinfo->reg_event_lock); + val = FIELD_GET(EVENT_COUNTER_LANE_SELECT, val); + pos = scnprintf(debugfs_buf, DWC_DEBUGFS_BUF_MAX, "Lane: %d\n", val); + + return simple_read_from_buffer(buf, count, ppos, debugfs_buf, pos); +} + +static ssize_t counter_lane_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct dwc_pcie_rasdes_priv *pdata = file->private_data; + struct dw_pcie *pci = pdata->pci; + struct dwc_pcie_rasdes_info *rinfo = pci->debugfs->rasdes_info; + u32 val, lane; + int ret; + + ret = kstrtou32_from_user(buf, count, 0, &lane); + if (ret) + return ret; + + mutex_lock(&rinfo->reg_event_lock); + set_event_number(pdata, pci, rinfo); + val = dw_pcie_readl_dbi(pci, rinfo->ras_cap_offset + RAS_DES_EVENT_COUNTER_CTRL_REG); + FIELD_MODIFY(EVENT_COUNTER_LANE_SELECT, &val, lane); + dw_pcie_writel_dbi(pci, rinfo->ras_cap_offset + RAS_DES_EVENT_COUNTER_CTRL_REG, val); + mutex_unlock(&rinfo->reg_event_lock); + + return count; +} + +static ssize_t counter_value_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct dwc_pcie_rasdes_priv *pdata = file->private_data; + struct dw_pcie *pci = pdata->pci; + struct dwc_pcie_rasdes_info *rinfo = pci->debugfs->rasdes_info; + char debugfs_buf[DWC_DEBUGFS_BUF_MAX]; + ssize_t pos; + u32 val; + + mutex_lock(&rinfo->reg_event_lock); + set_event_number(pdata, pci, rinfo); + val = dw_pcie_readl_dbi(pci, rinfo->ras_cap_offset + RAS_DES_EVENT_COUNTER_DATA_REG); + mutex_unlock(&rinfo->reg_event_lock); + pos = scnprintf(debugfs_buf, DWC_DEBUGFS_BUF_MAX, "Counter value: %d\n", val); + + return simple_read_from_buffer(buf, count, ppos, debugfs_buf, pos); +} + +static int ltssm_status_show(struct seq_file *s, void *v) +{ + struct dw_pcie *pci = s->private; + enum dw_pcie_ltssm val; + + val = dw_pcie_get_ltssm(pci); + seq_printf(s, "%s (0x%02x)\n", dw_pcie_ltssm_status_string(val), val); + + return 0; +} + +#define dwc_debugfs_create(name) \ +debugfs_create_file(#name, 0644, rasdes_debug, pci, \ + &dbg_ ## name ## _fops) + +#define DWC_DEBUGFS_FOPS(name) \ +static const struct file_operations dbg_ ## name ## _fops = { \ + .open = simple_open, \ + .read = name ## _read, \ + .write = name ## _write \ +} + +DWC_DEBUGFS_FOPS(lane_detect); +DWC_DEBUGFS_FOPS(rx_valid); + +static const struct file_operations dwc_pcie_err_inj_ops = { + .open = simple_open, + .write = err_inj_write, +}; + +static const struct file_operations dwc_pcie_counter_enable_ops = { + .open = simple_open, + .read = counter_enable_read, + .write = counter_enable_write, +}; + +static const struct file_operations dwc_pcie_counter_lane_ops = { + .open = simple_open, + .read = counter_lane_read, + .write = counter_lane_write, +}; + +static const struct file_operations dwc_pcie_counter_value_ops = { + .open = simple_open, + .read = counter_value_read, +}; + +DEFINE_SHOW_ATTRIBUTE(ltssm_status); + +static void dwc_pcie_rasdes_debugfs_deinit(struct dw_pcie *pci) +{ + struct dwc_pcie_rasdes_info *rinfo = pci->debugfs->rasdes_info; + + if (!rinfo) + return; + + mutex_destroy(&rinfo->reg_event_lock); +} + +static int dwc_pcie_rasdes_debugfs_init(struct dw_pcie *pci, struct dentry *dir) +{ + struct dentry *rasdes_debug, *rasdes_err_inj; + struct dentry *rasdes_event_counter, *rasdes_events; + struct dwc_pcie_rasdes_info *rasdes_info; + struct dwc_pcie_rasdes_priv *priv_tmp; + struct device *dev = pci->dev; + int ras_cap, i, ret; + + /* + * If a given SoC has no RAS DES capability, the following call is + * bound to return an error, breaking some existing platforms. So, + * return 0 here, as this is not necessarily an error. + */ + ras_cap = dw_pcie_find_rasdes_capability(pci); + if (!ras_cap) { + dev_dbg(dev, "no RAS DES capability available\n"); + return 0; + } + + rasdes_info = devm_kzalloc(dev, sizeof(*rasdes_info), GFP_KERNEL); + if (!rasdes_info) + return -ENOMEM; + + /* Create subdirectories for Debug, Error Injection, Statistics. */ + rasdes_debug = debugfs_create_dir("rasdes_debug", dir); + rasdes_err_inj = debugfs_create_dir("rasdes_err_inj", dir); + rasdes_event_counter = debugfs_create_dir("rasdes_event_counter", dir); + + mutex_init(&rasdes_info->reg_event_lock); + rasdes_info->ras_cap_offset = ras_cap; + pci->debugfs->rasdes_info = rasdes_info; + + /* Create debugfs files for Debug subdirectory. */ + dwc_debugfs_create(lane_detect); + dwc_debugfs_create(rx_valid); + + /* Create debugfs files for Error Injection subdirectory. */ + for (i = 0; i < ARRAY_SIZE(err_inj_list); i++) { + priv_tmp = devm_kzalloc(dev, sizeof(*priv_tmp), GFP_KERNEL); + if (!priv_tmp) { + ret = -ENOMEM; + goto err_deinit; + } + + priv_tmp->idx = i; + priv_tmp->pci = pci; + debugfs_create_file(err_inj_list[i].name, 0200, rasdes_err_inj, priv_tmp, + &dwc_pcie_err_inj_ops); + } + + /* Create debugfs files for Statistical Counter subdirectory. */ + for (i = 0; i < ARRAY_SIZE(event_list); i++) { + priv_tmp = devm_kzalloc(dev, sizeof(*priv_tmp), GFP_KERNEL); + if (!priv_tmp) { + ret = -ENOMEM; + goto err_deinit; + } + + priv_tmp->idx = i; + priv_tmp->pci = pci; + rasdes_events = debugfs_create_dir(event_list[i].name, rasdes_event_counter); + if (event_list[i].group_no == 0 || event_list[i].group_no == 4) { + debugfs_create_file("lane_select", 0644, rasdes_events, + priv_tmp, &dwc_pcie_counter_lane_ops); + } + debugfs_create_file("counter_value", 0444, rasdes_events, priv_tmp, + &dwc_pcie_counter_value_ops); + debugfs_create_file("counter_enable", 0644, rasdes_events, priv_tmp, + &dwc_pcie_counter_enable_ops); + } + + return 0; + +err_deinit: + dwc_pcie_rasdes_debugfs_deinit(pci); + return ret; +} + +static void dwc_pcie_ltssm_debugfs_init(struct dw_pcie *pci, struct dentry *dir) +{ + debugfs_create_file("ltssm_status", 0444, dir, pci, + <ssm_status_fops); +} + +static int dw_pcie_ptm_check_capability(void *drvdata) +{ + struct dw_pcie *pci = drvdata; + + pci->ptm_vsec_offset = dw_pcie_find_ptm_capability(pci); + + return pci->ptm_vsec_offset; +} + +static int dw_pcie_ptm_context_update_write(void *drvdata, u8 mode) +{ + struct dw_pcie *pci = drvdata; + u32 val; + + if (mode == PCIE_PTM_CONTEXT_UPDATE_AUTO) { + val = dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_RES_REQ_CTRL); + val |= PTM_REQ_AUTO_UPDATE_ENABLED; + dw_pcie_writel_dbi(pci, pci->ptm_vsec_offset + PTM_RES_REQ_CTRL, val); + } else if (mode == PCIE_PTM_CONTEXT_UPDATE_MANUAL) { + val = dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_RES_REQ_CTRL); + val &= ~PTM_REQ_AUTO_UPDATE_ENABLED; + val |= PTM_REQ_START_UPDATE; + dw_pcie_writel_dbi(pci, pci->ptm_vsec_offset + PTM_RES_REQ_CTRL, val); + } else { + return -EINVAL; + } + + return 0; +} + +static int dw_pcie_ptm_context_update_read(void *drvdata, u8 *mode) +{ + struct dw_pcie *pci = drvdata; + u32 val; + + val = dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_RES_REQ_CTRL); + if (FIELD_GET(PTM_REQ_AUTO_UPDATE_ENABLED, val)) + *mode = PCIE_PTM_CONTEXT_UPDATE_AUTO; + else + /* + * PTM_REQ_START_UPDATE is a self clearing register bit. So if + * PTM_REQ_AUTO_UPDATE_ENABLED is not set, then it implies that + * manual update is used. + */ + *mode = PCIE_PTM_CONTEXT_UPDATE_MANUAL; + + return 0; +} + +static int dw_pcie_ptm_context_valid_write(void *drvdata, bool valid) +{ + struct dw_pcie *pci = drvdata; + u32 val; + + if (valid) { + val = dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_RES_REQ_CTRL); + val |= PTM_RES_CCONTEXT_VALID; + dw_pcie_writel_dbi(pci, pci->ptm_vsec_offset + PTM_RES_REQ_CTRL, val); + } else { + val = dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_RES_REQ_CTRL); + val &= ~PTM_RES_CCONTEXT_VALID; + dw_pcie_writel_dbi(pci, pci->ptm_vsec_offset + PTM_RES_REQ_CTRL, val); + } + + return 0; +} + +static int dw_pcie_ptm_context_valid_read(void *drvdata, bool *valid) +{ + struct dw_pcie *pci = drvdata; + u32 val; + + val = dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_RES_REQ_CTRL); + *valid = !!FIELD_GET(PTM_RES_CCONTEXT_VALID, val); + + return 0; +} + +static int dw_pcie_ptm_local_clock_read(void *drvdata, u64 *clock) +{ + struct dw_pcie *pci = drvdata; + u32 msb, lsb; + + do { + msb = dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_LOCAL_MSB); + lsb = dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_LOCAL_LSB); + } while (msb != dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_LOCAL_MSB)); + + *clock = ((u64) msb) << 32 | lsb; + + return 0; +} + +static int dw_pcie_ptm_master_clock_read(void *drvdata, u64 *clock) +{ + struct dw_pcie *pci = drvdata; + u32 msb, lsb; + + do { + msb = dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_MASTER_MSB); + lsb = dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_MASTER_LSB); + } while (msb != dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_MASTER_MSB)); + + *clock = ((u64) msb) << 32 | lsb; + + return 0; +} + +static int dw_pcie_ptm_t1_read(void *drvdata, u64 *clock) +{ + struct dw_pcie *pci = drvdata; + u32 msb, lsb; + + do { + msb = dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_T1_T2_MSB); + lsb = dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_T1_T2_LSB); + } while (msb != dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_T1_T2_MSB)); + + *clock = ((u64) msb) << 32 | lsb; + + return 0; +} + +static int dw_pcie_ptm_t2_read(void *drvdata, u64 *clock) +{ + struct dw_pcie *pci = drvdata; + u32 msb, lsb; + + do { + msb = dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_T1_T2_MSB); + lsb = dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_T1_T2_LSB); + } while (msb != dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_T1_T2_MSB)); + + *clock = ((u64) msb) << 32 | lsb; + + return 0; +} + +static int dw_pcie_ptm_t3_read(void *drvdata, u64 *clock) +{ + struct dw_pcie *pci = drvdata; + u32 msb, lsb; + + do { + msb = dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_T3_T4_MSB); + lsb = dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_T3_T4_LSB); + } while (msb != dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_T3_T4_MSB)); + + *clock = ((u64) msb) << 32 | lsb; + + return 0; +} + +static int dw_pcie_ptm_t4_read(void *drvdata, u64 *clock) +{ + struct dw_pcie *pci = drvdata; + u32 msb, lsb; + + do { + msb = dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_T3_T4_MSB); + lsb = dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_T3_T4_LSB); + } while (msb != dw_pcie_readl_dbi(pci, pci->ptm_vsec_offset + PTM_T3_T4_MSB)); + + *clock = ((u64) msb) << 32 | lsb; + + return 0; +} + +static bool dw_pcie_ptm_context_update_visible(void *drvdata) +{ + struct dw_pcie *pci = drvdata; + + return pci->mode == DW_PCIE_EP_TYPE; +} + +static bool dw_pcie_ptm_context_valid_visible(void *drvdata) +{ + struct dw_pcie *pci = drvdata; + + return pci->mode == DW_PCIE_RC_TYPE; +} + +static bool dw_pcie_ptm_local_clock_visible(void *drvdata) +{ + /* PTM local clock is always visible */ + return true; +} + +static bool dw_pcie_ptm_master_clock_visible(void *drvdata) +{ + struct dw_pcie *pci = drvdata; + + return pci->mode == DW_PCIE_EP_TYPE; +} + +static bool dw_pcie_ptm_t1_visible(void *drvdata) +{ + struct dw_pcie *pci = drvdata; + + return pci->mode == DW_PCIE_EP_TYPE; +} + +static bool dw_pcie_ptm_t2_visible(void *drvdata) +{ + struct dw_pcie *pci = drvdata; + + return pci->mode == DW_PCIE_RC_TYPE; +} + +static bool dw_pcie_ptm_t3_visible(void *drvdata) +{ + struct dw_pcie *pci = drvdata; + + return pci->mode == DW_PCIE_RC_TYPE; +} + +static bool dw_pcie_ptm_t4_visible(void *drvdata) +{ + struct dw_pcie *pci = drvdata; + + return pci->mode == DW_PCIE_EP_TYPE; +} + +static const struct pcie_ptm_ops dw_pcie_ptm_ops = { + .check_capability = dw_pcie_ptm_check_capability, + .context_update_write = dw_pcie_ptm_context_update_write, + .context_update_read = dw_pcie_ptm_context_update_read, + .context_valid_write = dw_pcie_ptm_context_valid_write, + .context_valid_read = dw_pcie_ptm_context_valid_read, + .local_clock_read = dw_pcie_ptm_local_clock_read, + .master_clock_read = dw_pcie_ptm_master_clock_read, + .t1_read = dw_pcie_ptm_t1_read, + .t2_read = dw_pcie_ptm_t2_read, + .t3_read = dw_pcie_ptm_t3_read, + .t4_read = dw_pcie_ptm_t4_read, + .context_update_visible = dw_pcie_ptm_context_update_visible, + .context_valid_visible = dw_pcie_ptm_context_valid_visible, + .local_clock_visible = dw_pcie_ptm_local_clock_visible, + .master_clock_visible = dw_pcie_ptm_master_clock_visible, + .t1_visible = dw_pcie_ptm_t1_visible, + .t2_visible = dw_pcie_ptm_t2_visible, + .t3_visible = dw_pcie_ptm_t3_visible, + .t4_visible = dw_pcie_ptm_t4_visible, +}; + +void dwc_pcie_debugfs_deinit(struct dw_pcie *pci) +{ + if (!pci->debugfs) + return; + + pcie_ptm_destroy_debugfs(pci->ptm_debugfs); + dwc_pcie_rasdes_debugfs_deinit(pci); + debugfs_remove_recursive(pci->debugfs->debug_dir); +} + +void dwc_pcie_debugfs_init(struct dw_pcie *pci, enum dw_pcie_device_mode mode) +{ + char dirname[DWC_DEBUGFS_BUF_MAX]; + struct device *dev = pci->dev; + struct debugfs_info *debugfs; + struct dentry *dir; + int err; + + /* Create main directory for each platform driver. */ + snprintf(dirname, DWC_DEBUGFS_BUF_MAX, "dwc_pcie_%s", dev_name(dev)); + dir = debugfs_create_dir(dirname, NULL); + debugfs = devm_kzalloc(dev, sizeof(*debugfs), GFP_KERNEL); + if (!debugfs) + return; + + debugfs->debug_dir = dir; + pci->debugfs = debugfs; + err = dwc_pcie_rasdes_debugfs_init(pci, dir); + if (err) + dev_err(dev, "failed to initialize RAS DES debugfs, err=%d\n", + err); + + dwc_pcie_ltssm_debugfs_init(pci, dir); + + pci->mode = mode; + pci->ptm_debugfs = pcie_ptm_create_debugfs(pci->dev, pci, + &dw_pcie_ptm_ops); +} diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c new file mode 100644 index 000000000..de8ee3db4 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-designware-ep.c @@ -0,0 +1,1574 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Synopsys DesignWare PCIe Endpoint controller driver + * + * Copyright (C) 2017 Texas Instruments + * Author: Kishon Vijay Abraham I <kishon@ti.com> + */ + +#include <linux/align.h> +#include <linux/bitfield.h> +#include <linux/of.h> +#include <linux/overflow.h> +#include <linux/platform_device.h> + +#include "pcie-designware.h" +#include <linux/pci-epc.h> +#include <linux/pci-epf.h> + +/** + * dw_pcie_ep_get_func_from_ep - Get the struct dw_pcie_ep_func corresponding to + * the endpoint function + * @ep: DWC EP device + * @func_no: Function number of the endpoint device + * + * Return: struct dw_pcie_ep_func if success, NULL otherwise. + */ +struct dw_pcie_ep_func * +dw_pcie_ep_get_func_from_ep(struct dw_pcie_ep *ep, u8 func_no) +{ + struct dw_pcie_ep_func *ep_func; + + list_for_each_entry(ep_func, &ep->func_list, list) { + if (ep_func->func_no == func_no) + return ep_func; + } + + return NULL; +} + +static void __dw_pcie_ep_reset_bar(struct dw_pcie *pci, u8 func_no, + enum pci_barno bar, int flags) +{ + struct dw_pcie_ep *ep = &pci->ep; + u32 reg; + + reg = PCI_BASE_ADDRESS_0 + (4 * bar); + dw_pcie_dbi_ro_wr_en(pci); + dw_pcie_ep_writel_dbi2(ep, func_no, reg, 0x0); + dw_pcie_ep_writel_dbi(ep, func_no, reg, 0x0); + if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) { + dw_pcie_ep_writel_dbi2(ep, func_no, reg + 4, 0x0); + dw_pcie_ep_writel_dbi(ep, func_no, reg + 4, 0x0); + } + dw_pcie_dbi_ro_wr_dis(pci); +} + +/** + * dw_pcie_ep_reset_bar - Reset endpoint BAR + * @pci: DWC PCI device + * @bar: BAR number of the endpoint + */ +void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) +{ + u8 func_no, funcs; + + funcs = pci->ep.epc->max_functions; + + for (func_no = 0; func_no < funcs; func_no++) + __dw_pcie_ep_reset_bar(pci, func_no, bar, 0); +} +EXPORT_SYMBOL_GPL(dw_pcie_ep_reset_bar); + +static u8 dw_pcie_ep_find_capability(struct dw_pcie_ep *ep, u8 func_no, u8 cap) +{ + return PCI_FIND_NEXT_CAP(dw_pcie_ep_read_cfg, PCI_CAPABILITY_LIST, + cap, NULL, ep, func_no); +} + +static u16 dw_pcie_ep_find_ext_capability(struct dw_pcie_ep *ep, + u8 func_no, u8 cap) +{ + return PCI_FIND_NEXT_EXT_CAP(dw_pcie_ep_read_cfg, 0, + cap, NULL, ep, func_no); +} + +static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + struct pci_epf_header *hdr) +{ + struct dw_pcie_ep *ep = epc_get_drvdata(epc); + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + dw_pcie_dbi_ro_wr_en(pci); + dw_pcie_ep_writew_dbi(ep, func_no, PCI_VENDOR_ID, hdr->vendorid); + dw_pcie_ep_writew_dbi(ep, func_no, PCI_DEVICE_ID, hdr->deviceid); + dw_pcie_ep_writeb_dbi(ep, func_no, PCI_REVISION_ID, hdr->revid); + dw_pcie_ep_writeb_dbi(ep, func_no, PCI_CLASS_PROG, hdr->progif_code); + dw_pcie_ep_writew_dbi(ep, func_no, PCI_CLASS_DEVICE, + hdr->subclass_code | hdr->baseclass_code << 8); + dw_pcie_ep_writeb_dbi(ep, func_no, PCI_CACHE_LINE_SIZE, + hdr->cache_line_size); + dw_pcie_ep_writew_dbi(ep, func_no, PCI_SUBSYSTEM_VENDOR_ID, + hdr->subsys_vendor_id); + dw_pcie_ep_writew_dbi(ep, func_no, PCI_SUBSYSTEM_ID, hdr->subsys_id); + dw_pcie_ep_writeb_dbi(ep, func_no, PCI_INTERRUPT_PIN, + hdr->interrupt_pin); + dw_pcie_dbi_ro_wr_dis(pci); + + return 0; +} + +/* BAR Match Mode inbound iATU mapping */ +static int dw_pcie_ep_ib_atu_bar(struct dw_pcie_ep *ep, u8 func_no, int type, + dma_addr_t parent_bus_addr, enum pci_barno bar, + size_t size) +{ + int ret; + u32 free_win; + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + + if (!ep_func) + return -EINVAL; + + if (!ep_func->bar_to_atu[bar]) + free_win = find_first_zero_bit(ep->ib_window_map, pci->num_ib_windows); + else + free_win = ep_func->bar_to_atu[bar] - 1; + + if (free_win >= pci->num_ib_windows) { + dev_err(pci->dev, "No free inbound window\n"); + return -EINVAL; + } + + ret = dw_pcie_prog_ep_inbound_atu(pci, func_no, free_win, type, + parent_bus_addr, bar, size); + if (ret < 0) { + dev_err(pci->dev, "Failed to program IB window\n"); + return ret; + } + + /* + * Always increment free_win before assignment, since value 0 is used to identify + * unallocated mapping. + */ + ep_func->bar_to_atu[bar] = free_win + 1; + set_bit(free_win, ep->ib_window_map); + + return 0; +} + +static void dw_pcie_ep_clear_ib_maps(struct dw_pcie_ep *ep, u8 func_no, enum pci_barno bar) +{ + struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct device *dev = pci->dev; + unsigned int i, num; + u32 atu_index; + u32 *indexes; + + if (!ep_func) + return; + + /* Tear down the BAR Match Mode mapping, if any. */ + if (ep_func->bar_to_atu[bar]) { + atu_index = ep_func->bar_to_atu[bar] - 1; + dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, atu_index); + clear_bit(atu_index, ep->ib_window_map); + ep_func->bar_to_atu[bar] = 0; + return; + } + + /* Tear down all Address Match Mode mappings, if any. */ + indexes = ep_func->ib_atu_indexes[bar]; + num = ep_func->num_ib_atu_indexes[bar]; + ep_func->ib_atu_indexes[bar] = NULL; + ep_func->num_ib_atu_indexes[bar] = 0; + if (!indexes) + return; + for (i = 0; i < num; i++) { + dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, indexes[i]); + clear_bit(indexes[i], ep->ib_window_map); + } + devm_kfree(dev, indexes); +} + +static u64 dw_pcie_ep_read_bar_assigned(struct dw_pcie_ep *ep, u8 func_no, + enum pci_barno bar, int flags) +{ + u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar); + u32 lo, hi; + u64 addr; + + lo = dw_pcie_ep_readl_dbi(ep, func_no, reg); + + if (flags & PCI_BASE_ADDRESS_SPACE) + return lo & PCI_BASE_ADDRESS_IO_MASK; + + addr = lo & PCI_BASE_ADDRESS_MEM_MASK; + if (!(flags & PCI_BASE_ADDRESS_MEM_TYPE_64)) + return addr; + + hi = dw_pcie_ep_readl_dbi(ep, func_no, reg + 4); + return addr | ((u64)hi << 32); +} + +static int dw_pcie_ep_validate_submap(struct dw_pcie_ep *ep, + const struct pci_epf_bar_submap *submap, + unsigned int num_submap, size_t bar_size) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + u32 align = pci->region_align; + size_t off = 0; + unsigned int i; + size_t size; + + if (!align || !IS_ALIGNED(bar_size, align)) + return -EINVAL; + + /* + * The submap array order defines the BAR layout (submap[0] starts + * at offset 0 and each entry immediately follows the previous + * one). Here, validate that it forms a strict, gapless + * decomposition of the BAR: + * - each entry has a non-zero size + * - sizes, implicit offsets and phys_addr are aligned to + * pci->region_align + * - each entry lies within the BAR range + * - the entries exactly cover the whole BAR + * + * Note: dw_pcie_prog_inbound_atu() also checks alignment for the + * PCI address and the target phys_addr, but validating up-front + * avoids partially programming iATU windows in vain. + */ + for (i = 0; i < num_submap; i++) { + size = submap[i].size; + + if (!size) + return -EINVAL; + + if (!IS_ALIGNED(size, align) || !IS_ALIGNED(off, align)) + return -EINVAL; + + if (!IS_ALIGNED(submap[i].phys_addr, align)) + return -EINVAL; + + if (off > bar_size || size > bar_size - off) + return -EINVAL; + + off += size; + } + if (off != bar_size) + return -EINVAL; + + return 0; +} + +/* Address Match Mode inbound iATU mapping */ +static int dw_pcie_ep_ib_atu_addr(struct dw_pcie_ep *ep, u8 func_no, int type, + const struct pci_epf_bar *epf_bar) +{ + struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + const struct pci_epf_bar_submap *submap = epf_bar->submap; + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + enum pci_barno bar = epf_bar->barno; + struct device *dev = pci->dev; + u64 pci_addr, parent_bus_addr; + u64 size, base, off = 0; + int free_win, ret; + unsigned int i; + u32 *indexes; + + if (!ep_func || !epf_bar->num_submap || !submap || !epf_bar->size) + return -EINVAL; + + ret = dw_pcie_ep_validate_submap(ep, submap, epf_bar->num_submap, + epf_bar->size); + if (ret) + return ret; + + base = dw_pcie_ep_read_bar_assigned(ep, func_no, bar, epf_bar->flags); + if (!base) { + dev_err(dev, + "BAR%u not assigned, cannot set up sub-range mappings\n", + bar); + return -EINVAL; + } + + indexes = devm_kcalloc(dev, epf_bar->num_submap, sizeof(*indexes), + GFP_KERNEL); + if (!indexes) + return -ENOMEM; + + ep_func->ib_atu_indexes[bar] = indexes; + ep_func->num_ib_atu_indexes[bar] = 0; + + for (i = 0; i < epf_bar->num_submap; i++) { + size = submap[i].size; + parent_bus_addr = submap[i].phys_addr; + + if (off > (~0ULL) - base) { + ret = -EINVAL; + goto err; + } + + pci_addr = base + off; + off += size; + + free_win = find_first_zero_bit(ep->ib_window_map, + pci->num_ib_windows); + if (free_win >= pci->num_ib_windows) { + ret = -ENOSPC; + goto err; + } + + ret = dw_pcie_prog_inbound_atu(pci, free_win, type, + parent_bus_addr, pci_addr, size); + if (ret) + goto err; + + set_bit(free_win, ep->ib_window_map); + indexes[i] = free_win; + ep_func->num_ib_atu_indexes[bar] = i + 1; + } + return 0; +err: + dw_pcie_ep_clear_ib_maps(ep, func_no, bar); + return ret; +} + +static int dw_pcie_ep_outbound_atu(struct dw_pcie_ep *ep, + struct dw_pcie_ob_atu_cfg *atu) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + u32 free_win; + int ret; + + free_win = find_first_zero_bit(ep->ob_window_map, pci->num_ob_windows); + if (free_win >= pci->num_ob_windows) { + dev_err(pci->dev, "No free outbound window\n"); + return -EINVAL; + } + + atu->index = free_win; + ret = dw_pcie_prog_outbound_atu(pci, atu); + if (ret) + return ret; + + set_bit(free_win, ep->ob_window_map); + ep->outbound_addr[free_win] = atu->parent_bus_addr; + + return 0; +} + +static void dw_pcie_ep_clear_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + struct pci_epf_bar *epf_bar) +{ + struct dw_pcie_ep *ep = epc_get_drvdata(epc); + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + enum pci_barno bar = epf_bar->barno; + struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + + if (!ep_func || !ep_func->epf_bar[bar]) + return; + + __dw_pcie_ep_reset_bar(pci, func_no, bar, epf_bar->flags); + + dw_pcie_ep_clear_ib_maps(ep, func_no, bar); + + ep_func->epf_bar[bar] = NULL; +} + +static unsigned int dw_pcie_ep_get_rebar_offset(struct dw_pcie_ep *ep, u8 func_no, + enum pci_barno bar) +{ + u32 reg, bar_index; + unsigned int offset, nbars; + int i; + + offset = dw_pcie_ep_find_ext_capability(ep, func_no, PCI_EXT_CAP_ID_REBAR); + if (!offset) + return offset; + + reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); + nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, reg); + + for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL) { + reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); + bar_index = FIELD_GET(PCI_REBAR_CTRL_BAR_IDX, reg); + if (bar_index == bar) + return offset; + } + + return 0; +} + +static int dw_pcie_ep_set_bar_resizable(struct dw_pcie_ep *ep, u8 func_no, + struct pci_epf_bar *epf_bar) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + enum pci_barno bar = epf_bar->barno; + size_t size = epf_bar->size; + int flags = epf_bar->flags; + u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar); + unsigned int rebar_offset; + u32 rebar_cap, rebar_ctrl; + int ret; + + rebar_offset = dw_pcie_ep_get_rebar_offset(ep, func_no, bar); + if (!rebar_offset) + return -EINVAL; + + ret = pci_epc_bar_size_to_rebar_cap(size, &rebar_cap); + if (ret) + return ret; + + dw_pcie_dbi_ro_wr_en(pci); + + /* + * A BAR mask should not be written for a resizable BAR. The BAR mask + * is automatically derived by the controller every time the "selected + * size" bits are updated, see "Figure 3-26 Resizable BAR Example for + * 32-bit Memory BAR0" in DWC EP databook 5.96a. We simply need to write + * BIT(0) to set the BAR enable bit. + */ + dw_pcie_ep_writel_dbi2(ep, func_no, reg, BIT(0)); + dw_pcie_ep_writel_dbi(ep, func_no, reg, flags); + + if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) { + dw_pcie_ep_writel_dbi2(ep, func_no, reg + 4, 0); + dw_pcie_ep_writel_dbi(ep, func_no, reg + 4, 0); + } + + /* + * Bits 31:0 in PCI_REBAR_CAP define "supported sizes" bits for sizes + * 1 MB to 128 TB. Bits 31:16 in PCI_REBAR_CTRL define "supported sizes" + * bits for sizes 256 TB to 8 EB. Disallow sizes 256 TB to 8 EB. + */ + rebar_ctrl = dw_pcie_ep_readl_dbi(ep, func_no, rebar_offset + PCI_REBAR_CTRL); + rebar_ctrl &= ~GENMASK(31, 16); + dw_pcie_ep_writel_dbi(ep, func_no, rebar_offset + PCI_REBAR_CTRL, rebar_ctrl); + + /* + * The "selected size" (bits 13:8) in PCI_REBAR_CTRL are automatically + * updated when writing PCI_REBAR_CAP, see "Figure 3-26 Resizable BAR + * Example for 32-bit Memory BAR0" in DWC EP databook 5.96a. + */ + dw_pcie_ep_writel_dbi(ep, func_no, rebar_offset + PCI_REBAR_CAP, rebar_cap); + + dw_pcie_dbi_ro_wr_dis(pci); + + return 0; +} + +static int dw_pcie_ep_set_bar_programmable(struct dw_pcie_ep *ep, u8 func_no, + struct pci_epf_bar *epf_bar) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + enum pci_barno bar = epf_bar->barno; + size_t size = epf_bar->size; + int flags = epf_bar->flags; + u32 reg = PCI_BASE_ADDRESS_0 + (4 * bar); + + dw_pcie_dbi_ro_wr_en(pci); + + dw_pcie_ep_writel_dbi2(ep, func_no, reg, lower_32_bits(size - 1)); + dw_pcie_ep_writel_dbi(ep, func_no, reg, flags); + + if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) { + dw_pcie_ep_writel_dbi2(ep, func_no, reg + 4, upper_32_bits(size - 1)); + dw_pcie_ep_writel_dbi(ep, func_no, reg + 4, 0); + } + + dw_pcie_dbi_ro_wr_dis(pci); + + return 0; +} + +static enum pci_epc_bar_type dw_pcie_ep_get_bar_type(struct dw_pcie_ep *ep, + enum pci_barno bar) +{ + const struct pci_epc_features *epc_features; + + if (!ep->ops->get_features) + return BAR_PROGRAMMABLE; + + epc_features = ep->ops->get_features(ep); + + return epc_features->bar[bar].type; +} + +static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + struct pci_epf_bar *epf_bar) +{ + struct dw_pcie_ep *ep = epc_get_drvdata(epc); + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + enum pci_barno bar = epf_bar->barno; + size_t size = epf_bar->size; + enum pci_epc_bar_type bar_type; + int flags = epf_bar->flags; + int ret, type; + + if (!ep_func) + return -EINVAL; + + /* + * DWC does not allow BAR pairs to overlap, e.g. you cannot combine BARs + * 1 and 2 to form a 64-bit BAR. + */ + if ((flags & PCI_BASE_ADDRESS_MEM_TYPE_64) && (bar & 1)) + return -EINVAL; + + /* + * Certain EPF drivers dynamically change the physical address of a BAR + * (i.e. they call set_bar() twice, without ever calling clear_bar(), as + * calling clear_bar() would clear the BAR's PCI address assigned by the + * host). + */ + if (ep_func->epf_bar[bar]) { + /* + * We can only dynamically change a BAR if the new BAR size and + * BAR flags do not differ from the existing configuration. + * + * Note: this safety check only works when the caller uses + * a new struct pci_epf_bar in the second set_bar() call. + * If the same instance is updated in place and passed in, + * we cannot reliably detect invalid barno/size/flags + * changes here. + */ + if (ep_func->epf_bar[bar]->barno != bar || + ep_func->epf_bar[bar]->size != size || + ep_func->epf_bar[bar]->flags != flags) + return -EINVAL; + + /* + * When dynamically changing a BAR, tear down any existing + * mappings before re-programming. This is redundant when + * both the old and new mappings are BAR Match Mode, but + * required to handle in-place updates and match-mode + * changes reliably. + */ + dw_pcie_ep_clear_ib_maps(ep, func_no, bar); + + /* + * When dynamically changing a BAR, skip writing the BAR reg, as + * that would clear the BAR's PCI address assigned by the host. + */ + goto config_atu; + } else { + /* + * Subrange mapping is an update-only operation. The BAR + * must have been configured once without submaps so that + * subsequent set_bar() calls can update inbound mappings + * without touching the BAR register (and clobbering the + * host-assigned address). + */ + if (epf_bar->num_submap) + return -EINVAL; + } + + bar_type = dw_pcie_ep_get_bar_type(ep, bar); + switch (bar_type) { + case BAR_FIXED: + /* + * There is no need to write a BAR mask for a fixed BAR (except + * to write 1 to the LSB of the BAR mask register, to enable the + * BAR). Write the BAR mask regardless. (The fixed bits in the + * BAR mask register will be read-only anyway.) + */ + fallthrough; + case BAR_PROGRAMMABLE: + ret = dw_pcie_ep_set_bar_programmable(ep, func_no, epf_bar); + break; + case BAR_RESIZABLE: + ret = dw_pcie_ep_set_bar_resizable(ep, func_no, epf_bar); + break; + default: + ret = -EINVAL; + dev_err(pci->dev, "Invalid BAR type\n"); + break; + } + + if (ret) + return ret; + +config_atu: + if (!(flags & PCI_BASE_ADDRESS_SPACE)) + type = PCIE_TLP_TYPE_MEM_RDWR; + else + type = PCIE_TLP_TYPE_IO_RDWR; + + if (epf_bar->num_submap) + ret = dw_pcie_ep_ib_atu_addr(ep, func_no, type, epf_bar); + else + ret = dw_pcie_ep_ib_atu_bar(ep, func_no, type, + epf_bar->phys_addr, bar, size); + + if (ret) + return ret; + + ep_func->epf_bar[bar] = epf_bar; + + return 0; +} + +static int dw_pcie_find_index(struct dw_pcie_ep *ep, phys_addr_t addr, + u32 *atu_index) +{ + u32 index; + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + for_each_set_bit(index, ep->ob_window_map, pci->num_ob_windows) { + if (ep->outbound_addr[index] != addr) + continue; + *atu_index = index; + return 0; + } + + return -EINVAL; +} + +static u64 dw_pcie_ep_align_addr(struct pci_epc *epc, u64 pci_addr, + size_t *pci_size, size_t *offset) +{ + struct dw_pcie_ep *ep = epc_get_drvdata(epc); + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + u64 mask = pci->region_align - 1; + size_t ofst = pci_addr & mask; + + *pci_size = ALIGN(ofst + *pci_size, epc->mem->window.page_size); + *offset = ofst; + + return pci_addr & ~mask; +} + +static void dw_pcie_ep_unmap_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + phys_addr_t addr) +{ + int ret; + u32 atu_index; + struct dw_pcie_ep *ep = epc_get_drvdata(epc); + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + ret = dw_pcie_find_index(ep, addr - pci->parent_bus_offset, + &atu_index); + if (ret < 0) + return; + + ep->outbound_addr[atu_index] = 0; + dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_OB, atu_index); + clear_bit(atu_index, ep->ob_window_map); +} + +static int dw_pcie_ep_map_addr(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + phys_addr_t addr, u64 pci_addr, size_t size) +{ + int ret; + struct dw_pcie_ep *ep = epc_get_drvdata(epc); + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct dw_pcie_ob_atu_cfg atu = { 0 }; + + atu.func_no = func_no; + atu.type = PCIE_TLP_TYPE_MEM_RDWR; + atu.parent_bus_addr = addr - pci->parent_bus_offset; + atu.pci_addr = pci_addr; + atu.size = size; + ret = dw_pcie_ep_outbound_atu(ep, &atu); + if (ret) { + dev_err(pci->dev, "Failed to enable address\n"); + return ret; + } + + return 0; +} + +static int dw_pcie_ep_get_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no) +{ + struct dw_pcie_ep *ep = epc_get_drvdata(epc); + struct dw_pcie_ep_func *ep_func; + u32 val, reg; + + ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + if (!ep_func || !ep_func->msi_cap) + return -EINVAL; + + reg = ep_func->msi_cap + PCI_MSI_FLAGS; + val = dw_pcie_ep_readw_dbi(ep, func_no, reg); + if (!(val & PCI_MSI_FLAGS_ENABLE)) + return -EINVAL; + + val = FIELD_GET(PCI_MSI_FLAGS_QSIZE, val); + + return 1 << val; +} + +static int dw_pcie_ep_set_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + u8 nr_irqs) +{ + struct dw_pcie_ep *ep = epc_get_drvdata(epc); + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct dw_pcie_ep_func *ep_func; + u8 mmc = order_base_2(nr_irqs); + u32 val, reg; + + ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + if (!ep_func || !ep_func->msi_cap) + return -EINVAL; + + reg = ep_func->msi_cap + PCI_MSI_FLAGS; + val = dw_pcie_ep_readw_dbi(ep, func_no, reg); + FIELD_MODIFY(PCI_MSI_FLAGS_QMASK, &val, mmc); + dw_pcie_dbi_ro_wr_en(pci); + dw_pcie_ep_writew_dbi(ep, func_no, reg, val); + dw_pcie_dbi_ro_wr_dis(pci); + + return 0; +} + +static int dw_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no) +{ + struct dw_pcie_ep *ep = epc_get_drvdata(epc); + struct dw_pcie_ep_func *ep_func; + u32 val, reg; + + ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + if (!ep_func || !ep_func->msix_cap) + return -EINVAL; + + reg = ep_func->msix_cap + PCI_MSIX_FLAGS; + val = dw_pcie_ep_readw_dbi(ep, func_no, reg); + if (!(val & PCI_MSIX_FLAGS_ENABLE)) + return -EINVAL; + + val &= PCI_MSIX_FLAGS_QSIZE; + + return val + 1; +} + +static int dw_pcie_ep_set_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + u16 nr_irqs, enum pci_barno bir, u32 offset) +{ + struct dw_pcie_ep *ep = epc_get_drvdata(epc); + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct dw_pcie_ep_func *ep_func; + u32 val, reg; + + ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + if (!ep_func || !ep_func->msix_cap) + return -EINVAL; + + dw_pcie_dbi_ro_wr_en(pci); + + reg = ep_func->msix_cap + PCI_MSIX_FLAGS; + val = dw_pcie_ep_readw_dbi(ep, func_no, reg); + val &= ~PCI_MSIX_FLAGS_QSIZE; + val |= nr_irqs - 1; /* encoded as N-1 */ + dw_pcie_ep_writew_dbi(ep, func_no, reg, val); + + reg = ep_func->msix_cap + PCI_MSIX_TABLE; + val = offset | bir; + dw_pcie_ep_writel_dbi(ep, func_no, reg, val); + + reg = ep_func->msix_cap + PCI_MSIX_PBA; + val = (offset + (nr_irqs * PCI_MSIX_ENTRY_SIZE)) | bir; + dw_pcie_ep_writel_dbi(ep, func_no, reg, val); + + dw_pcie_dbi_ro_wr_dis(pci); + + return 0; +} + +static int dw_pcie_ep_raise_irq(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + unsigned int type, u16 interrupt_num) +{ + struct dw_pcie_ep *ep = epc_get_drvdata(epc); + + if (!ep->ops->raise_irq) + return -EINVAL; + + return ep->ops->raise_irq(ep, func_no, type, interrupt_num); +} + +static void dw_pcie_ep_stop(struct pci_epc *epc) +{ + struct dw_pcie_ep *ep = epc_get_drvdata(epc); + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + /* + * Tear down the dedicated outbound window used for MSI + * generation. This avoids leaking an iATU window across + * endpoint stop/start cycles. + */ + if (ep->msi_iatu_mapped) { + dw_pcie_ep_unmap_addr(epc, 0, 0, ep->msi_mem_phys); + ep->msi_iatu_mapped = false; + } + + dw_pcie_stop_link(pci); +} + +static int dw_pcie_ep_start(struct pci_epc *epc) +{ + struct dw_pcie_ep *ep = epc_get_drvdata(epc); + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + return dw_pcie_start_link(pci); +} + +static const struct pci_epc_features* +dw_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no) +{ + struct dw_pcie_ep *ep = epc_get_drvdata(epc); + + if (!ep->ops->get_features) + return NULL; + + return ep->ops->get_features(ep); +} + +static const struct pci_epc_bar_rsvd_region * +dw_pcie_ep_find_bar_rsvd_region(struct dw_pcie_ep *ep, + enum pci_epc_bar_rsvd_region_type type, + enum pci_barno *bar, + resource_size_t *bar_offset) +{ + const struct pci_epc_features *features; + const struct pci_epc_bar_desc *bar_desc; + const struct pci_epc_bar_rsvd_region *r; + int i, j; + + if (!ep->ops->get_features) + return NULL; + + features = ep->ops->get_features(ep); + if (!features) + return NULL; + + for (i = BAR_0; i <= BAR_5; i++) { + bar_desc = &features->bar[i]; + + if (!bar_desc->nr_rsvd_regions || !bar_desc->rsvd_regions) + continue; + + for (j = 0; j < bar_desc->nr_rsvd_regions; j++) { + r = &bar_desc->rsvd_regions[j]; + + if (r->type != type) + continue; + + if (bar) + *bar = i; + if (bar_offset) + *bar_offset = r->offset; + return r; + } + } + + return NULL; +} + +static int +dw_pcie_ep_get_aux_resources_count(struct pci_epc *epc, u8 func_no, + u8 vfunc_no) +{ + struct dw_pcie_ep *ep = epc_get_drvdata(epc); + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct dw_edma_chip *edma = &pci->edma; + + if (!pci->edma_reg_size) + return 0; + + if (edma->db_offset == ~0) + return 0; + + return 1; +} + +static int +dw_pcie_ep_get_aux_resources(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + struct pci_epc_aux_resource *resources, + int num_resources) +{ + struct dw_pcie_ep *ep = epc_get_drvdata(epc); + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + const struct pci_epc_bar_rsvd_region *rsvd; + struct dw_edma_chip *edma = &pci->edma; + enum pci_barno dma_ctrl_bar = NO_BAR; + resource_size_t db_offset = edma->db_offset; + resource_size_t dma_ctrl_bar_offset = 0; + resource_size_t dma_reg_size; + int count; + + count = dw_pcie_ep_get_aux_resources_count(epc, func_no, vfunc_no); + if (count < 0) + return count; + + if (num_resources < count) + return -ENOSPC; + + if (!count) + return 0; + + dma_reg_size = pci->edma_reg_size; + + rsvd = dw_pcie_ep_find_bar_rsvd_region(ep, + PCI_EPC_BAR_RSVD_DMA_CTRL_MMIO, + &dma_ctrl_bar, + &dma_ctrl_bar_offset); + if (rsvd && rsvd->size < dma_reg_size) + dma_reg_size = rsvd->size; + + /* + * For interrupt-emulation doorbells, report a standalone resource + * instead of bundling it into the DMA controller MMIO resource. + */ + if (range_end_overflows_t(resource_size_t, db_offset, + sizeof(u32), dma_reg_size)) + return -EINVAL; + + resources[0] = (struct pci_epc_aux_resource) { + .type = PCI_EPC_AUX_DOORBELL_MMIO, + .phys_addr = pci->edma_reg_phys + db_offset, + .size = sizeof(u32), + .bar = dma_ctrl_bar, + .bar_offset = dma_ctrl_bar != NO_BAR ? + dma_ctrl_bar_offset + db_offset : 0, + .u.db_mmio = { + .irq = edma->db_irq, + .data = 0, /* write 0 to assert */ + }, + }; + + return 0; +} + +static const struct pci_epc_ops epc_ops = { + .write_header = dw_pcie_ep_write_header, + .set_bar = dw_pcie_ep_set_bar, + .clear_bar = dw_pcie_ep_clear_bar, + .align_addr = dw_pcie_ep_align_addr, + .map_addr = dw_pcie_ep_map_addr, + .unmap_addr = dw_pcie_ep_unmap_addr, + .set_msi = dw_pcie_ep_set_msi, + .get_msi = dw_pcie_ep_get_msi, + .set_msix = dw_pcie_ep_set_msix, + .get_msix = dw_pcie_ep_get_msix, + .raise_irq = dw_pcie_ep_raise_irq, + .start = dw_pcie_ep_start, + .stop = dw_pcie_ep_stop, + .get_features = dw_pcie_ep_get_features, + .get_aux_resources_count = dw_pcie_ep_get_aux_resources_count, + .get_aux_resources = dw_pcie_ep_get_aux_resources, +}; + +/** + * dw_pcie_ep_raise_intx_irq - Raise INTx IRQ to the host + * @ep: DWC EP device + * @func_no: Function number of the endpoint + * + * Return: 0 if success, errno otherwise. + */ +int dw_pcie_ep_raise_intx_irq(struct dw_pcie_ep *ep, u8 func_no) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct device *dev = pci->dev; + + dev_err(dev, "EP cannot raise INTX IRQs\n"); + + return -EINVAL; +} +EXPORT_SYMBOL_GPL(dw_pcie_ep_raise_intx_irq); + +/** + * dw_pcie_ep_raise_msi_irq - Raise MSI IRQ to the host + * @ep: DWC EP device + * @func_no: Function number of the endpoint + * @interrupt_num: Interrupt number to be raised + * + * Return: 0 if success, errno otherwise. + */ +int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, + u8 interrupt_num) +{ + u32 msg_addr_lower, msg_addr_upper, reg; + struct dw_pcie_ep_func *ep_func; + struct pci_epc *epc = ep->epc; + size_t map_size = sizeof(u32); + size_t offset; + u16 msg_ctrl, msg_data; + bool has_upper; + u64 msg_addr; + int ret; + + ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + if (!ep_func || !ep_func->msi_cap) + return -EINVAL; + + /* Raise MSI per the PCI Local Bus Specification Revision 3.0, 6.8.1. */ + reg = ep_func->msi_cap + PCI_MSI_FLAGS; + msg_ctrl = dw_pcie_ep_readw_dbi(ep, func_no, reg); + has_upper = !!(msg_ctrl & PCI_MSI_FLAGS_64BIT); + reg = ep_func->msi_cap + PCI_MSI_ADDRESS_LO; + msg_addr_lower = dw_pcie_ep_readl_dbi(ep, func_no, reg); + if (has_upper) { + reg = ep_func->msi_cap + PCI_MSI_ADDRESS_HI; + msg_addr_upper = dw_pcie_ep_readl_dbi(ep, func_no, reg); + reg = ep_func->msi_cap + PCI_MSI_DATA_64; + msg_data = dw_pcie_ep_readw_dbi(ep, func_no, reg); + } else { + msg_addr_upper = 0; + reg = ep_func->msi_cap + PCI_MSI_DATA_32; + msg_data = dw_pcie_ep_readw_dbi(ep, func_no, reg); + } + msg_addr = ((u64)msg_addr_upper) << 32 | msg_addr_lower; + + msg_addr = dw_pcie_ep_align_addr(epc, msg_addr, &map_size, &offset); + + /* + * Program the outbound iATU once and keep it enabled. + * + * The spec warns that updating iATU registers while there are + * operations in flight on the AXI bridge interface is not + * supported, so we avoid reprogramming the region on every MSI, + * specifically unmapping immediately after writel(). + */ + if (ep->msi_iatu_mapped && (ep->msi_msg_addr != msg_addr || + ep->msi_map_size != map_size)) { + /* + * The host changed the MSI target address or the required + * mapping size changed. Reprogramming the iATU when there are + * operations in flight is unsafe on this controller. However, + * there is no unified way to check if we have operations in + * flight, thus we don't know if we should WARN() or not. + */ + /* flush posted write before unmap */ + readl(ep->msi_mem + ep->msi_iatu_mapped_offset); + dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); + ep->msi_iatu_mapped = false; + } + + if (!ep->msi_iatu_mapped) { + ret = dw_pcie_ep_map_addr(epc, func_no, 0, + ep->msi_mem_phys, msg_addr, + map_size); + if (ret) + return ret; + + ep->msi_iatu_mapped = true; + ep->msi_iatu_mapped_offset = offset; + ep->msi_msg_addr = msg_addr; + ep->msi_map_size = map_size; + } + + writel(msg_data | (interrupt_num - 1), ep->msi_mem + offset); + + return 0; +} +EXPORT_SYMBOL_GPL(dw_pcie_ep_raise_msi_irq); + +/** + * dw_pcie_ep_raise_msix_irq_doorbell - Raise MSI-X to the host using Doorbell + * method + * @ep: DWC EP device + * @func_no: Function number of the endpoint device + * @interrupt_num: Interrupt number to be raised + * + * Return: 0 if success, errno otherwise. + */ +int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no, + u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct dw_pcie_ep_func *ep_func; + u32 msg_data; + + ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + if (!ep_func || !ep_func->msix_cap) + return -EINVAL; + + msg_data = (func_no << PCIE_MSIX_DOORBELL_PF_SHIFT) | + (interrupt_num - 1); + + dw_pcie_writel_dbi(pci, PCIE_MSIX_DOORBELL, msg_data); + + return 0; +} + +/** + * dw_pcie_ep_raise_msix_irq - Raise MSI-X to the host + * @ep: DWC EP device + * @func_no: Function number of the endpoint device + * @interrupt_num: Interrupt number to be raised + * + * Return: 0 if success, errno otherwise. + */ +int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, + u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct pci_epf_msix_tbl *msix_tbl; + struct dw_pcie_ep_func *ep_func; + struct pci_epc *epc = ep->epc; + size_t map_size = sizeof(u32); + size_t offset; + u32 reg, msg_data, vec_ctrl; + u32 tbl_offset; + u64 msg_addr; + int ret; + u8 bir; + + ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + if (!ep_func || !ep_func->msix_cap) + return -EINVAL; + + reg = ep_func->msix_cap + PCI_MSIX_TABLE; + tbl_offset = dw_pcie_ep_readl_dbi(ep, func_no, reg); + bir = FIELD_GET(PCI_MSIX_TABLE_BIR, tbl_offset); + tbl_offset &= PCI_MSIX_TABLE_OFFSET; + + msix_tbl = ep_func->epf_bar[bir]->addr + tbl_offset; + msg_addr = msix_tbl[(interrupt_num - 1)].msg_addr; + msg_data = msix_tbl[(interrupt_num - 1)].msg_data; + vec_ctrl = msix_tbl[(interrupt_num - 1)].vector_ctrl; + + if (vec_ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT) { + dev_dbg(pci->dev, "MSI-X entry ctrl set\n"); + return -EPERM; + } + + /* + * ep->msi_iatu_mapped means that an MSI target address is cached, + * unmap it first so that we can reuse ep->msi_mem_phys for MSI-X. + */ + if (ep->msi_iatu_mapped) { + /* flush posted write before unmap */ + readl(ep->msi_mem + ep->msi_iatu_mapped_offset); + dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); + ep->msi_iatu_mapped = false; + } + + msg_addr = dw_pcie_ep_align_addr(epc, msg_addr, &map_size, &offset); + ret = dw_pcie_ep_map_addr(epc, func_no, 0, ep->msi_mem_phys, msg_addr, + map_size); + if (ret) + return ret; + + writel(msg_data, ep->msi_mem + offset); + + /* flush posted write before unmap */ + readl(ep->msi_mem + offset); + + dw_pcie_ep_unmap_addr(epc, func_no, 0, ep->msi_mem_phys); + + return 0; +} +EXPORT_SYMBOL_GPL(dw_pcie_ep_raise_msix_irq); + +/** + * dw_pcie_ep_cleanup - Cleanup DWC EP resources after fundamental reset + * @ep: DWC EP device + * + * Cleans up the DWC EP specific resources like eDMA etc... after fundamental + * reset like PERST#. Note that this API is only applicable for drivers + * supporting PERST# or any other methods of fundamental reset. + */ +void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + if (ep->msi_iatu_mapped) { + dw_pcie_ep_unmap_addr(ep->epc, 0, 0, ep->msi_mem_phys); + ep->msi_iatu_mapped = false; + } + + dwc_pcie_debugfs_deinit(pci); + dw_pcie_edma_remove(pci); +} +EXPORT_SYMBOL_GPL(dw_pcie_ep_cleanup); + +/** + * dw_pcie_ep_deinit - Deinitialize the endpoint device + * @ep: DWC EP device + * + * Deinitialize the endpoint device. EPC device is not destroyed since that will + * be taken care by Devres. + */ +void dw_pcie_ep_deinit(struct dw_pcie_ep *ep) +{ + struct pci_epc *epc = ep->epc; + + dw_pcie_ep_cleanup(ep); + + pci_epc_mem_free_addr(epc, ep->msi_mem_phys, ep->msi_mem, + epc->mem->window.page_size); + + pci_epc_mem_exit(epc); +} +EXPORT_SYMBOL_GPL(dw_pcie_ep_deinit); + +static void dw_pcie_ep_init_rebar_registers(struct dw_pcie_ep *ep, u8 func_no) +{ + struct dw_pcie_ep_func *ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + unsigned int offset, nbars; + enum pci_barno bar; + u32 reg, i, val; + + if (!ep_func) + return; + + offset = dw_pcie_ep_find_ext_capability(ep, func_no, PCI_EXT_CAP_ID_REBAR); + + if (offset) { + reg = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); + nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, reg); + + /* + * PCIe r6.0, sec 7.8.6.2 require us to support at least one + * size in the range from 1 MB to 512 GB. Advertise support + * for 1 MB BAR size only. + * + * For a BAR that has been configured via dw_pcie_ep_set_bar(), + * advertise support for only that size instead. + */ + for (i = 0; i < nbars; i++, offset += PCI_REBAR_CTRL) { + /* + * While the RESBAR_CAP_REG_* fields are sticky, the + * RESBAR_CTRL_REG_BAR_SIZE field is non-sticky (it is + * sticky in certain versions of DWC PCIe, but not all). + * + * RESBAR_CTRL_REG_BAR_SIZE is updated automatically by + * the controller when RESBAR_CAP_REG is written, which + * is why RESBAR_CAP_REG is written here. + */ + val = dw_pcie_ep_readl_dbi(ep, func_no, offset + PCI_REBAR_CTRL); + bar = FIELD_GET(PCI_REBAR_CTRL_BAR_IDX, val); + if (ep_func->epf_bar[bar]) + pci_epc_bar_size_to_rebar_cap(ep_func->epf_bar[bar]->size, &val); + else + val = BIT(4); + + dw_pcie_ep_writel_dbi(ep, func_no, offset + PCI_REBAR_CAP, val); + } + } +} + +static void dw_pcie_ep_init_non_sticky_registers(struct dw_pcie *pci) +{ + struct dw_pcie_ep *ep = &pci->ep; + u8 funcs = ep->epc->max_functions; + u32 func0_lnkcap, lnkcap; + u8 func_no, offset; + + dw_pcie_dbi_ro_wr_en(pci); + + for (func_no = 0; func_no < funcs; func_no++) + dw_pcie_ep_init_rebar_registers(ep, func_no); + + dw_pcie_setup(pci); + + /* + * PCIe r7.0, section 7.5.3.6 states that for multi-function + * endpoints, max link width and speed fields must report same + * values for all functions. However, dw_pcie_setup() programs + * these fields only for function 0. Hence, mirror these fields + * to all other functions as well. + */ + if (funcs > 1) { + offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + func0_lnkcap = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP); + func0_lnkcap = FIELD_GET(PCI_EXP_LNKCAP_MLW | + PCI_EXP_LNKCAP_SLS, func0_lnkcap); + + for (func_no = 1; func_no < funcs; func_no++) { + offset = dw_pcie_ep_find_capability(ep, func_no, + PCI_CAP_ID_EXP); + lnkcap = dw_pcie_ep_readl_dbi(ep, func_no, + offset + PCI_EXP_LNKCAP); + FIELD_MODIFY(PCI_EXP_LNKCAP_MLW | PCI_EXP_LNKCAP_SLS, + &lnkcap, func0_lnkcap); + dw_pcie_ep_writel_dbi(ep, func_no, + offset + PCI_EXP_LNKCAP, lnkcap); + } + } + + dw_pcie_dbi_ro_wr_dis(pci); +} + +static void dw_pcie_ep_disable_bars(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + enum pci_epc_bar_type bar_type; + enum pci_barno bar; + + for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) { + bar_type = dw_pcie_ep_get_bar_type(ep, bar); + + /* + * Reserved BARs should not get disabled by default. All other + * BAR types are disabled by default. + * + * This is in line with the current EPC core design, where all + * BARs are disabled by default, and then the EPF driver enables + * the BARs it wishes to use. + */ + if (bar_type != BAR_RESERVED) + dw_pcie_ep_reset_bar(pci, bar); + } +} + +/** + * dw_pcie_ep_init_registers - Initialize DWC EP specific registers + * @ep: DWC EP device + * + * Initialize the registers (CSRs) specific to DWC EP. This API should be called + * only when the endpoint receives an active refclk (either from host or + * generated locally). + */ +int dw_pcie_ep_init_registers(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct dw_pcie_ep_func *ep_func; + struct device *dev = pci->dev; + struct pci_epc *epc = ep->epc; + u32 ptm_cap_base, reg; + u8 hdr_type; + u8 func_no; + void *addr; + int ret; + + hdr_type = dw_pcie_readb_dbi(pci, PCI_HEADER_TYPE) & + PCI_HEADER_TYPE_MASK; + if (hdr_type != PCI_HEADER_TYPE_NORMAL) { + dev_err(pci->dev, + "PCIe controller is not set to EP mode (hdr_type:0x%x)!\n", + hdr_type); + return -EIO; + } + + dw_pcie_version_detect(pci); + + dw_pcie_iatu_detect(pci); + + ret = dw_pcie_edma_detect(pci); + if (ret) + return ret; + + ret = -ENOMEM; + if (!ep->ib_window_map) { + ep->ib_window_map = devm_bitmap_zalloc(dev, pci->num_ib_windows, + GFP_KERNEL); + if (!ep->ib_window_map) + goto err_remove_edma; + } + + if (!ep->ob_window_map) { + ep->ob_window_map = devm_bitmap_zalloc(dev, pci->num_ob_windows, + GFP_KERNEL); + if (!ep->ob_window_map) + goto err_remove_edma; + } + + if (!ep->outbound_addr) { + addr = devm_kcalloc(dev, pci->num_ob_windows, sizeof(phys_addr_t), + GFP_KERNEL); + if (!addr) + goto err_remove_edma; + ep->outbound_addr = addr; + } + + for (func_no = 0; func_no < epc->max_functions; func_no++) { + + ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no); + if (ep_func) + continue; + + ep_func = devm_kzalloc(dev, sizeof(*ep_func), GFP_KERNEL); + if (!ep_func) + goto err_remove_edma; + + ep_func->func_no = func_no; + ep_func->msi_cap = dw_pcie_ep_find_capability(ep, func_no, + PCI_CAP_ID_MSI); + ep_func->msix_cap = dw_pcie_ep_find_capability(ep, func_no, + PCI_CAP_ID_MSIX); + + list_add_tail(&ep_func->list, &ep->func_list); + } + + if (ep->ops->init) { + ret = ep->ops->init(ep); + if (ret) + goto err_remove_edma; + } + + dw_pcie_ep_disable_bars(ep); + + /* + * PCIe r6.0, section 7.9.15 states that for endpoints that support + * PTM, this capability structure is required in exactly one + * function, which controls the PTM behavior of all PTM capable + * functions. This indicates the PTM capability structure + * represents controller-level registers rather than per-function + * registers. + * + * Therefore, PTM capability registers are configured using the + * standard DBI accessors, instead of func_no indexed per-function + * accessors. + */ + ptm_cap_base = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_PTM); + + /* + * PTM responder capability can be disabled only after disabling + * PTM root capability. + */ + if (ptm_cap_base) { + dw_pcie_dbi_ro_wr_en(pci); + reg = dw_pcie_readl_dbi(pci, ptm_cap_base + PCI_PTM_CAP); + reg &= ~PCI_PTM_CAP_ROOT; + dw_pcie_writel_dbi(pci, ptm_cap_base + PCI_PTM_CAP, reg); + + reg = dw_pcie_readl_dbi(pci, ptm_cap_base + PCI_PTM_CAP); + reg &= ~(PCI_PTM_CAP_RES | PCI_PTM_GRANULARITY_MASK); + dw_pcie_writel_dbi(pci, ptm_cap_base + PCI_PTM_CAP, reg); + dw_pcie_dbi_ro_wr_dis(pci); + } + + dw_pcie_ep_init_non_sticky_registers(pci); + + dwc_pcie_debugfs_init(pci, DW_PCIE_EP_TYPE); + + return 0; + +err_remove_edma: + dw_pcie_edma_remove(pci); + + return ret; +} +EXPORT_SYMBOL_GPL(dw_pcie_ep_init_registers); + +/** + * dw_pcie_ep_linkup - Notify EPF drivers about Link Up event + * @ep: DWC EP device + */ +void dw_pcie_ep_linkup(struct dw_pcie_ep *ep) +{ + struct pci_epc *epc = ep->epc; + + pci_epc_linkup(epc); +} +EXPORT_SYMBOL_GPL(dw_pcie_ep_linkup); + +/** + * dw_pcie_ep_linkdown - Notify EPF drivers about Link Down event + * @ep: DWC EP device + * + * Non-sticky registers are also initialized before sending the notification to + * the EPF drivers. This is needed since the registers need to be initialized + * before the link comes back again. + */ +void dw_pcie_ep_linkdown(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct pci_epc *epc = ep->epc; + + /* + * Initialize the non-sticky DWC registers as they would've reset post + * Link Down. This is specifically needed for drivers not supporting + * PERST# as they have no way to reinitialize the registers before the + * link comes back again. + */ + dw_pcie_ep_init_non_sticky_registers(pci); + + pci_epc_linkdown(epc); +} +EXPORT_SYMBOL_GPL(dw_pcie_ep_linkdown); + +static int dw_pcie_ep_get_resources(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct device *dev = pci->dev; + struct platform_device *pdev = to_platform_device(dev); + struct device_node *np = dev->of_node; + struct pci_epc *epc = ep->epc; + struct resource *res; + int ret; + + ret = dw_pcie_get_resources(pci); + if (ret) + return ret; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space"); + if (!res) + return -EINVAL; + + ep->phys_base = res->start; + ep->addr_size = resource_size(res); + + /* + * artpec6_pcie_cpu_addr_fixup() uses ep->phys_base, so call + * dw_pcie_parent_bus_offset() after setting ep->phys_base. + */ + pci->parent_bus_offset = dw_pcie_parent_bus_offset(pci, "addr_space", + ep->phys_base); + + ret = of_property_read_u8(np, "max-functions", &epc->max_functions); + if (ret < 0) + epc->max_functions = 1; + + return 0; +} + +/** + * dw_pcie_ep_init - Initialize the endpoint device + * @ep: DWC EP device + * + * Initialize the endpoint device. Allocate resources and create the EPC + * device with the endpoint framework. + * + * Return: 0 if success, errno otherwise. + */ +int dw_pcie_ep_init(struct dw_pcie_ep *ep) +{ + int ret; + struct pci_epc *epc; + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct device *dev = pci->dev; + + INIT_LIST_HEAD(&ep->func_list); + ep->msi_iatu_mapped = false; + ep->msi_msg_addr = 0; + ep->msi_map_size = 0; + + epc = devm_pci_epc_create(dev, &epc_ops); + if (IS_ERR(epc)) { + dev_err(dev, "Failed to create epc device\n"); + return PTR_ERR(epc); + } + + ep->epc = epc; + epc_set_drvdata(epc, ep); + + ret = dw_pcie_ep_get_resources(ep); + if (ret) + return ret; + + if (ep->ops->pre_init) { + ret = ep->ops->pre_init(ep); + if (ret) + return ret; + } + + ret = pci_epc_mem_init(epc, ep->phys_base, ep->addr_size, + ep->page_size); + if (ret < 0) { + dev_err(dev, "Failed to initialize address space\n"); + return ret; + } + + ep->msi_mem = pci_epc_mem_alloc_addr(epc, &ep->msi_mem_phys, + epc->mem->window.page_size); + if (!ep->msi_mem) { + ret = -ENOMEM; + dev_err(dev, "Failed to reserve memory for MSI/MSI-X\n"); + goto err_exit_epc_mem; + } + + return 0; + +err_exit_epc_mem: + pci_epc_mem_exit(epc); + + return ret; +} +EXPORT_SYMBOL_GPL(dw_pcie_ep_init); diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c new file mode 100644 index 000000000..f5a38e6fd --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-designware-host.c @@ -0,0 +1,1338 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Synopsys DesignWare PCIe host controller driver + * + * Copyright (C) 2013 Samsung Electronics Co., Ltd. + * https://www.samsung.com + * + * Author: Jingoo Han <jg1.han@samsung.com> + */ + +#include <linux/align.h> +#include <linux/iopoll.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/irqdomain.h> +#include <linux/msi.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/pci_regs.h> +#include <linux/platform_device.h> + +#include "../pci-host-common.h" +#include "../../pci.h" +#include "pcie-designware.h" + +static struct pci_ops dw_pcie_ops; +static struct pci_ops dw_pcie_ecam_ops; +static struct pci_ops dw_child_pcie_ops; + +#ifdef CONFIG_SMP +static void dw_irq_noop(struct irq_data *d) { } +#endif + +static bool dw_pcie_init_dev_msi_info(struct device *dev, struct irq_domain *domain, + struct irq_domain *real_parent, struct msi_domain_info *info) +{ + if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info)) + return false; + +#ifdef CONFIG_SMP + info->chip->irq_ack = dw_irq_noop; + info->chip->irq_pre_redirect = irq_chip_pre_redirect_parent; +#else + info->chip->irq_ack = irq_chip_ack_parent; +#endif + return true; +} + +#define DW_PCIE_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_PCI_MSI_MASK_PARENT) +#define DW_PCIE_MSI_FLAGS_SUPPORTED (MSI_FLAG_MULTI_PCI_MSI | \ + MSI_FLAG_PCI_MSIX | \ + MSI_GENERIC_FLAGS_MASK) + +#define IS_256MB_ALIGNED(x) IS_ALIGNED(x, SZ_256M) + +static const struct msi_parent_ops dw_pcie_msi_parent_ops = { + .required_flags = DW_PCIE_MSI_FLAGS_REQUIRED, + .supported_flags = DW_PCIE_MSI_FLAGS_SUPPORTED, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .prefix = "DW-", + .init_dev_msi_info = dw_pcie_init_dev_msi_info, +}; + +/* MSI int handler */ +void dw_handle_msi_irq(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + unsigned int i, num_ctrls; + + num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL; + + for (i = 0; i < num_ctrls; i++) { + unsigned int reg_off = i * MSI_REG_CTRL_BLOCK_SIZE; + unsigned int irq_off = i * MAX_MSI_IRQS_PER_CTRL; + unsigned long status, pos; + + status = dw_pcie_readl_dbi(pci, PCIE_MSI_INTR0_STATUS + reg_off); + if (!status) + continue; + + for_each_set_bit(pos, &status, MAX_MSI_IRQS_PER_CTRL) + generic_handle_demux_domain_irq(pp->irq_domain, irq_off + pos); + } +} + +/* Chained MSI interrupt service routine */ +static void dw_chained_msi_isr(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + struct dw_pcie_rp *pp; + + chained_irq_enter(chip, desc); + + pp = irq_desc_get_handler_data(desc); + dw_handle_msi_irq(pp); + + chained_irq_exit(chip, desc); +} + +static void dw_pci_setup_msi_msg(struct irq_data *d, struct msi_msg *msg) +{ + struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + u64 msi_target = (u64)pp->msi_data; + + msg->address_lo = lower_32_bits(msi_target); + msg->address_hi = upper_32_bits(msi_target); + msg->data = d->hwirq; + + dev_dbg(pci->dev, "msi#%d address_hi %#x address_lo %#x\n", + (int)d->hwirq, msg->address_hi, msg->address_lo); +} + +static void dw_pci_bottom_mask(struct irq_data *d) +{ + struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + unsigned int res, bit, ctrl; + + guard(raw_spinlock)(&pp->lock); + ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL; + res = ctrl * MSI_REG_CTRL_BLOCK_SIZE; + bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; + + pp->irq_mask[ctrl] |= BIT(bit); + dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]); +} + +static void dw_pci_bottom_unmask(struct irq_data *d) +{ + struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + unsigned int res, bit, ctrl; + + guard(raw_spinlock)(&pp->lock); + ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL; + res = ctrl * MSI_REG_CTRL_BLOCK_SIZE; + bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; + + pp->irq_mask[ctrl] &= ~BIT(bit); + dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]); +} + +static void dw_pci_bottom_ack(struct irq_data *d) +{ + struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + unsigned int res, bit, ctrl; + + ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL; + res = ctrl * MSI_REG_CTRL_BLOCK_SIZE; + bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL; + + dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_STATUS + res, BIT(bit)); +} + +static struct irq_chip dw_pci_msi_bottom_irq_chip = { + .name = "DWPCI-MSI", + .irq_compose_msi_msg = dw_pci_setup_msi_msg, + .irq_mask = dw_pci_bottom_mask, + .irq_unmask = dw_pci_bottom_unmask, +#ifdef CONFIG_SMP + .irq_ack = dw_irq_noop, + .irq_pre_redirect = dw_pci_bottom_ack, + .irq_set_affinity = irq_chip_redirect_set_affinity, +#else + .irq_ack = dw_pci_bottom_ack, +#endif +}; + +static int dw_pcie_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct dw_pcie_rp *pp = domain->host_data; + int bit; + + scoped_guard (raw_spinlock_irq, &pp->lock) { + bit = bitmap_find_free_region(pp->msi_irq_in_use, pp->num_vectors, + order_base_2(nr_irqs)); + } + + if (bit < 0) + return -ENOSPC; + + for (unsigned int i = 0; i < nr_irqs; i++) { + irq_domain_set_info(domain, virq + i, bit + i, pp->msi_irq_chip, + pp, handle_edge_irq, NULL, NULL); + } + return 0; +} + +static void dw_pcie_irq_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct dw_pcie_rp *pp = domain->host_data; + + guard(raw_spinlock_irq)(&pp->lock); + bitmap_release_region(pp->msi_irq_in_use, d->hwirq, order_base_2(nr_irqs)); +} + +static const struct irq_domain_ops dw_pcie_msi_domain_ops = { + .alloc = dw_pcie_irq_domain_alloc, + .free = dw_pcie_irq_domain_free, +}; + +int dw_pcie_allocate_domains(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct irq_domain_info info = { + .fwnode = dev_fwnode(pci->dev), + .ops = &dw_pcie_msi_domain_ops, + .size = pp->num_vectors, + .host_data = pp, + }; + + pp->irq_domain = msi_create_parent_irq_domain(&info, &dw_pcie_msi_parent_ops); + if (!pp->irq_domain) { + dev_err(pci->dev, "Failed to create IRQ domain\n"); + return -ENOMEM; + } + + return 0; +} +EXPORT_SYMBOL_GPL(dw_pcie_allocate_domains); + +void dw_pcie_free_msi(struct dw_pcie_rp *pp) +{ + u32 ctrl; + + for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++) { + if (pp->msi_irq[ctrl] > 0) + irq_set_chained_handler_and_data(pp->msi_irq[ctrl], NULL, NULL); + } + + irq_domain_remove(pp->irq_domain); +} +EXPORT_SYMBOL_GPL(dw_pcie_free_msi); + +void dw_pcie_msi_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + u64 msi_target = (u64)pp->msi_data; + u32 ctrl, num_ctrls; + + if (!pci_msi_enabled() || !pp->use_imsi_rx) + return; + + num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL; + + /* Initialize IRQ Status array */ + for (ctrl = 0; ctrl < num_ctrls; ctrl++) { + dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + + (ctrl * MSI_REG_CTRL_BLOCK_SIZE), + pp->irq_mask[ctrl]); + dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_ENABLE + + (ctrl * MSI_REG_CTRL_BLOCK_SIZE), + ~0); + } + + /* Program the msi_data */ + dw_pcie_writel_dbi(pci, PCIE_MSI_ADDR_LO, lower_32_bits(msi_target)); + dw_pcie_writel_dbi(pci, PCIE_MSI_ADDR_HI, upper_32_bits(msi_target)); +} +EXPORT_SYMBOL_GPL(dw_pcie_msi_init); + +static int dw_pcie_parse_split_msi_irq(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct device *dev = pci->dev; + struct platform_device *pdev = to_platform_device(dev); + u32 ctrl, max_vectors; + int irq; + + /* Parse any "msiX" IRQs described in the devicetree */ + for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++) { + char msi_name[] = "msiX"; + + msi_name[3] = '0' + ctrl; + irq = platform_get_irq_byname_optional(pdev, msi_name); + if (irq == -ENXIO) + break; + if (irq < 0) + return dev_err_probe(dev, irq, + "Failed to parse MSI IRQ '%s'\n", + msi_name); + + pp->msi_irq[ctrl] = irq; + } + + /* If no "msiX" IRQs, caller should fallback to "msi" IRQ */ + if (ctrl == 0) + return -ENXIO; + + max_vectors = ctrl * MAX_MSI_IRQS_PER_CTRL; + if (pp->num_vectors > max_vectors) { + dev_warn(dev, "Exceeding number of MSI vectors, limiting to %u\n", + max_vectors); + pp->num_vectors = max_vectors; + } + if (!pp->num_vectors) + pp->num_vectors = max_vectors; + + return 0; +} + +int dw_pcie_msi_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct device *dev = pci->dev; + struct platform_device *pdev = to_platform_device(dev); + u64 *msi_vaddr = NULL; + int ret; + u32 ctrl, num_ctrls; + + for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++) + pp->irq_mask[ctrl] = ~0; + + if (!pp->msi_irq[0]) { + ret = dw_pcie_parse_split_msi_irq(pp); + if (ret < 0 && ret != -ENXIO) + return ret; + } + + if (!pp->num_vectors) + pp->num_vectors = MSI_DEF_NUM_VECTORS; + num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL; + + if (!pp->msi_irq[0]) { + pp->msi_irq[0] = platform_get_irq_byname_optional(pdev, "msi"); + if (pp->msi_irq[0] < 0) { + pp->msi_irq[0] = platform_get_irq(pdev, 0); + if (pp->msi_irq[0] < 0) + return pp->msi_irq[0]; + } + } + + dev_dbg(dev, "Using %d MSI vectors\n", pp->num_vectors); + + pp->msi_irq_chip = &dw_pci_msi_bottom_irq_chip; + + ret = dw_pcie_allocate_domains(pp); + if (ret) + return ret; + + for (ctrl = 0; ctrl < num_ctrls; ctrl++) { + if (pp->msi_irq[ctrl] > 0) + irq_set_chained_handler_and_data(pp->msi_irq[ctrl], + dw_chained_msi_isr, pp); + } + + /* + * Even though the iMSI-RX Module supports 64-bit addresses some + * peripheral PCIe devices may lack 64-bit message support. In + * order not to miss MSI TLPs from those devices the MSI target + * address has to be within the lowest 4GB. + * + * Per DWC databook r6.21a, section 3.10.2.3, the incoming MWr TLP + * targeting the MSI_CTRL_ADDR is terminated by the iMSI-RX and never + * appears on the AXI bus. So MSI_CTRL_ADDR address doesn't need to be + * mapped and can be any memory that doesn't get allocated for the BAR + * memory. Since most of the platforms provide 32-bit address for + * 'config' region, try cfg0_base as the first option for the MSI target + * address if it's a 32-bit address. Otherwise, try 32-bit and 64-bit + * coherent memory allocation one by one. + */ + if (!(pp->cfg0_base & GENMASK_ULL(63, 32))) { + pp->msi_data = pp->cfg0_base; + return 0; + } + + ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(32)); + if (!ret) + msi_vaddr = dmam_alloc_coherent(dev, sizeof(u64), &pp->msi_data, + GFP_KERNEL); + + if (!msi_vaddr) { + dev_warn(dev, "Failed to allocate 32-bit MSI address\n"); + dma_set_coherent_mask(dev, DMA_BIT_MASK(64)); + msi_vaddr = dmam_alloc_coherent(dev, sizeof(u64), &pp->msi_data, + GFP_KERNEL); + if (!msi_vaddr) { + dev_err(dev, "Failed to allocate MSI address\n"); + dw_pcie_free_msi(pp); + return -ENOMEM; + } + } + + return 0; +} +EXPORT_SYMBOL_GPL(dw_pcie_msi_host_init); + +static void dw_pcie_host_request_msg_tlp_res(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct resource_entry *win; + struct resource *res; + + win = resource_list_first_type(&pp->bridge->windows, IORESOURCE_MEM); + if (win) { + res = devm_kzalloc(pci->dev, sizeof(*res), GFP_KERNEL); + if (!res) + return; + + /* + * Allocate MSG TLP region of size 'region_align' at the end of + * the host bridge window. + */ + res->start = win->res->end - pci->region_align + 1; + res->end = win->res->end; + res->name = "msg"; + res->flags = win->res->flags | IORESOURCE_BUSY; + + if (!devm_request_resource(pci->dev, win->res, res)) + pp->msg_res = res; + } +} + +static int dw_pcie_config_ecam_iatu(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct dw_pcie_ob_atu_cfg atu = {0}; + resource_size_t bus_range_max; + struct resource_entry *bus; + int ret; + + bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS); + + /* + * Root bus under the host bridge doesn't require any iATU configuration + * as DBI region will be used to access root bus config space. + * Immediate bus under Root Bus, needs type 0 iATU configuration and + * remaining buses need type 1 iATU configuration. + */ + atu.index = 0; + atu.type = PCIE_TLP_TYPE_CFG0_RDWR; + atu.parent_bus_addr = pp->cfg0_base + SZ_1M; + /* 1MiB is to cover 1 (bus) * 32 (devices) * 8 (functions) */ + atu.size = SZ_1M; + atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE; + ret = dw_pcie_prog_outbound_atu(pci, &atu); + if (ret) + return ret; + + bus_range_max = resource_size(bus->res); + + if (bus_range_max < 2) + return 0; + + /* Configure remaining buses in type 1 iATU configuration */ + atu.index = 1; + atu.type = PCIE_TLP_TYPE_CFG1_RDWR; + atu.parent_bus_addr = pp->cfg0_base + SZ_2M; + atu.size = (SZ_1M * bus_range_max) - SZ_2M; + atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE; + + return dw_pcie_prog_outbound_atu(pci, &atu); +} + +static int dw_pcie_create_ecam_window(struct dw_pcie_rp *pp, struct resource *res) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct device *dev = pci->dev; + struct resource_entry *bus; + + bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS); + if (!bus) + return -ENODEV; + + pp->cfg = pci_ecam_create(dev, res, bus->res, &pci_generic_ecam_ops); + if (IS_ERR(pp->cfg)) + return PTR_ERR(pp->cfg); + + return 0; +} + +static bool dw_pcie_ecam_enabled(struct dw_pcie_rp *pp, struct resource *config_res) +{ + struct resource *bus_range; + u64 nr_buses; + + /* Vendor glue drivers may implement their own ECAM mechanism */ + if (pp->native_ecam) + return false; + + /* + * PCIe spec r6.0, sec 7.2.2 mandates the base address used for ECAM to + * be aligned on a 2^(n+20) byte boundary, where n is the number of bits + * used for representing 'bus' in BDF. Since the DWC cores always use 8 + * bits for representing 'bus', the base address has to be aligned to + * 2^28 byte boundary, which is 256 MiB. + */ + if (!IS_256MB_ALIGNED(config_res->start)) + return false; + + bus_range = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS)->res; + if (!bus_range) + return false; + + nr_buses = resource_size(config_res) >> PCIE_ECAM_BUS_SHIFT; + + return nr_buses >= resource_size(bus_range); +} + +static int dw_pcie_host_get_resources(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct device *dev = pci->dev; + struct platform_device *pdev = to_platform_device(dev); + struct resource_entry *win; + struct resource *res; + int ret; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config"); + if (!res) { + dev_err(dev, "Missing \"config\" reg space\n"); + return -ENODEV; + } + + pp->cfg0_size = resource_size(res); + pp->cfg0_base = res->start; + + pp->ecam_enabled = dw_pcie_ecam_enabled(pp, res); + if (pp->ecam_enabled) { + ret = dw_pcie_create_ecam_window(pp, res); + if (ret) + return ret; + + pp->bridge->ops = &dw_pcie_ecam_ops; + pp->bridge->sysdata = pp->cfg; + pp->cfg->priv = pp; + } else { + pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res); + if (IS_ERR(pp->va_cfg0_base)) + return PTR_ERR(pp->va_cfg0_base); + + /* Set default bus ops */ + pp->bridge->ops = &dw_pcie_ops; + pp->bridge->child_ops = &dw_child_pcie_ops; + pp->bridge->sysdata = pp; + } + + ret = dw_pcie_get_resources(pci); + if (ret) { + if (pp->cfg) + pci_ecam_free(pp->cfg); + return ret; + } + + /* Get the I/O range from DT */ + win = resource_list_first_type(&pp->bridge->windows, IORESOURCE_IO); + if (win) { + pp->io_size = resource_size(win->res); + pp->io_bus_addr = win->res->start - win->offset; + pp->io_base = pci_pio_to_address(win->res->start); + } + + /* + * visconti_pcie_cpu_addr_fixup() uses pp->io_base, so we have to + * call dw_pcie_parent_bus_offset() after setting pp->io_base. + */ + pci->parent_bus_offset = dw_pcie_parent_bus_offset(pci, "config", + pp->cfg0_base); + return 0; +} + +int dw_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct device *dev = pci->dev; + struct device_node *np = dev->of_node; + struct pci_host_bridge *bridge; + int ret; + + raw_spin_lock_init(&pp->lock); + + bridge = devm_pci_alloc_host_bridge(dev, 0); + if (!bridge) + return -ENOMEM; + + pp->bridge = bridge; + + ret = dw_pcie_host_get_resources(pp); + if (ret) + return ret; + + if (pci_msi_enabled()) { + pp->use_imsi_rx = !(pp->ops->msi_init || + of_property_present(np, "msi-parent") || + of_property_present(np, "msi-map")); + } + + if (pp->ops->init) { + ret = pp->ops->init(pp); + if (ret) + goto err_free_ecam; + } + + if (pci_msi_enabled()) { + /* + * For the use_imsi_rx case the default assignment is handled + * in the dw_pcie_msi_host_init(). + */ + if (!pp->use_imsi_rx && !pp->num_vectors) { + pp->num_vectors = MSI_DEF_NUM_VECTORS; + } else if (pp->num_vectors > MAX_MSI_IRQS) { + dev_err(dev, "Invalid number of vectors\n"); + ret = -EINVAL; + goto err_deinit_host; + } + + if (pp->ops->msi_init) { + ret = pp->ops->msi_init(pp); + if (ret < 0) + goto err_deinit_host; + } else if (pp->use_imsi_rx) { + ret = dw_pcie_msi_host_init(pp); + if (ret < 0) + goto err_deinit_host; + } + } + + dw_pcie_version_detect(pci); + + dw_pcie_iatu_detect(pci); + + if (pci->num_lanes < 1) + pci->num_lanes = dw_pcie_link_get_max_link_width(pci); + + ret = of_pci_get_equalization_presets(dev, &pp->presets, pci->num_lanes); + if (ret) + goto err_free_msi; + + /* + * Allocate the resource for MSG TLP before programming the iATU + * outbound window in dw_pcie_setup_rc(). Since the allocation depends + * on the value of 'region_align', this has to be done after + * dw_pcie_iatu_detect(). + * + * Glue drivers need to set 'use_atu_msg' before dw_pcie_host_init() to + * make use of the generic MSG TLP implementation. + */ + if (pp->use_atu_msg) + dw_pcie_host_request_msg_tlp_res(pp); + + ret = dw_pcie_edma_detect(pci); + if (ret) + goto err_free_msi; + + ret = dw_pcie_setup_rc(pp); + if (ret) + goto err_remove_edma; + + if (!dw_pcie_link_up(pci)) { + ret = dw_pcie_start_link(pci); + if (ret) + goto err_remove_edma; + } + + /* + * Only fail on timeout error. Other errors indicate the device may + * become available later, so continue without failing. + */ + ret = dw_pcie_wait_for_link(pci); + if (ret == -ETIMEDOUT) + goto err_stop_link; + + ret = pci_host_probe(bridge); + if (ret) + goto err_stop_link; + + if (pp->ops->post_init) + pp->ops->post_init(pp); + + dwc_pcie_debugfs_init(pci, DW_PCIE_RC_TYPE); + + return 0; + +err_stop_link: + dw_pcie_stop_link(pci); + +err_remove_edma: + dw_pcie_edma_remove(pci); + +err_free_msi: + if (pp->use_imsi_rx) + dw_pcie_free_msi(pp); + +err_deinit_host: + if (pp->ops->deinit) + pp->ops->deinit(pp); + +err_free_ecam: + if (pp->cfg) + pci_ecam_free(pp->cfg); + + return ret; +} +EXPORT_SYMBOL_GPL(dw_pcie_host_init); + +void dw_pcie_host_deinit(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + + dwc_pcie_debugfs_deinit(pci); + + pci_lock_rescan_remove(); + pci_stop_root_bus(pp->bridge->bus); + pci_remove_root_bus(pp->bridge->bus); + pci_unlock_rescan_remove(); + + dw_pcie_stop_link(pci); + + dw_pcie_edma_remove(pci); + + if (pp->use_imsi_rx) + dw_pcie_free_msi(pp); + + if (pp->ops->deinit) + pp->ops->deinit(pp); + + if (pp->cfg) + pci_ecam_free(pp->cfg); +} +EXPORT_SYMBOL_GPL(dw_pcie_host_deinit); + +static void __iomem *dw_pcie_other_conf_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + struct dw_pcie_rp *pp = bus->sysdata; + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct dw_pcie_ob_atu_cfg atu = { 0 }; + int type, ret; + u32 busdev; + + /* + * Checking whether the link is up here is a last line of defense + * against platforms that forward errors on the system bus as + * SError upon PCI configuration transactions issued when the link + * is down. This check is racy by definition and does not stop + * the system from triggering an SError if the link goes down + * after this check is performed. + */ + if (!dw_pcie_link_up(pci)) + return NULL; + + busdev = PCIE_ATU_BUS(bus->number) | PCIE_ATU_DEV(PCI_SLOT(devfn)) | + PCIE_ATU_FUNC(PCI_FUNC(devfn)); + + if (pci_is_root_bus(bus->parent)) + type = PCIE_TLP_TYPE_CFG0_RDWR; + else + type = PCIE_TLP_TYPE_CFG1_RDWR; + + atu.type = type; + atu.parent_bus_addr = pp->cfg0_base - pci->parent_bus_offset; + atu.pci_addr = busdev; + atu.size = pp->cfg0_size; + + ret = dw_pcie_prog_outbound_atu(pci, &atu); + if (ret) + return NULL; + + return pp->va_cfg0_base + where; +} + +static int dw_pcie_rd_other_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct dw_pcie_rp *pp = bus->sysdata; + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct dw_pcie_ob_atu_cfg atu = { 0 }; + int ret; + + ret = pci_generic_config_read(bus, devfn, where, size, val); + if (ret != PCIBIOS_SUCCESSFUL) + return ret; + + if (pp->cfg0_io_shared) { + atu.type = PCIE_TLP_TYPE_IO_RDWR; + atu.parent_bus_addr = pp->io_base - pci->parent_bus_offset; + atu.pci_addr = pp->io_bus_addr; + atu.size = pp->io_size; + + ret = dw_pcie_prog_outbound_atu(pci, &atu); + if (ret) + return PCIBIOS_SET_FAILED; + } + + return PCIBIOS_SUCCESSFUL; +} + +static int dw_pcie_wr_other_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + struct dw_pcie_rp *pp = bus->sysdata; + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct dw_pcie_ob_atu_cfg atu = { 0 }; + int ret; + + ret = pci_generic_config_write(bus, devfn, where, size, val); + if (ret != PCIBIOS_SUCCESSFUL) + return ret; + + if (pp->cfg0_io_shared) { + atu.type = PCIE_TLP_TYPE_IO_RDWR; + atu.parent_bus_addr = pp->io_base - pci->parent_bus_offset; + atu.pci_addr = pp->io_bus_addr; + atu.size = pp->io_size; + + ret = dw_pcie_prog_outbound_atu(pci, &atu); + if (ret) + return PCIBIOS_SET_FAILED; + } + + return PCIBIOS_SUCCESSFUL; +} + +static struct pci_ops dw_child_pcie_ops = { + .map_bus = dw_pcie_other_conf_map_bus, + .read = dw_pcie_rd_other_conf, + .write = dw_pcie_wr_other_conf, +}; + +void __iomem *dw_pcie_own_conf_map_bus(struct pci_bus *bus, unsigned int devfn, int where) +{ + struct dw_pcie_rp *pp = bus->sysdata; + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + + if (PCI_SLOT(devfn) > 0) + return NULL; + + return pci->dbi_base + where; +} +EXPORT_SYMBOL_GPL(dw_pcie_own_conf_map_bus); + +static void __iomem *dw_pcie_ecam_conf_map_bus(struct pci_bus *bus, unsigned int devfn, int where) +{ + struct pci_config_window *cfg = bus->sysdata; + struct dw_pcie_rp *pp = cfg->priv; + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + unsigned int busn = bus->number; + + if (busn > 0) + return pci_ecam_map_bus(bus, devfn, where); + + if (PCI_SLOT(devfn) > 0) + return NULL; + + return pci->dbi_base + where; +} + +static struct pci_ops dw_pcie_ops = { + .map_bus = dw_pcie_own_conf_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; + +static struct pci_ops dw_pcie_ecam_ops = { + .map_bus = dw_pcie_ecam_conf_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; + +static int dw_pcie_iatu_setup(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct dw_pcie_ob_atu_cfg atu = { 0 }; + struct resource_entry *entry; + int ob_iatu_index; + int ib_iatu_index; + int i, ret; + + if (!pci->num_ob_windows) { + dev_err(pci->dev, "No outbound iATU found\n"); + return -EINVAL; + } + + /* + * Ensure all out/inbound windows are disabled before proceeding with + * the MEM/IO (dma-)ranges setups. + */ + for (i = 0; i < pci->num_ob_windows; i++) + dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_OB, i); + + for (i = 0; i < pci->num_ib_windows; i++) + dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, i); + + /* + * NOTE: For outbound address translation, outbound iATU at index 0 is + * reserved for CFG IOs (dw_pcie_other_conf_map_bus()), thus start at + * index 1. + * + * If using ECAM, outbound iATU at index 0 and index 1 is reserved for + * CFG IOs. + */ + if (pp->ecam_enabled) { + ob_iatu_index = 2; + ret = dw_pcie_config_ecam_iatu(pp); + if (ret) { + dev_err(pci->dev, "Failed to configure iATU in ECAM mode\n"); + return ret; + } + } else { + ob_iatu_index = 1; + } + + resource_list_for_each_entry(entry, &pp->bridge->windows) { + resource_size_t res_size; + + if (resource_type(entry->res) != IORESOURCE_MEM) + continue; + + atu.type = PCIE_TLP_TYPE_MEM_RDWR; + atu.parent_bus_addr = entry->res->start - pci->parent_bus_offset; + atu.pci_addr = entry->res->start - entry->offset; + + /* Adjust iATU size if MSG TLP region was allocated before */ + if (pp->msg_res && pp->msg_res->parent == entry->res) + res_size = resource_size(entry->res) - + resource_size(pp->msg_res); + else + res_size = resource_size(entry->res); + + while (res_size > 0) { + /* + * Return failure if we run out of windows in the + * middle. Otherwise, we would end up only partially + * mapping a single resource. + */ + if (ob_iatu_index >= pci->num_ob_windows) { + dev_err(pci->dev, "Cannot add outbound window for region: %pr\n", + entry->res); + return -ENOMEM; + } + + atu.index = ob_iatu_index; + atu.size = MIN(pci->region_limit + 1, res_size); + + ret = dw_pcie_prog_outbound_atu(pci, &atu); + if (ret) { + dev_err(pci->dev, "Failed to set MEM range %pr\n", + entry->res); + return ret; + } + + ob_iatu_index++; + atu.parent_bus_addr += atu.size; + atu.pci_addr += atu.size; + res_size -= atu.size; + } + } + + if (pp->io_size) { + if (ob_iatu_index < pci->num_ob_windows) { + atu.index = ob_iatu_index; + atu.type = PCIE_TLP_TYPE_IO_RDWR; + atu.parent_bus_addr = pp->io_base - pci->parent_bus_offset; + atu.pci_addr = pp->io_bus_addr; + atu.size = pp->io_size; + + ret = dw_pcie_prog_outbound_atu(pci, &atu); + if (ret) { + dev_err(pci->dev, "Failed to set IO range %pr\n", + entry->res); + return ret; + } + ob_iatu_index++; + } else { + /* + * If there are not enough outbound windows to give I/O + * space its own iATU, the outbound iATU at index 0 will + * be shared between I/O space and CFG IOs, by + * temporarily reconfiguring the iATU to CFG space, in + * order to do a CFG IO, and then immediately restoring + * it to I/O space. This is only implemented when using + * dw_pcie_other_conf_map_bus(), which is not the case + * when using ECAM. + */ + if (pp->ecam_enabled) { + dev_err(pci->dev, "Cannot add outbound window for I/O\n"); + return -ENOMEM; + } + pp->cfg0_io_shared = true; + } + } + + if (pp->use_atu_msg) { + if (ob_iatu_index >= pci->num_ob_windows) { + dev_err(pci->dev, "Cannot add outbound window for MSG TLP\n"); + return -ENOMEM; + } + pp->msg_atu_index = ob_iatu_index++; + } + + ib_iatu_index = 0; + resource_list_for_each_entry(entry, &pp->bridge->dma_ranges) { + resource_size_t res_start, res_size, window_size; + + if (resource_type(entry->res) != IORESOURCE_MEM) + continue; + + res_size = resource_size(entry->res); + res_start = entry->res->start; + while (res_size > 0) { + /* + * Return failure if we run out of windows in the + * middle. Otherwise, we would end up only partially + * mapping a single resource. + */ + if (ib_iatu_index >= pci->num_ib_windows) { + dev_err(pci->dev, "Cannot add inbound window for region: %pr\n", + entry->res); + return -ENOMEM; + } + + window_size = MIN(pci->region_limit + 1, res_size); + ret = dw_pcie_prog_inbound_atu(pci, ib_iatu_index, + PCIE_TLP_TYPE_MEM_RDWR, res_start, + res_start - entry->offset, window_size); + if (ret) { + dev_err(pci->dev, "Failed to set DMA range %pr\n", + entry->res); + return ret; + } + + ib_iatu_index++; + res_start += window_size; + res_size -= window_size; + } + } + + return 0; +} + +static void dw_pcie_program_presets(struct dw_pcie_rp *pp, enum pci_bus_speed speed) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + u8 lane_eq_offset, lane_reg_size, cap_id; + u8 *presets; + u32 cap; + int i; + + if (speed == PCIE_SPEED_8_0GT) { + presets = (u8 *)pp->presets.eq_presets_8gts; + lane_eq_offset = PCI_SECPCI_LE_CTRL; + cap_id = PCI_EXT_CAP_ID_SECPCI; + /* For data rate of 8 GT/S each lane equalization control is 16bits wide*/ + lane_reg_size = 0x2; + } else if (speed == PCIE_SPEED_16_0GT) { + presets = pp->presets.eq_presets_Ngts[EQ_PRESET_TYPE_16GTS - 1]; + lane_eq_offset = PCI_PL_16GT_LE_CTRL; + cap_id = PCI_EXT_CAP_ID_PL_16GT; + lane_reg_size = 0x1; + } else if (speed == PCIE_SPEED_32_0GT) { + presets = pp->presets.eq_presets_Ngts[EQ_PRESET_TYPE_32GTS - 1]; + lane_eq_offset = PCI_PL_32GT_LE_CTRL; + cap_id = PCI_EXT_CAP_ID_PL_32GT; + lane_reg_size = 0x1; + } else if (speed == PCIE_SPEED_64_0GT) { + presets = pp->presets.eq_presets_Ngts[EQ_PRESET_TYPE_64GTS - 1]; + lane_eq_offset = PCI_PL_64GT_LE_CTRL; + cap_id = PCI_EXT_CAP_ID_PL_64GT; + lane_reg_size = 0x1; + } else { + return; + } + + if (presets[0] == PCI_EQ_RESV) + return; + + cap = dw_pcie_find_ext_capability(pci, cap_id); + if (!cap) + return; + + /* + * Write preset values to the registers byte-by-byte for the given + * number of lanes and register size. + */ + for (i = 0; i < pci->num_lanes * lane_reg_size; i++) + dw_pcie_writeb_dbi(pci, cap + lane_eq_offset + i, presets[i]); +} + +static void dw_pcie_config_presets(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + enum pci_bus_speed speed = pcie_get_link_speed(pci->max_link_speed); + + /* + * Lane equalization settings need to be applied for all data rates the + * controller supports and for all supported lanes. + */ + + if (speed >= PCIE_SPEED_8_0GT) + dw_pcie_program_presets(pp, PCIE_SPEED_8_0GT); + + if (speed >= PCIE_SPEED_16_0GT) + dw_pcie_program_presets(pp, PCIE_SPEED_16_0GT); + + if (speed >= PCIE_SPEED_32_0GT) + dw_pcie_program_presets(pp, PCIE_SPEED_32_0GT); + + if (speed >= PCIE_SPEED_64_0GT) + dw_pcie_program_presets(pp, PCIE_SPEED_64_0GT); +} + +int dw_pcie_setup_rc(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + u32 val; + int ret; + + /* + * Enable DBI read-only registers for writing/updating configuration. + * Write permission gets disabled towards the end of this function. + */ + dw_pcie_dbi_ro_wr_en(pci); + + dw_pcie_setup(pci); + + dw_pcie_msi_init(pp); + + /* Setup RC BARs */ + dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0x00000004); + dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_1, 0x00000000); + + /* Setup interrupt pins */ + val = dw_pcie_readl_dbi(pci, PCI_INTERRUPT_LINE); + val &= 0xffff00ff; + val |= 0x00000100; + dw_pcie_writel_dbi(pci, PCI_INTERRUPT_LINE, val); + + /* Setup bus numbers */ + val = dw_pcie_readl_dbi(pci, PCI_PRIMARY_BUS); + val &= 0xff000000; + val |= 0x00ff0100; + dw_pcie_writel_dbi(pci, PCI_PRIMARY_BUS, val); + + /* Setup command register */ + val = dw_pcie_readl_dbi(pci, PCI_COMMAND); + val &= 0xffff0000; + val |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY | + PCI_COMMAND_MASTER | PCI_COMMAND_SERR; + dw_pcie_writel_dbi(pci, PCI_COMMAND, val); + + dw_pcie_hide_unsupported_l1ss(pci); + + dw_pcie_config_presets(pp); + /* + * If the platform provides its own child bus config accesses, it means + * the platform uses its own address translation component rather than + * ATU, so we should not program the ATU here. + */ + if (pp->bridge->child_ops == &dw_child_pcie_ops || pp->ecam_enabled) { + ret = dw_pcie_iatu_setup(pp); + if (ret) + return ret; + } + + dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0); + + /* Program correct class for RC */ + dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI); + + val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); + val |= PORT_LOGIC_SPEED_CHANGE; + dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); + + dw_pcie_dbi_ro_wr_dis(pci); + + /* + * The iMSI-RX module does not support receiving MSI or MSI-X generated + * by the Root Port. If iMSI-RX is used as the MSI controller, remove + * the MSI and MSI-X capabilities of the Root Port to allow the drivers + * to fall back to INTx instead. + */ + if (pp->use_imsi_rx && !pp->keep_rp_msi_en) { + dw_pcie_remove_capability(pci, PCI_CAP_ID_MSI); + dw_pcie_remove_capability(pci, PCI_CAP_ID_MSIX); + } + + return 0; +} +EXPORT_SYMBOL_GPL(dw_pcie_setup_rc); + +static int dw_pcie_pme_turn_off(struct dw_pcie *pci) +{ + struct dw_pcie_ob_atu_cfg atu = { 0 }; + void __iomem *mem; + int ret; + + if (pci->num_ob_windows <= pci->pp.msg_atu_index) + return -ENOSPC; + + if (!pci->pp.msg_res) + return -ENOSPC; + + atu.code = PCIE_MSG_CODE_PME_TURN_OFF; + atu.routing = PCIE_MSG_TYPE_R_BC; + atu.type = PCIE_TLP_TYPE_MSG; + atu.size = resource_size(pci->pp.msg_res); + atu.index = pci->pp.msg_atu_index; + + atu.parent_bus_addr = pci->pp.msg_res->start - pci->parent_bus_offset; + + ret = dw_pcie_prog_outbound_atu(pci, &atu); + if (ret) + return ret; + + mem = ioremap(pci->pp.msg_res->start, pci->region_align); + if (!mem) + return -ENOMEM; + + /* A dummy write is converted to a Msg TLP */ + writel(0, mem); + + iounmap(mem); + + return 0; +} + +int dw_pcie_suspend_noirq(struct dw_pcie *pci) +{ + bool pme_capable = false; + int ret = 0; + u32 val; + + if (!dw_pcie_link_up(pci)) + goto stop_link; + + if (!pci_host_common_d3cold_possible(pci->pp.bridge, &pme_capable)) + return 0; + + if (pci->pp.ops->pme_turn_off) { + pci->pp.ops->pme_turn_off(&pci->pp); + } else { + ret = dw_pcie_pme_turn_off(pci); + if (ret) + return ret; + } + + /* + * Some SoCs do not support reading the LTSSM register after + * PME_Turn_Off broadcast. For those SoCs, skip waiting for L2/L3 Ready + * state and wait 10ms as recommended in PCIe spec r6.0, sec 5.3.3.2.1. + */ + if (pci->pp.skip_l23_ready) { + mdelay(PCIE_PME_TO_L2_TIMEOUT_US/1000); + goto stop_link; + } + + ret = read_poll_timeout(dw_pcie_get_ltssm, val, + val == DW_PCIE_LTSSM_L2_IDLE || + val <= DW_PCIE_LTSSM_DETECT_WAIT, + PCIE_PME_TO_L2_TIMEOUT_US/10, + PCIE_PME_TO_L2_TIMEOUT_US, false, pci); + if (ret) { + /* + * Failure is non-fatal since spec r7.0, sec 5.3.3.2.1, + * recommends proceeding with L2/L3 sequence even if one or more + * devices do not respond with PME_TO_Ack after 10ms timeout. + */ + dev_warn(pci->dev, "Timeout waiting for L2 entry! LTSSM: 0x%x\n", val); + ret = 0; + } + + /* + * Per PCIe r6.0, sec 5.3.3.2.1, software should wait at least + * 100ns after L2/L3 Ready before turning off refclock and + * main power. This is harmless when no endpoint is connected. + */ + udelay(1); + +stop_link: + /* + * TODO: "pme_capable" means some downstream device is wakeup- + * enabled and is capable of generating PME from D3cold, which + * requires auxiliary power. Instead of always skipping power off + * if PME is supported from D3cold, query the pwrctrl core and skip + * power off only if device supports PME from D3cold and Vaux is + * not supported. + */ + pci->pp.skip_pwrctrl_off = pme_capable; + dw_pcie_stop_link(pci); + if (pci->pp.ops->deinit) + pci->pp.ops->deinit(&pci->pp); + + pci->suspended = true; + + return ret; +} +EXPORT_SYMBOL_GPL(dw_pcie_suspend_noirq); + +int dw_pcie_resume_noirq(struct dw_pcie *pci) +{ + int ret; + + if (!pci->suspended) + return 0; + + if (pci->pp.ops->init) { + ret = pci->pp.ops->init(&pci->pp); + if (ret) { + dev_err(pci->dev, "Host init failed: %d\n", ret); + return ret; + } + } + + dw_pcie_setup_rc(&pci->pp); + + ret = dw_pcie_start_link(pci); + if (ret) + goto err_deinit; + + ret = dw_pcie_wait_for_link(pci); + if (ret == -ETIMEDOUT) + goto err_stop_link; + + if (pci->pp.ops->post_init) + pci->pp.ops->post_init(&pci->pp); + + pci->suspended = false; + + return 0; + +err_stop_link: + dw_pcie_stop_link(pci); + +err_deinit: + if (pci->pp.ops->deinit) + pci->pp.ops->deinit(&pci->pp); + + return ret; +} +EXPORT_SYMBOL_GPL(dw_pcie_resume_noirq); diff --git a/drivers/pci/controller/dwc/pcie-designware-plat.c b/drivers/pci/controller/dwc/pcie-designware-plat.c new file mode 100644 index 000000000..14d94e761 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-designware-plat.c @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe RC driver for Synopsys DesignWare Core + * + * Copyright (C) 2015-2016 Synopsys, Inc. (www.synopsys.com) + * + * Authors: Joao Pinto <Joao.Pinto@synopsys.com> + */ +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/of.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/resource.h> +#include <linux/types.h> + +#include "pcie-designware.h" + +struct dw_plat_pcie { + struct dw_pcie *pci; +}; + +struct dw_plat_pcie_of_data { + enum dw_pcie_device_mode mode; +}; + +static const struct dw_pcie_host_ops dw_plat_pcie_host_ops = { +}; + +static int dw_plat_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, + unsigned int type, u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + switch (type) { + case PCI_IRQ_INTX: + return dw_pcie_ep_raise_intx_irq(ep, func_no); + case PCI_IRQ_MSI: + return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); + case PCI_IRQ_MSIX: + return dw_pcie_ep_raise_msix_irq(ep, func_no, interrupt_num); + default: + dev_err(pci->dev, "UNKNOWN IRQ type\n"); + } + + return 0; +} + +static const struct pci_epc_features dw_plat_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, + .msi_capable = true, + .msix_capable = true, +}; + +static const struct pci_epc_features* +dw_plat_pcie_get_features(struct dw_pcie_ep *ep) +{ + return &dw_plat_pcie_epc_features; +} + +static const struct dw_pcie_ep_ops pcie_ep_ops = { + .raise_irq = dw_plat_pcie_ep_raise_irq, + .get_features = dw_plat_pcie_get_features, +}; + +static int dw_plat_add_pcie_port(struct dw_plat_pcie *dw_plat_pcie, + struct platform_device *pdev) +{ + struct dw_pcie *pci = dw_plat_pcie->pci; + struct dw_pcie_rp *pp = &pci->pp; + struct device *dev = &pdev->dev; + int ret; + + pp->irq = platform_get_irq(pdev, 1); + if (pp->irq < 0) + return pp->irq; + + pp->num_vectors = MAX_MSI_IRQS; + pp->ops = &dw_plat_pcie_host_ops; + + ret = dw_pcie_host_init(pp); + if (ret) { + dev_err(dev, "Failed to initialize host\n"); + return ret; + } + + return 0; +} + +static int dw_plat_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dw_plat_pcie *dw_plat_pcie; + struct dw_pcie *pci; + int ret; + const struct dw_plat_pcie_of_data *data; + enum dw_pcie_device_mode mode; + + data = of_device_get_match_data(dev); + if (!data) + return -EINVAL; + + mode = (enum dw_pcie_device_mode)data->mode; + + dw_plat_pcie = devm_kzalloc(dev, sizeof(*dw_plat_pcie), GFP_KERNEL); + if (!dw_plat_pcie) + return -ENOMEM; + + pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); + if (!pci) + return -ENOMEM; + + pci->dev = dev; + + dw_plat_pcie->pci = pci; + dw_plat_pcie->pci->mode = mode; + + platform_set_drvdata(pdev, dw_plat_pcie); + + switch (dw_plat_pcie->pci->mode) { + case DW_PCIE_RC_TYPE: + if (!IS_ENABLED(CONFIG_PCIE_DW_PLAT_HOST)) + return -ENODEV; + + ret = dw_plat_add_pcie_port(dw_plat_pcie, pdev); + break; + case DW_PCIE_EP_TYPE: + if (!IS_ENABLED(CONFIG_PCIE_DW_PLAT_EP)) + return -ENODEV; + + pci->ep.ops = &pcie_ep_ops; + ret = dw_pcie_ep_init(&pci->ep); + if (ret) + return ret; + + ret = dw_pcie_ep_init_registers(&pci->ep); + if (ret) { + dev_err(dev, "Failed to initialize DWC endpoint registers\n"); + dw_pcie_ep_deinit(&pci->ep); + } + + pci_epc_init_notify(pci->ep.epc); + + break; + default: + dev_err(dev, "INVALID device type %d\n", dw_plat_pcie->pci->mode); + ret = -EINVAL; + break; + } + + return ret; +} + +static const struct dw_plat_pcie_of_data dw_plat_pcie_rc_of_data = { + .mode = DW_PCIE_RC_TYPE, +}; + +static const struct dw_plat_pcie_of_data dw_plat_pcie_ep_of_data = { + .mode = DW_PCIE_EP_TYPE, +}; + +static const struct of_device_id dw_plat_pcie_of_match[] = { + { + .compatible = "snps,dw-pcie", + .data = &dw_plat_pcie_rc_of_data, + }, + { + .compatible = "snps,dw-pcie-ep", + .data = &dw_plat_pcie_ep_of_data, + }, + {}, +}; + +static struct platform_driver dw_plat_pcie_driver = { + .driver = { + .name = "dw-pcie", + .of_match_table = dw_plat_pcie_of_match, + .suppress_bind_attrs = true, + }, + .probe = dw_plat_pcie_probe, +}; +builtin_platform_driver(dw_plat_pcie_driver); diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c new file mode 100644 index 000000000..593388f29 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-designware.c @@ -0,0 +1,1364 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Synopsys DesignWare PCIe host controller driver + * + * Copyright (C) 2013 Samsung Electronics Co., Ltd. + * https://www.samsung.com + * + * Author: Jingoo Han <jg1.han@samsung.com> + */ + +#include <linux/align.h> +#include <linux/bitops.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/dma/edma.h> +#include <linux/gpio/consumer.h> +#include <linux/ioport.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/pcie-dwc.h> +#include <linux/platform_device.h> +#include <linux/sizes.h> +#include <linux/types.h> + +#include "../pci-host-common.h" +#include "../../pci.h" +#include "pcie-designware.h" + +static const char * const dw_pcie_app_clks[DW_PCIE_NUM_APP_CLKS] = { + [DW_PCIE_DBI_CLK] = "dbi", + [DW_PCIE_MSTR_CLK] = "mstr", + [DW_PCIE_SLV_CLK] = "slv", +}; + +static const char * const dw_pcie_core_clks[DW_PCIE_NUM_CORE_CLKS] = { + [DW_PCIE_PIPE_CLK] = "pipe", + [DW_PCIE_CORE_CLK] = "core", + [DW_PCIE_AUX_CLK] = "aux", + [DW_PCIE_REF_CLK] = "ref", +}; + +static const char * const dw_pcie_app_rsts[DW_PCIE_NUM_APP_RSTS] = { + [DW_PCIE_DBI_RST] = "dbi", + [DW_PCIE_MSTR_RST] = "mstr", + [DW_PCIE_SLV_RST] = "slv", +}; + +static const char * const dw_pcie_core_rsts[DW_PCIE_NUM_CORE_RSTS] = { + [DW_PCIE_NON_STICKY_RST] = "non-sticky", + [DW_PCIE_STICKY_RST] = "sticky", + [DW_PCIE_CORE_RST] = "core", + [DW_PCIE_PIPE_RST] = "pipe", + [DW_PCIE_PHY_RST] = "phy", + [DW_PCIE_HOT_RST] = "hot", + [DW_PCIE_PWR_RST] = "pwr", +}; + +static const struct dwc_pcie_vsec_id dwc_pcie_ptm_vsec_ids[] = { + { .vendor_id = PCI_VENDOR_ID_QCOM, /* EP */ + .vsec_id = 0x03, .vsec_rev = 0x1 }, + { .vendor_id = PCI_VENDOR_ID_QCOM, /* RC */ + .vsec_id = 0x04, .vsec_rev = 0x1 }, + { } +}; + +static int dw_pcie_get_clocks(struct dw_pcie *pci) +{ + int i, ret; + + for (i = 0; i < DW_PCIE_NUM_APP_CLKS; i++) + pci->app_clks[i].id = dw_pcie_app_clks[i]; + + for (i = 0; i < DW_PCIE_NUM_CORE_CLKS; i++) + pci->core_clks[i].id = dw_pcie_core_clks[i]; + + ret = devm_clk_bulk_get_optional(pci->dev, DW_PCIE_NUM_APP_CLKS, + pci->app_clks); + if (ret) + return ret; + + return devm_clk_bulk_get_optional(pci->dev, DW_PCIE_NUM_CORE_CLKS, + pci->core_clks); +} + +static int dw_pcie_get_resets(struct dw_pcie *pci) +{ + int i, ret; + + for (i = 0; i < DW_PCIE_NUM_APP_RSTS; i++) + pci->app_rsts[i].id = dw_pcie_app_rsts[i]; + + for (i = 0; i < DW_PCIE_NUM_CORE_RSTS; i++) + pci->core_rsts[i].id = dw_pcie_core_rsts[i]; + + ret = devm_reset_control_bulk_get_optional_shared(pci->dev, + DW_PCIE_NUM_APP_RSTS, + pci->app_rsts); + if (ret) + return ret; + + ret = devm_reset_control_bulk_get_optional_exclusive(pci->dev, + DW_PCIE_NUM_CORE_RSTS, + pci->core_rsts); + if (ret) + return ret; + + pci->pe_rst = devm_gpiod_get_optional(pci->dev, "reset", GPIOD_OUT_HIGH); + if (IS_ERR(pci->pe_rst)) + return PTR_ERR(pci->pe_rst); + + return 0; +} + +int dw_pcie_get_resources(struct dw_pcie *pci) +{ + struct platform_device *pdev = to_platform_device(pci->dev); + struct device_node *np = dev_of_node(pci->dev); + struct resource *res; + int ret; + + if (!pci->dbi_base) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi"); + pci->dbi_base = devm_pci_remap_cfg_resource(pci->dev, res); + if (IS_ERR(pci->dbi_base)) + return PTR_ERR(pci->dbi_base); + pci->dbi_phys_addr = res->start; + } + + /* DBI2 is mainly useful for the endpoint controller */ + if (!pci->dbi_base2) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi2"); + if (res) { + pci->dbi_base2 = devm_pci_remap_cfg_resource(pci->dev, res); + if (IS_ERR(pci->dbi_base2)) + return PTR_ERR(pci->dbi_base2); + } else { + pci->dbi_base2 = pci->dbi_base + SZ_4K; + } + } + + /* For non-unrolled iATU/eDMA platforms this range will be ignored */ + if (!pci->atu_base) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "atu"); + if (res) { + pci->atu_size = resource_size(res); + pci->atu_base = devm_ioremap_resource(pci->dev, res); + if (IS_ERR(pci->atu_base)) + return PTR_ERR(pci->atu_base); + pci->atu_phys_addr = res->start; + } else { + pci->atu_base = pci->dbi_base + DEFAULT_DBI_ATU_OFFSET; + } + } + + /* Set a default value suitable for at most 8 in and 8 out windows */ + if (!pci->atu_size) + pci->atu_size = SZ_4K; + + /* eDMA region can be mapped to a custom base address */ + if (!pci->edma.reg_base) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dma"); + if (res) { + pci->edma.reg_base = devm_ioremap_resource(pci->dev, res); + if (IS_ERR(pci->edma.reg_base)) + return PTR_ERR(pci->edma.reg_base); + pci->edma_reg_phys = res->start; + pci->edma_reg_size = resource_size(res); + } else if (pci->atu_size >= 2 * DEFAULT_DBI_DMA_OFFSET) { + pci->edma.reg_base = pci->atu_base + DEFAULT_DBI_DMA_OFFSET; + pci->edma_reg_phys = pci->atu_phys_addr + DEFAULT_DBI_DMA_OFFSET; + pci->edma_reg_size = pci->atu_size - DEFAULT_DBI_DMA_OFFSET; + } + } + + /* ELBI is an optional resource */ + if (!pci->elbi_base) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "elbi"); + if (res) { + pci->elbi_base = devm_ioremap_resource(pci->dev, res); + if (IS_ERR(pci->elbi_base)) + return PTR_ERR(pci->elbi_base); + } + } + + /* LLDD is supposed to manually switch the clocks and resets state */ + if (dw_pcie_cap_is(pci, REQ_RES)) { + ret = dw_pcie_get_clocks(pci); + if (ret) + return ret; + + ret = dw_pcie_get_resets(pci); + if (ret) + return ret; + } + + if (pci->max_link_speed < 1) + pci->max_link_speed = of_pci_get_max_link_speed(np); + + of_property_read_u32(np, "num-lanes", &pci->num_lanes); + + if (of_property_read_bool(np, "snps,enable-cdm-check")) + dw_pcie_cap_set(pci, CDM_CHECK); + + return 0; +} + +void dw_pcie_version_detect(struct dw_pcie *pci) +{ + u32 ver; + + /* The content of the CSR is zero on DWC PCIe older than v4.70a */ + ver = dw_pcie_readl_dbi(pci, PCIE_VERSION_NUMBER); + if (!ver) + return; + + if (pci->version && pci->version != ver) + dev_warn(pci->dev, "Versions don't match (%08x != %08x)\n", + pci->version, ver); + else + pci->version = ver; + + ver = dw_pcie_readl_dbi(pci, PCIE_VERSION_TYPE); + + if (pci->type && pci->type != ver) + dev_warn(pci->dev, "Types don't match (%08x != %08x)\n", + pci->type, ver); + else + pci->type = ver; +} + +u8 dw_pcie_find_capability(struct dw_pcie *pci, u8 cap) +{ + return PCI_FIND_NEXT_CAP(dw_pcie_read_cfg, PCI_CAPABILITY_LIST, cap, + NULL, pci); +} +EXPORT_SYMBOL_GPL(dw_pcie_find_capability); + +u16 dw_pcie_find_ext_capability(struct dw_pcie *pci, u8 cap) +{ + return PCI_FIND_NEXT_EXT_CAP(dw_pcie_read_cfg, 0, cap, NULL, pci); +} +EXPORT_SYMBOL_GPL(dw_pcie_find_ext_capability); + +void dw_pcie_remove_capability(struct dw_pcie *pci, u8 cap) +{ + u8 cap_pos, pre_pos, next_pos; + u16 reg; + + cap_pos = PCI_FIND_NEXT_CAP(dw_pcie_read_cfg, PCI_CAPABILITY_LIST, cap, + &pre_pos, pci); + if (!cap_pos) + return; + + reg = dw_pcie_readw_dbi(pci, cap_pos); + next_pos = (reg & 0xff00) >> 8; + + dw_pcie_dbi_ro_wr_en(pci); + if (pre_pos == PCI_CAPABILITY_LIST) + dw_pcie_writeb_dbi(pci, PCI_CAPABILITY_LIST, next_pos); + else + dw_pcie_writeb_dbi(pci, pre_pos + 1, next_pos); + dw_pcie_dbi_ro_wr_dis(pci); +} +EXPORT_SYMBOL_GPL(dw_pcie_remove_capability); + +void dw_pcie_remove_ext_capability(struct dw_pcie *pci, u8 cap) +{ + int cap_pos, next_pos, pre_pos; + u32 pre_header, header; + + cap_pos = PCI_FIND_NEXT_EXT_CAP(dw_pcie_read_cfg, 0, cap, &pre_pos, pci); + if (!cap_pos) + return; + + header = dw_pcie_readl_dbi(pci, cap_pos); + + /* + * If the first cap at offset PCI_CFG_SPACE_SIZE is removed, + * only set its capid to zero as it cannot be skipped. + */ + if (cap_pos == PCI_CFG_SPACE_SIZE) { + dw_pcie_dbi_ro_wr_en(pci); + dw_pcie_writel_dbi(pci, cap_pos, header & 0xffff0000); + dw_pcie_dbi_ro_wr_dis(pci); + return; + } + + pre_header = dw_pcie_readl_dbi(pci, pre_pos); + next_pos = PCI_EXT_CAP_NEXT(header); + + dw_pcie_dbi_ro_wr_en(pci); + dw_pcie_writel_dbi(pci, pre_pos, + (pre_header & 0xfffff) | (next_pos << 20)); + dw_pcie_dbi_ro_wr_dis(pci); +} +EXPORT_SYMBOL_GPL(dw_pcie_remove_ext_capability); + +static u16 __dw_pcie_find_vsec_capability(struct dw_pcie *pci, u16 vendor_id, + u16 vsec_id) +{ + u16 vsec = 0; + u32 header; + + if (vendor_id != dw_pcie_readw_dbi(pci, PCI_VENDOR_ID)) + return 0; + + while ((vsec = PCI_FIND_NEXT_EXT_CAP(dw_pcie_read_cfg, vsec, + PCI_EXT_CAP_ID_VNDR, NULL, pci))) { + header = dw_pcie_readl_dbi(pci, vsec + PCI_VNDR_HEADER); + if (PCI_VNDR_HEADER_ID(header) == vsec_id) + return vsec; + } + + return 0; +} + +static u16 dw_pcie_find_vsec_capability(struct dw_pcie *pci, + const struct dwc_pcie_vsec_id *vsec_ids) +{ + const struct dwc_pcie_vsec_id *vid; + u16 vsec; + u32 header; + + for (vid = vsec_ids; vid->vendor_id; vid++) { + vsec = __dw_pcie_find_vsec_capability(pci, vid->vendor_id, + vid->vsec_id); + if (vsec) { + header = dw_pcie_readl_dbi(pci, vsec + PCI_VNDR_HEADER); + if (PCI_VNDR_HEADER_REV(header) == vid->vsec_rev) + return vsec; + } + } + + return 0; +} + +u16 dw_pcie_find_rasdes_capability(struct dw_pcie *pci) +{ + return dw_pcie_find_vsec_capability(pci, dwc_pcie_rasdes_vsec_ids); +} +EXPORT_SYMBOL_GPL(dw_pcie_find_rasdes_capability); + +u16 dw_pcie_find_ptm_capability(struct dw_pcie *pci) +{ + return dw_pcie_find_vsec_capability(pci, dwc_pcie_ptm_vsec_ids); +} +EXPORT_SYMBOL_GPL(dw_pcie_find_ptm_capability); + +int dw_pcie_read(void __iomem *addr, int size, u32 *val) +{ + if (!IS_ALIGNED((uintptr_t)addr, size)) { + *val = 0; + return PCIBIOS_BAD_REGISTER_NUMBER; + } + + if (size == 4) { + *val = readl(addr); + } else if (size == 2) { + *val = readw(addr); + } else if (size == 1) { + *val = readb(addr); + } else { + *val = 0; + return PCIBIOS_BAD_REGISTER_NUMBER; + } + + return PCIBIOS_SUCCESSFUL; +} +EXPORT_SYMBOL_GPL(dw_pcie_read); + +int dw_pcie_write(void __iomem *addr, int size, u32 val) +{ + if (!IS_ALIGNED((uintptr_t)addr, size)) + return PCIBIOS_BAD_REGISTER_NUMBER; + + if (size == 4) + writel(val, addr); + else if (size == 2) + writew(val, addr); + else if (size == 1) + writeb(val, addr); + else + return PCIBIOS_BAD_REGISTER_NUMBER; + + return PCIBIOS_SUCCESSFUL; +} +EXPORT_SYMBOL_GPL(dw_pcie_write); + +u32 dw_pcie_read_dbi(struct dw_pcie *pci, u32 reg, size_t size) +{ + int ret; + u32 val; + + if (pci->ops && pci->ops->read_dbi) + return pci->ops->read_dbi(pci, pci->dbi_base, reg, size); + + ret = dw_pcie_read(pci->dbi_base + reg, size, &val); + if (ret) + dev_err(pci->dev, "Read DBI address failed\n"); + + return val; +} +EXPORT_SYMBOL_GPL(dw_pcie_read_dbi); + +void dw_pcie_write_dbi(struct dw_pcie *pci, u32 reg, size_t size, u32 val) +{ + int ret; + + if (pci->ops && pci->ops->write_dbi) { + pci->ops->write_dbi(pci, pci->dbi_base, reg, size, val); + return; + } + + ret = dw_pcie_write(pci->dbi_base + reg, size, val); + if (ret) + dev_err(pci->dev, "Write DBI address failed\n"); +} +EXPORT_SYMBOL_GPL(dw_pcie_write_dbi); + +void dw_pcie_write_dbi2(struct dw_pcie *pci, u32 reg, size_t size, u32 val) +{ + int ret; + + if (pci->ops && pci->ops->write_dbi2) { + pci->ops->write_dbi2(pci, pci->dbi_base2, reg, size, val); + return; + } + + ret = dw_pcie_write(pci->dbi_base2 + reg, size, val); + if (ret) + dev_err(pci->dev, "write DBI address failed\n"); +} +EXPORT_SYMBOL_GPL(dw_pcie_write_dbi2); + +static inline void __iomem *dw_pcie_select_atu(struct dw_pcie *pci, u32 dir, + u32 index) +{ + if (dw_pcie_cap_is(pci, IATU_UNROLL)) + return pci->atu_base + PCIE_ATU_UNROLL_BASE(dir, index); + + dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT, dir | index); + return pci->atu_base; +} + +static u32 dw_pcie_readl_atu(struct dw_pcie *pci, u32 dir, u32 index, u32 reg) +{ + void __iomem *base; + int ret; + u32 val; + + base = dw_pcie_select_atu(pci, dir, index); + + if (pci->ops && pci->ops->read_dbi) + return pci->ops->read_dbi(pci, base, reg, 4); + + ret = dw_pcie_read(base + reg, 4, &val); + if (ret) + dev_err(pci->dev, "Read ATU address failed\n"); + + return val; +} + +static void dw_pcie_writel_atu(struct dw_pcie *pci, u32 dir, u32 index, + u32 reg, u32 val) +{ + void __iomem *base; + int ret; + + base = dw_pcie_select_atu(pci, dir, index); + + if (pci->ops && pci->ops->write_dbi) { + pci->ops->write_dbi(pci, base, reg, 4, val); + return; + } + + ret = dw_pcie_write(base + reg, 4, val); + if (ret) + dev_err(pci->dev, "Write ATU address failed\n"); +} + +static inline u32 dw_pcie_readl_atu_ob(struct dw_pcie *pci, u32 index, u32 reg) +{ + return dw_pcie_readl_atu(pci, PCIE_ATU_REGION_DIR_OB, index, reg); +} + +static inline void dw_pcie_writel_atu_ob(struct dw_pcie *pci, u32 index, u32 reg, + u32 val) +{ + dw_pcie_writel_atu(pci, PCIE_ATU_REGION_DIR_OB, index, reg, val); +} + +static inline u32 dw_pcie_enable_ecrc(u32 val) +{ + /* + * DesignWare core versions prior to 5.10A have a design issue where the + * 'TD' bit in the Control register-1 of the ATU outbound region acts + * like an override for the ECRC setting, i.e., the presence of TLP + * Digest (ECRC) in the outgoing TLPs is solely determined by this + * bit. This is contrary to the PCIe spec which says that the + * enablement of the ECRC is solely determined by the AER + * registers. + * + * Because of this, even when the ECRC is enabled through AER + * registers, the transactions going through ATU won't have TLP + * Digest as there is no way the PCI core AER code could program + * the TD bit which is specific to the DesignWare core. + * + * The best way to handle this scenario is to program the TD bit + * always. It affects only the traffic from root port to downstream + * devices. + * + * At this point, + * When ECRC is enabled in AER registers, everything works normally + * When ECRC is NOT enabled in AER registers, then, + * on Root Port:- TLP Digest (DWord size) gets appended to each packet + * even through it is not required. Since downstream + * TLPs are mostly for configuration accesses and BAR + * accesses, they are not in critical path and won't + * have much negative effect on the performance. + * on End Point:- TLP Digest is received for some/all the packets coming + * from the root port. TLP Digest is ignored because, + * as per the PCIe Spec r5.0 v1.0 section 2.2.3 + * "TLP Digest Rules", when an endpoint receives TLP + * Digest when its ECRC check functionality is disabled + * in AER registers, received TLP Digest is just ignored. + * Since there is no issue or error reported either side, best way to + * handle the scenario is to program TD bit by default. + */ + + return val | PCIE_ATU_TD; +} + +int dw_pcie_prog_outbound_atu(struct dw_pcie *pci, + const struct dw_pcie_ob_atu_cfg *atu) +{ + u64 parent_bus_addr = atu->parent_bus_addr; + u32 retries, val; + u64 limit_addr; + + if (atu->index >= pci->num_ob_windows) + return -ENOSPC; + + limit_addr = parent_bus_addr + atu->size - 1; + + if ((limit_addr & ~pci->region_limit) != (parent_bus_addr & ~pci->region_limit) || + !IS_ALIGNED(parent_bus_addr, pci->region_align) || + !IS_ALIGNED(atu->pci_addr, pci->region_align) || !atu->size) { + return -EINVAL; + } + + dw_pcie_writel_atu_ob(pci, atu->index, PCIE_ATU_LOWER_BASE, + lower_32_bits(parent_bus_addr)); + dw_pcie_writel_atu_ob(pci, atu->index, PCIE_ATU_UPPER_BASE, + upper_32_bits(parent_bus_addr)); + + dw_pcie_writel_atu_ob(pci, atu->index, PCIE_ATU_LIMIT, + lower_32_bits(limit_addr)); + if (dw_pcie_ver_is_ge(pci, 460A)) + dw_pcie_writel_atu_ob(pci, atu->index, PCIE_ATU_UPPER_LIMIT, + upper_32_bits(limit_addr)); + + dw_pcie_writel_atu_ob(pci, atu->index, PCIE_ATU_LOWER_TARGET, + lower_32_bits(atu->pci_addr)); + dw_pcie_writel_atu_ob(pci, atu->index, PCIE_ATU_UPPER_TARGET, + upper_32_bits(atu->pci_addr)); + + val = atu->type | atu->routing | PCIE_ATU_FUNC_NUM(atu->func_no); + if (upper_32_bits(limit_addr) > upper_32_bits(parent_bus_addr) && + dw_pcie_ver_is_ge(pci, 460A)) + val |= PCIE_ATU_INCREASE_REGION_SIZE; + if (!dw_pcie_ver_is_ge(pci, 510A)) + val = dw_pcie_enable_ecrc(val); + dw_pcie_writel_atu_ob(pci, atu->index, PCIE_ATU_REGION_CTRL1, val); + + val = PCIE_ATU_ENABLE | atu->ctrl2; + if (atu->type == PCIE_TLP_TYPE_MSG) { + /* The data-less messages only for now */ + val |= PCIE_ATU_INHIBIT_PAYLOAD | atu->code; + } + dw_pcie_writel_atu_ob(pci, atu->index, PCIE_ATU_REGION_CTRL2, val); + + /* + * Make sure ATU enable takes effect before any subsequent config + * and I/O accesses. + */ + for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++) { + val = dw_pcie_readl_atu_ob(pci, atu->index, PCIE_ATU_REGION_CTRL2); + if (val & PCIE_ATU_ENABLE) + return 0; + + mdelay(LINK_WAIT_IATU); + } + + dev_err(pci->dev, "Outbound iATU is not being enabled\n"); + + return -ETIMEDOUT; +} + +static inline u32 dw_pcie_readl_atu_ib(struct dw_pcie *pci, u32 index, u32 reg) +{ + return dw_pcie_readl_atu(pci, PCIE_ATU_REGION_DIR_IB, index, reg); +} + +static inline void dw_pcie_writel_atu_ib(struct dw_pcie *pci, u32 index, u32 reg, + u32 val) +{ + dw_pcie_writel_atu(pci, PCIE_ATU_REGION_DIR_IB, index, reg, val); +} + +int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int type, + u64 parent_bus_addr, u64 pci_addr, u64 size) +{ + u64 limit_addr = pci_addr + size - 1; + u32 retries, val; + + if (index >= pci->num_ib_windows) + return -ENOSPC; + + if ((limit_addr & ~pci->region_limit) != (pci_addr & ~pci->region_limit) || + !IS_ALIGNED(parent_bus_addr, pci->region_align) || + !IS_ALIGNED(pci_addr, pci->region_align) || !size) { + return -EINVAL; + } + + dw_pcie_writel_atu_ib(pci, index, PCIE_ATU_LOWER_BASE, + lower_32_bits(pci_addr)); + dw_pcie_writel_atu_ib(pci, index, PCIE_ATU_UPPER_BASE, + upper_32_bits(pci_addr)); + + dw_pcie_writel_atu_ib(pci, index, PCIE_ATU_LIMIT, + lower_32_bits(limit_addr)); + if (dw_pcie_ver_is_ge(pci, 460A)) + dw_pcie_writel_atu_ib(pci, index, PCIE_ATU_UPPER_LIMIT, + upper_32_bits(limit_addr)); + + dw_pcie_writel_atu_ib(pci, index, PCIE_ATU_LOWER_TARGET, + lower_32_bits(parent_bus_addr)); + dw_pcie_writel_atu_ib(pci, index, PCIE_ATU_UPPER_TARGET, + upper_32_bits(parent_bus_addr)); + + val = type; + if (upper_32_bits(limit_addr) > upper_32_bits(pci_addr) && + dw_pcie_ver_is_ge(pci, 460A)) + val |= PCIE_ATU_INCREASE_REGION_SIZE; + dw_pcie_writel_atu_ib(pci, index, PCIE_ATU_REGION_CTRL1, val); + dw_pcie_writel_atu_ib(pci, index, PCIE_ATU_REGION_CTRL2, PCIE_ATU_ENABLE); + + /* + * Make sure ATU enable takes effect before any subsequent config + * and I/O accesses. + */ + for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++) { + val = dw_pcie_readl_atu_ib(pci, index, PCIE_ATU_REGION_CTRL2); + if (val & PCIE_ATU_ENABLE) + return 0; + + mdelay(LINK_WAIT_IATU); + } + + dev_err(pci->dev, "Inbound iATU is not being enabled\n"); + + return -ETIMEDOUT; +} + +int dw_pcie_prog_ep_inbound_atu(struct dw_pcie *pci, u8 func_no, int index, + int type, u64 parent_bus_addr, u8 bar, size_t size) +{ + u32 retries, val; + + if (!IS_ALIGNED(parent_bus_addr, pci->region_align) || + !IS_ALIGNED(parent_bus_addr, size)) + return -EINVAL; + + dw_pcie_writel_atu_ib(pci, index, PCIE_ATU_LOWER_TARGET, + lower_32_bits(parent_bus_addr)); + dw_pcie_writel_atu_ib(pci, index, PCIE_ATU_UPPER_TARGET, + upper_32_bits(parent_bus_addr)); + + dw_pcie_writel_atu_ib(pci, index, PCIE_ATU_REGION_CTRL1, type | + PCIE_ATU_FUNC_NUM(func_no)); + dw_pcie_writel_atu_ib(pci, index, PCIE_ATU_REGION_CTRL2, + PCIE_ATU_ENABLE | PCIE_ATU_FUNC_NUM_MATCH_EN | + PCIE_ATU_BAR_MODE_ENABLE | (bar << 8)); + + /* + * Make sure ATU enable takes effect before any subsequent config + * and I/O accesses. + */ + for (retries = 0; retries < LINK_WAIT_MAX_IATU_RETRIES; retries++) { + val = dw_pcie_readl_atu_ib(pci, index, PCIE_ATU_REGION_CTRL2); + if (val & PCIE_ATU_ENABLE) + return 0; + + mdelay(LINK_WAIT_IATU); + } + + dev_err(pci->dev, "Inbound iATU is not being enabled\n"); + + return -ETIMEDOUT; +} + +void dw_pcie_disable_atu(struct dw_pcie *pci, u32 dir, int index) +{ + dw_pcie_writel_atu(pci, dir, index, PCIE_ATU_REGION_CTRL2, 0); +} + +const char *dw_pcie_ltssm_status_string(enum dw_pcie_ltssm ltssm) +{ + const char *str; + + switch (ltssm) { +#define DW_PCIE_LTSSM_NAME(n) case n: str = #n; break + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_DETECT_QUIET); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_DETECT_ACT); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_POLL_ACTIVE); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_POLL_COMPLIANCE); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_POLL_CONFIG); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_PRE_DETECT_QUIET); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_DETECT_WAIT); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_CFG_LINKWD_START); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_CFG_LINKWD_ACEPT); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_CFG_LANENUM_WAI); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_CFG_LANENUM_ACEPT); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_CFG_COMPLETE); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_CFG_IDLE); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_RCVRY_LOCK); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_RCVRY_SPEED); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_RCVRY_RCVRCFG); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_RCVRY_IDLE); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_L0); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_L0S); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_L123_SEND_EIDLE); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_L1_IDLE); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_L2_IDLE); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_L2_WAKE); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_DISABLED_ENTRY); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_DISABLED_IDLE); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_DISABLED); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_LPBK_ENTRY); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_LPBK_ACTIVE); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_LPBK_EXIT); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_LPBK_EXIT_TIMEOUT); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_HOT_RESET_ENTRY); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_HOT_RESET); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_RCVRY_EQ0); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_RCVRY_EQ1); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_RCVRY_EQ2); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_RCVRY_EQ3); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_L1_1); + DW_PCIE_LTSSM_NAME(DW_PCIE_LTSSM_L1_2); + default: + str = "DW_PCIE_LTSSM_UNKNOWN"; + break; + } + + return str + strlen("DW_PCIE_LTSSM_"); +} + +/** + * dw_pcie_wait_for_link - Wait for the PCIe link to be up + * @pci: DWC instance + * + * Returns: 0 if link is up, -ENODEV if device is not found, -EIO if the device + * is found but not active and -ETIMEDOUT if the link fails to come up for other + * reasons. + */ +int dw_pcie_wait_for_link(struct dw_pcie *pci) +{ + u32 offset, val, ltssm; + int retries; + + /* Check if the link is up or not */ + for (retries = 0; retries < PCIE_LINK_WAIT_MAX_RETRIES; retries++) { + if (dw_pcie_link_up(pci)) + break; + + msleep(PCIE_LINK_WAIT_SLEEP_MS); + } + + if (retries >= PCIE_LINK_WAIT_MAX_RETRIES) { + /* + * If the link is in Detect.Quiet or Detect.Active state, it + * indicates that no device is detected. + */ + ltssm = dw_pcie_get_ltssm(pci); + if (ltssm == DW_PCIE_LTSSM_DETECT_QUIET || + ltssm == DW_PCIE_LTSSM_DETECT_ACT) { + dev_info(pci->dev, "Device not found\n"); + return -ENODEV; + + /* + * If the link is in POLL.{Active/Compliance} state, then the + * device is found to be connected to the bus, but it is not + * active i.e., the device firmware might not yet initialized. + */ + } else if (ltssm == DW_PCIE_LTSSM_POLL_ACTIVE || + ltssm == DW_PCIE_LTSSM_POLL_COMPLIANCE) { + dev_info(pci->dev, "Device found, but not active\n"); + return -EIO; + } + + dev_err(pci->dev, "Link failed to come up. LTSSM: %s\n", + dw_pcie_ltssm_status_string(ltssm)); + return -ETIMEDOUT; + } + + pci_host_common_link_train_delay(pci->max_link_speed); + + offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + val = dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKSTA); + + dev_info(pci->dev, "PCIe Gen.%u x%u link up\n", + FIELD_GET(PCI_EXP_LNKSTA_CLS, val), + FIELD_GET(PCI_EXP_LNKSTA_NLW, val)); + + return 0; +} +EXPORT_SYMBOL_GPL(dw_pcie_wait_for_link); + +bool dw_pcie_link_up(struct dw_pcie *pci) +{ + u32 val; + + if (pci->ops && pci->ops->link_up) + return pci->ops->link_up(pci); + + val = dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG1); + return ((val & PCIE_PORT_DEBUG1_LINK_UP) && + (!(val & PCIE_PORT_DEBUG1_LINK_IN_TRAINING))); +} +EXPORT_SYMBOL_GPL(dw_pcie_link_up); + +void dw_pcie_upconfig_setup(struct dw_pcie *pci) +{ + u32 val; + + val = dw_pcie_readl_dbi(pci, PCIE_PORT_MULTI_LANE_CTRL); + val |= PORT_MLTI_UPCFG_SUPPORT; + dw_pcie_writel_dbi(pci, PCIE_PORT_MULTI_LANE_CTRL, val); +} +EXPORT_SYMBOL_GPL(dw_pcie_upconfig_setup); + +static void dw_pcie_link_set_max_speed(struct dw_pcie *pci) +{ + u32 cap, ctrl2; + enum pci_bus_speed link_speed; + u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + u16 ctrl2_speed; + + cap = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP); + + /* + * Even if the platform doesn't want to limit the maximum link speed, + * just cache the hardware default value so that the vendor drivers can + * use it to do any link specific configuration. + */ + if (pci->max_link_speed < 1) { + pci->max_link_speed = FIELD_GET(PCI_EXP_LNKCAP_SLS, cap); + return; + } + + ctrl2 = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCTL2); + ctrl2 &= ~PCI_EXP_LNKCTL2_TLS; + + link_speed = pcie_get_link_speed(pci->max_link_speed); + ctrl2_speed = pci_bus_speed2lnkctl2(link_speed); + if (ctrl2_speed == 0) { + /* Use hardware capability */ + ctrl2_speed = FIELD_GET(PCI_EXP_LNKCAP_SLS, cap); + ctrl2 &= ~PCI_EXP_LNKCTL2_HASD; + } + + dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCTL2, ctrl2 | ctrl2_speed); + + cap &= ~((u32)PCI_EXP_LNKCAP_SLS); + dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, cap | ctrl2_speed); + +} + +int dw_pcie_link_get_max_link_width(struct dw_pcie *pci) +{ + u8 cap = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + u32 lnkcap = dw_pcie_readl_dbi(pci, cap + PCI_EXP_LNKCAP); + + return FIELD_GET(PCI_EXP_LNKCAP_MLW, lnkcap); +} + +static void dw_pcie_link_set_max_link_width(struct dw_pcie *pci, u32 num_lanes) +{ + u32 lnkcap, lwsc, plc; + u8 cap; + + if (!num_lanes) + return; + + /* Set the number of lanes */ + plc = dw_pcie_readl_dbi(pci, PCIE_PORT_LINK_CONTROL); + plc &= ~PORT_LINK_FAST_LINK_MODE; + plc &= ~PORT_LINK_MODE_MASK; + + /* Set link width speed control register */ + lwsc = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); + lwsc &= ~PORT_LOGIC_LINK_WIDTH_MASK; + lwsc |= PORT_LOGIC_LINK_WIDTH_1_LANES; + switch (num_lanes) { + case 1: + plc |= PORT_LINK_MODE_1_LANES; + break; + case 2: + plc |= PORT_LINK_MODE_2_LANES; + break; + case 4: + plc |= PORT_LINK_MODE_4_LANES; + break; + case 8: + plc |= PORT_LINK_MODE_8_LANES; + break; + case 16: + plc |= PORT_LINK_MODE_16_LANES; + break; + default: + dev_err(pci->dev, "num-lanes %u: invalid value\n", num_lanes); + return; + } + dw_pcie_writel_dbi(pci, PCIE_PORT_LINK_CONTROL, plc); + dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, lwsc); + + cap = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + lnkcap = dw_pcie_readl_dbi(pci, cap + PCI_EXP_LNKCAP); + FIELD_MODIFY(PCI_EXP_LNKCAP_MLW, &lnkcap, num_lanes); + dw_pcie_writel_dbi(pci, cap + PCI_EXP_LNKCAP, lnkcap); +} + +void dw_pcie_iatu_detect(struct dw_pcie *pci) +{ + int max_region, ob, ib; + u32 val, min, dir; + u64 max; + + val = dw_pcie_readl_dbi(pci, PCIE_ATU_VIEWPORT); + if (val == 0xFFFFFFFF) { + dw_pcie_cap_set(pci, IATU_UNROLL); + + max_region = min((int)pci->atu_size / 512, 256); + } else { + pci->atu_base = pci->dbi_base + PCIE_ATU_VIEWPORT_BASE; + pci->atu_size = PCIE_ATU_VIEWPORT_SIZE; + + dw_pcie_writel_dbi(pci, PCIE_ATU_VIEWPORT, 0xFF); + max_region = dw_pcie_readl_dbi(pci, PCIE_ATU_VIEWPORT) + 1; + } + + for (ob = 0; ob < max_region; ob++) { + dw_pcie_writel_atu_ob(pci, ob, PCIE_ATU_LOWER_TARGET, 0x11110000); + val = dw_pcie_readl_atu_ob(pci, ob, PCIE_ATU_LOWER_TARGET); + if (val != 0x11110000) + break; + } + + for (ib = 0; ib < max_region; ib++) { + dw_pcie_writel_atu_ib(pci, ib, PCIE_ATU_LOWER_TARGET, 0x11110000); + val = dw_pcie_readl_atu_ib(pci, ib, PCIE_ATU_LOWER_TARGET); + if (val != 0x11110000) + break; + } + + if (ob) { + dir = PCIE_ATU_REGION_DIR_OB; + } else if (ib) { + dir = PCIE_ATU_REGION_DIR_IB; + } else { + dev_err(pci->dev, "No iATU regions found\n"); + return; + } + + dw_pcie_writel_atu(pci, dir, 0, PCIE_ATU_LIMIT, 0x0); + min = dw_pcie_readl_atu(pci, dir, 0, PCIE_ATU_LIMIT); + + if (dw_pcie_ver_is_ge(pci, 460A)) { + dw_pcie_writel_atu(pci, dir, 0, PCIE_ATU_UPPER_LIMIT, 0xFFFFFFFF); + max = dw_pcie_readl_atu(pci, dir, 0, PCIE_ATU_UPPER_LIMIT); + } else { + max = 0; + } + + pci->num_ob_windows = ob; + pci->num_ib_windows = ib; + pci->region_align = 1 << fls(min); + pci->region_limit = (max << 32) | (SZ_4G - 1); + + dev_info(pci->dev, "iATU: unroll %s, %u ob, %u ib, align %uK, limit %lluG\n", + dw_pcie_cap_is(pci, IATU_UNROLL) ? "T" : "F", + pci->num_ob_windows, pci->num_ib_windows, + pci->region_align / SZ_1K, (pci->region_limit + 1) / SZ_1G); +} + +static u32 dw_pcie_readl_dma(struct dw_pcie *pci, u32 reg) +{ + u32 val = 0; + int ret; + + if (pci->ops && pci->ops->read_dbi) + return pci->ops->read_dbi(pci, pci->edma.reg_base, reg, 4); + + ret = dw_pcie_read(pci->edma.reg_base + reg, 4, &val); + if (ret) + dev_err(pci->dev, "Read DMA address failed\n"); + + return val; +} + +static int dw_pcie_edma_irq_vector(struct device *dev, unsigned int nr) +{ + struct platform_device *pdev = to_platform_device(dev); + char name[6]; + int ret; + + if (nr >= EDMA_MAX_WR_CH + EDMA_MAX_RD_CH) + return -EINVAL; + + ret = platform_get_irq_byname_optional(pdev, "dma"); + if (ret > 0) + return ret; + + snprintf(name, sizeof(name), "dma%u", nr); + + return platform_get_irq_byname_optional(pdev, name); +} + +static struct dw_edma_plat_ops dw_pcie_edma_ops = { + .irq_vector = dw_pcie_edma_irq_vector, +}; + +static void dw_pcie_edma_init_data(struct dw_pcie *pci) +{ + pci->edma.dev = pci->dev; + + if (!pci->edma.ops) + pci->edma.ops = &dw_pcie_edma_ops; + + pci->edma.flags |= DW_EDMA_CHIP_LOCAL; +} + +static int dw_pcie_edma_find_mf(struct dw_pcie *pci) +{ + u32 val; + + /* + * Bail out finding the mapping format if it is already set by the glue + * driver. Also ensure that the edma.reg_base is pointing to a valid + * memory region. + */ + if (pci->edma.mf != EDMA_MF_EDMA_LEGACY) + return pci->edma.reg_base ? 0 : -ENODEV; + + /* + * Indirect eDMA CSRs access has been completely removed since v5.40a + * thus no space is now reserved for the eDMA channels viewport and + * former DMA CTRL register is no longer fixed to FFs. + */ + if (dw_pcie_ver_is_ge(pci, 540A)) + val = 0xFFFFFFFF; + else + val = dw_pcie_readl_dbi(pci, PCIE_DMA_VIEWPORT_BASE + PCIE_DMA_CTRL); + + if (val == 0xFFFFFFFF && pci->edma.reg_base) { + pci->edma.mf = EDMA_MF_EDMA_UNROLL; + } else if (val != 0xFFFFFFFF) { + pci->edma.mf = EDMA_MF_EDMA_LEGACY; + + pci->edma.reg_base = pci->dbi_base + PCIE_DMA_VIEWPORT_BASE; + } else { + return -ENODEV; + } + + return 0; +} + +static int dw_pcie_edma_find_channels(struct dw_pcie *pci) +{ + u32 val; + + /* + * Autodetect the read/write channels count only for non-HDMA platforms. + * HDMA platforms with native CSR mapping doesn't support autodetect, + * so the glue drivers should've passed the valid count already. If not, + * the below sanity check will catch it. + */ + if (pci->edma.mf != EDMA_MF_HDMA_NATIVE) { + val = dw_pcie_readl_dma(pci, PCIE_DMA_CTRL); + + pci->edma.ll_wr_cnt = FIELD_GET(PCIE_DMA_NUM_WR_CHAN, val); + pci->edma.ll_rd_cnt = FIELD_GET(PCIE_DMA_NUM_RD_CHAN, val); + } + + /* Sanity check the channels count if the mapping was incorrect */ + if (!pci->edma.ll_wr_cnt || pci->edma.ll_wr_cnt > EDMA_MAX_WR_CH || + !pci->edma.ll_rd_cnt || pci->edma.ll_rd_cnt > EDMA_MAX_RD_CH) + return -EINVAL; + + return 0; +} + +static int dw_pcie_edma_find_chip(struct dw_pcie *pci) +{ + int ret; + + dw_pcie_edma_init_data(pci); + + ret = dw_pcie_edma_find_mf(pci); + if (ret) + return ret; + + return dw_pcie_edma_find_channels(pci); +} + +static int dw_pcie_edma_irq_verify(struct dw_pcie *pci) +{ + struct platform_device *pdev = to_platform_device(pci->dev); + u16 ch_cnt = pci->edma.ll_wr_cnt + pci->edma.ll_rd_cnt; + char name[15]; + int ret; + + if (pci->edma.nr_irqs > 1) + return pci->edma.nr_irqs != ch_cnt ? -EINVAL : 0; + + ret = platform_get_irq_byname_optional(pdev, "dma"); + if (ret > 0) { + pci->edma.nr_irqs = 1; + return 0; + } + + for (; pci->edma.nr_irqs < ch_cnt; pci->edma.nr_irqs++) { + snprintf(name, sizeof(name), "dma%d", pci->edma.nr_irqs); + + ret = platform_get_irq_byname_optional(pdev, name); + if (ret <= 0) + return -EINVAL; + } + + return 0; +} + +static int dw_pcie_edma_ll_alloc(struct dw_pcie *pci) +{ + struct dw_edma_region *ll; + dma_addr_t paddr; + int i; + + for (i = 0; i < pci->edma.ll_wr_cnt; i++) { + ll = &pci->edma.ll_region_wr[i]; + ll->sz = DMA_LLP_MEM_SIZE; + ll->vaddr.mem = dmam_alloc_coherent(pci->dev, ll->sz, + &paddr, GFP_KERNEL); + if (!ll->vaddr.mem) + return -ENOMEM; + + ll->paddr = paddr; + } + + for (i = 0; i < pci->edma.ll_rd_cnt; i++) { + ll = &pci->edma.ll_region_rd[i]; + ll->sz = DMA_LLP_MEM_SIZE; + ll->vaddr.mem = dmam_alloc_coherent(pci->dev, ll->sz, + &paddr, GFP_KERNEL); + if (!ll->vaddr.mem) + return -ENOMEM; + + ll->paddr = paddr; + } + + return 0; +} + +int dw_pcie_edma_detect(struct dw_pcie *pci) +{ + int ret; + + /* Don't fail if no eDMA was found (for the backward compatibility) */ + ret = dw_pcie_edma_find_chip(pci); + if (ret) + return 0; + + /* Don't fail on the IRQs verification (for the backward compatibility) */ + ret = dw_pcie_edma_irq_verify(pci); + if (ret) { + dev_err(pci->dev, "Invalid eDMA IRQs found\n"); + return 0; + } + + ret = dw_pcie_edma_ll_alloc(pci); + if (ret) { + dev_err(pci->dev, "Couldn't allocate LLP memory\n"); + return ret; + } + + /* Don't fail if the DW eDMA driver can't find the device */ + ret = dw_edma_probe(&pci->edma); + if (ret && ret != -ENODEV) { + dev_err(pci->dev, "Couldn't register eDMA device\n"); + return ret; + } + + dev_info(pci->dev, "eDMA: unroll %s, %hu wr, %hu rd\n", + pci->edma.mf == EDMA_MF_EDMA_UNROLL ? "T" : "F", + pci->edma.ll_wr_cnt, pci->edma.ll_rd_cnt); + + return 0; +} + +void dw_pcie_edma_remove(struct dw_pcie *pci) +{ + dw_edma_remove(&pci->edma); +} + +void dw_pcie_hide_unsupported_l1ss(struct dw_pcie *pci) +{ + u16 l1ss; + u32 l1ss_cap; + + if (pci->l1ss_support) + return; + + l1ss = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS); + if (!l1ss) + return; + + /* + * Unless the driver claims "l1ss_support", don't advertise L1 PM + * Substates because they require CLKREQ# and possibly other + * device-specific configuration. + */ + l1ss_cap = dw_pcie_readl_dbi(pci, l1ss + PCI_L1SS_CAP); + l1ss_cap &= ~(PCI_L1SS_CAP_PCIPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_1 | + PCI_L1SS_CAP_PCIPM_L1_2 | PCI_L1SS_CAP_ASPM_L1_2 | + PCI_L1SS_CAP_L1_PM_SS); + dw_pcie_writel_dbi(pci, l1ss + PCI_L1SS_CAP, l1ss_cap); +} + +/* TODO: Need to handle multi Root Ports */ +void dw_pcie_program_t_power_on(struct dw_pcie *pci, u32 t_power_on) +{ + u8 scale, value; + u16 offset; + u32 val; + + if (!t_power_on) + return; + + offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS); + if (!offset) + return; + + pcie_encode_t_power_on(t_power_on, &scale, &value); + + dw_pcie_dbi_ro_wr_en(pci); + + val = dw_pcie_readl_dbi(pci, offset + PCI_L1SS_CAP); + val &= ~(PCI_L1SS_CAP_P_PWR_ON_SCALE | PCI_L1SS_CAP_P_PWR_ON_VALUE); + FIELD_MODIFY(PCI_L1SS_CAP_P_PWR_ON_SCALE, &val, scale); + FIELD_MODIFY(PCI_L1SS_CAP_P_PWR_ON_VALUE, &val, value); + + dw_pcie_writel_dbi(pci, offset + PCI_L1SS_CAP, val); + + dw_pcie_dbi_ro_wr_dis(pci); +} + +void dw_pcie_setup(struct dw_pcie *pci) +{ + u32 val; + + dw_pcie_link_set_max_speed(pci); + + /* Configure Gen1 N_FTS */ + if (pci->n_fts[0]) { + val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR); + val &= ~(PORT_AFR_N_FTS_MASK | PORT_AFR_CC_N_FTS_MASK); + val |= PORT_AFR_N_FTS(pci->n_fts[0]); + val |= PORT_AFR_CC_N_FTS(pci->n_fts[0]); + dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val); + } + + /* Configure Gen2+ N_FTS */ + if (pci->n_fts[1]) { + val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); + val &= ~PORT_LOGIC_N_FTS_MASK; + val |= pci->n_fts[1]; + dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); + } + + if (dw_pcie_cap_is(pci, CDM_CHECK)) { + val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS); + val |= PCIE_PL_CHK_REG_CHK_REG_CONTINUOUS | + PCIE_PL_CHK_REG_CHK_REG_START; + dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, val); + } + + val = dw_pcie_readl_dbi(pci, PCIE_PORT_LINK_CONTROL); + val &= ~PORT_LINK_FAST_LINK_MODE; + val |= PORT_LINK_DLL_LINK_EN; + dw_pcie_writel_dbi(pci, PCIE_PORT_LINK_CONTROL, val); + + dw_pcie_link_set_max_link_width(pci, pci->num_lanes); +} + +resource_size_t dw_pcie_parent_bus_offset(struct dw_pcie *pci, + const char *reg_name, + resource_size_t cpu_phys_addr) +{ + struct device *dev = pci->dev; + struct device_node *np = dev->of_node; + int index; + u64 reg_addr, fixup_addr; + u64 (*fixup)(struct dw_pcie *pcie, u64 cpu_addr); + + /* Look up reg_name address on parent bus */ + index = of_property_match_string(np, "reg-names", reg_name); + + if (index < 0) { + dev_err(dev, "No %s in devicetree \"reg\" property\n", reg_name); + return 0; + } + + of_property_read_reg(np, index, ®_addr, NULL); + + fixup = pci->ops ? pci->ops->cpu_addr_fixup : NULL; + if (fixup) { + fixup_addr = fixup(pci, cpu_phys_addr); + if (reg_addr == fixup_addr) { + dev_info(dev, "%s reg[%d] %#010llx == %#010llx == fixup(cpu %#010llx); %ps is redundant with this devicetree\n", + reg_name, index, reg_addr, fixup_addr, + (unsigned long long) cpu_phys_addr, fixup); + } else { + dev_warn(dev, "%s reg[%d] %#010llx != %#010llx == fixup(cpu %#010llx); devicetree is broken\n", + reg_name, index, reg_addr, fixup_addr, + (unsigned long long) cpu_phys_addr); + reg_addr = fixup_addr; + } + + return cpu_phys_addr - reg_addr; + } + + if (pci->use_parent_dt_ranges) { + + /* + * This platform once had a fixup, presumably because it + * translates between CPU and PCI controller addresses. + * Log a note if devicetree didn't describe a translation. + */ + if (reg_addr == cpu_phys_addr) + dev_info(dev, "%s reg[%d] %#010llx == cpu %#010llx\n; no fixup was ever needed for this devicetree\n", + reg_name, index, reg_addr, + (unsigned long long) cpu_phys_addr); + } else { + if (reg_addr != cpu_phys_addr) { + dev_warn(dev, "%s reg[%d] %#010llx != cpu %#010llx; no fixup and devicetree \"ranges\" is broken, assuming no translation\n", + reg_name, index, reg_addr, + (unsigned long long) cpu_phys_addr); + return 0; + } + } + + return cpu_phys_addr - reg_addr; +} diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h new file mode 100644 index 000000000..0735ae940 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-designware.h @@ -0,0 +1,1016 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Synopsys DesignWare PCIe host controller driver + * + * Copyright (C) 2013 Samsung Electronics Co., Ltd. + * https://www.samsung.com + * + * Author: Jingoo Han <jg1.han@samsung.com> + */ + +#ifndef _PCIE_DESIGNWARE_H +#define _PCIE_DESIGNWARE_H + +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/clk.h> +#include <linux/dma-mapping.h> +#include <linux/dma/edma.h> +#include <linux/gpio/consumer.h> +#include <linux/irq.h> +#include <linux/msi.h> +#include <linux/pci.h> +#include <linux/pci-ecam.h> +#include <linux/reset.h> + +#include <linux/pci-epc.h> +#include <linux/pci-epf.h> + +#include "../../pci.h" + +/* DWC PCIe IP-core versions (native support since v4.70a) */ +#define DW_PCIE_VER_365A 0x3336352a +#define DW_PCIE_VER_460A 0x3436302a +#define DW_PCIE_VER_470A 0x3437302a +#define DW_PCIE_VER_480A 0x3438302a +#define DW_PCIE_VER_490A 0x3439302a +#define DW_PCIE_VER_500A 0x3530302a +#define DW_PCIE_VER_510A 0x3531302a +#define DW_PCIE_VER_520A 0x3532302a +#define DW_PCIE_VER_540A 0x3534302a +#define DW_PCIE_VER_562A 0x3536322a + +#define __dw_pcie_ver_cmp(_pci, _ver, _op) \ + ((_pci)->version _op DW_PCIE_VER_ ## _ver) + +#define dw_pcie_ver_is(_pci, _ver) __dw_pcie_ver_cmp(_pci, _ver, ==) + +#define dw_pcie_ver_is_ge(_pci, _ver) __dw_pcie_ver_cmp(_pci, _ver, >=) + +#define dw_pcie_ver_type_is(_pci, _ver, _type) \ + (__dw_pcie_ver_cmp(_pci, _ver, ==) && \ + __dw_pcie_ver_cmp(_pci, TYPE_ ## _type, ==)) + +#define dw_pcie_ver_type_is_ge(_pci, _ver, _type) \ + (__dw_pcie_ver_cmp(_pci, _ver, ==) && \ + __dw_pcie_ver_cmp(_pci, TYPE_ ## _type, >=)) + +/* DWC PCIe controller capabilities */ +#define DW_PCIE_CAP_REQ_RES 0 +#define DW_PCIE_CAP_IATU_UNROLL 1 +#define DW_PCIE_CAP_CDM_CHECK 2 + +#define dw_pcie_cap_is(_pci, _cap) \ + test_bit(DW_PCIE_CAP_ ## _cap, &(_pci)->caps) + +#define dw_pcie_cap_set(_pci, _cap) \ + set_bit(DW_PCIE_CAP_ ## _cap, &(_pci)->caps) + +/* Parameters for the waiting for iATU enabled routine */ +#define LINK_WAIT_MAX_IATU_RETRIES 5 +#define LINK_WAIT_IATU 9 + +/* Synopsys-specific PCIe configuration registers */ +#define PCIE_PORT_FORCE 0x708 +/* Bit[7:0] LINK_NUM: Link Number. Not used for endpoint */ +#define PORT_LINK_NUM_MASK GENMASK(7, 0) +#define PORT_FORCE_DO_DESKEW_FOR_SRIS BIT(23) + +#define PCIE_PORT_AFR 0x70C +#define PORT_AFR_N_FTS_MASK GENMASK(15, 8) +#define PORT_AFR_N_FTS(n) FIELD_PREP(PORT_AFR_N_FTS_MASK, n) +#define PORT_AFR_CC_N_FTS_MASK GENMASK(23, 16) +#define PORT_AFR_CC_N_FTS(n) FIELD_PREP(PORT_AFR_CC_N_FTS_MASK, n) +#define PORT_AFR_ENTER_ASPM BIT(30) +#define PORT_AFR_L0S_ENTRANCE_LAT_SHIFT 24 +#define PORT_AFR_L0S_ENTRANCE_LAT_MASK GENMASK(26, 24) +#define PORT_AFR_L1_ENTRANCE_LAT_SHIFT 27 +#define PORT_AFR_L1_ENTRANCE_LAT_MASK GENMASK(29, 27) + +#define PCIE_PORT_LINK_CONTROL 0x710 +#define PORT_LINK_DLL_LINK_EN BIT(5) +#define PORT_LINK_FAST_LINK_MODE BIT(7) +#define PORT_LINK_MODE_MASK GENMASK(21, 16) +#define PORT_LINK_MODE(n) FIELD_PREP(PORT_LINK_MODE_MASK, n) +#define PORT_LINK_MODE_1_LANES PORT_LINK_MODE(0x1) +#define PORT_LINK_MODE_2_LANES PORT_LINK_MODE(0x3) +#define PORT_LINK_MODE_4_LANES PORT_LINK_MODE(0x7) +#define PORT_LINK_MODE_8_LANES PORT_LINK_MODE(0xf) +#define PORT_LINK_MODE_16_LANES PORT_LINK_MODE(0x1f) + +#define PCIE_PORT_LANE_SKEW 0x714 +#define PORT_LANE_SKEW_INSERT_MASK GENMASK(23, 0) + +/* + * PCIE_TIMER_CTRL_MAX_FUNC_NUM: Timer Control and Max Function Number + * Register. + * + * This register holds the ack frequency, latency, replay, fast link + * scaling timers, and max function number values. + * + * Bit[30:29] FAST_LINK_SCALING_FACTOR: Fast Link Timer Scaling Factor. + * 0x0 (SF_1024): Scaling Factor is 1024 (1ms is 1us). + * When the LTSSM is in Config or L12 Entry State, 1ms + * timer is 2us, 2ms timer is 4us and 3ms timer is 6us. + * 0x1 (SF_256): Scaling Factor is 256 (1ms is 4us) + * 0x2 (SF_64): Scaling Factor is 64 (1ms is 16us) + * 0x3 (SF_16): Scaling Factor is 16 (1ms is 64us) + */ +#define PCIE_TIMER_CTRL_MAX_FUNC_NUM 0x718 +#define PORT_FLT_SF_MASK GENMASK(30, 29) +#define PORT_FLT_SF_VAL_1024 0x0 +#define PORT_FLT_SF_VAL_256 0x1 +#define PORT_FLT_SF_VAL_64 0x2 +#define PORT_FLT_SF_VAL_16 0x3 + +#define PCIE_PORT_DEBUG0 0x728 +#define PORT_LOGIC_LTSSM_STATE_MASK 0x3f +#define PORT_LOGIC_LTSSM_STATE_L0 0x11 +#define PCIE_PORT_DEBUG1 0x72C +#define PCIE_PORT_DEBUG1_LINK_UP BIT(4) +#define PCIE_PORT_DEBUG1_LINK_IN_TRAINING BIT(29) + +#define PCIE_LINK_WIDTH_SPEED_CONTROL 0x80C +#define PORT_LOGIC_N_FTS_MASK GENMASK(7, 0) +#define PORT_LOGIC_SPEED_CHANGE BIT(17) +#define PORT_LOGIC_LINK_WIDTH_MASK GENMASK(12, 8) +#define PORT_LOGIC_LINK_WIDTH(n) FIELD_PREP(PORT_LOGIC_LINK_WIDTH_MASK, n) +#define PORT_LOGIC_LINK_WIDTH_1_LANES PORT_LOGIC_LINK_WIDTH(0x1) +#define PORT_LOGIC_LINK_WIDTH_2_LANES PORT_LOGIC_LINK_WIDTH(0x2) +#define PORT_LOGIC_LINK_WIDTH_4_LANES PORT_LOGIC_LINK_WIDTH(0x4) +#define PORT_LOGIC_LINK_WIDTH_8_LANES PORT_LOGIC_LINK_WIDTH(0x8) + +#define PCIE_MSI_ADDR_LO 0x820 +#define PCIE_MSI_ADDR_HI 0x824 +#define PCIE_MSI_INTR0_ENABLE 0x828 +#define PCIE_MSI_INTR0_MASK 0x82C +#define PCIE_MSI_INTR0_STATUS 0x830 + +#define GEN3_RELATED_OFF 0x890 +#define GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL BIT(0) +#define GEN3_RELATED_OFF_EQ_PHASE_2_3 BIT(9) +#define GEN3_RELATED_OFF_RXEQ_RGRDLESS_RXTS BIT(13) +#define GEN3_RELATED_OFF_GEN3_EQ_DISABLE BIT(16) +#define GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT 24 +#define GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK GENMASK(25, 24) + +#define GEN3_EQ_CONTROL_OFF 0x8A8 +#define GEN3_EQ_CONTROL_OFF_FB_MODE GENMASK(3, 0) +#define GEN3_EQ_CONTROL_OFF_PHASE23_EXIT_MODE BIT(4) +#define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC GENMASK(23, 8) +#define GEN3_EQ_CONTROL_OFF_FOM_INC_INITIAL_EVAL BIT(24) + +#define GEN3_EQ_FB_MODE_DIR_CHANGE_OFF 0x8AC +#define GEN3_EQ_FMDC_T_MIN_PHASE23 GENMASK(4, 0) +#define GEN3_EQ_FMDC_N_EVALS GENMASK(9, 5) +#define GEN3_EQ_FMDC_MAX_PRE_CURSOR_DELTA GENMASK(13, 10) +#define GEN3_EQ_FMDC_MAX_POST_CURSOR_DELTA GENMASK(17, 14) + +#define COHERENCY_CONTROL_1_OFF 0x8E0 +#define CFG_MEMTYPE_BOUNDARY_LOW_ADDR_MASK GENMASK(31, 2) +#define CFG_MEMTYPE_VALUE BIT(0) + +#define COHERENCY_CONTROL_2_OFF 0x8E4 +#define COHERENCY_CONTROL_3_OFF 0x8E8 + +#define PCIE_PORT_MULTI_LANE_CTRL 0x8C0 +#define PORT_MLTI_UPCFG_SUPPORT BIT(7) + +#define PCIE_VERSION_NUMBER 0x8F8 +#define PCIE_VERSION_TYPE 0x8FC + +/* + * iATU inbound and outbound windows CSRs. Before the IP-core v4.80a each + * iATU region CSRs had been indirectly accessible by means of the dedicated + * viewport selector. The iATU/eDMA CSRs space was re-designed in DWC PCIe + * v4.80a in a way so the viewport was unrolled into the directly accessible + * iATU/eDMA CSRs space. + */ +#define PCIE_ATU_VIEWPORT 0x900 +#define PCIE_ATU_REGION_DIR_IB BIT(31) +#define PCIE_ATU_REGION_DIR_OB 0 +#define PCIE_ATU_VIEWPORT_BASE 0x904 +#define PCIE_ATU_UNROLL_BASE(dir, index) \ + (((index) << 9) | ((dir == PCIE_ATU_REGION_DIR_IB) ? BIT(8) : 0)) +#define PCIE_ATU_VIEWPORT_SIZE 0x2C +#define PCIE_ATU_REGION_CTRL1 0x000 +#define PCIE_ATU_INCREASE_REGION_SIZE BIT(13) +#define PCIE_ATU_TD BIT(8) +#define PCIE_ATU_FUNC_NUM(pf) ((pf) << 20) +#define PCIE_ATU_REGION_CTRL2 0x004 +#define PCIE_ATU_ENABLE BIT(31) +#define PCIE_ATU_BAR_MODE_ENABLE BIT(30) +#define PCIE_ATU_CFG_SHIFT_MODE_ENABLE BIT(28) +#define PCIE_ATU_INHIBIT_PAYLOAD BIT(22) +#define PCIE_ATU_FUNC_NUM_MATCH_EN BIT(19) +#define PCIE_ATU_LOWER_BASE 0x008 +#define PCIE_ATU_UPPER_BASE 0x00C +#define PCIE_ATU_LIMIT 0x010 +#define PCIE_ATU_LOWER_TARGET 0x014 +#define PCIE_ATU_BUS(x) FIELD_PREP(GENMASK(31, 24), x) +#define PCIE_ATU_DEV(x) FIELD_PREP(GENMASK(23, 19), x) +#define PCIE_ATU_FUNC(x) FIELD_PREP(GENMASK(18, 16), x) +#define PCIE_ATU_UPPER_TARGET 0x018 +#define PCIE_ATU_UPPER_LIMIT 0x020 + +#define PCIE_MISC_CONTROL_1_OFF 0x8BC +#define PCIE_DBI_RO_WR_EN BIT(0) + +#define PCIE_MSIX_DOORBELL 0x948 +#define PCIE_MSIX_DOORBELL_PF_SHIFT 24 + +/* + * eDMA CSRs. DW PCIe IP-core v4.70a and older had the eDMA registers accessible + * over the Port Logic registers space. Afterwards the unrolled mapping was + * introduced so eDMA and iATU could be accessed via a dedicated registers + * space. + */ +#define PCIE_DMA_VIEWPORT_BASE 0x970 +#define PCIE_DMA_UNROLL_BASE 0x80000 +#define PCIE_DMA_CTRL 0x008 +#define PCIE_DMA_NUM_WR_CHAN GENMASK(3, 0) +#define PCIE_DMA_NUM_RD_CHAN GENMASK(19, 16) + +#define PCIE_PL_CHK_REG_CONTROL_STATUS 0xB20 +#define PCIE_PL_CHK_REG_CHK_REG_START BIT(0) +#define PCIE_PL_CHK_REG_CHK_REG_CONTINUOUS BIT(1) +#define PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR BIT(16) +#define PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR BIT(17) +#define PCIE_PL_CHK_REG_CHK_REG_COMPLETE BIT(18) + +#define PCIE_PL_CHK_REG_ERR_ADDR 0xB28 + +/* + * 16.0 GT/s (Gen 4) lane margining register definitions + */ +#define GEN4_LANE_MARGINING_1_OFF 0xB80 +#define MARGINING_MAX_VOLTAGE_OFFSET GENMASK(29, 24) +#define MARGINING_NUM_VOLTAGE_STEPS GENMASK(22, 16) +#define MARGINING_MAX_TIMING_OFFSET GENMASK(13, 8) +#define MARGINING_NUM_TIMING_STEPS GENMASK(5, 0) + +#define GEN4_LANE_MARGINING_2_OFF 0xB84 +#define MARGINING_IND_ERROR_SAMPLER BIT(28) +#define MARGINING_SAMPLE_REPORTING_METHOD BIT(27) +#define MARGINING_IND_LEFT_RIGHT_TIMING BIT(26) +#define MARGINING_IND_UP_DOWN_VOLTAGE BIT(25) +#define MARGINING_VOLTAGE_SUPPORTED BIT(24) +#define MARGINING_MAXLANES GENMASK(20, 16) +#define MARGINING_SAMPLE_RATE_TIMING GENMASK(13, 8) +#define MARGINING_SAMPLE_RATE_VOLTAGE GENMASK(5, 0) +/* + * iATU Unroll-specific register definitions + * From 4.80 core version the address translation will be made by unroll + */ +#define PCIE_ATU_UNR_REGION_CTRL1 0x00 +#define PCIE_ATU_UNR_REGION_CTRL2 0x04 +#define PCIE_ATU_UNR_LOWER_BASE 0x08 +#define PCIE_ATU_UNR_UPPER_BASE 0x0C +#define PCIE_ATU_UNR_LOWER_LIMIT 0x10 +#define PCIE_ATU_UNR_LOWER_TARGET 0x14 +#define PCIE_ATU_UNR_UPPER_TARGET 0x18 +#define PCIE_ATU_UNR_UPPER_LIMIT 0x20 + +/* + * RAS-DES register definitions + */ +#define PCIE_RAS_DES_EVENT_COUNTER_CONTROL 0x8 +#define EVENT_COUNTER_ALL_CLEAR 0x3 +#define EVENT_COUNTER_ENABLE_ALL 0x7 +#define EVENT_COUNTER_ENABLE_SHIFT 2 +#define EVENT_COUNTER_EVENT_SEL_MASK GENMASK(7, 0) +#define EVENT_COUNTER_EVENT_SEL_SHIFT 16 +#define EVENT_COUNTER_EVENT_Tx_L0S 0x2 +#define EVENT_COUNTER_EVENT_Rx_L0S 0x3 +#define EVENT_COUNTER_EVENT_L1 0x5 +#define EVENT_COUNTER_EVENT_L1_1 0x7 +#define EVENT_COUNTER_EVENT_L1_2 0x8 +#define EVENT_COUNTER_GROUP_SEL_SHIFT 24 +#define EVENT_COUNTER_GROUP_5 0x5 + +#define PCIE_RAS_DES_EVENT_COUNTER_DATA 0xc + +/* PTM register definitions */ +#define PTM_RES_REQ_CTRL 0x8 +#define PTM_RES_CCONTEXT_VALID BIT(0) +#define PTM_REQ_AUTO_UPDATE_ENABLED BIT(0) +#define PTM_REQ_START_UPDATE BIT(1) + +#define PTM_LOCAL_LSB 0x10 +#define PTM_LOCAL_MSB 0x14 +#define PTM_T1_T2_LSB 0x18 +#define PTM_T1_T2_MSB 0x1c +#define PTM_T3_T4_LSB 0x28 +#define PTM_T3_T4_MSB 0x2c +#define PTM_MASTER_LSB 0x38 +#define PTM_MASTER_MSB 0x3c + +/* + * The default address offset between dbi_base and atu_base. Root controller + * drivers are not required to initialize atu_base if the offset matches this + * default; the driver core automatically derives atu_base from dbi_base using + * this offset, if atu_base not set. + */ +#define DEFAULT_DBI_ATU_OFFSET (0x3 << 20) +#define DEFAULT_DBI_DMA_OFFSET PCIE_DMA_UNROLL_BASE + +#define MAX_MSI_IRQS 256 +#define MAX_MSI_IRQS_PER_CTRL 32 +#define MAX_MSI_CTRLS (MAX_MSI_IRQS / MAX_MSI_IRQS_PER_CTRL) +#define MSI_REG_CTRL_BLOCK_SIZE 12 +#define MSI_DEF_NUM_VECTORS 32 + +/* Maximum number of inbound/outbound iATUs */ +#define MAX_IATU_IN 256 +#define MAX_IATU_OUT 256 + +/* Default eDMA LLP memory size */ +#define DMA_LLP_MEM_SIZE PAGE_SIZE + +/* Common struct pci_epc_feature bits among DWC EP glue drivers */ +#define DWC_EPC_COMMON_FEATURES .dynamic_inbound_mapping = true, \ + .subrange_mapping = true + +struct dw_pcie; +struct dw_pcie_rp; +struct dw_pcie_ep; + +enum dw_pcie_device_mode { + DW_PCIE_UNKNOWN_TYPE, + DW_PCIE_EP_TYPE, + DW_PCIE_LEG_EP_TYPE, + DW_PCIE_RC_TYPE, +}; + +enum dw_pcie_app_clk { + DW_PCIE_DBI_CLK, + DW_PCIE_MSTR_CLK, + DW_PCIE_SLV_CLK, + DW_PCIE_NUM_APP_CLKS +}; + +enum dw_pcie_core_clk { + DW_PCIE_PIPE_CLK, + DW_PCIE_CORE_CLK, + DW_PCIE_AUX_CLK, + DW_PCIE_REF_CLK, + DW_PCIE_NUM_CORE_CLKS +}; + +enum dw_pcie_app_rst { + DW_PCIE_DBI_RST, + DW_PCIE_MSTR_RST, + DW_PCIE_SLV_RST, + DW_PCIE_NUM_APP_RSTS +}; + +enum dw_pcie_core_rst { + DW_PCIE_NON_STICKY_RST, + DW_PCIE_STICKY_RST, + DW_PCIE_CORE_RST, + DW_PCIE_PIPE_RST, + DW_PCIE_PHY_RST, + DW_PCIE_HOT_RST, + DW_PCIE_PWR_RST, + DW_PCIE_NUM_CORE_RSTS +}; + +enum dw_pcie_ltssm { + /* Need to align with PCIE_PORT_DEBUG0 bits 0:5 */ + DW_PCIE_LTSSM_DETECT_QUIET = 0x0, + DW_PCIE_LTSSM_DETECT_ACT = 0x1, + DW_PCIE_LTSSM_POLL_ACTIVE = 0x2, + DW_PCIE_LTSSM_POLL_COMPLIANCE = 0x3, + DW_PCIE_LTSSM_POLL_CONFIG = 0x4, + DW_PCIE_LTSSM_PRE_DETECT_QUIET = 0x5, + DW_PCIE_LTSSM_DETECT_WAIT = 0x6, + DW_PCIE_LTSSM_CFG_LINKWD_START = 0x7, + DW_PCIE_LTSSM_CFG_LINKWD_ACEPT = 0x8, + DW_PCIE_LTSSM_CFG_LANENUM_WAI = 0x9, + DW_PCIE_LTSSM_CFG_LANENUM_ACEPT = 0xa, + DW_PCIE_LTSSM_CFG_COMPLETE = 0xb, + DW_PCIE_LTSSM_CFG_IDLE = 0xc, + DW_PCIE_LTSSM_RCVRY_LOCK = 0xd, + DW_PCIE_LTSSM_RCVRY_SPEED = 0xe, + DW_PCIE_LTSSM_RCVRY_RCVRCFG = 0xf, + DW_PCIE_LTSSM_RCVRY_IDLE = 0x10, + DW_PCIE_LTSSM_L0 = 0x11, + DW_PCIE_LTSSM_L0S = 0x12, + DW_PCIE_LTSSM_L123_SEND_EIDLE = 0x13, + DW_PCIE_LTSSM_L1_IDLE = 0x14, + DW_PCIE_LTSSM_L2_IDLE = 0x15, + DW_PCIE_LTSSM_L2_WAKE = 0x16, + DW_PCIE_LTSSM_DISABLED_ENTRY = 0x17, + DW_PCIE_LTSSM_DISABLED_IDLE = 0x18, + DW_PCIE_LTSSM_DISABLED = 0x19, + DW_PCIE_LTSSM_LPBK_ENTRY = 0x1a, + DW_PCIE_LTSSM_LPBK_ACTIVE = 0x1b, + DW_PCIE_LTSSM_LPBK_EXIT = 0x1c, + DW_PCIE_LTSSM_LPBK_EXIT_TIMEOUT = 0x1d, + DW_PCIE_LTSSM_HOT_RESET_ENTRY = 0x1e, + DW_PCIE_LTSSM_HOT_RESET = 0x1f, + DW_PCIE_LTSSM_RCVRY_EQ0 = 0x20, + DW_PCIE_LTSSM_RCVRY_EQ1 = 0x21, + DW_PCIE_LTSSM_RCVRY_EQ2 = 0x22, + DW_PCIE_LTSSM_RCVRY_EQ3 = 0x23, + + /* Vendor glue drivers provide pseudo L1 substates from get_ltssm() */ + DW_PCIE_LTSSM_L1_1 = 0x141, + DW_PCIE_LTSSM_L1_2 = 0x142, + + DW_PCIE_LTSSM_UNKNOWN = 0xFFFFFFFF, +}; + +struct dw_pcie_ob_atu_cfg { + int index; + int type; + u8 func_no; + u8 code; + u8 routing; + u32 ctrl2; + u64 parent_bus_addr; + u64 pci_addr; + u64 size; +}; + +struct dw_pcie_host_ops { + int (*init)(struct dw_pcie_rp *pp); + void (*deinit)(struct dw_pcie_rp *pp); + void (*post_init)(struct dw_pcie_rp *pp); + int (*msi_init)(struct dw_pcie_rp *pp); + void (*pme_turn_off)(struct dw_pcie_rp *pp); +}; + +struct dw_pcie_rp { + bool use_imsi_rx:1; + bool keep_rp_msi_en:1; + bool cfg0_io_shared:1; + u64 cfg0_base; + void __iomem *va_cfg0_base; + u32 cfg0_size; + resource_size_t io_base; + phys_addr_t io_bus_addr; + u32 io_size; + int irq; + const struct dw_pcie_host_ops *ops; + int msi_irq[MAX_MSI_CTRLS]; + struct irq_domain *irq_domain; + dma_addr_t msi_data; + struct irq_chip *msi_irq_chip; + u32 num_vectors; + u32 irq_mask[MAX_MSI_CTRLS]; + struct pci_host_bridge *bridge; + raw_spinlock_t lock; + DECLARE_BITMAP(msi_irq_in_use, MAX_MSI_IRQS); + bool use_atu_msg; + int msg_atu_index; + struct resource *msg_res; + struct pci_eq_presets presets; + struct pci_config_window *cfg; + bool ecam_enabled; + bool native_ecam; + bool skip_l23_ready; + bool skip_pwrctrl_off; +}; + +struct dw_pcie_ep_ops { + int (*pre_init)(struct dw_pcie_ep *ep); + int (*init)(struct dw_pcie_ep *ep); + int (*raise_irq)(struct dw_pcie_ep *ep, u8 func_no, + unsigned int type, u16 interrupt_num); + const struct pci_epc_features* (*get_features)(struct dw_pcie_ep *ep); + /* + * Provide a method to implement the different func config space + * access for different platform, if different func have different + * offset, return the offset of func. if use write a register way + * return a 0, and implement code in callback function of platform + * driver. + */ + unsigned int (*get_dbi_offset)(struct dw_pcie_ep *ep, u8 func_no); + unsigned int (*get_dbi2_offset)(struct dw_pcie_ep *ep, u8 func_no); +}; + +struct dw_pcie_ep_func { + struct list_head list; + u8 func_no; + u8 msi_cap; /* MSI capability offset */ + u8 msix_cap; /* MSI-X capability offset */ + u8 bar_to_atu[PCI_STD_NUM_BARS]; + struct pci_epf_bar *epf_bar[PCI_STD_NUM_BARS]; + + /* Only for Address Match Mode inbound iATU */ + u32 *ib_atu_indexes[PCI_STD_NUM_BARS]; + unsigned int num_ib_atu_indexes[PCI_STD_NUM_BARS]; +}; + +struct dw_pcie_ep { + struct pci_epc *epc; + struct list_head func_list; + const struct dw_pcie_ep_ops *ops; + phys_addr_t phys_base; + size_t addr_size; + size_t page_size; + phys_addr_t *outbound_addr; + unsigned long *ib_window_map; + unsigned long *ob_window_map; + void __iomem *msi_mem; + phys_addr_t msi_mem_phys; + + /* MSI outbound iATU state */ + bool msi_iatu_mapped; + size_t msi_iatu_mapped_offset; + u64 msi_msg_addr; + size_t msi_map_size; +}; + +struct dw_pcie_ops { + u64 (*cpu_addr_fixup)(struct dw_pcie *pcie, u64 cpu_addr); + u32 (*read_dbi)(struct dw_pcie *pcie, void __iomem *base, u32 reg, + size_t size); + void (*write_dbi)(struct dw_pcie *pcie, void __iomem *base, u32 reg, + size_t size, u32 val); + void (*write_dbi2)(struct dw_pcie *pcie, void __iomem *base, u32 reg, + size_t size, u32 val); + bool (*link_up)(struct dw_pcie *pcie); + enum dw_pcie_ltssm (*get_ltssm)(struct dw_pcie *pcie); + int (*start_link)(struct dw_pcie *pcie); + void (*stop_link)(struct dw_pcie *pcie); +}; + +struct debugfs_info { + struct dentry *debug_dir; + void *rasdes_info; +}; + +struct dw_pcie { + struct device *dev; + void __iomem *dbi_base; + resource_size_t dbi_phys_addr; + void __iomem *dbi_base2; + void __iomem *atu_base; + void __iomem *elbi_base; + resource_size_t atu_phys_addr; + size_t atu_size; + resource_size_t parent_bus_offset; + u32 num_ib_windows; + u32 num_ob_windows; + u32 region_align; + u64 region_limit; + struct dw_pcie_rp pp; + struct dw_pcie_ep ep; + const struct dw_pcie_ops *ops; + u32 version; + u32 type; + unsigned long caps; + int num_lanes; + int max_link_speed; + u8 n_fts[2]; + struct dw_edma_chip edma; + phys_addr_t edma_reg_phys; + resource_size_t edma_reg_size; + bool l1ss_support; /* L1 PM Substates support */ + struct clk_bulk_data app_clks[DW_PCIE_NUM_APP_CLKS]; + struct clk_bulk_data core_clks[DW_PCIE_NUM_CORE_CLKS]; + struct reset_control_bulk_data app_rsts[DW_PCIE_NUM_APP_RSTS]; + struct reset_control_bulk_data core_rsts[DW_PCIE_NUM_CORE_RSTS]; + struct gpio_desc *pe_rst; + bool suspended; + struct debugfs_info *debugfs; + enum dw_pcie_device_mode mode; + u16 ptm_vsec_offset; + struct pci_ptm_debugfs *ptm_debugfs; + + /* + * If iATU input addresses are offset from CPU physical addresses, + * we previously required .cpu_addr_fixup() to convert them. We + * now rely on the devicetree instead. If .cpu_addr_fixup() + * exists, we compare its results with devicetree. + * + * If .cpu_addr_fixup() does not exist, we assume the offset is + * zero and warn if devicetree claims otherwise. If we know all + * devicetrees correctly describe the offset, set + * use_parent_dt_ranges to true to avoid this warning. + */ + bool use_parent_dt_ranges; +}; + +#define to_dw_pcie_from_pp(port) container_of((port), struct dw_pcie, pp) + +#define to_dw_pcie_from_ep(endpoint) \ + container_of((endpoint), struct dw_pcie, ep) + +int dw_pcie_get_resources(struct dw_pcie *pci); + +void dw_pcie_version_detect(struct dw_pcie *pci); + +u8 dw_pcie_find_capability(struct dw_pcie *pci, u8 cap); +u16 dw_pcie_find_ext_capability(struct dw_pcie *pci, u8 cap); +void dw_pcie_remove_capability(struct dw_pcie *pci, u8 cap); +void dw_pcie_remove_ext_capability(struct dw_pcie *pci, u8 cap); +u16 dw_pcie_find_rasdes_capability(struct dw_pcie *pci); +u16 dw_pcie_find_ptm_capability(struct dw_pcie *pci); + +int dw_pcie_read(void __iomem *addr, int size, u32 *val); +int dw_pcie_write(void __iomem *addr, int size, u32 val); + +u32 dw_pcie_read_dbi(struct dw_pcie *pci, u32 reg, size_t size); +void dw_pcie_write_dbi(struct dw_pcie *pci, u32 reg, size_t size, u32 val); +void dw_pcie_write_dbi2(struct dw_pcie *pci, u32 reg, size_t size, u32 val); +bool dw_pcie_link_up(struct dw_pcie *pci); +void dw_pcie_upconfig_setup(struct dw_pcie *pci); +int dw_pcie_wait_for_link(struct dw_pcie *pci); +int dw_pcie_link_get_max_link_width(struct dw_pcie *pci); +int dw_pcie_prog_outbound_atu(struct dw_pcie *pci, + const struct dw_pcie_ob_atu_cfg *atu); +int dw_pcie_prog_inbound_atu(struct dw_pcie *pci, int index, int type, + u64 parent_bus_addr, u64 pci_addr, u64 size); +int dw_pcie_prog_ep_inbound_atu(struct dw_pcie *pci, u8 func_no, int index, + int type, u64 parent_bus_addr, + u8 bar, size_t size); +void dw_pcie_disable_atu(struct dw_pcie *pci, u32 dir, int index); +void dw_pcie_hide_unsupported_l1ss(struct dw_pcie *pci); +void dw_pcie_program_t_power_on(struct dw_pcie *pci, u32 t_power_on); +void dw_pcie_setup(struct dw_pcie *pci); +void dw_pcie_iatu_detect(struct dw_pcie *pci); +int dw_pcie_edma_detect(struct dw_pcie *pci); +void dw_pcie_edma_remove(struct dw_pcie *pci); +resource_size_t dw_pcie_parent_bus_offset(struct dw_pcie *pci, + const char *reg_name, + resource_size_t cpu_phy_addr); + +static inline void dw_pcie_writel_dbi(struct dw_pcie *pci, u32 reg, u32 val) +{ + dw_pcie_write_dbi(pci, reg, 0x4, val); +} + +static inline u32 dw_pcie_readl_dbi(struct dw_pcie *pci, u32 reg) +{ + return dw_pcie_read_dbi(pci, reg, 0x4); +} + +static inline void dw_pcie_writew_dbi(struct dw_pcie *pci, u32 reg, u16 val) +{ + dw_pcie_write_dbi(pci, reg, 0x2, val); +} + +static inline u16 dw_pcie_readw_dbi(struct dw_pcie *pci, u32 reg) +{ + return dw_pcie_read_dbi(pci, reg, 0x2); +} + +static inline void dw_pcie_writeb_dbi(struct dw_pcie *pci, u32 reg, u8 val) +{ + dw_pcie_write_dbi(pci, reg, 0x1, val); +} + +static inline u8 dw_pcie_readb_dbi(struct dw_pcie *pci, u32 reg) +{ + return dw_pcie_read_dbi(pci, reg, 0x1); +} + +static inline void dw_pcie_writel_dbi2(struct dw_pcie *pci, u32 reg, u32 val) +{ + dw_pcie_write_dbi2(pci, reg, 0x4, val); +} + +static inline int dw_pcie_read_cfg_byte(struct dw_pcie *pci, int where, + u8 *val) +{ + *val = dw_pcie_readb_dbi(pci, where); + return PCIBIOS_SUCCESSFUL; +} + +static inline int dw_pcie_read_cfg_word(struct dw_pcie *pci, int where, + u16 *val) +{ + *val = dw_pcie_readw_dbi(pci, where); + return PCIBIOS_SUCCESSFUL; +} + +static inline int dw_pcie_read_cfg_dword(struct dw_pcie *pci, int where, + u32 *val) +{ + *val = dw_pcie_readl_dbi(pci, where); + return PCIBIOS_SUCCESSFUL; +} + +static inline unsigned int dw_pcie_ep_get_dbi_offset(struct dw_pcie_ep *ep, + u8 func_no) +{ + unsigned int dbi_offset = 0; + + if (ep->ops->get_dbi_offset) + dbi_offset = ep->ops->get_dbi_offset(ep, func_no); + + return dbi_offset; +} + +static inline u32 dw_pcie_ep_read_dbi(struct dw_pcie_ep *ep, u8 func_no, + u32 reg, size_t size) +{ + unsigned int offset = dw_pcie_ep_get_dbi_offset(ep, func_no); + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + return dw_pcie_read_dbi(pci, offset + reg, size); +} + +static inline void dw_pcie_ep_write_dbi(struct dw_pcie_ep *ep, u8 func_no, + u32 reg, size_t size, u32 val) +{ + unsigned int offset = dw_pcie_ep_get_dbi_offset(ep, func_no); + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + dw_pcie_write_dbi(pci, offset + reg, size, val); +} + +static inline void dw_pcie_ep_writel_dbi(struct dw_pcie_ep *ep, u8 func_no, + u32 reg, u32 val) +{ + dw_pcie_ep_write_dbi(ep, func_no, reg, 0x4, val); +} + +static inline u32 dw_pcie_ep_readl_dbi(struct dw_pcie_ep *ep, u8 func_no, + u32 reg) +{ + return dw_pcie_ep_read_dbi(ep, func_no, reg, 0x4); +} + +static inline void dw_pcie_ep_writew_dbi(struct dw_pcie_ep *ep, u8 func_no, + u32 reg, u16 val) +{ + dw_pcie_ep_write_dbi(ep, func_no, reg, 0x2, val); +} + +static inline u16 dw_pcie_ep_readw_dbi(struct dw_pcie_ep *ep, u8 func_no, + u32 reg) +{ + return dw_pcie_ep_read_dbi(ep, func_no, reg, 0x2); +} + +static inline void dw_pcie_ep_writeb_dbi(struct dw_pcie_ep *ep, u8 func_no, + u32 reg, u8 val) +{ + dw_pcie_ep_write_dbi(ep, func_no, reg, 0x1, val); +} + +static inline u8 dw_pcie_ep_readb_dbi(struct dw_pcie_ep *ep, u8 func_no, + u32 reg) +{ + return dw_pcie_ep_read_dbi(ep, func_no, reg, 0x1); +} + +static inline int dw_pcie_ep_read_cfg_byte(struct dw_pcie_ep *ep, u8 func_no, + int where, u8 *val) +{ + *val = dw_pcie_ep_readb_dbi(ep, func_no, where); + return PCIBIOS_SUCCESSFUL; +} + +static inline int dw_pcie_ep_read_cfg_word(struct dw_pcie_ep *ep, u8 func_no, + int where, u16 *val) +{ + *val = dw_pcie_ep_readw_dbi(ep, func_no, where); + return PCIBIOS_SUCCESSFUL; +} + +static inline int dw_pcie_ep_read_cfg_dword(struct dw_pcie_ep *ep, u8 func_no, + int where, u32 *val) +{ + *val = dw_pcie_ep_readl_dbi(ep, func_no, where); + return PCIBIOS_SUCCESSFUL; +} + +static inline unsigned int dw_pcie_ep_get_dbi2_offset(struct dw_pcie_ep *ep, + u8 func_no) +{ + unsigned int dbi2_offset = 0; + + if (ep->ops->get_dbi2_offset) + dbi2_offset = ep->ops->get_dbi2_offset(ep, func_no); + else if (ep->ops->get_dbi_offset) /* for backward compatibility */ + dbi2_offset = ep->ops->get_dbi_offset(ep, func_no); + + return dbi2_offset; +} + +static inline void dw_pcie_ep_write_dbi2(struct dw_pcie_ep *ep, u8 func_no, + u32 reg, size_t size, u32 val) +{ + unsigned int offset = dw_pcie_ep_get_dbi2_offset(ep, func_no); + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + dw_pcie_write_dbi2(pci, offset + reg, size, val); +} + +static inline void dw_pcie_ep_writel_dbi2(struct dw_pcie_ep *ep, u8 func_no, + u32 reg, u32 val) +{ + dw_pcie_ep_write_dbi2(ep, func_no, reg, 0x4, val); +} + +static inline void dw_pcie_dbi_ro_wr_en(struct dw_pcie *pci) +{ + u32 reg; + u32 val; + + reg = PCIE_MISC_CONTROL_1_OFF; + val = dw_pcie_readl_dbi(pci, reg); + val |= PCIE_DBI_RO_WR_EN; + dw_pcie_writel_dbi(pci, reg, val); +} + +static inline void dw_pcie_dbi_ro_wr_dis(struct dw_pcie *pci) +{ + u32 reg; + u32 val; + + reg = PCIE_MISC_CONTROL_1_OFF; + val = dw_pcie_readl_dbi(pci, reg); + val &= ~PCIE_DBI_RO_WR_EN; + dw_pcie_writel_dbi(pci, reg, val); +} + +static inline int dw_pcie_start_link(struct dw_pcie *pci) +{ + if (pci->ops && pci->ops->start_link) + return pci->ops->start_link(pci); + + return 0; +} + +static inline void dw_pcie_stop_link(struct dw_pcie *pci) +{ + if (pci->ops && pci->ops->stop_link) + pci->ops->stop_link(pci); +} + +static inline enum dw_pcie_ltssm dw_pcie_get_ltssm(struct dw_pcie *pci) +{ + u32 val; + + if (pci->ops && pci->ops->get_ltssm) + return pci->ops->get_ltssm(pci); + + val = dw_pcie_readl_dbi(pci, PCIE_PORT_DEBUG0); + + return (enum dw_pcie_ltssm)FIELD_GET(PORT_LOGIC_LTSSM_STATE_MASK, val); +} + +const char *dw_pcie_ltssm_status_string(enum dw_pcie_ltssm ltssm); + +#ifdef CONFIG_PCIE_DW_HOST +int dw_pcie_suspend_noirq(struct dw_pcie *pci); +int dw_pcie_resume_noirq(struct dw_pcie *pci); +void dw_handle_msi_irq(struct dw_pcie_rp *pp); +void dw_pcie_msi_init(struct dw_pcie_rp *pp); +int dw_pcie_msi_host_init(struct dw_pcie_rp *pp); +void dw_pcie_free_msi(struct dw_pcie_rp *pp); +int dw_pcie_setup_rc(struct dw_pcie_rp *pp); +int dw_pcie_host_init(struct dw_pcie_rp *pp); +void dw_pcie_host_deinit(struct dw_pcie_rp *pp); +int dw_pcie_allocate_domains(struct dw_pcie_rp *pp); +void __iomem *dw_pcie_own_conf_map_bus(struct pci_bus *bus, unsigned int devfn, + int where); +#else +static inline int dw_pcie_suspend_noirq(struct dw_pcie *pci) +{ + return 0; +} + +static inline int dw_pcie_resume_noirq(struct dw_pcie *pci) +{ + return 0; +} + +static inline void dw_handle_msi_irq(struct dw_pcie_rp *pp) { } + +static inline void dw_pcie_msi_init(struct dw_pcie_rp *pp) +{ } + +static inline int dw_pcie_msi_host_init(struct dw_pcie_rp *pp) +{ + return -ENODEV; +} + +static inline void dw_pcie_free_msi(struct dw_pcie_rp *pp) +{ } + +static inline int dw_pcie_setup_rc(struct dw_pcie_rp *pp) +{ + return 0; +} + +static inline int dw_pcie_host_init(struct dw_pcie_rp *pp) +{ + return 0; +} + +static inline void dw_pcie_host_deinit(struct dw_pcie_rp *pp) +{ +} + +static inline int dw_pcie_allocate_domains(struct dw_pcie_rp *pp) +{ + return 0; +} +static inline void __iomem *dw_pcie_own_conf_map_bus(struct pci_bus *bus, + unsigned int devfn, + int where) +{ + return NULL; +} +#endif + +#ifdef CONFIG_PCIE_DW_EP +void dw_pcie_ep_linkup(struct dw_pcie_ep *ep); +void dw_pcie_ep_linkdown(struct dw_pcie_ep *ep); +int dw_pcie_ep_init(struct dw_pcie_ep *ep); +int dw_pcie_ep_init_registers(struct dw_pcie_ep *ep); +void dw_pcie_ep_deinit(struct dw_pcie_ep *ep); +void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep); +int dw_pcie_ep_raise_intx_irq(struct dw_pcie_ep *ep, u8 func_no); +int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, + u8 interrupt_num); +int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, + u16 interrupt_num); +int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, u8 func_no, + u16 interrupt_num); +void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar); +struct dw_pcie_ep_func * +dw_pcie_ep_get_func_from_ep(struct dw_pcie_ep *ep, u8 func_no); +#else +static inline void dw_pcie_ep_linkup(struct dw_pcie_ep *ep) +{ +} + +static inline void dw_pcie_ep_linkdown(struct dw_pcie_ep *ep) +{ +} + +static inline int dw_pcie_ep_init(struct dw_pcie_ep *ep) +{ + return 0; +} + +static inline int dw_pcie_ep_init_registers(struct dw_pcie_ep *ep) +{ + return 0; +} + +static inline void dw_pcie_ep_deinit(struct dw_pcie_ep *ep) +{ +} + +static inline void dw_pcie_ep_cleanup(struct dw_pcie_ep *ep) +{ +} + +static inline int dw_pcie_ep_raise_intx_irq(struct dw_pcie_ep *ep, u8 func_no) +{ + return 0; +} + +static inline int dw_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, u8 func_no, + u8 interrupt_num) +{ + return 0; +} + +static inline int dw_pcie_ep_raise_msix_irq(struct dw_pcie_ep *ep, u8 func_no, + u16 interrupt_num) +{ + return 0; +} + +static inline int dw_pcie_ep_raise_msix_irq_doorbell(struct dw_pcie_ep *ep, + u8 func_no, + u16 interrupt_num) +{ + return 0; +} + +static inline void dw_pcie_ep_reset_bar(struct dw_pcie *pci, enum pci_barno bar) +{ +} + +static inline struct dw_pcie_ep_func * +dw_pcie_ep_get_func_from_ep(struct dw_pcie_ep *ep, u8 func_no) +{ + return NULL; +} +#endif + +#ifdef CONFIG_PCIE_DW_DEBUGFS +void dwc_pcie_debugfs_init(struct dw_pcie *pci, enum dw_pcie_device_mode mode); +void dwc_pcie_debugfs_deinit(struct dw_pcie *pci); +#else +static inline void dwc_pcie_debugfs_init(struct dw_pcie *pci, + enum dw_pcie_device_mode mode) +{ +} +static inline void dwc_pcie_debugfs_deinit(struct dw_pcie *pci) +{ +} +#endif + +#endif /* _PCIE_DESIGNWARE_H */ diff --git a/drivers/pci/controller/dwc/pcie-dw-rockchip.c b/drivers/pci/controller/dwc/pcie-dw-rockchip.c new file mode 100644 index 000000000..af26a07c0 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-dw-rockchip.c @@ -0,0 +1,1000 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for Rockchip SoCs. + * + * Copyright (C) 2021 Rockchip Electronics Co., Ltd. + * http://www.rock-chips.com + * + * Author: Simon Xue <xxm@rock-chips.com> + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/gpio/consumer.h> +#include <linux/hw_bitfield.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqdomain.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_irq.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/reset.h> +#include <linux/workqueue.h> +#include <trace/events/pci_controller.h> + +#include "../../pci.h" +#include "../pci-host-common.h" +#include "pcie-designware.h" + +/* + * The upper 16 bits of PCIE_CLIENT_CONFIG are a write + * mask for the lower 16 bits. + */ + +#define to_rockchip_pcie(x) dev_get_drvdata((x)->dev) + +/* General Control Register */ +#define PCIE_CLIENT_GENERAL_CON 0x0 +#define PCIE_CLIENT_MODE_MASK GENMASK(7, 4) +#define PCIE_CLIENT_MODE_EP 0x0UL +#define PCIE_CLIENT_MODE_RC 0x4UL +#define PCIE_CLIENT_SET_MODE(x) FIELD_PREP_WM16(PCIE_CLIENT_MODE_MASK, (x)) +#define PCIE_CLIENT_LD_RQ_RST_GRT FIELD_PREP_WM16(BIT(3), 1) +#define PCIE_CLIENT_ENABLE_LTSSM FIELD_PREP_WM16(BIT(2), 1) +#define PCIE_CLIENT_DISABLE_LTSSM FIELD_PREP_WM16(BIT(2), 0) + +/* Interrupt Status Register Related to Legacy Interrupt */ +#define PCIE_CLIENT_INTR_STATUS_LEGACY 0x8 + +/* Interrupt Status Register Related to Miscellaneous Operation */ +#define PCIE_CLIENT_INTR_STATUS_MISC 0x10 +#define PCIE_RDLH_LINK_UP_CHGED BIT(1) +#define PCIE_LINK_REQ_RST_NOT_INT BIT(2) + +/* Interrupt Mask Register Related to Legacy Interrupt */ +#define PCIE_CLIENT_INTR_MASK_LEGACY 0x1c +#define PCIE_INTR_MASK GENMASK(7, 0) +#define PCIE_INTR_CLAMP(_x) ((BIT((_x)) & PCIE_INTR_MASK)) +#define PCIE_INTR_LEGACY_MASK(x) (PCIE_INTR_CLAMP((x)) | \ + (PCIE_INTR_CLAMP((x)) << 16)) +#define PCIE_INTR_LEGACY_UNMASK(x) (PCIE_INTR_CLAMP((x)) << 16) + +/* Interrupt Mask Register Related to Miscellaneous Operation */ +#define PCIE_CLIENT_INTR_MASK_MISC 0x24 + +/* Power Management Control Register */ +#define PCIE_CLIENT_POWER_CON 0x2c +#define PCIE_CLKREQ_READY FIELD_PREP_WM16(BIT(0), 1) +#define PCIE_CLKREQ_NOT_READY FIELD_PREP_WM16(BIT(0), 0) +#define PCIE_CLKREQ_PULL_DOWN FIELD_PREP_WM16(GENMASK(13, 12), 1) + +/* RASDES TBA information */ +#define PCIE_CLIENT_CDM_RASDES_TBA_INFO_CMN 0x154 +#define PCIE_CLIENT_CDM_RASDES_TBA_L1_1 BIT(4) +#define PCIE_CLIENT_CDM_RASDES_TBA_L1_2 BIT(5) + +/* Debug FIFO information */ +#define PCIE_CLIENT_DBG_FIFO_MODE_CON 0x310 +#define PCIE_CLIENT_DBG_EN 0xffff0007 +#define PCIE_CLIENT_DBG_DIS 0xffff0000 +#define PCIE_CLIENT_DBG_FIFO_PTN_HIT_D0 0x320 +#define PCIE_CLIENT_DBG_FIFO_PTN_HIT_D1 0x324 +#define PCIE_CLIENT_DBG_FIFO_TRN_HIT_D0 0x328 +#define PCIE_CLIENT_DBG_FIFO_TRN_HIT_D1 0x32c +#define PCIE_CLIENT_DBG_TRANSITION_DATA 0xffff0000 +#define PCIE_CLIENT_DBG_FIFO_STATUS 0x350 +#define PCIE_DBG_FIFO_RATE_MASK GENMASK(22, 20) +#define PCIE_DBG_FIFO_L1SUB_MASK GENMASK(10, 8) +#define PCIE_DBG_LTSSM_HISTORY_CNT 64 + +/* Hot Reset Control Register */ +#define PCIE_CLIENT_HOT_RESET_CTRL 0x180 +#define PCIE_LTSSM_APP_DLY2_EN BIT(1) +#define PCIE_LTSSM_APP_DLY2_DONE BIT(3) +#define PCIE_LTSSM_ENABLE_ENHANCE BIT(4) + +/* LTSSM Status Register */ +#define PCIE_CLIENT_LTSSM_STATUS 0x300 +#define PCIE_LINKUP 0x3 +#define PCIE_LINKUP_MASK GENMASK(17, 16) +#define PCIE_LTSSM_STATUS_MASK GENMASK(5, 0) + +#define PCIE_TYPE0_HDR_DBI2_OFFSET 0x100000 + +struct rockchip_pcie { + struct dw_pcie pci; + void __iomem *apb_base; + struct phy *phy; + struct clk_bulk_data *clks; + unsigned int clk_cnt; + struct reset_control *rst; + struct gpio_desc *rst_gpio; + struct irq_domain *irq_domain; + const struct rockchip_pcie_of_data *data; + bool supports_clkreq; + struct delayed_work trace_work; +}; + +struct rockchip_pcie_of_data { + enum dw_pcie_device_mode mode; + const struct pci_epc_features *epc_features; +}; + +static int rockchip_pcie_rc_reset_root_port(struct pci_host_bridge *bridge, + struct pci_dev *pdev); + +static int rockchip_pcie_readl_apb(struct rockchip_pcie *rockchip, u32 reg) +{ + return readl_relaxed(rockchip->apb_base + reg); +} + +static void rockchip_pcie_writel_apb(struct rockchip_pcie *rockchip, u32 val, + u32 reg) +{ + writel_relaxed(val, rockchip->apb_base + reg); +} + +static void rockchip_pcie_intx_handler(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc); + unsigned long reg, hwirq; + + chained_irq_enter(chip, desc); + + reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_LEGACY); + + for_each_set_bit(hwirq, ®, 4) + generic_handle_domain_irq(rockchip->irq_domain, hwirq); + + chained_irq_exit(chip, desc); +} + +static void rockchip_intx_mask(struct irq_data *data) +{ + rockchip_pcie_writel_apb(irq_data_get_irq_chip_data(data), + PCIE_INTR_LEGACY_MASK(data->hwirq), + PCIE_CLIENT_INTR_MASK_LEGACY); +}; + +static void rockchip_intx_unmask(struct irq_data *data) +{ + rockchip_pcie_writel_apb(irq_data_get_irq_chip_data(data), + PCIE_INTR_LEGACY_UNMASK(data->hwirq), + PCIE_CLIENT_INTR_MASK_LEGACY); +}; + +static struct irq_chip rockchip_intx_irq_chip = { + .name = "INTx", + .irq_mask = rockchip_intx_mask, + .irq_unmask = rockchip_intx_unmask, + .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, +}; + +static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &rockchip_intx_irq_chip, handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + + return 0; +} + +static const struct irq_domain_ops intx_domain_ops = { + .map = rockchip_pcie_intx_map, +}; + +static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip) +{ + struct device *dev = rockchip->pci.dev; + struct device_node *intc; + + intc = of_get_child_by_name(dev->of_node, "legacy-interrupt-controller"); + if (!intc) { + dev_err(dev, "missing child interrupt-controller node\n"); + return -EINVAL; + } + + rockchip->irq_domain = irq_domain_create_linear(of_fwnode_handle(intc), PCI_NUM_INTX, + &intx_domain_ops, rockchip); + of_node_put(intc); + if (!rockchip->irq_domain) { + dev_err(dev, "failed to get a INTx IRQ domain\n"); + return -EINVAL; + } + + return 0; +} + +static u32 rockchip_pcie_get_ltssm_reg(struct rockchip_pcie *rockchip) +{ + return rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_LTSSM_STATUS); +} + +static enum dw_pcie_ltssm rockchip_pcie_get_ltssm(struct dw_pcie *pci) +{ + struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); + u32 val = rockchip_pcie_readl_apb(rockchip, + PCIE_CLIENT_CDM_RASDES_TBA_INFO_CMN); + + if (val & PCIE_CLIENT_CDM_RASDES_TBA_L1_1) + return DW_PCIE_LTSSM_L1_1; + + if (val & PCIE_CLIENT_CDM_RASDES_TBA_L1_2) + return DW_PCIE_LTSSM_L1_2; + + return rockchip_pcie_get_ltssm_reg(rockchip) & PCIE_LTSSM_STATUS_MASK; +} + +#ifdef CONFIG_TRACING +static void rockchip_pcie_ltssm_trace_work(struct work_struct *work) +{ + struct rockchip_pcie *rockchip = container_of(work, + struct rockchip_pcie, + trace_work.work); + struct dw_pcie *pci = &rockchip->pci; + enum dw_pcie_ltssm state; + u32 i, l1ss, prev_val = DW_PCIE_LTSSM_UNKNOWN, rate, val; + + if (!trace_pcie_ltssm_state_transition_enabled()) + goto skip_trace; + + for (i = 0; i < PCIE_DBG_LTSSM_HISTORY_CNT; i++) { + val = rockchip_pcie_readl_apb(rockchip, + PCIE_CLIENT_DBG_FIFO_STATUS); + rate = FIELD_GET(PCIE_DBG_FIFO_RATE_MASK, val); + l1ss = FIELD_GET(PCIE_DBG_FIFO_L1SUB_MASK, val); + val = FIELD_GET(PCIE_LTSSM_STATUS_MASK, val); + + /* + * Hardware Mechanism: The ring FIFO employs two tracking + * counters: + * - 'last-read-point': maintains the user's last read position + * - 'last-valid-point': tracks the HW's last state update + * + * Software Handling: When two consecutive LTSSM states are + * identical, it indicates invalid subsequent data in the FIFO. + * In this case, we skip the remaining entries. The dual counter + * design ensures that on the next state transition, reading can + * resume from the last user position. + */ + if ((i > 0 && val == prev_val) || val > DW_PCIE_LTSSM_RCVRY_EQ3) + break; + + state = prev_val = val; + if (val == DW_PCIE_LTSSM_L1_IDLE) { + if (l1ss == 2) + state = DW_PCIE_LTSSM_L1_2; + else if (l1ss == 1) + state = DW_PCIE_LTSSM_L1_1; + } + + trace_pcie_ltssm_state_transition(dev_name(pci->dev), + dw_pcie_ltssm_status_string(state), + ((rate + 1) > pci->max_link_speed) ? + PCI_SPEED_UNKNOWN : PCIE_SPEED_2_5GT + rate); + } + +skip_trace: + schedule_delayed_work(&rockchip->trace_work, msecs_to_jiffies(5000)); +} + +static void rockchip_pcie_ltssm_trace(struct rockchip_pcie *rockchip, + bool enable) +{ + if (enable) { + rockchip_pcie_writel_apb(rockchip, + PCIE_CLIENT_DBG_TRANSITION_DATA, + PCIE_CLIENT_DBG_FIFO_PTN_HIT_D0); + rockchip_pcie_writel_apb(rockchip, + PCIE_CLIENT_DBG_TRANSITION_DATA, + PCIE_CLIENT_DBG_FIFO_PTN_HIT_D1); + rockchip_pcie_writel_apb(rockchip, + PCIE_CLIENT_DBG_TRANSITION_DATA, + PCIE_CLIENT_DBG_FIFO_TRN_HIT_D0); + rockchip_pcie_writel_apb(rockchip, + PCIE_CLIENT_DBG_TRANSITION_DATA, + PCIE_CLIENT_DBG_FIFO_TRN_HIT_D1); + rockchip_pcie_writel_apb(rockchip, + PCIE_CLIENT_DBG_EN, + PCIE_CLIENT_DBG_FIFO_MODE_CON); + + INIT_DELAYED_WORK(&rockchip->trace_work, + rockchip_pcie_ltssm_trace_work); + schedule_delayed_work(&rockchip->trace_work, 0); + } else { + rockchip_pcie_writel_apb(rockchip, + PCIE_CLIENT_DBG_DIS, + PCIE_CLIENT_DBG_FIFO_MODE_CON); + cancel_delayed_work_sync(&rockchip->trace_work); + } +} +#else +static void rockchip_pcie_ltssm_trace(struct rockchip_pcie *rockchip, + bool enable) +{ +} +#endif + +static void rockchip_pcie_enable_ltssm(struct rockchip_pcie *rockchip) +{ + rockchip_pcie_writel_apb(rockchip, PCIE_CLIENT_ENABLE_LTSSM, + PCIE_CLIENT_GENERAL_CON); +} + +static void rockchip_pcie_disable_ltssm(struct rockchip_pcie *rockchip) +{ + rockchip_pcie_writel_apb(rockchip, PCIE_CLIENT_DISABLE_LTSSM, + PCIE_CLIENT_GENERAL_CON); +} + +static bool rockchip_pcie_link_up(struct dw_pcie *pci) +{ + struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); + u32 val = rockchip_pcie_get_ltssm_reg(rockchip); + + return FIELD_GET(PCIE_LINKUP_MASK, val) == PCIE_LINKUP; +} + +/* + * See e.g. section '11.6.6.4 L1 Substate' in the RK3588 TRM V1.0 for the steps + * needed to support L1 substates. Currently, just enable L1 substates for RC + * mode if CLKREQ# is properly connected and supports-clkreq is present in DT. + * For EP mode, there are more things should be done to actually save power in + * L1 substates, so disable L1 substates until there is proper support. + */ +static void rockchip_pcie_configure_l1ss(struct dw_pcie *pci) +{ + struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); + + /* Enable L1 substates if CLKREQ# is properly connected */ + if (rockchip->supports_clkreq) { + rockchip_pcie_writel_apb(rockchip, PCIE_CLKREQ_READY, + PCIE_CLIENT_POWER_CON); + pci->l1ss_support = true; + return; + } + + /* + * Otherwise, assert CLKREQ# unconditionally. Since + * pci->l1ss_support is not set, the DWC core will prevent L1 + * Substates support from being advertised. + */ + rockchip_pcie_writel_apb(rockchip, + PCIE_CLKREQ_PULL_DOWN | PCIE_CLKREQ_NOT_READY, + PCIE_CLIENT_POWER_CON); +} + +static void rockchip_pcie_enable_l0s(struct dw_pcie *pci) +{ + u32 cap, lnkcap; + + /* Enable L0S capability for all SoCs */ + cap = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + if (cap) { + lnkcap = dw_pcie_readl_dbi(pci, cap + PCI_EXP_LNKCAP); + lnkcap |= PCI_EXP_LNKCAP_ASPM_L0S; + dw_pcie_dbi_ro_wr_en(pci); + dw_pcie_writel_dbi(pci, cap + PCI_EXP_LNKCAP, lnkcap); + dw_pcie_dbi_ro_wr_dis(pci); + } +} + +static int rockchip_pcie_start_link(struct dw_pcie *pci) +{ + struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); + + /* Reset device */ + gpiod_set_value_cansleep(rockchip->rst_gpio, 0); + + rockchip_pcie_enable_ltssm(rockchip); + + /* + * PCIe requires the refclk to be stable for 100µs prior to releasing + * PERST. See table 2-4 in section 2.6.2 AC Specifications of the PCI + * Express Card Electromechanical Specification, 1.1. However, we don't + * know if the refclk is coming from RC's PHY or external OSC. If it's + * from RC, so enabling LTSSM is the just right place to release #PERST. + * We need more extra time as before, rather than setting just + * 100us as we don't know how long should the device need to reset. + */ + msleep(PCIE_T_PVPERL_MS); + + rockchip_pcie_ltssm_trace(rockchip, true); + + gpiod_set_value_cansleep(rockchip->rst_gpio, 1); + + return 0; +} + +static void rockchip_pcie_stop_link(struct dw_pcie *pci) +{ + struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); + + rockchip_pcie_disable_ltssm(rockchip); + rockchip_pcie_ltssm_trace(rockchip, false); +} + +static int rockchip_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); + struct device *dev = rockchip->pci.dev; + int irq, ret; + + irq = of_irq_get_byname(dev->of_node, "legacy"); + if (irq < 0) + return irq; + + pci->dbi_base2 = pci->dbi_base + PCIE_TYPE0_HDR_DBI2_OFFSET; + + ret = rockchip_pcie_init_irq_domain(rockchip); + if (ret < 0) + dev_err(dev, "failed to init irq domain\n"); + + irq_set_chained_handler_and_data(irq, rockchip_pcie_intx_handler, + rockchip); + + rockchip_pcie_configure_l1ss(pci); + rockchip_pcie_enable_l0s(pci); + pp->bridge->reset_root_port = rockchip_pcie_rc_reset_root_port; + + /* Disable Root Ports BAR0 and BAR1 as they report bogus size */ + dw_pcie_writel_dbi2(pci, PCI_BASE_ADDRESS_0, 0x0); + dw_pcie_writel_dbi2(pci, PCI_BASE_ADDRESS_1, 0x0); + + return 0; +} + +static const struct dw_pcie_host_ops rockchip_pcie_host_ops = { + .init = rockchip_pcie_host_init, +}; + +/* + * ATS does not work on RK3588 when running in EP mode. + * + * After the host has enabled ATS on the EP side, it will send an IOTLB + * invalidation request to the EP side. However, the RK3588 will never send + * a completion back and eventually the host will print an IOTLB_INV_TIMEOUT + * error, and the EP will not be operational. If we hide the ATS capability, + * things work as expected. + */ +static void rockchip_pcie_ep_hide_broken_ats_cap_rk3588(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct device *dev = pci->dev; + + /* Only hide the ATS capability for RK3588 running in EP mode. */ + if (!of_device_is_compatible(dev->of_node, "rockchip,rk3588-pcie-ep")) + return; + + dw_pcie_remove_ext_capability(pci, PCI_EXT_CAP_ID_ATS); +} + +static int rockchip_pcie_ep_init(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + rockchip_pcie_enable_l0s(pci); + rockchip_pcie_ep_hide_broken_ats_cap_rk3588(ep); + + return 0; +}; + +static int rockchip_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no, + unsigned int type, u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + switch (type) { + case PCI_IRQ_INTX: + return dw_pcie_ep_raise_intx_irq(ep, func_no); + case PCI_IRQ_MSI: + return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); + case PCI_IRQ_MSIX: + return dw_pcie_ep_raise_msix_irq(ep, func_no, interrupt_num); + default: + dev_err(pci->dev, "UNKNOWN IRQ type\n"); + } + + return 0; +} + +static const struct pci_epc_features rockchip_pcie_epc_features_rk3568 = { + DWC_EPC_COMMON_FEATURES, + .linkup_notifier = true, + .msi_capable = true, + .msix_capable = true, + .align = SZ_64K, + .bar[BAR_0] = { .type = BAR_RESIZABLE, }, + .bar[BAR_1] = { .type = BAR_RESIZABLE, }, + .bar[BAR_2] = { .type = BAR_RESIZABLE, }, + .bar[BAR_3] = { .type = BAR_RESIZABLE, }, + .bar[BAR_4] = { .type = BAR_RESIZABLE, }, + .bar[BAR_5] = { .type = BAR_RESIZABLE, }, +}; + +static const struct pci_epc_bar_rsvd_region rk3588_bar4_rsvd[] = { + { + /* DMA_CAP (BAR4: DMA Port Logic Structure) */ + .type = PCI_EPC_BAR_RSVD_DMA_CTRL_MMIO, + .offset = 0x0, + .size = 0x2000, + }, +}; + +/* + * BAR4 on rk3588 exposes the ATU Port Logic Structure to the host regardless of + * iATU settings for BAR4. This means that BAR4 cannot be used by an EPF driver, + * so mark it as RESERVED. + */ +static const struct pci_epc_features rockchip_pcie_epc_features_rk3588 = { + DWC_EPC_COMMON_FEATURES, + .linkup_notifier = true, + .msi_capable = true, + .msix_capable = true, + .align = SZ_64K, + .bar[BAR_0] = { .type = BAR_RESIZABLE, }, + .bar[BAR_1] = { .type = BAR_RESIZABLE, }, + .bar[BAR_2] = { .type = BAR_RESIZABLE, }, + .bar[BAR_3] = { .type = BAR_RESIZABLE, }, + .bar[BAR_4] = { + .type = BAR_RESERVED, + .nr_rsvd_regions = ARRAY_SIZE(rk3588_bar4_rsvd), + .rsvd_regions = rk3588_bar4_rsvd, + }, + .bar[BAR_5] = { .type = BAR_RESIZABLE, }, +}; + +static const struct pci_epc_features * +rockchip_pcie_get_features(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); + + return rockchip->data->epc_features; +} + +static const struct dw_pcie_ep_ops rockchip_pcie_ep_ops = { + .init = rockchip_pcie_ep_init, + .raise_irq = rockchip_pcie_raise_irq, + .get_features = rockchip_pcie_get_features, +}; + +static int rockchip_pcie_clk_init(struct rockchip_pcie *rockchip) +{ + struct device *dev = rockchip->pci.dev; + int ret; + + ret = devm_clk_bulk_get_all(dev, &rockchip->clks); + if (ret < 0) + return dev_err_probe(dev, ret, "failed to get clocks\n"); + + rockchip->clk_cnt = ret; + + ret = clk_bulk_prepare_enable(rockchip->clk_cnt, rockchip->clks); + if (ret) + return dev_err_probe(dev, ret, "failed to enable clocks\n"); + + return 0; +} + +static int rockchip_pcie_resource_get(struct platform_device *pdev, + struct rockchip_pcie *rockchip) +{ + rockchip->apb_base = devm_platform_ioremap_resource_byname(pdev, "apb"); + if (IS_ERR(rockchip->apb_base)) + return dev_err_probe(&pdev->dev, PTR_ERR(rockchip->apb_base), + "failed to map apb registers\n"); + + rockchip->rst_gpio = devm_gpiod_get_optional(&pdev->dev, "reset", + GPIOD_OUT_LOW); + if (IS_ERR(rockchip->rst_gpio)) + return dev_err_probe(&pdev->dev, PTR_ERR(rockchip->rst_gpio), + "failed to get reset gpio\n"); + + rockchip->rst = devm_reset_control_array_get_exclusive(&pdev->dev); + if (IS_ERR(rockchip->rst)) + return dev_err_probe(&pdev->dev, PTR_ERR(rockchip->rst), + "failed to get reset lines\n"); + + rockchip->supports_clkreq = of_property_read_bool(pdev->dev.of_node, + "supports-clkreq"); + + return 0; +} + +static int rockchip_pcie_phy_init(struct rockchip_pcie *rockchip) +{ + struct device *dev = rockchip->pci.dev; + int ret; + + rockchip->phy = devm_phy_get(dev, "pcie-phy"); + if (IS_ERR(rockchip->phy)) + return dev_err_probe(dev, PTR_ERR(rockchip->phy), + "missing PHY\n"); + + ret = phy_init(rockchip->phy); + if (ret < 0) + return ret; + + ret = phy_power_on(rockchip->phy); + if (ret) + phy_exit(rockchip->phy); + + return ret; +} + +static void rockchip_pcie_phy_deinit(struct rockchip_pcie *rockchip) +{ + phy_power_off(rockchip->phy); + phy_exit(rockchip->phy); +} + +static const struct dw_pcie_ops dw_pcie_ops = { + .link_up = rockchip_pcie_link_up, + .start_link = rockchip_pcie_start_link, + .stop_link = rockchip_pcie_stop_link, + .get_ltssm = rockchip_pcie_get_ltssm, +}; + +static irqreturn_t rockchip_pcie_rc_sys_irq_thread(int irq, void *arg) +{ + struct rockchip_pcie *rockchip = arg; + struct dw_pcie *pci = &rockchip->pci; + struct dw_pcie_rp *pp = &pci->pp; + struct device *dev = pci->dev; + struct pci_dev *port; + u32 reg; + + reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC); + rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC); + + dev_dbg(dev, "PCIE_CLIENT_INTR_STATUS_MISC: %#x\n", reg); + dev_dbg(dev, "LTSSM_STATUS: %#x\n", rockchip_pcie_get_ltssm_reg(rockchip)); + + if (reg & PCIE_LINK_REQ_RST_NOT_INT) { + dev_dbg(dev, "hot reset or link-down reset\n"); + for_each_pci_bridge(port, pp->bridge->bus) { + if (pci_pcie_type(port) == PCI_EXP_TYPE_ROOT_PORT) + pci_host_handle_link_down(port); + } + } + + return IRQ_HANDLED; +} + +static irqreturn_t rockchip_pcie_ep_sys_irq_thread(int irq, void *arg) +{ + struct rockchip_pcie *rockchip = arg; + struct dw_pcie *pci = &rockchip->pci; + struct device *dev = pci->dev; + u32 reg, val; + + reg = rockchip_pcie_readl_apb(rockchip, PCIE_CLIENT_INTR_STATUS_MISC); + rockchip_pcie_writel_apb(rockchip, reg, PCIE_CLIENT_INTR_STATUS_MISC); + + dev_dbg(dev, "PCIE_CLIENT_INTR_STATUS_MISC: %#x\n", reg); + dev_dbg(dev, "LTSSM_STATUS: %#x\n", rockchip_pcie_get_ltssm_reg(rockchip)); + + if (reg & PCIE_LINK_REQ_RST_NOT_INT) { + dev_dbg(dev, "hot reset or link-down reset\n"); + dw_pcie_ep_linkdown(&pci->ep); + /* Stop delaying link training. */ + val = FIELD_PREP_WM16(PCIE_LTSSM_APP_DLY2_DONE, 1); + rockchip_pcie_writel_apb(rockchip, val, + PCIE_CLIENT_HOT_RESET_CTRL); + } + + if (reg & PCIE_RDLH_LINK_UP_CHGED) { + if (rockchip_pcie_link_up(pci)) { + dev_dbg(dev, "link up\n"); + dw_pcie_ep_linkup(&pci->ep); + } + } + + return IRQ_HANDLED; +} + +static int rockchip_pcie_configure_rc(struct platform_device *pdev, + struct rockchip_pcie *rockchip) +{ + struct device *dev = &pdev->dev; + struct dw_pcie_rp *pp; + int irq, ret; + u32 val; + + if (!IS_ENABLED(CONFIG_PCIE_ROCKCHIP_DW_HOST)) + return -ENODEV; + + irq = platform_get_irq_byname(pdev, "sys"); + if (irq < 0) + return irq; + + ret = devm_request_threaded_irq(dev, irq, NULL, + rockchip_pcie_rc_sys_irq_thread, + IRQF_ONESHOT, "pcie-sys-rc", rockchip); + if (ret) { + dev_err(dev, "failed to request PCIe sys IRQ\n"); + return ret; + } + + /* LTSSM enable control mode */ + val = FIELD_PREP_WM16(PCIE_LTSSM_ENABLE_ENHANCE, 1); + rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_HOT_RESET_CTRL); + + rockchip_pcie_writel_apb(rockchip, + PCIE_CLIENT_SET_MODE(PCIE_CLIENT_MODE_RC), + PCIE_CLIENT_GENERAL_CON); + + pp = &rockchip->pci.pp; + pp->ops = &rockchip_pcie_host_ops; + + ret = dw_pcie_host_init(pp); + if (ret) { + dev_err(dev, "failed to initialize host\n"); + return ret; + } + + /* unmask hot reset/link-down reset */ + val = FIELD_PREP_WM16(PCIE_LINK_REQ_RST_NOT_INT, 0); + rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_INTR_MASK_MISC); + + return ret; +} + +static int rockchip_pcie_configure_ep(struct platform_device *pdev, + struct rockchip_pcie *rockchip) +{ + struct device *dev = &pdev->dev; + int irq, ret; + u32 val; + + if (!IS_ENABLED(CONFIG_PCIE_ROCKCHIP_DW_EP)) + return -ENODEV; + + irq = platform_get_irq_byname(pdev, "sys"); + if (irq < 0) + return irq; + + ret = devm_request_threaded_irq(dev, irq, NULL, + rockchip_pcie_ep_sys_irq_thread, + IRQF_ONESHOT, "pcie-sys-ep", rockchip); + if (ret) { + dev_err(dev, "failed to request PCIe sys IRQ\n"); + return ret; + } + + /* + * LTSSM enable control mode, and automatically delay link training on + * hot reset/link-down reset. + */ + val = FIELD_PREP_WM16(PCIE_LTSSM_ENABLE_ENHANCE, 1) | + FIELD_PREP_WM16(PCIE_LTSSM_APP_DLY2_EN, 1); + rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_HOT_RESET_CTRL); + + rockchip_pcie_writel_apb(rockchip, + PCIE_CLIENT_SET_MODE(PCIE_CLIENT_MODE_EP), + PCIE_CLIENT_GENERAL_CON); + + rockchip->pci.ep.ops = &rockchip_pcie_ep_ops; + rockchip->pci.ep.page_size = SZ_64K; + + dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); + + ret = dw_pcie_ep_init(&rockchip->pci.ep); + if (ret) { + dev_err(dev, "failed to initialize endpoint\n"); + return ret; + } + + ret = dw_pcie_ep_init_registers(&rockchip->pci.ep); + if (ret) { + dev_err(dev, "failed to initialize DWC endpoint registers\n"); + dw_pcie_ep_deinit(&rockchip->pci.ep); + return ret; + } + + pci_epc_init_notify(rockchip->pci.ep.epc); + + /* unmask DLL up/down indicator and hot reset/link-down reset */ + val = FIELD_PREP_WM16(PCIE_RDLH_LINK_UP_CHGED, 0) | + FIELD_PREP_WM16(PCIE_LINK_REQ_RST_NOT_INT, 0); + rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_INTR_MASK_MISC); + + return ret; +} + +static int rockchip_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct rockchip_pcie *rockchip; + const struct rockchip_pcie_of_data *data; + int ret; + + data = of_device_get_match_data(dev); + if (!data) + return -EINVAL; + + rockchip = devm_kzalloc(dev, sizeof(*rockchip), GFP_KERNEL); + if (!rockchip) + return -ENOMEM; + + platform_set_drvdata(pdev, rockchip); + + rockchip->pci.dev = dev; + rockchip->pci.ops = &dw_pcie_ops; + rockchip->data = data; + + /* Default N_FTS value (210) is broken, override it to 255 */ + rockchip->pci.n_fts[0] = 255; /* Gen1 */ + rockchip->pci.n_fts[1] = 255; /* Gen2+ */ + + ret = rockchip_pcie_resource_get(pdev, rockchip); + if (ret) + return ret; + + ret = reset_control_assert(rockchip->rst); + if (ret) + return ret; + + /* DON'T MOVE ME: must be enable before PHY init */ + ret = devm_regulator_get_enable_optional(dev, "vpcie3v3"); + if (ret < 0 && ret != -ENODEV) + return dev_err_probe(dev, ret, + "failed to enable vpcie3v3 regulator\n"); + + ret = rockchip_pcie_phy_init(rockchip); + if (ret) + return dev_err_probe(dev, ret, + "failed to initialize the phy\n"); + + ret = reset_control_deassert(rockchip->rst); + if (ret) + goto deinit_phy; + + ret = rockchip_pcie_clk_init(rockchip); + if (ret) + goto deinit_phy; + + switch (data->mode) { + case DW_PCIE_RC_TYPE: + ret = rockchip_pcie_configure_rc(pdev, rockchip); + if (ret) + goto deinit_clk; + break; + case DW_PCIE_EP_TYPE: + ret = rockchip_pcie_configure_ep(pdev, rockchip); + if (ret) + goto deinit_clk; + break; + default: + dev_err(dev, "INVALID device type %d\n", data->mode); + ret = -EINVAL; + goto deinit_clk; + } + + return 0; + +deinit_clk: + clk_bulk_disable_unprepare(rockchip->clk_cnt, rockchip->clks); +deinit_phy: + rockchip_pcie_phy_deinit(rockchip); + + return ret; +} + +static int rockchip_pcie_rc_reset_root_port(struct pci_host_bridge *bridge, + struct pci_dev *pdev) +{ + struct pci_bus *bus = bridge->bus; + struct dw_pcie_rp *pp = bus->sysdata; + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct rockchip_pcie *rockchip = to_rockchip_pcie(pci); + struct device *dev = rockchip->pci.dev; + u32 val; + int ret; + + dw_pcie_stop_link(pci); + clk_bulk_disable_unprepare(rockchip->clk_cnt, rockchip->clks); + rockchip_pcie_phy_deinit(rockchip); + + ret = reset_control_assert(rockchip->rst); + if (ret) + return ret; + + ret = rockchip_pcie_phy_init(rockchip); + if (ret) + return ret; + + ret = reset_control_deassert(rockchip->rst); + if (ret) + goto deinit_phy; + + ret = rockchip_pcie_clk_init(rockchip); + if (ret) + goto deinit_phy; + + ret = pp->ops->init(pp); + if (ret) { + dev_err(dev, "Host init failed: %d\n", ret); + goto deinit_clk; + } + + /* LTSSM enable control mode */ + val = FIELD_PREP_WM16(PCIE_LTSSM_ENABLE_ENHANCE, 1); + rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_HOT_RESET_CTRL); + + rockchip_pcie_writel_apb(rockchip, + PCIE_CLIENT_SET_MODE(PCIE_CLIENT_MODE_RC), + PCIE_CLIENT_GENERAL_CON); + + ret = dw_pcie_setup_rc(pp); + if (ret) { + dev_err(dev, "Failed to setup RC: %d\n", ret); + goto deinit_clk; + } + + /* unmask hot reset/link-down reset */ + val = FIELD_PREP_WM16(PCIE_LINK_REQ_RST_NOT_INT, 0); + rockchip_pcie_writel_apb(rockchip, val, PCIE_CLIENT_INTR_MASK_MISC); + + ret = dw_pcie_start_link(pci); + if (ret) + goto deinit_clk; + + /* Ignore errors, the link may come up later */ + dw_pcie_wait_for_link(pci); + dev_dbg(dev, "Root Port reset completed\n"); + return ret; + +deinit_clk: + clk_bulk_disable_unprepare(rockchip->clk_cnt, rockchip->clks); +deinit_phy: + rockchip_pcie_phy_deinit(rockchip); + + return ret; +} + +static const struct rockchip_pcie_of_data rockchip_pcie_rc_of_data_rk3568 = { + .mode = DW_PCIE_RC_TYPE, +}; + +static const struct rockchip_pcie_of_data rockchip_pcie_ep_of_data_rk3568 = { + .mode = DW_PCIE_EP_TYPE, + .epc_features = &rockchip_pcie_epc_features_rk3568, +}; + +static const struct rockchip_pcie_of_data rockchip_pcie_ep_of_data_rk3588 = { + .mode = DW_PCIE_EP_TYPE, + .epc_features = &rockchip_pcie_epc_features_rk3588, +}; + +static const struct of_device_id rockchip_pcie_of_match[] = { + { + .compatible = "rockchip,rk3568-pcie", + .data = &rockchip_pcie_rc_of_data_rk3568, + }, + { + .compatible = "rockchip,rk3568-pcie-ep", + .data = &rockchip_pcie_ep_of_data_rk3568, + }, + { + .compatible = "rockchip,rk3588-pcie-ep", + .data = &rockchip_pcie_ep_of_data_rk3588, + }, + {}, +}; + +static struct platform_driver rockchip_pcie_driver = { + .driver = { + .name = "rockchip-dw-pcie", + .of_match_table = rockchip_pcie_of_match, + .suppress_bind_attrs = true, + }, + .probe = rockchip_pcie_probe, +}; +builtin_platform_driver(rockchip_pcie_driver); diff --git a/drivers/pci/controller/dwc/pcie-eswin.c b/drivers/pci/controller/dwc/pcie-eswin.c new file mode 100644 index 000000000..ce8d64f8a --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-eswin.c @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ESWIN PCIe Root Complex driver + * + * Copyright 2026, Beijing ESWIN Computing Technology Co., Ltd. + * + * Authors: Yu Ning <ningyu@eswincomputing.com> + * Senchuan Zhang <zhangsenchuan@eswincomputing.com> + * Yanghui Ou <ouyanghui@eswincomputing.com> + */ + +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/resource.h> +#include <linux/reset.h> +#include <linux/types.h> + +#include "pcie-designware.h" + +/* ELBI registers */ +#define PCIEELBI_CTRL0_OFFSET 0x0 +#define PCIEELBI_STATUS0_OFFSET 0x100 + +/* LTSSM register fields */ +#define PCIEELBI_APP_LTSSM_ENABLE BIT(5) + +/* APP_HOLD_PHY_RST register fields */ +#define PCIEELBI_APP_HOLD_PHY_RST BIT(6) + +/* PM_SEL_AUX_CLK register fields */ +#define PCIEELBI_PM_SEL_AUX_CLK BIT(16) + +/* DEV_TYPE register fields */ +#define PCIEELBI_CTRL0_DEV_TYPE GENMASK(3, 0) + +/* Vendor and device ID value */ +#define PCI_VENDOR_ID_ESWIN 0x1fe1 +#define PCI_DEVICE_ID_ESWIN_EIC7700 0x2030 + +#define ESWIN_NUM_RSTS ARRAY_SIZE(eswin_pcie_rsts) + +static const char * const eswin_pcie_rsts[] = { + "pwr", + "dbi", +}; + +struct eswin_pcie_data { + bool skip_l23; +}; + +struct eswin_pcie_port { + struct list_head list; + struct reset_control *perst; + int num_lanes; +}; + +struct eswin_pcie { + struct dw_pcie pci; + struct clk_bulk_data *clks; + struct reset_control_bulk_data resets[ESWIN_NUM_RSTS]; + struct list_head ports; + const struct eswin_pcie_data *data; + int num_clks; +}; + +#define to_eswin_pcie(x) dev_get_drvdata((x)->dev) + +static int eswin_pcie_start_link(struct dw_pcie *pci) +{ + u32 val; + + /* Enable LTSSM */ + val = readl_relaxed(pci->elbi_base + PCIEELBI_CTRL0_OFFSET); + val |= PCIEELBI_APP_LTSSM_ENABLE; + writel_relaxed(val, pci->elbi_base + PCIEELBI_CTRL0_OFFSET); + + return 0; +} + +static bool eswin_pcie_link_up(struct dw_pcie *pci) +{ + u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + u16 val = dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKSTA); + + return val & PCI_EXP_LNKSTA_DLLLA; +} + +static int eswin_pcie_perst_reset(struct eswin_pcie_port *port, + struct eswin_pcie *pcie) +{ + int ret; + + ret = reset_control_assert(port->perst); + if (ret) { + dev_err(pcie->pci.dev, "Failed to assert PERST#\n"); + return ret; + } + + /* Ensure that PERST# has been asserted for at least 100 ms */ + msleep(PCIE_T_PVPERL_MS); + + ret = reset_control_deassert(port->perst); + if (ret) { + dev_err(pcie->pci.dev, "Failed to deassert PERST#\n"); + return ret; + } + + return 0; +} + +static void eswin_pcie_assert(struct eswin_pcie *pcie) +{ + struct eswin_pcie_port *port; + + list_for_each_entry(port, &pcie->ports, list) + reset_control_assert(port->perst); + reset_control_bulk_assert(ESWIN_NUM_RSTS, pcie->resets); +} + +static int eswin_pcie_parse_port(struct eswin_pcie *pcie, + struct device_node *node) +{ + struct device *dev = pcie->pci.dev; + struct eswin_pcie_port *port; + + port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); + if (!port) + return -ENOMEM; + + port->perst = of_reset_control_get_exclusive(node, "perst"); + if (IS_ERR(port->perst)) { + dev_err(dev, "Failed to get PERST# reset\n"); + return PTR_ERR(port->perst); + } + + /* + * TODO: Since the Root Port node is separated out by pcie devicetree, + * the DWC core initialization code can't parse the num-lanes attribute + * in the Root Port. Before entering the DWC core initialization code, + * the platform driver code parses the Root Port node. The ESWIN only + * supports one Root Port node, and the num-lanes attribute is suitable + * for the case of one Root Port. + */ + if (!of_property_read_u32(node, "num-lanes", &port->num_lanes)) + pcie->pci.num_lanes = port->num_lanes; + + INIT_LIST_HEAD(&port->list); + list_add_tail(&port->list, &pcie->ports); + + return 0; +} + +static int eswin_pcie_parse_ports(struct eswin_pcie *pcie) +{ + struct eswin_pcie_port *port, *tmp; + struct device *dev = pcie->pci.dev; + int ret; + + for_each_available_child_of_node_scoped(dev->of_node, of_port) { + ret = eswin_pcie_parse_port(pcie, of_port); + if (ret) + goto err_port; + } + + return 0; + +err_port: + list_for_each_entry_safe(port, tmp, &pcie->ports, list) { + reset_control_put(port->perst); + list_del(&port->list); + } + + return ret; +} + +static int eswin_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct eswin_pcie *pcie = to_eswin_pcie(pci); + struct eswin_pcie_port *port, *tmp; + u32 val; + int ret; + + ret = clk_bulk_prepare_enable(pcie->num_clks, pcie->clks); + if (ret) + return ret; + + /* + * The PWR and DBI reset signals are respectively used to reset the + * PCIe controller and the DBI register. + * + * The PERST# signal is a reset signal that simultaneously controls the + * PCIe controller, PHY, and Endpoint. Before configuring the PHY, the + * PERST# signal must first be deasserted. + * + * The external reference clock is supplied simultaneously to the PHY + * and EP. When the PHY is configurable, the entire chip already has + * stable power and reference clock. The PHY will be ready within 20ms + * after writing app_hold_phy_rst register bit of ELBI register space. + */ + ret = reset_control_bulk_deassert(ESWIN_NUM_RSTS, pcie->resets); + if (ret) { + dev_err(pcie->pci.dev, "Failed to deassert resets\n"); + goto err_deassert; + } + + /* Configure Root Port type */ + val = readl_relaxed(pci->elbi_base + PCIEELBI_CTRL0_OFFSET); + FIELD_MODIFY(PCIEELBI_CTRL0_DEV_TYPE, &val, PCI_EXP_TYPE_ROOT_PORT); + writel_relaxed(val, pci->elbi_base + PCIEELBI_CTRL0_OFFSET); + + list_for_each_entry(port, &pcie->ports, list) { + ret = eswin_pcie_perst_reset(port, pcie); + if (ret) + goto err_perst; + } + + /* Configure app_hold_phy_rst */ + val = readl_relaxed(pci->elbi_base + PCIEELBI_CTRL0_OFFSET); + val &= ~PCIEELBI_APP_HOLD_PHY_RST; + writel_relaxed(val, pci->elbi_base + PCIEELBI_CTRL0_OFFSET); + + /* The maximum waiting time for the clock switch lock is 20ms */ + ret = readl_poll_timeout(pci->elbi_base + PCIEELBI_STATUS0_OFFSET, val, + !(val & PCIEELBI_PM_SEL_AUX_CLK), 1000, + 20000); + if (ret) { + dev_err(pci->dev, "Timeout waiting for PM_SEL_AUX_CLK ready\n"); + goto err_phy_init; + } + + /* + * Configure ESWIN VID:DID for Root Port as the default values are + * invalid. + */ + dw_pcie_dbi_ro_wr_en(pci); + dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, PCI_VENDOR_ID_ESWIN); + dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, PCI_DEVICE_ID_ESWIN_EIC7700); + dw_pcie_dbi_ro_wr_dis(pci); + + return 0; + +err_phy_init: + list_for_each_entry(port, &pcie->ports, list) + reset_control_assert(port->perst); +err_perst: + reset_control_bulk_assert(ESWIN_NUM_RSTS, pcie->resets); +err_deassert: + clk_bulk_disable_unprepare(pcie->num_clks, pcie->clks); + list_for_each_entry_safe(port, tmp, &pcie->ports, list) { + reset_control_put(port->perst); + list_del(&port->list); + } + + return ret; +} + +static void eswin_pcie_host_deinit(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct eswin_pcie *pcie = to_eswin_pcie(pci); + + eswin_pcie_assert(pcie); + clk_bulk_disable_unprepare(pcie->num_clks, pcie->clks); +} + +static void eswin_pcie_pme_turn_off(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct eswin_pcie *pcie = to_eswin_pcie(pci); + + /* + * The ESWIN EIC7700 SoC lacks hardware support for the L2/L3 low-power + * link states. It cannot enter the L2/L3 Ready state through the + * PME_Turn_Off/PME_To_Ack handshake protocol. To avoid this problem, + * the skip_l23_ready has been set. + */ + pp->skip_l23_ready = pcie->data->skip_l23; +} + +static const struct dw_pcie_host_ops eswin_pcie_host_ops = { + .init = eswin_pcie_host_init, + .deinit = eswin_pcie_host_deinit, + .pme_turn_off = eswin_pcie_pme_turn_off, +}; + +static const struct dw_pcie_ops dw_pcie_ops = { + .start_link = eswin_pcie_start_link, + .link_up = eswin_pcie_link_up, +}; + +static int eswin_pcie_probe(struct platform_device *pdev) +{ + const struct eswin_pcie_data *data; + struct eswin_pcie_port *port, *tmp; + struct device *dev = &pdev->dev; + struct eswin_pcie *pcie; + struct dw_pcie *pci; + int ret, i; + + data = of_device_get_match_data(dev); + if (!data) + return dev_err_probe(dev, -ENODATA, "No platform data\n"); + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + INIT_LIST_HEAD(&pcie->ports); + + pci = &pcie->pci; + pci->dev = dev; + pci->ops = &dw_pcie_ops; + pci->pp.ops = &eswin_pcie_host_ops; + pcie->data = data; + + pcie->num_clks = devm_clk_bulk_get_all(dev, &pcie->clks); + if (pcie->num_clks < 0) + return dev_err_probe(dev, pcie->num_clks, + "Failed to get pcie clocks\n"); + + for (i = 0; i < ESWIN_NUM_RSTS; i++) + pcie->resets[i].id = eswin_pcie_rsts[i]; + + ret = devm_reset_control_bulk_get_exclusive(dev, ESWIN_NUM_RSTS, + pcie->resets); + if (ret) + return dev_err_probe(dev, ret, "Failed to get resets\n"); + + ret = eswin_pcie_parse_ports(pcie); + if (ret) + return dev_err_probe(dev, ret, "Failed to parse Root Port\n"); + + platform_set_drvdata(pdev, pcie); + + pm_runtime_no_callbacks(dev); + devm_pm_runtime_enable(dev); + ret = pm_runtime_get_sync(dev); + if (ret < 0) + goto err_pm_runtime_put; + + ret = dw_pcie_host_init(&pci->pp); + if (ret) { + dev_err(dev, "Failed to init host\n"); + goto err_init; + } + + return 0; + +err_pm_runtime_put: + list_for_each_entry_safe(port, tmp, &pcie->ports, list) { + reset_control_put(port->perst); + list_del(&port->list); + } +err_init: + pm_runtime_put(dev); + + return ret; +} + +static int eswin_pcie_suspend_noirq(struct device *dev) +{ + struct eswin_pcie *pcie = dev_get_drvdata(dev); + + return dw_pcie_suspend_noirq(&pcie->pci); +} + +static int eswin_pcie_resume_noirq(struct device *dev) +{ + struct eswin_pcie *pcie = dev_get_drvdata(dev); + + return dw_pcie_resume_noirq(&pcie->pci); +} + +static DEFINE_NOIRQ_DEV_PM_OPS(eswin_pcie_pm, eswin_pcie_suspend_noirq, + eswin_pcie_resume_noirq); + +static const struct eswin_pcie_data eswin_eic7700_data = { + .skip_l23 = true, +}; + +static const struct of_device_id eswin_pcie_of_match[] = { + { .compatible = "eswin,eic7700-pcie", .data = &eswin_eic7700_data }, + {} +}; + +static struct platform_driver eswin_pcie_driver = { + .probe = eswin_pcie_probe, + .driver = { + .name = "eswin-pcie", + .of_match_table = eswin_pcie_of_match, + .suppress_bind_attrs = true, + .pm = &eswin_pcie_pm, + }, +}; +builtin_platform_driver(eswin_pcie_driver); + +MODULE_DESCRIPTION("ESWIN PCIe Root Complex driver"); +MODULE_AUTHOR("Yu Ning <ningyu@eswincomputing.com>"); +MODULE_AUTHOR("Senchuan Zhang <zhangsenchuan@eswincomputing.com>"); +MODULE_AUTHOR("Yanghui Ou <ouyanghui@eswincomputing.com>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pci/controller/dwc/pcie-fu740.c b/drivers/pci/controller/dwc/pcie-fu740.c new file mode 100644 index 000000000..d0a34f680 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-fu740.c @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * FU740 DesignWare PCIe Controller integration + * Copyright (C) 2019-2021 SiFive, Inc. + * Paul Walmsley + * Greentime Hu + * + * Based in part on the i.MX6 PCIe host controller shim which is: + * + * Copyright (C) 2013 Kosagi + * https://www.kosagi.com + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/gpio/consumer.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/resource.h> +#include <linux/types.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/reset.h> + +#include "pcie-designware.h" + +#define to_fu740_pcie(x) dev_get_drvdata((x)->dev) + +struct fu740_pcie { + struct dw_pcie pci; + void __iomem *mgmt_base; + struct gpio_desc *reset; + struct gpio_desc *pwren; + struct clk *pcie_aux; + struct reset_control *rst; +}; + +#define SIFIVE_DEVICESRESETREG 0x28 + +#define PCIEX8MGMT_PERST_N 0x0 +#define PCIEX8MGMT_APP_LTSSM_ENABLE 0x10 +#define PCIEX8MGMT_APP_HOLD_PHY_RST 0x18 +#define PCIEX8MGMT_DEVICE_TYPE 0x708 +#define PCIEX8MGMT_PHY0_CR_PARA_ADDR 0x860 +#define PCIEX8MGMT_PHY0_CR_PARA_RD_EN 0x870 +#define PCIEX8MGMT_PHY0_CR_PARA_RD_DATA 0x878 +#define PCIEX8MGMT_PHY0_CR_PARA_SEL 0x880 +#define PCIEX8MGMT_PHY0_CR_PARA_WR_DATA 0x888 +#define PCIEX8MGMT_PHY0_CR_PARA_WR_EN 0x890 +#define PCIEX8MGMT_PHY0_CR_PARA_ACK 0x898 +#define PCIEX8MGMT_PHY1_CR_PARA_ADDR 0x8a0 +#define PCIEX8MGMT_PHY1_CR_PARA_RD_EN 0x8b0 +#define PCIEX8MGMT_PHY1_CR_PARA_RD_DATA 0x8b8 +#define PCIEX8MGMT_PHY1_CR_PARA_SEL 0x8c0 +#define PCIEX8MGMT_PHY1_CR_PARA_WR_DATA 0x8c8 +#define PCIEX8MGMT_PHY1_CR_PARA_WR_EN 0x8d0 +#define PCIEX8MGMT_PHY1_CR_PARA_ACK 0x8d8 + +#define PCIEX8MGMT_PHY_CDR_TRACK_EN BIT(0) +#define PCIEX8MGMT_PHY_LOS_THRSHLD BIT(5) +#define PCIEX8MGMT_PHY_TERM_EN BIT(9) +#define PCIEX8MGMT_PHY_TERM_ACDC BIT(10) +#define PCIEX8MGMT_PHY_EN BIT(11) +#define PCIEX8MGMT_PHY_INIT_VAL (PCIEX8MGMT_PHY_CDR_TRACK_EN|\ + PCIEX8MGMT_PHY_LOS_THRSHLD|\ + PCIEX8MGMT_PHY_TERM_EN|\ + PCIEX8MGMT_PHY_TERM_ACDC|\ + PCIEX8MGMT_PHY_EN) + +#define PCIEX8MGMT_PHY_LANEN_DIG_ASIC_RX_OVRD_IN_3 0x1008 +#define PCIEX8MGMT_PHY_LANE_OFF 0x100 +#define PCIEX8MGMT_PHY_LANE0_BASE (PCIEX8MGMT_PHY_LANEN_DIG_ASIC_RX_OVRD_IN_3 + 0x100 * 0) +#define PCIEX8MGMT_PHY_LANE1_BASE (PCIEX8MGMT_PHY_LANEN_DIG_ASIC_RX_OVRD_IN_3 + 0x100 * 1) +#define PCIEX8MGMT_PHY_LANE2_BASE (PCIEX8MGMT_PHY_LANEN_DIG_ASIC_RX_OVRD_IN_3 + 0x100 * 2) +#define PCIEX8MGMT_PHY_LANE3_BASE (PCIEX8MGMT_PHY_LANEN_DIG_ASIC_RX_OVRD_IN_3 + 0x100 * 3) + +static void fu740_pcie_assert_reset(struct fu740_pcie *afp) +{ + /* Assert PERST_N GPIO */ + gpiod_set_value_cansleep(afp->reset, 0); + /* Assert controller PERST_N */ + writel_relaxed(0x0, afp->mgmt_base + PCIEX8MGMT_PERST_N); +} + +static void fu740_pcie_deassert_reset(struct fu740_pcie *afp) +{ + /* Deassert controller PERST_N */ + writel_relaxed(0x1, afp->mgmt_base + PCIEX8MGMT_PERST_N); + /* Deassert PERST_N GPIO */ + gpiod_set_value_cansleep(afp->reset, 1); +} + +static void fu740_pcie_power_on(struct fu740_pcie *afp) +{ + gpiod_set_value_cansleep(afp->pwren, 1); + /* + * Ensure that PERST has been asserted for at least 100 ms. + * Section 2.2 of PCI Express Card Electromechanical Specification + * Revision 3.0 + */ + msleep(100); +} + +static void fu740_pcie_drive_reset(struct fu740_pcie *afp) +{ + fu740_pcie_assert_reset(afp); + fu740_pcie_power_on(afp); + fu740_pcie_deassert_reset(afp); +} + +static void fu740_phyregwrite(const uint8_t phy, const uint16_t addr, + const uint16_t wrdata, struct fu740_pcie *afp) +{ + struct device *dev = afp->pci.dev; + void __iomem *phy_cr_para_addr; + void __iomem *phy_cr_para_wr_data; + void __iomem *phy_cr_para_wr_en; + void __iomem *phy_cr_para_ack; + int ret, val; + + /* Setup */ + if (phy) { + phy_cr_para_addr = afp->mgmt_base + PCIEX8MGMT_PHY1_CR_PARA_ADDR; + phy_cr_para_wr_data = afp->mgmt_base + PCIEX8MGMT_PHY1_CR_PARA_WR_DATA; + phy_cr_para_wr_en = afp->mgmt_base + PCIEX8MGMT_PHY1_CR_PARA_WR_EN; + phy_cr_para_ack = afp->mgmt_base + PCIEX8MGMT_PHY1_CR_PARA_ACK; + } else { + phy_cr_para_addr = afp->mgmt_base + PCIEX8MGMT_PHY0_CR_PARA_ADDR; + phy_cr_para_wr_data = afp->mgmt_base + PCIEX8MGMT_PHY0_CR_PARA_WR_DATA; + phy_cr_para_wr_en = afp->mgmt_base + PCIEX8MGMT_PHY0_CR_PARA_WR_EN; + phy_cr_para_ack = afp->mgmt_base + PCIEX8MGMT_PHY0_CR_PARA_ACK; + } + + writel_relaxed(addr, phy_cr_para_addr); + writel_relaxed(wrdata, phy_cr_para_wr_data); + writel_relaxed(1, phy_cr_para_wr_en); + + /* Wait for wait_idle */ + ret = readl_poll_timeout(phy_cr_para_ack, val, val, 10, 5000); + if (ret) + dev_warn(dev, "Wait for wait_idle state failed!\n"); + + /* Clear */ + writel_relaxed(0, phy_cr_para_wr_en); + + /* Wait for ~wait_idle */ + ret = readl_poll_timeout(phy_cr_para_ack, val, !val, 10, 5000); + if (ret) + dev_warn(dev, "Wait for !wait_idle state failed!\n"); +} + +static void fu740_pcie_init_phy(struct fu740_pcie *afp) +{ + /* Enable phy cr_para_sel interfaces */ + writel_relaxed(0x1, afp->mgmt_base + PCIEX8MGMT_PHY0_CR_PARA_SEL); + writel_relaxed(0x1, afp->mgmt_base + PCIEX8MGMT_PHY1_CR_PARA_SEL); + + /* + * Wait 10 cr_para cycles to guarantee that the registers are ready + * to be edited. + */ + ndelay(10); + + /* Set PHY AC termination mode */ + fu740_phyregwrite(0, PCIEX8MGMT_PHY_LANE0_BASE, PCIEX8MGMT_PHY_INIT_VAL, afp); + fu740_phyregwrite(0, PCIEX8MGMT_PHY_LANE1_BASE, PCIEX8MGMT_PHY_INIT_VAL, afp); + fu740_phyregwrite(0, PCIEX8MGMT_PHY_LANE2_BASE, PCIEX8MGMT_PHY_INIT_VAL, afp); + fu740_phyregwrite(0, PCIEX8MGMT_PHY_LANE3_BASE, PCIEX8MGMT_PHY_INIT_VAL, afp); + fu740_phyregwrite(1, PCIEX8MGMT_PHY_LANE0_BASE, PCIEX8MGMT_PHY_INIT_VAL, afp); + fu740_phyregwrite(1, PCIEX8MGMT_PHY_LANE1_BASE, PCIEX8MGMT_PHY_INIT_VAL, afp); + fu740_phyregwrite(1, PCIEX8MGMT_PHY_LANE2_BASE, PCIEX8MGMT_PHY_INIT_VAL, afp); + fu740_phyregwrite(1, PCIEX8MGMT_PHY_LANE3_BASE, PCIEX8MGMT_PHY_INIT_VAL, afp); +} + +static int fu740_pcie_start_link(struct dw_pcie *pci) +{ + struct device *dev = pci->dev; + struct fu740_pcie *afp = dev_get_drvdata(dev); + u8 cap_exp = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + int ret; + u32 orig, tmp; + + /* + * Force 2.5GT/s when starting the link, due to some devices not + * probing at higher speeds. This happens with the PCIe switch + * on the Unmatched board when U-Boot has not initialised the PCIe. + * The fix in U-Boot is to force 2.5GT/s, which then gets cleared + * by the soft reset done by this driver. + */ + dev_dbg(dev, "cap_exp at %x\n", cap_exp); + dw_pcie_dbi_ro_wr_en(pci); + + tmp = dw_pcie_readl_dbi(pci, cap_exp + PCI_EXP_LNKCAP); + orig = tmp & PCI_EXP_LNKCAP_SLS; + tmp &= ~PCI_EXP_LNKCAP_SLS; + tmp |= PCI_EXP_LNKCAP_SLS_2_5GB; + dw_pcie_writel_dbi(pci, cap_exp + PCI_EXP_LNKCAP, tmp); + + /* Enable LTSSM */ + writel_relaxed(0x1, afp->mgmt_base + PCIEX8MGMT_APP_LTSSM_ENABLE); + + ret = dw_pcie_wait_for_link(pci); + if (ret) { + dev_err(dev, "error: link did not start\n"); + goto err; + } + + tmp = dw_pcie_readl_dbi(pci, cap_exp + PCI_EXP_LNKCAP); + if ((tmp & PCI_EXP_LNKCAP_SLS) != orig) { + dev_dbg(dev, "changing speed back to original\n"); + + tmp &= ~PCI_EXP_LNKCAP_SLS; + tmp |= orig; + dw_pcie_writel_dbi(pci, cap_exp + PCI_EXP_LNKCAP, tmp); + + tmp = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); + tmp |= PORT_LOGIC_SPEED_CHANGE; + dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, tmp); + + ret = dw_pcie_wait_for_link(pci); + if (ret) { + dev_err(dev, "error: link did not start at new speed\n"); + goto err; + } + } + + ret = 0; +err: + WARN_ON(ret); /* we assume that errors will be very rare */ + dw_pcie_dbi_ro_wr_dis(pci); + return ret; +} + +static int fu740_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct fu740_pcie *afp = to_fu740_pcie(pci); + struct device *dev = pci->dev; + int ret; + + /* Power on reset */ + fu740_pcie_drive_reset(afp); + + /* Enable pcieauxclk */ + ret = clk_prepare_enable(afp->pcie_aux); + if (ret) { + dev_err(dev, "unable to enable pcie_aux clock\n"); + return ret; + } + + /* + * Assert hold_phy_rst (hold the controller LTSSM in reset after + * power_up_rst_n for register programming with cr_para) + */ + writel_relaxed(0x1, afp->mgmt_base + PCIEX8MGMT_APP_HOLD_PHY_RST); + + /* Deassert power_up_rst_n */ + ret = reset_control_deassert(afp->rst); + if (ret) { + dev_err(dev, "unable to deassert pcie_power_up_rst_n\n"); + return ret; + } + + fu740_pcie_init_phy(afp); + + /* Disable pcieauxclk */ + clk_disable_unprepare(afp->pcie_aux); + /* Clear hold_phy_rst */ + writel_relaxed(0x0, afp->mgmt_base + PCIEX8MGMT_APP_HOLD_PHY_RST); + /* Enable pcieauxclk */ + clk_prepare_enable(afp->pcie_aux); + /* Set RC mode */ + writel_relaxed(0x4, afp->mgmt_base + PCIEX8MGMT_DEVICE_TYPE); + + return 0; +} + +static const struct dw_pcie_host_ops fu740_pcie_host_ops = { + .init = fu740_pcie_host_init, +}; + +static const struct dw_pcie_ops dw_pcie_ops = { + .start_link = fu740_pcie_start_link, +}; + +static int fu740_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dw_pcie *pci; + struct fu740_pcie *afp; + + afp = devm_kzalloc(dev, sizeof(*afp), GFP_KERNEL); + if (!afp) + return -ENOMEM; + pci = &afp->pci; + pci->dev = dev; + pci->ops = &dw_pcie_ops; + pci->pp.ops = &fu740_pcie_host_ops; + pci->pp.num_vectors = MAX_MSI_IRQS; + + /* SiFive specific region: mgmt */ + afp->mgmt_base = devm_platform_ioremap_resource_byname(pdev, "mgmt"); + if (IS_ERR(afp->mgmt_base)) + return PTR_ERR(afp->mgmt_base); + + /* Fetch GPIOs */ + afp->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW); + if (IS_ERR(afp->reset)) + return dev_err_probe(dev, PTR_ERR(afp->reset), "unable to get reset-gpios\n"); + + afp->pwren = devm_gpiod_get_optional(dev, "pwren", GPIOD_OUT_LOW); + if (IS_ERR(afp->pwren)) + return dev_err_probe(dev, PTR_ERR(afp->pwren), "unable to get pwren-gpios\n"); + + /* Fetch clocks */ + afp->pcie_aux = devm_clk_get(dev, "pcie_aux"); + if (IS_ERR(afp->pcie_aux)) + return dev_err_probe(dev, PTR_ERR(afp->pcie_aux), + "pcie_aux clock source missing or invalid\n"); + + /* Fetch reset */ + afp->rst = devm_reset_control_get_exclusive(dev, NULL); + if (IS_ERR(afp->rst)) + return dev_err_probe(dev, PTR_ERR(afp->rst), "unable to get reset\n"); + + platform_set_drvdata(pdev, afp); + + return dw_pcie_host_init(&pci->pp); +} + +static void fu740_pcie_shutdown(struct platform_device *pdev) +{ + struct fu740_pcie *afp = platform_get_drvdata(pdev); + + /* Bring down link, so bootloader gets clean state in case of reboot */ + fu740_pcie_assert_reset(afp); +} + +static const struct of_device_id fu740_pcie_of_match[] = { + { .compatible = "sifive,fu740-pcie", }, + {}, +}; + +static struct platform_driver fu740_pcie_driver = { + .driver = { + .name = "fu740-pcie", + .of_match_table = fu740_pcie_of_match, + .suppress_bind_attrs = true, + }, + .probe = fu740_pcie_probe, + .shutdown = fu740_pcie_shutdown, +}; + +builtin_platform_driver(fu740_pcie_driver); diff --git a/drivers/pci/controller/dwc/pcie-hisi.c b/drivers/pci/controller/dwc/pcie-hisi.c new file mode 100644 index 000000000..3c17897e5 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-hisi.c @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for HiSilicon SoCs + * + * Copyright (C) 2015 HiSilicon Co., Ltd. http://www.hisilicon.com + * + * Authors: Zhou Wang <wangzhou1@hisilicon.com> + * Dacai Zhu <zhudacai@hisilicon.com> + * Gabriele Paoloni <gabriele.paoloni@huawei.com> + */ +#include <linux/interrupt.h> +#include <linux/init.h> +#include <linux/platform_device.h> +#include <linux/pci.h> +#include <linux/pci-acpi.h> +#include <linux/pci-ecam.h> +#include "../../pci.h" +#include "../pci-host-common.h" + +#if defined(CONFIG_PCI_HISI) || (defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS)) + +struct hisi_pcie { + void __iomem *reg_base; +}; + +static int hisi_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where, + int size, u32 *val) +{ + struct pci_config_window *cfg = bus->sysdata; + int dev = PCI_SLOT(devfn); + + if (bus->number == cfg->busr.start) { + /* access only one slot on each root port */ + if (dev > 0) + return PCIBIOS_DEVICE_NOT_FOUND; + else + return pci_generic_config_read32(bus, devfn, where, + size, val); + } + + return pci_generic_config_read(bus, devfn, where, size, val); +} + +static int hisi_pcie_wr_conf(struct pci_bus *bus, u32 devfn, + int where, int size, u32 val) +{ + struct pci_config_window *cfg = bus->sysdata; + int dev = PCI_SLOT(devfn); + + if (bus->number == cfg->busr.start) { + /* access only one slot on each root port */ + if (dev > 0) + return PCIBIOS_DEVICE_NOT_FOUND; + else + return pci_generic_config_write32(bus, devfn, where, + size, val); + } + + return pci_generic_config_write(bus, devfn, where, size, val); +} + +static void __iomem *hisi_pcie_map_bus(struct pci_bus *bus, unsigned int devfn, + int where) +{ + struct pci_config_window *cfg = bus->sysdata; + struct hisi_pcie *pcie = cfg->priv; + + if (bus->number == cfg->busr.start) + return pcie->reg_base + where; + else + return pci_ecam_map_bus(bus, devfn, where); +} + +#if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS) + +static int hisi_pcie_init(struct pci_config_window *cfg) +{ + struct device *dev = cfg->parent; + struct hisi_pcie *pcie; + struct acpi_device *adev = to_acpi_device(dev); + struct acpi_pci_root *root = acpi_driver_data(adev); + struct resource *res; + int ret; + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + /* + * Retrieve RC base and size from a HISI0081 device with _UID + * matching our segment. + */ + res = devm_kzalloc(dev, sizeof(*res), GFP_KERNEL); + if (!res) + return -ENOMEM; + + ret = acpi_get_rc_resources(dev, "HISI0081", root->segment, res); + if (ret) { + dev_err(dev, "can't get rc base address\n"); + return -ENOMEM; + } + + pcie->reg_base = devm_pci_remap_cfgspace(dev, res->start, resource_size(res)); + if (!pcie->reg_base) + return -ENOMEM; + + cfg->priv = pcie; + return 0; +} + +const struct pci_ecam_ops hisi_pcie_ops = { + .init = hisi_pcie_init, + .pci_ops = { + .map_bus = hisi_pcie_map_bus, + .read = hisi_pcie_rd_conf, + .write = hisi_pcie_wr_conf, + } +}; + +#endif + +#ifdef CONFIG_PCI_HISI + +static int hisi_pcie_platform_init(struct pci_config_window *cfg) +{ + struct device *dev = cfg->parent; + struct hisi_pcie *pcie; + struct platform_device *pdev = to_platform_device(dev); + struct resource *res; + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 1); + if (!res) { + dev_err(dev, "missing \"reg[1]\"property\n"); + return -EINVAL; + } + + pcie->reg_base = devm_pci_remap_cfgspace(dev, res->start, resource_size(res)); + if (!pcie->reg_base) + return -ENOMEM; + + cfg->priv = pcie; + return 0; +} + +static const struct pci_ecam_ops hisi_pcie_platform_ops = { + .init = hisi_pcie_platform_init, + .pci_ops = { + .map_bus = hisi_pcie_map_bus, + .read = hisi_pcie_rd_conf, + .write = hisi_pcie_wr_conf, + } +}; + +static const struct of_device_id hisi_pcie_almost_ecam_of_match[] = { + { + .compatible = "hisilicon,hip06-pcie-ecam", + .data = &hisi_pcie_platform_ops, + }, + { + .compatible = "hisilicon,hip07-pcie-ecam", + .data = &hisi_pcie_platform_ops, + }, + {}, +}; + +static struct platform_driver hisi_pcie_almost_ecam_driver = { + .probe = pci_host_common_probe, + .driver = { + .name = "hisi-pcie-almost-ecam", + .of_match_table = hisi_pcie_almost_ecam_of_match, + .suppress_bind_attrs = true, + }, +}; +builtin_platform_driver(hisi_pcie_almost_ecam_driver); + +#endif +#endif diff --git a/drivers/pci/controller/dwc/pcie-histb.c b/drivers/pci/controller/dwc/pcie-histb.c new file mode 100644 index 000000000..a52071589 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-histb.c @@ -0,0 +1,451 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for HiSilicon STB SoCs + * + * Copyright (C) 2016-2017 HiSilicon Co., Ltd. http://www.hisilicon.com + * + * Authors: Ruqiang Ju <juruqiang@hisilicon.com> + * Jianguo Sun <sunjianguo1@huawei.com> + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/gpio/consumer.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/pci.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/resource.h> +#include <linux/reset.h> + +#include "pcie-designware.h" + +#define to_histb_pcie(x) dev_get_drvdata((x)->dev) + +#define PCIE_SYS_CTRL0 0x0000 +#define PCIE_SYS_CTRL1 0x0004 +#define PCIE_SYS_CTRL7 0x001C +#define PCIE_SYS_CTRL13 0x0034 +#define PCIE_SYS_CTRL15 0x003C +#define PCIE_SYS_CTRL16 0x0040 +#define PCIE_SYS_CTRL17 0x0044 + +#define PCIE_SYS_STAT0 0x0100 +#define PCIE_SYS_STAT4 0x0110 + +#define PCIE_RDLH_LINK_UP BIT(5) +#define PCIE_XMLH_LINK_UP BIT(15) +#define PCIE_ELBI_SLV_DBI_ENABLE BIT(21) +#define PCIE_APP_LTSSM_ENABLE BIT(11) + +#define PCIE_DEVICE_TYPE_MASK GENMASK(31, 28) +#define PCIE_WM_EP 0 +#define PCIE_WM_LEGACY BIT(1) +#define PCIE_WM_RC BIT(30) + +#define PCIE_LTSSM_STATE_MASK GENMASK(5, 0) +#define PCIE_LTSSM_STATE_ACTIVE 0x11 + +struct histb_pcie { + struct dw_pcie *pci; + struct clk *aux_clk; + struct clk *pipe_clk; + struct clk *sys_clk; + struct clk *bus_clk; + struct phy *phy; + struct reset_control *soft_reset; + struct reset_control *sys_reset; + struct reset_control *bus_reset; + void __iomem *ctrl; + struct gpio_desc *reset_gpio; + struct regulator *vpcie; +}; + +static u32 histb_pcie_readl(struct histb_pcie *histb_pcie, u32 reg) +{ + return readl(histb_pcie->ctrl + reg); +} + +static void histb_pcie_writel(struct histb_pcie *histb_pcie, u32 reg, u32 val) +{ + writel(val, histb_pcie->ctrl + reg); +} + +static void histb_pcie_dbi_w_mode(struct dw_pcie_rp *pp, bool enable) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct histb_pcie *hipcie = to_histb_pcie(pci); + u32 val; + + val = histb_pcie_readl(hipcie, PCIE_SYS_CTRL0); + if (enable) + val |= PCIE_ELBI_SLV_DBI_ENABLE; + else + val &= ~PCIE_ELBI_SLV_DBI_ENABLE; + histb_pcie_writel(hipcie, PCIE_SYS_CTRL0, val); +} + +static void histb_pcie_dbi_r_mode(struct dw_pcie_rp *pp, bool enable) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct histb_pcie *hipcie = to_histb_pcie(pci); + u32 val; + + val = histb_pcie_readl(hipcie, PCIE_SYS_CTRL1); + if (enable) + val |= PCIE_ELBI_SLV_DBI_ENABLE; + else + val &= ~PCIE_ELBI_SLV_DBI_ENABLE; + histb_pcie_writel(hipcie, PCIE_SYS_CTRL1, val); +} + +static u32 histb_pcie_read_dbi(struct dw_pcie *pci, void __iomem *base, + u32 reg, size_t size) +{ + u32 val; + + histb_pcie_dbi_r_mode(&pci->pp, true); + dw_pcie_read(base + reg, size, &val); + histb_pcie_dbi_r_mode(&pci->pp, false); + + return val; +} + +static void histb_pcie_write_dbi(struct dw_pcie *pci, void __iomem *base, + u32 reg, size_t size, u32 val) +{ + histb_pcie_dbi_w_mode(&pci->pp, true); + dw_pcie_write(base + reg, size, val); + histb_pcie_dbi_w_mode(&pci->pp, false); +} + +static int histb_pcie_rd_own_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(bus->sysdata); + + if (PCI_SLOT(devfn)) + return PCIBIOS_DEVICE_NOT_FOUND; + + *val = dw_pcie_read_dbi(pci, where, size); + return PCIBIOS_SUCCESSFUL; +} + +static int histb_pcie_wr_own_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(bus->sysdata); + + if (PCI_SLOT(devfn)) + return PCIBIOS_DEVICE_NOT_FOUND; + + dw_pcie_write_dbi(pci, where, size, val); + return PCIBIOS_SUCCESSFUL; +} + +static struct pci_ops histb_pci_ops = { + .read = histb_pcie_rd_own_conf, + .write = histb_pcie_wr_own_conf, +}; + +static bool histb_pcie_link_up(struct dw_pcie *pci) +{ + struct histb_pcie *hipcie = to_histb_pcie(pci); + u32 regval; + u32 status; + + regval = histb_pcie_readl(hipcie, PCIE_SYS_STAT0); + status = histb_pcie_readl(hipcie, PCIE_SYS_STAT4); + status &= PCIE_LTSSM_STATE_MASK; + return ((regval & PCIE_XMLH_LINK_UP) && (regval & PCIE_RDLH_LINK_UP) && + (status == PCIE_LTSSM_STATE_ACTIVE)); +} + +static int histb_pcie_start_link(struct dw_pcie *pci) +{ + struct histb_pcie *hipcie = to_histb_pcie(pci); + u32 regval; + + /* assert LTSSM enable */ + regval = histb_pcie_readl(hipcie, PCIE_SYS_CTRL7); + regval |= PCIE_APP_LTSSM_ENABLE; + histb_pcie_writel(hipcie, PCIE_SYS_CTRL7, regval); + + return 0; +} + +static int histb_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct histb_pcie *hipcie = to_histb_pcie(pci); + u32 regval; + + pp->bridge->ops = &histb_pci_ops; + + /* PCIe RC work mode */ + regval = histb_pcie_readl(hipcie, PCIE_SYS_CTRL0); + regval &= ~PCIE_DEVICE_TYPE_MASK; + regval |= PCIE_WM_RC; + histb_pcie_writel(hipcie, PCIE_SYS_CTRL0, regval); + + return 0; +} + +static const struct dw_pcie_host_ops histb_pcie_host_ops = { + .init = histb_pcie_host_init, +}; + +static void histb_pcie_host_disable(struct histb_pcie *hipcie) +{ + reset_control_assert(hipcie->soft_reset); + reset_control_assert(hipcie->sys_reset); + reset_control_assert(hipcie->bus_reset); + + clk_disable_unprepare(hipcie->aux_clk); + clk_disable_unprepare(hipcie->pipe_clk); + clk_disable_unprepare(hipcie->sys_clk); + clk_disable_unprepare(hipcie->bus_clk); + + if (hipcie->reset_gpio) + gpiod_set_value_cansleep(hipcie->reset_gpio, 1); + + if (hipcie->vpcie) + regulator_disable(hipcie->vpcie); +} + +static int histb_pcie_host_enable(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct histb_pcie *hipcie = to_histb_pcie(pci); + struct device *dev = pci->dev; + int ret; + + /* power on PCIe device if have */ + if (hipcie->vpcie) { + ret = regulator_enable(hipcie->vpcie); + if (ret) { + dev_err(dev, "failed to enable regulator: %d\n", ret); + return ret; + } + } + + if (hipcie->reset_gpio) + gpiod_set_value_cansleep(hipcie->reset_gpio, 0); + + ret = clk_prepare_enable(hipcie->bus_clk); + if (ret) { + dev_err(dev, "cannot prepare/enable bus clk\n"); + goto err_bus_clk; + } + + ret = clk_prepare_enable(hipcie->sys_clk); + if (ret) { + dev_err(dev, "cannot prepare/enable sys clk\n"); + goto err_sys_clk; + } + + ret = clk_prepare_enable(hipcie->pipe_clk); + if (ret) { + dev_err(dev, "cannot prepare/enable pipe clk\n"); + goto err_pipe_clk; + } + + ret = clk_prepare_enable(hipcie->aux_clk); + if (ret) { + dev_err(dev, "cannot prepare/enable aux clk\n"); + goto err_aux_clk; + } + + reset_control_assert(hipcie->soft_reset); + reset_control_deassert(hipcie->soft_reset); + + reset_control_assert(hipcie->sys_reset); + reset_control_deassert(hipcie->sys_reset); + + reset_control_assert(hipcie->bus_reset); + reset_control_deassert(hipcie->bus_reset); + + return 0; + +err_aux_clk: + clk_disable_unprepare(hipcie->pipe_clk); +err_pipe_clk: + clk_disable_unprepare(hipcie->sys_clk); +err_sys_clk: + clk_disable_unprepare(hipcie->bus_clk); +err_bus_clk: + if (hipcie->vpcie) + regulator_disable(hipcie->vpcie); + + return ret; +} + +static const struct dw_pcie_ops dw_pcie_ops = { + .read_dbi = histb_pcie_read_dbi, + .write_dbi = histb_pcie_write_dbi, + .link_up = histb_pcie_link_up, + .start_link = histb_pcie_start_link, +}; + +static int histb_pcie_probe(struct platform_device *pdev) +{ + struct histb_pcie *hipcie; + struct dw_pcie *pci; + struct dw_pcie_rp *pp; + struct device *dev = &pdev->dev; + int ret; + + hipcie = devm_kzalloc(dev, sizeof(*hipcie), GFP_KERNEL); + if (!hipcie) + return -ENOMEM; + + pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); + if (!pci) + return -ENOMEM; + + hipcie->pci = pci; + pp = &pci->pp; + pci->dev = dev; + pci->ops = &dw_pcie_ops; + + hipcie->ctrl = devm_platform_ioremap_resource_byname(pdev, "control"); + if (IS_ERR(hipcie->ctrl)) { + dev_err(dev, "cannot get control reg base\n"); + return PTR_ERR(hipcie->ctrl); + } + + pci->dbi_base = devm_platform_ioremap_resource_byname(pdev, "rc-dbi"); + if (IS_ERR(pci->dbi_base)) { + dev_err(dev, "cannot get rc-dbi base\n"); + return PTR_ERR(pci->dbi_base); + } + + hipcie->vpcie = devm_regulator_get_optional(dev, "vpcie"); + if (IS_ERR(hipcie->vpcie)) { + if (PTR_ERR(hipcie->vpcie) != -ENODEV) + return PTR_ERR(hipcie->vpcie); + hipcie->vpcie = NULL; + } + + hipcie->reset_gpio = devm_gpiod_get_optional(dev, "reset", + GPIOD_OUT_HIGH); + ret = PTR_ERR_OR_ZERO(hipcie->reset_gpio); + if (ret) { + dev_err(dev, "unable to request reset gpio: %d\n", ret); + return ret; + } + + ret = gpiod_set_consumer_name(hipcie->reset_gpio, + "PCIe device power control"); + if (ret) { + dev_err(dev, "unable to set reset gpio name: %d\n", ret); + return ret; + } + + hipcie->aux_clk = devm_clk_get(dev, "aux"); + if (IS_ERR(hipcie->aux_clk)) { + dev_err(dev, "Failed to get PCIe aux clk\n"); + return PTR_ERR(hipcie->aux_clk); + } + + hipcie->pipe_clk = devm_clk_get(dev, "pipe"); + if (IS_ERR(hipcie->pipe_clk)) { + dev_err(dev, "Failed to get PCIe pipe clk\n"); + return PTR_ERR(hipcie->pipe_clk); + } + + hipcie->sys_clk = devm_clk_get(dev, "sys"); + if (IS_ERR(hipcie->sys_clk)) { + dev_err(dev, "Failed to get PCIEe sys clk\n"); + return PTR_ERR(hipcie->sys_clk); + } + + hipcie->bus_clk = devm_clk_get(dev, "bus"); + if (IS_ERR(hipcie->bus_clk)) { + dev_err(dev, "Failed to get PCIe bus clk\n"); + return PTR_ERR(hipcie->bus_clk); + } + + hipcie->soft_reset = devm_reset_control_get(dev, "soft"); + if (IS_ERR(hipcie->soft_reset)) { + dev_err(dev, "couldn't get soft reset\n"); + return PTR_ERR(hipcie->soft_reset); + } + + hipcie->sys_reset = devm_reset_control_get(dev, "sys"); + if (IS_ERR(hipcie->sys_reset)) { + dev_err(dev, "couldn't get sys reset\n"); + return PTR_ERR(hipcie->sys_reset); + } + + hipcie->bus_reset = devm_reset_control_get(dev, "bus"); + if (IS_ERR(hipcie->bus_reset)) { + dev_err(dev, "couldn't get bus reset\n"); + return PTR_ERR(hipcie->bus_reset); + } + + hipcie->phy = devm_phy_get(dev, "phy"); + if (IS_ERR(hipcie->phy)) { + dev_info(dev, "no pcie-phy found\n"); + hipcie->phy = NULL; + /* fall through here! + * if no pcie-phy found, phy init + * should be done under boot! + */ + } else { + phy_init(hipcie->phy); + } + + pp->ops = &histb_pcie_host_ops; + + platform_set_drvdata(pdev, hipcie); + + ret = histb_pcie_host_enable(pp); + if (ret) { + dev_err(dev, "failed to enable host\n"); + goto err_exit_phy; + } + + ret = dw_pcie_host_init(pp); + if (ret) { + dev_err(dev, "failed to initialize host\n"); + goto err_exit_phy; + } + + return 0; + +err_exit_phy: + phy_exit(hipcie->phy); + + return ret; +} + +static void histb_pcie_remove(struct platform_device *pdev) +{ + struct histb_pcie *hipcie = platform_get_drvdata(pdev); + + histb_pcie_host_disable(hipcie); + + phy_exit(hipcie->phy); +} + +static const struct of_device_id histb_pcie_of_match[] = { + { .compatible = "hisilicon,hi3798cv200-pcie", }, + {}, +}; +MODULE_DEVICE_TABLE(of, histb_pcie_of_match); + +static struct platform_driver histb_pcie_platform_driver = { + .probe = histb_pcie_probe, + .remove = histb_pcie_remove, + .driver = { + .name = "histb-pcie", + .of_match_table = histb_pcie_of_match, + }, +}; +module_platform_driver(histb_pcie_platform_driver); + +MODULE_DESCRIPTION("HiSilicon STB PCIe host controller driver"); diff --git a/drivers/pci/controller/dwc/pcie-intel-gw.c b/drivers/pci/controller/dwc/pcie-intel-gw.c new file mode 100644 index 000000000..348e579e6 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-intel-gw.c @@ -0,0 +1,474 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for Intel Gateway SoCs + * + * Copyright (c) 2019 Intel Corporation. + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/gpio/consumer.h> +#include <linux/iopoll.h> +#include <linux/pci_regs.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/property.h> +#include <linux/reset.h> + +#include "../../pci.h" +#include "pcie-designware.h" + +#define PORT_AFR_N_FTS_GEN12_DFT (SZ_128 - 1) +#define PORT_AFR_N_FTS_GEN3 180 +#define PORT_AFR_N_FTS_GEN4 196 + +/* PCIe Application logic Registers */ +#define PCIE_APP_CCR 0x10 +#define PCIE_APP_CCR_LTSSM_ENABLE BIT(0) + +#define PCIE_APP_MSG_CR 0x30 +#define PCIE_APP_MSG_XMT_PM_TURNOFF BIT(0) + +#define PCIE_APP_PMC 0x44 +#define PCIE_APP_PMC_IN_L2 BIT(20) + +#define PCIE_APP_IRNEN 0xF4 +#define PCIE_APP_IRNCR 0xF8 +#define PCIE_APP_IRN_AER_REPORT BIT(0) +#define PCIE_APP_IRN_PME BIT(2) +#define PCIE_APP_IRN_RX_VDM_MSG BIT(4) +#define PCIE_APP_IRN_PM_TO_ACK BIT(9) +#define PCIE_APP_IRN_LINK_AUTO_BW_STAT BIT(11) +#define PCIE_APP_IRN_BW_MGT BIT(12) +#define PCIE_APP_IRN_INTA BIT(13) +#define PCIE_APP_IRN_INTB BIT(14) +#define PCIE_APP_IRN_INTC BIT(15) +#define PCIE_APP_IRN_INTD BIT(16) +#define PCIE_APP_IRN_MSG_LTR BIT(18) +#define PCIE_APP_IRN_SYS_ERR_RC BIT(29) + +#define PCIE_APP_IRN_INT \ + (PCIE_APP_IRN_AER_REPORT | PCIE_APP_IRN_PME | \ + PCIE_APP_IRN_RX_VDM_MSG | PCIE_APP_IRN_SYS_ERR_RC | \ + PCIE_APP_IRN_PM_TO_ACK | PCIE_APP_IRN_MSG_LTR | \ + PCIE_APP_IRN_BW_MGT | PCIE_APP_IRN_LINK_AUTO_BW_STAT | \ + PCIE_APP_IRN_INTA | PCIE_APP_IRN_INTB | \ + PCIE_APP_IRN_INTC | PCIE_APP_IRN_INTD) + +#define RESET_INTERVAL_MS 100 + +struct intel_pcie { + struct dw_pcie pci; + void __iomem *app_base; + struct gpio_desc *reset_gpio; + u32 rst_intrvl; + struct clk *core_clk; + struct reset_control *core_rst; + struct phy *phy; +}; + +static void pcie_update_bits(void __iomem *base, u32 ofs, u32 mask, u32 val) +{ + u32 old; + + old = readl(base + ofs); + val = (old & ~mask) | (val & mask); + + if (val != old) + writel(val, base + ofs); +} + +static inline void pcie_app_wr(struct intel_pcie *pcie, u32 ofs, u32 val) +{ + writel(val, pcie->app_base + ofs); +} + +static void pcie_app_wr_mask(struct intel_pcie *pcie, u32 ofs, + u32 mask, u32 val) +{ + pcie_update_bits(pcie->app_base, ofs, mask, val); +} + +static inline u32 pcie_rc_cfg_rd(struct intel_pcie *pcie, u32 ofs) +{ + return dw_pcie_readl_dbi(&pcie->pci, ofs); +} + +static inline void pcie_rc_cfg_wr(struct intel_pcie *pcie, u32 ofs, u32 val) +{ + dw_pcie_writel_dbi(&pcie->pci, ofs, val); +} + +static void pcie_rc_cfg_wr_mask(struct intel_pcie *pcie, u32 ofs, + u32 mask, u32 val) +{ + pcie_update_bits(pcie->pci.dbi_base, ofs, mask, val); +} + +static void intel_pcie_ltssm_enable(struct intel_pcie *pcie) +{ + pcie_app_wr_mask(pcie, PCIE_APP_CCR, PCIE_APP_CCR_LTSSM_ENABLE, + PCIE_APP_CCR_LTSSM_ENABLE); +} + +static void intel_pcie_ltssm_disable(struct intel_pcie *pcie) +{ + pcie_app_wr_mask(pcie, PCIE_APP_CCR, PCIE_APP_CCR_LTSSM_ENABLE, 0); +} + +static void intel_pcie_link_setup(struct intel_pcie *pcie) +{ + u32 val; + u8 offset = dw_pcie_find_capability(&pcie->pci, PCI_CAP_ID_EXP); + + val = pcie_rc_cfg_rd(pcie, offset + PCI_EXP_LNKCTL); + + val &= ~(PCI_EXP_LNKCTL_LD | PCI_EXP_LNKCTL_ASPMC); + pcie_rc_cfg_wr(pcie, offset + PCI_EXP_LNKCTL, val); +} + +static void intel_pcie_init_n_fts(struct dw_pcie *pci) +{ + switch (pci->max_link_speed) { + case 3: + pci->n_fts[1] = PORT_AFR_N_FTS_GEN3; + break; + case 4: + pci->n_fts[1] = PORT_AFR_N_FTS_GEN4; + break; + default: + pci->n_fts[1] = PORT_AFR_N_FTS_GEN12_DFT; + break; + } + pci->n_fts[0] = PORT_AFR_N_FTS_GEN12_DFT; +} + +static int intel_pcie_ep_rst_init(struct intel_pcie *pcie) +{ + struct device *dev = pcie->pci.dev; + int ret; + + pcie->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW); + if (IS_ERR(pcie->reset_gpio)) { + ret = PTR_ERR(pcie->reset_gpio); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to request PCIe GPIO: %d\n", ret); + return ret; + } + + /* Make initial reset last for 100us */ + usleep_range(100, 200); + + return 0; +} + +static void intel_pcie_core_rst_assert(struct intel_pcie *pcie) +{ + reset_control_assert(pcie->core_rst); +} + +static void intel_pcie_core_rst_deassert(struct intel_pcie *pcie) +{ + /* + * One micro-second delay to make sure the reset pulse + * wide enough so that core reset is clean. + */ + udelay(1); + reset_control_deassert(pcie->core_rst); + + /* + * Some SoC core reset also reset PHY, more delay needed + * to make sure the reset process is done. + */ + usleep_range(1000, 2000); +} + +static void intel_pcie_device_rst_assert(struct intel_pcie *pcie) +{ + gpiod_set_value_cansleep(pcie->reset_gpio, 1); +} + +static void intel_pcie_device_rst_deassert(struct intel_pcie *pcie) +{ + msleep(pcie->rst_intrvl); + gpiod_set_value_cansleep(pcie->reset_gpio, 0); +} + +static void intel_pcie_core_irq_enable(struct intel_pcie *pcie) +{ + pcie_app_wr(pcie, PCIE_APP_IRNEN, 0); + pcie_app_wr(pcie, PCIE_APP_IRNCR, PCIE_APP_IRN_INT); + pcie_app_wr(pcie, PCIE_APP_IRNEN, PCIE_APP_IRN_INT); +} + +static void intel_pcie_core_irq_disable(struct intel_pcie *pcie) +{ + pcie_app_wr(pcie, PCIE_APP_IRNEN, 0); + pcie_app_wr(pcie, PCIE_APP_IRNCR, PCIE_APP_IRN_INT); +} + +static int intel_pcie_get_resources(struct platform_device *pdev) +{ + struct intel_pcie *pcie = platform_get_drvdata(pdev); + struct dw_pcie *pci = &pcie->pci; + struct device *dev = pci->dev; + int ret; + + pcie->core_clk = devm_clk_get(dev, NULL); + if (IS_ERR(pcie->core_clk)) { + ret = PTR_ERR(pcie->core_clk); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to get clks: %d\n", ret); + return ret; + } + + pcie->core_rst = devm_reset_control_get(dev, NULL); + if (IS_ERR(pcie->core_rst)) { + ret = PTR_ERR(pcie->core_rst); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to get resets: %d\n", ret); + return ret; + } + + ret = device_property_read_u32(dev, "reset-assert-ms", + &pcie->rst_intrvl); + if (ret) + pcie->rst_intrvl = RESET_INTERVAL_MS; + + pcie->app_base = devm_platform_ioremap_resource_byname(pdev, "app"); + if (IS_ERR(pcie->app_base)) + return PTR_ERR(pcie->app_base); + + pcie->phy = devm_phy_get(dev, "pcie"); + if (IS_ERR(pcie->phy)) { + ret = PTR_ERR(pcie->phy); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Couldn't get pcie-phy: %d\n", ret); + return ret; + } + + return 0; +} + +static int intel_pcie_wait_l2(struct intel_pcie *pcie) +{ + u32 value; + int ret; + struct dw_pcie *pci = &pcie->pci; + + if (pci->max_link_speed < 3) + return 0; + + /* Send PME_TURN_OFF message */ + pcie_app_wr_mask(pcie, PCIE_APP_MSG_CR, PCIE_APP_MSG_XMT_PM_TURNOFF, + PCIE_APP_MSG_XMT_PM_TURNOFF); + + /* Read PMC status and wait for falling into L2 link state */ + ret = readl_poll_timeout(pcie->app_base + PCIE_APP_PMC, value, + value & PCIE_APP_PMC_IN_L2, 20, + jiffies_to_usecs(5 * HZ)); + if (ret) + dev_err(pcie->pci.dev, "PCIe link enter L2 timeout!\n"); + + return ret; +} + +static void intel_pcie_turn_off(struct intel_pcie *pcie) +{ + if (dw_pcie_link_up(&pcie->pci)) + intel_pcie_wait_l2(pcie); + + /* Put endpoint device in reset state */ + intel_pcie_device_rst_assert(pcie); + pcie_rc_cfg_wr_mask(pcie, PCI_COMMAND, PCI_COMMAND_MEMORY, 0); +} + +static int intel_pcie_start_link(struct dw_pcie *pci) +{ + struct intel_pcie *pcie = dev_get_drvdata(pci->dev); + + intel_pcie_device_rst_deassert(pcie); + intel_pcie_ltssm_enable(pcie); + + return 0; +} + +static int intel_pcie_host_setup(struct intel_pcie *pcie) +{ + int ret; + struct dw_pcie *pci = &pcie->pci; + + intel_pcie_core_rst_assert(pcie); + intel_pcie_device_rst_assert(pcie); + intel_pcie_core_rst_deassert(pcie); + + /* Controller clock must be provided earlier than PHY */ + ret = clk_prepare_enable(pcie->core_clk); + if (ret) { + dev_err(pcie->pci.dev, "Core clock enable failed: %d\n", ret); + goto clk_err; + } + + ret = phy_init(pcie->phy); + if (ret) + goto phy_err; + + intel_pcie_ltssm_disable(pcie); + intel_pcie_link_setup(pcie); + intel_pcie_init_n_fts(pci); + + dw_pcie_upconfig_setup(pci); + + intel_pcie_core_irq_enable(pcie); + + return 0; + +phy_err: + clk_disable_unprepare(pcie->core_clk); +clk_err: + intel_pcie_core_rst_assert(pcie); + + return ret; +} + +static void __intel_pcie_remove(struct intel_pcie *pcie) +{ + intel_pcie_core_irq_disable(pcie); + intel_pcie_turn_off(pcie); + clk_disable_unprepare(pcie->core_clk); + intel_pcie_core_rst_assert(pcie); + phy_exit(pcie->phy); +} + +static void intel_pcie_remove(struct platform_device *pdev) +{ + struct intel_pcie *pcie = platform_get_drvdata(pdev); + struct dw_pcie_rp *pp = &pcie->pci.pp; + + dw_pcie_host_deinit(pp); + __intel_pcie_remove(pcie); +} + +static int intel_pcie_suspend_noirq(struct device *dev) +{ + struct intel_pcie *pcie = dev_get_drvdata(dev); + int ret; + + intel_pcie_core_irq_disable(pcie); + ret = intel_pcie_wait_l2(pcie); + if (ret) + return ret; + + phy_exit(pcie->phy); + clk_disable_unprepare(pcie->core_clk); + return ret; +} + +static int intel_pcie_resume_noirq(struct device *dev) +{ + struct intel_pcie *pcie = dev_get_drvdata(dev); + + return intel_pcie_host_setup(pcie); +} + +static int intel_pcie_rc_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct intel_pcie *pcie = dev_get_drvdata(pci->dev); + + return intel_pcie_host_setup(pcie); +} + +static const struct dw_pcie_ops intel_pcie_ops = { + .start_link = intel_pcie_start_link, +}; + +static const struct dw_pcie_host_ops intel_pcie_dw_ops = { + .init = intel_pcie_rc_init, +}; + +static int intel_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct intel_pcie *pcie; + struct dw_pcie_rp *pp; + struct resource *res; + struct dw_pcie *pci; + int ret; + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + platform_set_drvdata(pdev, pcie); + pci = &pcie->pci; + pci->dev = dev; + pci->use_parent_dt_ranges = true; + pp = &pci->pp; + + ret = intel_pcie_get_resources(pdev); + if (ret) + return ret; + + ret = intel_pcie_ep_rst_init(pcie); + if (ret) + return ret; + + pci->ops = &intel_pcie_ops; + pp->ops = &intel_pcie_dw_ops; + + /* + * If the 'atu' region is not available in the devicetree, use the + * default offset from DBI region for backwards compatibility. The + * 'atu' region should always be specified in the devicetree, as + * this is a hardware-specific address that should not be defined + * in the driver. + */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "atu"); + if (!res) { + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi"); + pci->dbi_base = devm_pci_remap_cfg_resource(pci->dev, res); + if (IS_ERR(pci->dbi_base)) + return PTR_ERR(pci->dbi_base); + + pci->dbi_phys_addr = res->start; + pci->atu_base = devm_ioremap(dev, res->start + 0xC0000, SZ_4K); + if (!pci->atu_base) { + dev_err(dev, "failed to remap ATU space\n"); + return -ENOMEM; + } + + pci->atu_size = SZ_4K; + pci->atu_phys_addr = res->start + 0xC0000; + dev_warn(dev, "ATU region not specified in DT. Using default offset\n"); + } + + ret = dw_pcie_host_init(pp); + if (ret) { + dev_err(dev, "Cannot initialize host\n"); + return ret; + } + + return 0; +} + +static const struct dev_pm_ops intel_pcie_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(intel_pcie_suspend_noirq, + intel_pcie_resume_noirq) +}; + +static const struct of_device_id of_intel_pcie_match[] = { + { .compatible = "intel,lgm-pcie" }, + {} +}; + +static struct platform_driver intel_pcie_driver = { + .probe = intel_pcie_probe, + .remove = intel_pcie_remove, + .driver = { + .name = "intel-gw-pcie", + .of_match_table = of_intel_pcie_match, + .pm = &intel_pcie_pm_ops, + }, +}; +builtin_platform_driver(intel_pcie_driver); diff --git a/drivers/pci/controller/dwc/pcie-keembay.c b/drivers/pci/controller/dwc/pcie-keembay.c new file mode 100644 index 000000000..b11e66c10 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-keembay.c @@ -0,0 +1,481 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PCIe controller driver for Intel Keem Bay + * Copyright (C) 2020 Intel Corporation + */ + +#include <linux/bitfield.h> +#include <linux/bits.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/err.h> +#include <linux/gpio/consumer.h> +#include <linux/init.h> +#include <linux/iopoll.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/property.h> + +#include "pcie-designware.h" + +/* PCIE_REGS_APB_SLV Registers */ +#define PCIE_REGS_PCIE_CFG 0x0004 +#define PCIE_DEVICE_TYPE BIT(8) +#define PCIE_RSTN BIT(0) +#define PCIE_REGS_PCIE_APP_CNTRL 0x0008 +#define APP_LTSSM_ENABLE BIT(0) +#define PCIE_REGS_INTERRUPT_ENABLE 0x0028 +#define MSI_CTRL_INT_EN BIT(8) +#define EDMA_INT_EN GENMASK(7, 0) +#define PCIE_REGS_INTERRUPT_STATUS 0x002c +#define MSI_CTRL_INT BIT(8) +#define PCIE_REGS_PCIE_SII_PM_STATE 0x00b0 +#define SMLH_LINK_UP BIT(19) +#define RDLH_LINK_UP BIT(8) +#define PCIE_REGS_PCIE_SII_LINK_UP (SMLH_LINK_UP | RDLH_LINK_UP) +#define PCIE_REGS_PCIE_PHY_CNTL 0x0164 +#define PHY0_SRAM_BYPASS BIT(8) +#define PCIE_REGS_PCIE_PHY_STAT 0x0168 +#define PHY0_MPLLA_STATE BIT(1) +#define PCIE_REGS_LJPLL_STA 0x016c +#define LJPLL_LOCK BIT(0) +#define PCIE_REGS_LJPLL_CNTRL_0 0x0170 +#define LJPLL_EN BIT(29) +#define LJPLL_FOUT_EN GENMASK(24, 21) +#define PCIE_REGS_LJPLL_CNTRL_2 0x0178 +#define LJPLL_REF_DIV GENMASK(17, 12) +#define LJPLL_FB_DIV GENMASK(11, 0) +#define PCIE_REGS_LJPLL_CNTRL_3 0x017c +#define LJPLL_POST_DIV3A GENMASK(24, 22) +#define LJPLL_POST_DIV2A GENMASK(18, 16) + +#define PERST_DELAY_US 1000 +#define AUX_CLK_RATE_HZ 24000000 + +struct keembay_pcie { + struct dw_pcie pci; + void __iomem *apb_base; + + struct clk *clk_master; + struct clk *clk_aux; + struct gpio_desc *reset; +}; + +struct keembay_pcie_of_data { + enum dw_pcie_device_mode mode; +}; + +static void keembay_ep_reset_assert(struct keembay_pcie *pcie) +{ + gpiod_set_value_cansleep(pcie->reset, 1); + usleep_range(PERST_DELAY_US, PERST_DELAY_US + 500); +} + +static void keembay_ep_reset_deassert(struct keembay_pcie *pcie) +{ + /* + * Ensure that PERST# is asserted for a minimum of 100ms. + * + * For more details, refer to PCI Express Card Electromechanical + * Specification Revision 1.1, Table-2.4. + */ + msleep(100); + + gpiod_set_value_cansleep(pcie->reset, 0); + usleep_range(PERST_DELAY_US, PERST_DELAY_US + 500); +} + +static void keembay_pcie_ltssm_set(struct keembay_pcie *pcie, bool enable) +{ + u32 val; + + val = readl(pcie->apb_base + PCIE_REGS_PCIE_APP_CNTRL); + if (enable) + val |= APP_LTSSM_ENABLE; + else + val &= ~APP_LTSSM_ENABLE; + writel(val, pcie->apb_base + PCIE_REGS_PCIE_APP_CNTRL); +} + +static bool keembay_pcie_link_up(struct dw_pcie *pci) +{ + struct keembay_pcie *pcie = dev_get_drvdata(pci->dev); + u32 val; + + val = readl(pcie->apb_base + PCIE_REGS_PCIE_SII_PM_STATE); + + return (val & PCIE_REGS_PCIE_SII_LINK_UP) == PCIE_REGS_PCIE_SII_LINK_UP; +} + +static int keembay_pcie_start_link(struct dw_pcie *pci) +{ + struct keembay_pcie *pcie = dev_get_drvdata(pci->dev); + u32 val; + int ret; + + if (pcie->pci.mode == DW_PCIE_EP_TYPE) + return 0; + + keembay_pcie_ltssm_set(pcie, false); + + ret = readl_poll_timeout(pcie->apb_base + PCIE_REGS_PCIE_PHY_STAT, + val, val & PHY0_MPLLA_STATE, 20, + 500 * USEC_PER_MSEC); + if (ret) { + dev_err(pci->dev, "MPLLA is not locked\n"); + return ret; + } + + keembay_pcie_ltssm_set(pcie, true); + + return 0; +} + +static void keembay_pcie_stop_link(struct dw_pcie *pci) +{ + struct keembay_pcie *pcie = dev_get_drvdata(pci->dev); + + keembay_pcie_ltssm_set(pcie, false); +} + +static const struct dw_pcie_ops keembay_pcie_ops = { + .link_up = keembay_pcie_link_up, + .start_link = keembay_pcie_start_link, + .stop_link = keembay_pcie_stop_link, +}; + +static inline void keembay_pcie_disable_clock(void *data) +{ + struct clk *clk = data; + + clk_disable_unprepare(clk); +} + +static inline struct clk *keembay_pcie_probe_clock(struct device *dev, + const char *id, u64 rate) +{ + struct clk *clk; + int ret; + + clk = devm_clk_get(dev, id); + if (IS_ERR(clk)) + return clk; + + if (rate) { + ret = clk_set_rate(clk, rate); + if (ret) + return ERR_PTR(ret); + } + + ret = clk_prepare_enable(clk); + if (ret) + return ERR_PTR(ret); + + ret = devm_add_action_or_reset(dev, keembay_pcie_disable_clock, clk); + if (ret) + return ERR_PTR(ret); + + return clk; +} + +static int keembay_pcie_probe_clocks(struct keembay_pcie *pcie) +{ + struct dw_pcie *pci = &pcie->pci; + struct device *dev = pci->dev; + + pcie->clk_master = keembay_pcie_probe_clock(dev, "master", 0); + if (IS_ERR(pcie->clk_master)) + return dev_err_probe(dev, PTR_ERR(pcie->clk_master), + "Failed to enable master clock"); + + pcie->clk_aux = keembay_pcie_probe_clock(dev, "aux", AUX_CLK_RATE_HZ); + if (IS_ERR(pcie->clk_aux)) + return dev_err_probe(dev, PTR_ERR(pcie->clk_aux), + "Failed to enable auxiliary clock"); + + return 0; +} + +/* + * Initialize the internal PCIe PLL in Host mode. + * See the following sections in Keem Bay data book, + * (1) 6.4.6.1 PCIe Subsystem Example Initialization, + * (2) 6.8 PCIe Low Jitter PLL for Ref Clk Generation. + */ +static int keembay_pcie_pll_init(struct keembay_pcie *pcie) +{ + struct dw_pcie *pci = &pcie->pci; + u32 val; + int ret; + + val = FIELD_PREP(LJPLL_REF_DIV, 0) | FIELD_PREP(LJPLL_FB_DIV, 0x32); + writel(val, pcie->apb_base + PCIE_REGS_LJPLL_CNTRL_2); + + val = FIELD_PREP(LJPLL_POST_DIV3A, 0x2) | + FIELD_PREP(LJPLL_POST_DIV2A, 0x2); + writel(val, pcie->apb_base + PCIE_REGS_LJPLL_CNTRL_3); + + val = FIELD_PREP(LJPLL_EN, 0x1) | FIELD_PREP(LJPLL_FOUT_EN, 0xc); + writel(val, pcie->apb_base + PCIE_REGS_LJPLL_CNTRL_0); + + ret = readl_poll_timeout(pcie->apb_base + PCIE_REGS_LJPLL_STA, + val, val & LJPLL_LOCK, 20, + 500 * USEC_PER_MSEC); + if (ret) + dev_err(pci->dev, "Low jitter PLL is not locked\n"); + + return ret; +} + +static void keembay_pcie_msi_irq_handler(struct irq_desc *desc) +{ + struct keembay_pcie *pcie = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + u32 val, mask, status; + struct dw_pcie_rp *pp; + + /* + * Keem Bay PCIe Controller provides an additional IP logic on top of + * standard DWC IP to clear MSI IRQ by writing '1' to the respective + * bit of the status register. + * + * So, a chained irq handler is defined to handle this additional + * IP logic. + */ + + chained_irq_enter(chip, desc); + + pp = &pcie->pci.pp; + val = readl(pcie->apb_base + PCIE_REGS_INTERRUPT_STATUS); + mask = readl(pcie->apb_base + PCIE_REGS_INTERRUPT_ENABLE); + + status = val & mask; + + if (status & MSI_CTRL_INT) { + dw_handle_msi_irq(pp); + writel(status, pcie->apb_base + PCIE_REGS_INTERRUPT_STATUS); + } + + chained_irq_exit(chip, desc); +} + +static int keembay_pcie_setup_msi_irq(struct keembay_pcie *pcie) +{ + struct dw_pcie *pci = &pcie->pci; + struct device *dev = pci->dev; + struct platform_device *pdev = to_platform_device(dev); + int irq; + + irq = platform_get_irq_byname(pdev, "pcie"); + if (irq < 0) + return irq; + + irq_set_chained_handler_and_data(irq, keembay_pcie_msi_irq_handler, + pcie); + + return 0; +} + +static int keembay_pcie_ep_init(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct keembay_pcie *pcie = dev_get_drvdata(pci->dev); + + writel(EDMA_INT_EN, pcie->apb_base + PCIE_REGS_INTERRUPT_ENABLE); + + return 0; +} + +static int keembay_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, + unsigned int type, u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + switch (type) { + case PCI_IRQ_INTX: + /* INTx interrupts are not supported in Keem Bay */ + dev_err(pci->dev, "INTx IRQ is not supported\n"); + return -EINVAL; + case PCI_IRQ_MSI: + return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); + case PCI_IRQ_MSIX: + return dw_pcie_ep_raise_msix_irq(ep, func_no, interrupt_num); + default: + dev_err(pci->dev, "Unknown IRQ type %d\n", type); + return -EINVAL; + } +} + +static const struct pci_epc_features keembay_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, + .msi_capable = true, + .msix_capable = true, + .bar[BAR_0] = { .only_64bit = true, }, + .bar[BAR_2] = { .only_64bit = true, }, + .bar[BAR_4] = { .only_64bit = true, }, + .align = SZ_16K, +}; + +static const struct pci_epc_features * +keembay_pcie_get_features(struct dw_pcie_ep *ep) +{ + return &keembay_pcie_epc_features; +} + +static const struct dw_pcie_ep_ops keembay_pcie_ep_ops = { + .init = keembay_pcie_ep_init, + .raise_irq = keembay_pcie_ep_raise_irq, + .get_features = keembay_pcie_get_features, +}; + +static const struct dw_pcie_host_ops keembay_pcie_host_ops = { +}; + +static int keembay_pcie_add_pcie_port(struct keembay_pcie *pcie, + struct platform_device *pdev) +{ + struct dw_pcie *pci = &pcie->pci; + struct dw_pcie_rp *pp = &pci->pp; + struct device *dev = &pdev->dev; + u32 val; + int ret; + + pp->ops = &keembay_pcie_host_ops; + pp->msi_irq[0] = -ENODEV; + + ret = keembay_pcie_setup_msi_irq(pcie); + if (ret) + return ret; + + pcie->reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH); + if (IS_ERR(pcie->reset)) + return PTR_ERR(pcie->reset); + + ret = keembay_pcie_probe_clocks(pcie); + if (ret) + return ret; + + val = readl(pcie->apb_base + PCIE_REGS_PCIE_PHY_CNTL); + val |= PHY0_SRAM_BYPASS; + writel(val, pcie->apb_base + PCIE_REGS_PCIE_PHY_CNTL); + + writel(PCIE_DEVICE_TYPE, pcie->apb_base + PCIE_REGS_PCIE_CFG); + + ret = keembay_pcie_pll_init(pcie); + if (ret) + return ret; + + val = readl(pcie->apb_base + PCIE_REGS_PCIE_CFG); + writel(val | PCIE_RSTN, pcie->apb_base + PCIE_REGS_PCIE_CFG); + keembay_ep_reset_deassert(pcie); + + ret = dw_pcie_host_init(pp); + if (ret) { + keembay_ep_reset_assert(pcie); + dev_err(dev, "Failed to initialize host: %d\n", ret); + return ret; + } + + val = readl(pcie->apb_base + PCIE_REGS_INTERRUPT_ENABLE); + if (IS_ENABLED(CONFIG_PCI_MSI)) + val |= MSI_CTRL_INT_EN; + writel(val, pcie->apb_base + PCIE_REGS_INTERRUPT_ENABLE); + + return 0; +} + +static int keembay_pcie_probe(struct platform_device *pdev) +{ + const struct keembay_pcie_of_data *data; + struct device *dev = &pdev->dev; + struct keembay_pcie *pcie; + struct dw_pcie *pci; + enum dw_pcie_device_mode mode; + int ret; + + data = device_get_match_data(dev); + if (!data) + return -ENODEV; + + mode = (enum dw_pcie_device_mode)data->mode; + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + pci = &pcie->pci; + pci->dev = dev; + pci->ops = &keembay_pcie_ops; + + pcie->pci.mode = mode; + + pcie->apb_base = devm_platform_ioremap_resource_byname(pdev, "apb"); + if (IS_ERR(pcie->apb_base)) + return PTR_ERR(pcie->apb_base); + + platform_set_drvdata(pdev, pcie); + + switch (pcie->pci.mode) { + case DW_PCIE_RC_TYPE: + if (!IS_ENABLED(CONFIG_PCIE_KEEMBAY_HOST)) + return -ENODEV; + + return keembay_pcie_add_pcie_port(pcie, pdev); + case DW_PCIE_EP_TYPE: + if (!IS_ENABLED(CONFIG_PCIE_KEEMBAY_EP)) + return -ENODEV; + + pci->ep.ops = &keembay_pcie_ep_ops; + ret = dw_pcie_ep_init(&pci->ep); + if (ret) + return ret; + + ret = dw_pcie_ep_init_registers(&pci->ep); + if (ret) { + dev_err(dev, "Failed to initialize DWC endpoint registers\n"); + dw_pcie_ep_deinit(&pci->ep); + return ret; + } + + pci_epc_init_notify(pci->ep.epc); + + break; + default: + dev_err(dev, "Invalid device type %d\n", pcie->pci.mode); + return -ENODEV; + } + + return 0; +} + +static const struct keembay_pcie_of_data keembay_pcie_rc_of_data = { + .mode = DW_PCIE_RC_TYPE, +}; + +static const struct keembay_pcie_of_data keembay_pcie_ep_of_data = { + .mode = DW_PCIE_EP_TYPE, +}; + +static const struct of_device_id keembay_pcie_of_match[] = { + { + .compatible = "intel,keembay-pcie", + .data = &keembay_pcie_rc_of_data, + }, + { + .compatible = "intel,keembay-pcie-ep", + .data = &keembay_pcie_ep_of_data, + }, + {} +}; + +static struct platform_driver keembay_pcie_driver = { + .driver = { + .name = "keembay-pcie", + .of_match_table = keembay_pcie_of_match, + .suppress_bind_attrs = true, + }, + .probe = keembay_pcie_probe, +}; +builtin_platform_driver(keembay_pcie_driver); diff --git a/drivers/pci/controller/dwc/pcie-kirin.c b/drivers/pci/controller/dwc/pcie-kirin.c new file mode 100644 index 000000000..91559c8b1 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-kirin.c @@ -0,0 +1,767 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for Kirin Phone SoCs + * + * Copyright (C) 2017 HiSilicon Electronics Co., Ltd. + * https://www.huawei.com + * + * Author: Xiaowei Song <songxiaowei@huawei.com> + */ + +#include <linux/clk.h> +#include <linux/compiler.h> +#include <linux/delay.h> +#include <linux/err.h> +#include <linux/gpio/consumer.h> +#include <linux/interrupt.h> +#include <linux/mfd/syscon.h> +#include <linux/of.h> +#include <linux/of_pci.h> +#include <linux/phy/phy.h> +#include <linux/pci.h> +#include <linux/pci_regs.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/resource.h> +#include <linux/types.h> +#include "pcie-designware.h" + +#define to_kirin_pcie(x) dev_get_drvdata((x)->dev) + +/* PCIe ELBI registers */ +#define SOC_PCIECTRL_CTRL0_ADDR 0x000 +#define SOC_PCIECTRL_CTRL1_ADDR 0x004 +#define PCIE_ELBI_SLV_DBI_ENABLE (0x1 << 21) + +/* info located in APB */ +#define PCIE_APP_LTSSM_ENABLE 0x01c +#define PCIE_APB_PHY_STATUS0 0x400 +#define PCIE_LINKUP_ENABLE (0x8020) +#define PCIE_LTSSM_ENABLE_BIT (0x1 << 11) + +/* info located in sysctrl */ +#define SCTRL_PCIE_CMOS_OFFSET 0x60 +#define SCTRL_PCIE_CMOS_BIT 0x10 +#define SCTRL_PCIE_ISO_OFFSET 0x44 +#define SCTRL_PCIE_ISO_BIT 0x30 +#define SCTRL_PCIE_HPCLK_OFFSET 0x190 +#define SCTRL_PCIE_HPCLK_BIT 0x184000 +#define SCTRL_PCIE_OE_OFFSET 0x14a +#define PCIE_DEBOUNCE_PARAM 0xF0F400 +#define PCIE_OE_BYPASS (0x3 << 28) + +/* + * Max number of connected PCI slots at an external PCI bridge + * + * This is used on HiKey 970, which has a PEX 8606 bridge with 4 connected + * lanes (lane 0 upstream, and the other three lanes, one connected to an + * in-board Ethernet adapter and the other two connected to M.2 and mini + * PCI slots. + * + * Each slot has a different clock source and uses a separate PERST# pin. + */ +#define MAX_PCI_SLOTS 3 + +enum pcie_kirin_phy_type { + PCIE_KIRIN_INTERNAL_PHY, + PCIE_KIRIN_EXTERNAL_PHY +}; + +struct kirin_pcie { + enum pcie_kirin_phy_type type; + + struct dw_pcie *pci; + struct regmap *apb; + struct phy *phy; + void *phy_priv; /* only for PCIE_KIRIN_INTERNAL_PHY */ + + /* DWC PERST# */ + struct gpio_desc *id_dwc_perst_gpio; + + /* Per-slot PERST# */ + int num_slots; + struct gpio_desc *id_reset_gpio[MAX_PCI_SLOTS]; + const char *reset_names[MAX_PCI_SLOTS]; + + /* Per-slot clkreq */ + int n_gpio_clkreq; + struct gpio_desc *id_clkreq_gpio[MAX_PCI_SLOTS]; + const char *clkreq_names[MAX_PCI_SLOTS]; +}; + +/* + * Kirin 960 PHY. Can't be split into a PHY driver without changing the + * DT schema. + */ + +#define REF_CLK_FREQ 100000000 + +/* PHY info located in APB */ +#define PCIE_APB_PHY_CTRL0 0x0 +#define PCIE_APB_PHY_CTRL1 0x4 +#define PCIE_APB_PHY_STATUS0 0x400 +#define PIPE_CLK_STABLE BIT(19) +#define PHY_REF_PAD_BIT BIT(8) +#define PHY_PWR_DOWN_BIT BIT(22) +#define PHY_RST_ACK_BIT BIT(16) + +/* peri_crg ctrl */ +#define CRGCTRL_PCIE_ASSERT_OFFSET 0x88 +#define CRGCTRL_PCIE_ASSERT_BIT 0x8c000000 + +/* Time for delay */ +#define REF_2_PERST_MIN 21000 +#define REF_2_PERST_MAX 25000 +#define PERST_2_ACCESS_MIN 10000 +#define PERST_2_ACCESS_MAX 12000 +#define PIPE_CLK_WAIT_MIN 550 +#define PIPE_CLK_WAIT_MAX 600 +#define TIME_CMOS_MIN 100 +#define TIME_CMOS_MAX 105 +#define TIME_PHY_PD_MIN 10 +#define TIME_PHY_PD_MAX 11 + +struct hi3660_pcie_phy { + struct device *dev; + void __iomem *base; + struct regmap *crgctrl; + struct regmap *sysctrl; + struct clk *apb_sys_clk; + struct clk *apb_phy_clk; + struct clk *phy_ref_clk; + struct clk *aclk; + struct clk *aux_clk; +}; + +/* Registers in PCIePHY */ +static inline void kirin_apb_phy_writel(struct hi3660_pcie_phy *hi3660_pcie_phy, + u32 val, u32 reg) +{ + writel(val, hi3660_pcie_phy->base + reg); +} + +static inline u32 kirin_apb_phy_readl(struct hi3660_pcie_phy *hi3660_pcie_phy, + u32 reg) +{ + return readl(hi3660_pcie_phy->base + reg); +} + +static int hi3660_pcie_phy_get_clk(struct hi3660_pcie_phy *phy) +{ + struct device *dev = phy->dev; + + phy->phy_ref_clk = devm_clk_get(dev, "pcie_phy_ref"); + if (IS_ERR(phy->phy_ref_clk)) + return PTR_ERR(phy->phy_ref_clk); + + phy->aux_clk = devm_clk_get(dev, "pcie_aux"); + if (IS_ERR(phy->aux_clk)) + return PTR_ERR(phy->aux_clk); + + phy->apb_phy_clk = devm_clk_get(dev, "pcie_apb_phy"); + if (IS_ERR(phy->apb_phy_clk)) + return PTR_ERR(phy->apb_phy_clk); + + phy->apb_sys_clk = devm_clk_get(dev, "pcie_apb_sys"); + if (IS_ERR(phy->apb_sys_clk)) + return PTR_ERR(phy->apb_sys_clk); + + phy->aclk = devm_clk_get(dev, "pcie_aclk"); + if (IS_ERR(phy->aclk)) + return PTR_ERR(phy->aclk); + + return 0; +} + +static int hi3660_pcie_phy_get_resource(struct hi3660_pcie_phy *phy) +{ + struct device *dev = phy->dev; + struct platform_device *pdev; + + /* registers */ + pdev = container_of(dev, struct platform_device, dev); + + phy->base = devm_platform_ioremap_resource_byname(pdev, "phy"); + if (IS_ERR(phy->base)) + return PTR_ERR(phy->base); + + phy->crgctrl = syscon_regmap_lookup_by_compatible("hisilicon,hi3660-crgctrl"); + if (IS_ERR(phy->crgctrl)) + return PTR_ERR(phy->crgctrl); + + phy->sysctrl = syscon_regmap_lookup_by_compatible("hisilicon,hi3660-sctrl"); + if (IS_ERR(phy->sysctrl)) + return PTR_ERR(phy->sysctrl); + + return 0; +} + +static int hi3660_pcie_phy_start(struct hi3660_pcie_phy *phy) +{ + struct device *dev = phy->dev; + u32 reg_val; + + reg_val = kirin_apb_phy_readl(phy, PCIE_APB_PHY_CTRL1); + reg_val &= ~PHY_REF_PAD_BIT; + kirin_apb_phy_writel(phy, reg_val, PCIE_APB_PHY_CTRL1); + + reg_val = kirin_apb_phy_readl(phy, PCIE_APB_PHY_CTRL0); + reg_val &= ~PHY_PWR_DOWN_BIT; + kirin_apb_phy_writel(phy, reg_val, PCIE_APB_PHY_CTRL0); + usleep_range(TIME_PHY_PD_MIN, TIME_PHY_PD_MAX); + + reg_val = kirin_apb_phy_readl(phy, PCIE_APB_PHY_CTRL1); + reg_val &= ~PHY_RST_ACK_BIT; + kirin_apb_phy_writel(phy, reg_val, PCIE_APB_PHY_CTRL1); + + usleep_range(PIPE_CLK_WAIT_MIN, PIPE_CLK_WAIT_MAX); + reg_val = kirin_apb_phy_readl(phy, PCIE_APB_PHY_STATUS0); + if (reg_val & PIPE_CLK_STABLE) + return dev_err_probe(dev, -ETIMEDOUT, + "PIPE clk is not stable\n"); + + return 0; +} + +static void hi3660_pcie_phy_oe_enable(struct hi3660_pcie_phy *phy) +{ + u32 val; + + regmap_read(phy->sysctrl, SCTRL_PCIE_OE_OFFSET, &val); + val |= PCIE_DEBOUNCE_PARAM; + val &= ~PCIE_OE_BYPASS; + regmap_write(phy->sysctrl, SCTRL_PCIE_OE_OFFSET, val); +} + +static int hi3660_pcie_phy_clk_ctrl(struct hi3660_pcie_phy *phy, bool enable) +{ + int ret = 0; + + if (!enable) + goto close_clk; + + ret = clk_set_rate(phy->phy_ref_clk, REF_CLK_FREQ); + if (ret) + return ret; + + ret = clk_prepare_enable(phy->phy_ref_clk); + if (ret) + return ret; + + ret = clk_prepare_enable(phy->apb_sys_clk); + if (ret) + goto apb_sys_fail; + + ret = clk_prepare_enable(phy->apb_phy_clk); + if (ret) + goto apb_phy_fail; + + ret = clk_prepare_enable(phy->aclk); + if (ret) + goto aclk_fail; + + ret = clk_prepare_enable(phy->aux_clk); + if (ret) + goto aux_clk_fail; + + return 0; + +close_clk: + clk_disable_unprepare(phy->aux_clk); +aux_clk_fail: + clk_disable_unprepare(phy->aclk); +aclk_fail: + clk_disable_unprepare(phy->apb_phy_clk); +apb_phy_fail: + clk_disable_unprepare(phy->apb_sys_clk); +apb_sys_fail: + clk_disable_unprepare(phy->phy_ref_clk); + + return ret; +} + +static int hi3660_pcie_phy_power_on(struct kirin_pcie *pcie) +{ + struct hi3660_pcie_phy *phy = pcie->phy_priv; + int ret; + + /* Power supply for Host */ + regmap_write(phy->sysctrl, + SCTRL_PCIE_CMOS_OFFSET, SCTRL_PCIE_CMOS_BIT); + usleep_range(TIME_CMOS_MIN, TIME_CMOS_MAX); + + hi3660_pcie_phy_oe_enable(phy); + + ret = hi3660_pcie_phy_clk_ctrl(phy, true); + if (ret) + return ret; + + /* ISO disable, PCIeCtrl, PHY assert and clk gate clear */ + regmap_write(phy->sysctrl, + SCTRL_PCIE_ISO_OFFSET, SCTRL_PCIE_ISO_BIT); + regmap_write(phy->crgctrl, + CRGCTRL_PCIE_ASSERT_OFFSET, CRGCTRL_PCIE_ASSERT_BIT); + regmap_write(phy->sysctrl, + SCTRL_PCIE_HPCLK_OFFSET, SCTRL_PCIE_HPCLK_BIT); + + ret = hi3660_pcie_phy_start(phy); + if (ret) + goto disable_clks; + + return 0; + +disable_clks: + hi3660_pcie_phy_clk_ctrl(phy, false); + return ret; +} + +static int hi3660_pcie_phy_init(struct platform_device *pdev, + struct kirin_pcie *pcie) +{ + struct device *dev = &pdev->dev; + struct hi3660_pcie_phy *phy; + int ret; + + phy = devm_kzalloc(dev, sizeof(*phy), GFP_KERNEL); + if (!phy) + return -ENOMEM; + + pcie->phy_priv = phy; + phy->dev = dev; + + ret = hi3660_pcie_phy_get_clk(phy); + if (ret) + return ret; + + return hi3660_pcie_phy_get_resource(phy); +} + +static int hi3660_pcie_phy_power_off(struct kirin_pcie *pcie) +{ + struct hi3660_pcie_phy *phy = pcie->phy_priv; + + /* Drop power supply for Host */ + regmap_write(phy->sysctrl, SCTRL_PCIE_CMOS_OFFSET, 0x00); + + hi3660_pcie_phy_clk_ctrl(phy, false); + + return 0; +} + +/* + * The non-PHY part starts here + */ + +static const struct regmap_config pcie_kirin_regmap_conf = { + .name = "kirin_pcie_apb", + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, +}; + +static int kirin_pcie_get_gpio_enable(struct kirin_pcie *pcie, + struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + int ret, i; + + /* This is an optional property */ + ret = gpiod_count(dev, "hisilicon,clken"); + if (ret < 0) + return 0; + + if (ret > MAX_PCI_SLOTS) + return dev_err_probe(dev, -EINVAL, + "Too many GPIO clock requests!\n"); + + pcie->n_gpio_clkreq = ret; + + for (i = 0; i < pcie->n_gpio_clkreq; i++) { + pcie->id_clkreq_gpio[i] = devm_gpiod_get_index(dev, + "hisilicon,clken", i, + GPIOD_OUT_LOW); + if (IS_ERR(pcie->id_clkreq_gpio[i])) + return dev_err_probe(dev, PTR_ERR(pcie->id_clkreq_gpio[i]), + "unable to get a valid clken gpio\n"); + + pcie->clkreq_names[i] = devm_kasprintf(dev, GFP_KERNEL, + "pcie_clkreq_%d", i); + if (!pcie->clkreq_names[i]) + return -ENOMEM; + + gpiod_set_consumer_name(pcie->id_clkreq_gpio[i], + pcie->clkreq_names[i]); + } + + return 0; +} + +static int kirin_pcie_parse_port(struct kirin_pcie *pcie, + struct platform_device *pdev, + struct device_node *node) +{ + struct device *dev = &pdev->dev; + int ret, slot, i; + + for_each_available_child_of_node_scoped(node, parent) { + for_each_available_child_of_node_scoped(parent, child) { + i = pcie->num_slots; + + pcie->id_reset_gpio[i] = devm_fwnode_gpiod_get_index(dev, + of_fwnode_handle(child), + "reset", 0, GPIOD_OUT_LOW, + NULL); + if (IS_ERR(pcie->id_reset_gpio[i])) { + if (PTR_ERR(pcie->id_reset_gpio[i]) == -ENOENT) + continue; + return dev_err_probe(dev, PTR_ERR(pcie->id_reset_gpio[i]), + "unable to get a valid reset gpio\n"); + } + + if (pcie->num_slots + 1 >= MAX_PCI_SLOTS) + return dev_err_probe(dev, -EINVAL, + "Too many PCI slots!\n"); + + pcie->num_slots++; + + ret = of_pci_get_devfn(child); + if (ret < 0) + return dev_err_probe(dev, ret, + "failed to parse devfn\n"); + + slot = PCI_SLOT(ret); + + pcie->reset_names[i] = devm_kasprintf(dev, GFP_KERNEL, + "pcie_perst_%d", + slot); + if (!pcie->reset_names[i]) + return -ENOMEM; + + gpiod_set_consumer_name(pcie->id_reset_gpio[i], + pcie->reset_names[i]); + } + } + + return 0; +} + +static long kirin_pcie_get_resource(struct kirin_pcie *kirin_pcie, + struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *node = dev->of_node; + void __iomem *apb_base; + int ret; + + apb_base = devm_platform_ioremap_resource_byname(pdev, "apb"); + if (IS_ERR(apb_base)) + return PTR_ERR(apb_base); + + kirin_pcie->apb = devm_regmap_init_mmio(dev, apb_base, + &pcie_kirin_regmap_conf); + if (IS_ERR(kirin_pcie->apb)) + return PTR_ERR(kirin_pcie->apb); + + /* pcie internal PERST# gpio */ + kirin_pcie->id_dwc_perst_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW); + if (IS_ERR(kirin_pcie->id_dwc_perst_gpio)) + return dev_err_probe(dev, PTR_ERR(kirin_pcie->id_dwc_perst_gpio), + "unable to get a valid gpio pin\n"); + gpiod_set_consumer_name(kirin_pcie->id_dwc_perst_gpio, "pcie_perst_bridge"); + + ret = kirin_pcie_get_gpio_enable(kirin_pcie, pdev); + if (ret) + return ret; + + /* Parse OF children */ + for_each_available_child_of_node_scoped(node, child) { + ret = kirin_pcie_parse_port(kirin_pcie, pdev, child); + if (ret) + return ret; + } + + return 0; +} + +static void kirin_pcie_sideband_dbi_w_mode(struct kirin_pcie *kirin_pcie, + bool on) +{ + u32 val; + + regmap_read(kirin_pcie->apb, SOC_PCIECTRL_CTRL0_ADDR, &val); + if (on) + val = val | PCIE_ELBI_SLV_DBI_ENABLE; + else + val = val & ~PCIE_ELBI_SLV_DBI_ENABLE; + + regmap_write(kirin_pcie->apb, SOC_PCIECTRL_CTRL0_ADDR, val); +} + +static void kirin_pcie_sideband_dbi_r_mode(struct kirin_pcie *kirin_pcie, + bool on) +{ + u32 val; + + regmap_read(kirin_pcie->apb, SOC_PCIECTRL_CTRL1_ADDR, &val); + if (on) + val = val | PCIE_ELBI_SLV_DBI_ENABLE; + else + val = val & ~PCIE_ELBI_SLV_DBI_ENABLE; + + regmap_write(kirin_pcie->apb, SOC_PCIECTRL_CTRL1_ADDR, val); +} + +static int kirin_pcie_rd_own_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(bus->sysdata); + + if (PCI_SLOT(devfn)) + return PCIBIOS_DEVICE_NOT_FOUND; + + *val = dw_pcie_read_dbi(pci, where, size); + return PCIBIOS_SUCCESSFUL; +} + +static int kirin_pcie_wr_own_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(bus->sysdata); + + if (PCI_SLOT(devfn)) + return PCIBIOS_DEVICE_NOT_FOUND; + + dw_pcie_write_dbi(pci, where, size, val); + return PCIBIOS_SUCCESSFUL; +} + +static int kirin_pcie_add_bus(struct pci_bus *bus) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(bus->sysdata); + struct kirin_pcie *kirin_pcie = to_kirin_pcie(pci); + int i, ret; + + if (!kirin_pcie->num_slots) + return 0; + + /* Send PERST# to each slot */ + for (i = 0; i < kirin_pcie->num_slots; i++) { + ret = gpiod_direction_output_raw(kirin_pcie->id_reset_gpio[i], 1); + if (ret) { + dev_err(pci->dev, "PERST# %s error: %d\n", + kirin_pcie->reset_names[i], ret); + } + } + usleep_range(PERST_2_ACCESS_MIN, PERST_2_ACCESS_MAX); + + return 0; +} + +static struct pci_ops kirin_pci_ops = { + .read = kirin_pcie_rd_own_conf, + .write = kirin_pcie_wr_own_conf, + .add_bus = kirin_pcie_add_bus, +}; + +static u32 kirin_pcie_read_dbi(struct dw_pcie *pci, void __iomem *base, + u32 reg, size_t size) +{ + struct kirin_pcie *kirin_pcie = to_kirin_pcie(pci); + u32 ret; + + kirin_pcie_sideband_dbi_r_mode(kirin_pcie, true); + dw_pcie_read(base + reg, size, &ret); + kirin_pcie_sideband_dbi_r_mode(kirin_pcie, false); + + return ret; +} + +static void kirin_pcie_write_dbi(struct dw_pcie *pci, void __iomem *base, + u32 reg, size_t size, u32 val) +{ + struct kirin_pcie *kirin_pcie = to_kirin_pcie(pci); + + kirin_pcie_sideband_dbi_w_mode(kirin_pcie, true); + dw_pcie_write(base + reg, size, val); + kirin_pcie_sideband_dbi_w_mode(kirin_pcie, false); +} + +static bool kirin_pcie_link_up(struct dw_pcie *pci) +{ + struct kirin_pcie *kirin_pcie = to_kirin_pcie(pci); + u32 val; + + regmap_read(kirin_pcie->apb, PCIE_APB_PHY_STATUS0, &val); + return (val & PCIE_LINKUP_ENABLE) == PCIE_LINKUP_ENABLE; +} + +static int kirin_pcie_start_link(struct dw_pcie *pci) +{ + struct kirin_pcie *kirin_pcie = to_kirin_pcie(pci); + + /* assert LTSSM enable */ + regmap_write(kirin_pcie->apb, PCIE_APP_LTSSM_ENABLE, + PCIE_LTSSM_ENABLE_BIT); + + return 0; +} + +static int kirin_pcie_host_init(struct dw_pcie_rp *pp) +{ + pp->bridge->ops = &kirin_pci_ops; + + return 0; +} + +static const struct dw_pcie_ops kirin_dw_pcie_ops = { + .read_dbi = kirin_pcie_read_dbi, + .write_dbi = kirin_pcie_write_dbi, + .link_up = kirin_pcie_link_up, + .start_link = kirin_pcie_start_link, +}; + +static const struct dw_pcie_host_ops kirin_pcie_host_ops = { + .init = kirin_pcie_host_init, +}; + +static int kirin_pcie_power_off(struct kirin_pcie *kirin_pcie) +{ + int i; + + if (kirin_pcie->type == PCIE_KIRIN_INTERNAL_PHY) + return hi3660_pcie_phy_power_off(kirin_pcie); + + for (i = 0; i < kirin_pcie->n_gpio_clkreq; i++) + gpiod_direction_output_raw(kirin_pcie->id_clkreq_gpio[i], 1); + + phy_power_off(kirin_pcie->phy); + phy_exit(kirin_pcie->phy); + + return 0; +} + +static int kirin_pcie_power_on(struct platform_device *pdev, + struct kirin_pcie *kirin_pcie) +{ + struct device *dev = &pdev->dev; + int ret; + + if (kirin_pcie->type == PCIE_KIRIN_INTERNAL_PHY) { + ret = hi3660_pcie_phy_init(pdev, kirin_pcie); + if (ret) + return ret; + + ret = hi3660_pcie_phy_power_on(kirin_pcie); + if (ret) + return ret; + } else { + kirin_pcie->phy = devm_of_phy_get(dev, dev->of_node, NULL); + if (IS_ERR(kirin_pcie->phy)) + return PTR_ERR(kirin_pcie->phy); + + ret = phy_init(kirin_pcie->phy); + if (ret) + goto err; + + ret = phy_power_on(kirin_pcie->phy); + if (ret) + goto err; + } + + /* perst assert Endpoint */ + usleep_range(REF_2_PERST_MIN, REF_2_PERST_MAX); + + ret = gpiod_direction_output_raw(kirin_pcie->id_dwc_perst_gpio, 1); + if (ret) + goto err; + + usleep_range(PERST_2_ACCESS_MIN, PERST_2_ACCESS_MAX); + + return 0; +err: + kirin_pcie_power_off(kirin_pcie); + + return ret; +} + +static void kirin_pcie_remove(struct platform_device *pdev) +{ + struct kirin_pcie *kirin_pcie = platform_get_drvdata(pdev); + + dw_pcie_host_deinit(&kirin_pcie->pci->pp); + + kirin_pcie_power_off(kirin_pcie); +} + +struct kirin_pcie_data { + enum pcie_kirin_phy_type phy_type; +}; + +static const struct kirin_pcie_data kirin_960_data = { + .phy_type = PCIE_KIRIN_INTERNAL_PHY, +}; + +static const struct kirin_pcie_data kirin_970_data = { + .phy_type = PCIE_KIRIN_EXTERNAL_PHY, +}; + +static const struct of_device_id kirin_pcie_match[] = { + { .compatible = "hisilicon,kirin960-pcie", .data = &kirin_960_data }, + { .compatible = "hisilicon,kirin970-pcie", .data = &kirin_970_data }, + {}, +}; + +static int kirin_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + const struct kirin_pcie_data *data; + struct kirin_pcie *kirin_pcie; + struct dw_pcie *pci; + int ret; + + data = of_device_get_match_data(dev); + if (!data) + return dev_err_probe(dev, -EINVAL, "OF data missing\n"); + + kirin_pcie = devm_kzalloc(dev, sizeof(struct kirin_pcie), GFP_KERNEL); + if (!kirin_pcie) + return -ENOMEM; + + pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); + if (!pci) + return -ENOMEM; + + pci->dev = dev; + pci->ops = &kirin_dw_pcie_ops; + pci->pp.ops = &kirin_pcie_host_ops; + kirin_pcie->pci = pci; + kirin_pcie->type = data->phy_type; + + ret = kirin_pcie_get_resource(kirin_pcie, pdev); + if (ret) + return ret; + + platform_set_drvdata(pdev, kirin_pcie); + + ret = kirin_pcie_power_on(pdev, kirin_pcie); + if (ret) + return ret; + + return dw_pcie_host_init(&pci->pp); +} + +static struct platform_driver kirin_pcie_driver = { + .probe = kirin_pcie_probe, + .remove = kirin_pcie_remove, + .driver = { + .name = "kirin-pcie", + .of_match_table = kirin_pcie_match, + .suppress_bind_attrs = true, + }, +}; +module_platform_driver(kirin_pcie_driver); + +MODULE_DEVICE_TABLE(of, kirin_pcie_match); +MODULE_DESCRIPTION("PCIe host controller driver for Kirin Phone SoCs"); +MODULE_AUTHOR("Xiaowei Song <songxiaowei@huawei.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/dwc/pcie-nxp-s32g.c b/drivers/pci/controller/dwc/pcie-nxp-s32g.c new file mode 100644 index 000000000..31e1169b8 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-nxp-s32g.c @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for NXP S32G SoCs + * + * Copyright 2019-2025 NXP + */ + +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/of_address.h> +#include <linux/pci.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/sizes.h> +#include <linux/types.h> + +#include "pcie-designware.h" + +/* PCIe controller Sub-System */ + +/* PCIe controller 0 General Control 1 */ +#define PCIE_S32G_PE0_GEN_CTRL_1 0x50 +#define DEVICE_TYPE_MASK GENMASK(3, 0) +#define SRIS_MODE BIT(8) + +/* PCIe controller 0 General Control 3 */ +#define PCIE_S32G_PE0_GEN_CTRL_3 0x58 +#define LTSSM_EN BIT(0) + +/* PCIe Controller 0 Interrupt Status */ +#define PCIE_S32G_PE0_INT_STS 0xE8 +#define HP_INT_STS BIT(6) + +/* Boundary between peripheral space and physical memory space */ +#define S32G_MEMORY_BOUNDARY_ADDR 0x80000000 + +struct s32g_pcie_port { + struct list_head list; + struct phy *phy; +}; + +struct s32g_pcie { + struct dw_pcie pci; + void __iomem *ctrl_base; + struct list_head ports; +}; + +#define to_s32g_from_dw_pcie(x) \ + container_of(x, struct s32g_pcie, pci) + +static void s32g_pcie_writel_ctrl(struct s32g_pcie *s32g_pp, u32 reg, u32 val) +{ + writel(val, s32g_pp->ctrl_base + reg); +} + +static u32 s32g_pcie_readl_ctrl(struct s32g_pcie *s32g_pp, u32 reg) +{ + return readl(s32g_pp->ctrl_base + reg); +} + +static void s32g_pcie_enable_ltssm(struct s32g_pcie *s32g_pp) +{ + u32 reg; + + reg = s32g_pcie_readl_ctrl(s32g_pp, PCIE_S32G_PE0_GEN_CTRL_3); + reg |= LTSSM_EN; + s32g_pcie_writel_ctrl(s32g_pp, PCIE_S32G_PE0_GEN_CTRL_3, reg); +} + +static void s32g_pcie_disable_ltssm(struct s32g_pcie *s32g_pp) +{ + u32 reg; + + reg = s32g_pcie_readl_ctrl(s32g_pp, PCIE_S32G_PE0_GEN_CTRL_3); + reg &= ~LTSSM_EN; + s32g_pcie_writel_ctrl(s32g_pp, PCIE_S32G_PE0_GEN_CTRL_3, reg); +} + +static int s32g_pcie_start_link(struct dw_pcie *pci) +{ + struct s32g_pcie *s32g_pp = to_s32g_from_dw_pcie(pci); + + s32g_pcie_enable_ltssm(s32g_pp); + + return 0; +} + +static void s32g_pcie_stop_link(struct dw_pcie *pci) +{ + struct s32g_pcie *s32g_pp = to_s32g_from_dw_pcie(pci); + + s32g_pcie_disable_ltssm(s32g_pp); +} + +static struct dw_pcie_ops s32g_pcie_ops = { + .start_link = s32g_pcie_start_link, + .stop_link = s32g_pcie_stop_link, +}; + +/* Configure the AMBA AXI Coherency Extensions (ACE) interface */ +static void s32g_pcie_reset_mstr_ace(struct dw_pcie *pci) +{ + u32 ddr_base_low = lower_32_bits(S32G_MEMORY_BOUNDARY_ADDR); + u32 ddr_base_high = upper_32_bits(S32G_MEMORY_BOUNDARY_ADDR); + + dw_pcie_dbi_ro_wr_en(pci); + dw_pcie_writel_dbi(pci, COHERENCY_CONTROL_3_OFF, 0x0); + + /* + * Ncore is a cache-coherent interconnect module that enables the + * integration of heterogeneous coherent and non-coherent agents in + * the chip. Ncore transactions to peripheral should be non-coherent + * or it might drop them. + * + * One example where this is needed are PCIe MSIs, which use NoSnoop=0 + * and might end up routed to Ncore. PCIe coherent traffic (e.g. MSIs) + * that targets peripheral space will be dropped by Ncore because + * peripherals on S32G are not coherent as slaves. We add a hard + * boundary in the PCIe controller coherency control registers to + * separate physical memory space from peripheral space. + * + * Define the start of DDR as seen by Linux as this boundary between + * "memory" and "peripherals", with peripherals being below. + */ + dw_pcie_writel_dbi(pci, COHERENCY_CONTROL_1_OFF, + (ddr_base_low & CFG_MEMTYPE_BOUNDARY_LOW_ADDR_MASK)); + dw_pcie_writel_dbi(pci, COHERENCY_CONTROL_2_OFF, ddr_base_high); + dw_pcie_dbi_ro_wr_dis(pci); +} + +static int s32g_init_pcie_controller(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct s32g_pcie *s32g_pp = to_s32g_from_dw_pcie(pci); + u32 val; + + /* Set RP mode */ + val = s32g_pcie_readl_ctrl(s32g_pp, PCIE_S32G_PE0_GEN_CTRL_1); + FIELD_MODIFY(DEVICE_TYPE_MASK, &val, PCI_EXP_TYPE_ROOT_PORT); + + /* Use default CRNS */ + val &= ~SRIS_MODE; + + s32g_pcie_writel_ctrl(s32g_pp, PCIE_S32G_PE0_GEN_CTRL_1, val); + + /* + * Make sure we use the coherency defaults (just in case the settings + * have been changed from their reset values) + */ + s32g_pcie_reset_mstr_ace(pci); + + dw_pcie_dbi_ro_wr_en(pci); + + val = dw_pcie_readl_dbi(pci, PCIE_PORT_FORCE); + val |= PORT_FORCE_DO_DESKEW_FOR_SRIS; + dw_pcie_writel_dbi(pci, PCIE_PORT_FORCE, val); + + val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); + val |= GEN3_RELATED_OFF_EQ_PHASE_2_3; + dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); + + dw_pcie_dbi_ro_wr_dis(pci); + + return 0; +} + +static const struct dw_pcie_host_ops s32g_pcie_host_ops = { + .init = s32g_init_pcie_controller, +}; + +static int s32g_init_pcie_phy(struct s32g_pcie *s32g_pp) +{ + struct dw_pcie *pci = &s32g_pp->pci; + struct device *dev = pci->dev; + struct s32g_pcie_port *port, *tmp; + int ret; + + list_for_each_entry(port, &s32g_pp->ports, list) { + ret = phy_init(port->phy); + if (ret) { + dev_err(dev, "Failed to init serdes PHY\n"); + goto err_phy_revert; + } + + ret = phy_set_mode_ext(port->phy, PHY_MODE_PCIE, 0); + if (ret) { + dev_err(dev, "Failed to set mode on serdes PHY\n"); + goto err_phy_exit; + } + + ret = phy_power_on(port->phy); + if (ret) { + dev_err(dev, "Failed to power on serdes PHY\n"); + goto err_phy_exit; + } + } + + return 0; + +err_phy_exit: + phy_exit(port->phy); + +err_phy_revert: + list_for_each_entry_continue_reverse(port, &s32g_pp->ports, list) { + phy_power_off(port->phy); + phy_exit(port->phy); + } + + list_for_each_entry_safe(port, tmp, &s32g_pp->ports, list) + list_del(&port->list); + + return ret; +} + +static void s32g_deinit_pcie_phy(struct s32g_pcie *s32g_pp) +{ + struct s32g_pcie_port *port, *tmp; + + list_for_each_entry_safe(port, tmp, &s32g_pp->ports, list) { + phy_power_off(port->phy); + phy_exit(port->phy); + list_del(&port->list); + } +} + +static int s32g_pcie_init(struct device *dev, struct s32g_pcie *s32g_pp) +{ + s32g_pcie_disable_ltssm(s32g_pp); + + return s32g_init_pcie_phy(s32g_pp); +} + +static void s32g_pcie_deinit(struct s32g_pcie *s32g_pp) +{ + s32g_pcie_disable_ltssm(s32g_pp); + + s32g_deinit_pcie_phy(s32g_pp); +} + +static int s32g_pcie_parse_port(struct s32g_pcie *s32g_pp, struct device_node *node) +{ + struct device *dev = s32g_pp->pci.dev; + struct s32g_pcie_port *port; + int num_lanes; + + port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); + if (!port) + return -ENOMEM; + + port->phy = devm_of_phy_get(dev, node, NULL); + if (IS_ERR(port->phy)) + return dev_err_probe(dev, PTR_ERR(port->phy), + "Failed to get serdes PHY\n"); + + INIT_LIST_HEAD(&port->list); + list_add_tail(&port->list, &s32g_pp->ports); + + /* + * The DWC core initialization code cannot yet parse the num-lanes + * attribute in the Root Port node. The S32G only supports one Root + * Port for now so its driver can parse the node and set the num_lanes + * field of struct dwc_pcie before calling dw_pcie_host_init(). + */ + if (!of_property_read_u32(node, "num-lanes", &num_lanes)) + s32g_pp->pci.num_lanes = num_lanes; + + return 0; +} + +static int s32g_pcie_parse_ports(struct device *dev, struct s32g_pcie *s32g_pp) +{ + struct s32g_pcie_port *port, *tmp; + int ret = -ENOENT; + + for_each_available_child_of_node_scoped(dev->of_node, of_port) { + if (!of_node_is_type(of_port, "pci")) + continue; + + ret = s32g_pcie_parse_port(s32g_pp, of_port); + if (ret) + break; + } + + if (ret) + list_for_each_entry_safe(port, tmp, &s32g_pp->ports, list) + list_del(&port->list); + + return ret; +} + +static int s32g_pcie_get_resources(struct platform_device *pdev, + struct s32g_pcie *s32g_pp) +{ + struct dw_pcie *pci = &s32g_pp->pci; + struct device *dev = &pdev->dev; + int ret; + + pci->dev = dev; + pci->ops = &s32g_pcie_ops; + + s32g_pp->ctrl_base = devm_platform_ioremap_resource_byname(pdev, "ctrl"); + if (IS_ERR(s32g_pp->ctrl_base)) + return PTR_ERR(s32g_pp->ctrl_base); + + INIT_LIST_HEAD(&s32g_pp->ports); + + ret = s32g_pcie_parse_ports(dev, s32g_pp); + if (ret) + return dev_err_probe(dev, ret, + "Failed to parse Root Port: %d\n", ret); + + platform_set_drvdata(pdev, s32g_pp); + + return 0; +} + +static int s32g_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct s32g_pcie *s32g_pp; + struct dw_pcie_rp *pp; + int ret; + + s32g_pp = devm_kzalloc(dev, sizeof(*s32g_pp), GFP_KERNEL); + if (!s32g_pp) + return -ENOMEM; + + ret = s32g_pcie_get_resources(pdev, s32g_pp); + if (ret) + return ret; + + pm_runtime_no_callbacks(dev); + devm_pm_runtime_enable(dev); + ret = pm_runtime_get_sync(dev); + if (ret < 0) + goto err_pm_runtime_put; + + ret = s32g_pcie_init(dev, s32g_pp); + if (ret) + goto err_pm_runtime_put; + + pp = &s32g_pp->pci.pp; + pp->ops = &s32g_pcie_host_ops; + pp->use_atu_msg = true; + + ret = dw_pcie_host_init(pp); + if (ret) + goto err_pcie_deinit; + + return 0; + +err_pcie_deinit: + s32g_pcie_deinit(s32g_pp); +err_pm_runtime_put: + pm_runtime_put(dev); + + return ret; +} + +static int s32g_pcie_suspend_noirq(struct device *dev) +{ + struct s32g_pcie *s32g_pp = dev_get_drvdata(dev); + struct dw_pcie *pci = &s32g_pp->pci; + + return dw_pcie_suspend_noirq(pci); +} + +static int s32g_pcie_resume_noirq(struct device *dev) +{ + struct s32g_pcie *s32g_pp = dev_get_drvdata(dev); + struct dw_pcie *pci = &s32g_pp->pci; + + return dw_pcie_resume_noirq(pci); +} + +static const struct dev_pm_ops s32g_pcie_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(s32g_pcie_suspend_noirq, + s32g_pcie_resume_noirq) +}; + +static const struct of_device_id s32g_pcie_of_match[] = { + { .compatible = "nxp,s32g2-pcie" }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, s32g_pcie_of_match); + +static struct platform_driver s32g_pcie_driver = { + .driver = { + .name = "s32g-pcie", + .of_match_table = s32g_pcie_of_match, + .suppress_bind_attrs = true, + .pm = pm_sleep_ptr(&s32g_pcie_pm_ops), + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, + .probe = s32g_pcie_probe, +}; + +builtin_platform_driver(s32g_pcie_driver); + +MODULE_AUTHOR("Ionut Vicovan <Ionut.Vicovan@nxp.com>"); +MODULE_DESCRIPTION("NXP S32G PCIe Host controller driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pci/controller/dwc/pcie-qcom-common.c b/drivers/pci/controller/dwc/pcie-qcom-common.c new file mode 100644 index 000000000..0da73caf2 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-qcom-common.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#include <linux/pci.h> + +#include "pcie-designware.h" +#include "pcie-qcom-common.h" + +void qcom_pcie_common_set_equalization(struct dw_pcie *pci) +{ + struct device *dev = pci->dev; + u32 reg; + u16 speed; + + /* + * GEN3_RELATED_OFF register is repurposed to apply equalization + * settings at various data transmission rates through registers namely + * GEN3_EQ_*. The RATE_SHADOW_SEL bit field of GEN3_RELATED_OFF + * determines the data rate for which these equalization settings are + * applied. + */ + + for (speed = PCIE_SPEED_8_0GT; speed <= pcie_get_link_speed(pci->max_link_speed); speed++) { + if (speed > PCIE_SPEED_32_0GT) { + dev_warn(dev, "Skipped equalization settings for unsupported data rate\n"); + break; + } + + reg = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); + reg &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; + FIELD_MODIFY(GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK, ®, + speed - PCIE_SPEED_8_0GT); + dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, reg); + + reg = dw_pcie_readl_dbi(pci, GEN3_EQ_FB_MODE_DIR_CHANGE_OFF); + FIELD_MODIFY(GEN3_EQ_FMDC_T_MIN_PHASE23, ®, 0x1); + FIELD_MODIFY(GEN3_EQ_FMDC_N_EVALS, ®, 0xd); + FIELD_MODIFY(GEN3_EQ_FMDC_MAX_PRE_CURSOR_DELTA, ®, 0x5); + FIELD_MODIFY(GEN3_EQ_FMDC_MAX_POST_CURSOR_DELTA, ®, 0x5); + dw_pcie_writel_dbi(pci, GEN3_EQ_FB_MODE_DIR_CHANGE_OFF, reg); + + reg = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); + reg &= ~(GEN3_EQ_CONTROL_OFF_FB_MODE | + GEN3_EQ_CONTROL_OFF_PHASE23_EXIT_MODE | + GEN3_EQ_CONTROL_OFF_FOM_INC_INITIAL_EVAL | + GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC); + dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, reg); + } +} +EXPORT_SYMBOL_GPL(qcom_pcie_common_set_equalization); + +void qcom_pcie_common_set_16gt_lane_margining(struct dw_pcie *pci) +{ + u32 reg; + + reg = dw_pcie_readl_dbi(pci, GEN4_LANE_MARGINING_1_OFF); + FIELD_MODIFY(MARGINING_MAX_VOLTAGE_OFFSET, ®, 0x24); + FIELD_MODIFY(MARGINING_NUM_VOLTAGE_STEPS, ®, 0x78); + FIELD_MODIFY(MARGINING_MAX_TIMING_OFFSET, ®, 0x32); + FIELD_MODIFY(MARGINING_NUM_TIMING_STEPS, ®, 0x10); + dw_pcie_writel_dbi(pci, GEN4_LANE_MARGINING_1_OFF, reg); + + reg = dw_pcie_readl_dbi(pci, GEN4_LANE_MARGINING_2_OFF); + reg |= MARGINING_IND_ERROR_SAMPLER | + MARGINING_SAMPLE_REPORTING_METHOD | + MARGINING_IND_LEFT_RIGHT_TIMING | + MARGINING_VOLTAGE_SUPPORTED; + reg &= ~MARGINING_IND_UP_DOWN_VOLTAGE; + FIELD_MODIFY(MARGINING_MAXLANES, ®, pci->num_lanes); + FIELD_MODIFY(MARGINING_SAMPLE_RATE_TIMING, ®, 0x3f); + FIELD_MODIFY(MARGINING_SAMPLE_RATE_VOLTAGE, ®, 0x3f); + dw_pcie_writel_dbi(pci, GEN4_LANE_MARGINING_2_OFF, reg); +} +EXPORT_SYMBOL_GPL(qcom_pcie_common_set_16gt_lane_margining); diff --git a/drivers/pci/controller/dwc/pcie-qcom-common.h b/drivers/pci/controller/dwc/pcie-qcom-common.h new file mode 100644 index 000000000..7f5ca2fd9 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-qcom-common.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. + */ + +#ifndef _PCIE_QCOM_COMMON_H +#define _PCIE_QCOM_COMMON_H + +struct dw_pcie; + +void qcom_pcie_common_set_equalization(struct dw_pcie *pci); +void qcom_pcie_common_set_16gt_lane_margining(struct dw_pcie *pci); + +#endif diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c new file mode 100644 index 000000000..56184e6ca --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c @@ -0,0 +1,987 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Qualcomm PCIe Endpoint controller driver + * + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + * Author: Siddartha Mohanadoss <smohanad@codeaurora.org + * + * Copyright (c) 2021, Linaro Ltd. + * Author: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org + */ + +#include <linux/clk.h> +#include <linux/debugfs.h> +#include <linux/delay.h> +#include <linux/gpio/consumer.h> +#include <linux/interconnect.h> +#include <linux/mfd/syscon.h> +#include <linux/phy/pcie.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/pm_domain.h> +#include <linux/regmap.h> +#include <linux/reset.h> +#include <linux/module.h> + +#include "../../pci.h" +#include "pcie-designware.h" +#include "pcie-qcom-common.h" + +/* PARF registers */ +#define PARF_SYS_CTRL 0x00 +#define PARF_DB_CTRL 0x10 +#define PARF_PM_CTRL 0x20 +#define PARF_MHI_CLOCK_RESET_CTRL 0x174 +#define PARF_MHI_BASE_ADDR_LOWER 0x178 +#define PARF_MHI_BASE_ADDR_UPPER 0x17c +#define PARF_DEBUG_INT_EN 0x190 +#define PARF_AXI_MSTR_RD_HALT_NO_WRITES 0x1a4 +#define PARF_AXI_MSTR_WR_ADDR_HALT 0x1a8 +#define PARF_Q2A_FLUSH 0x1ac +#define PARF_LTSSM 0x1b0 +#define PARF_CFG_BITS 0x210 +#define PARF_INT_ALL_STATUS 0x224 +#define PARF_INT_ALL_CLEAR 0x228 +#define PARF_INT_ALL_MASK 0x22c +#define PARF_SLV_ADDR_MSB_CTRL 0x2c0 +#define PARF_DBI_BASE_ADDR 0x350 +#define PARF_DBI_BASE_ADDR_HI 0x354 +#define PARF_SLV_ADDR_SPACE_SIZE 0x358 +#define PARF_SLV_ADDR_SPACE_SIZE_HI 0x35c +#define PARF_NO_SNOOP_OVERRIDE 0x3d4 +#define PARF_ATU_BASE_ADDR 0x634 +#define PARF_ATU_BASE_ADDR_HI 0x638 +#define PARF_SRIS_MODE 0x644 +#define PARF_DEBUG_CNT_PM_LINKST_IN_L2 0xc04 +#define PARF_DEBUG_CNT_PM_LINKST_IN_L1 0xc0c +#define PARF_DEBUG_CNT_PM_LINKST_IN_L0S 0xc10 +#define PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L1 0xc84 +#define PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L2 0xc88 +#define PARF_DEVICE_TYPE 0x1000 +#define PARF_BDF_TO_SID_CFG 0x2c00 +#define PARF_INT_ALL_5_MASK 0x2dcc +#define PARF_INT_ALL_3_MASK 0x2e18 + +/* PARF_INT_ALL_{STATUS/CLEAR/MASK} register fields */ +#define PARF_INT_ALL_LINK_DOWN BIT(1) +#define PARF_INT_ALL_BME BIT(2) +#define PARF_INT_ALL_PM_TURNOFF BIT(3) +#define PARF_INT_ALL_DEBUG BIT(4) +#define PARF_INT_ALL_LTR BIT(5) +#define PARF_INT_ALL_MHI_Q6 BIT(6) +#define PARF_INT_ALL_MHI_A7 BIT(7) +#define PARF_INT_ALL_DSTATE_CHANGE BIT(8) +#define PARF_INT_ALL_L1SUB_TIMEOUT BIT(9) +#define PARF_INT_ALL_MMIO_WRITE BIT(10) +#define PARF_INT_ALL_CFG_WRITE BIT(11) +#define PARF_INT_ALL_BRIDGE_FLUSH_N BIT(12) +#define PARF_INT_ALL_LINK_UP BIT(13) +#define PARF_INT_ALL_AER_LEGACY BIT(14) +#define PARF_INT_ALL_PLS_ERR BIT(15) +#define PARF_INT_ALL_PME_LEGACY BIT(16) +#define PARF_INT_ALL_PLS_PME BIT(17) +#define PARF_INT_ALL_EDMA BIT(22) + +/* PARF_BDF_TO_SID_CFG register fields */ +#define PARF_BDF_TO_SID_BYPASS BIT(0) + +/* PARF_DEBUG_INT_EN register fields */ +#define PARF_DEBUG_INT_PM_DSTATE_CHANGE BIT(1) +#define PARF_DEBUG_INT_CFG_BUS_MASTER_EN BIT(2) +#define PARF_DEBUG_INT_RADM_PM_TURNOFF BIT(3) + +/* PARF_NO_SNOOP_OVERRIDE register fields */ +#define WR_NO_SNOOP_OVERRIDE_EN BIT(1) +#define RD_NO_SNOOP_OVERRIDE_EN BIT(3) + +/* PARF_DEVICE_TYPE register fields */ +#define PARF_DEVICE_TYPE_EP 0x0 + +/* PARF_PM_CTRL register fields */ +#define PARF_PM_CTRL_REQ_EXIT_L1 BIT(1) +#define PARF_PM_CTRL_READY_ENTR_L23 BIT(2) +#define PARF_PM_CTRL_REQ_NOT_ENTR_L1 BIT(5) + +/* PARF_MHI_CLOCK_RESET_CTRL fields */ +#define PARF_MSTR_AXI_CLK_EN BIT(1) + +/* PARF_AXI_MSTR_RD_HALT_NO_WRITES register fields */ +#define PARF_AXI_MSTR_RD_HALT_NO_WRITE_EN BIT(0) + +/* PARF_AXI_MSTR_WR_ADDR_HALT register fields */ +#define PARF_AXI_MSTR_WR_ADDR_HALT_EN BIT(31) + +/* PARF_Q2A_FLUSH register fields */ +#define PARF_Q2A_FLUSH_EN BIT(16) + +/* PARF_SYS_CTRL register fields */ +#define PARF_SYS_CTRL_AUX_PWR_DET BIT(4) +#define PARF_SYS_CTRL_CORE_CLK_CGC_DIS BIT(6) +#define PARF_SYS_CTRL_MSTR_ACLK_CGC_DIS BIT(10) +#define PARF_SYS_CTRL_SLV_DBI_WAKE_DISABLE BIT(11) + +/* PARF_DB_CTRL register fields */ +#define PARF_DB_CTRL_INSR_DBNCR_BLOCK BIT(0) +#define PARF_DB_CTRL_RMVL_DBNCR_BLOCK BIT(1) +#define PARF_DB_CTRL_DBI_WKP_BLOCK BIT(4) +#define PARF_DB_CTRL_SLV_WKP_BLOCK BIT(5) +#define PARF_DB_CTRL_MST_WKP_BLOCK BIT(6) + +/* PARF_CFG_BITS register fields */ +#define PARF_CFG_BITS_REQ_EXIT_L1SS_MSI_LTR_EN BIT(1) + +/* PARF_INT_ALL_5_MASK fields */ +#define PARF_INT_ALL_5_MHI_RAM_DATA_PARITY_ERR BIT(0) + +/* PARF_INT_ALL_3_MASK fields */ +#define PARF_INT_ALL_3_PTM_UPDATING BIT(4) + +/* ELBI registers */ +#define ELBI_SYS_STTS 0x08 +#define ELBI_CS2_ENABLE 0xa4 + +/* DBI registers */ +#define DBI_CON_STATUS 0x44 + +/* DBI register fields */ +#define DBI_CON_STATUS_POWER_STATE_MASK GENMASK(1, 0) + +#define XMLH_LINK_UP 0x400 +#define CORE_RESET_TIME_US_MIN 1000 +#define CORE_RESET_TIME_US_MAX 1005 +#define WAKE_DELAY_US 2000 /* 2 ms */ + +#define QCOM_PCIE_LINK_SPEED_TO_BW(speed) \ + Mbps_to_icc(PCIE_SPEED2MBS_ENC(pcie_get_link_speed(speed))) + +#define to_pcie_ep(x) dev_get_drvdata((x)->dev) + +enum qcom_pcie_ep_link_status { + QCOM_PCIE_EP_LINK_DISABLED, + QCOM_PCIE_EP_LINK_ENABLED, + QCOM_PCIE_EP_LINK_UP, + QCOM_PCIE_EP_LINK_DOWN, +}; + +/** + * struct qcom_pcie_ep_cfg - Per SoC config struct + * @hdma_support: HDMA support on this SoC + * @override_no_snoop: Override NO_SNOOP attribute in TLP to enable cache snooping + * @disable_mhi_ram_parity_check: Disable MHI RAM data parity error check + * @firmware_managed: Set if the controller is firmware managed + */ +struct qcom_pcie_ep_cfg { + bool hdma_support; + bool override_no_snoop; + bool disable_mhi_ram_parity_check; + bool firmware_managed; +}; + +/** + * struct qcom_pcie_ep - Qualcomm PCIe Endpoint Controller + * @pci: Designware PCIe controller struct + * @parf: Qualcomm PCIe specific PARF register base + * @mmio: MMIO register base + * @perst_map: PERST regmap + * @mmio_res: MMIO region resource + * @core_reset: PCIe Endpoint core reset + * @reset: PERST# GPIO + * @wake: WAKE# GPIO + * @phy: PHY controller block + * @debugfs: PCIe Endpoint Debugfs directory + * @icc_mem: Handle to an interconnect path between PCIe and MEM + * @clks: PCIe clocks + * @num_clks: PCIe clocks count + * @perst_en: Flag for PERST enable + * @perst_sep_en: Flag for PERST separation enable + * @cfg: PCIe EP config struct + * @link_status: PCIe Link status + * @global_irq: Qualcomm PCIe specific Global IRQ + * @perst_irq: PERST# IRQ + */ +struct qcom_pcie_ep { + struct dw_pcie pci; + + void __iomem *parf; + void __iomem *mmio; + struct regmap *perst_map; + struct resource *mmio_res; + + struct reset_control *core_reset; + struct gpio_desc *reset; + struct gpio_desc *wake; + struct phy *phy; + struct dentry *debugfs; + + struct icc_path *icc_mem; + + struct clk_bulk_data *clks; + int num_clks; + + u32 perst_en; + u32 perst_sep_en; + + const struct qcom_pcie_ep_cfg *cfg; + enum qcom_pcie_ep_link_status link_status; + int global_irq; + int perst_irq; +}; + +static int qcom_pcie_ep_core_reset(struct qcom_pcie_ep *pcie_ep) +{ + struct dw_pcie *pci = &pcie_ep->pci; + struct device *dev = pci->dev; + int ret; + + ret = reset_control_assert(pcie_ep->core_reset); + if (ret) { + dev_err(dev, "Cannot assert core reset\n"); + return ret; + } + + usleep_range(CORE_RESET_TIME_US_MIN, CORE_RESET_TIME_US_MAX); + + ret = reset_control_deassert(pcie_ep->core_reset); + if (ret) { + dev_err(dev, "Cannot de-assert core reset\n"); + return ret; + } + + usleep_range(CORE_RESET_TIME_US_MIN, CORE_RESET_TIME_US_MAX); + + return 0; +} + +/* + * Delatch PERST_EN and PERST_SEPARATION_ENABLE with TCSR to avoid + * device reset during host reboot and hibernation. The driver is + * expected to handle this situation. + */ +static void qcom_pcie_ep_configure_tcsr(struct qcom_pcie_ep *pcie_ep) +{ + if (pcie_ep->perst_map) { + regmap_write(pcie_ep->perst_map, pcie_ep->perst_en, 0); + regmap_write(pcie_ep->perst_map, pcie_ep->perst_sep_en, 0); + } +} + +static bool qcom_pcie_dw_link_up(struct dw_pcie *pci) +{ + u32 reg; + + reg = readl_relaxed(pci->elbi_base + ELBI_SYS_STTS); + + return reg & XMLH_LINK_UP; +} + +static int qcom_pcie_dw_start_link(struct dw_pcie *pci) +{ + struct qcom_pcie_ep *pcie_ep = to_pcie_ep(pci); + + enable_irq(pcie_ep->perst_irq); + + return 0; +} + +static void qcom_pcie_dw_stop_link(struct dw_pcie *pci) +{ + struct qcom_pcie_ep *pcie_ep = to_pcie_ep(pci); + + disable_irq(pcie_ep->perst_irq); +} + +static void qcom_pcie_dw_write_dbi2(struct dw_pcie *pci, void __iomem *base, + u32 reg, size_t size, u32 val) +{ + int ret; + + writel(1, pci->elbi_base + ELBI_CS2_ENABLE); + + ret = dw_pcie_write(pci->dbi_base2 + reg, size, val); + if (ret) + dev_err(pci->dev, "Failed to write DBI2 register (0x%x): %d\n", reg, ret); + + writel(0, pci->elbi_base + ELBI_CS2_ENABLE); +} + +static void qcom_pcie_ep_icc_update(struct qcom_pcie_ep *pcie_ep) +{ + struct dw_pcie *pci = &pcie_ep->pci; + u32 offset, status; + int speed, width; + int ret; + + if (!pcie_ep->icc_mem) + return; + + offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + status = readw(pci->dbi_base + offset + PCI_EXP_LNKSTA); + + speed = FIELD_GET(PCI_EXP_LNKSTA_CLS, status); + width = FIELD_GET(PCI_EXP_LNKSTA_NLW, status); + + ret = icc_set_bw(pcie_ep->icc_mem, 0, width * QCOM_PCIE_LINK_SPEED_TO_BW(speed)); + if (ret) + dev_err(pci->dev, "failed to set interconnect bandwidth: %d\n", + ret); +} + +static int qcom_pcie_enable_resources(struct qcom_pcie_ep *pcie_ep) +{ + struct dw_pcie *pci = &pcie_ep->pci; + int ret; + + ret = clk_bulk_prepare_enable(pcie_ep->num_clks, pcie_ep->clks); + if (ret) + return ret; + + ret = qcom_pcie_ep_core_reset(pcie_ep); + if (ret) + goto err_disable_clk; + + ret = phy_init(pcie_ep->phy); + if (ret) + goto err_disable_clk; + + ret = phy_set_mode_ext(pcie_ep->phy, PHY_MODE_PCIE, PHY_MODE_PCIE_EP); + if (ret) + goto err_phy_exit; + + ret = phy_power_on(pcie_ep->phy); + if (ret) + goto err_phy_exit; + + /* + * Some Qualcomm platforms require interconnect bandwidth constraints + * to be set before enabling interconnect clocks. + * + * Set an initial peak bandwidth corresponding to single-lane Gen 1 + * for the pcie-mem path. + */ + ret = icc_set_bw(pcie_ep->icc_mem, 0, QCOM_PCIE_LINK_SPEED_TO_BW(1)); + if (ret) { + dev_err(pci->dev, "failed to set interconnect bandwidth: %d\n", + ret); + goto err_phy_off; + } + + return 0; + +err_phy_off: + phy_power_off(pcie_ep->phy); +err_phy_exit: + phy_exit(pcie_ep->phy); +err_disable_clk: + clk_bulk_disable_unprepare(pcie_ep->num_clks, pcie_ep->clks); + + return ret; +} + +static void qcom_pcie_disable_resources(struct qcom_pcie_ep *pcie_ep) +{ + struct device *dev = pcie_ep->pci.dev; + + pm_runtime_put(dev); + + /* Skip resource disablement if controller is firmware-managed */ + if (pcie_ep->cfg && pcie_ep->cfg->firmware_managed) + return; + + icc_set_bw(pcie_ep->icc_mem, 0, 0); + phy_power_off(pcie_ep->phy); + phy_exit(pcie_ep->phy); + clk_bulk_disable_unprepare(pcie_ep->num_clks, pcie_ep->clks); +} + +static int qcom_pcie_perst_deassert(struct dw_pcie *pci) +{ + struct qcom_pcie_ep *pcie_ep = to_pcie_ep(pci); + struct device *dev = pci->dev; + u32 val, offset; + int ret; + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + dev_err(dev, "Failed to enable device: %d\n", ret); + return ret; + } + + /* Skip resource enablement if controller is firmware-managed */ + if (pcie_ep->cfg && pcie_ep->cfg->firmware_managed) + goto skip_resources_enable; + + ret = qcom_pcie_enable_resources(pcie_ep); + if (ret) { + dev_err(dev, "Failed to enable resources: %d\n", ret); + pm_runtime_put(dev); + return ret; + } + +skip_resources_enable: + /* Perform cleanup that requires refclk */ + pci_epc_deinit_notify(pci->ep.epc); + dw_pcie_ep_cleanup(&pci->ep); + + /* Assert WAKE# to RC to indicate device is ready */ + gpiod_set_value_cansleep(pcie_ep->wake, 1); + usleep_range(WAKE_DELAY_US, WAKE_DELAY_US + 500); + gpiod_set_value_cansleep(pcie_ep->wake, 0); + + qcom_pcie_ep_configure_tcsr(pcie_ep); + + /* Disable BDF to SID mapping */ + val = readl_relaxed(pcie_ep->parf + PARF_BDF_TO_SID_CFG); + val |= PARF_BDF_TO_SID_BYPASS; + writel_relaxed(val, pcie_ep->parf + PARF_BDF_TO_SID_CFG); + + /* Enable debug IRQ */ + val = readl_relaxed(pcie_ep->parf + PARF_DEBUG_INT_EN); + val |= PARF_DEBUG_INT_RADM_PM_TURNOFF | + PARF_DEBUG_INT_CFG_BUS_MASTER_EN | + PARF_DEBUG_INT_PM_DSTATE_CHANGE; + writel_relaxed(val, pcie_ep->parf + PARF_DEBUG_INT_EN); + + /* Configure PCIe to endpoint mode */ + writel_relaxed(PARF_DEVICE_TYPE_EP, pcie_ep->parf + PARF_DEVICE_TYPE); + + /* Allow entering L1 state */ + val = readl_relaxed(pcie_ep->parf + PARF_PM_CTRL); + val &= ~PARF_PM_CTRL_REQ_NOT_ENTR_L1; + writel_relaxed(val, pcie_ep->parf + PARF_PM_CTRL); + + /* Read halts write */ + val = readl_relaxed(pcie_ep->parf + PARF_AXI_MSTR_RD_HALT_NO_WRITES); + val &= ~PARF_AXI_MSTR_RD_HALT_NO_WRITE_EN; + writel_relaxed(val, pcie_ep->parf + PARF_AXI_MSTR_RD_HALT_NO_WRITES); + + /* Write after write halt */ + val = readl_relaxed(pcie_ep->parf + PARF_AXI_MSTR_WR_ADDR_HALT); + val |= PARF_AXI_MSTR_WR_ADDR_HALT_EN; + writel_relaxed(val, pcie_ep->parf + PARF_AXI_MSTR_WR_ADDR_HALT); + + /* Q2A flush disable */ + val = readl_relaxed(pcie_ep->parf + PARF_Q2A_FLUSH); + val &= ~PARF_Q2A_FLUSH_EN; + writel_relaxed(val, pcie_ep->parf + PARF_Q2A_FLUSH); + + /* + * Disable Master AXI clock during idle. Do not allow DBI access + * to take the core out of L1. Disable core clock gating that + * gates PIPE clock from propagating to core clock. Report to the + * host that Vaux is present. + */ + val = readl_relaxed(pcie_ep->parf + PARF_SYS_CTRL); + val &= ~PARF_SYS_CTRL_MSTR_ACLK_CGC_DIS; + val |= PARF_SYS_CTRL_SLV_DBI_WAKE_DISABLE | + PARF_SYS_CTRL_CORE_CLK_CGC_DIS | + PARF_SYS_CTRL_AUX_PWR_DET; + writel_relaxed(val, pcie_ep->parf + PARF_SYS_CTRL); + + /* Disable the debouncers */ + val = readl_relaxed(pcie_ep->parf + PARF_DB_CTRL); + val |= PARF_DB_CTRL_INSR_DBNCR_BLOCK | PARF_DB_CTRL_RMVL_DBNCR_BLOCK | + PARF_DB_CTRL_DBI_WKP_BLOCK | PARF_DB_CTRL_SLV_WKP_BLOCK | + PARF_DB_CTRL_MST_WKP_BLOCK; + writel_relaxed(val, pcie_ep->parf + PARF_DB_CTRL); + + /* Request to exit from L1SS for MSI and LTR MSG */ + val = readl_relaxed(pcie_ep->parf + PARF_CFG_BITS); + val |= PARF_CFG_BITS_REQ_EXIT_L1SS_MSI_LTR_EN; + writel_relaxed(val, pcie_ep->parf + PARF_CFG_BITS); + + dw_pcie_dbi_ro_wr_en(pci); + + /* Set the L0s Exit Latency to 2us-4us = 0x6 */ + offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + val = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP); + FIELD_MODIFY(PCI_EXP_LNKCAP_L0SEL, &val, 0x6); + dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, val); + + /* Set the L1 Exit Latency to be 32us-64 us = 0x6 */ + offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + val = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP); + FIELD_MODIFY(PCI_EXP_LNKCAP_L1EL, &val, 0x6); + dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, val); + + dw_pcie_dbi_ro_wr_dis(pci); + + writel_relaxed(0, pcie_ep->parf + PARF_INT_ALL_MASK); + val = PARF_INT_ALL_LINK_DOWN | PARF_INT_ALL_BME | + PARF_INT_ALL_PM_TURNOFF | PARF_INT_ALL_DSTATE_CHANGE | + PARF_INT_ALL_LINK_UP | PARF_INT_ALL_EDMA; + writel_relaxed(val, pcie_ep->parf + PARF_INT_ALL_MASK); + + if (pcie_ep->cfg && pcie_ep->cfg->disable_mhi_ram_parity_check) { + val = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_5_MASK); + val &= ~PARF_INT_ALL_5_MHI_RAM_DATA_PARITY_ERR; + writel_relaxed(val, pcie_ep->parf + PARF_INT_ALL_5_MASK); + } + + val = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_3_MASK); + val &= ~PARF_INT_ALL_3_PTM_UPDATING; + writel_relaxed(val, pcie_ep->parf + PARF_INT_ALL_3_MASK); + + ret = dw_pcie_ep_init_registers(&pcie_ep->pci.ep); + if (ret) { + dev_err(dev, "Failed to complete initialization: %d\n", ret); + goto err_disable_resources; + } + + qcom_pcie_common_set_equalization(pci); + + if (pcie_get_link_speed(pci->max_link_speed) == PCIE_SPEED_16_0GT) + qcom_pcie_common_set_16gt_lane_margining(pci); + + /* + * The physical address of the MMIO region which is exposed as the BAR + * should be written to MHI BASE registers. + */ + writel_relaxed(pcie_ep->mmio_res->start, + pcie_ep->parf + PARF_MHI_BASE_ADDR_LOWER); + writel_relaxed(0, pcie_ep->parf + PARF_MHI_BASE_ADDR_UPPER); + + /* Gate Master AXI clock to MHI bus during L1SS */ + val = readl_relaxed(pcie_ep->parf + PARF_MHI_CLOCK_RESET_CTRL); + val &= ~PARF_MSTR_AXI_CLK_EN; + writel_relaxed(val, pcie_ep->parf + PARF_MHI_CLOCK_RESET_CTRL); + + pci_epc_init_notify(pcie_ep->pci.ep.epc); + + /* Enable LTSSM */ + val = readl_relaxed(pcie_ep->parf + PARF_LTSSM); + val |= BIT(8); + writel_relaxed(val, pcie_ep->parf + PARF_LTSSM); + + if (pcie_ep->cfg && pcie_ep->cfg->override_no_snoop) + writel_relaxed(WR_NO_SNOOP_OVERRIDE_EN | RD_NO_SNOOP_OVERRIDE_EN, + pcie_ep->parf + PARF_NO_SNOOP_OVERRIDE); + + return 0; + +err_disable_resources: + qcom_pcie_disable_resources(pcie_ep); + + return ret; +} + +static void qcom_pcie_perst_assert(struct dw_pcie *pci) +{ + struct qcom_pcie_ep *pcie_ep = to_pcie_ep(pci); + + qcom_pcie_disable_resources(pcie_ep); + pcie_ep->link_status = QCOM_PCIE_EP_LINK_DISABLED; +} + +/* Common DWC controller ops */ +static const struct dw_pcie_ops pci_ops = { + .link_up = qcom_pcie_dw_link_up, + .start_link = qcom_pcie_dw_start_link, + .stop_link = qcom_pcie_dw_stop_link, + .write_dbi2 = qcom_pcie_dw_write_dbi2, +}; + +static int qcom_pcie_ep_get_io_resources(struct platform_device *pdev, + struct qcom_pcie_ep *pcie_ep) +{ + struct device *dev = &pdev->dev; + struct dw_pcie *pci = &pcie_ep->pci; + struct device_node *syscon; + struct resource *res; + int ret; + + pcie_ep->parf = devm_platform_ioremap_resource_byname(pdev, "parf"); + if (IS_ERR(pcie_ep->parf)) + return PTR_ERR(pcie_ep->parf); + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi"); + pci->dbi_base = devm_pci_remap_cfg_resource(dev, res); + if (IS_ERR(pci->dbi_base)) + return PTR_ERR(pci->dbi_base); + pci->dbi_base2 = pci->dbi_base; + + pcie_ep->mmio_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "mmio"); + if (!pcie_ep->mmio_res) { + dev_err(dev, "Failed to get mmio resource\n"); + return -EINVAL; + } + + pcie_ep->mmio = devm_pci_remap_cfg_resource(dev, pcie_ep->mmio_res); + if (IS_ERR(pcie_ep->mmio)) + return PTR_ERR(pcie_ep->mmio); + + syscon = of_parse_phandle(dev->of_node, "qcom,perst-regs", 0); + if (!syscon) { + dev_dbg(dev, "PERST separation not available\n"); + return 0; + } + + pcie_ep->perst_map = syscon_node_to_regmap(syscon); + of_node_put(syscon); + if (IS_ERR(pcie_ep->perst_map)) + return PTR_ERR(pcie_ep->perst_map); + + ret = of_property_read_u32_index(dev->of_node, "qcom,perst-regs", + 1, &pcie_ep->perst_en); + if (ret < 0) { + dev_err(dev, "No Perst Enable offset in syscon\n"); + return ret; + } + + ret = of_property_read_u32_index(dev->of_node, "qcom,perst-regs", + 2, &pcie_ep->perst_sep_en); + if (ret < 0) { + dev_err(dev, "No Perst Separation Enable offset in syscon\n"); + return ret; + } + + return 0; +} + +static int qcom_pcie_ep_get_resources(struct platform_device *pdev, + struct qcom_pcie_ep *pcie_ep) +{ + struct device *dev = &pdev->dev; + int ret; + + ret = qcom_pcie_ep_get_io_resources(pdev, pcie_ep); + if (ret) { + dev_err(dev, "Failed to get io resources %d\n", ret); + return ret; + } + + pcie_ep->reset = devm_gpiod_get(dev, "reset", GPIOD_IN); + if (IS_ERR(pcie_ep->reset)) + return PTR_ERR(pcie_ep->reset); + + pcie_ep->wake = devm_gpiod_get_optional(dev, "wake", GPIOD_OUT_LOW); + if (IS_ERR(pcie_ep->wake)) + return PTR_ERR(pcie_ep->wake); + + if (pcie_ep->cfg && pcie_ep->cfg->firmware_managed) + return 0; + + pcie_ep->num_clks = devm_clk_bulk_get_all(dev, &pcie_ep->clks); + if (pcie_ep->num_clks < 0) { + dev_err(dev, "Failed to get clocks\n"); + return pcie_ep->num_clks; + } + + pcie_ep->core_reset = devm_reset_control_get_exclusive(dev, "core"); + if (IS_ERR(pcie_ep->core_reset)) + return PTR_ERR(pcie_ep->core_reset); + + pcie_ep->phy = devm_phy_optional_get(dev, "pciephy"); + if (IS_ERR(pcie_ep->phy)) + ret = PTR_ERR(pcie_ep->phy); + + pcie_ep->icc_mem = devm_of_icc_get(dev, "pcie-mem"); + if (IS_ERR(pcie_ep->icc_mem)) + ret = PTR_ERR(pcie_ep->icc_mem); + + return ret; +} + +/* TODO: Notify clients about PCIe state change */ +static irqreturn_t qcom_pcie_ep_global_irq_thread(int irq, void *data) +{ + struct qcom_pcie_ep *pcie_ep = data; + struct dw_pcie *pci = &pcie_ep->pci; + struct device *dev = pci->dev; + u32 status = readl_relaxed(pcie_ep->parf + PARF_INT_ALL_STATUS); + u32 dstate, val; + + writel_relaxed(status, pcie_ep->parf + PARF_INT_ALL_CLEAR); + + if (FIELD_GET(PARF_INT_ALL_LINK_DOWN, status)) { + dev_dbg(dev, "Received Linkdown event\n"); + pcie_ep->link_status = QCOM_PCIE_EP_LINK_DOWN; + dw_pcie_ep_linkdown(&pci->ep); + } else if (FIELD_GET(PARF_INT_ALL_BME, status)) { + dev_dbg(dev, "Received Bus Master Enable event\n"); + pcie_ep->link_status = QCOM_PCIE_EP_LINK_ENABLED; + qcom_pcie_ep_icc_update(pcie_ep); + pci_epc_bus_master_enable_notify(pci->ep.epc); + } else if (FIELD_GET(PARF_INT_ALL_PM_TURNOFF, status)) { + dev_dbg(dev, "Received PM Turn-off event! Entering L23\n"); + val = readl_relaxed(pcie_ep->parf + PARF_PM_CTRL); + val |= PARF_PM_CTRL_READY_ENTR_L23; + writel_relaxed(val, pcie_ep->parf + PARF_PM_CTRL); + } else if (FIELD_GET(PARF_INT_ALL_DSTATE_CHANGE, status)) { + dstate = dw_pcie_readl_dbi(pci, DBI_CON_STATUS) & + DBI_CON_STATUS_POWER_STATE_MASK; + dev_dbg(dev, "Received D%d state event\n", dstate); + if (dstate == 3) { + val = readl_relaxed(pcie_ep->parf + PARF_PM_CTRL); + val |= PARF_PM_CTRL_REQ_EXIT_L1; + writel_relaxed(val, pcie_ep->parf + PARF_PM_CTRL); + } + } else if (FIELD_GET(PARF_INT_ALL_LINK_UP, status)) { + dev_dbg(dev, "Received Linkup event. Enumeration complete!\n"); + dw_pcie_ep_linkup(&pci->ep); + pcie_ep->link_status = QCOM_PCIE_EP_LINK_UP; + } else { + dev_WARN_ONCE(dev, 1, "Received unknown event. INT_STATUS: 0x%08x\n", + status); + } + + return IRQ_HANDLED; +} + +static irqreturn_t qcom_pcie_ep_perst_irq_thread(int irq, void *data) +{ + struct qcom_pcie_ep *pcie_ep = data; + struct dw_pcie *pci = &pcie_ep->pci; + struct device *dev = pci->dev; + u32 perst; + + perst = gpiod_get_value(pcie_ep->reset); + if (perst) { + dev_dbg(dev, "PERST asserted by host. Shutting down the PCIe link!\n"); + qcom_pcie_perst_assert(pci); + } else { + dev_dbg(dev, "PERST de-asserted by host. Starting link training!\n"); + qcom_pcie_perst_deassert(pci); + } + + irq_set_irq_type(gpiod_to_irq(pcie_ep->reset), + (perst ? IRQF_TRIGGER_HIGH : IRQF_TRIGGER_LOW)); + + return IRQ_HANDLED; +} + +static int qcom_pcie_ep_enable_irq_resources(struct platform_device *pdev, + struct qcom_pcie_ep *pcie_ep) +{ + struct device *dev = pcie_ep->pci.dev; + char *name; + int ret; + + name = devm_kasprintf(dev, GFP_KERNEL, "qcom_pcie_ep_global_irq%d", + pcie_ep->pci.ep.epc->domain_nr); + if (!name) + return -ENOMEM; + + pcie_ep->global_irq = platform_get_irq_byname(pdev, "global"); + if (pcie_ep->global_irq < 0) + return pcie_ep->global_irq; + + ret = devm_request_threaded_irq(&pdev->dev, pcie_ep->global_irq, NULL, + qcom_pcie_ep_global_irq_thread, + IRQF_ONESHOT, + name, pcie_ep); + if (ret) { + dev_err(&pdev->dev, "Failed to request Global IRQ\n"); + return ret; + } + + name = devm_kasprintf(dev, GFP_KERNEL, "qcom_pcie_ep_perst_irq%d", + pcie_ep->pci.ep.epc->domain_nr); + if (!name) + return -ENOMEM; + + pcie_ep->perst_irq = gpiod_to_irq(pcie_ep->reset); + irq_set_status_flags(pcie_ep->perst_irq, IRQ_NOAUTOEN); + ret = devm_request_threaded_irq(&pdev->dev, pcie_ep->perst_irq, NULL, + qcom_pcie_ep_perst_irq_thread, + IRQF_TRIGGER_HIGH | IRQF_ONESHOT, + name, pcie_ep); + if (ret) { + dev_err(&pdev->dev, "Failed to request PERST IRQ\n"); + disable_irq(pcie_ep->global_irq); + return ret; + } + + return 0; +} + +static int qcom_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, + unsigned int type, u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + switch (type) { + case PCI_IRQ_INTX: + return dw_pcie_ep_raise_intx_irq(ep, func_no); + case PCI_IRQ_MSI: + return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); + default: + dev_err(pci->dev, "Unknown IRQ type\n"); + return -EINVAL; + } +} + +static int qcom_pcie_ep_link_transition_count(struct seq_file *s, void *data) +{ + struct qcom_pcie_ep *pcie_ep = (struct qcom_pcie_ep *) + dev_get_drvdata(s->private); + + seq_printf(s, "L0s transition count: %u\n", + readl_relaxed(pcie_ep->mmio + PARF_DEBUG_CNT_PM_LINKST_IN_L0S)); + + seq_printf(s, "L1 transition count: %u\n", + readl_relaxed(pcie_ep->mmio + PARF_DEBUG_CNT_PM_LINKST_IN_L1)); + + seq_printf(s, "L1.1 transition count: %u\n", + readl_relaxed(pcie_ep->mmio + PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L1)); + + seq_printf(s, "L1.2 transition count: %u\n", + readl_relaxed(pcie_ep->mmio + PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L2)); + + seq_printf(s, "L2 transition count: %u\n", + readl_relaxed(pcie_ep->mmio + PARF_DEBUG_CNT_PM_LINKST_IN_L2)); + + return 0; +} + +static void qcom_pcie_ep_init_debugfs(struct qcom_pcie_ep *pcie_ep) +{ + struct dw_pcie *pci = &pcie_ep->pci; + + debugfs_create_devm_seqfile(pci->dev, "link_transition_count", pcie_ep->debugfs, + qcom_pcie_ep_link_transition_count); +} + +static const struct pci_epc_features qcom_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, + .linkup_notifier = true, + .msi_capable = true, + .align = SZ_4K, + .bar[BAR_0] = { .only_64bit = true, }, + .bar[BAR_2] = { .only_64bit = true, }, +}; + +static const struct pci_epc_features * +qcom_pcie_epc_get_features(struct dw_pcie_ep *pci_ep) +{ + return &qcom_pcie_epc_features; +} + +static const struct dw_pcie_ep_ops pci_ep_ops = { + .raise_irq = qcom_pcie_ep_raise_irq, + .get_features = qcom_pcie_epc_get_features, +}; + +static int qcom_pcie_ep_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct qcom_pcie_ep *pcie_ep; + char *name; + int ret; + + pcie_ep = devm_kzalloc(dev, sizeof(*pcie_ep), GFP_KERNEL); + if (!pcie_ep) + return -ENOMEM; + + pcie_ep->pci.dev = dev; + pcie_ep->pci.ops = &pci_ops; + pcie_ep->pci.ep.ops = &pci_ep_ops; + + pcie_ep->cfg = of_device_get_match_data(dev); + if (pcie_ep->cfg && pcie_ep->cfg->hdma_support) { + pcie_ep->pci.edma.ll_wr_cnt = 8; + pcie_ep->pci.edma.ll_rd_cnt = 8; + pcie_ep->pci.edma.mf = EDMA_MF_HDMA_NATIVE; + } + + platform_set_drvdata(pdev, pcie_ep); + + pm_runtime_get_noresume(dev); + pm_runtime_set_active(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; + + ret = qcom_pcie_ep_get_resources(pdev, pcie_ep); + if (ret) + return ret; + + ret = dw_pcie_ep_init(&pcie_ep->pci.ep); + if (ret) { + dev_err(dev, "Failed to initialize endpoint: %d\n", ret); + return ret; + } + + ret = qcom_pcie_ep_enable_irq_resources(pdev, pcie_ep); + if (ret) + goto err_ep_deinit; + + name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node); + if (!name) { + ret = -ENOMEM; + goto err_disable_irqs; + } + + ret = pm_runtime_put_sync(dev); + if (ret < 0) { + dev_err(dev, "Failed to suspend device: %d\n", ret); + goto err_disable_irqs; + } + + pcie_ep->debugfs = debugfs_create_dir(name, NULL); + qcom_pcie_ep_init_debugfs(pcie_ep); + + return 0; + +err_disable_irqs: + disable_irq(pcie_ep->global_irq); + disable_irq(pcie_ep->perst_irq); + +err_ep_deinit: + dw_pcie_ep_deinit(&pcie_ep->pci.ep); + + return ret; +} + +static void qcom_pcie_ep_remove(struct platform_device *pdev) +{ + struct qcom_pcie_ep *pcie_ep = platform_get_drvdata(pdev); + + disable_irq(pcie_ep->global_irq); + disable_irq(pcie_ep->perst_irq); + + debugfs_remove_recursive(pcie_ep->debugfs); + + if (pcie_ep->link_status == QCOM_PCIE_EP_LINK_DISABLED) + return; + + qcom_pcie_disable_resources(pcie_ep); +} + +static const struct qcom_pcie_ep_cfg cfg_1_34_0 = { + .hdma_support = true, + .override_no_snoop = true, + .disable_mhi_ram_parity_check = true, +}; + +static const struct qcom_pcie_ep_cfg cfg_1_34_0_fw_managed = { + .hdma_support = true, + .override_no_snoop = true, + .disable_mhi_ram_parity_check = true, + .firmware_managed = true, +}; + +static const struct of_device_id qcom_pcie_ep_match[] = { + { .compatible = "qcom,sa8255p-pcie-ep", .data = &cfg_1_34_0_fw_managed}, + { .compatible = "qcom,sa8775p-pcie-ep", .data = &cfg_1_34_0}, + { .compatible = "qcom,sdx55-pcie-ep", }, + { .compatible = "qcom,sm8450-pcie-ep", }, + { .compatible = "qcom,sar2130p-pcie-ep", }, + { } +}; +MODULE_DEVICE_TABLE(of, qcom_pcie_ep_match); + +static struct platform_driver qcom_pcie_ep_driver = { + .probe = qcom_pcie_ep_probe, + .remove = qcom_pcie_ep_remove, + .driver = { + .name = "qcom-pcie-ep", + .of_match_table = qcom_pcie_ep_match, + }, +}; +builtin_platform_driver(qcom_pcie_ep_driver); + +MODULE_AUTHOR("Siddartha Mohanadoss <smohanad@codeaurora.org>"); +MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>"); +MODULE_DESCRIPTION("Qualcomm PCIe Endpoint controller driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c new file mode 100644 index 000000000..b58a607b7 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-qcom.c @@ -0,0 +1,2523 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Qualcomm PCIe root complex driver + * + * Copyright (c) 2014-2015, The Linux Foundation. All rights reserved. + * Copyright 2015 Linaro Limited. + * + * Author: Stanimir Varbanov <svarbanov@mm-sol.com> + */ + +#include <linux/clk.h> +#include <linux/crc8.h> +#include <linux/debugfs.h> +#include <linux/delay.h> +#include <linux/gpio/consumer.h> +#include <linux/interconnect.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/limits.h> +#include <linux/init.h> +#include <linux/of.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/pci-ecam.h> +#include <linux/pci-pwrctrl.h> +#include <linux/pm_opp.h> +#include <linux/pm_runtime.h> +#include <linux/platform_device.h> +#include <linux/phy/pcie.h> +#include <linux/phy/phy.h> +#include <linux/regulator/consumer.h> +#include <linux/reset.h> +#include <linux/slab.h> +#include <linux/types.h> +#include <linux/units.h> + +#include "../../pci.h" +#include "../pci-host-common.h" +#include "pcie-designware.h" +#include "pcie-qcom-common.h" + +/* PARF registers */ +#define PARF_SYS_CTRL 0x00 +#define PARF_PM_CTRL 0x20 +#define PARF_PCS_DEEMPH 0x34 +#define PARF_PCS_SWING 0x38 +#define PARF_PHY_CTRL 0x40 +#define PARF_PHY_REFCLK 0x4c +#define PARF_CONFIG_BITS 0x50 +#define PARF_DBI_BASE_ADDR 0x168 +#define PARF_SLV_ADDR_SPACE_SIZE 0x16c +#define PARF_MHI_CLOCK_RESET_CTRL 0x174 +#define PARF_AXI_MSTR_WR_ADDR_HALT 0x178 +#define PARF_AXI_MSTR_WR_ADDR_HALT_V2 0x1a8 +#define PARF_Q2A_FLUSH 0x1ac +#define PARF_LTSSM 0x1b0 +#define PARF_INT_ALL_STATUS 0x224 +#define PARF_INT_ALL_CLEAR 0x228 +#define PARF_INT_ALL_MASK 0x22c +#define PARF_STATUS 0x230 +#define PARF_SID_OFFSET 0x234 +#define PARF_BDF_TRANSLATE_CFG 0x24c +#define PARF_DBI_BASE_ADDR_V2 0x350 +#define PARF_DBI_BASE_ADDR_V2_HI 0x354 +#define PARF_SLV_ADDR_SPACE_SIZE_V2 0x358 +#define PARF_SLV_ADDR_SPACE_SIZE_V2_HI 0x35c +#define PARF_NO_SNOOP_OVERRIDE 0x3d4 +#define PARF_ATU_BASE_ADDR 0x634 +#define PARF_ATU_BASE_ADDR_HI 0x638 +#define PARF_DEVICE_TYPE 0x1000 +#define PARF_BDF_TO_SID_TABLE_N 0x2000 +#define PARF_BDF_TO_SID_CFG 0x2c00 + +/* ELBI registers */ +#define ELBI_SYS_CTRL 0x04 +#define ELBI_SYS_STTS 0x08 + +/* DBI registers */ +#define AXI_MSTR_RESP_COMP_CTRL0 0x818 +#define AXI_MSTR_RESP_COMP_CTRL1 0x81c + +/* MHI registers */ +#define PARF_DEBUG_CNT_PM_LINKST_IN_L2 0xc04 +#define PARF_DEBUG_CNT_PM_LINKST_IN_L1 0xc0c +#define PARF_DEBUG_CNT_PM_LINKST_IN_L0S 0xc10 +#define PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L1 0xc84 +#define PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L2 0xc88 + +/* PARF_SYS_CTRL register fields */ +#define MAC_PHY_POWERDOWN_IN_P2_D_MUX_EN BIT(29) +#define MST_WAKEUP_EN BIT(13) +#define SLV_WAKEUP_EN BIT(12) +#define MSTR_ACLK_CGC_DIS BIT(10) +#define SLV_ACLK_CGC_DIS BIT(9) +#define CORE_CLK_CGC_DIS BIT(6) +#define AUX_PWR_DET BIT(4) +#define L23_CLK_RMV_DIS BIT(2) +#define L1_CLK_RMV_DIS BIT(1) + +/* PARF_PM_CTRL register fields */ +#define REQ_NOT_ENTR_L1 BIT(5) + +/* PARF_PCS_DEEMPH register fields */ +#define PCS_DEEMPH_TX_DEEMPH_GEN1(x) FIELD_PREP(GENMASK(21, 16), x) +#define PCS_DEEMPH_TX_DEEMPH_GEN2_3_5DB(x) FIELD_PREP(GENMASK(13, 8), x) +#define PCS_DEEMPH_TX_DEEMPH_GEN2_6DB(x) FIELD_PREP(GENMASK(5, 0), x) + +/* PARF_PCS_SWING register fields */ +#define PCS_SWING_TX_SWING_FULL(x) FIELD_PREP(GENMASK(14, 8), x) +#define PCS_SWING_TX_SWING_LOW(x) FIELD_PREP(GENMASK(6, 0), x) + +/* PARF_PHY_CTRL register fields */ +#define PHY_CTRL_PHY_TX0_TERM_OFFSET_MASK GENMASK(20, 16) +#define PHY_CTRL_PHY_TX0_TERM_OFFSET(x) FIELD_PREP(PHY_CTRL_PHY_TX0_TERM_OFFSET_MASK, x) +#define PHY_TEST_PWR_DOWN BIT(0) + +/* PARF_PHY_REFCLK register fields */ +#define PHY_REFCLK_SSP_EN BIT(16) +#define PHY_REFCLK_USE_PAD BIT(12) + +/* PARF_CONFIG_BITS register fields */ +#define PHY_RX0_EQ(x) FIELD_PREP(GENMASK(26, 24), x) + +/* PARF_SLV_ADDR_SPACE_SIZE register value */ +#define SLV_ADDR_SPACE_SZ 0x80000000 + +/* PARF_MHI_CLOCK_RESET_CTRL register fields */ +#define AHB_CLK_EN BIT(0) +#define MSTR_AXI_CLK_EN BIT(1) +#define BYPASS BIT(4) + +/* PARF_AXI_MSTR_WR_ADDR_HALT register fields */ +#define EN BIT(31) + +/* PARF_LTSSM register fields */ +#define LTSSM_EN BIT(8) +#define PARF_LTSSM_STATE_MASK GENMASK(5, 0) +#define SW_CLEAR_FLUSH_MODE BIT(10) +#define FLUSH_MODE BIT(11) + +/* PARF_INT_ALL_{STATUS/CLEAR/MASK} register fields */ +#define INT_ALL_LINK_DOWN 1 +#define PARF_INT_ALL_LINK_DOWN BIT(INT_ALL_LINK_DOWN) +#define PARF_INT_MSI_DEV_0_7 GENMASK(30, 23) + +/* PARF_NO_SNOOP_OVERRIDE register fields */ +#define WR_NO_SNOOP_OVERRIDE_EN BIT(1) +#define RD_NO_SNOOP_OVERRIDE_EN BIT(3) + +/* PARF_DEVICE_TYPE register fields */ +#define DEVICE_TYPE_RC 0x4 + +/* PARF_BDF_TO_SID_CFG fields */ +#define BDF_TO_SID_BYPASS BIT(0) + +/* PARF_STATUS fields */ +#define FLUSH_COMPLETED BIT(8) + +/* ELBI_SYS_CTRL register fields */ +#define ELBI_SYS_CTRL_LT_ENABLE BIT(0) +#define ELBI_SYS_CTRL_PME_TURNOFF_MSG BIT(4) + +/* ELBI_SYS_STTS register fields */ +#define ELBI_SYS_STTS_LTSSM_STATE_MASK GENMASK(17, 12) + +/* AXI_MSTR_RESP_COMP_CTRL0 register fields */ +#define CFG_REMOTE_RD_REQ_BRIDGE_SIZE_2K 0x4 +#define CFG_REMOTE_RD_REQ_BRIDGE_SIZE_4K 0x5 + +/* AXI_MSTR_RESP_COMP_CTRL1 register fields */ +#define CFG_BRIDGE_SB_INIT BIT(0) + +/* PCI_EXP_SLTCAP register fields */ +#define PCIE_CAP_SLOT_POWER_LIMIT_VAL FIELD_PREP(PCI_EXP_SLTCAP_SPLV, 250) +#define PCIE_CAP_SLOT_POWER_LIMIT_SCALE FIELD_PREP(PCI_EXP_SLTCAP_SPLS, 1) +#define PCIE_CAP_SLOT_VAL (PCI_EXP_SLTCAP_ABP | \ + PCI_EXP_SLTCAP_PCP | \ + PCI_EXP_SLTCAP_MRLSP | \ + PCI_EXP_SLTCAP_AIP | \ + PCI_EXP_SLTCAP_PIP | \ + PCI_EXP_SLTCAP_HPS | \ + PCI_EXP_SLTCAP_EIP | \ + PCIE_CAP_SLOT_POWER_LIMIT_VAL | \ + PCIE_CAP_SLOT_POWER_LIMIT_SCALE) + +#define PERST_DELAY_US 1000 +#define FLUSH_TIMEOUT_US 100 + +#define QCOM_PCIE_CRC8_POLYNOMIAL (BIT(2) | BIT(1) | BIT(0)) + +#define QCOM_PCIE_LINK_SPEED_TO_BW(speed) \ + Mbps_to_icc(PCIE_SPEED2MBS_ENC(pcie_get_link_speed(speed))) + +struct qcom_pcie_resources_1_0_0 { + struct clk_bulk_data *clks; + int num_clks; + struct reset_control *core; + struct regulator *vdda; +}; + +#define QCOM_PCIE_2_1_0_MAX_RESETS 6 +#define QCOM_PCIE_2_1_0_MAX_SUPPLY 3 +struct qcom_pcie_resources_2_1_0 { + struct clk_bulk_data *clks; + int num_clks; + struct reset_control_bulk_data resets[QCOM_PCIE_2_1_0_MAX_RESETS]; + int num_resets; + struct regulator_bulk_data supplies[QCOM_PCIE_2_1_0_MAX_SUPPLY]; +}; + +#define QCOM_PCIE_2_3_2_MAX_SUPPLY 2 +struct qcom_pcie_resources_2_3_2 { + struct clk_bulk_data *clks; + int num_clks; + struct regulator_bulk_data supplies[QCOM_PCIE_2_3_2_MAX_SUPPLY]; +}; + +#define QCOM_PCIE_2_3_3_MAX_RESETS 7 +struct qcom_pcie_resources_2_3_3 { + struct clk_bulk_data *clks; + int num_clks; + struct reset_control_bulk_data rst[QCOM_PCIE_2_3_3_MAX_RESETS]; +}; + +#define QCOM_PCIE_2_4_0_MAX_RESETS 12 +struct qcom_pcie_resources_2_4_0 { + struct clk_bulk_data *clks; + int num_clks; + struct reset_control_bulk_data resets[QCOM_PCIE_2_4_0_MAX_RESETS]; + int num_resets; +}; + +#define QCOM_PCIE_2_7_0_MAX_SUPPLIES 2 +struct qcom_pcie_resources_2_7_0 { + struct clk_bulk_data *clks; + int num_clks; + struct regulator_bulk_data supplies[QCOM_PCIE_2_7_0_MAX_SUPPLIES]; + struct reset_control *rst; +}; + +struct qcom_pcie_resources_2_9_0 { + struct clk_bulk_data *clks; + int num_clks; + struct reset_control *rst; +}; + +union qcom_pcie_resources { + struct qcom_pcie_resources_1_0_0 v1_0_0; + struct qcom_pcie_resources_2_1_0 v2_1_0; + struct qcom_pcie_resources_2_3_2 v2_3_2; + struct qcom_pcie_resources_2_3_3 v2_3_3; + struct qcom_pcie_resources_2_4_0 v2_4_0; + struct qcom_pcie_resources_2_7_0 v2_7_0; + struct qcom_pcie_resources_2_9_0 v2_9_0; +}; + +struct qcom_pcie; + +struct qcom_pcie_ops { + int (*get_resources)(struct qcom_pcie *pcie); + int (*init)(struct qcom_pcie *pcie); + int (*post_init)(struct qcom_pcie *pcie); + void (*host_post_init)(struct qcom_pcie *pcie); + void (*deinit)(struct qcom_pcie *pcie); + void (*ltssm_enable)(struct qcom_pcie *pcie); + int (*config_sid)(struct qcom_pcie *pcie); + enum dw_pcie_ltssm (*get_ltssm)(struct qcom_pcie *pcie); +}; + + /** + * struct qcom_pcie_cfg - Per SoC config struct + * @ops: qcom PCIe ops structure + * @override_no_snoop: Override NO_SNOOP attribute in TLP to enable cache + * snooping + * @firmware_managed: Set if the Root Complex is firmware managed + */ +struct qcom_pcie_cfg { + const struct qcom_pcie_ops *ops; + bool override_no_snoop; + bool firmware_managed; + bool no_l0s; +}; + +struct qcom_pcie_perst { + struct list_head list; + struct gpio_desc *desc; +}; + +struct qcom_pcie_port { + struct list_head list; + struct phy *phy; + u32 l1ss_t_power_on; + struct list_head perst; +}; + +struct qcom_pcie { + struct dw_pcie *pci; + void __iomem *parf; /* DT parf */ + void __iomem *mhi; + union qcom_pcie_resources res; + struct icc_path *icc_mem; + struct icc_path *icc_cpu; + const struct qcom_pcie_cfg *cfg; + struct dentry *debugfs; + struct list_head ports; + struct gpio_desc *reset; + int global_irq; + bool use_pm_opp; +}; + +#define to_qcom_pcie(x) dev_get_drvdata((x)->dev) +static int qcom_pcie_reset_root_port(struct pci_host_bridge *bridge, + struct pci_dev *pdev); + +static void __qcom_pcie_perst_assert(struct qcom_pcie *pcie, bool assert) +{ + struct qcom_pcie_perst *perst; + struct qcom_pcie_port *port; + int val = assert ? 1 : 0; + + list_for_each_entry(port, &pcie->ports, list) { + list_for_each_entry(perst, &port->perst, list) + gpiod_set_value_cansleep(perst->desc, val); + } + + usleep_range(PERST_DELAY_US, PERST_DELAY_US + 500); +} + +static void qcom_pcie_perst_assert(struct qcom_pcie *pcie) +{ + __qcom_pcie_perst_assert(pcie, true); +} + +static void qcom_pcie_perst_deassert(struct qcom_pcie *pcie) +{ + /* Ensure that PERST# has been asserted for at least 100 ms */ + msleep(PCIE_T_PVPERL_MS); + __qcom_pcie_perst_assert(pcie, false); +} + +static int qcom_pcie_start_link(struct dw_pcie *pci) +{ + struct qcom_pcie *pcie = to_qcom_pcie(pci); + + qcom_pcie_common_set_equalization(pci); + + if (pcie_get_link_speed(pci->max_link_speed) == PCIE_SPEED_16_0GT) + qcom_pcie_common_set_16gt_lane_margining(pci); + + /* Enable Link Training state machine */ + if (pcie->cfg->ops->ltssm_enable) + pcie->cfg->ops->ltssm_enable(pcie); + + return 0; +} + +static void qcom_pcie_clear_aspm_l0s(struct dw_pcie *pci) +{ + struct qcom_pcie *pcie = to_qcom_pcie(pci); + u16 offset; + u32 val; + + if (!pcie->cfg->no_l0s) + return; + + offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + + dw_pcie_dbi_ro_wr_en(pci); + + val = readl(pci->dbi_base + offset + PCI_EXP_LNKCAP); + val &= ~PCI_EXP_LNKCAP_ASPM_L0S; + writel(val, pci->dbi_base + offset + PCI_EXP_LNKCAP); + + dw_pcie_dbi_ro_wr_dis(pci); +} + +static void qcom_pcie_set_slot_cap(struct dw_pcie *pci) +{ + u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + u32 val; + + dw_pcie_dbi_ro_wr_en(pci); + + /* + * Qcom PCIe Root Ports do not support generating command completion + * notifications for the Hot-Plug commands. So set the NCCS field to + * avoid waiting for the completions. + */ + val = readl(pci->dbi_base + offset + PCI_EXP_SLTCAP); + val |= PCI_EXP_SLTCAP_NCCS; + + /* + * Qcom PCIe Root Ports do not support Attention Button, so clear + * Attention Button Present in Slot Capabilities. + */ + val &= ~PCI_EXP_SLTCAP_ABP; + writel(val, pci->dbi_base + offset + PCI_EXP_SLTCAP); + + dw_pcie_dbi_ro_wr_dis(pci); +} + +static void qcom_pcie_configure_dbi_base(struct qcom_pcie *pcie) +{ + struct dw_pcie *pci = pcie->pci; + + if (pci->dbi_phys_addr) { + /* + * PARF_DBI_BASE_ADDR register is in CPU domain and require to + * be programmed with CPU physical address. + */ + writel(lower_32_bits(pci->dbi_phys_addr), pcie->parf + + PARF_DBI_BASE_ADDR); + writel(SLV_ADDR_SPACE_SZ, pcie->parf + + PARF_SLV_ADDR_SPACE_SIZE); + } +} + +static void qcom_pcie_configure_dbi_atu_base(struct qcom_pcie *pcie) +{ + struct dw_pcie *pci = pcie->pci; + + if (pci->dbi_phys_addr) { + /* + * PARF_DBI_BASE_ADDR_V2 and PARF_ATU_BASE_ADDR registers are + * in CPU domain and require to be programmed with CPU + * physical addresses. + */ + writel(lower_32_bits(pci->dbi_phys_addr), pcie->parf + + PARF_DBI_BASE_ADDR_V2); + writel(upper_32_bits(pci->dbi_phys_addr), pcie->parf + + PARF_DBI_BASE_ADDR_V2_HI); + + if (pci->atu_phys_addr) { + writel(lower_32_bits(pci->atu_phys_addr), pcie->parf + + PARF_ATU_BASE_ADDR); + writel(upper_32_bits(pci->atu_phys_addr), pcie->parf + + PARF_ATU_BASE_ADDR_HI); + } + + writel(0x0, pcie->parf + PARF_SLV_ADDR_SPACE_SIZE_V2); + writel(SLV_ADDR_SPACE_SZ, pcie->parf + + PARF_SLV_ADDR_SPACE_SIZE_V2_HI); + } +} + +static void qcom_pcie_2_1_0_ltssm_enable(struct qcom_pcie *pcie) +{ + struct dw_pcie *pci = pcie->pci; + u32 val; + + if (!pci->elbi_base) { + dev_err(pci->dev, "ELBI is not present\n"); + return; + } + /* enable link training */ + val = readl(pci->elbi_base + ELBI_SYS_CTRL); + val |= ELBI_SYS_CTRL_LT_ENABLE; + writel(val, pci->elbi_base + ELBI_SYS_CTRL); +} + +static enum dw_pcie_ltssm qcom_pcie_2_1_0_get_ltssm(struct qcom_pcie *pcie) +{ + struct dw_pcie *pci = pcie->pci; + u32 val; + + val = readl(pci->elbi_base + ELBI_SYS_STTS); + return (enum dw_pcie_ltssm)FIELD_GET(ELBI_SYS_STTS_LTSSM_STATE_MASK, val); +} + +static int qcom_pcie_get_resources_2_1_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_1_0 *res = &pcie->res.v2_1_0; + struct dw_pcie *pci = pcie->pci; + struct device *dev = pci->dev; + bool is_apq = of_device_is_compatible(dev->of_node, "qcom,pcie-apq8064"); + int ret; + + res->supplies[0].supply = "vdda"; + res->supplies[1].supply = "vdda_phy"; + res->supplies[2].supply = "vdda_refclk"; + ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(res->supplies), + res->supplies); + if (ret) + return ret; + + res->num_clks = devm_clk_bulk_get_all(dev, &res->clks); + if (res->num_clks < 0) { + dev_err(dev, "Failed to get clocks\n"); + return res->num_clks; + } + + res->resets[0].id = "pci"; + res->resets[1].id = "axi"; + res->resets[2].id = "ahb"; + res->resets[3].id = "por"; + res->resets[4].id = "phy"; + res->resets[5].id = "ext"; + + /* ext is optional on APQ8016 */ + res->num_resets = is_apq ? 5 : 6; + ret = devm_reset_control_bulk_get_exclusive(dev, res->num_resets, res->resets); + if (ret < 0) + return ret; + + return 0; +} + +static void qcom_pcie_deinit_2_1_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_1_0 *res = &pcie->res.v2_1_0; + + clk_bulk_disable_unprepare(res->num_clks, res->clks); + reset_control_bulk_assert(res->num_resets, res->resets); + + writel(1, pcie->parf + PARF_PHY_CTRL); + + regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies); +} + +static int qcom_pcie_init_2_1_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_1_0 *res = &pcie->res.v2_1_0; + struct dw_pcie *pci = pcie->pci; + struct device *dev = pci->dev; + int ret; + + /* reset the PCIe interface as uboot can leave it undefined state */ + ret = reset_control_bulk_assert(res->num_resets, res->resets); + if (ret < 0) { + dev_err(dev, "cannot assert resets\n"); + return ret; + } + + ret = regulator_bulk_enable(ARRAY_SIZE(res->supplies), res->supplies); + if (ret < 0) { + dev_err(dev, "cannot enable regulators\n"); + return ret; + } + + ret = reset_control_bulk_deassert(res->num_resets, res->resets); + if (ret < 0) { + dev_err(dev, "cannot deassert resets\n"); + regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies); + return ret; + } + + return 0; +} + +static int qcom_pcie_post_init_2_1_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_1_0 *res = &pcie->res.v2_1_0; + struct dw_pcie *pci = pcie->pci; + struct device *dev = pci->dev; + struct device_node *node = dev->of_node; + u32 val; + int ret; + + /* Force PHY out of lowest power state */ + val = readl(pcie->parf + PARF_PHY_CTRL); + val &= ~PHY_TEST_PWR_DOWN; + writel(val, pcie->parf + PARF_PHY_CTRL); + + ret = clk_bulk_prepare_enable(res->num_clks, res->clks); + if (ret) + return ret; + + if (of_device_is_compatible(node, "qcom,pcie-ipq8064") || + of_device_is_compatible(node, "qcom,pcie-ipq8064-v2")) { + writel(PCS_DEEMPH_TX_DEEMPH_GEN1(24) | + PCS_DEEMPH_TX_DEEMPH_GEN2_3_5DB(24) | + PCS_DEEMPH_TX_DEEMPH_GEN2_6DB(34), + pcie->parf + PARF_PCS_DEEMPH); + writel(PCS_SWING_TX_SWING_FULL(120) | + PCS_SWING_TX_SWING_LOW(120), + pcie->parf + PARF_PCS_SWING); + writel(PHY_RX0_EQ(4), pcie->parf + PARF_CONFIG_BITS); + } + + if (of_device_is_compatible(node, "qcom,pcie-ipq8064")) { + /* set TX termination offset */ + val = readl(pcie->parf + PARF_PHY_CTRL); + val &= ~PHY_CTRL_PHY_TX0_TERM_OFFSET_MASK; + val |= PHY_CTRL_PHY_TX0_TERM_OFFSET(7); + writel(val, pcie->parf + PARF_PHY_CTRL); + } + + /* enable external reference clock */ + val = readl(pcie->parf + PARF_PHY_REFCLK); + /* USE_PAD is required only for ipq806x */ + if (!of_device_is_compatible(node, "qcom,pcie-apq8064")) + val &= ~PHY_REFCLK_USE_PAD; + val |= PHY_REFCLK_SSP_EN; + writel(val, pcie->parf + PARF_PHY_REFCLK); + + /* wait for clock acquisition */ + usleep_range(1000, 1500); + + /* Set the Max TLP size to 2K, instead of using default of 4K */ + writel(CFG_REMOTE_RD_REQ_BRIDGE_SIZE_2K, + pci->dbi_base + AXI_MSTR_RESP_COMP_CTRL0); + writel(CFG_BRIDGE_SB_INIT, + pci->dbi_base + AXI_MSTR_RESP_COMP_CTRL1); + + qcom_pcie_set_slot_cap(pcie->pci); + + return 0; +} + +static int qcom_pcie_get_resources_1_0_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_1_0_0 *res = &pcie->res.v1_0_0; + struct dw_pcie *pci = pcie->pci; + struct device *dev = pci->dev; + + res->vdda = devm_regulator_get(dev, "vdda"); + if (IS_ERR(res->vdda)) + return PTR_ERR(res->vdda); + + res->num_clks = devm_clk_bulk_get_all(dev, &res->clks); + if (res->num_clks < 0) { + dev_err(dev, "Failed to get clocks\n"); + return res->num_clks; + } + + res->core = devm_reset_control_get_exclusive(dev, "core"); + return PTR_ERR_OR_ZERO(res->core); +} + +static void qcom_pcie_deinit_1_0_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_1_0_0 *res = &pcie->res.v1_0_0; + + reset_control_assert(res->core); + clk_bulk_disable_unprepare(res->num_clks, res->clks); + regulator_disable(res->vdda); +} + +static int qcom_pcie_init_1_0_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_1_0_0 *res = &pcie->res.v1_0_0; + struct dw_pcie *pci = pcie->pci; + struct device *dev = pci->dev; + int ret; + + ret = reset_control_deassert(res->core); + if (ret) { + dev_err(dev, "cannot deassert core reset\n"); + return ret; + } + + ret = clk_bulk_prepare_enable(res->num_clks, res->clks); + if (ret) { + dev_err(dev, "cannot prepare/enable clocks\n"); + goto err_assert_reset; + } + + ret = regulator_enable(res->vdda); + if (ret) { + dev_err(dev, "cannot enable vdda regulator\n"); + goto err_disable_clks; + } + + return 0; + +err_disable_clks: + clk_bulk_disable_unprepare(res->num_clks, res->clks); +err_assert_reset: + reset_control_assert(res->core); + + return ret; +} + +static int qcom_pcie_post_init_1_0_0(struct qcom_pcie *pcie) +{ + qcom_pcie_configure_dbi_base(pcie); + + if (IS_ENABLED(CONFIG_PCI_MSI)) { + u32 val = readl(pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT); + + val |= EN; + writel(val, pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT); + } + + qcom_pcie_set_slot_cap(pcie->pci); + + return 0; +} + +static void qcom_pcie_2_3_2_ltssm_enable(struct qcom_pcie *pcie) +{ + u32 val; + + /* enable link training */ + val = readl(pcie->parf + PARF_LTSSM); + val |= LTSSM_EN; + writel(val, pcie->parf + PARF_LTSSM); +} + +static int qcom_pcie_get_resources_2_3_2(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_3_2 *res = &pcie->res.v2_3_2; + struct dw_pcie *pci = pcie->pci; + struct device *dev = pci->dev; + int ret; + + res->supplies[0].supply = "vdda"; + res->supplies[1].supply = "vddpe-3v3"; + ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(res->supplies), + res->supplies); + if (ret) + return ret; + + res->num_clks = devm_clk_bulk_get_all(dev, &res->clks); + if (res->num_clks < 0) { + dev_err(dev, "Failed to get clocks\n"); + return res->num_clks; + } + + return 0; +} + +static void qcom_pcie_deinit_2_3_2(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_3_2 *res = &pcie->res.v2_3_2; + u32 val; + + /* Force PHY to lowest power state*/ + val = readl(pcie->parf + PARF_PHY_CTRL); + val |= PHY_TEST_PWR_DOWN; + writel(val, pcie->parf + PARF_PHY_CTRL); + + clk_bulk_disable_unprepare(res->num_clks, res->clks); + regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies); +} + +static int qcom_pcie_init_2_3_2(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_3_2 *res = &pcie->res.v2_3_2; + struct dw_pcie *pci = pcie->pci; + struct device *dev = pci->dev; + int ret; + + ret = regulator_bulk_enable(ARRAY_SIZE(res->supplies), res->supplies); + if (ret < 0) { + dev_err(dev, "cannot enable regulators\n"); + return ret; + } + + ret = clk_bulk_prepare_enable(res->num_clks, res->clks); + if (ret) { + dev_err(dev, "cannot prepare/enable clocks\n"); + regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies); + return ret; + } + + return 0; +} + +static int qcom_pcie_post_init_2_3_2(struct qcom_pcie *pcie) +{ + u32 val; + + /* Force PHY out of lowest power state */ + val = readl(pcie->parf + PARF_PHY_CTRL); + val &= ~PHY_TEST_PWR_DOWN; + writel(val, pcie->parf + PARF_PHY_CTRL); + + qcom_pcie_configure_dbi_base(pcie); + + /* MAC PHY_POWERDOWN MUX DISABLE */ + val = readl(pcie->parf + PARF_SYS_CTRL); + val &= ~MAC_PHY_POWERDOWN_IN_P2_D_MUX_EN; + writel(val, pcie->parf + PARF_SYS_CTRL); + + val = readl(pcie->parf + PARF_MHI_CLOCK_RESET_CTRL); + val |= BYPASS; + writel(val, pcie->parf + PARF_MHI_CLOCK_RESET_CTRL); + + val = readl(pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2); + val |= EN; + writel(val, pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2); + + qcom_pcie_set_slot_cap(pcie->pci); + + return 0; +} + +static int qcom_pcie_get_resources_2_4_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_4_0 *res = &pcie->res.v2_4_0; + struct dw_pcie *pci = pcie->pci; + struct device *dev = pci->dev; + bool is_ipq = of_device_is_compatible(dev->of_node, "qcom,pcie-ipq4019"); + int ret; + + res->num_clks = devm_clk_bulk_get_all(dev, &res->clks); + if (res->num_clks < 0) { + dev_err(dev, "Failed to get clocks\n"); + return res->num_clks; + } + + res->resets[0].id = "axi_m"; + res->resets[1].id = "axi_s"; + res->resets[2].id = "axi_m_sticky"; + res->resets[3].id = "pipe_sticky"; + res->resets[4].id = "pwr"; + res->resets[5].id = "ahb"; + res->resets[6].id = "pipe"; + res->resets[7].id = "axi_m_vmid"; + res->resets[8].id = "axi_s_xpu"; + res->resets[9].id = "parf"; + res->resets[10].id = "phy"; + res->resets[11].id = "phy_ahb"; + + res->num_resets = is_ipq ? 12 : 6; + + ret = devm_reset_control_bulk_get_exclusive(dev, res->num_resets, res->resets); + if (ret < 0) + return ret; + + return 0; +} + +static void qcom_pcie_deinit_2_4_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_4_0 *res = &pcie->res.v2_4_0; + u32 val; + + /* Force PHY to lowest power state*/ + val = readl(pcie->parf + PARF_PHY_CTRL); + val |= PHY_TEST_PWR_DOWN; + writel(val, pcie->parf + PARF_PHY_CTRL); + + reset_control_bulk_assert(res->num_resets, res->resets); + clk_bulk_disable_unprepare(res->num_clks, res->clks); +} + +static int qcom_pcie_init_2_4_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_4_0 *res = &pcie->res.v2_4_0; + struct dw_pcie *pci = pcie->pci; + struct device *dev = pci->dev; + int ret; + + ret = reset_control_bulk_assert(res->num_resets, res->resets); + if (ret < 0) { + dev_err(dev, "cannot assert resets\n"); + return ret; + } + + usleep_range(10000, 12000); + + ret = reset_control_bulk_deassert(res->num_resets, res->resets); + if (ret < 0) { + dev_err(dev, "cannot deassert resets\n"); + return ret; + } + + usleep_range(10000, 12000); + + ret = clk_bulk_prepare_enable(res->num_clks, res->clks); + if (ret) { + reset_control_bulk_assert(res->num_resets, res->resets); + return ret; + } + + return 0; +} + +static int qcom_pcie_get_resources_2_3_3(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_3_3 *res = &pcie->res.v2_3_3; + struct dw_pcie *pci = pcie->pci; + struct device *dev = pci->dev; + int ret; + + res->num_clks = devm_clk_bulk_get_all(dev, &res->clks); + if (res->num_clks < 0) { + dev_err(dev, "Failed to get clocks\n"); + return res->num_clks; + } + + res->rst[0].id = "axi_m"; + res->rst[1].id = "axi_s"; + res->rst[2].id = "pipe"; + res->rst[3].id = "axi_m_sticky"; + res->rst[4].id = "sticky"; + res->rst[5].id = "ahb"; + res->rst[6].id = "sleep"; + + ret = devm_reset_control_bulk_get_exclusive(dev, ARRAY_SIZE(res->rst), res->rst); + if (ret < 0) + return ret; + + return 0; +} + +static void qcom_pcie_deinit_2_3_3(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_3_3 *res = &pcie->res.v2_3_3; + u32 val; + + /* Force PHY to lowest power state */ + val = readl(pcie->parf + PARF_PHY_CTRL); + val |= PHY_TEST_PWR_DOWN; + writel(val, pcie->parf + PARF_PHY_CTRL); + + clk_bulk_disable_unprepare(res->num_clks, res->clks); +} + +static int qcom_pcie_init_2_3_3(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_3_3 *res = &pcie->res.v2_3_3; + struct dw_pcie *pci = pcie->pci; + struct device *dev = pci->dev; + int ret; + + ret = reset_control_bulk_assert(ARRAY_SIZE(res->rst), res->rst); + if (ret < 0) { + dev_err(dev, "cannot assert resets\n"); + return ret; + } + + usleep_range(2000, 2500); + + ret = reset_control_bulk_deassert(ARRAY_SIZE(res->rst), res->rst); + if (ret < 0) { + dev_err(dev, "cannot deassert resets\n"); + return ret; + } + + /* + * Don't have a way to see if the reset has completed. + * Wait for some time. + */ + usleep_range(2000, 2500); + + ret = clk_bulk_prepare_enable(res->num_clks, res->clks); + if (ret) { + dev_err(dev, "cannot prepare/enable clocks\n"); + goto err_assert_resets; + } + + return 0; + +err_assert_resets: + /* + * Not checking for failure, will anyway return + * the original failure in 'ret'. + */ + reset_control_bulk_assert(ARRAY_SIZE(res->rst), res->rst); + + return ret; +} + +static int qcom_pcie_post_init_2_3_3(struct qcom_pcie *pcie) +{ + struct dw_pcie *pci = pcie->pci; + u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + u32 val; + + /* Force PHY out of lowest power state */ + val = readl(pcie->parf + PARF_PHY_CTRL); + val &= ~PHY_TEST_PWR_DOWN; + writel(val, pcie->parf + PARF_PHY_CTRL); + + qcom_pcie_configure_dbi_atu_base(pcie); + + writel(MST_WAKEUP_EN | SLV_WAKEUP_EN | MSTR_ACLK_CGC_DIS + | SLV_ACLK_CGC_DIS | CORE_CLK_CGC_DIS | + AUX_PWR_DET | L23_CLK_RMV_DIS | L1_CLK_RMV_DIS, + pcie->parf + PARF_SYS_CTRL); + writel(0, pcie->parf + PARF_Q2A_FLUSH); + + writel(PCI_COMMAND_MASTER, pci->dbi_base + PCI_COMMAND); + + dw_pcie_dbi_ro_wr_en(pci); + + writel(PCIE_CAP_SLOT_VAL, pci->dbi_base + offset + PCI_EXP_SLTCAP); + + val = readl(pci->dbi_base + offset + PCI_EXP_LNKCAP); + val &= ~PCI_EXP_LNKCAP_ASPMS; + writel(val, pci->dbi_base + offset + PCI_EXP_LNKCAP); + + writel(PCI_EXP_DEVCTL2_COMP_TMOUT_DIS, pci->dbi_base + offset + + PCI_EXP_DEVCTL2); + + dw_pcie_dbi_ro_wr_dis(pci); + + return 0; +} + +static int qcom_pcie_get_resources_2_7_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0; + struct dw_pcie *pci = pcie->pci; + struct device *dev = pci->dev; + int ret; + + res->rst = devm_reset_control_array_get_exclusive(dev); + if (IS_ERR(res->rst)) + return PTR_ERR(res->rst); + + res->supplies[0].supply = "vdda"; + res->supplies[1].supply = "vddpe-3v3"; + ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(res->supplies), + res->supplies); + if (ret) + return ret; + + res->num_clks = devm_clk_bulk_get_all(dev, &res->clks); + if (res->num_clks < 0) { + dev_err(dev, "Failed to get clocks\n"); + return res->num_clks; + } + + return 0; +} + +static int qcom_pcie_init_2_7_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0; + struct dw_pcie *pci = pcie->pci; + struct device *dev = pci->dev; + u32 val; + int ret; + + ret = regulator_bulk_enable(ARRAY_SIZE(res->supplies), res->supplies); + if (ret < 0) { + dev_err(dev, "cannot enable regulators\n"); + return ret; + } + + ret = clk_bulk_prepare_enable(res->num_clks, res->clks); + if (ret < 0) + goto err_disable_regulators; + + ret = reset_control_assert(res->rst); + if (ret) { + dev_err(dev, "reset assert failed (%d)\n", ret); + goto err_disable_clocks; + } + + usleep_range(1000, 1500); + + ret = reset_control_deassert(res->rst); + if (ret) { + dev_err(dev, "reset deassert failed (%d)\n", ret); + goto err_disable_clocks; + } + + /* Wait for reset to complete, required on SM8450 */ + usleep_range(1000, 1500); + + /* configure PCIe to RC mode */ + writel(DEVICE_TYPE_RC, pcie->parf + PARF_DEVICE_TYPE); + + /* Force PHY out of lowest power state */ + val = readl(pcie->parf + PARF_PHY_CTRL); + val &= ~PHY_TEST_PWR_DOWN; + writel(val, pcie->parf + PARF_PHY_CTRL); + + qcom_pcie_configure_dbi_atu_base(pcie); + + /* MAC PHY_POWERDOWN MUX DISABLE */ + val = readl(pcie->parf + PARF_SYS_CTRL); + val &= ~MAC_PHY_POWERDOWN_IN_P2_D_MUX_EN; + writel(val, pcie->parf + PARF_SYS_CTRL); + + val = readl(pcie->parf + PARF_MHI_CLOCK_RESET_CTRL); + val |= BYPASS; + writel(val, pcie->parf + PARF_MHI_CLOCK_RESET_CTRL); + + /* Enable L1 and L1SS */ + val = readl(pcie->parf + PARF_PM_CTRL); + val &= ~REQ_NOT_ENTR_L1; + writel(val, pcie->parf + PARF_PM_CTRL); + + pci->l1ss_support = true; + + val = readl(pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2); + val |= EN; + writel(val, pcie->parf + PARF_AXI_MSTR_WR_ADDR_HALT_V2); + + return 0; +err_disable_clocks: + clk_bulk_disable_unprepare(res->num_clks, res->clks); +err_disable_regulators: + regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies); + + return ret; +} + +static int qcom_pcie_post_init_2_7_0(struct qcom_pcie *pcie) +{ + const struct qcom_pcie_cfg *pcie_cfg = pcie->cfg; + + if (pcie_cfg->override_no_snoop) + writel(WR_NO_SNOOP_OVERRIDE_EN | RD_NO_SNOOP_OVERRIDE_EN, + pcie->parf + PARF_NO_SNOOP_OVERRIDE); + + qcom_pcie_set_slot_cap(pcie->pci); + + return 0; +} + +static int qcom_pcie_enable_aspm(struct pci_dev *pdev, void *userdata) +{ + /* + * Downstream devices need to be in D0 state before enabling PCI PM + * substates. + */ + pci_set_power_state_locked(pdev, PCI_D0); + pci_enable_link_state_locked(pdev, PCIE_LINK_STATE_ALL); + + return 0; +} + +static void qcom_pcie_host_post_init_2_7_0(struct qcom_pcie *pcie) +{ + struct dw_pcie_rp *pp = &pcie->pci->pp; + + pci_walk_bus(pp->bridge->bus, qcom_pcie_enable_aspm, NULL); +} + +static void qcom_pcie_deinit_2_7_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0; + u32 val; + + /* Force PHY to lowest power state */ + val = readl(pcie->parf + PARF_PHY_CTRL); + val |= PHY_TEST_PWR_DOWN; + writel(val, pcie->parf + PARF_PHY_CTRL); + + clk_bulk_disable_unprepare(res->num_clks, res->clks); + + regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies); +} + +static int qcom_pcie_config_sid_1_9_0(struct qcom_pcie *pcie) +{ + /* iommu map structure */ + struct { + u32 bdf; + u32 phandle; + u32 smmu_sid; + u32 smmu_sid_len; + } *map; + void __iomem *bdf_to_sid_base = pcie->parf + PARF_BDF_TO_SID_TABLE_N; + struct device *dev = pcie->pci->dev; + u8 qcom_pcie_crc8_table[CRC8_TABLE_SIZE]; + int i, nr_map, size = 0; + u32 smmu_sid_base; + u32 val; + + of_get_property(dev->of_node, "iommu-map", &size); + if (!size) + return 0; + + /* Enable BDF to SID translation by disabling bypass mode (default) */ + val = readl(pcie->parf + PARF_BDF_TO_SID_CFG); + val &= ~BDF_TO_SID_BYPASS; + writel(val, pcie->parf + PARF_BDF_TO_SID_CFG); + + map = kzalloc(size, GFP_KERNEL); + if (!map) + return -ENOMEM; + + of_property_read_u32_array(dev->of_node, "iommu-map", (u32 *)map, + size / sizeof(u32)); + + nr_map = size / (sizeof(*map)); + + crc8_populate_msb(qcom_pcie_crc8_table, QCOM_PCIE_CRC8_POLYNOMIAL); + + /* Registers need to be zero out first */ + memset_io(bdf_to_sid_base, 0, CRC8_TABLE_SIZE * sizeof(u32)); + + /* Extract the SMMU SID base from the first entry of iommu-map */ + smmu_sid_base = map[0].smmu_sid; + + /* Look for an available entry to hold the mapping */ + for (i = 0; i < nr_map; i++) { + __be16 bdf_be = cpu_to_be16(map[i].bdf); + u32 val; + u8 hash; + + hash = crc8(qcom_pcie_crc8_table, (u8 *)&bdf_be, sizeof(bdf_be), 0); + + val = readl(bdf_to_sid_base + hash * sizeof(u32)); + + /* If the register is already populated, look for next available entry */ + while (val) { + u8 current_hash = hash++; + u8 next_mask = 0xff; + + /* If NEXT field is NULL then update it with next hash */ + if (!(val & next_mask)) { + val |= (u32)hash; + writel(val, bdf_to_sid_base + current_hash * sizeof(u32)); + } + + val = readl(bdf_to_sid_base + hash * sizeof(u32)); + } + + /* BDF [31:16] | SID [15:8] | NEXT [7:0] */ + val = map[i].bdf << 16 | (map[i].smmu_sid - smmu_sid_base) << 8 | 0; + writel(val, bdf_to_sid_base + hash * sizeof(u32)); + } + + kfree(map); + + return 0; +} + +static int qcom_pcie_get_resources_2_9_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_9_0 *res = &pcie->res.v2_9_0; + struct dw_pcie *pci = pcie->pci; + struct device *dev = pci->dev; + + res->num_clks = devm_clk_bulk_get_all(dev, &res->clks); + if (res->num_clks < 0) { + dev_err(dev, "Failed to get clocks\n"); + return res->num_clks; + } + + res->rst = devm_reset_control_array_get_exclusive(dev); + if (IS_ERR(res->rst)) + return PTR_ERR(res->rst); + + return 0; +} + +static void qcom_pcie_deinit_2_9_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_9_0 *res = &pcie->res.v2_9_0; + u32 val; + + /* Force PHY to lowest power state */ + val = readl(pcie->parf + PARF_PHY_CTRL); + val |= PHY_TEST_PWR_DOWN; + writel(val, pcie->parf + PARF_PHY_CTRL); + + clk_bulk_disable_unprepare(res->num_clks, res->clks); +} + +static int qcom_pcie_init_2_9_0(struct qcom_pcie *pcie) +{ + struct qcom_pcie_resources_2_9_0 *res = &pcie->res.v2_9_0; + struct device *dev = pcie->pci->dev; + int ret; + + ret = reset_control_assert(res->rst); + if (ret) { + dev_err(dev, "reset assert failed (%d)\n", ret); + return ret; + } + + /* + * Delay periods before and after reset deassert are working values + * from downstream Codeaurora kernel + */ + usleep_range(2000, 2500); + + ret = reset_control_deassert(res->rst); + if (ret) { + dev_err(dev, "reset deassert failed (%d)\n", ret); + return ret; + } + + usleep_range(2000, 2500); + + return clk_bulk_prepare_enable(res->num_clks, res->clks); +} + +static int qcom_pcie_post_init_2_9_0(struct qcom_pcie *pcie) +{ + struct dw_pcie *pci = pcie->pci; + u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + u32 val; + int i; + + /* Force PHY out of lowest power state */ + val = readl(pcie->parf + PARF_PHY_CTRL); + val &= ~PHY_TEST_PWR_DOWN; + writel(val, pcie->parf + PARF_PHY_CTRL); + + qcom_pcie_configure_dbi_atu_base(pcie); + + writel(DEVICE_TYPE_RC, pcie->parf + PARF_DEVICE_TYPE); + writel(BYPASS | MSTR_AXI_CLK_EN | AHB_CLK_EN, + pcie->parf + PARF_MHI_CLOCK_RESET_CTRL); + writel(GEN3_RELATED_OFF_RXEQ_RGRDLESS_RXTS | + GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL, + pci->dbi_base + GEN3_RELATED_OFF); + + writel(MST_WAKEUP_EN | SLV_WAKEUP_EN | MSTR_ACLK_CGC_DIS | + SLV_ACLK_CGC_DIS | CORE_CLK_CGC_DIS | + AUX_PWR_DET | L23_CLK_RMV_DIS | L1_CLK_RMV_DIS, + pcie->parf + PARF_SYS_CTRL); + + writel(0, pcie->parf + PARF_Q2A_FLUSH); + + dw_pcie_dbi_ro_wr_en(pci); + + writel(PCIE_CAP_SLOT_VAL, pci->dbi_base + offset + PCI_EXP_SLTCAP); + + val = readl(pci->dbi_base + offset + PCI_EXP_LNKCAP); + val &= ~PCI_EXP_LNKCAP_ASPMS; + writel(val, pci->dbi_base + offset + PCI_EXP_LNKCAP); + + writel(PCI_EXP_DEVCTL2_COMP_TMOUT_DIS, pci->dbi_base + offset + + PCI_EXP_DEVCTL2); + + dw_pcie_dbi_ro_wr_dis(pci); + + for (i = 0; i < 256; i++) + writel(0, pcie->parf + PARF_BDF_TO_SID_TABLE_N + (4 * i)); + + return 0; +} + +static bool qcom_pcie_link_up(struct dw_pcie *pci) +{ + u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + u16 val = readw(pci->dbi_base + offset + PCI_EXP_LNKSTA); + + return val & PCI_EXP_LNKSTA_DLLLA; +} + +static enum dw_pcie_ltssm qcom_pcie_get_ltssm(struct dw_pcie *pci) +{ + struct qcom_pcie *pcie = to_qcom_pcie(pci); + u32 val; + + if (pcie->cfg->ops->get_ltssm) + return pcie->cfg->ops->get_ltssm(pcie); + + val = readl(pcie->parf + PARF_LTSSM); + + return (enum dw_pcie_ltssm)FIELD_GET(PARF_LTSSM_STATE_MASK, val); +} + +static void qcom_pcie_phy_power_off(struct qcom_pcie *pcie) +{ + struct qcom_pcie_port *port; + + list_for_each_entry(port, &pcie->ports, list) + phy_power_off(port->phy); +} + +static int qcom_pcie_phy_power_on(struct qcom_pcie *pcie) +{ + struct qcom_pcie_port *port; + int ret; + + list_for_each_entry(port, &pcie->ports, list) { + ret = phy_set_mode_ext(port->phy, PHY_MODE_PCIE, PHY_MODE_PCIE_RC); + if (ret) + return ret; + + ret = phy_power_on(port->phy); + if (ret) { + qcom_pcie_phy_power_off(pcie); + return ret; + } + } + + return 0; +} + +static void qcom_pcie_configure_ports(struct qcom_pcie *pcie) +{ + struct qcom_pcie_port *port; + + list_for_each_entry(port, &pcie->ports, list) + dw_pcie_program_t_power_on(pcie->pci, port->l1ss_t_power_on); +} + +static int qcom_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct qcom_pcie *pcie = to_qcom_pcie(pci); + int ret; + + qcom_pcie_perst_assert(pcie); + + ret = pcie->cfg->ops->init(pcie); + if (ret) + return ret; + + ret = qcom_pcie_phy_power_on(pcie); + if (ret) + goto err_deinit; + + if (!pci->suspended) { + ret = pci_pwrctrl_create_devices(pci->dev); + if (ret) + goto err_disable_phy; + } + + if (!pp->skip_pwrctrl_off) { + ret = pci_pwrctrl_power_on_devices(pci->dev); + if (ret) + goto err_pwrctrl_destroy; + } + + if (pcie->cfg->ops->post_init) { + ret = pcie->cfg->ops->post_init(pcie); + if (ret) + goto err_pwrctrl_power_off; + } + + qcom_pcie_clear_aspm_l0s(pcie->pci); + dw_pcie_remove_capability(pcie->pci, PCI_CAP_ID_MSIX); + dw_pcie_remove_ext_capability(pcie->pci, PCI_EXT_CAP_ID_DPC); + + qcom_pcie_configure_ports(pcie); + + qcom_pcie_perst_deassert(pcie); + + if (pcie->cfg->ops->config_sid) { + ret = pcie->cfg->ops->config_sid(pcie); + if (ret) + goto err_assert_reset; + } + + pp->bridge->reset_root_port = qcom_pcie_reset_root_port; + + return 0; + +err_assert_reset: + qcom_pcie_perst_assert(pcie); +err_pwrctrl_power_off: + if (!pp->skip_pwrctrl_off) + pci_pwrctrl_power_off_devices(pci->dev); +err_pwrctrl_destroy: + if (ret != -EPROBE_DEFER && !pci->suspended) + pci_pwrctrl_destroy_devices(pci->dev); +err_disable_phy: + qcom_pcie_phy_power_off(pcie); +err_deinit: + pcie->cfg->ops->deinit(pcie); + + return ret; +} + +static void qcom_pcie_host_deinit(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct qcom_pcie *pcie = to_qcom_pcie(pci); + + qcom_pcie_perst_assert(pcie); + + if (!pci->pp.skip_pwrctrl_off) { + /* + * No need to destroy pwrctrl devices as this function only + * gets called during system suspend as of now. + */ + pci_pwrctrl_power_off_devices(pci->dev); + } + + qcom_pcie_phy_power_off(pcie); + pcie->cfg->ops->deinit(pcie); +} + +static void qcom_pcie_host_post_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct qcom_pcie *pcie = to_qcom_pcie(pci); + + /* + * During system suspend, the Qcom RC driver may turn off the + * analog circuitry of PHY and remove controller votes to save + * power. If the link is in L1SS and the endpoint asserts CLKREQ# + * to exit L1SS, the time required to wake the system and restore + * the PHY/REFCLK may exceed the L1SS exit timing (L10_REFCLK_ON + + * T_COMMONMODE), resulting in Link Down (LDn) and a reset of the + * endpoint. Set this flag to indicate this limitation to client + * drivers so that they can avoid relying on device state being + * preserved during system suspend. + */ + pp->bridge->broken_l1ss_resume = true; + + if (pcie->cfg->ops->host_post_init) + pcie->cfg->ops->host_post_init(pcie); +} + +static void qcom_pcie_host_pme_turn_off(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + + writel(ELBI_SYS_CTRL_PME_TURNOFF_MSG, pci->elbi_base + ELBI_SYS_CTRL); +} + +static const struct dw_pcie_host_ops qcom_pcie_dw_ops = { + .init = qcom_pcie_host_init, + .deinit = qcom_pcie_host_deinit, + .post_init = qcom_pcie_host_post_init, + .pme_turn_off = qcom_pcie_host_pme_turn_off, +}; + +/* Qcom IP rev.: 2.1.0 Synopsys IP rev.: 4.01a */ +static const struct qcom_pcie_ops ops_2_1_0 = { + .get_resources = qcom_pcie_get_resources_2_1_0, + .init = qcom_pcie_init_2_1_0, + .post_init = qcom_pcie_post_init_2_1_0, + .deinit = qcom_pcie_deinit_2_1_0, + .ltssm_enable = qcom_pcie_2_1_0_ltssm_enable, + .get_ltssm = qcom_pcie_2_1_0_get_ltssm, +}; + +/* Qcom IP rev.: 1.0.0 Synopsys IP rev.: 4.11a */ +static const struct qcom_pcie_ops ops_1_0_0 = { + .get_resources = qcom_pcie_get_resources_1_0_0, + .init = qcom_pcie_init_1_0_0, + .post_init = qcom_pcie_post_init_1_0_0, + .deinit = qcom_pcie_deinit_1_0_0, + .ltssm_enable = qcom_pcie_2_1_0_ltssm_enable, + .get_ltssm = qcom_pcie_2_1_0_get_ltssm, +}; + +/* Qcom IP rev.: 2.3.2 Synopsys IP rev.: 4.21a */ +static const struct qcom_pcie_ops ops_2_3_2 = { + .get_resources = qcom_pcie_get_resources_2_3_2, + .init = qcom_pcie_init_2_3_2, + .post_init = qcom_pcie_post_init_2_3_2, + .deinit = qcom_pcie_deinit_2_3_2, + .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable, +}; + +/* Qcom IP rev.: 2.4.0 Synopsys IP rev.: 4.20a */ +static const struct qcom_pcie_ops ops_2_4_0 = { + .get_resources = qcom_pcie_get_resources_2_4_0, + .init = qcom_pcie_init_2_4_0, + .post_init = qcom_pcie_post_init_2_3_2, + .deinit = qcom_pcie_deinit_2_4_0, + .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable, +}; + +/* Qcom IP rev.: 2.3.3 Synopsys IP rev.: 4.30a */ +static const struct qcom_pcie_ops ops_2_3_3 = { + .get_resources = qcom_pcie_get_resources_2_3_3, + .init = qcom_pcie_init_2_3_3, + .post_init = qcom_pcie_post_init_2_3_3, + .deinit = qcom_pcie_deinit_2_3_3, + .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable, +}; + +/* Qcom IP rev.: 2.7.0 Synopsys IP rev.: 4.30a */ +static const struct qcom_pcie_ops ops_2_7_0 = { + .get_resources = qcom_pcie_get_resources_2_7_0, + .init = qcom_pcie_init_2_7_0, + .post_init = qcom_pcie_post_init_2_7_0, + .deinit = qcom_pcie_deinit_2_7_0, + .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable, +}; + +/* Qcom IP rev.: 1.9.0 */ +static const struct qcom_pcie_ops ops_1_9_0 = { + .get_resources = qcom_pcie_get_resources_2_7_0, + .init = qcom_pcie_init_2_7_0, + .post_init = qcom_pcie_post_init_2_7_0, + .host_post_init = qcom_pcie_host_post_init_2_7_0, + .deinit = qcom_pcie_deinit_2_7_0, + .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable, + .config_sid = qcom_pcie_config_sid_1_9_0, +}; + +/* Qcom IP rev.: 1.21.0 Synopsys IP rev.: 5.60a */ +static const struct qcom_pcie_ops ops_1_21_0 = { + .get_resources = qcom_pcie_get_resources_2_7_0, + .init = qcom_pcie_init_2_7_0, + .post_init = qcom_pcie_post_init_2_7_0, + .host_post_init = qcom_pcie_host_post_init_2_7_0, + .deinit = qcom_pcie_deinit_2_7_0, + .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable, +}; + +/* Qcom IP rev.: 2.9.0 Synopsys IP rev.: 5.00a */ +static const struct qcom_pcie_ops ops_2_9_0 = { + .get_resources = qcom_pcie_get_resources_2_9_0, + .init = qcom_pcie_init_2_9_0, + .post_init = qcom_pcie_post_init_2_9_0, + .deinit = qcom_pcie_deinit_2_9_0, + .ltssm_enable = qcom_pcie_2_3_2_ltssm_enable, +}; + +static const struct qcom_pcie_cfg cfg_1_0_0 = { + .ops = &ops_1_0_0, +}; + +static const struct qcom_pcie_cfg cfg_1_9_0 = { + .ops = &ops_1_9_0, +}; + +static const struct qcom_pcie_cfg cfg_1_34_0 = { + .ops = &ops_1_9_0, + .override_no_snoop = true, + .no_l0s = true, +}; + +static const struct qcom_pcie_cfg cfg_2_1_0 = { + .ops = &ops_2_1_0, +}; + +static const struct qcom_pcie_cfg cfg_2_3_2 = { + .ops = &ops_2_3_2, + .no_l0s = true, +}; + +static const struct qcom_pcie_cfg cfg_2_3_3 = { + .ops = &ops_2_3_3, +}; + +static const struct qcom_pcie_cfg cfg_2_4_0 = { + .ops = &ops_2_4_0, +}; + +static const struct qcom_pcie_cfg cfg_2_7_0 = { + .ops = &ops_2_7_0, +}; + +static const struct qcom_pcie_cfg cfg_2_9_0 = { + .ops = &ops_2_9_0, +}; + +static const struct qcom_pcie_cfg cfg_sc8280xp = { + .ops = &ops_1_21_0, + .no_l0s = true, +}; + +static const struct qcom_pcie_cfg cfg_fw_managed = { + .firmware_managed = true, +}; + +static const struct dw_pcie_ops dw_pcie_ops = { + .link_up = qcom_pcie_link_up, + .start_link = qcom_pcie_start_link, + .get_ltssm = qcom_pcie_get_ltssm, +}; + +static int qcom_pcie_icc_init(struct qcom_pcie *pcie) +{ + struct dw_pcie *pci = pcie->pci; + int ret; + + pcie->icc_mem = devm_of_icc_get(pci->dev, "pcie-mem"); + if (IS_ERR(pcie->icc_mem)) + return PTR_ERR(pcie->icc_mem); + + pcie->icc_cpu = devm_of_icc_get(pci->dev, "cpu-pcie"); + if (IS_ERR(pcie->icc_cpu)) + return PTR_ERR(pcie->icc_cpu); + /* + * Some Qualcomm platforms require interconnect bandwidth constraints + * to be set before enabling interconnect clocks. + * + * Set an initial peak bandwidth corresponding to single-lane Gen 1 + * for the pcie-mem path. + */ + ret = icc_set_bw(pcie->icc_mem, 0, QCOM_PCIE_LINK_SPEED_TO_BW(1)); + if (ret) { + dev_err(pci->dev, "Failed to set bandwidth for PCIe-MEM interconnect path: %d\n", + ret); + return ret; + } + + /* + * Since the CPU-PCIe path is only used for activities like register + * access of the host controller and endpoint Config/BAR space access, + * HW team has recommended to use a minimal bandwidth of 1KBps just to + * keep the path active. + */ + ret = icc_set_bw(pcie->icc_cpu, 0, kBps_to_icc(1)); + if (ret) { + dev_err(pci->dev, "Failed to set bandwidth for CPU-PCIe interconnect path: %d\n", + ret); + icc_set_bw(pcie->icc_mem, 0, 0); + return ret; + } + + return 0; +} + +static void qcom_pcie_icc_opp_update(struct qcom_pcie *pcie) +{ + u32 offset, status, width, speed; + struct dw_pcie *pci = pcie->pci; + struct dev_pm_opp_key key = {}; + unsigned long freq_kbps; + struct dev_pm_opp *opp; + int ret, freq_mbps; + + offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + status = readw(pci->dbi_base + offset + PCI_EXP_LNKSTA); + + /* Only update constraints if link is up. */ + if (!(status & PCI_EXP_LNKSTA_DLLLA)) + return; + + speed = FIELD_GET(PCI_EXP_LNKSTA_CLS, status); + width = FIELD_GET(PCI_EXP_LNKSTA_NLW, status); + + if (pcie->icc_mem) { + ret = icc_set_bw(pcie->icc_mem, 0, + width * QCOM_PCIE_LINK_SPEED_TO_BW(speed)); + if (ret) { + dev_err(pci->dev, "Failed to set bandwidth for PCIe-MEM interconnect path: %d\n", + ret); + } + } else if (pcie->use_pm_opp) { + freq_mbps = pcie_dev_speed_mbps(pcie_get_link_speed(speed)); + if (freq_mbps < 0) + return; + + freq_kbps = freq_mbps * KILO; + opp = dev_pm_opp_find_level_exact(pci->dev, speed); + if (IS_ERR(opp)) { + /* opp-level is not defined use only frequency */ + opp = dev_pm_opp_find_freq_exact(pci->dev, freq_kbps * width, + true); + } else { + /* put opp-level OPP */ + dev_pm_opp_put(opp); + + key.freq = freq_kbps * width; + key.level = speed; + key.bw = 0; + opp = dev_pm_opp_find_key_exact(pci->dev, &key, true); + } + if (!IS_ERR(opp)) { + ret = dev_pm_opp_set_opp(pci->dev, opp); + if (ret) + dev_err(pci->dev, "Failed to set OPP for freq (%lu): %d\n", + freq_kbps * width, ret); + dev_pm_opp_put(opp); + } + } +} + +static int qcom_pcie_set_max_opp(struct device *dev) +{ + unsigned long max_freq = ULONG_MAX; + struct dev_pm_opp *opp; + int ret; + + opp = dev_pm_opp_find_freq_floor(dev, &max_freq); + if (IS_ERR(opp)) + return PTR_ERR(opp); + + ret = dev_pm_opp_set_opp(dev, opp); + dev_pm_opp_put(opp); + + return ret; +} + +/* + * Qcom PCIe controllers only support one Root Port per controller instance. So + * this function ignores the 'pci_dev' associated with the Root Port and just + * resets the host bridge, which in turn resets the Root Port also. + */ +static int qcom_pcie_reset_root_port(struct pci_host_bridge *bridge, + struct pci_dev *pdev) +{ + struct device *dev = bridge->dev.parent; + struct qcom_pcie *pcie = dev_get_drvdata(dev); + struct dw_pcie *pci = pcie->pci; + struct dw_pcie_rp *pp = &pci->pp; + u32 val; + int ret; + + /* Wait for the pending transactions to be completed */ + ret = readl_relaxed_poll_timeout(pcie->parf + PARF_STATUS, val, + val & FLUSH_COMPLETED, 10, + FLUSH_TIMEOUT_US); + if (ret) { + dev_err(dev, "Flush completion failed: %d\n", ret); + return ret; + } + + /* Clear the FLUSH_MODE to allow the core to be reset */ + val = readl(pcie->parf + PARF_LTSSM); + val |= SW_CLEAR_FLUSH_MODE; + writel(val, pcie->parf + PARF_LTSSM); + + /* Wait for the FLUSH_MODE to clear */ + ret = readl_relaxed_poll_timeout(pcie->parf + PARF_LTSSM, val, + !(val & FLUSH_MODE), 10, + FLUSH_TIMEOUT_US); + if (ret) { + dev_err(dev, "Flush mode clear failed: %d\n", ret); + return ret; + } + + qcom_pcie_host_deinit(pp); + + ret = qcom_pcie_host_init(pp); + if (ret) { + dev_err(dev, "Host init failed\n"); + return ret; + } + + ret = dw_pcie_setup_rc(pp); + if (ret) + return ret; + + /* + * Re-enable global IRQ events as the PARF_INT_ALL_MASK register is + * non-sticky. + */ + if (pcie->global_irq) + writel_relaxed(PARF_INT_ALL_LINK_DOWN | PARF_INT_MSI_DEV_0_7, + pcie->parf + PARF_INT_ALL_MASK); + + qcom_pcie_start_link(pci); + + ret = dw_pcie_wait_for_link(pci); + if (ret) + return ret; + + dev_dbg(dev, "Root Port reset completed\n"); + + return 0; +} + +static int qcom_pcie_link_transition_count(struct seq_file *s, void *data) +{ + struct qcom_pcie *pcie = (struct qcom_pcie *)dev_get_drvdata(s->private); + + seq_printf(s, "L0s transition count: %u\n", + readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_PM_LINKST_IN_L0S)); + + seq_printf(s, "L1 transition count: %u\n", + readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_PM_LINKST_IN_L1)); + + seq_printf(s, "L1.1 transition count: %u\n", + readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L1)); + + seq_printf(s, "L1.2 transition count: %u\n", + readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_AUX_CLK_IN_L1SUB_L2)); + + seq_printf(s, "L2 transition count: %u\n", + readl_relaxed(pcie->mhi + PARF_DEBUG_CNT_PM_LINKST_IN_L2)); + + return 0; +} + +static void qcom_pcie_init_debugfs(struct qcom_pcie *pcie) +{ + struct dw_pcie *pci = pcie->pci; + struct device *dev = pci->dev; + char *name; + + name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node); + if (!name) + return; + + pcie->debugfs = debugfs_create_dir(name, NULL); + debugfs_create_devm_seqfile(dev, "link_transition_count", pcie->debugfs, + qcom_pcie_link_transition_count); +} + +static irqreturn_t qcom_pcie_global_irq_thread(int irq, void *data) +{ + struct qcom_pcie *pcie = data; + struct dw_pcie_rp *pp = &pcie->pci->pp; + struct device *dev = pcie->pci->dev; + struct pci_dev *port; + unsigned long status = readl_relaxed(pcie->parf + PARF_INT_ALL_STATUS); + + writel_relaxed(status, pcie->parf + PARF_INT_ALL_CLEAR); + + if (test_and_clear_bit(INT_ALL_LINK_DOWN, &status)) { + dev_dbg(dev, "Received Link down event\n"); + for_each_pci_bridge(port, pp->bridge->bus) { + if (pci_pcie_type(port) == PCI_EXP_TYPE_ROOT_PORT) + pci_host_handle_link_down(port); + } + } + + return IRQ_HANDLED; +} + +static void qcom_pci_free_msi(void *ptr) +{ + struct dw_pcie_rp *pp = (struct dw_pcie_rp *)ptr; + + if (pp && pp->use_imsi_rx) + dw_pcie_free_msi(pp); +} + +static int qcom_pcie_ecam_host_init(struct pci_config_window *cfg) +{ + struct device *dev = cfg->parent; + struct dw_pcie_rp *pp; + struct dw_pcie *pci; + int ret; + + pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); + if (!pci) + return -ENOMEM; + + pci->dev = dev; + pp = &pci->pp; + pci->dbi_base = cfg->win; + pp->num_vectors = MSI_DEF_NUM_VECTORS; + + /* + * dw_pcie_msi_host_init() is called directly here, bypassing + * dw_pcie_host_init() where pp->lock is normally initialized. + */ + raw_spin_lock_init(&pp->lock); + + ret = dw_pcie_msi_host_init(pp); + if (ret) + return ret; + + pp->use_imsi_rx = true; + dw_pcie_msi_init(pp); + + return devm_add_action_or_reset(dev, qcom_pci_free_msi, pp); +} + +static const struct pci_ecam_ops pci_qcom_ecam_ops = { + .init = qcom_pcie_ecam_host_init, + .pci_ops = { + .map_bus = pci_ecam_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, + } +}; + +/* Check if @node is a child of @dev in DT */ +static bool qcom_pcie_is_child_node(struct device *dev, + struct device_node *node) +{ + struct device_node *parent; + + for (parent = of_get_parent(node); parent; + parent = of_get_next_parent(parent)) { + if (parent == dev->of_node) { + of_node_put(parent); + return true; + } + } + + return false; +} + +/* Parse PERST# from all nodes in depth first manner starting from @np */ +static int qcom_pcie_parse_perst(struct qcom_pcie *pcie, + struct qcom_pcie_port *port, + struct device_node *np) +{ + struct device *dev = pcie->pci->dev; + struct qcom_pcie_perst *perst; + struct device_node *gpio_np; + struct gpio_desc *reset; + int ret; + + if (pcie->reset) { + dev_warn_once(dev, + "Reusing PERST# from Root Complex node. DT needs to be fixed!\n"); + reset = pcie->reset; + goto skip_perst_parsing; + } + + if (!of_find_property(np, "reset-gpios", NULL)) + goto parse_child_node; + + /* + * Skip GPIOs provided by a PCIe device which is a child of the Root + * Complex (e.g., a PCIe switch with GPIO controller capability). Such + * controllers won't be available at RC probe time and their PERST# + * should be controlled by the respective PCI client driver + * implementation. + */ + gpio_np = of_parse_phandle(np, "reset-gpios", 0); + if (!gpio_np) { + dev_err(dev, "Failed to parse GPIO provider\n"); + return -EINVAL; + } + + if (qcom_pcie_is_child_node(dev, gpio_np)) { + of_node_put(gpio_np); + goto parse_child_node; + } + of_node_put(gpio_np); + + reset = devm_fwnode_gpiod_get(dev, of_fwnode_handle(np), "reset", + GPIOD_OUT_HIGH, "PERST#"); + if (IS_ERR(reset)) { + /* + * FIXME: GPIOLIB currently supports exclusive GPIO access only. + * Non exclusive access is broken. But shared PERST# requires + * non-exclusive access. So once GPIOLIB properly supports it, + * implement it here. + */ + if (PTR_ERR(reset) == -EBUSY) + dev_err(dev, "Shared PERST# is not supported\n"); + + return PTR_ERR(reset); + } + +skip_perst_parsing: + perst = devm_kzalloc(dev, sizeof(*perst), GFP_KERNEL); + if (!perst) + return -ENOMEM; + + INIT_LIST_HEAD(&perst->list); + perst->desc = reset; + list_add_tail(&perst->list, &port->perst); + +parse_child_node: + for_each_available_child_of_node_scoped(np, child) { + ret = qcom_pcie_parse_perst(pcie, port, child); + if (ret) + return ret; + } + + return 0; +} + +static int qcom_pcie_parse_port(struct qcom_pcie *pcie, struct device_node *node) +{ + struct device *dev = pcie->pci->dev; + struct qcom_pcie_port *port; + struct phy *phy; + int ret; + + phy = devm_of_phy_get(dev, node, NULL); + if (IS_ERR(phy)) + return PTR_ERR(phy); + + port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); + if (!port) + return -ENOMEM; + + ret = phy_init(phy); + if (ret) + return ret; + + INIT_LIST_HEAD(&port->perst); + + ret = qcom_pcie_parse_perst(pcie, port, node); + if (ret) + return ret; + + /* TODO: Move to DWC core after multi Root Port support is added */ + of_property_read_u32(node, "t-power-on-us", &port->l1ss_t_power_on); + + port->phy = phy; + INIT_LIST_HEAD(&port->list); + list_add_tail(&port->list, &pcie->ports); + + return 0; +} + +static int qcom_pcie_parse_ports(struct qcom_pcie *pcie) +{ + struct qcom_pcie_perst *perst, *tmp_perst; + struct qcom_pcie_port *port, *tmp_port; + struct device *dev = pcie->pci->dev; + int ret = -ENODEV; + + if (of_find_property(dev->of_node, "perst-gpios", NULL)) { + pcie->reset = devm_gpiod_get_optional(dev, "perst", + GPIOD_OUT_HIGH); + if (IS_ERR(pcie->reset)) + return PTR_ERR(pcie->reset); + } + + for_each_available_child_of_node_scoped(dev->of_node, of_port) { + if (!of_node_is_type(of_port, "pci")) + continue; + ret = qcom_pcie_parse_port(pcie, of_port); + if (ret) + goto err_port_del; + } + + return ret; + +err_port_del: + list_for_each_entry_safe(port, tmp_port, &pcie->ports, list) { + list_for_each_entry_safe(perst, tmp_perst, &port->perst, list) + list_del(&perst->list); + phy_exit(port->phy); + list_del(&port->list); + } + + return ret; +} + +static int qcom_pcie_parse_legacy_binding(struct qcom_pcie *pcie) +{ + struct device *dev = pcie->pci->dev; + struct qcom_pcie_perst *perst; + struct qcom_pcie_port *port; + struct phy *phy; + int ret; + + phy = devm_phy_optional_get(dev, "pciephy"); + if (IS_ERR(phy)) + return PTR_ERR(phy); + + ret = phy_init(phy); + if (ret) + return ret; + + port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); + if (!port) + return -ENOMEM; + + perst = devm_kzalloc(dev, sizeof(*perst), GFP_KERNEL); + if (!perst) + return -ENOMEM; + + port->phy = phy; + INIT_LIST_HEAD(&port->list); + list_add_tail(&port->list, &pcie->ports); + + perst->desc = pcie->reset; + INIT_LIST_HEAD(&port->perst); + INIT_LIST_HEAD(&perst->list); + list_add_tail(&perst->list, &port->perst); + + return 0; +} + +static int qcom_pcie_probe(struct platform_device *pdev) +{ + struct qcom_pcie_perst *perst, *tmp_perst; + struct qcom_pcie_port *port, *tmp_port; + const struct qcom_pcie_cfg *pcie_cfg; + struct device *dev = &pdev->dev; + struct qcom_pcie *pcie; + struct dw_pcie_rp *pp; + struct resource *res; + struct dw_pcie *pci; + int ret, irq; + + pcie_cfg = of_device_get_match_data(dev); + if (!pcie_cfg) { + dev_err(dev, "No platform data\n"); + return -ENODATA; + } + + if (!pcie_cfg->firmware_managed && !pcie_cfg->ops) { + dev_err(dev, "No platform ops\n"); + return -ENODATA; + } + + pm_runtime_enable(dev); + ret = pm_runtime_get_sync(dev); + if (ret < 0) + goto err_pm_runtime_put; + + if (pcie_cfg->firmware_managed) { + struct pci_host_bridge *bridge; + struct pci_config_window *cfg; + + bridge = devm_pci_alloc_host_bridge(dev, 0); + if (!bridge) { + ret = -ENOMEM; + goto err_pm_runtime_put; + } + + /* Parse and map our ECAM configuration space area */ + cfg = pci_host_common_ecam_create(dev, bridge, + &pci_qcom_ecam_ops); + if (IS_ERR(cfg)) { + ret = PTR_ERR(cfg); + goto err_pm_runtime_put; + } + + bridge->sysdata = cfg; + bridge->ops = (struct pci_ops *)&pci_qcom_ecam_ops.pci_ops; + bridge->msi_domain = true; + + ret = pci_host_probe(bridge); + if (ret) + goto err_pm_runtime_put; + + return 0; + } + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) { + ret = -ENOMEM; + goto err_pm_runtime_put; + } + + pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); + if (!pci) { + ret = -ENOMEM; + goto err_pm_runtime_put; + } + + INIT_LIST_HEAD(&pcie->ports); + + pci->dev = dev; + pci->ops = &dw_pcie_ops; + pp = &pci->pp; + + pcie->pci = pci; + + pcie->cfg = pcie_cfg; + + pcie->parf = devm_platform_ioremap_resource_byname(pdev, "parf"); + if (IS_ERR(pcie->parf)) { + ret = PTR_ERR(pcie->parf); + goto err_pm_runtime_put; + } + + /* MHI region is optional */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mhi"); + if (res) { + pcie->mhi = devm_ioremap_resource(dev, res); + if (IS_ERR(pcie->mhi)) { + ret = PTR_ERR(pcie->mhi); + goto err_pm_runtime_put; + } + } + + /* OPP table is optional */ + ret = devm_pm_opp_of_add_table(dev); + if (ret && ret != -ENODEV) { + dev_err_probe(dev, ret, "Failed to add OPP table\n"); + goto err_pm_runtime_put; + } + + /* + * Before the PCIe link is initialized, vote for highest OPP in the OPP + * table, so that we are voting for maximum voltage corner for the + * link to come up in maximum supported speed. At the end of the + * probe(), OPP will be updated using qcom_pcie_icc_opp_update(). + */ + if (!ret) { + ret = qcom_pcie_set_max_opp(dev); + if (ret) { + dev_err_probe(dev, ret, "Failed to set max OPP\n"); + goto err_pm_runtime_put; + } + + pcie->use_pm_opp = true; + } else { + /* Skip ICC init if OPP is supported as it is handled by OPP */ + ret = qcom_pcie_icc_init(pcie); + if (ret) + goto err_pm_runtime_put; + } + + ret = pcie->cfg->ops->get_resources(pcie); + if (ret) + goto err_pm_runtime_put; + + pp->ops = &qcom_pcie_dw_ops; + + ret = qcom_pcie_parse_ports(pcie); + if (ret) { + if (ret != -ENODEV) { + dev_err_probe(pci->dev, ret, + "Failed to parse Root Port: %d\n", ret); + goto err_pm_runtime_put; + } + + /* + * In the case of properties not populated in Root Port node, + * fallback to the legacy method of parsing the Host Bridge + * node. This is to maintain DT backwards compatibility. + */ + ret = qcom_pcie_parse_legacy_binding(pcie); + if (ret) + goto err_pm_runtime_put; + } + + platform_set_drvdata(pdev, pcie); + + ret = dw_pcie_host_init(pp); + if (ret) { + dev_err_probe(dev, ret, "cannot initialize host\n"); + goto err_phy_exit; + } + + irq = platform_get_irq_byname_optional(pdev, "global"); + if (irq > 0) { + const char *name; + + name = devm_kasprintf(dev, GFP_KERNEL, "qcom_pcie_global_irq%d", + pci_domain_nr(pp->bridge->bus)); + if (!name) { + ret = -ENOMEM; + goto err_host_deinit; + } + + ret = devm_request_threaded_irq(&pdev->dev, irq, NULL, + qcom_pcie_global_irq_thread, + IRQF_ONESHOT, name, pcie); + if (ret) { + dev_err_probe(&pdev->dev, ret, + "Failed to request Global IRQ\n"); + goto err_host_deinit; + } + + writel_relaxed(PARF_INT_ALL_LINK_DOWN | PARF_INT_MSI_DEV_0_7, + pcie->parf + PARF_INT_ALL_MASK); + + pcie->global_irq = irq; + } + + qcom_pcie_icc_opp_update(pcie); + + if (pcie->mhi) + qcom_pcie_init_debugfs(pcie); + + return 0; + +err_host_deinit: + dw_pcie_host_deinit(pp); +err_phy_exit: + list_for_each_entry_safe(port, tmp_port, &pcie->ports, list) { + list_for_each_entry_safe(perst, tmp_perst, &port->perst, list) + list_del(&perst->list); + phy_exit(port->phy); + list_del(&port->list); + } +err_pm_runtime_put: + pm_runtime_put(dev); + pm_runtime_disable(dev); + + return ret; +} + +static int qcom_pcie_suspend_noirq(struct device *dev) +{ + struct qcom_pcie *pcie; + int ret = 0; + + pcie = dev_get_drvdata(dev); + if (!pcie) + return 0; + + ret = dw_pcie_suspend_noirq(pcie->pci); + if (ret) + return ret; + + if (pcie->pci->suspended) { + ret = icc_disable(pcie->icc_mem); + if (ret) + dev_err(dev, "Failed to disable PCIe-MEM interconnect path: %d\n", ret); + + ret = icc_disable(pcie->icc_cpu); + if (ret) + dev_err(dev, "Failed to disable CPU-PCIe interconnect path: %d\n", ret); + + if (pcie->use_pm_opp) + dev_pm_opp_set_opp(pcie->pci->dev, NULL); + } else { + /* + * Set minimum bandwidth required to keep data path + * functional during suspend. + */ + if (pcie->icc_mem) { + ret = icc_set_bw(pcie->icc_mem, 0, kBps_to_icc(1)); + if (ret) { + dev_err(dev, + "Failed to set bandwidth for PCIe-MEM interconnect path: %d\n", + ret); + return ret; + } + } + + /* + * Only disable CPU-PCIe interconnect path if the suspend + * is non-S2RAM. On some platforms, DBI access can happen + * very late during S2RAM and a non-active CPU-PCIe + * interconnect path may lead to NoC error. + */ + if (pm_suspend_target_state != PM_SUSPEND_MEM) { + ret = icc_disable(pcie->icc_cpu); + if (ret) + dev_err(dev, "Failed to disable CPU-PCIe interconnect path: %d\n", + ret); + + if (pcie->use_pm_opp) + dev_pm_opp_set_opp(pcie->pci->dev, NULL); + } + } + return ret; +} + +static int qcom_pcie_resume_noirq(struct device *dev) +{ + struct qcom_pcie *pcie; + int ret; + + pcie = dev_get_drvdata(dev); + if (!pcie) + return 0; + + if (pcie->pci->suspended) { + if (pcie->use_pm_opp) { + ret = qcom_pcie_set_max_opp(dev); + if (ret) { + dev_err(dev, "Failed to set max OPP: %d\n", ret); + return ret; + } + } + + ret = icc_enable(pcie->icc_cpu); + if (ret) { + dev_err(dev, "Failed to enable CPU-PCIe interconnect path: %d\n", ret); + return ret; + } + + ret = icc_enable(pcie->icc_mem); + if (ret) { + dev_err(dev, "Failed to enable PCIe-MEM interconnect path: %d\n", ret); + goto disable_icc_cpu; + } + + /* + * Ignore -ENODEV & -EIO here since it is expected when no + * endpoint is connected to the PCIe link. + */ + ret = dw_pcie_resume_noirq(pcie->pci); + if (ret && ret != -ENODEV && ret != -EIO) + goto disable_icc_mem; + } else { + if (pm_suspend_target_state != PM_SUSPEND_MEM) { + if (pcie->use_pm_opp) { + ret = qcom_pcie_set_max_opp(dev); + if (ret) { + dev_err(dev, "Failed to set max OPP: %d\n", ret); + return ret; + } + } + + ret = icc_enable(pcie->icc_cpu); + if (ret) { + dev_err(dev, "Failed to enable CPU-PCIe interconnect path: %d\n", + ret); + return ret; + } + } + } + + qcom_pcie_icc_opp_update(pcie); + + return 0; +disable_icc_mem: + icc_disable(pcie->icc_mem); +disable_icc_cpu: + icc_disable(pcie->icc_cpu); + + return ret; +} + +static const struct of_device_id qcom_pcie_match[] = { + { .compatible = "qcom,hawi-pcie", .data = &cfg_1_9_0 }, + { .compatible = "qcom,pcie-apq8064", .data = &cfg_2_1_0 }, + { .compatible = "qcom,pcie-apq8084", .data = &cfg_1_0_0 }, + { .compatible = "qcom,pcie-ipq4019", .data = &cfg_2_4_0 }, + { .compatible = "qcom,pcie-ipq5018", .data = &cfg_2_9_0 }, + { .compatible = "qcom,pcie-ipq6018", .data = &cfg_2_9_0 }, + { .compatible = "qcom,pcie-ipq8064", .data = &cfg_2_1_0 }, + { .compatible = "qcom,pcie-ipq8064-v2", .data = &cfg_2_1_0 }, + { .compatible = "qcom,pcie-ipq8074", .data = &cfg_2_3_3 }, + { .compatible = "qcom,pcie-ipq8074-gen3", .data = &cfg_2_9_0 }, + { .compatible = "qcom,pcie-ipq9574", .data = &cfg_2_9_0 }, + { .compatible = "qcom,pcie-msm8996", .data = &cfg_2_3_2 }, + { .compatible = "qcom,pcie-qcs404", .data = &cfg_2_4_0 }, + { .compatible = "qcom,pcie-sa8255p", .data = &cfg_fw_managed }, + { .compatible = "qcom,pcie-sa8540p", .data = &cfg_sc8280xp }, + { .compatible = "qcom,pcie-sa8775p", .data = &cfg_1_34_0}, + { .compatible = "qcom,pcie-sc7280", .data = &cfg_1_9_0 }, + { .compatible = "qcom,pcie-sc8180x", .data = &cfg_1_9_0 }, + { .compatible = "qcom,pcie-sc8280xp", .data = &cfg_sc8280xp }, + { .compatible = "qcom,pcie-sdm845", .data = &cfg_2_7_0 }, + { .compatible = "qcom,pcie-sdx55", .data = &cfg_1_9_0 }, + { .compatible = "qcom,pcie-sm8150", .data = &cfg_1_9_0 }, + { .compatible = "qcom,pcie-sm8250", .data = &cfg_1_9_0 }, + { .compatible = "qcom,pcie-sm8350", .data = &cfg_1_9_0 }, + { .compatible = "qcom,pcie-sm8450-pcie0", .data = &cfg_1_9_0 }, + { .compatible = "qcom,pcie-sm8450-pcie1", .data = &cfg_1_9_0 }, + { .compatible = "qcom,pcie-sm8550", .data = &cfg_1_9_0 }, + { .compatible = "qcom,pcie-x1e80100", .data = &cfg_sc8280xp }, + { } +}; + +static void qcom_fixup_class(struct pci_dev *dev) +{ + dev->class = PCI_CLASS_BRIDGE_PCI_NORMAL; +} +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x0101, qcom_fixup_class); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x0104, qcom_fixup_class); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x0106, qcom_fixup_class); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x0107, qcom_fixup_class); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x0302, qcom_fixup_class); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x1000, qcom_fixup_class); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_QCOM, 0x1001, qcom_fixup_class); + +static const struct dev_pm_ops qcom_pcie_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(qcom_pcie_suspend_noirq, qcom_pcie_resume_noirq) +}; + +static struct platform_driver qcom_pcie_driver = { + .probe = qcom_pcie_probe, + .driver = { + .name = "qcom-pcie", + .suppress_bind_attrs = true, + .of_match_table = qcom_pcie_match, + .pm = &qcom_pcie_pm_ops, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, +}; +builtin_platform_driver(qcom_pcie_driver); diff --git a/drivers/pci/controller/dwc/pcie-rcar-gen4.c b/drivers/pci/controller/dwc/pcie-rcar-gen4.c new file mode 100644 index 000000000..fbe465a29 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-rcar-gen4.c @@ -0,0 +1,907 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PCIe controller driver for Renesas R-Car Gen4 Series SoCs + * Copyright (C) 2022-2023 Renesas Electronics Corporation + * + * The r8a779g0 (R-Car V4H) controller requires a specific firmware to be + * provided, to initialize the PHY. Otherwise, the PCIe controller will not + * work. + */ + +#include <linux/delay.h> +#include <linux/firmware.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/irqchip/arm-gic-v3.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/reset.h> + +#include "../../pci.h" +#include "pcie-designware.h" + +/* Renesas-specific */ +/* PCIe Mode Setting Register 0 */ +#define PCIEMSR0 0x0000 +#define APP_SRIS_MODE BIT(6) +#define DEVICE_TYPE_EP 0 +#define DEVICE_TYPE_RC BIT(4) +#define BIFUR_MOD_SET_ON BIT(0) + +/* MSI Capability */ +#define MSICAP0 0x0050 +#define MSICAP0_MSIE BIT(16) + +/* PCIe Interrupt Status 0 */ +#define PCIEINTSTS0 0x0084 + +/* PCIe Interrupt Status 0 Enable */ +#define PCIEINTSTS0EN 0x0310 +#define MSI_CTRL_INT BIT(26) +#define SMLH_LINK_UP BIT(7) +#define RDLH_LINK_UP BIT(6) + +/* PCIe DMA Interrupt Status Enable */ +#define PCIEDMAINTSTSEN 0x0314 +#define PCIEDMAINTSTSEN_INIT GENMASK(15, 0) + +/* Port Logic Registers 89 */ +#define PRTLGC89 0x0b70 + +/* Port Logic Registers 90 */ +#define PRTLGC90 0x0b74 + +/* PCIe Reset Control Register 1 */ +#define PCIERSTCTRL1 0x0014 +#define APP_HOLD_PHY_RST BIT(16) +#define APP_LTSSM_ENABLE BIT(0) + +/* INTC address */ +#define AXIINTCADDR 0x0a00 + +/* INTC control & mask */ +#define AXIINTCCONT 0x0a04 +#define INTC_EN BIT(31) +#define INTC_MASK GENMASK(11, 3) + +/* PCIe Power Management Control */ +#define PCIEPWRMNGCTRL 0x0070 +#define APP_CLK_REQ_N BIT(11) +#define APP_CLK_PM_EN BIT(10) + +#define RCAR_NUM_SPEED_CHANGE_RETRIES 10 +#define RCAR_MAX_LINK_SPEED 4 + +#define RCAR_GEN4_PCIE_EP_FUNC_DBI_OFFSET 0x1000 +#define RCAR_GEN4_PCIE_EP_FUNC_DBI2_OFFSET 0x800 + +#define RCAR_GEN4_PCIE_FIRMWARE_NAME "rcar_gen4_pcie.bin" +#define RCAR_GEN4_PCIE_FIRMWARE_BASE_ADDR 0xc000 +MODULE_FIRMWARE(RCAR_GEN4_PCIE_FIRMWARE_NAME); + +struct rcar_gen4_pcie; +struct rcar_gen4_pcie_drvdata { + void (*additional_common_init)(struct rcar_gen4_pcie *rcar); + int (*ltssm_control)(struct rcar_gen4_pcie *rcar, bool enable); + enum dw_pcie_device_mode mode; +}; + +struct rcar_gen4_pcie { + struct dw_pcie dw; + void __iomem *base; + void __iomem *phy_base; + struct platform_device *pdev; + const struct rcar_gen4_pcie_drvdata *drvdata; +}; +#define to_rcar_gen4_pcie(_dw) container_of(_dw, struct rcar_gen4_pcie, dw) + +/* Common */ +static bool rcar_gen4_pcie_link_up(struct dw_pcie *dw) +{ + struct rcar_gen4_pcie *rcar = to_rcar_gen4_pcie(dw); + u32 val, mask; + + val = readl(rcar->base + PCIEINTSTS0); + mask = RDLH_LINK_UP | SMLH_LINK_UP; + + return (val & mask) == mask; +} + +/* + * Manually initiate the speed change. Return 0 if change succeeded; otherwise + * -ETIMEDOUT. + */ +static int rcar_gen4_pcie_speed_change(struct dw_pcie *dw) +{ + u32 val; + int i; + + val = dw_pcie_readl_dbi(dw, PCIE_LINK_WIDTH_SPEED_CONTROL); + val &= ~PORT_LOGIC_SPEED_CHANGE; + dw_pcie_writel_dbi(dw, PCIE_LINK_WIDTH_SPEED_CONTROL, val); + + val = dw_pcie_readl_dbi(dw, PCIE_LINK_WIDTH_SPEED_CONTROL); + val |= PORT_LOGIC_SPEED_CHANGE; + dw_pcie_writel_dbi(dw, PCIE_LINK_WIDTH_SPEED_CONTROL, val); + + for (i = 0; i < RCAR_NUM_SPEED_CHANGE_RETRIES; i++) { + val = dw_pcie_readl_dbi(dw, PCIE_LINK_WIDTH_SPEED_CONTROL); + if (!(val & PORT_LOGIC_SPEED_CHANGE)) + return 0; + usleep_range(10000, 11000); + } + + return -ETIMEDOUT; +} + +/* + * Enable LTSSM of this controller and manually initiate the speed change. + * Always return 0. + */ +static int rcar_gen4_pcie_start_link(struct dw_pcie *dw) +{ + struct rcar_gen4_pcie *rcar = to_rcar_gen4_pcie(dw); + int i, changes, ret; + + if (rcar->drvdata->ltssm_control) { + ret = rcar->drvdata->ltssm_control(rcar, true); + if (ret) + return ret; + } + + /* + * Require direct speed change with retrying here if the max_link_speed + * is PCIe Gen2 or higher. + */ + changes = min_not_zero(dw->max_link_speed, RCAR_MAX_LINK_SPEED) - 1; + + /* + * Since dw_pcie_setup_rc() sets it once, PCIe Gen2 will be trained. + * So, this needs remaining times for up to PCIe Gen4 if RC mode. + */ + if (changes && rcar->drvdata->mode == DW_PCIE_RC_TYPE) + changes--; + + for (i = 0; i < changes; i++) { + /* It may not be connected in EP mode yet. So, break the loop */ + if (rcar_gen4_pcie_speed_change(dw)) + break; + } + + return 0; +} + +static void rcar_gen4_pcie_stop_link(struct dw_pcie *dw) +{ + struct rcar_gen4_pcie *rcar = to_rcar_gen4_pcie(dw); + + if (rcar->drvdata->ltssm_control) + rcar->drvdata->ltssm_control(rcar, false); +} + +static int rcar_gen4_pcie_common_init(struct rcar_gen4_pcie *rcar) +{ + struct dw_pcie *dw = &rcar->dw; + u32 val; + int ret; + + ret = clk_bulk_prepare_enable(DW_PCIE_NUM_CORE_CLKS, dw->core_clks); + if (ret) { + dev_err(dw->dev, "Enabling core clocks failed\n"); + return ret; + } + + if (!reset_control_status(dw->core_rsts[DW_PCIE_PWR_RST].rstc)) { + reset_control_assert(dw->core_rsts[DW_PCIE_PWR_RST].rstc); + /* + * R-Car V4H Reference Manual R19UH0186EJ0130 Rev.1.30 Apr. + * 21, 2025 page 585 Figure 9.3.2 Software Reset flow (B) + * indicates that for peripherals in HSC domain, after + * reset has been asserted by writing a matching reset bit + * into register SRCR, it is mandatory to wait 1ms. + */ + fsleep(1000); + } + + val = readl(rcar->base + PCIEMSR0); + if (rcar->drvdata->mode == DW_PCIE_RC_TYPE) { + val |= DEVICE_TYPE_RC; + } else if (rcar->drvdata->mode == DW_PCIE_EP_TYPE) { + val |= DEVICE_TYPE_EP; + } else { + ret = -EINVAL; + goto err_unprepare; + } + + if (dw->num_lanes < 4) + val |= BIFUR_MOD_SET_ON; + + writel(val, rcar->base + PCIEMSR0); + + ret = reset_control_deassert(dw->core_rsts[DW_PCIE_PWR_RST].rstc); + if (ret) + goto err_unprepare; + + /* + * Assure the reset is latched and the core is ready for DBI access. + * On R-Car V4H, the PCIe reset is asynchronous and does not take + * effect immediately, but needs a short time to complete. In case + * DBI access happens in that short time, that access generates an + * SError. To make sure that condition can never happen, read back the + * state of the reset, which should turn the asynchronous reset into + * synchronous one, and wait a little over 1ms to add additional + * safety margin. + */ + reset_control_status(dw->core_rsts[DW_PCIE_PWR_RST].rstc); + fsleep(1000); + + if (rcar->drvdata->additional_common_init) + rcar->drvdata->additional_common_init(rcar); + + return 0; + +err_unprepare: + clk_bulk_disable_unprepare(DW_PCIE_NUM_CORE_CLKS, dw->core_clks); + + return ret; +} + +static void rcar_gen4_pcie_common_deinit(struct rcar_gen4_pcie *rcar) +{ + struct dw_pcie *dw = &rcar->dw; + + reset_control_assert(dw->core_rsts[DW_PCIE_PWR_RST].rstc); + clk_bulk_disable_unprepare(DW_PCIE_NUM_CORE_CLKS, dw->core_clks); +} + +static int rcar_gen4_pcie_prepare(struct rcar_gen4_pcie *rcar) +{ + struct device *dev = rcar->dw.dev; + int err; + + pm_runtime_enable(dev); + err = pm_runtime_resume_and_get(dev); + if (err < 0) { + dev_err(dev, "Runtime resume failed\n"); + pm_runtime_disable(dev); + } + + return err; +} + +static void rcar_gen4_pcie_unprepare(struct rcar_gen4_pcie *rcar) +{ + struct device *dev = rcar->dw.dev; + + pm_runtime_put(dev); + pm_runtime_disable(dev); +} + +static int rcar_gen4_pcie_get_resources(struct rcar_gen4_pcie *rcar) +{ + rcar->phy_base = devm_platform_ioremap_resource_byname(rcar->pdev, "phy"); + if (IS_ERR(rcar->phy_base)) + return PTR_ERR(rcar->phy_base); + + /* Renesas-specific registers */ + rcar->base = devm_platform_ioremap_resource_byname(rcar->pdev, "app"); + + return PTR_ERR_OR_ZERO(rcar->base); +} + +static const struct dw_pcie_ops dw_pcie_ops = { + .start_link = rcar_gen4_pcie_start_link, + .stop_link = rcar_gen4_pcie_stop_link, + .link_up = rcar_gen4_pcie_link_up, +}; + +static struct rcar_gen4_pcie *rcar_gen4_pcie_alloc(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct rcar_gen4_pcie *rcar; + + rcar = devm_kzalloc(dev, sizeof(*rcar), GFP_KERNEL); + if (!rcar) + return ERR_PTR(-ENOMEM); + + rcar->dw.ops = &dw_pcie_ops; + rcar->dw.dev = dev; + rcar->pdev = pdev; + rcar->dw.edma.mf = EDMA_MF_EDMA_UNROLL; + dw_pcie_cap_set(&rcar->dw, REQ_RES); + platform_set_drvdata(pdev, rcar); + + return rcar; +} + +static int rcar_gen4_pcie_host_msi_addr(struct dw_pcie_rp *pp, u32 *msi_addr) +{ + struct dw_pcie *dw = to_dw_pcie_from_pp(pp); + struct device_node *msi_node = NULL; + struct device *dev = dw->dev; + struct resource res; + u64 addr; + int ret; + + /* + * Either the "msi-parent" or the "msi-map" phandle needs to exist + * to obtain the MSI node. + */ + of_msi_xlate(dev, &msi_node, 0); + if (!msi_node) + return -ENODEV; + + /* Check if "msi-parent" or the "msi-map" points to ARM GICv3 ITS. */ + if (!of_device_is_compatible(msi_node, "arm,gic-v3-its")) + return dev_err_probe(dev, -ENODEV, "Compatible MSI controller not found\n"); + + /* Derive GITS_TRANSLATER address from GICv3 */ + ret = of_address_to_resource(msi_node, 0, &res); + if (ret < 0) + return dev_err_probe(dev, ret, "MSI controller resources not obtained\n"); + + addr = res.start + GITS_TRANSLATER; + if (addr >= SZ_4G) + return dev_err_probe(dev, -EINVAL, "MSI controller address above 32bit range\n"); + + *msi_addr = addr; + return 0; +} + +static int rcar_gen4_pcie_host_msi_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *dw = to_dw_pcie_from_pp(pp); + struct rcar_gen4_pcie *rcar = to_rcar_gen4_pcie(dw); + u32 val; + int ret; + + /* Make sure MSICAP0 MSIE is configured. */ + val = dw_pcie_readl_dbi(dw, MSICAP0); + if (pci_msi_enabled()) + val |= MSICAP0_MSIE; + else + val &= ~MSICAP0_MSIE; + dw_pcie_writel_dbi(dw, MSICAP0, val); + + if (!pci_msi_enabled() || pp->use_imsi_rx) { + /* Clear AXIINTC mapping. */ + writel(0, rcar->base + AXIINTCADDR); + writel(0, rcar->base + AXIINTCCONT); + } else { + ret = rcar_gen4_pcie_host_msi_addr(pp, &val); + if (ret) + goto err; + + /* Point AXIINTC to GIC ITS and enable. */ + writel(val, rcar->base + AXIINTCADDR); + writel(INTC_EN | INTC_MASK, rcar->base + AXIINTCCONT); + } + + /* Configure MSI interrupt signal */ + val = readl(rcar->base + PCIEINTSTS0EN); + if (pci_msi_enabled()) + val |= MSI_CTRL_INT; + else + val &= ~MSI_CTRL_INT; + writel(val, rcar->base + PCIEINTSTS0EN); + + return 0; + +err: + /* Deconfigure MSICAP0 MSIE. */ + val = dw_pcie_readl_dbi(dw, MSICAP0); + val &= ~MSICAP0_MSIE; + dw_pcie_writel_dbi(dw, MSICAP0, val); + + /* Clear AXIINTC mapping. */ + writel(0, rcar->base + AXIINTCADDR); + writel(0, rcar->base + AXIINTCCONT); + + /* Deconfigure MSI interrupt signal */ + val = readl(rcar->base + PCIEINTSTS0EN); + val &= ~MSI_CTRL_INT; + writel(val, rcar->base + PCIEINTSTS0EN); + + return ret; +} + +/* Host mode */ +static int rcar_gen4_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *dw = to_dw_pcie_from_pp(pp); + struct rcar_gen4_pcie *rcar = to_rcar_gen4_pcie(dw); + int ret; + + gpiod_set_value_cansleep(dw->pe_rst, 1); + + ret = rcar_gen4_pcie_common_init(rcar); + if (ret) + return ret; + + /* + * According to the section 3.5.7.2 "RC Mode" in DWC PCIe Dual Mode + * Rev.5.20a and 3.5.6.1 "RC mode" in DWC PCIe RC databook v5.20a, we + * should disable two BARs to avoid unnecessary memory assignment + * during device enumeration. + */ + dw_pcie_writel_dbi2(dw, PCI_BASE_ADDRESS_0, 0x0); + dw_pcie_writel_dbi2(dw, PCI_BASE_ADDRESS_1, 0x0); + + ret = rcar_gen4_pcie_host_msi_init(pp); + if (ret) + goto err; + + msleep(PCIE_T_PVPERL_MS); /* pe_rst requires 100msec delay */ + + gpiod_set_value_cansleep(dw->pe_rst, 0); + + return 0; + +err: + rcar_gen4_pcie_common_deinit(rcar); + return ret; +} + +static void rcar_gen4_pcie_host_deinit(struct dw_pcie_rp *pp) +{ + struct dw_pcie *dw = to_dw_pcie_from_pp(pp); + struct rcar_gen4_pcie *rcar = to_rcar_gen4_pcie(dw); + + gpiod_set_value_cansleep(dw->pe_rst, 1); + rcar_gen4_pcie_common_deinit(rcar); +} + +static const struct dw_pcie_host_ops rcar_gen4_pcie_host_ops = { + .init = rcar_gen4_pcie_host_init, + .deinit = rcar_gen4_pcie_host_deinit, +}; + +static int rcar_gen4_add_dw_pcie_rp(struct rcar_gen4_pcie *rcar) +{ + struct dw_pcie_rp *pp = &rcar->dw.pp; + + if (!IS_ENABLED(CONFIG_PCIE_RCAR_GEN4_HOST)) + return -ENODEV; + + pp->num_vectors = MAX_MSI_IRQS; + pp->ops = &rcar_gen4_pcie_host_ops; + + return dw_pcie_host_init(pp); +} + +static void rcar_gen4_remove_dw_pcie_rp(struct rcar_gen4_pcie *rcar) +{ + dw_pcie_host_deinit(&rcar->dw.pp); +} + +/* Endpoint mode */ +static int rcar_gen4_pcie_ep_pre_init(struct dw_pcie_ep *ep) +{ + struct dw_pcie *dw = to_dw_pcie_from_ep(ep); + struct rcar_gen4_pcie *rcar = to_rcar_gen4_pcie(dw); + int ret; + + ret = rcar_gen4_pcie_common_init(rcar); + if (ret) + return ret; + + writel(PCIEDMAINTSTSEN_INIT, rcar->base + PCIEDMAINTSTSEN); + + return 0; +} + +static void rcar_gen4_pcie_ep_deinit(struct rcar_gen4_pcie *rcar) +{ + writel(0, rcar->base + PCIEDMAINTSTSEN); + rcar_gen4_pcie_common_deinit(rcar); +} + +static int rcar_gen4_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, + unsigned int type, u16 interrupt_num) +{ + struct dw_pcie *dw = to_dw_pcie_from_ep(ep); + + switch (type) { + case PCI_IRQ_INTX: + return dw_pcie_ep_raise_intx_irq(ep, func_no); + case PCI_IRQ_MSI: + return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); + default: + dev_err(dw->dev, "Unknown IRQ type\n"); + return -EINVAL; + } + + return 0; +} + +static const struct pci_epc_features rcar_gen4_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, + .msi_capable = true, + .bar[BAR_0] = { .type = BAR_RESIZABLE, }, + .bar[BAR_1] = { .type = BAR_DISABLED, }, + .bar[BAR_2] = { .type = BAR_RESIZABLE, }, + .bar[BAR_3] = { .type = BAR_DISABLED, }, + .bar[BAR_4] = { .type = BAR_FIXED, .fixed_size = 256 }, + .bar[BAR_5] = { .type = BAR_DISABLED, }, + .align = SZ_4K, +}; + +static const struct pci_epc_features* +rcar_gen4_pcie_ep_get_features(struct dw_pcie_ep *ep) +{ + return &rcar_gen4_pcie_epc_features; +} + +static unsigned int rcar_gen4_pcie_ep_get_dbi_offset(struct dw_pcie_ep *ep, + u8 func_no) +{ + return func_no * RCAR_GEN4_PCIE_EP_FUNC_DBI_OFFSET; +} + +static unsigned int rcar_gen4_pcie_ep_get_dbi2_offset(struct dw_pcie_ep *ep, + u8 func_no) +{ + return func_no * RCAR_GEN4_PCIE_EP_FUNC_DBI2_OFFSET; +} + +static const struct dw_pcie_ep_ops pcie_ep_ops = { + .pre_init = rcar_gen4_pcie_ep_pre_init, + .raise_irq = rcar_gen4_pcie_ep_raise_irq, + .get_features = rcar_gen4_pcie_ep_get_features, + .get_dbi_offset = rcar_gen4_pcie_ep_get_dbi_offset, + .get_dbi2_offset = rcar_gen4_pcie_ep_get_dbi2_offset, +}; + +static int rcar_gen4_add_dw_pcie_ep(struct rcar_gen4_pcie *rcar) +{ + struct dw_pcie_ep *ep = &rcar->dw.ep; + struct device *dev = rcar->dw.dev; + int ret; + + if (!IS_ENABLED(CONFIG_PCIE_RCAR_GEN4_EP)) + return -ENODEV; + + ep->ops = &pcie_ep_ops; + + ret = dw_pcie_ep_init(ep); + if (ret) { + rcar_gen4_pcie_ep_deinit(rcar); + return ret; + } + + ret = dw_pcie_ep_init_registers(ep); + if (ret) { + dev_err(dev, "Failed to initialize DWC endpoint registers\n"); + dw_pcie_ep_deinit(ep); + rcar_gen4_pcie_ep_deinit(rcar); + } + + pci_epc_init_notify(ep->epc); + + return ret; +} + +static void rcar_gen4_remove_dw_pcie_ep(struct rcar_gen4_pcie *rcar) +{ + dw_pcie_ep_deinit(&rcar->dw.ep); + rcar_gen4_pcie_ep_deinit(rcar); +} + +/* Common */ +static int rcar_gen4_add_dw_pcie(struct rcar_gen4_pcie *rcar) +{ + rcar->drvdata = of_device_get_match_data(&rcar->pdev->dev); + if (!rcar->drvdata) + return -EINVAL; + + switch (rcar->drvdata->mode) { + case DW_PCIE_RC_TYPE: + return rcar_gen4_add_dw_pcie_rp(rcar); + case DW_PCIE_EP_TYPE: + return rcar_gen4_add_dw_pcie_ep(rcar); + default: + return -EINVAL; + } +} + +static int rcar_gen4_pcie_probe(struct platform_device *pdev) +{ + struct rcar_gen4_pcie *rcar; + int err; + + rcar = rcar_gen4_pcie_alloc(pdev); + if (IS_ERR(rcar)) + return PTR_ERR(rcar); + + err = rcar_gen4_pcie_get_resources(rcar); + if (err) + return err; + + err = rcar_gen4_pcie_prepare(rcar); + if (err) + return err; + + err = rcar_gen4_add_dw_pcie(rcar); + if (err) + goto err_unprepare; + + return 0; + +err_unprepare: + rcar_gen4_pcie_unprepare(rcar); + + return err; +} + +static void rcar_gen4_remove_dw_pcie(struct rcar_gen4_pcie *rcar) +{ + switch (rcar->drvdata->mode) { + case DW_PCIE_RC_TYPE: + rcar_gen4_remove_dw_pcie_rp(rcar); + break; + case DW_PCIE_EP_TYPE: + rcar_gen4_remove_dw_pcie_ep(rcar); + break; + default: + break; + } +} + +static void rcar_gen4_pcie_remove(struct platform_device *pdev) +{ + struct rcar_gen4_pcie *rcar = platform_get_drvdata(pdev); + + rcar_gen4_remove_dw_pcie(rcar); + rcar_gen4_pcie_unprepare(rcar); +} + +static int r8a779f0_pcie_ltssm_control(struct rcar_gen4_pcie *rcar, bool enable) +{ + u32 val; + + val = readl(rcar->base + PCIERSTCTRL1); + if (enable) { + val |= APP_LTSSM_ENABLE; + val &= ~APP_HOLD_PHY_RST; + } else { + /* + * Since the datasheet of R-Car doesn't mention how to assert + * the APP_HOLD_PHY_RST, don't assert it again. Otherwise, + * hang-up issue happened in the dw_edma_core_off() when + * the controller didn't detect a PCI device. + */ + val &= ~APP_LTSSM_ENABLE; + } + writel(val, rcar->base + PCIERSTCTRL1); + + return 0; +} + +static void rcar_gen4_pcie_additional_common_init(struct rcar_gen4_pcie *rcar) +{ + struct dw_pcie *dw = &rcar->dw; + u32 val; + + val = dw_pcie_readl_dbi(dw, PCIE_PORT_LANE_SKEW); + val &= ~PORT_LANE_SKEW_INSERT_MASK; + if (dw->num_lanes < 4) + val |= BIT(6); + dw_pcie_writel_dbi(dw, PCIE_PORT_LANE_SKEW, val); + + val = readl(rcar->base + PCIEPWRMNGCTRL); + val |= APP_CLK_REQ_N | APP_CLK_PM_EN; + writel(val, rcar->base + PCIEPWRMNGCTRL); +} + +static void rcar_gen4_pcie_phy_reg_update_bits(struct rcar_gen4_pcie *rcar, + u32 offset, u32 mask, u32 val) +{ + u32 tmp; + + tmp = readl(rcar->phy_base + offset); + tmp &= ~mask; + tmp |= val; + writel(tmp, rcar->phy_base + offset); +} + +/* + * SoC datasheet suggests checking port logic register bits during firmware + * write. If read returns non-zero value, then this function returns -EAGAIN + * indicating that the write needs to be done again. If read returns zero, + * then return 0 to indicate success. + */ +static int rcar_gen4_pcie_reg_test_bit(struct rcar_gen4_pcie *rcar, + u32 offset, u32 mask) +{ + struct dw_pcie *dw = &rcar->dw; + + if (dw_pcie_readl_dbi(dw, offset) & mask) + return -EAGAIN; + + return 0; +} + +static int rcar_gen4_pcie_download_phy_firmware(struct rcar_gen4_pcie *rcar) +{ + /* The check_addr values are magical numbers in the datasheet */ + static const u32 check_addr[] = { + 0x00101018, + 0x00101118, + 0x00101021, + 0x00101121, + }; + struct dw_pcie *dw = &rcar->dw; + const struct firmware *fw; + unsigned int i, timeout; + u32 data; + int ret; + + ret = request_firmware(&fw, RCAR_GEN4_PCIE_FIRMWARE_NAME, dw->dev); + if (ret) { + dev_err(dw->dev, "Failed to load firmware (%s): %d\n", + RCAR_GEN4_PCIE_FIRMWARE_NAME, ret); + return ret; + } + + for (i = 0; i < (fw->size / 2); i++) { + data = fw->data[(i * 2) + 1] << 8 | fw->data[i * 2]; + timeout = 100; + do { + dw_pcie_writel_dbi(dw, PRTLGC89, RCAR_GEN4_PCIE_FIRMWARE_BASE_ADDR + i); + dw_pcie_writel_dbi(dw, PRTLGC90, data); + if (!rcar_gen4_pcie_reg_test_bit(rcar, PRTLGC89, BIT(30))) + break; + if (!(--timeout)) { + ret = -ETIMEDOUT; + goto exit; + } + usleep_range(100, 200); + } while (1); + } + + rcar_gen4_pcie_phy_reg_update_bits(rcar, 0x0f8, BIT(17), BIT(17)); + + for (i = 0; i < ARRAY_SIZE(check_addr); i++) { + timeout = 100; + do { + dw_pcie_writel_dbi(dw, PRTLGC89, check_addr[i]); + ret = rcar_gen4_pcie_reg_test_bit(rcar, PRTLGC89, BIT(30)); + ret |= rcar_gen4_pcie_reg_test_bit(rcar, PRTLGC90, BIT(0)); + if (!ret) + break; + if (!(--timeout)) { + ret = -ETIMEDOUT; + goto exit; + } + usleep_range(100, 200); + } while (1); + } + +exit: + release_firmware(fw); + + return ret; +} + +static int rcar_gen4_pcie_ltssm_control(struct rcar_gen4_pcie *rcar, bool enable) +{ + struct dw_pcie *dw = &rcar->dw; + u32 val; + int ret; + + if (!enable) { + val = readl(rcar->base + PCIERSTCTRL1); + val &= ~APP_LTSSM_ENABLE; + writel(val, rcar->base + PCIERSTCTRL1); + + return 0; + } + + val = dw_pcie_readl_dbi(dw, PCIE_PORT_FORCE); + val |= PORT_FORCE_DO_DESKEW_FOR_SRIS; + dw_pcie_writel_dbi(dw, PCIE_PORT_FORCE, val); + + val = readl(rcar->base + PCIEMSR0); + val |= APP_SRIS_MODE; + writel(val, rcar->base + PCIEMSR0); + + /* + * The R-Car Gen4 datasheet doesn't describe the PHY registers' name. + * But, the initialization procedure describes these offsets. So, + * this driver has magical offset numbers. + */ + rcar_gen4_pcie_phy_reg_update_bits(rcar, 0x700, BIT(28), 0); + rcar_gen4_pcie_phy_reg_update_bits(rcar, 0x700, BIT(20), 0); + rcar_gen4_pcie_phy_reg_update_bits(rcar, 0x700, BIT(12), 0); + rcar_gen4_pcie_phy_reg_update_bits(rcar, 0x700, BIT(4), 0); + + rcar_gen4_pcie_phy_reg_update_bits(rcar, 0x148, GENMASK(23, 22), BIT(22)); + rcar_gen4_pcie_phy_reg_update_bits(rcar, 0x148, GENMASK(18, 16), GENMASK(17, 16)); + rcar_gen4_pcie_phy_reg_update_bits(rcar, 0x148, GENMASK(7, 6), BIT(6)); + rcar_gen4_pcie_phy_reg_update_bits(rcar, 0x148, GENMASK(2, 0), GENMASK(1, 0)); + rcar_gen4_pcie_phy_reg_update_bits(rcar, 0x1d4, GENMASK(16, 15), GENMASK(16, 15)); + rcar_gen4_pcie_phy_reg_update_bits(rcar, 0x514, BIT(26), BIT(26)); + rcar_gen4_pcie_phy_reg_update_bits(rcar, 0x0f8, BIT(16), 0); + rcar_gen4_pcie_phy_reg_update_bits(rcar, 0x0f8, BIT(19), BIT(19)); + + val = readl(rcar->base + PCIERSTCTRL1); + val &= ~APP_HOLD_PHY_RST; + writel(val, rcar->base + PCIERSTCTRL1); + + ret = readl_poll_timeout(rcar->phy_base + 0x0f8, val, val & BIT(18), 100, 10000); + if (ret < 0) + return ret; + + ret = rcar_gen4_pcie_download_phy_firmware(rcar); + if (ret) + return ret; + + val = readl(rcar->base + PCIERSTCTRL1); + val |= APP_LTSSM_ENABLE; + writel(val, rcar->base + PCIERSTCTRL1); + + return 0; +} + +static struct rcar_gen4_pcie_drvdata drvdata_r8a779f0_pcie = { + .ltssm_control = r8a779f0_pcie_ltssm_control, + .mode = DW_PCIE_RC_TYPE, +}; + +static struct rcar_gen4_pcie_drvdata drvdata_r8a779f0_pcie_ep = { + .ltssm_control = r8a779f0_pcie_ltssm_control, + .mode = DW_PCIE_EP_TYPE, +}; + +static struct rcar_gen4_pcie_drvdata drvdata_rcar_gen4_pcie = { + .additional_common_init = rcar_gen4_pcie_additional_common_init, + .ltssm_control = rcar_gen4_pcie_ltssm_control, + .mode = DW_PCIE_RC_TYPE, +}; + +static struct rcar_gen4_pcie_drvdata drvdata_rcar_gen4_pcie_ep = { + .additional_common_init = rcar_gen4_pcie_additional_common_init, + .ltssm_control = rcar_gen4_pcie_ltssm_control, + .mode = DW_PCIE_EP_TYPE, +}; + +static const struct of_device_id rcar_gen4_pcie_of_match[] = { + { + .compatible = "renesas,r8a779f0-pcie", + .data = &drvdata_r8a779f0_pcie, + }, + { + .compatible = "renesas,r8a779f0-pcie-ep", + .data = &drvdata_r8a779f0_pcie_ep, + }, + { + .compatible = "renesas,rcar-gen4-pcie", + .data = &drvdata_rcar_gen4_pcie, + }, + { + .compatible = "renesas,rcar-gen4-pcie-ep", + .data = &drvdata_rcar_gen4_pcie_ep, + }, + {}, +}; +MODULE_DEVICE_TABLE(of, rcar_gen4_pcie_of_match); + +static struct platform_driver rcar_gen4_pcie_driver = { + .driver = { + .name = "pcie-rcar-gen4", + .of_match_table = rcar_gen4_pcie_of_match, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, + .probe = rcar_gen4_pcie_probe, + .remove = rcar_gen4_pcie_remove, +}; +module_platform_driver(rcar_gen4_pcie_driver); + +MODULE_DESCRIPTION("Renesas R-Car Gen4 PCIe controller driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pci/controller/dwc/pcie-sophgo.c b/drivers/pci/controller/dwc/pcie-sophgo.c new file mode 100644 index 000000000..044088898 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-sophgo.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Sophgo DesignWare based PCIe host controller driver + */ + +#include <linux/bits.h> +#include <linux/clk.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqdomain.h> +#include <linux/module.h> +#include <linux/property.h> +#include <linux/platform_device.h> + +#include "pcie-designware.h" + +#define to_sophgo_pcie(x) dev_get_drvdata((x)->dev) + +#define PCIE_INT_SIGNAL 0xc48 +#define PCIE_INT_EN 0xca0 + +#define PCIE_INT_SIGNAL_INTX GENMASK(8, 5) + +#define PCIE_INT_EN_INTX GENMASK(4, 1) +#define PCIE_INT_EN_INT_MSI BIT(5) + +struct sophgo_pcie { + struct dw_pcie pci; + void __iomem *app_base; + struct clk_bulk_data *clks; + unsigned int clk_cnt; + struct irq_domain *irq_domain; +}; + +static int sophgo_pcie_readl_app(struct sophgo_pcie *sophgo, u32 reg) +{ + return readl_relaxed(sophgo->app_base + reg); +} + +static void sophgo_pcie_writel_app(struct sophgo_pcie *sophgo, u32 val, u32 reg) +{ + writel_relaxed(val, sophgo->app_base + reg); +} + +static void sophgo_pcie_intx_handler(struct irq_desc *desc) +{ + struct dw_pcie_rp *pp = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct sophgo_pcie *sophgo = to_sophgo_pcie(pci); + unsigned long hwirq, reg; + + chained_irq_enter(chip, desc); + + reg = sophgo_pcie_readl_app(sophgo, PCIE_INT_SIGNAL); + reg = FIELD_GET(PCIE_INT_SIGNAL_INTX, reg); + + for_each_set_bit(hwirq, ®, PCI_NUM_INTX) + generic_handle_domain_irq(sophgo->irq_domain, hwirq); + + chained_irq_exit(chip, desc); +} + +static void sophgo_intx_irq_mask(struct irq_data *d) +{ + struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct sophgo_pcie *sophgo = to_sophgo_pcie(pci); + unsigned long flags; + u32 val; + + raw_spin_lock_irqsave(&pp->lock, flags); + + val = sophgo_pcie_readl_app(sophgo, PCIE_INT_EN); + val &= ~FIELD_PREP(PCIE_INT_EN_INTX, BIT(d->hwirq)); + sophgo_pcie_writel_app(sophgo, val, PCIE_INT_EN); + + raw_spin_unlock_irqrestore(&pp->lock, flags); +}; + +static void sophgo_intx_irq_unmask(struct irq_data *d) +{ + struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct sophgo_pcie *sophgo = to_sophgo_pcie(pci); + unsigned long flags; + u32 val; + + raw_spin_lock_irqsave(&pp->lock, flags); + + val = sophgo_pcie_readl_app(sophgo, PCIE_INT_EN); + val |= FIELD_PREP(PCIE_INT_EN_INTX, BIT(d->hwirq)); + sophgo_pcie_writel_app(sophgo, val, PCIE_INT_EN); + + raw_spin_unlock_irqrestore(&pp->lock, flags); +}; + +static struct irq_chip sophgo_intx_irq_chip = { + .name = "INTx", + .irq_mask = sophgo_intx_irq_mask, + .irq_unmask = sophgo_intx_irq_unmask, +}; + +static int sophgo_pcie_intx_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &sophgo_intx_irq_chip, handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + + return 0; +} + +static const struct irq_domain_ops intx_domain_ops = { + .map = sophgo_pcie_intx_map, +}; + +static int sophgo_pcie_init_irq_domain(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct sophgo_pcie *sophgo = to_sophgo_pcie(pci); + struct device *dev = sophgo->pci.dev; + struct fwnode_handle *intc; + int irq; + + intc = device_get_named_child_node(dev, "interrupt-controller"); + if (!intc) { + dev_err(dev, "missing child interrupt-controller node\n"); + return -ENODEV; + } + + irq = fwnode_irq_get(intc, 0); + if (irq < 0) { + dev_err(dev, "failed to get INTx irq number\n"); + fwnode_handle_put(intc); + return irq; + } + + sophgo->irq_domain = irq_domain_create_linear(intc, PCI_NUM_INTX, + &intx_domain_ops, pp); + fwnode_handle_put(intc); + if (!sophgo->irq_domain) { + dev_err(dev, "failed to get a INTx irq domain\n"); + return -EINVAL; + } + + return irq; +} + +static void sophgo_pcie_msi_enable(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct sophgo_pcie *sophgo = to_sophgo_pcie(pci); + unsigned long flags; + u32 val; + + raw_spin_lock_irqsave(&pp->lock, flags); + + val = sophgo_pcie_readl_app(sophgo, PCIE_INT_EN); + val |= PCIE_INT_EN_INT_MSI; + sophgo_pcie_writel_app(sophgo, val, PCIE_INT_EN); + + raw_spin_unlock_irqrestore(&pp->lock, flags); +} + +static void sophgo_pcie_disable_l0s_l1(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + u32 offset, val; + + offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + + dw_pcie_dbi_ro_wr_en(pci); + + val = dw_pcie_readl_dbi(pci, PCI_EXP_LNKCAP + offset); + val &= ~(PCI_EXP_LNKCAP_ASPM_L0S | PCI_EXP_LNKCAP_ASPM_L1); + dw_pcie_writel_dbi(pci, PCI_EXP_LNKCAP + offset, val); + + dw_pcie_dbi_ro_wr_dis(pci); +} + +static int sophgo_pcie_host_init(struct dw_pcie_rp *pp) +{ + int irq; + + irq = sophgo_pcie_init_irq_domain(pp); + if (irq < 0) + return irq; + + irq_set_chained_handler_and_data(irq, sophgo_pcie_intx_handler, pp); + + sophgo_pcie_disable_l0s_l1(pp); + + sophgo_pcie_msi_enable(pp); + + return 0; +} + +static const struct dw_pcie_host_ops sophgo_pcie_host_ops = { + .init = sophgo_pcie_host_init, +}; + +static int sophgo_pcie_clk_init(struct sophgo_pcie *sophgo) +{ + struct device *dev = sophgo->pci.dev; + int ret; + + ret = devm_clk_bulk_get_all_enabled(dev, &sophgo->clks); + if (ret < 0) + return dev_err_probe(dev, ret, "failed to get clocks\n"); + + sophgo->clk_cnt = ret; + + return 0; +} + +static int sophgo_pcie_resource_get(struct platform_device *pdev, + struct sophgo_pcie *sophgo) +{ + sophgo->app_base = devm_platform_ioremap_resource_byname(pdev, "app"); + if (IS_ERR(sophgo->app_base)) + return dev_err_probe(&pdev->dev, PTR_ERR(sophgo->app_base), + "failed to map app registers\n"); + + return 0; +} + +static int sophgo_pcie_configure_rc(struct sophgo_pcie *sophgo) +{ + struct dw_pcie_rp *pp; + + pp = &sophgo->pci.pp; + pp->ops = &sophgo_pcie_host_ops; + + return dw_pcie_host_init(pp); +} + +static int sophgo_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct sophgo_pcie *sophgo; + int ret; + + sophgo = devm_kzalloc(dev, sizeof(*sophgo), GFP_KERNEL); + if (!sophgo) + return -ENOMEM; + + platform_set_drvdata(pdev, sophgo); + + sophgo->pci.dev = dev; + + ret = sophgo_pcie_resource_get(pdev, sophgo); + if (ret) + return ret; + + ret = sophgo_pcie_clk_init(sophgo); + if (ret) + return ret; + + return sophgo_pcie_configure_rc(sophgo); +} + +static const struct of_device_id sophgo_pcie_of_match[] = { + { .compatible = "sophgo,sg2044-pcie" }, + { } +}; +MODULE_DEVICE_TABLE(of, sophgo_pcie_of_match); + +static struct platform_driver sophgo_pcie_driver = { + .driver = { + .name = "sophgo-pcie", + .of_match_table = sophgo_pcie_of_match, + .suppress_bind_attrs = true, + }, + .probe = sophgo_pcie_probe, +}; +builtin_platform_driver(sophgo_pcie_driver); diff --git a/drivers/pci/controller/dwc/pcie-spacemit-k1.c b/drivers/pci/controller/dwc/pcie-spacemit-k1.c new file mode 100644 index 000000000..0564c46e2 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-spacemit-k1.c @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * SpacemiT K1 PCIe host driver + * + * Copyright (C) 2025 by RISCstar Solutions Corporation. All rights reserved. + * Copyright (c) 2023, spacemit Corporation. + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/gfp.h> +#include <linux/mfd/syscon.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/reset.h> +#include <linux/types.h> + +#include "pcie-designware.h" + +/* Offsets and field definitions for link management registers */ +#define K1_PHY_AHB_IRQ_EN 0x0000 +#define PCIE_INTERRUPT_EN BIT(0) + +#define K1_PHY_AHB_LINK_STS 0x0004 +#define SMLH_LINK_UP BIT(1) +#define RDLH_LINK_UP BIT(12) + +#define INTR_ENABLE 0x0014 +#define MSI_CTRL_INT BIT(11) + +/* Some controls require APMU regmap access */ +#define SYSCON_APMU "spacemit,apmu" + +/* Offsets and field definitions for APMU registers */ +#define PCIE_CLK_RESET_CONTROL 0x0000 +#define LTSSM_EN BIT(6) +#define PCIE_AUX_PWR_DET BIT(9) +#define PCIE_RC_PERST BIT(12) /* 1: assert PERST# */ +#define APP_HOLD_PHY_RST BIT(30) +#define DEVICE_TYPE_RC BIT(31) /* 0: endpoint; 1: RC */ + +#define PCIE_CONTROL_LOGIC 0x0004 +#define PCIE_SOFT_RESET BIT(0) + +struct k1_pcie { + struct dw_pcie pci; + struct phy *phy; + void __iomem *link; + struct regmap *pmu; /* Errors ignored; MMIO-backed regmap */ + u32 pmu_off; +}; + +#define to_k1_pcie(dw_pcie) \ + platform_get_drvdata(to_platform_device((dw_pcie)->dev)) + +static void k1_pcie_toggle_soft_reset(struct k1_pcie *k1) +{ + u32 offset; + u32 val; + + /* + * Write, then read back to guarantee it has reached the device + * before we start the delay. + */ + offset = k1->pmu_off + PCIE_CONTROL_LOGIC; + regmap_set_bits(k1->pmu, offset, PCIE_SOFT_RESET); + regmap_read(k1->pmu, offset, &val); + + mdelay(2); + + regmap_clear_bits(k1->pmu, offset, PCIE_SOFT_RESET); +} + +/* Enable app clocks, deassert resets */ +static int k1_pcie_enable_resources(struct k1_pcie *k1) +{ + struct dw_pcie *pci = &k1->pci; + int ret; + + ret = clk_bulk_prepare_enable(ARRAY_SIZE(pci->app_clks), pci->app_clks); + if (ret) + return ret; + + ret = reset_control_bulk_deassert(ARRAY_SIZE(pci->app_rsts), + pci->app_rsts); + if (ret) + goto err_disable_clks; + + return 0; + +err_disable_clks: + clk_bulk_disable_unprepare(ARRAY_SIZE(pci->app_clks), pci->app_clks); + + return ret; +} + +/* Assert resets, disable app clocks */ +static void k1_pcie_disable_resources(struct k1_pcie *k1) +{ + struct dw_pcie *pci = &k1->pci; + + reset_control_bulk_assert(ARRAY_SIZE(pci->app_rsts), pci->app_rsts); + clk_bulk_disable_unprepare(ARRAY_SIZE(pci->app_clks), pci->app_clks); +} + +/* FIXME: Disable ASPM L1 to avoid errors reported on some NVMe drives */ +static void k1_pcie_disable_aspm_l1(struct k1_pcie *k1) +{ + struct dw_pcie *pci = &k1->pci; + u8 offset; + u32 val; + + offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + offset += PCI_EXP_LNKCAP; + + dw_pcie_dbi_ro_wr_en(pci); + val = dw_pcie_readl_dbi(pci, offset); + val &= ~PCI_EXP_LNKCAP_ASPM_L1; + dw_pcie_writel_dbi(pci, offset, val); + dw_pcie_dbi_ro_wr_dis(pci); +} + +static int k1_pcie_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct k1_pcie *k1 = to_k1_pcie(pci); + u32 reset_ctrl; + u32 val; + int ret; + + k1_pcie_toggle_soft_reset(k1); + + ret = k1_pcie_enable_resources(k1); + if (ret) + return ret; + + /* Set the PCI vendor and device ID */ + dw_pcie_dbi_ro_wr_en(pci); + dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, PCI_VENDOR_ID_SPACEMIT); + dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, PCI_DEVICE_ID_SPACEMIT_K1); + dw_pcie_dbi_ro_wr_dis(pci); + + /* + * Start by asserting fundamental reset (drive PERST# low). The + * PCI CEM spec says that PERST# should be deasserted at least + * 100ms after the power becomes stable, so we'll insert that + * delay first. Write, then read it back to guarantee the write + * reaches the device before we start the delay. + */ + reset_ctrl = k1->pmu_off + PCIE_CLK_RESET_CONTROL; + regmap_set_bits(k1->pmu, reset_ctrl, PCIE_RC_PERST); + regmap_read(k1->pmu, reset_ctrl, &val); + mdelay(PCIE_T_PVPERL_MS); + + /* + * Put the controller in root complex mode, and indicate that + * Vaux (3.3v) is present. + */ + regmap_set_bits(k1->pmu, reset_ctrl, DEVICE_TYPE_RC | PCIE_AUX_PWR_DET); + + ret = phy_init(k1->phy); + if (ret) { + k1_pcie_disable_resources(k1); + + return ret; + } + + /* Deassert fundamental reset (drive PERST# high) */ + regmap_clear_bits(k1->pmu, reset_ctrl, PCIE_RC_PERST); + + /* Finally, as a workaround, disable ASPM L1 */ + k1_pcie_disable_aspm_l1(k1); + + return 0; +} + +static void k1_pcie_deinit(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct k1_pcie *k1 = to_k1_pcie(pci); + + /* Assert fundamental reset (drive PERST# low) */ + regmap_set_bits(k1->pmu, k1->pmu_off + PCIE_CLK_RESET_CONTROL, + PCIE_RC_PERST); + + phy_exit(k1->phy); + + k1_pcie_disable_resources(k1); +} + +static const struct dw_pcie_host_ops k1_pcie_host_ops = { + .init = k1_pcie_init, + .deinit = k1_pcie_deinit, +}; + +static bool k1_pcie_link_up(struct dw_pcie *pci) +{ + struct k1_pcie *k1 = to_k1_pcie(pci); + u32 val; + + val = readl_relaxed(k1->link + K1_PHY_AHB_LINK_STS); + + return (val & RDLH_LINK_UP) && (val & SMLH_LINK_UP); +} + +static int k1_pcie_start_link(struct dw_pcie *pci) +{ + struct k1_pcie *k1 = to_k1_pcie(pci); + u32 val; + + /* Stop holding the PHY in reset, and enable link training */ + regmap_update_bits(k1->pmu, k1->pmu_off + PCIE_CLK_RESET_CONTROL, + APP_HOLD_PHY_RST | LTSSM_EN, LTSSM_EN); + + /* Enable the MSI interrupt */ + writel_relaxed(MSI_CTRL_INT, k1->link + INTR_ENABLE); + + /* Top-level interrupt enable */ + val = readl_relaxed(k1->link + K1_PHY_AHB_IRQ_EN); + val |= PCIE_INTERRUPT_EN; + writel_relaxed(val, k1->link + K1_PHY_AHB_IRQ_EN); + + return 0; +} + +static void k1_pcie_stop_link(struct dw_pcie *pci) +{ + struct k1_pcie *k1 = to_k1_pcie(pci); + u32 val; + + /* Disable interrupts */ + val = readl_relaxed(k1->link + K1_PHY_AHB_IRQ_EN); + val &= ~PCIE_INTERRUPT_EN; + writel_relaxed(val, k1->link + K1_PHY_AHB_IRQ_EN); + + writel_relaxed(0, k1->link + INTR_ENABLE); + + /* Disable the link and hold the PHY in reset */ + regmap_update_bits(k1->pmu, k1->pmu_off + PCIE_CLK_RESET_CONTROL, + APP_HOLD_PHY_RST | LTSSM_EN, APP_HOLD_PHY_RST); +} + +static const struct dw_pcie_ops k1_pcie_ops = { + .link_up = k1_pcie_link_up, + .start_link = k1_pcie_start_link, + .stop_link = k1_pcie_stop_link, +}; + +static int k1_pcie_parse_port(struct k1_pcie *k1) +{ + struct device *dev = k1->pci.dev; + struct device_node *root_port; + struct phy *phy; + + /* We assume only one root port */ + root_port = of_get_next_available_child(dev_of_node(dev), NULL); + if (!root_port) + return -EINVAL; + + phy = devm_of_phy_get(dev, root_port, NULL); + + of_node_put(root_port); + + if (IS_ERR(phy)) + return PTR_ERR(phy); + + k1->phy = phy; + + return 0; +} + +static int k1_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct k1_pcie *k1; + int ret; + + k1 = devm_kzalloc(dev, sizeof(*k1), GFP_KERNEL); + if (!k1) + return -ENOMEM; + + k1->pmu = syscon_regmap_lookup_by_phandle_args(dev_of_node(dev), + SYSCON_APMU, 1, + &k1->pmu_off); + if (IS_ERR(k1->pmu)) + return dev_err_probe(dev, PTR_ERR(k1->pmu), + "failed to lookup PMU registers\n"); + + k1->link = devm_platform_ioremap_resource_byname(pdev, "link"); + if (IS_ERR(k1->link)) + return dev_err_probe(dev, PTR_ERR(k1->link), + "failed to map \"link\" registers\n"); + + k1->pci.dev = dev; + k1->pci.ops = &k1_pcie_ops; + k1->pci.pp.num_vectors = MAX_MSI_IRQS; + dw_pcie_cap_set(&k1->pci, REQ_RES); + + k1->pci.pp.ops = &k1_pcie_host_ops; + + /* Hold the PHY in reset until we start the link */ + regmap_set_bits(k1->pmu, k1->pmu_off + PCIE_CLK_RESET_CONTROL, + APP_HOLD_PHY_RST); + + ret = devm_regulator_get_enable(dev, "vpcie3v3"); + if (ret) + return dev_err_probe(dev, ret, + "failed to get \"vpcie3v3\" supply\n"); + + pm_runtime_set_active(dev); + pm_runtime_no_callbacks(dev); + devm_pm_runtime_enable(dev); + + platform_set_drvdata(pdev, k1); + + ret = k1_pcie_parse_port(k1); + if (ret) + return dev_err_probe(dev, ret, "failed to parse root port\n"); + + ret = dw_pcie_host_init(&k1->pci.pp); + if (ret) + return dev_err_probe(dev, ret, "failed to initialize host\n"); + + return 0; +} + +static void k1_pcie_remove(struct platform_device *pdev) +{ + struct k1_pcie *k1 = platform_get_drvdata(pdev); + + dw_pcie_host_deinit(&k1->pci.pp); +} + +static const struct of_device_id k1_pcie_of_match_table[] = { + { .compatible = "spacemit,k1-pcie", }, + { } +}; +MODULE_DEVICE_TABLE(of, k1_pcie_of_match_table); + +static struct platform_driver k1_pcie_driver = { + .probe = k1_pcie_probe, + .remove = k1_pcie_remove, + .driver = { + .name = "spacemit-k1-pcie", + .of_match_table = k1_pcie_of_match_table, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, +}; +module_platform_driver(k1_pcie_driver); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SpacemiT K1 PCIe host driver"); diff --git a/drivers/pci/controller/dwc/pcie-spear13xx.c b/drivers/pci/controller/dwc/pcie-spear13xx.c new file mode 100644 index 000000000..01794a9d3 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-spear13xx.c @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for ST Microelectronics SPEAr13xx SoCs + * + * SPEAr13xx PCIe Glue Layer Source Code + * + * Copyright (C) 2010-2014 ST Microelectronics + * Pratyush Anand <pratyush.anand@gmail.com> + * Mohit Kumar <mohit.kumar.dhaka@gmail.com> + */ + +#include <linux/clk.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/of.h> +#include <linux/pci.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/resource.h> + +#include "pcie-designware.h" + +struct spear13xx_pcie { + struct dw_pcie *pci; + void __iomem *app_base; + struct phy *phy; + struct clk *clk; +}; + +struct pcie_app_reg { + u32 app_ctrl_0; /* cr0 */ + u32 app_ctrl_1; /* cr1 */ + u32 app_status_0; /* cr2 */ + u32 app_status_1; /* cr3 */ + u32 msg_status; /* cr4 */ + u32 msg_payload; /* cr5 */ + u32 int_sts; /* cr6 */ + u32 int_clr; /* cr7 */ + u32 int_mask; /* cr8 */ + u32 mst_bmisc; /* cr9 */ + u32 phy_ctrl; /* cr10 */ + u32 phy_status; /* cr11 */ + u32 cxpl_debug_info_0; /* cr12 */ + u32 cxpl_debug_info_1; /* cr13 */ + u32 ven_msg_ctrl_0; /* cr14 */ + u32 ven_msg_ctrl_1; /* cr15 */ + u32 ven_msg_data_0; /* cr16 */ + u32 ven_msg_data_1; /* cr17 */ + u32 ven_msi_0; /* cr18 */ + u32 ven_msi_1; /* cr19 */ + u32 mst_rmisc; /* cr20 */ +}; + +/* CR0 ID */ +#define APP_LTSSM_ENABLE_ID 3 +#define DEVICE_TYPE_RC (4 << 25) +#define MISCTRL_EN_ID 30 +#define REG_TRANSLATION_ENABLE 31 + +/* CR3 ID */ +#define XMLH_LINK_UP (1 << 6) + +/* CR6 */ +#define MSI_CTRL_INT (1 << 26) + +#define to_spear13xx_pcie(x) dev_get_drvdata((x)->dev) + +static int spear13xx_pcie_start_link(struct dw_pcie *pci) +{ + struct spear13xx_pcie *spear13xx_pcie = to_spear13xx_pcie(pci); + struct pcie_app_reg __iomem *app_reg = spear13xx_pcie->app_base; + + /* enable ltssm */ + writel(DEVICE_TYPE_RC | (1 << MISCTRL_EN_ID) + | (1 << APP_LTSSM_ENABLE_ID) + | ((u32)1 << REG_TRANSLATION_ENABLE), + &app_reg->app_ctrl_0); + + return 0; +} + +static irqreturn_t spear13xx_pcie_irq_handler(int irq, void *arg) +{ + struct spear13xx_pcie *spear13xx_pcie = arg; + struct pcie_app_reg __iomem *app_reg = spear13xx_pcie->app_base; + struct dw_pcie *pci = spear13xx_pcie->pci; + struct dw_pcie_rp *pp = &pci->pp; + unsigned int status; + + status = readl(&app_reg->int_sts); + + if (status & MSI_CTRL_INT) { + BUG_ON(!IS_ENABLED(CONFIG_PCI_MSI)); + dw_handle_msi_irq(pp); + } + + writel(status, &app_reg->int_clr); + + return IRQ_HANDLED; +} + +static void spear13xx_pcie_enable_interrupts(struct spear13xx_pcie *spear13xx_pcie) +{ + struct pcie_app_reg __iomem *app_reg = spear13xx_pcie->app_base; + + /* Enable MSI interrupt */ + if (IS_ENABLED(CONFIG_PCI_MSI)) + writel(readl(&app_reg->int_mask) | + MSI_CTRL_INT, &app_reg->int_mask); +} + +static bool spear13xx_pcie_link_up(struct dw_pcie *pci) +{ + struct spear13xx_pcie *spear13xx_pcie = to_spear13xx_pcie(pci); + struct pcie_app_reg __iomem *app_reg = spear13xx_pcie->app_base; + + return readl(&app_reg->app_status_1) & XMLH_LINK_UP; +} + +static int spear13xx_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct spear13xx_pcie *spear13xx_pcie = to_spear13xx_pcie(pci); + u32 exp_cap_off = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + u32 val; + + spear13xx_pcie->app_base = pci->dbi_base + 0x2000; + + /* + * this controller support only 128 bytes read size, however its + * default value in capability register is 512 bytes. So force + * it to 128 here. + */ + val = dw_pcie_readw_dbi(pci, exp_cap_off + PCI_EXP_DEVCTL); + val &= ~PCI_EXP_DEVCTL_READRQ; + dw_pcie_writew_dbi(pci, exp_cap_off + PCI_EXP_DEVCTL, val); + + dw_pcie_writew_dbi(pci, PCI_VENDOR_ID, 0x104A); + dw_pcie_writew_dbi(pci, PCI_DEVICE_ID, 0xCD80); + + spear13xx_pcie_enable_interrupts(spear13xx_pcie); + + return 0; +} + +static const struct dw_pcie_host_ops spear13xx_pcie_host_ops = { + .init = spear13xx_pcie_host_init, +}; + +static int spear13xx_add_pcie_port(struct spear13xx_pcie *spear13xx_pcie, + struct platform_device *pdev) +{ + struct dw_pcie *pci = spear13xx_pcie->pci; + struct dw_pcie_rp *pp = &pci->pp; + struct device *dev = &pdev->dev; + int ret; + + pp->irq = platform_get_irq(pdev, 0); + if (pp->irq < 0) + return pp->irq; + + ret = devm_request_irq(dev, pp->irq, spear13xx_pcie_irq_handler, + IRQF_SHARED | IRQF_NO_THREAD, + "spear1340-pcie", spear13xx_pcie); + if (ret) { + dev_err(dev, "failed to request irq %d\n", pp->irq); + return ret; + } + + pp->ops = &spear13xx_pcie_host_ops; + pp->msi_irq[0] = -ENODEV; + + ret = dw_pcie_host_init(pp); + if (ret) { + dev_err(dev, "failed to initialize host\n"); + return ret; + } + + return 0; +} + +static const struct dw_pcie_ops dw_pcie_ops = { + .link_up = spear13xx_pcie_link_up, + .start_link = spear13xx_pcie_start_link, +}; + +static int spear13xx_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dw_pcie *pci; + struct spear13xx_pcie *spear13xx_pcie; + struct device_node *np = dev->of_node; + int ret; + + spear13xx_pcie = devm_kzalloc(dev, sizeof(*spear13xx_pcie), GFP_KERNEL); + if (!spear13xx_pcie) + return -ENOMEM; + + pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL); + if (!pci) + return -ENOMEM; + + pci->dev = dev; + pci->ops = &dw_pcie_ops; + + spear13xx_pcie->pci = pci; + + spear13xx_pcie->phy = devm_phy_get(dev, "pcie-phy"); + if (IS_ERR(spear13xx_pcie->phy)) { + ret = PTR_ERR(spear13xx_pcie->phy); + if (ret == -EPROBE_DEFER) + dev_info(dev, "probe deferred\n"); + else + dev_err(dev, "couldn't get pcie-phy\n"); + return ret; + } + + phy_init(spear13xx_pcie->phy); + + spear13xx_pcie->clk = devm_clk_get(dev, NULL); + if (IS_ERR(spear13xx_pcie->clk)) { + dev_err(dev, "couldn't get clk for pcie\n"); + return PTR_ERR(spear13xx_pcie->clk); + } + ret = clk_prepare_enable(spear13xx_pcie->clk); + if (ret) { + dev_err(dev, "couldn't enable clk for pcie\n"); + return ret; + } + + if (of_property_read_bool(np, "st,pcie-is-gen1")) + pci->max_link_speed = 1; + + platform_set_drvdata(pdev, spear13xx_pcie); + + ret = spear13xx_add_pcie_port(spear13xx_pcie, pdev); + if (ret < 0) + goto fail_clk; + + return 0; + +fail_clk: + clk_disable_unprepare(spear13xx_pcie->clk); + + return ret; +} + +static const struct of_device_id spear13xx_pcie_of_match[] = { + { .compatible = "st,spear1340-pcie", }, + {}, +}; + +static struct platform_driver spear13xx_pcie_driver = { + .probe = spear13xx_pcie_probe, + .driver = { + .name = "spear-pcie", + .of_match_table = spear13xx_pcie_of_match, + .suppress_bind_attrs = true, + }, +}; + +builtin_platform_driver(spear13xx_pcie_driver); diff --git a/drivers/pci/controller/dwc/pcie-stm32-ep.c b/drivers/pci/controller/dwc/pcie-stm32-ep.c new file mode 100644 index 000000000..a7988dff1 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-stm32-ep.c @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * STMicroelectronics STM32MP25 PCIe endpoint driver. + * + * Copyright (C) 2025 STMicroelectronics + * Author: Christian Bruel <christian.bruel@foss.st.com> + */ + +#include <linux/clk.h> +#include <linux/gpio/consumer.h> +#include <linux/mfd/syscon.h> +#include <linux/of_platform.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/regmap.h> +#include <linux/reset.h> +#include "pcie-designware.h" +#include "pcie-stm32.h" + +struct stm32_pcie { + struct dw_pcie pci; + struct regmap *regmap; + struct reset_control *rst; + struct phy *phy; + struct clk *clk; + struct gpio_desc *perst_gpio; + unsigned int perst_irq; +}; + +static int stm32_pcie_start_link(struct dw_pcie *pci) +{ + struct stm32_pcie *stm32_pcie = to_stm32_pcie(pci); + + enable_irq(stm32_pcie->perst_irq); + + return 0; +} + +static void stm32_pcie_stop_link(struct dw_pcie *pci) +{ + struct stm32_pcie *stm32_pcie = to_stm32_pcie(pci); + + disable_irq(stm32_pcie->perst_irq); +} + +static int stm32_pcie_raise_irq(struct dw_pcie_ep *ep, u8 func_no, + unsigned int type, u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + switch (type) { + case PCI_IRQ_INTX: + return dw_pcie_ep_raise_intx_irq(ep, func_no); + case PCI_IRQ_MSI: + return dw_pcie_ep_raise_msi_irq(ep, func_no, interrupt_num); + default: + dev_err(pci->dev, "UNKNOWN IRQ type\n"); + return -EINVAL; + } +} + +static const struct pci_epc_features stm32_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, + .msi_capable = true, + .align = SZ_64K, +}; + +static const struct pci_epc_features* +stm32_pcie_get_features(struct dw_pcie_ep *ep) +{ + return &stm32_pcie_epc_features; +} + +static const struct dw_pcie_ep_ops stm32_pcie_ep_ops = { + .raise_irq = stm32_pcie_raise_irq, + .get_features = stm32_pcie_get_features, +}; + +static const struct dw_pcie_ops dw_pcie_ops = { + .start_link = stm32_pcie_start_link, + .stop_link = stm32_pcie_stop_link, +}; + +static int stm32_pcie_enable_resources(struct stm32_pcie *stm32_pcie) +{ + int ret; + + ret = phy_init(stm32_pcie->phy); + if (ret) + return ret; + + ret = clk_prepare_enable(stm32_pcie->clk); + if (ret) + phy_exit(stm32_pcie->phy); + + return ret; +} + +static void stm32_pcie_disable_resources(struct stm32_pcie *stm32_pcie) +{ + clk_disable_unprepare(stm32_pcie->clk); + + phy_exit(stm32_pcie->phy); +} + +static void stm32_pcie_perst_assert(struct dw_pcie *pci) +{ + struct stm32_pcie *stm32_pcie = to_stm32_pcie(pci); + struct dw_pcie_ep *ep = &stm32_pcie->pci.ep; + struct device *dev = pci->dev; + + dev_dbg(dev, "PERST asserted by host\n"); + + regmap_update_bits(stm32_pcie->regmap, SYSCFG_PCIECR, + STM32MP25_PCIECR_LTSSM_EN, 0); + + pci_epc_deinit_notify(ep->epc); + + stm32_pcie_disable_resources(stm32_pcie); + + pm_runtime_put_sync(dev); +} + +static void stm32_pcie_perst_deassert(struct dw_pcie *pci) +{ + struct stm32_pcie *stm32_pcie = to_stm32_pcie(pci); + struct device *dev = pci->dev; + struct dw_pcie_ep *ep = &pci->ep; + int ret; + + dev_dbg(dev, "PERST de-asserted by host\n"); + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + dev_err(dev, "Failed to resume runtime PM: %d\n", ret); + return; + } + + ret = stm32_pcie_enable_resources(stm32_pcie); + if (ret) { + dev_err(dev, "Failed to enable resources: %d\n", ret); + goto err_pm_put_sync; + } + + /* + * Reprogram the configuration space registers here because the DBI + * registers were reset by the PHY RCC during phy_init(). + */ + ret = dw_pcie_ep_init_registers(ep); + if (ret) { + dev_err(dev, "Failed to complete initialization: %d\n", ret); + goto err_disable_resources; + } + + pci_epc_init_notify(ep->epc); + + /* Enable link training */ + regmap_update_bits(stm32_pcie->regmap, SYSCFG_PCIECR, + STM32MP25_PCIECR_LTSSM_EN, + STM32MP25_PCIECR_LTSSM_EN); + + return; + +err_disable_resources: + stm32_pcie_disable_resources(stm32_pcie); + +err_pm_put_sync: + pm_runtime_put_sync(dev); +} + +static irqreturn_t stm32_pcie_ep_perst_irq_thread(int irq, void *data) +{ + struct stm32_pcie *stm32_pcie = data; + struct dw_pcie *pci = &stm32_pcie->pci; + u32 perst; + + perst = gpiod_get_value(stm32_pcie->perst_gpio); + if (perst) + stm32_pcie_perst_assert(pci); + else + stm32_pcie_perst_deassert(pci); + + irq_set_irq_type(gpiod_to_irq(stm32_pcie->perst_gpio), + (perst ? IRQF_TRIGGER_HIGH : IRQF_TRIGGER_LOW)); + + return IRQ_HANDLED; +} + +static int stm32_add_pcie_ep(struct stm32_pcie *stm32_pcie, + struct platform_device *pdev) +{ + struct dw_pcie_ep *ep = &stm32_pcie->pci.ep; + struct device *dev = &pdev->dev; + int ret; + + ret = regmap_update_bits(stm32_pcie->regmap, SYSCFG_PCIECR, + STM32MP25_PCIECR_TYPE_MASK, + STM32MP25_PCIECR_EP); + if (ret) + return ret; + + reset_control_assert(stm32_pcie->rst); + reset_control_deassert(stm32_pcie->rst); + + ep->ops = &stm32_pcie_ep_ops; + + ep->page_size = stm32_pcie_epc_features.align; + + ret = dw_pcie_ep_init(ep); + if (ret) { + dev_err(dev, "Failed to initialize ep: %d\n", ret); + return ret; + } + + ret = stm32_pcie_enable_resources(stm32_pcie); + if (ret) { + dev_err(dev, "Failed to enable resources: %d\n", ret); + dw_pcie_ep_deinit(ep); + return ret; + } + + return 0; +} + +static int stm32_pcie_probe(struct platform_device *pdev) +{ + struct stm32_pcie *stm32_pcie; + struct device *dev = &pdev->dev; + int ret; + + stm32_pcie = devm_kzalloc(dev, sizeof(*stm32_pcie), GFP_KERNEL); + if (!stm32_pcie) + return -ENOMEM; + + stm32_pcie->pci.dev = dev; + stm32_pcie->pci.ops = &dw_pcie_ops; + + stm32_pcie->regmap = syscon_regmap_lookup_by_compatible("st,stm32mp25-syscfg"); + if (IS_ERR(stm32_pcie->regmap)) + return dev_err_probe(dev, PTR_ERR(stm32_pcie->regmap), + "No syscfg specified\n"); + + stm32_pcie->phy = devm_phy_get(dev, NULL); + if (IS_ERR(stm32_pcie->phy)) + return dev_err_probe(dev, PTR_ERR(stm32_pcie->phy), + "failed to get pcie-phy\n"); + + stm32_pcie->clk = devm_clk_get(dev, NULL); + if (IS_ERR(stm32_pcie->clk)) + return dev_err_probe(dev, PTR_ERR(stm32_pcie->clk), + "Failed to get PCIe clock source\n"); + + stm32_pcie->rst = devm_reset_control_get_exclusive(dev, NULL); + if (IS_ERR(stm32_pcie->rst)) + return dev_err_probe(dev, PTR_ERR(stm32_pcie->rst), + "Failed to get PCIe reset\n"); + + stm32_pcie->perst_gpio = devm_gpiod_get(dev, "reset", GPIOD_IN); + if (IS_ERR(stm32_pcie->perst_gpio)) + return dev_err_probe(dev, PTR_ERR(stm32_pcie->perst_gpio), + "Failed to get reset GPIO\n"); + + ret = phy_set_mode(stm32_pcie->phy, PHY_MODE_PCIE); + if (ret) + return ret; + + platform_set_drvdata(pdev, stm32_pcie); + + pm_runtime_get_noresume(dev); + + ret = devm_pm_runtime_enable(dev); + if (ret < 0) { + pm_runtime_put_noidle(&pdev->dev); + return dev_err_probe(dev, ret, "Failed to enable runtime PM\n"); + } + + stm32_pcie->perst_irq = gpiod_to_irq(stm32_pcie->perst_gpio); + + /* Will be enabled in start_link when device is initialized. */ + irq_set_status_flags(stm32_pcie->perst_irq, IRQ_NOAUTOEN); + + ret = devm_request_threaded_irq(dev, stm32_pcie->perst_irq, NULL, + stm32_pcie_ep_perst_irq_thread, + IRQF_TRIGGER_HIGH | IRQF_ONESHOT, + "perst_irq", stm32_pcie); + if (ret) { + pm_runtime_put_noidle(&pdev->dev); + return dev_err_probe(dev, ret, "Failed to request PERST IRQ\n"); + } + + ret = stm32_add_pcie_ep(stm32_pcie, pdev); + if (ret) + pm_runtime_put_noidle(&pdev->dev); + + return ret; +} + +static void stm32_pcie_remove(struct platform_device *pdev) +{ + struct stm32_pcie *stm32_pcie = platform_get_drvdata(pdev); + struct dw_pcie *pci = &stm32_pcie->pci; + struct dw_pcie_ep *ep = &pci->ep; + + dw_pcie_stop_link(pci); + + pci_epc_deinit_notify(ep->epc); + dw_pcie_ep_deinit(ep); + + stm32_pcie_disable_resources(stm32_pcie); + + pm_runtime_put_sync(&pdev->dev); +} + +static const struct of_device_id stm32_pcie_ep_of_match[] = { + { .compatible = "st,stm32mp25-pcie-ep" }, + {}, +}; + +static struct platform_driver stm32_pcie_ep_driver = { + .probe = stm32_pcie_probe, + .remove = stm32_pcie_remove, + .driver = { + .name = "stm32-ep-pcie", + .of_match_table = stm32_pcie_ep_of_match, + }, +}; + +module_platform_driver(stm32_pcie_ep_driver); + +MODULE_AUTHOR("Christian Bruel <christian.bruel@foss.st.com>"); +MODULE_DESCRIPTION("STM32MP25 PCIe Endpoint Controller driver"); +MODULE_LICENSE("GPL"); +MODULE_DEVICE_TABLE(of, stm32_pcie_ep_of_match); diff --git a/drivers/pci/controller/dwc/pcie-stm32.c b/drivers/pci/controller/dwc/pcie-stm32.c new file mode 100644 index 000000000..349618ea5 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-stm32.c @@ -0,0 +1,369 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * STMicroelectronics STM32MP25 PCIe root complex driver. + * + * Copyright (C) 2025 STMicroelectronics + * Author: Christian Bruel <christian.bruel@foss.st.com> + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/gpio/consumer.h> +#include <linux/irq.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_platform.h> +#include <linux/phy/phy.h> +#include <linux/pinctrl/consumer.h> +#include <linux/platform_device.h> +#include <linux/pm.h> +#include <linux/pm_runtime.h> +#include <linux/pm_wakeirq.h> +#include <linux/regmap.h> +#include <linux/reset.h> +#include <linux/stddef.h> + +#include "../../pci.h" + +#include "pcie-designware.h" +#include "pcie-stm32.h" + +struct stm32_pcie { + struct dw_pcie pci; + struct regmap *regmap; + struct reset_control *rst; + struct phy *phy; + struct clk *clk; + struct gpio_desc *perst_gpio; + struct gpio_desc *wake_gpio; +}; + +static void stm32_pcie_deassert_perst(struct stm32_pcie *stm32_pcie) +{ + if (stm32_pcie->perst_gpio) { + msleep(PCIE_T_PVPERL_MS); + gpiod_set_value(stm32_pcie->perst_gpio, 0); + } + + msleep(PCIE_RESET_CONFIG_WAIT_MS); +} + +static void stm32_pcie_assert_perst(struct stm32_pcie *stm32_pcie) +{ + gpiod_set_value(stm32_pcie->perst_gpio, 1); +} + +static int stm32_pcie_start_link(struct dw_pcie *pci) +{ + struct stm32_pcie *stm32_pcie = to_stm32_pcie(pci); + + return regmap_update_bits(stm32_pcie->regmap, SYSCFG_PCIECR, + STM32MP25_PCIECR_LTSSM_EN, + STM32MP25_PCIECR_LTSSM_EN); +} + +static void stm32_pcie_stop_link(struct dw_pcie *pci) +{ + struct stm32_pcie *stm32_pcie = to_stm32_pcie(pci); + + regmap_update_bits(stm32_pcie->regmap, SYSCFG_PCIECR, + STM32MP25_PCIECR_LTSSM_EN, 0); +} + +static int stm32_pcie_suspend_noirq(struct device *dev) +{ + struct stm32_pcie *stm32_pcie = dev_get_drvdata(dev); + int ret; + + ret = dw_pcie_suspend_noirq(&stm32_pcie->pci); + if (ret) + return ret; + + stm32_pcie_assert_perst(stm32_pcie); + + clk_disable_unprepare(stm32_pcie->clk); + + if (!device_wakeup_path(dev)) + phy_exit(stm32_pcie->phy); + + return pinctrl_pm_select_sleep_state(dev); +} + +static int stm32_pcie_resume_noirq(struct device *dev) +{ + struct stm32_pcie *stm32_pcie = dev_get_drvdata(dev); + int ret; + + /* + * The core clock is gated with CLKREQ# from the COMBOPHY REFCLK, + * thus if no device is present, must deassert it with a GPIO from + * pinctrl pinmux before accessing the DBI registers. + */ + ret = pinctrl_pm_select_init_state(dev); + if (ret) { + dev_err(dev, "Failed to activate pinctrl pm state: %d\n", ret); + return ret; + } + + if (!device_wakeup_path(dev)) { + ret = phy_init(stm32_pcie->phy); + if (ret) { + pinctrl_pm_select_default_state(dev); + return ret; + } + } + + ret = clk_prepare_enable(stm32_pcie->clk); + if (ret) + goto err_phy_exit; + + stm32_pcie_deassert_perst(stm32_pcie); + + ret = dw_pcie_resume_noirq(&stm32_pcie->pci); + if (ret) + goto err_disable_clk; + + pinctrl_pm_select_default_state(dev); + + return 0; + +err_disable_clk: + stm32_pcie_assert_perst(stm32_pcie); + clk_disable_unprepare(stm32_pcie->clk); + +err_phy_exit: + phy_exit(stm32_pcie->phy); + pinctrl_pm_select_default_state(dev); + + return ret; +} + +static const struct dev_pm_ops stm32_pcie_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(stm32_pcie_suspend_noirq, + stm32_pcie_resume_noirq) +}; + +static const struct dw_pcie_host_ops stm32_pcie_host_ops = { +}; + +static const struct dw_pcie_ops dw_pcie_ops = { + .start_link = stm32_pcie_start_link, + .stop_link = stm32_pcie_stop_link +}; + +static int stm32_add_pcie_port(struct stm32_pcie *stm32_pcie) +{ + struct device *dev = stm32_pcie->pci.dev; + unsigned int wake_irq; + int ret; + + ret = phy_set_mode(stm32_pcie->phy, PHY_MODE_PCIE); + if (ret) + return ret; + + ret = phy_init(stm32_pcie->phy); + if (ret) + return ret; + + ret = regmap_update_bits(stm32_pcie->regmap, SYSCFG_PCIECR, + STM32MP25_PCIECR_TYPE_MASK, + STM32MP25_PCIECR_RC); + if (ret) + goto err_phy_exit; + + stm32_pcie_deassert_perst(stm32_pcie); + + if (stm32_pcie->wake_gpio) { + wake_irq = gpiod_to_irq(stm32_pcie->wake_gpio); + ret = dev_pm_set_dedicated_wake_irq(dev, wake_irq); + if (ret) { + dev_err(dev, "Failed to enable wakeup irq %d\n", ret); + goto err_assert_perst; + } + irq_set_irq_type(wake_irq, IRQ_TYPE_EDGE_FALLING); + } + + return 0; + +err_assert_perst: + stm32_pcie_assert_perst(stm32_pcie); + +err_phy_exit: + phy_exit(stm32_pcie->phy); + + return ret; +} + +static void stm32_remove_pcie_port(struct stm32_pcie *stm32_pcie) +{ + dev_pm_clear_wake_irq(stm32_pcie->pci.dev); + + stm32_pcie_assert_perst(stm32_pcie); + + phy_exit(stm32_pcie->phy); +} + +static int stm32_pcie_parse_port(struct stm32_pcie *stm32_pcie) +{ + struct device *dev = stm32_pcie->pci.dev; + struct device_node *root_port; + + root_port = of_get_next_available_child(dev->of_node, NULL); + + stm32_pcie->phy = devm_of_phy_get(dev, root_port, NULL); + if (IS_ERR(stm32_pcie->phy)) { + of_node_put(root_port); + return dev_err_probe(dev, PTR_ERR(stm32_pcie->phy), + "Failed to get pcie-phy\n"); + } + + stm32_pcie->perst_gpio = devm_fwnode_gpiod_get(dev, of_fwnode_handle(root_port), + "reset", GPIOD_OUT_HIGH, NULL); + if (IS_ERR(stm32_pcie->perst_gpio)) { + if (PTR_ERR(stm32_pcie->perst_gpio) != -ENOENT) { + of_node_put(root_port); + return dev_err_probe(dev, PTR_ERR(stm32_pcie->perst_gpio), + "Failed to get reset GPIO\n"); + } + stm32_pcie->perst_gpio = NULL; + } + + stm32_pcie->wake_gpio = devm_fwnode_gpiod_get(dev, of_fwnode_handle(root_port), + "wake", GPIOD_IN, NULL); + + if (IS_ERR(stm32_pcie->wake_gpio)) { + if (PTR_ERR(stm32_pcie->wake_gpio) != -ENOENT) { + of_node_put(root_port); + return dev_err_probe(dev, PTR_ERR(stm32_pcie->wake_gpio), + "Failed to get wake GPIO\n"); + } + stm32_pcie->wake_gpio = NULL; + } + + of_node_put(root_port); + + return 0; +} + +static int stm32_pcie_probe(struct platform_device *pdev) +{ + struct stm32_pcie *stm32_pcie; + struct device *dev = &pdev->dev; + int ret; + + stm32_pcie = devm_kzalloc(dev, sizeof(*stm32_pcie), GFP_KERNEL); + if (!stm32_pcie) + return -ENOMEM; + + stm32_pcie->pci.dev = dev; + stm32_pcie->pci.ops = &dw_pcie_ops; + stm32_pcie->pci.pp.ops = &stm32_pcie_host_ops; + + stm32_pcie->regmap = syscon_regmap_lookup_by_compatible("st,stm32mp25-syscfg"); + if (IS_ERR(stm32_pcie->regmap)) + return dev_err_probe(dev, PTR_ERR(stm32_pcie->regmap), + "No syscfg specified\n"); + + stm32_pcie->clk = devm_clk_get(dev, NULL); + if (IS_ERR(stm32_pcie->clk)) + return dev_err_probe(dev, PTR_ERR(stm32_pcie->clk), + "Failed to get PCIe clock source\n"); + + stm32_pcie->rst = devm_reset_control_get_exclusive(dev, NULL); + if (IS_ERR(stm32_pcie->rst)) + return dev_err_probe(dev, PTR_ERR(stm32_pcie->rst), + "Failed to get PCIe reset\n"); + + ret = stm32_pcie_parse_port(stm32_pcie); + if (ret) + return ret; + + platform_set_drvdata(pdev, stm32_pcie); + + ret = stm32_add_pcie_port(stm32_pcie); + if (ret) + return ret; + + reset_control_assert(stm32_pcie->rst); + reset_control_deassert(stm32_pcie->rst); + + ret = clk_prepare_enable(stm32_pcie->clk); + if (ret) { + dev_err(dev, "Core clock enable failed %d\n", ret); + goto err_remove_port; + } + + ret = pm_runtime_set_active(dev); + if (ret < 0) { + dev_err_probe(dev, ret, "Failed to activate runtime PM\n"); + goto err_disable_clk; + } + + pm_runtime_no_callbacks(dev); + + ret = devm_pm_runtime_enable(dev); + if (ret < 0) { + dev_err_probe(dev, ret, "Failed to enable runtime PM\n"); + goto err_disable_clk; + } + + ret = dw_pcie_host_init(&stm32_pcie->pci.pp); + if (ret) + goto err_disable_clk; + + if (stm32_pcie->wake_gpio) + device_init_wakeup(dev, true); + + return 0; + +err_disable_clk: + clk_disable_unprepare(stm32_pcie->clk); + +err_remove_port: + stm32_remove_pcie_port(stm32_pcie); + + return ret; +} + +static void stm32_pcie_remove(struct platform_device *pdev) +{ + struct stm32_pcie *stm32_pcie = platform_get_drvdata(pdev); + struct dw_pcie_rp *pp = &stm32_pcie->pci.pp; + + if (stm32_pcie->wake_gpio) + device_init_wakeup(&pdev->dev, false); + + dw_pcie_host_deinit(pp); + + clk_disable_unprepare(stm32_pcie->clk); + + stm32_remove_pcie_port(stm32_pcie); + + pm_runtime_put_noidle(&pdev->dev); +} + +static const struct of_device_id stm32_pcie_of_match[] = { + { .compatible = "st,stm32mp25-pcie-rc" }, + {}, +}; + +static struct platform_driver stm32_pcie_driver = { + .probe = stm32_pcie_probe, + .remove = stm32_pcie_remove, + .driver = { + .name = "stm32-pcie", + .of_match_table = stm32_pcie_of_match, + .pm = &stm32_pcie_pm_ops, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, +}; + +module_platform_driver(stm32_pcie_driver); + +MODULE_AUTHOR("Christian Bruel <christian.bruel@foss.st.com>"); +MODULE_DESCRIPTION("STM32MP25 PCIe Controller driver"); +MODULE_LICENSE("GPL"); +MODULE_DEVICE_TABLE(of, stm32_pcie_of_match); diff --git a/drivers/pci/controller/dwc/pcie-stm32.h b/drivers/pci/controller/dwc/pcie-stm32.h new file mode 100644 index 000000000..419cf1ff6 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-stm32.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * ST PCIe driver definitions for STM32-MP25 SoC + * + * Copyright (C) 2025 STMicroelectronics - All Rights Reserved + * Author: Christian Bruel <christian.bruel@foss.st.com> + */ + +#include <linux/bits.h> +#include <linux/device.h> + +#define to_stm32_pcie(x) dev_get_drvdata((x)->dev) + +#define STM32MP25_PCIECR_TYPE_MASK GENMASK(11, 8) +#define STM32MP25_PCIECR_EP 0 +#define STM32MP25_PCIECR_LTSSM_EN BIT(2) +#define STM32MP25_PCIECR_RC BIT(10) + +#define SYSCFG_PCIECR 0x6000 diff --git a/drivers/pci/controller/dwc/pcie-tegra194-acpi.c b/drivers/pci/controller/dwc/pcie-tegra194-acpi.c new file mode 100644 index 000000000..2d737b49e --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-tegra194-acpi.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * ACPI quirks for Tegra194 PCIe host controller + * + * Copyright (C) 2021 NVIDIA Corporation. + * + * Author: Vidya Sagar <vidyas@nvidia.com> + */ + +#include <linux/pci.h> +#include <linux/pci-acpi.h> +#include <linux/pci-ecam.h> + +#include "pcie-designware.h" + +struct tegra194_pcie_ecam { + void __iomem *config_base; + void __iomem *iatu_base; + void __iomem *dbi_base; +}; + +static int tegra194_acpi_init(struct pci_config_window *cfg) +{ + struct device *dev = cfg->parent; + struct tegra194_pcie_ecam *pcie_ecam; + + pcie_ecam = devm_kzalloc(dev, sizeof(*pcie_ecam), GFP_KERNEL); + if (!pcie_ecam) + return -ENOMEM; + + pcie_ecam->config_base = cfg->win; + pcie_ecam->iatu_base = cfg->win + SZ_256K; + pcie_ecam->dbi_base = cfg->win + SZ_512K; + cfg->priv = pcie_ecam; + + return 0; +} + +static void atu_reg_write(struct tegra194_pcie_ecam *pcie_ecam, int index, + u32 val, u32 reg) +{ + u32 offset = PCIE_ATU_UNROLL_BASE(PCIE_ATU_REGION_DIR_OB, index) + + PCIE_ATU_VIEWPORT_BASE; + + writel(val, pcie_ecam->iatu_base + offset + reg); +} + +static void program_outbound_atu(struct tegra194_pcie_ecam *pcie_ecam, + int index, int type, u64 cpu_addr, + u64 pci_addr, u64 size) +{ + atu_reg_write(pcie_ecam, index, lower_32_bits(cpu_addr), + PCIE_ATU_LOWER_BASE); + atu_reg_write(pcie_ecam, index, upper_32_bits(cpu_addr), + PCIE_ATU_UPPER_BASE); + atu_reg_write(pcie_ecam, index, lower_32_bits(pci_addr), + PCIE_ATU_LOWER_TARGET); + atu_reg_write(pcie_ecam, index, lower_32_bits(cpu_addr + size - 1), + PCIE_ATU_LIMIT); + atu_reg_write(pcie_ecam, index, upper_32_bits(pci_addr), + PCIE_ATU_UPPER_TARGET); + atu_reg_write(pcie_ecam, index, type, PCIE_ATU_REGION_CTRL1); + atu_reg_write(pcie_ecam, index, PCIE_ATU_ENABLE, PCIE_ATU_REGION_CTRL2); +} + +static void __iomem *tegra194_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + struct pci_config_window *cfg = bus->sysdata; + struct tegra194_pcie_ecam *pcie_ecam = cfg->priv; + u32 busdev; + int type; + + if (bus->number < cfg->busr.start || bus->number > cfg->busr.end) + return NULL; + + if (bus->number == cfg->busr.start) { + if (PCI_SLOT(devfn) == 0) + return pcie_ecam->dbi_base + where; + else + return NULL; + } + + busdev = PCIE_ATU_BUS(bus->number) | PCIE_ATU_DEV(PCI_SLOT(devfn)) | + PCIE_ATU_FUNC(PCI_FUNC(devfn)); + + if (bus->parent->number == cfg->busr.start) { + if (PCI_SLOT(devfn) == 0) + type = PCIE_TLP_TYPE_CFG0_RDWR; + else + return NULL; + } else { + type = PCIE_TLP_TYPE_CFG1_RDWR; + } + + program_outbound_atu(pcie_ecam, 0, type, cfg->res.start, busdev, + SZ_256K); + + return pcie_ecam->config_base + where; +} + +const struct pci_ecam_ops tegra194_pcie_ops = { + .init = tegra194_acpi_init, + .pci_ops = { + .map_bus = tegra194_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, + } +}; diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c new file mode 100644 index 000000000..63a0f56ca --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-tegra194.c @@ -0,0 +1,2589 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * PCIe host controller driver for the following SoCs + * Tegra194 + * Tegra234 + * + * Copyright (C) 2019-2022 NVIDIA Corporation. + * + * Author: Vidya Sagar <vidyas@nvidia.com> + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/debugfs.h> +#include <linux/delay.h> +#include <linux/gpio/consumer.h> +#include <linux/interconnect.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/phy/phy.h> +#include <linux/pinctrl/consumer.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/random.h> +#include <linux/reset.h> +#include <linux/resource.h> +#include <linux/types.h> +#include "pcie-designware.h" +#include <soc/tegra/bpmp.h> +#include <soc/tegra/bpmp-abi.h> +#include "../../pci.h" + +#define TEGRA194_DWC_IP_VER DW_PCIE_VER_500A +#define TEGRA234_DWC_IP_VER DW_PCIE_VER_562A + +#define APPL_PINMUX 0x0 +#define APPL_PINMUX_PEX_RST BIT(0) +#define APPL_PINMUX_CLKREQ_OVERRIDE_EN BIT(2) +#define APPL_PINMUX_CLKREQ_OVERRIDE BIT(3) +#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN BIT(4) +#define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE BIT(5) +#define APPL_PINMUX_CLKREQ_DEFAULT_VALUE BIT(13) + +#define APPL_CTRL 0x4 +#define APPL_CTRL_SYS_PRE_DET_STATE BIT(6) +#define APPL_CTRL_LTSSM_EN BIT(7) +#define APPL_CTRL_HW_HOT_RST_EN BIT(20) +#define APPL_CTRL_HW_HOT_RST_MODE_MASK GENMASK(1, 0) +#define APPL_CTRL_HW_HOT_RST_MODE_SHIFT 22 +#define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST 0x1 +#define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN 0x2 + +#define APPL_INTR_EN_L0_0 0x8 +#define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN BIT(0) +#define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN BIT(4) +#define APPL_INTR_EN_L0_0_INT_INT_EN BIT(8) +#define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN BIT(15) +#define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN BIT(19) +#define APPL_INTR_EN_L0_0_SYS_INTR_EN BIT(30) +#define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN BIT(31) + +#define APPL_INTR_STATUS_L0 0xC +#define APPL_INTR_STATUS_L0_LINK_STATE_INT BIT(0) +#define APPL_INTR_STATUS_L0_INT_INT BIT(8) +#define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT BIT(15) +#define APPL_INTR_STATUS_L0_PEX_RST_INT BIT(16) +#define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT BIT(18) + +#define APPL_INTR_EN_L1_0_0 0x1C +#define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN BIT(1) +#define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN BIT(3) +#define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN BIT(30) + +#define APPL_INTR_STATUS_L1_0_0 0x20 +#define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED BIT(1) +#define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED BIT(3) +#define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE BIT(30) + +#define APPL_INTR_STATUS_L1_1 0x2C +#define APPL_INTR_STATUS_L1_2 0x30 +#define APPL_INTR_STATUS_L1_3 0x34 +#define APPL_INTR_STATUS_L1_6 0x3C +#define APPL_INTR_STATUS_L1_7 0x40 +#define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED BIT(1) + +#define APPL_INTR_EN_L1_8_0 0x44 +#define APPL_INTR_EN_L1_8_BW_MGT_INT_EN BIT(2) +#define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN BIT(3) +#define APPL_INTR_EN_L1_8_EDMA_INT_EN BIT(6) +#define APPL_INTR_EN_L1_8_INTX_EN BIT(11) +#define APPL_INTR_EN_L1_8_AER_INT_EN BIT(15) + +#define APPL_INTR_STATUS_L1_8_0 0x4C +#define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK GENMASK(11, 6) +#define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS BIT(2) +#define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS BIT(3) + +#define APPL_INTR_STATUS_L1_9 0x54 +#define APPL_INTR_STATUS_L1_10 0x58 +#define APPL_INTR_STATUS_L1_11 0x64 +#define APPL_INTR_STATUS_L1_13 0x74 +#define APPL_INTR_STATUS_L1_14 0x78 +#define APPL_INTR_STATUS_L1_15 0x7C +#define APPL_INTR_STATUS_L1_17 0x88 + +#define APPL_INTR_EN_L1_18 0x90 +#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT BIT(2) +#define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR BIT(1) +#define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0) + +#define APPL_INTR_STATUS_L1_18 0x94 +#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT BIT(2) +#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR BIT(1) +#define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0) + +#define APPL_MSI_CTRL_1 0xAC + +#define APPL_MSI_CTRL_2 0xB0 + +#define APPL_LEGACY_INTX 0xB8 + +#define APPL_LTR_MSG_1 0xC4 +#define LTR_MSG_REQ BIT(15) +#define LTR_NOSNOOP_MSG_REQ BIT(31) + +#define APPL_LTR_MSG_2 0xC8 +#define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE BIT(3) + +#define APPL_LINK_STATUS 0xCC +#define APPL_LINK_STATUS_RDLH_LINK_UP BIT(0) + +#define APPL_DEBUG 0xD0 +#define APPL_DEBUG_PM_LINKST_IN_L2_LAT BIT(21) +#define APPL_DEBUG_PM_LINKST_IN_L0 0x11 +#define APPL_DEBUG_LTSSM_STATE_MASK GENMASK(8, 3) +#define APPL_DEBUG_LTSSM_STATE_SHIFT 3 +#define LTSSM_STATE_DETECT_QUIET 0x00 +#define LTSSM_STATE_DETECT_ACT 0x08 +#define LTSSM_STATE_PRE_DETECT_QUIET 0x28 +#define LTSSM_STATE_DETECT_WAIT 0x30 +#define LTSSM_STATE_L2_IDLE 0xa8 + +#define APPL_RADM_STATUS 0xE4 +#define APPL_PM_XMT_TURNOFF_STATE BIT(0) + +#define APPL_DM_TYPE 0x100 +#define APPL_DM_TYPE_MASK GENMASK(3, 0) +#define APPL_DM_TYPE_RP 0x4 +#define APPL_DM_TYPE_EP 0x0 + +#define APPL_CFG_BASE_ADDR 0x104 +#define APPL_CFG_BASE_ADDR_MASK GENMASK(31, 12) + +#define APPL_CFG_IATU_DMA_BASE_ADDR 0x108 +#define APPL_CFG_IATU_DMA_BASE_ADDR_MASK GENMASK(31, 18) + +#define APPL_CFG_MISC 0x110 +#define APPL_CFG_MISC_SLV_EP_MODE BIT(14) +#define APPL_CFG_MISC_ARCACHE_MASK GENMASK(13, 10) +#define APPL_CFG_MISC_ARCACHE_SHIFT 10 +#define APPL_CFG_MISC_ARCACHE_VAL 3 + +#define APPL_CFG_SLCG_OVERRIDE 0x114 +#define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER BIT(0) + +#define APPL_CAR_RESET_OVRD 0x12C +#define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N BIT(0) + +#define IO_BASE_IO_DECODE BIT(0) +#define IO_BASE_IO_DECODE_BIT8 BIT(8) + +#define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE BIT(0) +#define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE BIT(16) + +#define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF 0x718 +#define CFG_TIMER_CTRL_ACK_NAK_SHIFT (19) + +#define N_FTS_VAL 52 +#define FTS_VAL 52 + +#define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0 +#define AMBA_ERROR_RESPONSE_RRS_SHIFT 3 +#define AMBA_ERROR_RESPONSE_RRS_MASK GENMASK(1, 0) +#define AMBA_ERROR_RESPONSE_RRS_OKAY 0 +#define AMBA_ERROR_RESPONSE_RRS_OKAY_FFFFFFFF 1 +#define AMBA_ERROR_RESPONSE_RRS_OKAY_FFFF0001 2 + +#define MSIX_ADDR_MATCH_LOW_OFF 0x940 +#define MSIX_ADDR_MATCH_LOW_OFF_EN BIT(0) +#define MSIX_ADDR_MATCH_LOW_OFF_MASK GENMASK(31, 2) + +#define MSIX_ADDR_MATCH_HIGH_OFF 0x944 +#define MSIX_ADDR_MATCH_HIGH_OFF_MASK GENMASK(31, 0) + +#define PORT_LOGIC_MSIX_DOORBELL 0x948 + +#define CAP_SPCIE_CAP_OFF 0x154 +#define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK GENMASK(3, 0) +#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK GENMASK(11, 8) +#define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT 8 + +#define LTSSM_DELAY_US 10000 /* 10 ms */ +#define LTSSM_TIMEOUT_US 120000 /* 120 ms */ + +#define GEN3_GEN4_EQ_PRESET_INIT 5 + +#define GEN1_CORE_CLK_FREQ 62500000 +#define GEN2_CORE_CLK_FREQ 125000000 +#define GEN3_CORE_CLK_FREQ 250000000 +#define GEN4_CORE_CLK_FREQ 500000000 + +#define LTR_MSG_TIMEOUT (100 * 1000) + +#define PERST_DEBOUNCE_TIME (5 * 1000) + +#define EP_STATE_DISABLED 0 +#define EP_STATE_ENABLED 1 + +static const unsigned int pcie_gen_freq[] = { + GEN1_CORE_CLK_FREQ, /* PCI_EXP_LNKSTA_CLS == 0; undefined */ + GEN1_CORE_CLK_FREQ, + GEN2_CORE_CLK_FREQ, + GEN3_CORE_CLK_FREQ, + GEN4_CORE_CLK_FREQ +}; + +struct tegra_pcie_dw_of_data { + u32 version; + enum dw_pcie_device_mode mode; + bool has_msix_doorbell_access_fix; + bool has_sbr_reset_fix; + bool has_l1ss_exit_fix; + bool has_ltr_req_fix; + bool disable_l1_2; + u32 cdm_chk_int_en_bit; + u32 gen4_preset_vec; + u8 n_fts[2]; +}; + +struct tegra_pcie_dw { + struct device *dev; + struct resource *appl_res; + struct resource *dbi_res; + struct resource *atu_dma_res; + void __iomem *appl_base; + struct clk *core_clk; + struct clk *core_clk_m; + struct reset_control *core_apb_rst; + struct reset_control *core_rst; + struct dw_pcie pci; + struct tegra_bpmp *bpmp; + + struct tegra_pcie_dw_of_data *of_data; + + bool supports_clkreq; + bool enable_cdm_check; + bool enable_srns; + bool link_state; + bool update_fc_fixup; + bool enable_ext_refclk; + u8 init_link_width; + u32 msi_ctrl_int; + u32 num_lanes; + u32 cid; + u32 ras_des_cap; + u32 pcie_cap_base; + u32 aspm_cmrt; + u32 aspm_pwr_on_t; + u32 aspm_l0s_enter_lat; + u32 aspm_l1_enter_lat; + + struct regulator *pex_ctl_supply; + struct regulator *slot_ctl_3v3; + struct regulator *slot_ctl_12v; + + unsigned int phy_count; + struct phy **phys; + + struct dentry *debugfs; + + /* Endpoint mode specific */ + struct gpio_desc *pex_rst_gpiod; + struct gpio_desc *pex_refclk_sel_gpiod; + unsigned int pex_rst_irq; + int ep_state; + long link_status; + struct icc_path *icc_path; +}; + +static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci) +{ + return container_of(pci, struct tegra_pcie_dw, pci); +} + +static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value, + const u32 reg) +{ + writel_relaxed(value, pcie->appl_base + reg); +} + +static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg) +{ + return readl_relaxed(pcie->appl_base + reg); +} + +static void tegra_pcie_icc_set(struct tegra_pcie_dw *pcie) +{ + struct dw_pcie *pci = &pcie->pci; + u32 val, speed, width; + + val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); + + speed = FIELD_GET(PCI_EXP_LNKSTA_CLS, val); + width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val); + + val = width * PCIE_SPEED2MBS_ENC(pcie_get_link_speed(speed)); + + if (icc_set_bw(pcie->icc_path, Mbps_to_icc(val), 0)) + dev_err(pcie->dev, "can't set bw[%u]\n", val); + + if (speed >= ARRAY_SIZE(pcie_gen_freq)) + speed = 0; + + clk_set_rate(pcie->core_clk, pcie_gen_freq[speed]); +} + +static void apply_bad_link_workaround(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + u32 current_link_width; + u16 val; + + /* + * NOTE:- Since this scenario is uncommon and link as such is not + * stable anyway, not waiting to confirm if link is really + * transitioning to Gen-2 speed + */ + val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); + if (val & PCI_EXP_LNKSTA_LBMS) { + current_link_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val); + if (pcie->init_link_width > current_link_width) { + dev_warn(pci->dev, "PCIe link is bad, width reduced\n"); + val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + + PCI_EXP_LNKCTL2); + val &= ~PCI_EXP_LNKCTL2_TLS; + val |= PCI_EXP_LNKCTL2_TLS_2_5GT; + dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + + PCI_EXP_LNKCTL2, val); + + val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + + PCI_EXP_LNKCTL); + val |= PCI_EXP_LNKCTL_RL; + dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + + PCI_EXP_LNKCTL, val); + } + } +} + +static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg) +{ + struct tegra_pcie_dw *pcie = arg; + struct dw_pcie *pci = &pcie->pci; + struct dw_pcie_rp *pp = &pci->pp; + u32 val, status_l0, status_l1; + u16 val_w; + + status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0); + if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) { + status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); + appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0); + if (!pcie->of_data->has_sbr_reset_fix && + status_l1 & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) { + /* SBR & Surprise Link Down WAR */ + val = appl_readl(pcie, APPL_CAR_RESET_OVRD); + val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N; + appl_writel(pcie, val, APPL_CAR_RESET_OVRD); + udelay(1); + val = appl_readl(pcie, APPL_CAR_RESET_OVRD); + val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N; + appl_writel(pcie, val, APPL_CAR_RESET_OVRD); + + val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); + val |= PORT_LOGIC_SPEED_CHANGE; + dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); + } + } + + if (status_l0 & APPL_INTR_STATUS_L0_INT_INT) { + status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0); + if (status_l1 & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) { + appl_writel(pcie, + APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS, + APPL_INTR_STATUS_L1_8_0); + apply_bad_link_workaround(pp); + } + if (status_l1 & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) { + val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + + PCI_EXP_LNKSTA); + val_w |= PCI_EXP_LNKSTA_LBMS; + dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + + PCI_EXP_LNKSTA, val_w); + + appl_writel(pcie, + APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS, + APPL_INTR_STATUS_L1_8_0); + + val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + + PCI_EXP_LNKSTA); + dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w & + PCI_EXP_LNKSTA_CLS); + } + } + + if (status_l0 & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) { + status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_18); + val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS); + if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) { + dev_info(pci->dev, "CDM check complete\n"); + val |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE; + } + if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) { + dev_err(pci->dev, "CDM comparison mismatch\n"); + val |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR; + } + if (status_l1 & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) { + dev_err(pci->dev, "CDM Logic error\n"); + val |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR; + } + dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, val); + val = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR); + dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", val); + } + + return IRQ_HANDLED; +} + +static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie) +{ + u32 val; + + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); + appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2); + + val = appl_readl(pcie, APPL_CTRL); + val |= APPL_CTRL_LTSSM_EN; + appl_writel(pcie, val, APPL_CTRL); +} + +static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg) +{ + struct tegra_pcie_dw *pcie = arg; + struct dw_pcie_ep *ep = &pcie->pci.ep; + struct dw_pcie *pci = &pcie->pci; + u32 val; + + if (test_and_clear_bit(0, &pcie->link_status)) + dw_pcie_ep_linkup(ep); + + tegra_pcie_icc_set(pcie); + + if (pcie->of_data->has_ltr_req_fix) + return IRQ_HANDLED; + + /* If EP doesn't advertise L1SS, just return */ + if (!pci->l1ss_support) + return IRQ_HANDLED; + + /* Check if BME is set to '1' */ + val = dw_pcie_readl_dbi(pci, PCI_COMMAND); + if (val & PCI_COMMAND_MASTER) { + ktime_t timeout; + + /* Send LTR upstream */ + val = appl_readl(pcie, APPL_LTR_MSG_2); + val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE; + appl_writel(pcie, val, APPL_LTR_MSG_2); + + timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT); + for (;;) { + val = appl_readl(pcie, APPL_LTR_MSG_2); + if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)) + break; + if (ktime_after(ktime_get(), timeout)) + break; + usleep_range(1000, 1100); + } + if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE) + dev_err(pcie->dev, "Failed to send LTR message\n"); + } + + return IRQ_HANDLED; +} + +static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg) +{ + struct tegra_pcie_dw *pcie = arg; + int spurious = 1; + u32 status_l0, status_l1, link_status; + + status_l0 = appl_readl(pcie, APPL_INTR_STATUS_L0); + if (status_l0 & APPL_INTR_STATUS_L0_LINK_STATE_INT) { + status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0); + appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_0_0); + + if (status_l1 & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE) + pex_ep_event_hot_rst_done(pcie); + + if (status_l1 & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) { + link_status = appl_readl(pcie, APPL_LINK_STATUS); + if (link_status & APPL_LINK_STATUS_RDLH_LINK_UP) { + dev_dbg(pcie->dev, "Link is up with Host\n"); + set_bit(0, &pcie->link_status); + return IRQ_WAKE_THREAD; + } + } + + spurious = 0; + } + + if (status_l0 & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) { + status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_15); + appl_writel(pcie, status_l1, APPL_INTR_STATUS_L1_15); + + if (status_l1 & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED) + return IRQ_WAKE_THREAD; + + spurious = 0; + } + + if (status_l0 & APPL_INTR_STATUS_L0_INT_INT) { + status_l1 = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0); + + /* + * Interrupt is handled by DMA driver; don't treat it as + * spurious + */ + if (status_l1 & APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK) + spurious = 0; + } + + if (spurious) { + dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n", + status_l0); + appl_writel(pcie, status_l0, APPL_INTR_STATUS_L0); + } + + return IRQ_HANDLED; +} + +static int tegra_pcie_dw_rd_own_conf(struct pci_bus *bus, u32 devfn, int where, + int size, u32 *val) +{ + struct dw_pcie_rp *pp = bus->sysdata; + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + + /* + * This is an endpoint mode specific register happen to appear even + * when controller is operating in root port mode and system hangs + * when it is accessed with link being in ASPM-L1 state. + * So skip accessing it altogether + */ + if (!pcie->of_data->has_msix_doorbell_access_fix && + !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) { + *val = 0x00000000; + return PCIBIOS_SUCCESSFUL; + } + + return pci_generic_config_read(bus, devfn, where, size, val); +} + +static int tegra_pcie_dw_wr_own_conf(struct pci_bus *bus, u32 devfn, int where, + int size, u32 val) +{ + struct dw_pcie_rp *pp = bus->sysdata; + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + + /* + * This is an endpoint mode specific register happen to appear even + * when controller is operating in root port mode and system hangs + * when it is accessed with link being in ASPM-L1 state. + * So skip accessing it altogether + */ + if (!pcie->of_data->has_msix_doorbell_access_fix && + !PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) + return PCIBIOS_SUCCESSFUL; + + return pci_generic_config_write(bus, devfn, where, size, val); +} + +static struct pci_ops tegra_pci_ops = { + .map_bus = dw_pcie_own_conf_map_bus, + .read = tegra_pcie_dw_rd_own_conf, + .write = tegra_pcie_dw_wr_own_conf, +}; + +#if defined(CONFIG_PCIEASPM) +static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event) +{ + u32 val; + + val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap + + PCIE_RAS_DES_EVENT_COUNTER_CONTROL); + val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT); + val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; + val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT; + val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; + dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap + + PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val); + val = dw_pcie_readl_dbi(&pcie->pci, pcie->ras_des_cap + + PCIE_RAS_DES_EVENT_COUNTER_DATA); + + return val; +} + +static int aspm_state_cnt(struct seq_file *s, void *data) +{ + struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *) + dev_get_drvdata(s->private); + u32 val; + + seq_printf(s, "Tx L0s entry count : %u\n", + event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S)); + + seq_printf(s, "Rx L0s entry count : %u\n", + event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S)); + + seq_printf(s, "Link L1 entry count : %u\n", + event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1)); + + seq_printf(s, "Link L1.1 entry count : %u\n", + event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1)); + + seq_printf(s, "Link L1.2 entry count : %u\n", + event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2)); + + /* Clear all counters */ + dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap + + PCIE_RAS_DES_EVENT_COUNTER_CONTROL, + EVENT_COUNTER_ALL_CLEAR); + + /* Re-enable counting */ + val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; + val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; + dw_pcie_writel_dbi(&pcie->pci, pcie->ras_des_cap + + PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val); + + return 0; +} + +static void init_host_aspm(struct tegra_pcie_dw *pcie) +{ + struct dw_pcie *pci = &pcie->pci; + u32 l1ss, val; + + l1ss = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS); + + pcie->ras_des_cap = dw_pcie_find_ext_capability(&pcie->pci, + PCI_EXT_CAP_ID_VNDR); + + /* Enable ASPM counters */ + val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT; + val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT; + dw_pcie_writel_dbi(pci, pcie->ras_des_cap + + PCIE_RAS_DES_EVENT_COUNTER_CONTROL, val); + + /* Program T_cmrt and T_pwr_on values */ + val = dw_pcie_readl_dbi(pci, l1ss + PCI_L1SS_CAP); + val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE); + val |= (pcie->aspm_cmrt << 8); + val |= (pcie->aspm_pwr_on_t << 19); + dw_pcie_writel_dbi(pci, l1ss + PCI_L1SS_CAP, val); + + if (pcie->supports_clkreq) + pci->l1ss_support = true; + + /* + * Disable L1.2 capability advertisement for Tegra234 Endpoint mode. + * Tegra234 has a hardware bug where during L1.2 exit, the UPHY PLL is + * powered up immediately without waiting for REFCLK to stabilize. This + * causes the PLL to fail to lock to the correct frequency, resulting in + * PCIe link loss. Since there is no hardware fix available, we prevent + * the Endpoint from advertising L1.2 support by clearing the L1.2 bits + * in the L1 PM Substates Capabilities register. This ensures the host + * will not attempt to enter L1.2 state with this Endpoint. + */ + if (pcie->of_data->disable_l1_2 && + pcie->of_data->mode == DW_PCIE_EP_TYPE) { + val = dw_pcie_readl_dbi(pci, l1ss + PCI_L1SS_CAP); + val &= ~(PCI_L1SS_CAP_PCIPM_L1_2 | PCI_L1SS_CAP_ASPM_L1_2); + dw_pcie_writel_dbi(pci, l1ss + PCI_L1SS_CAP, val); + } + + /* Program L0s and L1 entrance latencies */ + val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR); + val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK; + val |= (pcie->aspm_l0s_enter_lat << PORT_AFR_L0S_ENTRANCE_LAT_SHIFT); + val &= ~PORT_AFR_L1_ENTRANCE_LAT_MASK; + val |= (pcie->aspm_l1_enter_lat << PORT_AFR_L1_ENTRANCE_LAT_SHIFT); + val |= PORT_AFR_ENTER_ASPM; + dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val); +} + +static void init_debugfs(struct tegra_pcie_dw *pcie) +{ + struct device *dev = pcie->dev; + char *name; + + name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node); + if (!name) + return; + + pcie->debugfs = debugfs_create_dir(name, NULL); + + debugfs_create_devm_seqfile(dev, "aspm_state_cnt", pcie->debugfs, + aspm_state_cnt); +} +#else +static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; } +static inline void init_debugfs(struct tegra_pcie_dw *pcie) { return; } +#endif + +static void tegra_pcie_enable_system_interrupts(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + u32 val; + u16 val_w; + + val = appl_readl(pcie, APPL_INTR_EN_L0_0); + val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN; + appl_writel(pcie, val, APPL_INTR_EN_L0_0); + + if (!pcie->of_data->has_sbr_reset_fix) { + val = appl_readl(pcie, APPL_INTR_EN_L1_0_0); + val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN; + appl_writel(pcie, val, APPL_INTR_EN_L1_0_0); + } + + if (pcie->enable_cdm_check) { + val = appl_readl(pcie, APPL_INTR_EN_L0_0); + val |= pcie->of_data->cdm_chk_int_en_bit; + appl_writel(pcie, val, APPL_INTR_EN_L0_0); + + val = appl_readl(pcie, APPL_INTR_EN_L1_18); + val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR; + val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR; + appl_writel(pcie, val, APPL_INTR_EN_L1_18); + } + + val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base + + PCI_EXP_LNKSTA); + pcie->init_link_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, val_w); + + val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base + + PCI_EXP_LNKCTL); + val_w |= PCI_EXP_LNKCTL_LBMIE; + dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL, + val_w); +} + +static void tegra_pcie_enable_intx_interrupts(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + u32 val; + + /* Enable INTX interrupt generation */ + val = appl_readl(pcie, APPL_INTR_EN_L0_0); + val |= APPL_INTR_EN_L0_0_SYS_INTR_EN; + val |= APPL_INTR_EN_L0_0_INT_INT_EN; + appl_writel(pcie, val, APPL_INTR_EN_L0_0); + + val = appl_readl(pcie, APPL_INTR_EN_L1_8_0); + val |= APPL_INTR_EN_L1_8_INTX_EN; + val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN; + val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN; + val |= APPL_INTR_EN_L1_8_EDMA_INT_EN; + if (IS_ENABLED(CONFIG_PCIEAER)) + val |= APPL_INTR_EN_L1_8_AER_INT_EN; + appl_writel(pcie, val, APPL_INTR_EN_L1_8_0); +} + +static void tegra_pcie_enable_msi_interrupts(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + u32 val; + + /* Enable MSI interrupt generation */ + val = appl_readl(pcie, APPL_INTR_EN_L0_0); + val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN; + val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN; + appl_writel(pcie, val, APPL_INTR_EN_L0_0); +} + +static void tegra_pcie_enable_interrupts(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + + /* Clear interrupt statuses before enabling interrupts */ + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); + + tegra_pcie_enable_system_interrupts(pp); + tegra_pcie_enable_intx_interrupts(pp); + if (IS_ENABLED(CONFIG_PCI_MSI)) + tegra_pcie_enable_msi_interrupts(pp); +} + +static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie) +{ + struct dw_pcie *pci = &pcie->pci; + u32 val, offset, i; + + /* Program init preset */ + for (i = 0; i < pcie->num_lanes; i++) { + val = dw_pcie_readw_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2)); + val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK; + val |= GEN3_GEN4_EQ_PRESET_INIT; + val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK; + val |= (GEN3_GEN4_EQ_PRESET_INIT << + CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT); + dw_pcie_writew_dbi(pci, CAP_SPCIE_CAP_OFF + (i * 2), val); + + offset = dw_pcie_find_ext_capability(pci, + PCI_EXT_CAP_ID_PL_16GT) + + PCI_PL_16GT_LE_CTRL; + val = dw_pcie_readb_dbi(pci, offset + i); + val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK; + val |= GEN3_GEN4_EQ_PRESET_INIT; + val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK; + val |= (GEN3_GEN4_EQ_PRESET_INIT << + PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT); + dw_pcie_writeb_dbi(pci, offset + i, val); + } + + val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); + val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; + dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); + + val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); + FIELD_MODIFY(GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC, &val, 0x3ff); + val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE; + dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); + + val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); + val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; + val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT); + dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); + + val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF); + FIELD_MODIFY(GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC, &val, + pcie->of_data->gen4_preset_vec); + val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE; + dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val); + + val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); + val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK; + dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); +} + +static int tegra_pcie_dw_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + u32 val; + u16 val_16; + + pp->bridge->ops = &tegra_pci_ops; + + if (!pcie->pcie_cap_base) + pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci, + PCI_CAP_ID_EXP); + + val = dw_pcie_readl_dbi(pci, PCI_IO_BASE); + val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8); + dw_pcie_writel_dbi(pci, PCI_IO_BASE, val); + + val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE); + val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE; + val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE; + dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val); + + dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0); + + /* Enable as 0xFFFF0001 response for RRS */ + val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT); + val &= ~(AMBA_ERROR_RESPONSE_RRS_MASK << AMBA_ERROR_RESPONSE_RRS_SHIFT); + val |= (AMBA_ERROR_RESPONSE_RRS_OKAY_FFFF0001 << + AMBA_ERROR_RESPONSE_RRS_SHIFT); + dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val); + + /* Clear Slot Clock Configuration bit if SRNS configuration */ + if (pcie->enable_srns) { + val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + + PCI_EXP_LNKSTA); + val_16 &= ~PCI_EXP_LNKSTA_SLC; + dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA, + val_16); + } + + config_gen3_gen4_eq_presets(pcie); + + init_host_aspm(pcie); + + if (!pcie->of_data->has_l1ss_exit_fix) { + val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); + val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; + dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); + } + + if (pcie->update_fc_fixup) { + val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF); + val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT; + dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val); + } + + clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ); + if (clk_prepare_enable(pcie->core_clk_m)) + dev_err(pci->dev, "Failed to enable core monitor clock\n"); + + return 0; +} + +static int tegra_pcie_dw_start_link(struct dw_pcie *pci) +{ + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + struct dw_pcie_rp *pp = &pci->pp; + u32 val, offset, tmp; + bool retry = true; + + if (pcie->of_data->mode == DW_PCIE_EP_TYPE) { + enable_irq(pcie->pex_rst_irq); + return 0; + } + +retry_link: + /* Assert RST */ + val = appl_readl(pcie, APPL_PINMUX); + val &= ~APPL_PINMUX_PEX_RST; + appl_writel(pcie, val, APPL_PINMUX); + + usleep_range(100, 200); + + /* Enable LTSSM */ + val = appl_readl(pcie, APPL_CTRL); + val |= APPL_CTRL_LTSSM_EN; + appl_writel(pcie, val, APPL_CTRL); + + /* De-assert RST */ + val = appl_readl(pcie, APPL_PINMUX); + val |= APPL_PINMUX_PEX_RST; + appl_writel(pcie, val, APPL_PINMUX); + + msleep(100); + + if (dw_pcie_wait_for_link(pci)) { + if (!retry) + return 0; + /* + * There are some endpoints which can't get the link up if + * root port has Data Link Feature (DLF) enabled. + * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info + * on Scaled Flow Control and DLF. + * So, need to confirm that is indeed the case here and attempt + * link up once again with DLF disabled. + */ + val = appl_readl(pcie, APPL_DEBUG); + val &= APPL_DEBUG_LTSSM_STATE_MASK; + val >>= APPL_DEBUG_LTSSM_STATE_SHIFT; + tmp = appl_readl(pcie, APPL_LINK_STATUS); + tmp &= APPL_LINK_STATUS_RDLH_LINK_UP; + if (!(val == 0x11 && !tmp)) { + /* Link is down for all good reasons */ + return 0; + } + + dev_info(pci->dev, "Link is down in DLL"); + dev_info(pci->dev, "Trying again with DLFE disabled\n"); + /* Disable LTSSM */ + val = appl_readl(pcie, APPL_CTRL); + val &= ~APPL_CTRL_LTSSM_EN; + appl_writel(pcie, val, APPL_CTRL); + + reset_control_assert(pcie->core_rst); + reset_control_deassert(pcie->core_rst); + + offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF); + val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP); + val &= ~PCI_DLF_EXCHANGE_ENABLE; + dw_pcie_writel_dbi(pci, offset + PCI_DLF_CAP, val); + + /* + * core_clk_m is enabled as part of host_init callback in + * dw_pcie_host_init(). Disable the clock since below + * tegra_pcie_dw_host_init() will enable it again. + */ + clk_disable_unprepare(pcie->core_clk_m); + tegra_pcie_dw_host_init(pp); + dw_pcie_setup_rc(pp); + + retry = false; + goto retry_link; + } + + tegra_pcie_icc_set(pcie); + + tegra_pcie_enable_interrupts(pp); + + return 0; +} + +static bool tegra_pcie_dw_link_up(struct dw_pcie *pci) +{ + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA); + + return val & PCI_EXP_LNKSTA_DLLLA; +} + +static void tegra_pcie_dw_stop_link(struct dw_pcie *pci) +{ + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + + if (pcie->of_data->mode == DW_PCIE_EP_TYPE) + disable_irq(pcie->pex_rst_irq); +} + +static const struct dw_pcie_ops tegra_dw_pcie_ops = { + .link_up = tegra_pcie_dw_link_up, + .start_link = tegra_pcie_dw_start_link, + .stop_link = tegra_pcie_dw_stop_link, +}; + +static const struct dw_pcie_host_ops tegra_pcie_dw_host_ops = { + .init = tegra_pcie_dw_host_init, +}; + +static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie) +{ + unsigned int phy_count = pcie->phy_count; + + while (phy_count--) { + phy_power_off(pcie->phys[phy_count]); + phy_exit(pcie->phys[phy_count]); + } +} + +static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie) +{ + unsigned int i; + int ret; + + for (i = 0; i < pcie->phy_count; i++) { + ret = phy_init(pcie->phys[i]); + if (ret < 0) + goto phy_power_off; + + ret = phy_power_on(pcie->phys[i]); + if (ret < 0) + goto phy_exit; + + if (pcie->of_data->mode == DW_PCIE_EP_TYPE) + phy_calibrate(pcie->phys[i]); + } + + return 0; + +phy_power_off: + while (i--) { + phy_power_off(pcie->phys[i]); +phy_exit: + phy_exit(pcie->phys[i]); + } + + return ret; +} + +static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie) +{ + struct platform_device *pdev = to_platform_device(pcie->dev); + struct device_node *np = pcie->dev->of_node; + u32 val; + int ret; + + pcie->dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi"); + if (!pcie->dbi_res) { + dev_err(pcie->dev, "Failed to find \"dbi\" region\n"); + return -ENODEV; + } + + ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt); + if (ret < 0) { + dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret); + return ret; + } + + ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us", + &pcie->aspm_pwr_on_t); + if (ret < 0) + dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n", + ret); + + ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us", + &pcie->aspm_l0s_enter_lat); + if (ret < 0) + dev_info(pcie->dev, + "Failed to read ASPM L0s Entrance latency: %d\n", ret); + + /* Default to max latency of 7. */ + pcie->aspm_l1_enter_lat = 7; + ret = of_property_read_u32(np, "aspm-l1-entry-delay-ns", &val); + if (!ret) { + u32 us = DIV_ROUND_UP(val, 1000); + + pcie->aspm_l1_enter_lat = min_t(u32, order_base_2(us), 7); + } + + ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes); + if (ret < 0) { + dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret); + return ret; + } + + ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid); + if (ret) { + dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret); + return ret; + } + + ret = of_property_count_strings(np, "phy-names"); + if (ret < 0) { + dev_err(pcie->dev, "Failed to find PHY entries: %d\n", + ret); + return ret; + } + pcie->phy_count = ret; + + if (of_property_read_bool(np, "nvidia,update-fc-fixup")) + pcie->update_fc_fixup = true; + + /* RP using an external REFCLK is supported only in Tegra234 */ + if (pcie->of_data->version == TEGRA194_DWC_IP_VER) { + if (pcie->of_data->mode == DW_PCIE_EP_TYPE) + pcie->enable_ext_refclk = true; + } else { + pcie->enable_ext_refclk = + of_property_read_bool(pcie->dev->of_node, + "nvidia,enable-ext-refclk"); + } + + pcie->supports_clkreq = + of_property_read_bool(pcie->dev->of_node, "supports-clkreq"); + + pcie->enable_cdm_check = + of_property_read_bool(np, "snps,enable-cdm-check"); + + if (pcie->of_data->version == TEGRA234_DWC_IP_VER) + pcie->enable_srns = + of_property_read_bool(np, "nvidia,enable-srns"); + + if (pcie->of_data->mode == DW_PCIE_RC_TYPE) + return 0; + + /* Endpoint mode specific DT entries */ + pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN); + if (IS_ERR(pcie->pex_rst_gpiod)) { + int err = PTR_ERR(pcie->pex_rst_gpiod); + const char *level = KERN_ERR; + + if (err == -EPROBE_DEFER) + level = KERN_DEBUG; + + dev_printk(level, pcie->dev, + dev_fmt("Failed to get PERST GPIO: %d\n"), + err); + return err; + } + + pcie->pex_refclk_sel_gpiod = devm_gpiod_get_optional(pcie->dev, + "nvidia,refclk-select", + GPIOD_OUT_HIGH); + if (IS_ERR(pcie->pex_refclk_sel_gpiod)) { + int err = PTR_ERR(pcie->pex_refclk_sel_gpiod); + const char *level = KERN_ERR; + + if (err == -EPROBE_DEFER) + level = KERN_DEBUG; + + dev_printk(level, pcie->dev, + dev_fmt("Failed to get REFCLK select GPIOs: %d\n"), + err); + pcie->pex_refclk_sel_gpiod = NULL; + } + + return 0; +} + +static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie, + bool enable) +{ + struct mrq_uphy_response resp; + struct tegra_bpmp_message msg; + struct mrq_uphy_request req; + int err; + + /* + * Controller-5 doesn't need to have its state set by BPMP-FW in + * Tegra194 + */ + if (pcie->of_data->version == TEGRA194_DWC_IP_VER && pcie->cid == 5) + return 0; + + memset(&req, 0, sizeof(req)); + memset(&resp, 0, sizeof(resp)); + + req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE; + req.controller_state.pcie_controller = pcie->cid; + req.controller_state.enable = enable; + + memset(&msg, 0, sizeof(msg)); + msg.mrq = MRQ_UPHY; + msg.tx.data = &req; + msg.tx.size = sizeof(req); + msg.rx.data = &resp; + msg.rx.size = sizeof(resp); + + err = tegra_bpmp_transfer(pcie->bpmp, &msg); + if (err) + return err; + if (msg.rx.ret) + return -EINVAL; + + return 0; +} + +static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie, + bool enable) +{ + struct mrq_uphy_response resp; + struct tegra_bpmp_message msg; + struct mrq_uphy_request req; + int err; + + memset(&req, 0, sizeof(req)); + memset(&resp, 0, sizeof(resp)); + + if (enable) { + req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT; + req.ep_ctrlr_pll_init.ep_controller = pcie->cid; + } else { + req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF; + req.ep_ctrlr_pll_off.ep_controller = pcie->cid; + } + + memset(&msg, 0, sizeof(msg)); + msg.mrq = MRQ_UPHY; + msg.tx.data = &req; + msg.tx.size = sizeof(req); + msg.rx.data = &resp; + msg.rx.size = sizeof(resp); + + err = tegra_bpmp_transfer(pcie->bpmp, &msg); + if (err) + return err; + if (msg.rx.ret) + return -EINVAL; + + return 0; +} + +static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie) +{ + pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3"); + if (IS_ERR(pcie->slot_ctl_3v3)) { + if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV) + return PTR_ERR(pcie->slot_ctl_3v3); + + pcie->slot_ctl_3v3 = NULL; + } + + pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v"); + if (IS_ERR(pcie->slot_ctl_12v)) { + if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV) + return PTR_ERR(pcie->slot_ctl_12v); + + pcie->slot_ctl_12v = NULL; + } + + return 0; +} + +static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie) +{ + int ret; + + if (pcie->slot_ctl_3v3) { + ret = regulator_enable(pcie->slot_ctl_3v3); + if (ret < 0) { + dev_err(pcie->dev, + "Failed to enable 3.3V slot supply: %d\n", ret); + return ret; + } + } + + if (pcie->slot_ctl_12v) { + ret = regulator_enable(pcie->slot_ctl_12v); + if (ret < 0) { + dev_err(pcie->dev, + "Failed to enable 12V slot supply: %d\n", ret); + goto fail_12v_enable; + } + } + + /* + * According to PCI Express Card Electromechanical Specification + * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive) + * should be a minimum of 100ms. + */ + if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v) + msleep(100); + + return 0; + +fail_12v_enable: + if (pcie->slot_ctl_3v3) + regulator_disable(pcie->slot_ctl_3v3); + return ret; +} + +static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie) +{ + if (pcie->slot_ctl_12v) + regulator_disable(pcie->slot_ctl_12v); + if (pcie->slot_ctl_3v3) + regulator_disable(pcie->slot_ctl_3v3); +} + +static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie, + bool en_hw_hot_rst) +{ + int ret; + u32 val; + + ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true); + if (ret) { + dev_err(pcie->dev, + "Failed to enable controller %u: %d\n", pcie->cid, ret); + return ret; + } + + if (pcie->enable_ext_refclk) { + ret = tegra_pcie_bpmp_set_pll_state(pcie, true); + if (ret) { + dev_err(pcie->dev, "Failed to init UPHY: %d\n", ret); + goto fail_pll_init; + } + } + + ret = tegra_pcie_enable_slot_regulators(pcie); + if (ret < 0) + goto fail_slot_reg_en; + + ret = regulator_enable(pcie->pex_ctl_supply); + if (ret < 0) { + dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret); + goto fail_reg_en; + } + + ret = clk_prepare_enable(pcie->core_clk); + if (ret) { + dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret); + goto fail_core_clk; + } + + ret = reset_control_deassert(pcie->core_apb_rst); + if (ret) { + dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n", + ret); + goto fail_core_apb_rst; + } + + if (en_hw_hot_rst || pcie->of_data->has_sbr_reset_fix) { + /* Enable HW_HOT_RST mode */ + val = appl_readl(pcie, APPL_CTRL); + val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << + APPL_CTRL_HW_HOT_RST_MODE_SHIFT); + val |= (APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN << + APPL_CTRL_HW_HOT_RST_MODE_SHIFT); + val |= APPL_CTRL_HW_HOT_RST_EN; + appl_writel(pcie, val, APPL_CTRL); + } + + ret = tegra_pcie_enable_phy(pcie); + if (ret) { + dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret); + goto fail_phy; + } + + /* Update CFG base address */ + appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK, + APPL_CFG_BASE_ADDR); + + /* Configure this core for RP mode operation */ + appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE); + + appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE); + + val = appl_readl(pcie, APPL_CTRL); + appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL); + + val = appl_readl(pcie, APPL_CFG_MISC); + val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT); + appl_writel(pcie, val, APPL_CFG_MISC); + + if (pcie->enable_srns || pcie->enable_ext_refclk) { + /* + * When Tegra PCIe RP is using external clock, it cannot supply + * same clock to its downstream hierarchy. Hence, gate PCIe RP + * REFCLK out pads when RP & EP are using separate clocks or RP + * is using an external REFCLK. + */ + val = appl_readl(pcie, APPL_PINMUX); + val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN; + val &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE; + appl_writel(pcie, val, APPL_PINMUX); + } + + if (!pcie->supports_clkreq) { + val = appl_readl(pcie, APPL_PINMUX); + val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN; + val &= ~APPL_PINMUX_CLKREQ_OVERRIDE; + val &= ~APPL_PINMUX_CLKREQ_DEFAULT_VALUE; + appl_writel(pcie, val, APPL_PINMUX); + } + + /* Update iATU_DMA base address */ + appl_writel(pcie, + pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK, + APPL_CFG_IATU_DMA_BASE_ADDR); + + reset_control_deassert(pcie->core_rst); + + return ret; + +fail_phy: + reset_control_assert(pcie->core_apb_rst); +fail_core_apb_rst: + clk_disable_unprepare(pcie->core_clk); +fail_core_clk: + regulator_disable(pcie->pex_ctl_supply); +fail_reg_en: + tegra_pcie_disable_slot_regulators(pcie); +fail_slot_reg_en: + if (pcie->enable_ext_refclk) + tegra_pcie_bpmp_set_pll_state(pcie, false); +fail_pll_init: + tegra_pcie_bpmp_set_ctrl_state(pcie, false); + + return ret; +} + +static void tegra_pcie_unconfig_controller(struct tegra_pcie_dw *pcie) +{ + int ret; + + ret = reset_control_assert(pcie->core_rst); + if (ret) + dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n", ret); + + tegra_pcie_disable_phy(pcie); + + ret = reset_control_assert(pcie->core_apb_rst); + if (ret) + dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret); + + clk_disable_unprepare(pcie->core_clk); + + ret = regulator_disable(pcie->pex_ctl_supply); + if (ret) + dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret); + + tegra_pcie_disable_slot_regulators(pcie); + + if (pcie->enable_ext_refclk) { + ret = tegra_pcie_bpmp_set_pll_state(pcie, false); + if (ret) + dev_err(pcie->dev, "Failed to deinit UPHY: %d\n", ret); + } + + ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false); + if (ret) + dev_err(pcie->dev, "Failed to disable controller %d: %d\n", + pcie->cid, ret); +} + +static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie) +{ + struct dw_pcie *pci = &pcie->pci; + struct dw_pcie_rp *pp = &pci->pp; + int ret; + + ret = tegra_pcie_config_controller(pcie, false); + if (ret < 0) + return ret; + + pp->ops = &tegra_pcie_dw_host_ops; + + ret = dw_pcie_host_init(pp); + if (ret < 0) { + dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret); + goto fail_host_init; + } + + return 0; + +fail_host_init: + tegra_pcie_unconfig_controller(pcie); + return ret; +} + +static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie) +{ + u32 val; + + if (!tegra_pcie_dw_link_up(&pcie->pci)) + return 0; + + val = appl_readl(pcie, APPL_RADM_STATUS); + val |= APPL_PM_XMT_TURNOFF_STATE; + appl_writel(pcie, val, APPL_RADM_STATUS); + + return readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val, + val & APPL_DEBUG_PM_LINKST_IN_L2_LAT, + PCIE_PME_TO_L2_TIMEOUT_US/10, + PCIE_PME_TO_L2_TIMEOUT_US); +} + +static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie) +{ + u32 data; + int err; + + if (!tegra_pcie_dw_link_up(&pcie->pci)) { + dev_dbg(pcie->dev, "PCIe link is not up...!\n"); + return; + } + + /* + * PCIe controller exits from L2 only if reset is applied, so + * controller doesn't handle interrupts. But in cases where + * L2 entry fails, PERST# is asserted which can trigger surprise + * link down AER. However this function call happens in + * suspend_noirq(), so AER interrupt will not be processed. + * Disable all interrupts to avoid such a scenario. + */ + appl_writel(pcie, 0x0, APPL_INTR_EN_L0_0); + + if (tegra_pcie_try_link_l2(pcie)) { + dev_info(pcie->dev, "Link didn't transition to L2 state\n"); + /* + * TX lane clock freq will reset to Gen1 only if link is in L2 + * or detect state. + * So apply pex_rst to end point to force RP to go into detect + * state + */ + data = appl_readl(pcie, APPL_PINMUX); + data &= ~APPL_PINMUX_PEX_RST; + appl_writel(pcie, data, APPL_PINMUX); + + err = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, data, + ((data & APPL_DEBUG_LTSSM_STATE_MASK) == LTSSM_STATE_DETECT_QUIET) || + ((data & APPL_DEBUG_LTSSM_STATE_MASK) == LTSSM_STATE_DETECT_ACT) || + ((data & APPL_DEBUG_LTSSM_STATE_MASK) == LTSSM_STATE_PRE_DETECT_QUIET) || + ((data & APPL_DEBUG_LTSSM_STATE_MASK) == LTSSM_STATE_DETECT_WAIT), + LTSSM_DELAY_US, LTSSM_TIMEOUT_US); + if (err) + dev_info(pcie->dev, "LTSSM state: 0x%x detect timeout: %d\n", data, err); + + /* + * Deassert LTSSM state to stop the state toggling between + * Polling and Detect. + */ + data = readl(pcie->appl_base + APPL_CTRL); + data &= ~APPL_CTRL_LTSSM_EN; + writel(data, pcie->appl_base + APPL_CTRL); + } + /* + * DBI registers may not be accessible after this as PLL-E would be + * down depending on how CLKREQ is pulled by end point + */ + data = appl_readl(pcie, APPL_PINMUX); + data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE); + /* Cut REFCLK to slot */ + data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN; + data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE; + appl_writel(pcie, data, APPL_PINMUX); +} + +static void tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie) +{ + clk_disable_unprepare(pcie->core_clk_m); + dw_pcie_host_deinit(&pcie->pci.pp); + tegra_pcie_dw_pme_turnoff(pcie); + tegra_pcie_unconfig_controller(pcie); +} + +static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie) +{ + struct device *dev = pcie->dev; + int ret; + + pm_runtime_enable(dev); + + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n", + ret); + goto fail_pm_get_sync; + } + + ret = pinctrl_pm_select_default_state(dev); + if (ret < 0) { + dev_err(dev, "Failed to configure sideband pins: %d\n", ret); + goto fail_pm_get_sync; + } + + ret = tegra_pcie_init_controller(pcie); + if (ret < 0) { + dev_err(dev, "Failed to initialize controller: %d\n", ret); + goto fail_pm_get_sync; + } + + pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci); + if (!pcie->link_state) { + ret = -ENOMEDIUM; + goto fail_host_init; + } + + init_debugfs(pcie); + + return ret; + +fail_host_init: + tegra_pcie_deinit_controller(pcie); +fail_pm_get_sync: + pm_runtime_put_sync(dev); + pm_runtime_disable(dev); + return ret; +} + +static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie) +{ + u32 val; + int ret; + + if (pcie->ep_state == EP_STATE_DISABLED) + return; + + ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val, + ((val & APPL_DEBUG_LTSSM_STATE_MASK) == LTSSM_STATE_DETECT_QUIET) || + ((val & APPL_DEBUG_LTSSM_STATE_MASK) == LTSSM_STATE_DETECT_ACT) || + ((val & APPL_DEBUG_LTSSM_STATE_MASK) == LTSSM_STATE_PRE_DETECT_QUIET) || + ((val & APPL_DEBUG_LTSSM_STATE_MASK) == LTSSM_STATE_DETECT_WAIT) || + ((val & APPL_DEBUG_LTSSM_STATE_MASK) == LTSSM_STATE_L2_IDLE), + LTSSM_DELAY_US, LTSSM_TIMEOUT_US); + if (ret) + dev_info(pcie->dev, "LTSSM state: 0x%x detect timeout: %d\n", val, ret); + + /* + * Deassert LTSSM state to stop the state toggling between + * Polling and Detect. + */ + val = appl_readl(pcie, APPL_CTRL); + val &= ~APPL_CTRL_LTSSM_EN; + appl_writel(pcie, val, APPL_CTRL); + + reset_control_assert(pcie->core_rst); + + tegra_pcie_disable_phy(pcie); + + reset_control_assert(pcie->core_apb_rst); + + clk_disable_unprepare(pcie->core_clk); + + pm_runtime_put_sync(pcie->dev); + + if (pcie->enable_ext_refclk) { + ret = tegra_pcie_bpmp_set_pll_state(pcie, false); + if (ret) + dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", + ret); + } + + ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false); + if (ret) + dev_err(pcie->dev, "Failed to disable controller: %d\n", ret); + + pcie->ep_state = EP_STATE_DISABLED; + dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n"); +} + +static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie) +{ + struct dw_pcie *pci = &pcie->pci; + struct dw_pcie_ep *ep = &pci->ep; + struct device *dev = pcie->dev; + u32 val; + int ret; + u16 val_16; + + if (pcie->ep_state == EP_STATE_ENABLED) + return; + + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) { + dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n", + ret); + return; + } + + ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true); + if (ret) { + dev_err(pcie->dev, "Failed to enable controller %u: %d\n", + pcie->cid, ret); + goto fail_set_ctrl_state; + } + + if (pcie->enable_ext_refclk) { + ret = tegra_pcie_bpmp_set_pll_state(pcie, true); + if (ret) { + dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n", + ret); + goto fail_pll_init; + } + } + + ret = clk_prepare_enable(pcie->core_clk); + if (ret) { + dev_err(dev, "Failed to enable core clock: %d\n", ret); + goto fail_core_clk_enable; + } + + ret = reset_control_deassert(pcie->core_apb_rst); + if (ret) { + dev_err(dev, "Failed to deassert core APB reset: %d\n", ret); + goto fail_core_apb_rst; + } + + ret = tegra_pcie_enable_phy(pcie); + if (ret) { + dev_err(dev, "Failed to enable PHY: %d\n", ret); + goto fail_phy; + } + + /* Clear any stale interrupt statuses */ + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15); + appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17); + + /* configure this core for EP mode operation */ + val = appl_readl(pcie, APPL_DM_TYPE); + val &= ~APPL_DM_TYPE_MASK; + val |= APPL_DM_TYPE_EP; + appl_writel(pcie, val, APPL_DM_TYPE); + + appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE); + + val = appl_readl(pcie, APPL_CTRL); + val |= APPL_CTRL_SYS_PRE_DET_STATE; + val |= APPL_CTRL_HW_HOT_RST_EN; + val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << APPL_CTRL_HW_HOT_RST_MODE_SHIFT); + val |= (APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST_LTSSM_EN << APPL_CTRL_HW_HOT_RST_MODE_SHIFT); + appl_writel(pcie, val, APPL_CTRL); + + val = appl_readl(pcie, APPL_CFG_MISC); + val |= APPL_CFG_MISC_SLV_EP_MODE; + val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT); + appl_writel(pcie, val, APPL_CFG_MISC); + + val = appl_readl(pcie, APPL_PINMUX); + val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN; + val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE; + appl_writel(pcie, val, APPL_PINMUX); + + appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK, + APPL_CFG_BASE_ADDR); + + appl_writel(pcie, pcie->atu_dma_res->start & + APPL_CFG_IATU_DMA_BASE_ADDR_MASK, + APPL_CFG_IATU_DMA_BASE_ADDR); + + val = appl_readl(pcie, APPL_INTR_EN_L0_0); + val |= APPL_INTR_EN_L0_0_SYS_INTR_EN; + val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN; + val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN; + val |= APPL_INTR_EN_L0_0_INT_INT_EN; + appl_writel(pcie, val, APPL_INTR_EN_L0_0); + + val = appl_readl(pcie, APPL_INTR_EN_L1_0_0); + val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN; + val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN; + appl_writel(pcie, val, APPL_INTR_EN_L1_0_0); + + val = appl_readl(pcie, APPL_INTR_EN_L1_8_0); + val |= APPL_INTR_EN_L1_8_EDMA_INT_EN; + appl_writel(pcie, val, APPL_INTR_EN_L1_8_0); + + /* 110us for both snoop and no-snoop */ + val = FIELD_PREP(PCI_LTR_VALUE_MASK, 110) | + FIELD_PREP(PCI_LTR_SCALE_MASK, 2) | + LTR_MSG_REQ | + FIELD_PREP(PCI_LTR_NOSNOOP_VALUE, 110) | + FIELD_PREP(PCI_LTR_NOSNOOP_SCALE, 2) | + LTR_NOSNOOP_MSG_REQ; + appl_writel(pcie, val, APPL_LTR_MSG_1); + + reset_control_deassert(pcie->core_rst); + + /* Perform cleanup that requires refclk and core reset deasserted */ + pci_epc_deinit_notify(pcie->pci.ep.epc); + dw_pcie_ep_cleanup(&pcie->pci.ep); + + val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL); + val &= ~PORT_LOGIC_SPEED_CHANGE; + dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val); + + if (pcie->update_fc_fixup) { + val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF); + val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT; + dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val); + } + + config_gen3_gen4_eq_presets(pcie); + + init_host_aspm(pcie); + + if (!pcie->of_data->has_l1ss_exit_fix) { + val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF); + val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL; + dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val); + } + + pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci, + PCI_CAP_ID_EXP); + + /* Clear Slot Clock Configuration bit if SRNS configuration */ + if (pcie->enable_srns) { + val_16 = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + + PCI_EXP_LNKSTA); + val_16 &= ~PCI_EXP_LNKSTA_SLC; + dw_pcie_writew_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA, + val_16); + } + + clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ); + + val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK); + val |= MSIX_ADDR_MATCH_LOW_OFF_EN; + dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val); + val = (upper_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK); + dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val); + + ret = dw_pcie_ep_init_registers(ep); + if (ret) { + dev_err(dev, "Failed to complete initialization: %d\n", ret); + goto fail_init_complete; + } + + pci_epc_init_notify(ep->epc); + + /* Program the private control to allow sending LTR upstream */ + if (pcie->of_data->has_ltr_req_fix) { + val = appl_readl(pcie, APPL_LTR_MSG_2); + val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE; + appl_writel(pcie, val, APPL_LTR_MSG_2); + } + + /* Enable LTSSM */ + val = appl_readl(pcie, APPL_CTRL); + val |= APPL_CTRL_LTSSM_EN; + appl_writel(pcie, val, APPL_CTRL); + + pcie->ep_state = EP_STATE_ENABLED; + dev_dbg(dev, "Initialization of endpoint is completed\n"); + + return; + +fail_init_complete: + reset_control_assert(pcie->core_rst); + tegra_pcie_disable_phy(pcie); +fail_phy: + reset_control_assert(pcie->core_apb_rst); +fail_core_apb_rst: + clk_disable_unprepare(pcie->core_clk); +fail_core_clk_enable: + tegra_pcie_bpmp_set_pll_state(pcie, false); +fail_pll_init: + tegra_pcie_bpmp_set_ctrl_state(pcie, false); +fail_set_ctrl_state: + pm_runtime_put_sync(dev); +} + +static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg) +{ + struct tegra_pcie_dw *pcie = arg; + + if (gpiod_get_value(pcie->pex_rst_gpiod)) + pex_ep_event_pex_rst_assert(pcie); + else + pex_ep_event_pex_rst_deassert(pcie); + + return IRQ_HANDLED; +} + +static int tegra_pcie_ep_raise_intx_irq(struct tegra_pcie_dw *pcie, u16 irq) +{ + /* Tegra194 supports only INTA */ + if (irq > 1) + return -EINVAL; + + appl_writel(pcie, 1, APPL_LEGACY_INTX); + usleep_range(1000, 2000); + appl_writel(pcie, 0, APPL_LEGACY_INTX); + return 0; +} + +static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq) +{ + if (unlikely(irq > 32)) + return -EINVAL; + + appl_writel(pcie, BIT(irq - 1), APPL_MSI_CTRL_1); + + return 0; +} + +static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq) +{ + struct dw_pcie_ep *ep = &pcie->pci.ep; + + writel(irq, ep->msi_mem); + + return 0; +} + +static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, + unsigned int type, u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct tegra_pcie_dw *pcie = to_tegra_pcie(pci); + + switch (type) { + case PCI_IRQ_INTX: + return tegra_pcie_ep_raise_intx_irq(pcie, interrupt_num); + + case PCI_IRQ_MSI: + return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num); + + case PCI_IRQ_MSIX: + return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num); + + default: + dev_err(pci->dev, "Unknown IRQ type\n"); + return -EPERM; + } + + return 0; +} + +static const struct pci_epc_bar_rsvd_region tegra194_bar2_rsvd[] = { + { + /* MSI-X table structure */ + .type = PCI_EPC_BAR_RSVD_MSIX_TBL_RAM, + .offset = 0x0, + .size = SZ_64K, + }, + { + /* MSI-X PBA structure */ + .type = PCI_EPC_BAR_RSVD_MSIX_PBA_RAM, + .offset = 0x10000, + .size = SZ_64K, + }, +}; + +static const struct pci_epc_bar_rsvd_region tegra194_bar4_rsvd[] = { + { + /* DMA_CAP (BAR4: DMA Port Logic Structure) */ + .type = PCI_EPC_BAR_RSVD_DMA_CTRL_MMIO, + .offset = 0x0, + .size = SZ_4K, + }, +}; + +/* Tegra EP: BAR0 = 64-bit programmable BAR, BAR2 = 64-bit MSI-X table, BAR4 = 64-bit DMA regs. */ +static const struct pci_epc_features tegra_pcie_epc_features = { + DWC_EPC_COMMON_FEATURES, + .linkup_notifier = true, + .msi_capable = true, + .bar[BAR_0] = { .only_64bit = true, }, + .bar[BAR_2] = { + .type = BAR_RESERVED, + .only_64bit = true, + .nr_rsvd_regions = ARRAY_SIZE(tegra194_bar2_rsvd), + .rsvd_regions = tegra194_bar2_rsvd, + }, + .bar[BAR_4] = { + .type = BAR_RESERVED, + .only_64bit = true, + .nr_rsvd_regions = ARRAY_SIZE(tegra194_bar4_rsvd), + .rsvd_regions = tegra194_bar4_rsvd, + }, + .align = SZ_64K, +}; + +static const struct pci_epc_features* +tegra_pcie_ep_get_features(struct dw_pcie_ep *ep) +{ + return &tegra_pcie_epc_features; +} + +static const struct dw_pcie_ep_ops pcie_ep_ops = { + .raise_irq = tegra_pcie_ep_raise_irq, + .get_features = tegra_pcie_ep_get_features, +}; + +static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie, + struct platform_device *pdev) +{ + struct dw_pcie *pci = &pcie->pci; + struct device *dev = pcie->dev; + struct dw_pcie_ep *ep; + char *name; + int ret; + + ep = &pci->ep; + ep->ops = &pcie_ep_ops; + + ep->page_size = SZ_64K; + + ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME); + if (ret < 0) { + dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n", + ret); + return ret; + } + + ret = gpiod_to_irq(pcie->pex_rst_gpiod); + if (ret < 0) { + dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret); + return ret; + } + pcie->pex_rst_irq = (unsigned int)ret; + + name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq", + pcie->cid); + if (!name) { + dev_err(dev, "Failed to create PERST IRQ string\n"); + return -ENOMEM; + } + + irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN); + + pcie->ep_state = EP_STATE_DISABLED; + + ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL, + tegra_pcie_ep_pex_rst_irq, + IRQF_TRIGGER_RISING | + IRQF_TRIGGER_FALLING | IRQF_ONESHOT, + name, (void *)pcie); + if (ret < 0) { + dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret); + return ret; + } + + pm_runtime_enable(dev); + + ret = dw_pcie_ep_init(ep); + if (ret) { + dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n", + ret); + pm_runtime_disable(dev); + return ret; + } + + return 0; +} + +static int tegra_pcie_dw_probe(struct platform_device *pdev) +{ + const struct tegra_pcie_dw_of_data *data; + struct device *dev = &pdev->dev; + struct resource *atu_dma_res; + struct tegra_pcie_dw *pcie; + struct dw_pcie_rp *pp; + struct dw_pcie *pci; + struct phy **phys; + char *name; + int ret; + u32 i; + + data = of_device_get_match_data(dev); + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + pci = &pcie->pci; + pci->dev = &pdev->dev; + pci->ops = &tegra_dw_pcie_ops; + pcie->dev = &pdev->dev; + pcie->of_data = (struct tegra_pcie_dw_of_data *)data; + pci->n_fts[0] = pcie->of_data->n_fts[0]; + pci->n_fts[1] = pcie->of_data->n_fts[1]; + pp = &pci->pp; + pp->num_vectors = MAX_MSI_IRQS; + + ret = tegra_pcie_dw_parse_dt(pcie); + if (ret < 0) { + const char *level = KERN_ERR; + + if (ret == -EPROBE_DEFER) + level = KERN_DEBUG; + + dev_printk(level, dev, + dev_fmt("Failed to parse device tree: %d\n"), + ret); + return ret; + } + + ret = tegra_pcie_get_slot_regulators(pcie); + if (ret < 0) { + const char *level = KERN_ERR; + + if (ret == -EPROBE_DEFER) + level = KERN_DEBUG; + + dev_printk(level, dev, + dev_fmt("Failed to get slot regulators: %d\n"), + ret); + return ret; + } + + if (pcie->pex_refclk_sel_gpiod) + gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1); + + pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl"); + if (IS_ERR(pcie->pex_ctl_supply)) { + ret = PTR_ERR(pcie->pex_ctl_supply); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to get regulator: %pe\n", + pcie->pex_ctl_supply); + return ret; + } + + pcie->core_clk = devm_clk_get(dev, "core"); + if (IS_ERR(pcie->core_clk)) { + dev_err(dev, "Failed to get core clock: %pe\n", + pcie->core_clk); + return PTR_ERR(pcie->core_clk); + } + + pcie->core_clk_m = devm_clk_get_optional(dev, "core_m"); + if (IS_ERR(pcie->core_clk_m)) + return dev_err_probe(dev, PTR_ERR(pcie->core_clk_m), + "Failed to get monitor clock\n"); + + pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "appl"); + if (!pcie->appl_res) { + dev_err(dev, "Failed to find \"appl\" region\n"); + return -ENODEV; + } + + pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res); + if (IS_ERR(pcie->appl_base)) + return PTR_ERR(pcie->appl_base); + + pcie->core_apb_rst = devm_reset_control_get(dev, "apb"); + if (IS_ERR(pcie->core_apb_rst)) { + dev_err(dev, "Failed to get APB reset: %pe\n", + pcie->core_apb_rst); + return PTR_ERR(pcie->core_apb_rst); + } + + phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL); + if (!phys) + return -ENOMEM; + + for (i = 0; i < pcie->phy_count; i++) { + name = kasprintf(GFP_KERNEL, "p2u-%u", i); + if (!name) { + dev_err(dev, "Failed to create P2U string\n"); + return -ENOMEM; + } + phys[i] = devm_phy_get(dev, name); + kfree(name); + if (IS_ERR(phys[i])) { + ret = PTR_ERR(phys[i]); + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to get PHY: %d\n", ret); + return ret; + } + } + + pcie->phys = phys; + + atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "atu_dma"); + if (!atu_dma_res) { + dev_err(dev, "Failed to find \"atu_dma\" region\n"); + return -ENODEV; + } + pcie->atu_dma_res = atu_dma_res; + + pci->atu_size = resource_size(atu_dma_res); + pci->atu_base = devm_ioremap_resource(dev, atu_dma_res); + if (IS_ERR(pci->atu_base)) + return PTR_ERR(pci->atu_base); + + pcie->core_rst = devm_reset_control_get(dev, "core"); + if (IS_ERR(pcie->core_rst)) { + dev_err(dev, "Failed to get core reset: %pe\n", + pcie->core_rst); + return PTR_ERR(pcie->core_rst); + } + + pp->irq = platform_get_irq_byname(pdev, "intr"); + if (pp->irq < 0) + return pp->irq; + + pcie->bpmp = tegra_bpmp_get(dev); + if (IS_ERR(pcie->bpmp)) + return PTR_ERR(pcie->bpmp); + + platform_set_drvdata(pdev, pcie); + + pcie->icc_path = devm_of_icc_get(&pdev->dev, "write"); + ret = PTR_ERR_OR_ZERO(pcie->icc_path); + if (ret) { + tegra_bpmp_put(pcie->bpmp); + dev_err_probe(&pdev->dev, ret, "failed to get write interconnect\n"); + return ret; + } + + switch (pcie->of_data->mode) { + case DW_PCIE_RC_TYPE: + ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler, + IRQF_SHARED, "tegra-pcie-intr", pcie); + if (ret) { + dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq, + ret); + goto fail; + } + + ret = tegra_pcie_config_rp(pcie); + if (ret && ret != -ENOMEDIUM) + goto fail; + else + return 0; + break; + + case DW_PCIE_EP_TYPE: + ret = devm_request_threaded_irq(dev, pp->irq, + tegra_pcie_ep_hard_irq, + tegra_pcie_ep_irq_thread, + IRQF_SHARED, + "tegra-pcie-ep-intr", pcie); + if (ret) { + dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq, + ret); + goto fail; + } + + ret = tegra_pcie_config_ep(pcie, pdev); + if (ret < 0) + goto fail; + else + return 0; + break; + + default: + dev_err(dev, "Invalid PCIe device type %d\n", + pcie->of_data->mode); + ret = -EINVAL; + } + +fail: + tegra_bpmp_put(pcie->bpmp); + return ret; +} + +static void tegra_pcie_dw_remove(struct platform_device *pdev) +{ + struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev); + struct dw_pcie_ep *ep = &pcie->pci.ep; + + if (pcie->of_data->mode == DW_PCIE_RC_TYPE) { + if (!pcie->link_state) + return; + + debugfs_remove_recursive(pcie->debugfs); + tegra_pcie_deinit_controller(pcie); + pm_runtime_put_sync(pcie->dev); + } else { + disable_irq(pcie->pex_rst_irq); + pex_ep_event_pex_rst_assert(pcie); + dw_pcie_ep_deinit(ep); + } + + pm_runtime_disable(pcie->dev); + tegra_bpmp_put(pcie->bpmp); + if (pcie->pex_refclk_sel_gpiod) + gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0); +} + +static int tegra_pcie_dw_suspend(struct device *dev) +{ + struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); + + if (pcie->of_data->mode == DW_PCIE_EP_TYPE) { + if (pcie->ep_state == EP_STATE_ENABLED) { + dev_err(dev, "Tegra PCIe is in EP mode, suspend not allowed\n"); + return -EPERM; + } + + disable_irq(pcie->pex_rst_irq); + return 0; + } + + return 0; +} + +static int tegra_pcie_dw_suspend_late(struct device *dev) +{ + struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); + u32 val; + + if (!pcie->link_state) + return 0; + + /* Enable HW_HOT_RST mode */ + if (!pcie->of_data->has_sbr_reset_fix) { + val = appl_readl(pcie, APPL_CTRL); + val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << + APPL_CTRL_HW_HOT_RST_MODE_SHIFT); + val |= APPL_CTRL_HW_HOT_RST_EN; + appl_writel(pcie, val, APPL_CTRL); + } + + return 0; +} + +static int tegra_pcie_dw_suspend_noirq(struct device *dev) +{ + struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); + + if (pcie->of_data->mode == DW_PCIE_EP_TYPE) + return 0; + + if (!pcie->link_state) + return 0; + + clk_disable_unprepare(pcie->core_clk_m); + tegra_pcie_dw_pme_turnoff(pcie); + tegra_pcie_unconfig_controller(pcie); + + return 0; +} + +static int tegra_pcie_dw_resume_noirq(struct device *dev) +{ + struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); + int ret; + + if (pcie->of_data->mode == DW_PCIE_EP_TYPE) + return 0; + + if (!pcie->link_state) + return 0; + + ret = tegra_pcie_config_controller(pcie, true); + if (ret < 0) + return ret; + + ret = tegra_pcie_dw_host_init(&pcie->pci.pp); + if (ret < 0) { + dev_err(dev, "Failed to init host: %d\n", ret); + goto fail_host_init; + } + + dw_pcie_setup_rc(&pcie->pci.pp); + + ret = tegra_pcie_dw_start_link(&pcie->pci); + if (ret < 0) + goto fail_host_init; + + return 0; + +fail_host_init: + tegra_pcie_unconfig_controller(pcie); + return ret; +} + +static int tegra_pcie_dw_resume_early(struct device *dev) +{ + struct tegra_pcie_dw *pcie = dev_get_drvdata(dev); + u32 val; + + if (pcie->of_data->mode == DW_PCIE_EP_TYPE) { + enable_irq(pcie->pex_rst_irq); + return 0; + } + + if (!pcie->link_state) + return 0; + + /* Disable HW_HOT_RST mode */ + if (!pcie->of_data->has_sbr_reset_fix) { + val = appl_readl(pcie, APPL_CTRL); + val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK << + APPL_CTRL_HW_HOT_RST_MODE_SHIFT); + val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST << + APPL_CTRL_HW_HOT_RST_MODE_SHIFT; + val &= ~APPL_CTRL_HW_HOT_RST_EN; + appl_writel(pcie, val, APPL_CTRL); + } + + return 0; +} + +static void tegra_pcie_dw_shutdown(struct platform_device *pdev) +{ + struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev); + + if (pcie->of_data->mode == DW_PCIE_RC_TYPE) { + if (!pcie->link_state) + return; + + debugfs_remove_recursive(pcie->debugfs); + + disable_irq(pcie->pci.pp.irq); + if (IS_ENABLED(CONFIG_PCI_MSI)) + disable_irq(pcie->pci.pp.msi_irq[0]); + + tegra_pcie_dw_pme_turnoff(pcie); + tegra_pcie_unconfig_controller(pcie); + pm_runtime_put_sync(pcie->dev); + } else { + disable_irq(pcie->pex_rst_irq); + pex_ep_event_pex_rst_assert(pcie); + } +} + +static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_rc_of_data = { + .version = TEGRA194_DWC_IP_VER, + .mode = DW_PCIE_RC_TYPE, + .cdm_chk_int_en_bit = BIT(19), + /* Gen4 - 5, 6, 8 and 9 presets enabled */ + .gen4_preset_vec = 0x360, + .n_fts = { 52, 52 }, +}; + +static const struct tegra_pcie_dw_of_data tegra194_pcie_dw_ep_of_data = { + .version = TEGRA194_DWC_IP_VER, + .mode = DW_PCIE_EP_TYPE, + .cdm_chk_int_en_bit = BIT(19), + /* Gen4 - 5, 6, 8 and 9 presets enabled */ + .gen4_preset_vec = 0x360, + .n_fts = { 52, 52 }, +}; + +static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_rc_of_data = { + .version = TEGRA234_DWC_IP_VER, + .mode = DW_PCIE_RC_TYPE, + .has_msix_doorbell_access_fix = true, + .has_sbr_reset_fix = true, + .has_l1ss_exit_fix = true, + .cdm_chk_int_en_bit = BIT(18), + /* Gen4 - 6, 8 and 9 presets enabled */ + .gen4_preset_vec = 0x340, + .n_fts = { 52, 80 }, +}; + +static const struct tegra_pcie_dw_of_data tegra234_pcie_dw_ep_of_data = { + .version = TEGRA234_DWC_IP_VER, + .mode = DW_PCIE_EP_TYPE, + .has_l1ss_exit_fix = true, + .has_ltr_req_fix = true, + .disable_l1_2 = true, + .cdm_chk_int_en_bit = BIT(18), + /* Gen4 - 6, 8 and 9 presets enabled */ + .gen4_preset_vec = 0x340, + .n_fts = { 52, 80 }, +}; + +static const struct of_device_id tegra_pcie_dw_of_match[] = { + { + .compatible = "nvidia,tegra194-pcie", + .data = &tegra194_pcie_dw_rc_of_data, + }, + { + .compatible = "nvidia,tegra194-pcie-ep", + .data = &tegra194_pcie_dw_ep_of_data, + }, + { + .compatible = "nvidia,tegra234-pcie", + .data = &tegra234_pcie_dw_rc_of_data, + }, + { + .compatible = "nvidia,tegra234-pcie-ep", + .data = &tegra234_pcie_dw_ep_of_data, + }, + {} +}; + +static const struct dev_pm_ops tegra_pcie_dw_pm_ops = { + .suspend = tegra_pcie_dw_suspend, + .suspend_late = tegra_pcie_dw_suspend_late, + .suspend_noirq = tegra_pcie_dw_suspend_noirq, + .resume_noirq = tegra_pcie_dw_resume_noirq, + .resume_early = tegra_pcie_dw_resume_early, +}; + +static struct platform_driver tegra_pcie_dw_driver = { + .probe = tegra_pcie_dw_probe, + .remove = tegra_pcie_dw_remove, + .shutdown = tegra_pcie_dw_shutdown, + .driver = { + .name = "tegra194-pcie", + .pm = &tegra_pcie_dw_pm_ops, + .of_match_table = tegra_pcie_dw_of_match, + }, +}; +module_platform_driver(tegra_pcie_dw_driver); + +MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match); + +MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>"); +MODULE_DESCRIPTION("NVIDIA PCIe host controller driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/dwc/pcie-ultrarisc.c b/drivers/pci/controller/dwc/pcie-ultrarisc.c new file mode 100644 index 000000000..51e891804 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-ultrarisc.c @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DWC PCIe RC driver for UltraRISC SoCs + * + * Copyright (C) 2026 UltraRISC Technology (Shanghai) Co., Ltd. + */ + +#include <linux/clk.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of_device.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/resource.h> + +#include "pcie-designware.h" + +#define PCIE_CUS_CORE 0x400000 + +#define LTSSM_ENABLE BIT(7) +#define FAST_LINK_MODE BIT(12) +#define HOLD_PHY_RST BIT(14) +#define L1SUB_DISABLE BIT(15) + +#define ULTRARISC_PCIE_COMP_TIMEOUT_65_210MS 0x6 + +struct ultrarisc_pcie { + struct dw_pcie pci; + struct clk_bulk_data *clks; + int num_clks; +}; + +static struct pci_ops ultrarisc_pci_ops = { + .map_bus = dw_pcie_own_conf_map_bus, + .read = pci_generic_config_read32, + .write = pci_generic_config_write32, +}; + +static int ultrarisc_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct pci_host_bridge *bridge = pp->bridge; + u8 cap_exp; + u32 val; + + bridge->ops = &ultrarisc_pci_ops; + + if (dw_pcie_link_up(pci)) + return 0; + + val = dw_pcie_readl_dbi(pci, PCIE_CUS_CORE); + val &= ~FAST_LINK_MODE; + dw_pcie_writel_dbi(pci, PCIE_CUS_CORE, val); + + val = dw_pcie_readl_dbi(pci, PCIE_TIMER_CTRL_MAX_FUNC_NUM); + FIELD_MODIFY(PORT_FLT_SF_MASK, &val, PORT_FLT_SF_VAL_64); + dw_pcie_writel_dbi(pci, PCIE_TIMER_CTRL_MAX_FUNC_NUM, val); + + cap_exp = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP); + val = dw_pcie_readl_dbi(pci, cap_exp + PCI_EXP_LNKCTL2); + FIELD_MODIFY(PCI_EXP_LNKCTL2_TLS, &val, PCI_EXP_LNKCTL2_TLS_16_0GT); + dw_pcie_writel_dbi(pci, cap_exp + PCI_EXP_LNKCTL2, val); + + val = dw_pcie_readl_dbi(pci, PCIE_PORT_FORCE); + FIELD_MODIFY(PORT_LINK_NUM_MASK, &val, 0); + dw_pcie_writel_dbi(pci, PCIE_PORT_FORCE, val); + + val = dw_pcie_readl_dbi(pci, cap_exp + PCI_EXP_DEVCTL2); + FIELD_MODIFY(PCI_EXP_DEVCTL2_COMP_TIMEOUT, &val, + ULTRARISC_PCIE_COMP_TIMEOUT_65_210MS); + dw_pcie_writel_dbi(pci, cap_exp + PCI_EXP_DEVCTL2, val); + + val = dw_pcie_readl_dbi(pci, PCIE_CUS_CORE); + val &= ~(HOLD_PHY_RST | L1SUB_DISABLE); + dw_pcie_writel_dbi(pci, PCIE_CUS_CORE, val); + + return 0; +} + +static void ultrarisc_pcie_pme_turn_off(struct dw_pcie_rp *pp) +{ + /* + * DP1000 does not support sending PME_Turn_Off from the RC. + * Keep this callback empty to skip the generic MSG TLP path. + */ +} + +static const struct dw_pcie_host_ops ultrarisc_pcie_host_ops = { + .init = ultrarisc_pcie_host_init, + .pme_turn_off = ultrarisc_pcie_pme_turn_off, +}; + +static int ultrarisc_pcie_start_link(struct dw_pcie *pci) +{ + u32 val; + + val = dw_pcie_readl_dbi(pci, PCIE_CUS_CORE); + val |= LTSSM_ENABLE; + dw_pcie_writel_dbi(pci, PCIE_CUS_CORE, val); + + return 0; +} + +static const struct dw_pcie_ops dw_pcie_ops = { + .start_link = ultrarisc_pcie_start_link, +}; + +static int ultrarisc_pcie_enable_clks(struct ultrarisc_pcie *ultra) +{ + return clk_bulk_prepare_enable(ultra->num_clks, ultra->clks); +} + +static void ultrarisc_pcie_disable_clks(void *data) +{ + struct ultrarisc_pcie *ultra = data; + + clk_bulk_disable_unprepare(ultra->num_clks, ultra->clks); +} + +static int ultrarisc_pcie_init_clks(struct ultrarisc_pcie *ultra) +{ + struct device *dev = ultra->pci.dev; + int ret; + + ultra->num_clks = devm_clk_bulk_get_all(dev, &ultra->clks); + if (ultra->num_clks < 0) + return dev_err_probe(dev, ultra->num_clks, "Failed to get clocks\n"); + + ret = ultrarisc_pcie_enable_clks(ultra); + if (ret) + return dev_err_probe(dev, ret, "Failed to enable clocks\n"); + + return devm_add_action_or_reset(dev, ultrarisc_pcie_disable_clks, ultra); +} + +static int ultrarisc_pcie_probe(struct platform_device *pdev) +{ + struct ultrarisc_pcie *ultra; + struct device *dev = &pdev->dev; + struct dw_pcie_rp *pp; + struct dw_pcie *pci; + int ret; + + ultra = devm_kzalloc(dev, sizeof(*ultra), GFP_KERNEL); + if (!ultra) + return -ENOMEM; + + pci = &ultra->pci; + pci->dev = dev; + pci->ops = &dw_pcie_ops; + + /* Set a default value suitable for at most 16 in and 16 out windows */ + pci->atu_size = SZ_8K; + + pp = &pci->pp; + + platform_set_drvdata(pdev, ultra); + + ret = ultrarisc_pcie_init_clks(ultra); + if (ret) + return ret; + + pp->num_vectors = MAX_MSI_IRQS; + /* No L2/L3 Ready indication is available on this platform */ + pp->skip_l23_ready = true; + pp->ops = &ultrarisc_pcie_host_ops; + + ret = dw_pcie_host_init(pp); + if (ret) { + dev_err(dev, "Failed to initialize host\n"); + return ret; + } + + return 0; +} + +static int ultrarisc_pcie_suspend_noirq(struct device *dev) +{ + struct ultrarisc_pcie *ultra = dev_get_drvdata(dev); + struct dw_pcie *pci = &ultra->pci; + int ret; + + /* + * A failed resume leaves the DWC suspended and the clocks disabled. + * A later suspend must not access the controller or disable them again. + */ + if (pci->suspended) + return 0; + + ret = dw_pcie_suspend_noirq(pci); + if (ret) + return ret; + + if (pci->suspended) + ultrarisc_pcie_disable_clks(ultra); + + return 0; +} + +static int ultrarisc_pcie_resume_noirq(struct device *dev) +{ + struct ultrarisc_pcie *ultra = dev_get_drvdata(dev); + struct dw_pcie *pci = &ultra->pci; + int ret; + + if (pci->suspended) { + ret = ultrarisc_pcie_enable_clks(ultra); + if (ret) + return ret; + + ret = dw_pcie_resume_noirq(pci); + if (ret) { + ultrarisc_pcie_disable_clks(ultra); + return ret; + } + } + + return 0; +} + +static const struct dev_pm_ops ultrarisc_pcie_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(ultrarisc_pcie_suspend_noirq, + ultrarisc_pcie_resume_noirq) +}; + +static const struct of_device_id ultrarisc_pcie_of_match[] = { + { + .compatible = "ultrarisc,dp1000-pcie", + }, + {}, +}; +MODULE_DEVICE_TABLE(of, ultrarisc_pcie_of_match); + +static struct platform_driver ultrarisc_pcie_driver = { + .driver = { + .name = "ultrarisc-pcie", + .of_match_table = ultrarisc_pcie_of_match, + .suppress_bind_attrs = true, + .pm = &ultrarisc_pcie_pm_ops, + }, + .probe = ultrarisc_pcie_probe, +}; +module_platform_driver(ultrarisc_pcie_driver); + +MODULE_DESCRIPTION("UltraRISC DP1000 DWC PCIe host controller"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pci/controller/dwc/pcie-uniphier-ep.c b/drivers/pci/controller/dwc/pcie-uniphier-ep.c new file mode 100644 index 000000000..89fb78200 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-uniphier-ep.c @@ -0,0 +1,461 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe endpoint controller driver for UniPhier SoCs + * Copyright 2018 Socionext Inc. + * Author: Kunihiko Hayashi <hayashi.kunihiko@socionext.com> + */ + +#include <linux/bitops.h> +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/init.h> +#include <linux/iopoll.h> +#include <linux/of.h> +#include <linux/pci.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/reset.h> + +#include "pcie-designware.h" + +/* Link Glue registers */ +#define PCL_RSTCTRL0 0x0010 +#define PCL_RSTCTRL_AXI_REG BIT(3) +#define PCL_RSTCTRL_AXI_SLAVE BIT(2) +#define PCL_RSTCTRL_AXI_MASTER BIT(1) +#define PCL_RSTCTRL_PIPE3 BIT(0) + +#define PCL_RSTCTRL1 0x0020 +#define PCL_RSTCTRL_PERST BIT(0) + +#define PCL_RSTCTRL2 0x0024 +#define PCL_RSTCTRL_PHY_RESET BIT(0) + +#define PCL_PINCTRL0 0x002c +#define PCL_PERST_PLDN_REGEN BIT(12) +#define PCL_PERST_NOE_REGEN BIT(11) +#define PCL_PERST_OUT_REGEN BIT(8) +#define PCL_PERST_PLDN_REGVAL BIT(4) +#define PCL_PERST_NOE_REGVAL BIT(3) +#define PCL_PERST_OUT_REGVAL BIT(0) + +#define PCL_PIPEMON 0x0044 +#define PCL_PCLK_ALIVE BIT(15) + +#define PCL_MODE 0x8000 +#define PCL_MODE_REGEN BIT(8) +#define PCL_MODE_REGVAL BIT(0) + +#define PCL_APP_CLK_CTRL 0x8004 +#define PCL_APP_CLK_REQ BIT(0) + +#define PCL_APP_READY_CTRL 0x8008 +#define PCL_APP_LTSSM_ENABLE BIT(0) + +#define PCL_APP_MSI0 0x8040 +#define PCL_APP_VEN_MSI_TC_MASK GENMASK(10, 8) +#define PCL_APP_VEN_MSI_VECTOR_MASK GENMASK(4, 0) + +#define PCL_APP_MSI1 0x8044 +#define PCL_APP_MSI_REQ BIT(0) + +#define PCL_APP_INTX 0x8074 +#define PCL_APP_INTX_SYS_INT BIT(0) + +#define PCL_APP_PM0 0x8078 +#define PCL_SYS_AUX_PWR_DET BIT(8) + +/* assertion time of INTx in usec */ +#define PCL_INTX_WIDTH_USEC 30 + +struct uniphier_pcie_ep_priv { + void __iomem *base; + struct dw_pcie pci; + struct clk *clk, *clk_gio; + struct reset_control *rst, *rst_gio; + struct phy *phy; + const struct uniphier_pcie_ep_soc_data *data; +}; + +struct uniphier_pcie_ep_soc_data { + bool has_gio; + void (*init)(struct uniphier_pcie_ep_priv *priv); + int (*wait)(struct uniphier_pcie_ep_priv *priv); + const struct pci_epc_features features; +}; + +#define to_uniphier_pcie(x) dev_get_drvdata((x)->dev) + +static void uniphier_pcie_ltssm_enable(struct uniphier_pcie_ep_priv *priv, + bool enable) +{ + u32 val; + + val = readl(priv->base + PCL_APP_READY_CTRL); + if (enable) + val |= PCL_APP_LTSSM_ENABLE; + else + val &= ~PCL_APP_LTSSM_ENABLE; + writel(val, priv->base + PCL_APP_READY_CTRL); +} + +static void uniphier_pcie_phy_reset(struct uniphier_pcie_ep_priv *priv, + bool assert) +{ + u32 val; + + val = readl(priv->base + PCL_RSTCTRL2); + if (assert) + val |= PCL_RSTCTRL_PHY_RESET; + else + val &= ~PCL_RSTCTRL_PHY_RESET; + writel(val, priv->base + PCL_RSTCTRL2); +} + +static void uniphier_pcie_pro5_init_ep(struct uniphier_pcie_ep_priv *priv) +{ + u32 val; + + /* set EP mode */ + val = readl(priv->base + PCL_MODE); + val |= PCL_MODE_REGEN | PCL_MODE_REGVAL; + writel(val, priv->base + PCL_MODE); + + /* clock request */ + val = readl(priv->base + PCL_APP_CLK_CTRL); + val &= ~PCL_APP_CLK_REQ; + writel(val, priv->base + PCL_APP_CLK_CTRL); + + /* deassert PIPE3 and AXI reset */ + val = readl(priv->base + PCL_RSTCTRL0); + val |= PCL_RSTCTRL_AXI_REG | PCL_RSTCTRL_AXI_SLAVE + | PCL_RSTCTRL_AXI_MASTER | PCL_RSTCTRL_PIPE3; + writel(val, priv->base + PCL_RSTCTRL0); + + uniphier_pcie_ltssm_enable(priv, false); + + msleep(100); +} + +static void uniphier_pcie_nx1_init_ep(struct uniphier_pcie_ep_priv *priv) +{ + u32 val; + + /* set EP mode */ + val = readl(priv->base + PCL_MODE); + val |= PCL_MODE_REGEN | PCL_MODE_REGVAL; + writel(val, priv->base + PCL_MODE); + + /* use auxiliary power detection */ + val = readl(priv->base + PCL_APP_PM0); + val |= PCL_SYS_AUX_PWR_DET; + writel(val, priv->base + PCL_APP_PM0); + + /* assert PERST# */ + val = readl(priv->base + PCL_PINCTRL0); + val &= ~(PCL_PERST_NOE_REGVAL | PCL_PERST_OUT_REGVAL + | PCL_PERST_PLDN_REGVAL); + val |= PCL_PERST_NOE_REGEN | PCL_PERST_OUT_REGEN + | PCL_PERST_PLDN_REGEN; + writel(val, priv->base + PCL_PINCTRL0); + + uniphier_pcie_ltssm_enable(priv, false); + + usleep_range(100000, 200000); + + /* deassert PERST# */ + val = readl(priv->base + PCL_PINCTRL0); + val |= PCL_PERST_OUT_REGVAL | PCL_PERST_OUT_REGEN; + writel(val, priv->base + PCL_PINCTRL0); +} + +static int uniphier_pcie_nx1_wait_ep(struct uniphier_pcie_ep_priv *priv) +{ + u32 status; + int ret; + + /* wait PIPE clock */ + ret = readl_poll_timeout(priv->base + PCL_PIPEMON, status, + status & PCL_PCLK_ALIVE, 100000, 1000000); + if (ret) { + dev_err(priv->pci.dev, + "Failed to initialize controller in EP mode\n"); + return ret; + } + + return 0; +} + +static int uniphier_pcie_start_link(struct dw_pcie *pci) +{ + struct uniphier_pcie_ep_priv *priv = to_uniphier_pcie(pci); + + uniphier_pcie_ltssm_enable(priv, true); + + return 0; +} + +static void uniphier_pcie_stop_link(struct dw_pcie *pci) +{ + struct uniphier_pcie_ep_priv *priv = to_uniphier_pcie(pci); + + uniphier_pcie_ltssm_enable(priv, false); +} + +static int uniphier_pcie_ep_raise_intx_irq(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct uniphier_pcie_ep_priv *priv = to_uniphier_pcie(pci); + u32 val; + + /* + * This makes pulse signal to send INTx to the RC, so this should + * be cleared as soon as possible. This sequence is covered with + * mutex in pci_epc_raise_irq(). + */ + /* assert INTx */ + val = readl(priv->base + PCL_APP_INTX); + val |= PCL_APP_INTX_SYS_INT; + writel(val, priv->base + PCL_APP_INTX); + + udelay(PCL_INTX_WIDTH_USEC); + + /* deassert INTx */ + val &= ~PCL_APP_INTX_SYS_INT; + writel(val, priv->base + PCL_APP_INTX); + + return 0; +} + +static int uniphier_pcie_ep_raise_msi_irq(struct dw_pcie_ep *ep, + u8 func_no, u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct uniphier_pcie_ep_priv *priv = to_uniphier_pcie(pci); + u32 val; + + val = FIELD_PREP(PCL_APP_VEN_MSI_TC_MASK, func_no) + | FIELD_PREP(PCL_APP_VEN_MSI_VECTOR_MASK, interrupt_num - 1); + writel(val, priv->base + PCL_APP_MSI0); + + val = readl(priv->base + PCL_APP_MSI1); + val |= PCL_APP_MSI_REQ; + writel(val, priv->base + PCL_APP_MSI1); + + return 0; +} + +static int uniphier_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no, + unsigned int type, u16 interrupt_num) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + + switch (type) { + case PCI_IRQ_INTX: + return uniphier_pcie_ep_raise_intx_irq(ep); + case PCI_IRQ_MSI: + return uniphier_pcie_ep_raise_msi_irq(ep, func_no, + interrupt_num); + default: + dev_err(pci->dev, "UNKNOWN IRQ type (%d)\n", type); + } + + return 0; +} + +static const struct pci_epc_features* +uniphier_pcie_get_features(struct dw_pcie_ep *ep) +{ + struct dw_pcie *pci = to_dw_pcie_from_ep(ep); + struct uniphier_pcie_ep_priv *priv = to_uniphier_pcie(pci); + + return &priv->data->features; +} + +static const struct dw_pcie_ep_ops uniphier_pcie_ep_ops = { + .raise_irq = uniphier_pcie_ep_raise_irq, + .get_features = uniphier_pcie_get_features, +}; + +static int uniphier_pcie_ep_enable(struct uniphier_pcie_ep_priv *priv) +{ + int ret; + + ret = clk_prepare_enable(priv->clk); + if (ret) + return ret; + + ret = clk_prepare_enable(priv->clk_gio); + if (ret) + goto out_clk_disable; + + ret = reset_control_deassert(priv->rst); + if (ret) + goto out_clk_gio_disable; + + ret = reset_control_deassert(priv->rst_gio); + if (ret) + goto out_rst_assert; + + if (priv->data->init) + priv->data->init(priv); + + uniphier_pcie_phy_reset(priv, true); + + ret = phy_init(priv->phy); + if (ret) + goto out_rst_gio_assert; + + uniphier_pcie_phy_reset(priv, false); + + if (priv->data->wait) { + ret = priv->data->wait(priv); + if (ret) + goto out_phy_exit; + } + + return 0; + +out_phy_exit: + phy_exit(priv->phy); +out_rst_gio_assert: + reset_control_assert(priv->rst_gio); +out_rst_assert: + reset_control_assert(priv->rst); +out_clk_gio_disable: + clk_disable_unprepare(priv->clk_gio); +out_clk_disable: + clk_disable_unprepare(priv->clk); + + return ret; +} + +static const struct dw_pcie_ops dw_pcie_ops = { + .start_link = uniphier_pcie_start_link, + .stop_link = uniphier_pcie_stop_link, +}; + +static int uniphier_pcie_ep_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct uniphier_pcie_ep_priv *priv; + int ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->data = of_device_get_match_data(dev); + if (WARN_ON(!priv->data)) + return -EINVAL; + + priv->pci.dev = dev; + priv->pci.ops = &dw_pcie_ops; + + priv->base = devm_platform_ioremap_resource_byname(pdev, "link"); + if (IS_ERR(priv->base)) + return PTR_ERR(priv->base); + + if (priv->data->has_gio) { + priv->clk_gio = devm_clk_get(dev, "gio"); + if (IS_ERR(priv->clk_gio)) + return PTR_ERR(priv->clk_gio); + + priv->rst_gio = devm_reset_control_get_shared(dev, "gio"); + if (IS_ERR(priv->rst_gio)) + return PTR_ERR(priv->rst_gio); + } + + priv->clk = devm_clk_get(dev, "link"); + if (IS_ERR(priv->clk)) + return PTR_ERR(priv->clk); + + priv->rst = devm_reset_control_get_shared(dev, "link"); + if (IS_ERR(priv->rst)) + return PTR_ERR(priv->rst); + + priv->phy = devm_phy_optional_get(dev, "pcie-phy"); + if (IS_ERR(priv->phy)) { + ret = PTR_ERR(priv->phy); + dev_err(dev, "Failed to get phy (%d)\n", ret); + return ret; + } + + platform_set_drvdata(pdev, priv); + + ret = uniphier_pcie_ep_enable(priv); + if (ret) + return ret; + + priv->pci.ep.ops = &uniphier_pcie_ep_ops; + ret = dw_pcie_ep_init(&priv->pci.ep); + if (ret) + return ret; + + ret = dw_pcie_ep_init_registers(&priv->pci.ep); + if (ret) { + dev_err(dev, "Failed to initialize DWC endpoint registers\n"); + dw_pcie_ep_deinit(&priv->pci.ep); + return ret; + } + + pci_epc_init_notify(priv->pci.ep.epc); + + return 0; +} + +static const struct uniphier_pcie_ep_soc_data uniphier_pro5_data = { + .has_gio = true, + .init = uniphier_pcie_pro5_init_ep, + .wait = NULL, + .features = { + DWC_EPC_COMMON_FEATURES, + .linkup_notifier = false, + .msi_capable = true, + .msix_capable = false, + .align = 1 << 16, + .bar[BAR_0] = { .only_64bit = true, }, + .bar[BAR_2] = { .only_64bit = true, }, + .bar[BAR_4] = { .type = BAR_DISABLED, }, + .bar[BAR_5] = { .type = BAR_DISABLED, }, + }, +}; + +static const struct uniphier_pcie_ep_soc_data uniphier_nx1_data = { + .has_gio = false, + .init = uniphier_pcie_nx1_init_ep, + .wait = uniphier_pcie_nx1_wait_ep, + .features = { + DWC_EPC_COMMON_FEATURES, + .linkup_notifier = false, + .msi_capable = true, + .msix_capable = false, + .align = 1 << 12, + .bar[BAR_0] = { .only_64bit = true, }, + .bar[BAR_2] = { .only_64bit = true, }, + .bar[BAR_4] = { .only_64bit = true, }, + }, +}; + +static const struct of_device_id uniphier_pcie_ep_match[] = { + { + .compatible = "socionext,uniphier-pro5-pcie-ep", + .data = &uniphier_pro5_data, + }, + { + .compatible = "socionext,uniphier-nx1-pcie-ep", + .data = &uniphier_nx1_data, + }, + { /* sentinel */ }, +}; + +static struct platform_driver uniphier_pcie_ep_driver = { + .probe = uniphier_pcie_ep_probe, + .driver = { + .name = "uniphier-pcie-ep", + .of_match_table = uniphier_pcie_ep_match, + .suppress_bind_attrs = true, + }, +}; +builtin_platform_driver(uniphier_pcie_ep_driver); diff --git a/drivers/pci/controller/dwc/pcie-uniphier.c b/drivers/pci/controller/dwc/pcie-uniphier.c new file mode 100644 index 000000000..297e7a3d9 --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-uniphier.c @@ -0,0 +1,409 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for UniPhier SoCs + * Copyright 2018 Socionext Inc. + * Author: Kunihiko Hayashi <hayashi.kunihiko@socionext.com> + */ + +#include <linux/bitops.h> +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqdomain.h> +#include <linux/of_irq.h> +#include <linux/pci.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/reset.h> + +#include "pcie-designware.h" + +#define PCL_PINCTRL0 0x002c +#define PCL_PERST_PLDN_REGEN BIT(12) +#define PCL_PERST_NOE_REGEN BIT(11) +#define PCL_PERST_OUT_REGEN BIT(8) +#define PCL_PERST_PLDN_REGVAL BIT(4) +#define PCL_PERST_NOE_REGVAL BIT(3) +#define PCL_PERST_OUT_REGVAL BIT(0) + +#define PCL_PIPEMON 0x0044 +#define PCL_PCLK_ALIVE BIT(15) + +#define PCL_MODE 0x8000 +#define PCL_MODE_REGEN BIT(8) +#define PCL_MODE_REGVAL BIT(0) + +#define PCL_APP_READY_CTRL 0x8008 +#define PCL_APP_LTSSM_ENABLE BIT(0) + +#define PCL_APP_PM0 0x8078 +#define PCL_SYS_AUX_PWR_DET BIT(8) + +#define PCL_RCV_INT 0x8108 +#define PCL_RCV_INT_ALL_ENABLE GENMASK(20, 17) +#define PCL_CFG_BW_MGT_STATUS BIT(4) +#define PCL_CFG_LINK_AUTO_BW_STATUS BIT(3) +#define PCL_CFG_AER_RC_ERR_MSI_STATUS BIT(2) +#define PCL_CFG_PME_MSI_STATUS BIT(1) + +#define PCL_RCV_INTX 0x810c +#define PCL_RCV_INTX_ALL_ENABLE GENMASK(19, 16) +#define PCL_RCV_INTX_ALL_MASK GENMASK(11, 8) +#define PCL_RCV_INTX_MASK_SHIFT 8 +#define PCL_RCV_INTX_ALL_STATUS GENMASK(3, 0) +#define PCL_RCV_INTX_STATUS_SHIFT 0 + +#define PCL_STATUS_LINK 0x8140 +#define PCL_RDLH_LINK_UP BIT(1) +#define PCL_XMLH_LINK_UP BIT(0) + +struct uniphier_pcie { + struct dw_pcie pci; + void __iomem *base; + struct clk *clk; + struct reset_control *rst; + struct phy *phy; + struct irq_domain *intx_irq_domain; +}; + +#define to_uniphier_pcie(x) dev_get_drvdata((x)->dev) + +static void uniphier_pcie_ltssm_enable(struct uniphier_pcie *pcie, + bool enable) +{ + u32 val; + + val = readl(pcie->base + PCL_APP_READY_CTRL); + if (enable) + val |= PCL_APP_LTSSM_ENABLE; + else + val &= ~PCL_APP_LTSSM_ENABLE; + writel(val, pcie->base + PCL_APP_READY_CTRL); +} + +static void uniphier_pcie_init_rc(struct uniphier_pcie *pcie) +{ + u32 val; + + /* set RC MODE */ + val = readl(pcie->base + PCL_MODE); + val |= PCL_MODE_REGEN; + val &= ~PCL_MODE_REGVAL; + writel(val, pcie->base + PCL_MODE); + + /* use auxiliary power detection */ + val = readl(pcie->base + PCL_APP_PM0); + val |= PCL_SYS_AUX_PWR_DET; + writel(val, pcie->base + PCL_APP_PM0); + + /* assert PERST# */ + val = readl(pcie->base + PCL_PINCTRL0); + val &= ~(PCL_PERST_NOE_REGVAL | PCL_PERST_OUT_REGVAL + | PCL_PERST_PLDN_REGVAL); + val |= PCL_PERST_NOE_REGEN | PCL_PERST_OUT_REGEN + | PCL_PERST_PLDN_REGEN; + writel(val, pcie->base + PCL_PINCTRL0); + + uniphier_pcie_ltssm_enable(pcie, false); + + usleep_range(100000, 200000); + + /* deassert PERST# */ + val = readl(pcie->base + PCL_PINCTRL0); + val |= PCL_PERST_OUT_REGVAL | PCL_PERST_OUT_REGEN; + writel(val, pcie->base + PCL_PINCTRL0); +} + +static int uniphier_pcie_wait_rc(struct uniphier_pcie *pcie) +{ + u32 status; + int ret; + + /* wait PIPE clock */ + ret = readl_poll_timeout(pcie->base + PCL_PIPEMON, status, + status & PCL_PCLK_ALIVE, 100000, 1000000); + if (ret) { + dev_err(pcie->pci.dev, + "Failed to initialize controller in RC mode\n"); + return ret; + } + + return 0; +} + +static bool uniphier_pcie_link_up(struct dw_pcie *pci) +{ + struct uniphier_pcie *pcie = to_uniphier_pcie(pci); + u32 val, mask; + + val = readl(pcie->base + PCL_STATUS_LINK); + mask = PCL_RDLH_LINK_UP | PCL_XMLH_LINK_UP; + + return (val & mask) == mask; +} + +static int uniphier_pcie_start_link(struct dw_pcie *pci) +{ + struct uniphier_pcie *pcie = to_uniphier_pcie(pci); + + uniphier_pcie_ltssm_enable(pcie, true); + + return 0; +} + +static void uniphier_pcie_stop_link(struct dw_pcie *pci) +{ + struct uniphier_pcie *pcie = to_uniphier_pcie(pci); + + uniphier_pcie_ltssm_enable(pcie, false); +} + +static void uniphier_pcie_irq_enable(struct uniphier_pcie *pcie) +{ + writel(PCL_RCV_INT_ALL_ENABLE, pcie->base + PCL_RCV_INT); + writel(PCL_RCV_INTX_ALL_ENABLE, pcie->base + PCL_RCV_INTX); +} + + +static void uniphier_pcie_irq_mask(struct irq_data *d) +{ + struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct uniphier_pcie *pcie = to_uniphier_pcie(pci); + unsigned long flags; + u32 val; + + raw_spin_lock_irqsave(&pp->lock, flags); + + val = readl(pcie->base + PCL_RCV_INTX); + val |= BIT(irqd_to_hwirq(d) + PCL_RCV_INTX_MASK_SHIFT); + writel(val, pcie->base + PCL_RCV_INTX); + + raw_spin_unlock_irqrestore(&pp->lock, flags); +} + +static void uniphier_pcie_irq_unmask(struct irq_data *d) +{ + struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d); + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct uniphier_pcie *pcie = to_uniphier_pcie(pci); + unsigned long flags; + u32 val; + + raw_spin_lock_irqsave(&pp->lock, flags); + + val = readl(pcie->base + PCL_RCV_INTX); + val &= ~BIT(irqd_to_hwirq(d) + PCL_RCV_INTX_MASK_SHIFT); + writel(val, pcie->base + PCL_RCV_INTX); + + raw_spin_unlock_irqrestore(&pp->lock, flags); +} + +static struct irq_chip uniphier_pcie_irq_chip = { + .name = "PCI", + .irq_mask = uniphier_pcie_irq_mask, + .irq_unmask = uniphier_pcie_irq_unmask, +}; + +static int uniphier_pcie_intx_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &uniphier_pcie_irq_chip, + handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + + return 0; +} + +static const struct irq_domain_ops uniphier_intx_domain_ops = { + .map = uniphier_pcie_intx_map, +}; + +static void uniphier_pcie_irq_handler(struct irq_desc *desc) +{ + struct dw_pcie_rp *pp = irq_desc_get_handler_data(desc); + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct uniphier_pcie *pcie = to_uniphier_pcie(pci); + struct irq_chip *chip = irq_desc_get_chip(desc); + unsigned long reg; + u32 val, bit; + + /* INT for debug */ + val = readl(pcie->base + PCL_RCV_INT); + + if (val & PCL_CFG_BW_MGT_STATUS) + dev_dbg(pci->dev, "Link Bandwidth Management Event\n"); + if (val & PCL_CFG_LINK_AUTO_BW_STATUS) + dev_dbg(pci->dev, "Link Autonomous Bandwidth Event\n"); + if (val & PCL_CFG_AER_RC_ERR_MSI_STATUS) + dev_dbg(pci->dev, "Root Error\n"); + if (val & PCL_CFG_PME_MSI_STATUS) + dev_dbg(pci->dev, "PME Interrupt\n"); + + writel(val, pcie->base + PCL_RCV_INT); + + /* INTx */ + chained_irq_enter(chip, desc); + + val = readl(pcie->base + PCL_RCV_INTX); + reg = FIELD_GET(PCL_RCV_INTX_ALL_STATUS, val); + + for_each_set_bit(bit, ®, PCI_NUM_INTX) + generic_handle_domain_irq(pcie->intx_irq_domain, bit); + + chained_irq_exit(chip, desc); +} + +static int uniphier_pcie_config_intx_irq(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct uniphier_pcie *pcie = to_uniphier_pcie(pci); + struct device_node *np = pci->dev->of_node; + struct device_node *np_intc; + int ret = 0; + + np_intc = of_get_child_by_name(np, "legacy-interrupt-controller"); + if (!np_intc) { + dev_err(pci->dev, "Failed to get legacy-interrupt-controller node\n"); + return -EINVAL; + } + + pp->irq = irq_of_parse_and_map(np_intc, 0); + if (!pp->irq) { + dev_err(pci->dev, "Failed to get an IRQ entry in legacy-interrupt-controller\n"); + ret = -EINVAL; + goto out_put_node; + } + + pcie->intx_irq_domain = irq_domain_create_linear(of_fwnode_handle(np_intc), PCI_NUM_INTX, + &uniphier_intx_domain_ops, pp); + if (!pcie->intx_irq_domain) { + dev_err(pci->dev, "Failed to get INTx domain\n"); + ret = -ENODEV; + goto out_put_node; + } + + irq_set_chained_handler_and_data(pp->irq, uniphier_pcie_irq_handler, + pp); + +out_put_node: + of_node_put(np_intc); + return ret; +} + +static int uniphier_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct uniphier_pcie *pcie = to_uniphier_pcie(pci); + int ret; + + ret = uniphier_pcie_config_intx_irq(pp); + if (ret) + return ret; + + uniphier_pcie_irq_enable(pcie); + + return 0; +} + +static const struct dw_pcie_host_ops uniphier_pcie_host_ops = { + .init = uniphier_pcie_host_init, +}; + +static int uniphier_pcie_host_enable(struct uniphier_pcie *pcie) +{ + int ret; + + ret = clk_prepare_enable(pcie->clk); + if (ret) + return ret; + + ret = reset_control_deassert(pcie->rst); + if (ret) + goto out_clk_disable; + + uniphier_pcie_init_rc(pcie); + + ret = phy_init(pcie->phy); + if (ret) + goto out_rst_assert; + + ret = uniphier_pcie_wait_rc(pcie); + if (ret) + goto out_phy_exit; + + return 0; + +out_phy_exit: + phy_exit(pcie->phy); +out_rst_assert: + reset_control_assert(pcie->rst); +out_clk_disable: + clk_disable_unprepare(pcie->clk); + + return ret; +} + +static const struct dw_pcie_ops dw_pcie_ops = { + .start_link = uniphier_pcie_start_link, + .stop_link = uniphier_pcie_stop_link, + .link_up = uniphier_pcie_link_up, +}; + +static int uniphier_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct uniphier_pcie *pcie; + int ret; + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + pcie->pci.dev = dev; + pcie->pci.ops = &dw_pcie_ops; + + pcie->base = devm_platform_ioremap_resource_byname(pdev, "link"); + if (IS_ERR(pcie->base)) + return PTR_ERR(pcie->base); + + pcie->clk = devm_clk_get(dev, NULL); + if (IS_ERR(pcie->clk)) + return PTR_ERR(pcie->clk); + + pcie->rst = devm_reset_control_get_shared(dev, NULL); + if (IS_ERR(pcie->rst)) + return PTR_ERR(pcie->rst); + + pcie->phy = devm_phy_optional_get(dev, "pcie-phy"); + if (IS_ERR(pcie->phy)) + return PTR_ERR(pcie->phy); + + platform_set_drvdata(pdev, pcie); + + ret = uniphier_pcie_host_enable(pcie); + if (ret) + return ret; + + pcie->pci.pp.ops = &uniphier_pcie_host_ops; + + return dw_pcie_host_init(&pcie->pci.pp); +} + +static const struct of_device_id uniphier_pcie_match[] = { + { .compatible = "socionext,uniphier-pcie", }, + { /* sentinel */ }, +}; + +static struct platform_driver uniphier_pcie_driver = { + .probe = uniphier_pcie_probe, + .driver = { + .name = "uniphier-pcie", + .of_match_table = uniphier_pcie_match, + }, +}; +builtin_platform_driver(uniphier_pcie_driver); diff --git a/drivers/pci/controller/dwc/pcie-visconti.c b/drivers/pci/controller/dwc/pcie-visconti.c new file mode 100644 index 000000000..f21775c3b --- /dev/null +++ b/drivers/pci/controller/dwc/pcie-visconti.c @@ -0,0 +1,328 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DWC PCIe RC driver for Toshiba Visconti ARM SoC + * + * Copyright (C) 2021 Toshiba Electronic Device & Storage Corporation + * Copyright (C) 2021 TOSHIBA CORPORATION + * + * Nobuhiro Iwamatsu <nobuhiro1.iwamatsu@toshiba.co.jp> + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/interrupt.h> +#include <linux/init.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/of_platform.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/resource.h> +#include <linux/types.h> + +#include "pcie-designware.h" +#include "../../pci.h" + +struct visconti_pcie { + struct dw_pcie pci; + void __iomem *ulreg_base; + void __iomem *smu_base; + void __iomem *mpu_base; + struct clk *refclk; + struct clk *coreclk; + struct clk *auxclk; +}; + +#define PCIE_UL_REG_S_PCIE_MODE 0x00F4 +#define PCIE_UL_REG_S_PCIE_MODE_EP 0x00 +#define PCIE_UL_REG_S_PCIE_MODE_RC 0x04 + +#define PCIE_UL_REG_S_PERSTN_CTRL 0x00F8 +#define PCIE_UL_IOM_PCIE_PERSTN_I_EN BIT(3) +#define PCIE_UL_DIRECT_PERSTN_EN BIT(2) +#define PCIE_UL_PERSTN_OUT BIT(1) +#define PCIE_UL_DIRECT_PERSTN BIT(0) +#define PCIE_UL_REG_S_PERSTN_CTRL_INIT (PCIE_UL_IOM_PCIE_PERSTN_I_EN | \ + PCIE_UL_DIRECT_PERSTN_EN | \ + PCIE_UL_DIRECT_PERSTN) + +#define PCIE_UL_REG_S_PHY_INIT_02 0x0104 +#define PCIE_UL_PHY0_SRAM_EXT_LD_DONE BIT(0) + +#define PCIE_UL_REG_S_PHY_INIT_03 0x0108 +#define PCIE_UL_PHY0_SRAM_INIT_DONE BIT(0) + +#define PCIE_UL_REG_S_INT_EVENT_MASK1 0x0138 +#define PCIE_UL_CFG_PME_INT BIT(0) +#define PCIE_UL_CFG_LINK_EQ_REQ_INT BIT(1) +#define PCIE_UL_EDMA_INT0 BIT(2) +#define PCIE_UL_EDMA_INT1 BIT(3) +#define PCIE_UL_EDMA_INT2 BIT(4) +#define PCIE_UL_EDMA_INT3 BIT(5) +#define PCIE_UL_S_INT_EVENT_MASK1_ALL (PCIE_UL_CFG_PME_INT | \ + PCIE_UL_CFG_LINK_EQ_REQ_INT | \ + PCIE_UL_EDMA_INT0 | \ + PCIE_UL_EDMA_INT1 | \ + PCIE_UL_EDMA_INT2 | \ + PCIE_UL_EDMA_INT3) + +#define PCIE_UL_REG_S_SB_MON 0x0198 +#define PCIE_UL_REG_S_SIG_MON 0x019C +#define PCIE_UL_CORE_RST_N_MON BIT(0) + +#define PCIE_UL_REG_V_SII_DBG_00 0x0844 +#define PCIE_UL_REG_V_SII_GEN_CTRL_01 0x0860 +#define PCIE_UL_APP_LTSSM_ENABLE BIT(0) + +#define PCIE_UL_REG_V_PHY_ST_00 0x0864 +#define PCIE_UL_SMLH_LINK_UP BIT(0) + +#define PCIE_UL_REG_V_PHY_ST_02 0x0868 +#define PCIE_UL_S_DETECT_ACT 0x01 +#define PCIE_UL_S_L0 0x11 + +#define PISMU_CKON_PCIE 0x0038 +#define PISMU_CKON_PCIE_AUX_CLK BIT(1) +#define PISMU_CKON_PCIE_MSTR_ACLK BIT(0) + +#define PISMU_RSOFF_PCIE 0x0538 +#define PISMU_RSOFF_PCIE_ULREG_RST_N BIT(1) +#define PISMU_RSOFF_PCIE_PWR_UP_RST_N BIT(0) + +#define PCIE_MPU_REG_MP_EN 0x0 +#define MPU_MP_EN_DISABLE BIT(0) + +/* Access registers in PCIe ulreg */ +static void visconti_ulreg_writel(struct visconti_pcie *pcie, u32 val, u32 reg) +{ + writel_relaxed(val, pcie->ulreg_base + reg); +} + +static u32 visconti_ulreg_readl(struct visconti_pcie *pcie, u32 reg) +{ + return readl_relaxed(pcie->ulreg_base + reg); +} + +/* Access registers in PCIe smu */ +static void visconti_smu_writel(struct visconti_pcie *pcie, u32 val, u32 reg) +{ + writel_relaxed(val, pcie->smu_base + reg); +} + +/* Access registers in PCIe mpu */ +static void visconti_mpu_writel(struct visconti_pcie *pcie, u32 val, u32 reg) +{ + writel_relaxed(val, pcie->mpu_base + reg); +} + +static u32 visconti_mpu_readl(struct visconti_pcie *pcie, u32 reg) +{ + return readl_relaxed(pcie->mpu_base + reg); +} + +static bool visconti_pcie_link_up(struct dw_pcie *pci) +{ + struct visconti_pcie *pcie = dev_get_drvdata(pci->dev); + void __iomem *addr = pcie->ulreg_base; + u32 val = readl_relaxed(addr + PCIE_UL_REG_V_PHY_ST_02); + + return val & PCIE_UL_S_L0; +} + +static int visconti_pcie_start_link(struct dw_pcie *pci) +{ + struct visconti_pcie *pcie = dev_get_drvdata(pci->dev); + void __iomem *addr = pcie->ulreg_base; + u32 val; + int ret; + + visconti_ulreg_writel(pcie, PCIE_UL_APP_LTSSM_ENABLE, + PCIE_UL_REG_V_SII_GEN_CTRL_01); + + ret = readl_relaxed_poll_timeout(addr + PCIE_UL_REG_V_PHY_ST_02, + val, (val & PCIE_UL_S_L0), + 90000, 100000); + if (ret) + return ret; + + visconti_ulreg_writel(pcie, PCIE_UL_S_INT_EVENT_MASK1_ALL, + PCIE_UL_REG_S_INT_EVENT_MASK1); + + if (dw_pcie_link_up(pci)) { + val = visconti_mpu_readl(pcie, PCIE_MPU_REG_MP_EN); + visconti_mpu_writel(pcie, val & ~MPU_MP_EN_DISABLE, + PCIE_MPU_REG_MP_EN); + } + + return 0; +} + +static void visconti_pcie_stop_link(struct dw_pcie *pci) +{ + struct visconti_pcie *pcie = dev_get_drvdata(pci->dev); + u32 val; + + val = visconti_ulreg_readl(pcie, PCIE_UL_REG_V_SII_GEN_CTRL_01); + val &= ~PCIE_UL_APP_LTSSM_ENABLE; + visconti_ulreg_writel(pcie, val, PCIE_UL_REG_V_SII_GEN_CTRL_01); + + val = visconti_mpu_readl(pcie, PCIE_MPU_REG_MP_EN); + visconti_mpu_writel(pcie, val | MPU_MP_EN_DISABLE, PCIE_MPU_REG_MP_EN); +} + +/* + * In this SoC specification, the CPU bus outputs the offset value from + * 0x40000000 to the PCIe bus, so 0x40000000 is subtracted from the CPU + * bus address. This 0x40000000 is also based on io_base from DT. + */ +static u64 visconti_pcie_cpu_addr_fixup(struct dw_pcie *pci, u64 cpu_addr) +{ + struct dw_pcie_rp *pp = &pci->pp; + + return cpu_addr & ~pp->io_base; +} + +static const struct dw_pcie_ops dw_pcie_ops = { + .cpu_addr_fixup = visconti_pcie_cpu_addr_fixup, + .link_up = visconti_pcie_link_up, + .start_link = visconti_pcie_start_link, + .stop_link = visconti_pcie_stop_link, +}; + +static int visconti_pcie_host_init(struct dw_pcie_rp *pp) +{ + struct dw_pcie *pci = to_dw_pcie_from_pp(pp); + struct visconti_pcie *pcie = dev_get_drvdata(pci->dev); + void __iomem *addr; + int err; + u32 val; + + visconti_smu_writel(pcie, + PISMU_CKON_PCIE_AUX_CLK | PISMU_CKON_PCIE_MSTR_ACLK, + PISMU_CKON_PCIE); + ndelay(250); + + visconti_smu_writel(pcie, PISMU_RSOFF_PCIE_ULREG_RST_N, + PISMU_RSOFF_PCIE); + visconti_ulreg_writel(pcie, PCIE_UL_REG_S_PCIE_MODE_RC, + PCIE_UL_REG_S_PCIE_MODE); + + val = PCIE_UL_REG_S_PERSTN_CTRL_INIT; + visconti_ulreg_writel(pcie, val, PCIE_UL_REG_S_PERSTN_CTRL); + udelay(100); + + val |= PCIE_UL_PERSTN_OUT; + visconti_ulreg_writel(pcie, val, PCIE_UL_REG_S_PERSTN_CTRL); + udelay(100); + + visconti_smu_writel(pcie, PISMU_RSOFF_PCIE_PWR_UP_RST_N, + PISMU_RSOFF_PCIE); + + addr = pcie->ulreg_base + PCIE_UL_REG_S_PHY_INIT_03; + err = readl_relaxed_poll_timeout(addr, val, + (val & PCIE_UL_PHY0_SRAM_INIT_DONE), + 100, 1000); + if (err) + return err; + + visconti_ulreg_writel(pcie, PCIE_UL_PHY0_SRAM_EXT_LD_DONE, + PCIE_UL_REG_S_PHY_INIT_02); + + addr = pcie->ulreg_base + PCIE_UL_REG_S_SIG_MON; + return readl_relaxed_poll_timeout(addr, val, + (val & PCIE_UL_CORE_RST_N_MON), 100, + 1000); +} + +static const struct dw_pcie_host_ops visconti_pcie_host_ops = { + .init = visconti_pcie_host_init, +}; + +static int visconti_get_resources(struct platform_device *pdev, + struct visconti_pcie *pcie) +{ + struct device *dev = &pdev->dev; + + pcie->ulreg_base = devm_platform_ioremap_resource_byname(pdev, "ulreg"); + if (IS_ERR(pcie->ulreg_base)) + return PTR_ERR(pcie->ulreg_base); + + pcie->smu_base = devm_platform_ioremap_resource_byname(pdev, "smu"); + if (IS_ERR(pcie->smu_base)) + return PTR_ERR(pcie->smu_base); + + pcie->mpu_base = devm_platform_ioremap_resource_byname(pdev, "mpu"); + if (IS_ERR(pcie->mpu_base)) + return PTR_ERR(pcie->mpu_base); + + pcie->refclk = devm_clk_get(dev, "ref"); + if (IS_ERR(pcie->refclk)) + return dev_err_probe(dev, PTR_ERR(pcie->refclk), + "Failed to get ref clock\n"); + + pcie->coreclk = devm_clk_get(dev, "core"); + if (IS_ERR(pcie->coreclk)) + return dev_err_probe(dev, PTR_ERR(pcie->coreclk), + "Failed to get core clock\n"); + + pcie->auxclk = devm_clk_get(dev, "aux"); + if (IS_ERR(pcie->auxclk)) + return dev_err_probe(dev, PTR_ERR(pcie->auxclk), + "Failed to get aux clock\n"); + + return 0; +} + +static int visconti_add_pcie_port(struct visconti_pcie *pcie, + struct platform_device *pdev) +{ + struct dw_pcie *pci = &pcie->pci; + struct dw_pcie_rp *pp = &pci->pp; + + pp->irq = platform_get_irq_byname(pdev, "intr"); + if (pp->irq < 0) + return pp->irq; + + pp->ops = &visconti_pcie_host_ops; + + return dw_pcie_host_init(pp); +} + +static int visconti_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct visconti_pcie *pcie; + struct dw_pcie *pci; + int ret; + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + pci = &pcie->pci; + pci->dev = dev; + pci->ops = &dw_pcie_ops; + + ret = visconti_get_resources(pdev, pcie); + if (ret) + return ret; + + platform_set_drvdata(pdev, pcie); + + return visconti_add_pcie_port(pcie, pdev); +} + +static const struct of_device_id visconti_pcie_match[] = { + { .compatible = "toshiba,visconti-pcie" }, + {}, +}; + +static struct platform_driver visconti_pcie_driver = { + .probe = visconti_pcie_probe, + .driver = { + .name = "visconti-pcie", + .of_match_table = visconti_pcie_match, + .suppress_bind_attrs = true, + }, +}; +builtin_platform_driver(visconti_pcie_driver); diff --git a/drivers/pci/controller/mobiveil/Kconfig b/drivers/pci/controller/mobiveil/Kconfig new file mode 100644 index 000000000..c50c46259 --- /dev/null +++ b/drivers/pci/controller/mobiveil/Kconfig @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "Mobiveil-based PCIe controllers" + depends on PCI + +config PCIE_MOBIVEIL + bool + +config PCIE_MOBIVEIL_HOST + bool + depends on PCI_MSI + select IRQ_MSI_LIB + select PCIE_MOBIVEIL + +config PCIE_LAYERSCAPE_GEN4 + bool "Freescale Layerscape Gen4 PCIe controller" + depends on ARCH_LAYERSCAPE || COMPILE_TEST + depends on PCI_MSI + select PCIE_MOBIVEIL_HOST + help + Say Y here if you want PCIe Gen4 controller support on + Layerscape SoCs. + +config PCIE_MOBIVEIL_PLAT + bool "Mobiveil AXI PCIe controller" + depends on ARCH_ZYNQMP || COMPILE_TEST + depends on OF + depends on PCI_MSI + select PCIE_MOBIVEIL_HOST + help + Say Y here if you want to enable support for the Mobiveil AXI PCIe + Soft IP. It has up to 8 outbound and inbound windows + for address translation and it is a PCIe Gen4 IP. + +endmenu diff --git a/drivers/pci/controller/mobiveil/Makefile b/drivers/pci/controller/mobiveil/Makefile new file mode 100644 index 000000000..99d879de3 --- /dev/null +++ b/drivers/pci/controller/mobiveil/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_PCIE_MOBIVEIL) += pcie-mobiveil.o +obj-$(CONFIG_PCIE_MOBIVEIL_HOST) += pcie-mobiveil-host.o +obj-$(CONFIG_PCIE_MOBIVEIL_PLAT) += pcie-mobiveil-plat.o +obj-$(CONFIG_PCIE_LAYERSCAPE_GEN4) += pcie-layerscape-gen4.o diff --git a/drivers/pci/controller/mobiveil/pcie-layerscape-gen4.c b/drivers/pci/controller/mobiveil/pcie-layerscape-gen4.c new file mode 100644 index 000000000..4919b27ea --- /dev/null +++ b/drivers/pci/controller/mobiveil/pcie-layerscape-gen4.c @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe Gen4 host controller driver for NXP Layerscape SoCs + * + * Copyright 2019-2020 NXP + * + * Author: Zhiqiang Hou <Zhiqiang.Hou@nxp.com> + */ + +#include <linux/kernel.h> +#include <linux/interrupt.h> +#include <linux/init.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/of_irq.h> +#include <linux/of_address.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/resource.h> +#include <linux/mfd/syscon.h> +#include <linux/regmap.h> + +#include "pcie-mobiveil.h" + +/* LUT and PF control registers */ +#define PCIE_LUT_OFF 0x80000 +#define PCIE_PF_OFF 0xc0000 +#define PCIE_PF_INT_STAT 0x18 +#define PF_INT_STAT_PABRST BIT(31) + +#define PCIE_PF_DBG 0x7fc +#define PF_DBG_LTSSM_MASK 0x3f +#define PF_DBG_LTSSM_L0 0x2d /* L0 state */ +#define PF_DBG_WE BIT(31) +#define PF_DBG_PABR BIT(27) + +#define to_ls_g4_pcie(x) platform_get_drvdata((x)->pdev) + +struct ls_g4_pcie { + struct mobiveil_pcie pci; + struct delayed_work dwork; + int irq; +}; + +static inline u32 ls_g4_pcie_pf_readl(struct ls_g4_pcie *pcie, u32 off) +{ + return ioread32(pcie->pci.csr_axi_slave_base + PCIE_PF_OFF + off); +} + +static inline void ls_g4_pcie_pf_writel(struct ls_g4_pcie *pcie, + u32 off, u32 val) +{ + iowrite32(val, pcie->pci.csr_axi_slave_base + PCIE_PF_OFF + off); +} + +static bool ls_g4_pcie_link_up(struct mobiveil_pcie *pci) +{ + struct ls_g4_pcie *pcie = to_ls_g4_pcie(pci); + u32 state; + + state = ls_g4_pcie_pf_readl(pcie, PCIE_PF_DBG); + return (state & PF_DBG_LTSSM_MASK) == PF_DBG_LTSSM_L0; +} + +static void ls_g4_pcie_disable_interrupt(struct ls_g4_pcie *pcie) +{ + struct mobiveil_pcie *mv_pci = &pcie->pci; + + mobiveil_csr_writel(mv_pci, 0, PAB_INTP_AMBA_MISC_ENB); +} + +static void ls_g4_pcie_enable_interrupt(struct ls_g4_pcie *pcie) +{ + struct mobiveil_pcie *mv_pci = &pcie->pci; + u32 val; + + /* Clear the interrupt status */ + mobiveil_csr_writel(mv_pci, 0xffffffff, PAB_INTP_AMBA_MISC_STAT); + + val = PAB_INTP_INTX_MASK | PAB_INTP_MSI | PAB_INTP_RESET | + PAB_INTP_PCIE_UE | PAB_INTP_IE_PMREDI | PAB_INTP_IE_EC; + mobiveil_csr_writel(mv_pci, val, PAB_INTP_AMBA_MISC_ENB); +} + +static int ls_g4_pcie_reinit_hw(struct ls_g4_pcie *pcie) +{ + struct mobiveil_pcie *mv_pci = &pcie->pci; + struct device *dev = &mv_pci->pdev->dev; + u32 val, act_stat; + int to = 100; + + /* Poll for pab_csb_reset to set and PAB activity to clear */ + do { + usleep_range(10, 15); + val = ls_g4_pcie_pf_readl(pcie, PCIE_PF_INT_STAT); + act_stat = mobiveil_csr_readl(mv_pci, PAB_ACTIVITY_STAT); + } while (((val & PF_INT_STAT_PABRST) == 0 || act_stat) && to--); + if (to < 0) { + dev_err(dev, "Poll PABRST&PABACT timeout\n"); + return -EIO; + } + + /* clear PEX_RESET bit in PEX_PF0_DBG register */ + val = ls_g4_pcie_pf_readl(pcie, PCIE_PF_DBG); + val |= PF_DBG_WE; + ls_g4_pcie_pf_writel(pcie, PCIE_PF_DBG, val); + + val = ls_g4_pcie_pf_readl(pcie, PCIE_PF_DBG); + val |= PF_DBG_PABR; + ls_g4_pcie_pf_writel(pcie, PCIE_PF_DBG, val); + + val = ls_g4_pcie_pf_readl(pcie, PCIE_PF_DBG); + val &= ~PF_DBG_WE; + ls_g4_pcie_pf_writel(pcie, PCIE_PF_DBG, val); + + mobiveil_host_init(mv_pci, true); + + to = 100; + while (!ls_g4_pcie_link_up(mv_pci) && to--) + usleep_range(200, 250); + if (to < 0) { + dev_err(dev, "PCIe link training timeout\n"); + return -EIO; + } + + return 0; +} + +static irqreturn_t ls_g4_pcie_isr(int irq, void *dev_id) +{ + struct ls_g4_pcie *pcie = (struct ls_g4_pcie *)dev_id; + struct mobiveil_pcie *mv_pci = &pcie->pci; + u32 val; + + val = mobiveil_csr_readl(mv_pci, PAB_INTP_AMBA_MISC_STAT); + if (!val) + return IRQ_NONE; + + if (val & PAB_INTP_RESET) { + ls_g4_pcie_disable_interrupt(pcie); + schedule_delayed_work(&pcie->dwork, msecs_to_jiffies(1)); + } + + mobiveil_csr_writel(mv_pci, val, PAB_INTP_AMBA_MISC_STAT); + + return IRQ_HANDLED; +} + +static int ls_g4_pcie_interrupt_init(struct mobiveil_pcie *mv_pci) +{ + struct ls_g4_pcie *pcie = to_ls_g4_pcie(mv_pci); + struct platform_device *pdev = mv_pci->pdev; + struct device *dev = &pdev->dev; + int ret; + + pcie->irq = platform_get_irq_byname(pdev, "intr"); + if (pcie->irq < 0) + return pcie->irq; + + ret = devm_request_irq(dev, pcie->irq, ls_g4_pcie_isr, + IRQF_SHARED, pdev->name, pcie); + if (ret) { + dev_err(dev, "Can't register PCIe IRQ, errno = %d\n", ret); + return ret; + } + + return 0; +} + +static void ls_g4_pcie_reset(struct work_struct *work) +{ + struct delayed_work *dwork = to_delayed_work(work); + struct ls_g4_pcie *pcie = container_of(dwork, struct ls_g4_pcie, dwork); + struct mobiveil_pcie *mv_pci = &pcie->pci; + u16 ctrl; + + ctrl = mobiveil_csr_readw(mv_pci, PCI_BRIDGE_CONTROL); + ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET; + mobiveil_csr_writew(mv_pci, ctrl, PCI_BRIDGE_CONTROL); + + if (!ls_g4_pcie_reinit_hw(pcie)) + return; + + ls_g4_pcie_enable_interrupt(pcie); +} + +static const struct mobiveil_rp_ops ls_g4_pcie_rp_ops = { + .interrupt_init = ls_g4_pcie_interrupt_init, +}; + +static const struct mobiveil_pab_ops ls_g4_pcie_pab_ops = { + .link_up = ls_g4_pcie_link_up, +}; + +static int __init ls_g4_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct pci_host_bridge *bridge; + struct mobiveil_pcie *mv_pci; + struct ls_g4_pcie *pcie; + struct device_node *np = dev->of_node; + int ret; + + if (!of_parse_phandle(np, "msi-parent", 0)) { + dev_err(dev, "Failed to find msi-parent\n"); + return -EINVAL; + } + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!bridge) + return -ENOMEM; + + pcie = pci_host_bridge_priv(bridge); + mv_pci = &pcie->pci; + + mv_pci->pdev = pdev; + mv_pci->ops = &ls_g4_pcie_pab_ops; + mv_pci->rp.ops = &ls_g4_pcie_rp_ops; + mv_pci->rp.bridge = bridge; + + platform_set_drvdata(pdev, pcie); + + INIT_DELAYED_WORK(&pcie->dwork, ls_g4_pcie_reset); + + ret = mobiveil_pcie_host_probe(mv_pci); + if (ret) { + dev_err(dev, "Fail to probe\n"); + return ret; + } + + ls_g4_pcie_enable_interrupt(pcie); + + return 0; +} + +static const struct of_device_id ls_g4_pcie_of_match[] = { + { .compatible = "fsl,lx2160a-pcie", }, + { }, +}; + +static struct platform_driver ls_g4_pcie_driver = { + .driver = { + .name = "layerscape-pcie-gen4", + .of_match_table = ls_g4_pcie_of_match, + .suppress_bind_attrs = true, + }, +}; + +builtin_platform_driver_probe(ls_g4_pcie_driver, ls_g4_pcie_probe); diff --git a/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c b/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c new file mode 100644 index 000000000..dbc72c73f --- /dev/null +++ b/drivers/pci/controller/mobiveil/pcie-mobiveil-host.c @@ -0,0 +1,582 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for Mobiveil PCIe Host controller + * + * Copyright (c) 2018 Mobiveil Inc. + * Copyright 2019-2020 NXP + * + * Author: Subrahmanya Lingappa <l.subrahmanya@mobiveil.co.in> + * Hou Zhiqiang <Zhiqiang.Hou@nxp.com> + */ + +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqdomain.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +#include "pcie-mobiveil.h" + +static bool mobiveil_pcie_valid_device(struct pci_bus *bus, unsigned int devfn) +{ + /* Only one device down on each root port */ + if (pci_is_root_bus(bus) && (devfn > 0)) + return false; + + /* + * Do not read more than one device on the bus directly + * attached to RC + */ + if ((bus->primary == to_pci_host_bridge(bus->bridge)->busnr) && (PCI_SLOT(devfn) > 0)) + return false; + + return true; +} + +/* + * mobiveil_pcie_map_bus - routine to get the configuration base of either + * root port or endpoint + */ +static void __iomem *mobiveil_pcie_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + struct mobiveil_pcie *pcie = bus->sysdata; + struct mobiveil_root_port *rp = &pcie->rp; + u32 value; + + if (!mobiveil_pcie_valid_device(bus, devfn)) + return NULL; + + /* RC config access */ + if (pci_is_root_bus(bus)) + return pcie->csr_axi_slave_base + where; + + /* + * EP config access (in Config/APIO space) + * Program PEX Address base (31..16 bits) with appropriate value + * (BDF) in PAB_AXI_AMAP_PEX_WIN_L0 Register. + * Relies on pci_lock serialization + */ + value = bus->number << PAB_BUS_SHIFT | + PCI_SLOT(devfn) << PAB_DEVICE_SHIFT | + PCI_FUNC(devfn) << PAB_FUNCTION_SHIFT; + + mobiveil_csr_writel(pcie, value, PAB_AXI_AMAP_PEX_WIN_L(WIN_NUM_0)); + + return rp->config_axi_slave_base + where; +} + +static struct pci_ops mobiveil_pcie_ops = { + .map_bus = mobiveil_pcie_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; + +static void mobiveil_pcie_isr(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + struct mobiveil_pcie *pcie = irq_desc_get_handler_data(desc); + struct device *dev = &pcie->pdev->dev; + struct mobiveil_root_port *rp = &pcie->rp; + struct mobiveil_msi *msi = &rp->msi; + u32 msi_data, msi_addr_lo, msi_addr_hi; + u32 intr_status, msi_status; + unsigned long shifted_status; + u32 bit, val, mask; + + /* + * The core provides a single interrupt for both INTx/MSI messages. + * So we'll read both INTx and MSI status + */ + + chained_irq_enter(chip, desc); + + /* read INTx status */ + val = mobiveil_csr_readl(pcie, PAB_INTP_AMBA_MISC_STAT); + mask = mobiveil_csr_readl(pcie, PAB_INTP_AMBA_MISC_ENB); + intr_status = val & mask; + + /* Handle INTx */ + if (intr_status & PAB_INTP_INTX_MASK) { + shifted_status = mobiveil_csr_readl(pcie, + PAB_INTP_AMBA_MISC_STAT); + shifted_status &= PAB_INTP_INTX_MASK; + shifted_status >>= PAB_INTX_START; + do { + for_each_set_bit(bit, &shifted_status, PCI_NUM_INTX) { + int ret; + ret = generic_handle_domain_irq(rp->intx_domain, + bit + 1); + if (ret) + dev_err_ratelimited(dev, "unexpected IRQ, INT%d\n", + bit); + + /* clear interrupt handled */ + mobiveil_csr_writel(pcie, + 1 << (PAB_INTX_START + bit), + PAB_INTP_AMBA_MISC_STAT); + } + + shifted_status = mobiveil_csr_readl(pcie, + PAB_INTP_AMBA_MISC_STAT); + shifted_status &= PAB_INTP_INTX_MASK; + shifted_status >>= PAB_INTX_START; + } while (shifted_status != 0); + } + + /* read extra MSI status register */ + msi_status = readl_relaxed(pcie->apb_csr_base + MSI_STATUS_OFFSET); + + /* handle MSI interrupts */ + while (msi_status & 1) { + msi_data = readl_relaxed(pcie->apb_csr_base + MSI_DATA_OFFSET); + + /* + * MSI_STATUS_OFFSET register gets updated to zero + * once we pop not only the MSI data but also address + * from MSI hardware FIFO. So keeping these following + * two dummy reads. + */ + msi_addr_lo = readl_relaxed(pcie->apb_csr_base + + MSI_ADDR_L_OFFSET); + msi_addr_hi = readl_relaxed(pcie->apb_csr_base + + MSI_ADDR_H_OFFSET); + dev_dbg(dev, "MSI registers, data: %08x, addr: %08x:%08x\n", + msi_data, msi_addr_hi, msi_addr_lo); + + generic_handle_domain_irq(msi->dev_domain, msi_data); + + msi_status = readl_relaxed(pcie->apb_csr_base + + MSI_STATUS_OFFSET); + } + + /* Clear the interrupt status */ + mobiveil_csr_writel(pcie, intr_status, PAB_INTP_AMBA_MISC_STAT); + chained_irq_exit(chip, desc); +} + +static int mobiveil_pcie_parse_dt(struct mobiveil_pcie *pcie) +{ + struct device *dev = &pcie->pdev->dev; + struct platform_device *pdev = pcie->pdev; + struct device_node *node = dev->of_node; + struct mobiveil_root_port *rp = &pcie->rp; + struct resource *res; + + /* map config resource */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "config_axi_slave"); + rp->config_axi_slave_base = devm_pci_remap_cfg_resource(dev, res); + if (IS_ERR(rp->config_axi_slave_base)) + return PTR_ERR(rp->config_axi_slave_base); + rp->ob_io_res = res; + + /* map csr resource */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "csr_axi_slave"); + pcie->csr_axi_slave_base = devm_pci_remap_cfg_resource(dev, res); + if (IS_ERR(pcie->csr_axi_slave_base)) + return PTR_ERR(pcie->csr_axi_slave_base); + pcie->pcie_reg_base = res->start; + + /* read the number of windows requested */ + if (of_property_read_u32(node, "apio-wins", &pcie->apio_wins)) + pcie->apio_wins = MAX_PIO_WINDOWS; + + if (of_property_read_u32(node, "ppio-wins", &pcie->ppio_wins)) + pcie->ppio_wins = MAX_PIO_WINDOWS; + + return 0; +} + +static void mobiveil_pcie_enable_msi(struct mobiveil_pcie *pcie) +{ + phys_addr_t msg_addr = pcie->pcie_reg_base; + struct mobiveil_msi *msi = &pcie->rp.msi; + + msi->num_of_vectors = PCI_NUM_MSI; + msi->msi_pages_phys = (phys_addr_t)msg_addr; + + writel_relaxed(lower_32_bits(msg_addr), + pcie->apb_csr_base + MSI_BASE_LO_OFFSET); + writel_relaxed(upper_32_bits(msg_addr), + pcie->apb_csr_base + MSI_BASE_HI_OFFSET); + writel_relaxed(4096, pcie->apb_csr_base + MSI_SIZE_OFFSET); + writel_relaxed(1, pcie->apb_csr_base + MSI_ENABLE_OFFSET); +} + +int mobiveil_host_init(struct mobiveil_pcie *pcie, bool reinit) +{ + struct mobiveil_root_port *rp = &pcie->rp; + struct pci_host_bridge *bridge = rp->bridge; + u32 value, pab_ctrl, type; + struct resource_entry *win; + + pcie->ib_wins_configured = 0; + pcie->ob_wins_configured = 0; + + if (!reinit) { + /* setup bus numbers */ + value = mobiveil_csr_readl(pcie, PCI_PRIMARY_BUS); + value &= 0xff000000; + value |= 0x00ff0100; + mobiveil_csr_writel(pcie, value, PCI_PRIMARY_BUS); + } + + /* + * program Bus Master Enable Bit in Command Register in PAB Config + * Space + */ + value = mobiveil_csr_readl(pcie, PCI_COMMAND); + value |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER; + mobiveil_csr_writel(pcie, value, PCI_COMMAND); + + /* + * program PIO Enable Bit to 1 (and PEX PIO Enable to 1) in PAB_CTRL + * register + */ + pab_ctrl = mobiveil_csr_readl(pcie, PAB_CTRL); + pab_ctrl |= (1 << AMBA_PIO_ENABLE_SHIFT) | (1 << PEX_PIO_ENABLE_SHIFT); + mobiveil_csr_writel(pcie, pab_ctrl, PAB_CTRL); + + /* + * program PIO Enable Bit to 1 and Config Window Enable Bit to 1 in + * PAB_AXI_PIO_CTRL Register + */ + value = mobiveil_csr_readl(pcie, PAB_AXI_PIO_CTRL); + value |= APIO_EN_MASK; + mobiveil_csr_writel(pcie, value, PAB_AXI_PIO_CTRL); + + /* Enable PCIe PIO master */ + value = mobiveil_csr_readl(pcie, PAB_PEX_PIO_CTRL); + value |= 1 << PIO_ENABLE_SHIFT; + mobiveil_csr_writel(pcie, value, PAB_PEX_PIO_CTRL); + + /* + * we'll program one outbound window for config reads and + * another default inbound window for all the upstream traffic + * rest of the outbound windows will be configured according to + * the "ranges" field defined in device tree + */ + + /* config outbound translation window */ + program_ob_windows(pcie, WIN_NUM_0, rp->ob_io_res->start, 0, + CFG_WINDOW_TYPE, resource_size(rp->ob_io_res)); + + /* memory inbound translation window */ + program_ib_windows(pcie, WIN_NUM_0, 0, 0, MEM_WINDOW_TYPE, IB_WIN_SIZE); + + /* Get the I/O and memory ranges from DT */ + resource_list_for_each_entry(win, &bridge->windows) { + if (resource_type(win->res) == IORESOURCE_MEM) + type = MEM_WINDOW_TYPE; + else if (resource_type(win->res) == IORESOURCE_IO) + type = IO_WINDOW_TYPE; + else + continue; + + /* configure outbound translation window */ + program_ob_windows(pcie, pcie->ob_wins_configured, + win->res->start, + win->res->start - win->offset, + type, resource_size(win->res)); + } + + /* fixup for PCIe class register */ + value = mobiveil_csr_readl(pcie, PAB_INTP_AXI_PIO_CLASS); + value &= 0xff; + value |= PCI_CLASS_BRIDGE_PCI_NORMAL << 8; + mobiveil_csr_writel(pcie, value, PAB_INTP_AXI_PIO_CLASS); + + return 0; +} + +static void mobiveil_mask_intx_irq(struct irq_data *data) +{ + struct mobiveil_pcie *pcie = irq_data_get_irq_chip_data(data); + struct mobiveil_root_port *rp; + unsigned long flags; + u32 mask, shifted_val; + + rp = &pcie->rp; + mask = 1 << ((data->hwirq + PAB_INTX_START) - 1); + raw_spin_lock_irqsave(&rp->intx_mask_lock, flags); + shifted_val = mobiveil_csr_readl(pcie, PAB_INTP_AMBA_MISC_ENB); + shifted_val &= ~mask; + mobiveil_csr_writel(pcie, shifted_val, PAB_INTP_AMBA_MISC_ENB); + raw_spin_unlock_irqrestore(&rp->intx_mask_lock, flags); +} + +static void mobiveil_unmask_intx_irq(struct irq_data *data) +{ + struct mobiveil_pcie *pcie = irq_data_get_irq_chip_data(data); + struct mobiveil_root_port *rp; + unsigned long flags; + u32 shifted_val, mask; + + rp = &pcie->rp; + mask = 1 << ((data->hwirq + PAB_INTX_START) - 1); + raw_spin_lock_irqsave(&rp->intx_mask_lock, flags); + shifted_val = mobiveil_csr_readl(pcie, PAB_INTP_AMBA_MISC_ENB); + shifted_val |= mask; + mobiveil_csr_writel(pcie, shifted_val, PAB_INTP_AMBA_MISC_ENB); + raw_spin_unlock_irqrestore(&rp->intx_mask_lock, flags); +} + +static struct irq_chip intx_irq_chip = { + .name = "mobiveil_pcie:intx", + .irq_enable = mobiveil_unmask_intx_irq, + .irq_disable = mobiveil_mask_intx_irq, + .irq_mask = mobiveil_mask_intx_irq, + .irq_unmask = mobiveil_unmask_intx_irq, +}; + +/* routine to setup the INTx related data */ +static int mobiveil_pcie_intx_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &intx_irq_chip, handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + + return 0; +} + +/* INTx domain operations structure */ +static const struct irq_domain_ops intx_domain_ops = { + .map = mobiveil_pcie_intx_map, +}; + +#define MOBIVEIL_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_NO_AFFINITY) + +#define MOBIVEIL_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_PCI_MSIX) + +static const struct msi_parent_ops mobiveil_msi_parent_ops = { + .required_flags = MOBIVEIL_MSI_FLAGS_REQUIRED, + .supported_flags = MOBIVEIL_MSI_FLAGS_SUPPORTED, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .prefix = "Mobiveil-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +static void mobiveil_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct mobiveil_pcie *pcie = irq_data_get_irq_chip_data(data); + phys_addr_t addr = pcie->pcie_reg_base + (data->hwirq * sizeof(int)); + + msg->address_lo = lower_32_bits(addr); + msg->address_hi = upper_32_bits(addr); + msg->data = data->hwirq; + + dev_dbg(&pcie->pdev->dev, "msi#%d address_hi %#x address_lo %#x\n", + (int)data->hwirq, msg->address_hi, msg->address_lo); +} + +static struct irq_chip mobiveil_msi_bottom_irq_chip = { + .name = "Mobiveil MSI", + .irq_compose_msi_msg = mobiveil_compose_msi_msg, +}; + +static int mobiveil_irq_msi_domain_alloc(struct irq_domain *domain, + unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct mobiveil_pcie *pcie = domain->host_data; + struct mobiveil_msi *msi = &pcie->rp.msi; + unsigned long bit; + + WARN_ON(nr_irqs != 1); + mutex_lock(&msi->lock); + + bit = find_first_zero_bit(msi->msi_irq_in_use, msi->num_of_vectors); + if (bit >= msi->num_of_vectors) { + mutex_unlock(&msi->lock); + return -ENOSPC; + } + + set_bit(bit, msi->msi_irq_in_use); + + mutex_unlock(&msi->lock); + + irq_domain_set_info(domain, virq, bit, &mobiveil_msi_bottom_irq_chip, + domain->host_data, handle_level_irq, NULL, NULL); + return 0; +} + +static void mobiveil_irq_msi_domain_free(struct irq_domain *domain, + unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct mobiveil_pcie *pcie = irq_data_get_irq_chip_data(d); + struct mobiveil_msi *msi = &pcie->rp.msi; + + mutex_lock(&msi->lock); + + if (!test_bit(d->hwirq, msi->msi_irq_in_use)) + dev_err(&pcie->pdev->dev, "trying to free unused MSI#%lu\n", + d->hwirq); + else + __clear_bit(d->hwirq, msi->msi_irq_in_use); + + mutex_unlock(&msi->lock); +} +static const struct irq_domain_ops msi_domain_ops = { + .alloc = mobiveil_irq_msi_domain_alloc, + .free = mobiveil_irq_msi_domain_free, +}; + +static int mobiveil_allocate_msi_domains(struct mobiveil_pcie *pcie) +{ + struct device *dev = &pcie->pdev->dev; + struct mobiveil_msi *msi = &pcie->rp.msi; + + mutex_init(&msi->lock); + + struct irq_domain_info info = { + .fwnode = dev_fwnode(dev), + .ops = &msi_domain_ops, + .host_data = pcie, + .size = msi->num_of_vectors, + }; + + msi->dev_domain = msi_create_parent_irq_domain(&info, &mobiveil_msi_parent_ops); + if (!msi->dev_domain) { + dev_err(dev, "failed to create MSI domain\n"); + return -ENOMEM; + } + + return 0; +} + +static int mobiveil_pcie_init_irq_domain(struct mobiveil_pcie *pcie) +{ + struct device *dev = &pcie->pdev->dev; + struct mobiveil_root_port *rp = &pcie->rp; + + /* setup INTx */ + rp->intx_domain = irq_domain_create_linear(dev_fwnode(dev), PCI_NUM_INTX, &intx_domain_ops, + pcie); + if (!rp->intx_domain) { + dev_err(dev, "Failed to get a INTx IRQ domain\n"); + return -ENOMEM; + } + + raw_spin_lock_init(&rp->intx_mask_lock); + + /* setup MSI */ + return mobiveil_allocate_msi_domains(pcie); +} + +static int mobiveil_pcie_integrated_interrupt_init(struct mobiveil_pcie *pcie) +{ + struct platform_device *pdev = pcie->pdev; + struct device *dev = &pdev->dev; + struct mobiveil_root_port *rp = &pcie->rp; + struct resource *res; + int ret; + + /* map MSI config resource */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "apb_csr"); + pcie->apb_csr_base = devm_pci_remap_cfg_resource(dev, res); + if (IS_ERR(pcie->apb_csr_base)) + return PTR_ERR(pcie->apb_csr_base); + + /* setup MSI hardware registers */ + mobiveil_pcie_enable_msi(pcie); + + rp->irq = platform_get_irq(pdev, 0); + if (rp->irq < 0) + return rp->irq; + + /* initialize the IRQ domains */ + ret = mobiveil_pcie_init_irq_domain(pcie); + if (ret) { + dev_err(dev, "Failed creating IRQ Domain\n"); + return ret; + } + + irq_set_chained_handler_and_data(rp->irq, mobiveil_pcie_isr, pcie); + + /* Enable interrupts */ + mobiveil_csr_writel(pcie, (PAB_INTP_INTX_MASK | PAB_INTP_MSI_MASK), + PAB_INTP_AMBA_MISC_ENB); + + + return 0; +} + +static int mobiveil_pcie_interrupt_init(struct mobiveil_pcie *pcie) +{ + struct mobiveil_root_port *rp = &pcie->rp; + + if (rp->ops->interrupt_init) + return rp->ops->interrupt_init(pcie); + + return mobiveil_pcie_integrated_interrupt_init(pcie); +} + +static bool mobiveil_pcie_is_bridge(struct mobiveil_pcie *pcie) +{ + u32 header_type; + + header_type = mobiveil_csr_readb(pcie, PCI_HEADER_TYPE); + header_type &= PCI_HEADER_TYPE_MASK; + + return header_type == PCI_HEADER_TYPE_BRIDGE; +} + +int mobiveil_pcie_host_probe(struct mobiveil_pcie *pcie) +{ + struct mobiveil_root_port *rp = &pcie->rp; + struct pci_host_bridge *bridge = rp->bridge; + struct device *dev = &pcie->pdev->dev; + int ret; + + ret = mobiveil_pcie_parse_dt(pcie); + if (ret) { + dev_err(dev, "Parsing DT failed, ret: %x\n", ret); + return ret; + } + + if (!mobiveil_pcie_is_bridge(pcie)) + return -ENODEV; + + /* + * configure all inbound and outbound windows and prepare the RC for + * config access + */ + ret = mobiveil_host_init(pcie, false); + if (ret) { + dev_err(dev, "Failed to initialize host\n"); + return ret; + } + + ret = mobiveil_pcie_interrupt_init(pcie); + if (ret) { + dev_err(dev, "Interrupt init failed\n"); + return ret; + } + + /* Initialize bridge */ + bridge->sysdata = pcie; + bridge->ops = &mobiveil_pcie_ops; + + ret = mobiveil_bringup_link(pcie); + if (ret) { + dev_info(dev, "link bring-up failed\n"); + return ret; + } + + return pci_host_probe(bridge); +} diff --git a/drivers/pci/controller/mobiveil/pcie-mobiveil-plat.c b/drivers/pci/controller/mobiveil/pcie-mobiveil-plat.c new file mode 100644 index 000000000..c5bb87ff6 --- /dev/null +++ b/drivers/pci/controller/mobiveil/pcie-mobiveil-plat.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for Mobiveil PCIe Host controller + * + * Copyright (c) 2018 Mobiveil Inc. + * Copyright 2019 NXP + * + * Author: Subrahmanya Lingappa <l.subrahmanya@mobiveil.co.in> + * Hou Zhiqiang <Zhiqiang.Hou@nxp.com> + */ + +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +#include "pcie-mobiveil.h" + +static int mobiveil_pcie_probe(struct platform_device *pdev) +{ + struct mobiveil_pcie *pcie; + struct pci_host_bridge *bridge; + struct device *dev = &pdev->dev; + + /* allocate the PCIe port */ + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!bridge) + return -ENOMEM; + + pcie = pci_host_bridge_priv(bridge); + pcie->rp.bridge = bridge; + + pcie->pdev = pdev; + + return mobiveil_pcie_host_probe(pcie); +} + +static const struct of_device_id mobiveil_pcie_of_match[] = { + {.compatible = "mbvl,gpex40-pcie",}, + {}, +}; + +MODULE_DEVICE_TABLE(of, mobiveil_pcie_of_match); + +static struct platform_driver mobiveil_pcie_driver = { + .probe = mobiveil_pcie_probe, + .driver = { + .name = "mobiveil-pcie", + .of_match_table = mobiveil_pcie_of_match, + .suppress_bind_attrs = true, + }, +}; + +builtin_platform_driver(mobiveil_pcie_driver); + +MODULE_DESCRIPTION("Mobiveil PCIe host controller driver"); +MODULE_AUTHOR("Subrahmanya Lingappa <l.subrahmanya@mobiveil.co.in>"); diff --git a/drivers/pci/controller/mobiveil/pcie-mobiveil.c b/drivers/pci/controller/mobiveil/pcie-mobiveil.c new file mode 100644 index 000000000..e8346851c --- /dev/null +++ b/drivers/pci/controller/mobiveil/pcie-mobiveil.c @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host controller driver for Mobiveil PCIe Host controller + * + * Copyright (c) 2018 Mobiveil Inc. + * Copyright 2019 NXP + * + * Author: Subrahmanya Lingappa <l.subrahmanya@mobiveil.co.in> + * Hou Zhiqiang <Zhiqiang.Hou@nxp.com> + */ + +#include <linux/delay.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/platform_device.h> + +#include "pcie-mobiveil.h" + +/* + * mobiveil_pcie_sel_page - routine to access paged register + * + * Registers whose address greater than PAGED_ADDR_BNDRY (0xc00) are paged, + * for this scheme to work extracted higher 6 bits of the offset will be + * written to pg_sel field of PAB_CTRL register and rest of the lower 10 + * bits enabled with PAGED_ADDR_BNDRY are used as offset of the register. + */ +static void mobiveil_pcie_sel_page(struct mobiveil_pcie *pcie, u8 pg_idx) +{ + u32 val; + + val = readl(pcie->csr_axi_slave_base + PAB_CTRL); + val &= ~(PAGE_SEL_MASK << PAGE_SEL_SHIFT); + val |= (pg_idx & PAGE_SEL_MASK) << PAGE_SEL_SHIFT; + + writel(val, pcie->csr_axi_slave_base + PAB_CTRL); +} + +static void __iomem *mobiveil_pcie_comp_addr(struct mobiveil_pcie *pcie, + u32 off) +{ + if (off < PAGED_ADDR_BNDRY) { + /* For directly accessed registers, clear the pg_sel field */ + mobiveil_pcie_sel_page(pcie, 0); + return pcie->csr_axi_slave_base + off; + } + + mobiveil_pcie_sel_page(pcie, OFFSET_TO_PAGE_IDX(off)); + return pcie->csr_axi_slave_base + OFFSET_TO_PAGE_ADDR(off); +} + +static int mobiveil_pcie_read(void __iomem *addr, int size, u32 *val) +{ + if ((uintptr_t)addr & (size - 1)) { + *val = 0; + return PCIBIOS_BAD_REGISTER_NUMBER; + } + + switch (size) { + case 4: + *val = readl(addr); + break; + case 2: + *val = readw(addr); + break; + case 1: + *val = readb(addr); + break; + default: + *val = 0; + return PCIBIOS_BAD_REGISTER_NUMBER; + } + + return PCIBIOS_SUCCESSFUL; +} + +static int mobiveil_pcie_write(void __iomem *addr, int size, u32 val) +{ + if ((uintptr_t)addr & (size - 1)) + return PCIBIOS_BAD_REGISTER_NUMBER; + + switch (size) { + case 4: + writel(val, addr); + break; + case 2: + writew(val, addr); + break; + case 1: + writeb(val, addr); + break; + default: + return PCIBIOS_BAD_REGISTER_NUMBER; + } + + return PCIBIOS_SUCCESSFUL; +} + +u32 mobiveil_csr_read(struct mobiveil_pcie *pcie, u32 off, size_t size) +{ + void __iomem *addr; + u32 val; + int ret; + + addr = mobiveil_pcie_comp_addr(pcie, off); + + ret = mobiveil_pcie_read(addr, size, &val); + if (ret) + dev_err(&pcie->pdev->dev, "read CSR address failed\n"); + + return val; +} + +void mobiveil_csr_write(struct mobiveil_pcie *pcie, u32 val, u32 off, + size_t size) +{ + void __iomem *addr; + int ret; + + addr = mobiveil_pcie_comp_addr(pcie, off); + + ret = mobiveil_pcie_write(addr, size, val); + if (ret) + dev_err(&pcie->pdev->dev, "write CSR address failed\n"); +} + +bool mobiveil_pcie_link_up(struct mobiveil_pcie *pcie) +{ + if (pcie->ops->link_up) + return pcie->ops->link_up(pcie); + + return (mobiveil_csr_readl(pcie, LTSSM_STATUS) & + LTSSM_STATUS_L0_MASK) == LTSSM_STATUS_L0; +} + +void program_ib_windows(struct mobiveil_pcie *pcie, int win_num, + u64 cpu_addr, u64 pci_addr, u32 type, u64 size) +{ + u32 value; + u64 size64 = ~(size - 1); + + if (win_num >= pcie->ppio_wins) { + dev_err(&pcie->pdev->dev, + "ERROR: max inbound windows reached !\n"); + return; + } + + value = mobiveil_csr_readl(pcie, PAB_PEX_AMAP_CTRL(win_num)); + value &= ~(AMAP_CTRL_TYPE_MASK << AMAP_CTRL_TYPE_SHIFT | WIN_SIZE_MASK); + value |= type << AMAP_CTRL_TYPE_SHIFT | 1 << AMAP_CTRL_EN_SHIFT | + (lower_32_bits(size64) & WIN_SIZE_MASK); + mobiveil_csr_writel(pcie, value, PAB_PEX_AMAP_CTRL(win_num)); + + mobiveil_csr_writel(pcie, upper_32_bits(size64), + PAB_EXT_PEX_AMAP_SIZEN(win_num)); + + mobiveil_csr_writel(pcie, lower_32_bits(cpu_addr), + PAB_PEX_AMAP_AXI_WIN(win_num)); + mobiveil_csr_writel(pcie, upper_32_bits(cpu_addr), + PAB_EXT_PEX_AMAP_AXI_WIN(win_num)); + + mobiveil_csr_writel(pcie, lower_32_bits(pci_addr), + PAB_PEX_AMAP_PEX_WIN_L(win_num)); + mobiveil_csr_writel(pcie, upper_32_bits(pci_addr), + PAB_PEX_AMAP_PEX_WIN_H(win_num)); + + pcie->ib_wins_configured++; +} + +/* + * routine to program the outbound windows + */ +void program_ob_windows(struct mobiveil_pcie *pcie, int win_num, + u64 cpu_addr, u64 pci_addr, u32 type, u64 size) +{ + u32 value; + u64 size64 = ~(size - 1); + + if (win_num >= pcie->apio_wins) { + dev_err(&pcie->pdev->dev, + "ERROR: max outbound windows reached !\n"); + return; + } + + /* + * program Enable Bit to 1, Type Bit to (00) base 2, AXI Window Size Bit + * to 4 KB in PAB_AXI_AMAP_CTRL register + */ + value = mobiveil_csr_readl(pcie, PAB_AXI_AMAP_CTRL(win_num)); + value &= ~(WIN_TYPE_MASK << WIN_TYPE_SHIFT | WIN_SIZE_MASK); + value |= 1 << WIN_ENABLE_SHIFT | type << WIN_TYPE_SHIFT | + (lower_32_bits(size64) & WIN_SIZE_MASK); + mobiveil_csr_writel(pcie, value, PAB_AXI_AMAP_CTRL(win_num)); + + mobiveil_csr_writel(pcie, upper_32_bits(size64), + PAB_EXT_AXI_AMAP_SIZE(win_num)); + + /* + * program AXI window base with appropriate value in + * PAB_AXI_AMAP_AXI_WIN0 register + */ + mobiveil_csr_writel(pcie, + lower_32_bits(cpu_addr) & (~AXI_WINDOW_ALIGN_MASK), + PAB_AXI_AMAP_AXI_WIN(win_num)); + mobiveil_csr_writel(pcie, upper_32_bits(cpu_addr), + PAB_EXT_AXI_AMAP_AXI_WIN(win_num)); + + mobiveil_csr_writel(pcie, lower_32_bits(pci_addr), + PAB_AXI_AMAP_PEX_WIN_L(win_num)); + mobiveil_csr_writel(pcie, upper_32_bits(pci_addr), + PAB_AXI_AMAP_PEX_WIN_H(win_num)); + + pcie->ob_wins_configured++; +} + +int mobiveil_bringup_link(struct mobiveil_pcie *pcie) +{ + int retries; + + /* check if the link is up or not */ + for (retries = 0; retries < PCIE_LINK_WAIT_MAX_RETRIES; retries++) { + if (mobiveil_pcie_link_up(pcie)) + return 0; + + msleep(PCIE_LINK_WAIT_SLEEP_MS); + } + + dev_err(&pcie->pdev->dev, "link never came up\n"); + + return -ETIMEDOUT; +} diff --git a/drivers/pci/controller/mobiveil/pcie-mobiveil.h b/drivers/pci/controller/mobiveil/pcie-mobiveil.h new file mode 100644 index 000000000..11010a99e --- /dev/null +++ b/drivers/pci/controller/mobiveil/pcie-mobiveil.h @@ -0,0 +1,219 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * PCIe host controller driver for Mobiveil PCIe Host controller + * + * Copyright (c) 2018 Mobiveil Inc. + * Copyright 2019 NXP + * + * Author: Subrahmanya Lingappa <l.subrahmanya@mobiveil.co.in> + * Hou Zhiqiang <Zhiqiang.Hou@nxp.com> + */ + +#ifndef _PCIE_MOBIVEIL_H +#define _PCIE_MOBIVEIL_H + +#include <linux/pci.h> +#include <linux/irq.h> +#include <linux/msi.h> +#include "../../pci.h" + +/* register offsets and bit positions */ + +/* + * translation tables are grouped into windows, each window registers are + * grouped into blocks of 4 or 16 registers each + */ +#define PAB_REG_BLOCK_SIZE 16 +#define PAB_EXT_REG_BLOCK_SIZE 4 + +#define PAB_REG_ADDR(offset, win) \ + (offset + (win * PAB_REG_BLOCK_SIZE)) +#define PAB_EXT_REG_ADDR(offset, win) \ + (offset + (win * PAB_EXT_REG_BLOCK_SIZE)) + +#define LTSSM_STATUS 0x0404 +#define LTSSM_STATUS_L0_MASK 0x3f +#define LTSSM_STATUS_L0 0x2d + +#define PAB_CTRL 0x0808 +#define AMBA_PIO_ENABLE_SHIFT 0 +#define PEX_PIO_ENABLE_SHIFT 1 +#define PAGE_SEL_SHIFT 13 +#define PAGE_SEL_MASK 0x3f +#define PAGE_LO_MASK 0x3ff +#define PAGE_SEL_OFFSET_SHIFT 10 + +#define PAB_ACTIVITY_STAT 0x81c + +#define PAB_AXI_PIO_CTRL 0x0840 +#define APIO_EN_MASK 0xf + +#define PAB_PEX_PIO_CTRL 0x08c0 +#define PIO_ENABLE_SHIFT 0 + +#define PAB_INTP_AMBA_MISC_ENB 0x0b0c +#define PAB_INTP_AMBA_MISC_STAT 0x0b1c +#define PAB_INTP_RESET BIT(1) +#define PAB_INTP_MSI BIT(3) +#define PAB_INTP_INTA BIT(5) +#define PAB_INTP_INTB BIT(6) +#define PAB_INTP_INTC BIT(7) +#define PAB_INTP_INTD BIT(8) +#define PAB_INTP_PCIE_UE BIT(9) +#define PAB_INTP_IE_PMREDI BIT(29) +#define PAB_INTP_IE_EC BIT(30) +#define PAB_INTP_MSI_MASK PAB_INTP_MSI +#define PAB_INTP_INTX_MASK (PAB_INTP_INTA | PAB_INTP_INTB |\ + PAB_INTP_INTC | PAB_INTP_INTD) + +#define PAB_AXI_AMAP_CTRL(win) PAB_REG_ADDR(0x0ba0, win) +#define WIN_ENABLE_SHIFT 0 +#define WIN_TYPE_SHIFT 1 +#define WIN_TYPE_MASK 0x3 +#define WIN_SIZE_MASK 0xfffffc00 + +#define PAB_EXT_AXI_AMAP_SIZE(win) PAB_EXT_REG_ADDR(0xbaf0, win) + +#define PAB_EXT_AXI_AMAP_AXI_WIN(win) PAB_EXT_REG_ADDR(0x80a0, win) +#define PAB_AXI_AMAP_AXI_WIN(win) PAB_REG_ADDR(0x0ba4, win) +#define AXI_WINDOW_ALIGN_MASK 3 + +#define PAB_AXI_AMAP_PEX_WIN_L(win) PAB_REG_ADDR(0x0ba8, win) +#define PAB_BUS_SHIFT 24 +#define PAB_DEVICE_SHIFT 19 +#define PAB_FUNCTION_SHIFT 16 + +#define PAB_AXI_AMAP_PEX_WIN_H(win) PAB_REG_ADDR(0x0bac, win) +#define PAB_INTP_AXI_PIO_CLASS 0x474 + +#define PAB_PEX_AMAP_CTRL(win) PAB_REG_ADDR(0x4ba0, win) +#define AMAP_CTRL_EN_SHIFT 0 +#define AMAP_CTRL_TYPE_SHIFT 1 +#define AMAP_CTRL_TYPE_MASK 3 + +#define PAB_EXT_PEX_AMAP_SIZEN(win) PAB_EXT_REG_ADDR(0xbef0, win) +#define PAB_EXT_PEX_AMAP_AXI_WIN(win) PAB_EXT_REG_ADDR(0xb4a0, win) +#define PAB_PEX_AMAP_AXI_WIN(win) PAB_REG_ADDR(0x4ba4, win) +#define PAB_PEX_AMAP_PEX_WIN_L(win) PAB_REG_ADDR(0x4ba8, win) +#define PAB_PEX_AMAP_PEX_WIN_H(win) PAB_REG_ADDR(0x4bac, win) + +/* starting offset of INTX bits in status register */ +#define PAB_INTX_START 5 + +/* supported number of MSI interrupts */ +#define PCI_NUM_MSI 16 + +/* MSI registers */ +#define MSI_BASE_LO_OFFSET 0x04 +#define MSI_BASE_HI_OFFSET 0x08 +#define MSI_SIZE_OFFSET 0x0c +#define MSI_ENABLE_OFFSET 0x14 +#define MSI_STATUS_OFFSET 0x18 +#define MSI_DATA_OFFSET 0x20 +#define MSI_ADDR_L_OFFSET 0x24 +#define MSI_ADDR_H_OFFSET 0x28 + +/* outbound and inbound window definitions */ +#define WIN_NUM_0 0 +#define WIN_NUM_1 1 +#define CFG_WINDOW_TYPE 0 +#define IO_WINDOW_TYPE 1 +#define MEM_WINDOW_TYPE 2 +#define IB_WIN_SIZE ((u64)256 * 1024 * 1024 * 1024) +#define MAX_PIO_WINDOWS 8 + +#define PAGED_ADDR_BNDRY 0xc00 +#define OFFSET_TO_PAGE_ADDR(off) \ + ((off & PAGE_LO_MASK) | PAGED_ADDR_BNDRY) +#define OFFSET_TO_PAGE_IDX(off) \ + ((off >> PAGE_SEL_OFFSET_SHIFT) & PAGE_SEL_MASK) + +struct mobiveil_msi { /* MSI information */ + struct mutex lock; /* protect bitmap variable */ + struct irq_domain *dev_domain; + phys_addr_t msi_pages_phys; + int num_of_vectors; + DECLARE_BITMAP(msi_irq_in_use, PCI_NUM_MSI); +}; + +struct mobiveil_pcie; + +struct mobiveil_rp_ops { + int (*interrupt_init)(struct mobiveil_pcie *pcie); +}; + +struct mobiveil_root_port { + void __iomem *config_axi_slave_base; /* endpoint config base */ + struct resource *ob_io_res; + const struct mobiveil_rp_ops *ops; + int irq; + raw_spinlock_t intx_mask_lock; + struct irq_domain *intx_domain; + struct mobiveil_msi msi; + struct pci_host_bridge *bridge; +}; + +struct mobiveil_pab_ops { + bool (*link_up)(struct mobiveil_pcie *pcie); +}; + +struct mobiveil_pcie { + struct platform_device *pdev; + void __iomem *csr_axi_slave_base; /* root port config base */ + void __iomem *apb_csr_base; /* MSI register base */ + phys_addr_t pcie_reg_base; /* Physical PCIe Controller Base */ + int apio_wins; + int ppio_wins; + int ob_wins_configured; /* configured outbound windows */ + int ib_wins_configured; /* configured inbound windows */ + const struct mobiveil_pab_ops *ops; + struct mobiveil_root_port rp; +}; + +int mobiveil_pcie_host_probe(struct mobiveil_pcie *pcie); +int mobiveil_host_init(struct mobiveil_pcie *pcie, bool reinit); +bool mobiveil_pcie_link_up(struct mobiveil_pcie *pcie); +int mobiveil_bringup_link(struct mobiveil_pcie *pcie); +void program_ob_windows(struct mobiveil_pcie *pcie, int win_num, u64 cpu_addr, + u64 pci_addr, u32 type, u64 size); +void program_ib_windows(struct mobiveil_pcie *pcie, int win_num, u64 cpu_addr, + u64 pci_addr, u32 type, u64 size); +u32 mobiveil_csr_read(struct mobiveil_pcie *pcie, u32 off, size_t size); +void mobiveil_csr_write(struct mobiveil_pcie *pcie, u32 val, u32 off, + size_t size); + +static inline u32 mobiveil_csr_readl(struct mobiveil_pcie *pcie, u32 off) +{ + return mobiveil_csr_read(pcie, off, 0x4); +} + +static inline u16 mobiveil_csr_readw(struct mobiveil_pcie *pcie, u32 off) +{ + return mobiveil_csr_read(pcie, off, 0x2); +} + +static inline u8 mobiveil_csr_readb(struct mobiveil_pcie *pcie, u32 off) +{ + return mobiveil_csr_read(pcie, off, 0x1); +} + + +static inline void mobiveil_csr_writel(struct mobiveil_pcie *pcie, u32 val, + u32 off) +{ + mobiveil_csr_write(pcie, val, off, 0x4); +} + +static inline void mobiveil_csr_writew(struct mobiveil_pcie *pcie, u16 val, + u32 off) +{ + mobiveil_csr_write(pcie, val, off, 0x2); +} + +static inline void mobiveil_csr_writeb(struct mobiveil_pcie *pcie, u8 val, + u32 off) +{ + mobiveil_csr_write(pcie, val, off, 0x1); +} + +#endif /* _PCIE_MOBIVEIL_H */ diff --git a/drivers/pci/controller/pci-aardvark.c b/drivers/pci/controller/pci-aardvark.c new file mode 100644 index 000000000..ecb81ac73 --- /dev/null +++ b/drivers/pci/controller/pci-aardvark.c @@ -0,0 +1,1987 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Driver for the Aardvark PCIe controller, used on Marvell Armada + * 3700. + * + * Copyright (C) 2016 Marvell + * + * Author: Hezi Shahmoon <hezi.shahmoon@marvell.com> + */ + +#include <linux/bitfield.h> +#include <linux/delay.h> +#include <linux/gpio/consumer.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/irqdomain.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/pci-ecam.h> +#include <linux/init.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/msi.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> + +#include "pci-host-common.h" +#include "../pci.h" +#include "../pci-bridge-emul.h" + +/* PCIe core registers */ +#define PCIE_CORE_DEV_ID_REG 0x0 +#define PCIE_CORE_CMD_STATUS_REG 0x4 +#define PCIE_CORE_DEV_REV_REG 0x8 +#define PCIE_CORE_SSDEV_ID_REG 0x2c +#define PCIE_CORE_PCIEXP_CAP 0xc0 +#define PCIE_CORE_PCIERR_CAP 0x100 +#define PCIE_CORE_ERR_CAPCTL_REG 0x118 +#define PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX BIT(5) +#define PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX_EN BIT(6) +#define PCIE_CORE_ERR_CAPCTL_ECRC_CHCK BIT(7) +#define PCIE_CORE_ERR_CAPCTL_ECRC_CHCK_RCV BIT(8) +/* PIO registers base address and register offsets */ +#define PIO_BASE_ADDR 0x4000 +#define PIO_CTRL (PIO_BASE_ADDR + 0x0) +#define PIO_CTRL_TYPE_MASK GENMASK(3, 0) +#define PIO_CTRL_ADDR_WIN_DISABLE BIT(24) +#define PIO_STAT (PIO_BASE_ADDR + 0x4) +#define PIO_COMPLETION_STATUS_SHIFT 7 +#define PIO_COMPLETION_STATUS_MASK GENMASK(9, 7) +#define PIO_COMPLETION_STATUS_OK 0 +#define PIO_COMPLETION_STATUS_UR 1 +#define PIO_COMPLETION_STATUS_RRS 2 +#define PIO_COMPLETION_STATUS_CA 4 +#define PIO_NON_POSTED_REQ BIT(10) +#define PIO_ERR_STATUS BIT(11) +#define PIO_ADDR_LS (PIO_BASE_ADDR + 0x8) +#define PIO_ADDR_MS (PIO_BASE_ADDR + 0xc) +#define PIO_WR_DATA (PIO_BASE_ADDR + 0x10) +#define PIO_WR_DATA_STRB (PIO_BASE_ADDR + 0x14) +#define PIO_RD_DATA (PIO_BASE_ADDR + 0x18) +#define PIO_START (PIO_BASE_ADDR + 0x1c) +#define PIO_ISR (PIO_BASE_ADDR + 0x20) +#define PIO_ISRM (PIO_BASE_ADDR + 0x24) + +/* Aardvark Control registers */ +#define CONTROL_BASE_ADDR 0x4800 +#define PCIE_CORE_CTRL0_REG (CONTROL_BASE_ADDR + 0x0) +#define PCIE_GEN_SEL_MSK 0x3 +#define PCIE_GEN_SEL_SHIFT 0x0 +#define SPEED_GEN_1 0 +#define SPEED_GEN_2 1 +#define SPEED_GEN_3 2 +#define IS_RC_MSK 1 +#define IS_RC_SHIFT 2 +#define LANE_CNT_MSK 0x18 +#define LANE_CNT_SHIFT 0x3 +#define LANE_COUNT_1 (0 << LANE_CNT_SHIFT) +#define LANE_COUNT_2 (1 << LANE_CNT_SHIFT) +#define LANE_COUNT_4 (2 << LANE_CNT_SHIFT) +#define LANE_COUNT_8 (3 << LANE_CNT_SHIFT) +#define LINK_TRAINING_EN BIT(6) +#define LEGACY_INTA BIT(28) +#define LEGACY_INTB BIT(29) +#define LEGACY_INTC BIT(30) +#define LEGACY_INTD BIT(31) +#define PCIE_CORE_CTRL1_REG (CONTROL_BASE_ADDR + 0x4) +#define HOT_RESET_GEN BIT(0) +#define PCIE_CORE_CTRL2_REG (CONTROL_BASE_ADDR + 0x8) +#define PCIE_CORE_CTRL2_RESERVED 0x7 +#define PCIE_CORE_CTRL2_TD_ENABLE BIT(4) +#define PCIE_CORE_CTRL2_STRICT_ORDER_ENABLE BIT(5) +#define PCIE_CORE_CTRL2_OB_WIN_ENABLE BIT(6) +#define PCIE_CORE_CTRL2_MSI_ENABLE BIT(10) +#define PCIE_CORE_REF_CLK_REG (CONTROL_BASE_ADDR + 0x14) +#define PCIE_CORE_REF_CLK_TX_ENABLE BIT(1) +#define PCIE_CORE_REF_CLK_RX_ENABLE BIT(2) +#define PCIE_MSG_LOG_REG (CONTROL_BASE_ADDR + 0x30) +#define PCIE_ISR0_REG (CONTROL_BASE_ADDR + 0x40) +#define PCIE_MSG_PM_PME_MASK BIT(7) +#define PCIE_ISR0_MASK_REG (CONTROL_BASE_ADDR + 0x44) +#define PCIE_ISR0_MSI_INT_PENDING BIT(24) +#define PCIE_ISR0_CORR_ERR BIT(11) +#define PCIE_ISR0_NFAT_ERR BIT(12) +#define PCIE_ISR0_FAT_ERR BIT(13) +#define PCIE_ISR0_ERR_MASK GENMASK(13, 11) +#define PCIE_ISR0_INTX_ASSERT(val) BIT(16 + (val)) +#define PCIE_ISR0_INTX_DEASSERT(val) BIT(20 + (val)) +#define PCIE_ISR0_ALL_MASK GENMASK(31, 0) +#define PCIE_ISR1_REG (CONTROL_BASE_ADDR + 0x48) +#define PCIE_ISR1_MASK_REG (CONTROL_BASE_ADDR + 0x4C) +#define PCIE_ISR1_POWER_STATE_CHANGE BIT(4) +#define PCIE_ISR1_FLUSH BIT(5) +#define PCIE_ISR1_INTX_ASSERT(val) BIT(8 + (val)) +#define PCIE_ISR1_ALL_MASK GENMASK(31, 0) +#define PCIE_MSI_ADDR_LOW_REG (CONTROL_BASE_ADDR + 0x50) +#define PCIE_MSI_ADDR_HIGH_REG (CONTROL_BASE_ADDR + 0x54) +#define PCIE_MSI_STATUS_REG (CONTROL_BASE_ADDR + 0x58) +#define PCIE_MSI_MASK_REG (CONTROL_BASE_ADDR + 0x5C) +#define PCIE_MSI_ALL_MASK GENMASK(31, 0) +#define PCIE_MSI_PAYLOAD_REG (CONTROL_BASE_ADDR + 0x9C) +#define PCIE_MSI_DATA_MASK GENMASK(15, 0) + +/* PCIe window configuration */ +#define OB_WIN_BASE_ADDR 0x4c00 +#define OB_WIN_BLOCK_SIZE 0x20 +#define OB_WIN_COUNT 8 +#define OB_WIN_REG_ADDR(win, offset) (OB_WIN_BASE_ADDR + \ + OB_WIN_BLOCK_SIZE * (win) + \ + (offset)) +#define OB_WIN_MATCH_LS(win) OB_WIN_REG_ADDR(win, 0x00) +#define OB_WIN_ENABLE BIT(0) +#define OB_WIN_MATCH_MS(win) OB_WIN_REG_ADDR(win, 0x04) +#define OB_WIN_REMAP_LS(win) OB_WIN_REG_ADDR(win, 0x08) +#define OB_WIN_REMAP_MS(win) OB_WIN_REG_ADDR(win, 0x0c) +#define OB_WIN_MASK_LS(win) OB_WIN_REG_ADDR(win, 0x10) +#define OB_WIN_MASK_MS(win) OB_WIN_REG_ADDR(win, 0x14) +#define OB_WIN_ACTIONS(win) OB_WIN_REG_ADDR(win, 0x18) +#define OB_WIN_DEFAULT_ACTIONS (OB_WIN_ACTIONS(OB_WIN_COUNT-1) + 0x4) +#define OB_WIN_FUNC_NUM_MASK GENMASK(31, 24) +#define OB_WIN_FUNC_NUM_SHIFT 24 +#define OB_WIN_FUNC_NUM_ENABLE BIT(23) +#define OB_WIN_BUS_NUM_BITS_MASK GENMASK(22, 20) +#define OB_WIN_BUS_NUM_BITS_SHIFT 20 +#define OB_WIN_MSG_CODE_ENABLE BIT(22) +#define OB_WIN_MSG_CODE_MASK GENMASK(21, 14) +#define OB_WIN_MSG_CODE_SHIFT 14 +#define OB_WIN_MSG_PAYLOAD_LEN BIT(12) +#define OB_WIN_ATTR_ENABLE BIT(11) +#define OB_WIN_ATTR_TC_MASK GENMASK(10, 8) +#define OB_WIN_ATTR_TC_SHIFT 8 +#define OB_WIN_ATTR_RELAXED BIT(7) +#define OB_WIN_ATTR_NOSNOOP BIT(6) +#define OB_WIN_ATTR_POISON BIT(5) +#define OB_WIN_ATTR_IDO BIT(4) +#define OB_WIN_TYPE_MASK GENMASK(3, 0) +#define OB_WIN_TYPE_SHIFT 0 +#define OB_WIN_TYPE_MEM 0x0 +#define OB_WIN_TYPE_IO 0x4 +#define OB_WIN_TYPE_CONFIG_TYPE0 0x8 +#define OB_WIN_TYPE_CONFIG_TYPE1 0x9 +#define OB_WIN_TYPE_MSG 0xc + +/* LMI registers base address and register offsets */ +#define LMI_BASE_ADDR 0x6000 +#define CFG_REG (LMI_BASE_ADDR + 0x0) +#define LTSSM_SHIFT 24 +#define LTSSM_MASK 0x3f +#define RC_BAR_CONFIG 0x300 + +/* LTSSM values in CFG_REG */ +enum { + LTSSM_DETECT_QUIET = 0x0, + LTSSM_DETECT_ACTIVE = 0x1, + LTSSM_POLLING_ACTIVE = 0x2, + LTSSM_POLLING_COMPLIANCE = 0x3, + LTSSM_POLLING_CONFIGURATION = 0x4, + LTSSM_CONFIG_LINKWIDTH_START = 0x5, + LTSSM_CONFIG_LINKWIDTH_ACCEPT = 0x6, + LTSSM_CONFIG_LANENUM_ACCEPT = 0x7, + LTSSM_CONFIG_LANENUM_WAIT = 0x8, + LTSSM_CONFIG_COMPLETE = 0x9, + LTSSM_CONFIG_IDLE = 0xa, + LTSSM_RECOVERY_RCVR_LOCK = 0xb, + LTSSM_RECOVERY_SPEED = 0xc, + LTSSM_RECOVERY_RCVR_CFG = 0xd, + LTSSM_RECOVERY_IDLE = 0xe, + LTSSM_L0 = 0x10, + LTSSM_RX_L0S_ENTRY = 0x11, + LTSSM_RX_L0S_IDLE = 0x12, + LTSSM_RX_L0S_FTS = 0x13, + LTSSM_TX_L0S_ENTRY = 0x14, + LTSSM_TX_L0S_IDLE = 0x15, + LTSSM_TX_L0S_FTS = 0x16, + LTSSM_L1_ENTRY = 0x17, + LTSSM_L1_IDLE = 0x18, + LTSSM_L2_IDLE = 0x19, + LTSSM_L2_TRANSMIT_WAKE = 0x1a, + LTSSM_DISABLED = 0x20, + LTSSM_LOOPBACK_ENTRY_MASTER = 0x21, + LTSSM_LOOPBACK_ACTIVE_MASTER = 0x22, + LTSSM_LOOPBACK_EXIT_MASTER = 0x23, + LTSSM_LOOPBACK_ENTRY_SLAVE = 0x24, + LTSSM_LOOPBACK_ACTIVE_SLAVE = 0x25, + LTSSM_LOOPBACK_EXIT_SLAVE = 0x26, + LTSSM_HOT_RESET = 0x27, + LTSSM_RECOVERY_EQUALIZATION_PHASE0 = 0x28, + LTSSM_RECOVERY_EQUALIZATION_PHASE1 = 0x29, + LTSSM_RECOVERY_EQUALIZATION_PHASE2 = 0x2a, + LTSSM_RECOVERY_EQUALIZATION_PHASE3 = 0x2b, +}; + +#define VENDOR_ID_REG (LMI_BASE_ADDR + 0x44) + +/* PCIe core controller registers */ +#define CTRL_CORE_BASE_ADDR 0x18000 +#define CTRL_CONFIG_REG (CTRL_CORE_BASE_ADDR + 0x0) +#define CTRL_MODE_SHIFT 0x0 +#define CTRL_MODE_MASK 0x1 +#define PCIE_CORE_MODE_DIRECT 0x0 +#define PCIE_CORE_MODE_COMMAND 0x1 + +/* PCIe Central Interrupts Registers */ +#define CENTRAL_INT_BASE_ADDR 0x1b000 +#define HOST_CTRL_INT_STATUS_REG (CENTRAL_INT_BASE_ADDR + 0x0) +#define HOST_CTRL_INT_MASK_REG (CENTRAL_INT_BASE_ADDR + 0x4) +#define PCIE_IRQ_CMDQ_INT BIT(0) +#define PCIE_IRQ_MSI_STATUS_INT BIT(1) +#define PCIE_IRQ_CMD_SENT_DONE BIT(3) +#define PCIE_IRQ_DMA_INT BIT(4) +#define PCIE_IRQ_IB_DXFERDONE BIT(5) +#define PCIE_IRQ_OB_DXFERDONE BIT(6) +#define PCIE_IRQ_OB_RXFERDONE BIT(7) +#define PCIE_IRQ_COMPQ_INT BIT(12) +#define PCIE_IRQ_DIR_RD_DDR_DET BIT(13) +#define PCIE_IRQ_DIR_WR_DDR_DET BIT(14) +#define PCIE_IRQ_CORE_INT BIT(16) +#define PCIE_IRQ_CORE_INT_PIO BIT(17) +#define PCIE_IRQ_DPMU_INT BIT(18) +#define PCIE_IRQ_PCIE_MIS_INT BIT(19) +#define PCIE_IRQ_MSI_INT1_DET BIT(20) +#define PCIE_IRQ_MSI_INT2_DET BIT(21) +#define PCIE_IRQ_RC_DBELL_DET BIT(22) +#define PCIE_IRQ_EP_STATUS BIT(23) +#define PCIE_IRQ_ALL_MASK GENMASK(31, 0) +#define PCIE_IRQ_ENABLE_INTS_MASK PCIE_IRQ_CORE_INT + +/* Transaction types */ +#define PCIE_CONFIG_RD_TYPE0 0x8 +#define PCIE_CONFIG_RD_TYPE1 0x9 +#define PCIE_CONFIG_WR_TYPE0 0xa +#define PCIE_CONFIG_WR_TYPE1 0xb + +#define PIO_RETRY_CNT 750000 /* 1.5 s */ +#define PIO_RETRY_DELAY 2 /* 2 us*/ + +#define RETRAIN_WAIT_MAX_RETRIES 10 +#define RETRAIN_WAIT_USLEEP_US 2000 + +#define MSI_IRQ_NUM 32 + +#define CFG_RD_RRS_VAL 0xffff0001 + +struct advk_pcie { + struct platform_device *pdev; + void __iomem *base; + struct { + phys_addr_t match; + phys_addr_t remap; + phys_addr_t mask; + u32 actions; + } wins[OB_WIN_COUNT]; + u8 wins_count; + struct irq_domain *rp_irq_domain; + struct irq_domain *irq_domain; + struct irq_chip irq_chip; + raw_spinlock_t irq_lock; + struct irq_domain *msi_inner_domain; + raw_spinlock_t msi_irq_lock; + DECLARE_BITMAP(msi_used, MSI_IRQ_NUM); + struct mutex msi_used_lock; + int link_gen; + struct pci_bridge_emul bridge; + struct gpio_desc *reset_gpio; + struct phy *phy; +}; + +static inline void advk_writel(struct advk_pcie *pcie, u32 val, u64 reg) +{ + writel(val, pcie->base + reg); +} + +static inline u32 advk_readl(struct advk_pcie *pcie, u64 reg) +{ + return readl(pcie->base + reg); +} + +static u8 advk_pcie_ltssm_state(struct advk_pcie *pcie) +{ + u32 val; + u8 ltssm_state; + + val = advk_readl(pcie, CFG_REG); + ltssm_state = (val >> LTSSM_SHIFT) & LTSSM_MASK; + return ltssm_state; +} + +static inline bool advk_pcie_link_up(struct advk_pcie *pcie) +{ + /* check if LTSSM is in normal operation - some L* state */ + u8 ltssm_state = advk_pcie_ltssm_state(pcie); + return ltssm_state >= LTSSM_L0 && ltssm_state < LTSSM_DISABLED; +} + +static inline bool advk_pcie_link_active(struct advk_pcie *pcie) +{ + /* + * According to PCIe Base specification 3.0, Table 4-14: Link + * Status Mapped to the LTSSM, and 4.2.6.3.6 Configuration.Idle + * is Link Up mapped to LTSSM Configuration.Idle, Recovery, L0, + * L0s, L1 and L2 states. And according to 3.2.1. Data Link + * Control and Management State Machine Rules is DL Up status + * reported in DL Active state. + */ + u8 ltssm_state = advk_pcie_ltssm_state(pcie); + return ltssm_state >= LTSSM_CONFIG_IDLE && ltssm_state < LTSSM_DISABLED; +} + +static inline bool advk_pcie_link_training(struct advk_pcie *pcie) +{ + /* + * According to PCIe Base specification 3.0, Table 4-14: Link + * Status Mapped to the LTSSM is Link Training mapped to LTSSM + * Configuration and Recovery states. + */ + u8 ltssm_state = advk_pcie_ltssm_state(pcie); + return ((ltssm_state >= LTSSM_CONFIG_LINKWIDTH_START && + ltssm_state < LTSSM_L0) || + (ltssm_state >= LTSSM_RECOVERY_EQUALIZATION_PHASE0 && + ltssm_state <= LTSSM_RECOVERY_EQUALIZATION_PHASE3)); +} + +static int advk_pcie_wait_for_link(struct advk_pcie *pcie) +{ + int retries; + + /* check if the link is up or not */ + for (retries = 0; retries < PCIE_LINK_WAIT_MAX_RETRIES; retries++) { + if (advk_pcie_link_up(pcie)) { + pci_host_common_link_train_delay(pcie->link_gen); + return 0; + } + + msleep(PCIE_LINK_WAIT_SLEEP_MS); + } + + return -ETIMEDOUT; +} + +static void advk_pcie_wait_for_retrain(struct advk_pcie *pcie) +{ + size_t retries; + + for (retries = 0; retries < RETRAIN_WAIT_MAX_RETRIES; ++retries) { + if (advk_pcie_link_training(pcie)) + break; + udelay(RETRAIN_WAIT_USLEEP_US); + } +} + +static void advk_pcie_issue_perst(struct advk_pcie *pcie) +{ + if (!pcie->reset_gpio) + return; + + /* 10ms delay is needed for some cards */ + dev_info(&pcie->pdev->dev, "issuing PERST via reset GPIO for 10ms\n"); + gpiod_set_value_cansleep(pcie->reset_gpio, 1); + usleep_range(10000, 11000); + gpiod_set_value_cansleep(pcie->reset_gpio, 0); +} + +static void advk_pcie_train_link(struct advk_pcie *pcie) +{ + struct device *dev = &pcie->pdev->dev; + u32 reg; + int ret; + + /* + * Setup PCIe rev / gen compliance based on device tree property + * 'max-link-speed' which also forces maximal link speed. + */ + reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG); + reg &= ~PCIE_GEN_SEL_MSK; + if (pcie->link_gen == 3) + reg |= SPEED_GEN_3; + else if (pcie->link_gen == 2) + reg |= SPEED_GEN_2; + else + reg |= SPEED_GEN_1; + advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG); + + /* + * Set maximal link speed value also into PCIe Link Control 2 register. + * Armada 3700 Functional Specification says that default value is based + * on SPEED_GEN but tests showed that default value is always 8.0 GT/s. + */ + reg = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + PCI_EXP_LNKCTL2); + reg &= ~PCI_EXP_LNKCTL2_TLS; + if (pcie->link_gen == 3) + reg |= PCI_EXP_LNKCTL2_TLS_8_0GT; + else if (pcie->link_gen == 2) + reg |= PCI_EXP_LNKCTL2_TLS_5_0GT; + else + reg |= PCI_EXP_LNKCTL2_TLS_2_5GT; + advk_writel(pcie, reg, PCIE_CORE_PCIEXP_CAP + PCI_EXP_LNKCTL2); + + /* Enable link training after selecting PCIe generation */ + reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG); + reg |= LINK_TRAINING_EN; + advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG); + + /* + * Reset PCIe card via PERST# signal. Some cards are not detected + * during link training when they are in some non-initial state. + */ + advk_pcie_issue_perst(pcie); + + /* + * PERST# signal could have been asserted by pinctrl subsystem before + * probe() callback has been called or issued explicitly by reset gpio + * function advk_pcie_issue_perst(), making the endpoint going into + * fundamental reset. As required by PCI Express spec (PCI Express + * Base Specification, REV. 4.0 PCI Express, February 19 2014, 6.6.1 + * Conventional Reset) a delay for at least 100ms after such a reset + * before sending a Configuration Request to the device is needed. + * So wait until PCIe link is up. Function advk_pcie_wait_for_link() + * waits for link at least 900ms. + */ + ret = advk_pcie_wait_for_link(pcie); + if (ret < 0) + dev_err(dev, "link never came up\n"); + else + dev_info(dev, "link up\n"); +} + +/* + * Set PCIe address window register which could be used for memory + * mapping. + */ +static void advk_pcie_set_ob_win(struct advk_pcie *pcie, u8 win_num, + phys_addr_t match, phys_addr_t remap, + phys_addr_t mask, u32 actions) +{ + advk_writel(pcie, OB_WIN_ENABLE | + lower_32_bits(match), OB_WIN_MATCH_LS(win_num)); + advk_writel(pcie, upper_32_bits(match), OB_WIN_MATCH_MS(win_num)); + advk_writel(pcie, lower_32_bits(remap), OB_WIN_REMAP_LS(win_num)); + advk_writel(pcie, upper_32_bits(remap), OB_WIN_REMAP_MS(win_num)); + advk_writel(pcie, lower_32_bits(mask), OB_WIN_MASK_LS(win_num)); + advk_writel(pcie, upper_32_bits(mask), OB_WIN_MASK_MS(win_num)); + advk_writel(pcie, actions, OB_WIN_ACTIONS(win_num)); +} + +static void advk_pcie_disable_ob_win(struct advk_pcie *pcie, u8 win_num) +{ + advk_writel(pcie, 0, OB_WIN_MATCH_LS(win_num)); + advk_writel(pcie, 0, OB_WIN_MATCH_MS(win_num)); + advk_writel(pcie, 0, OB_WIN_REMAP_LS(win_num)); + advk_writel(pcie, 0, OB_WIN_REMAP_MS(win_num)); + advk_writel(pcie, 0, OB_WIN_MASK_LS(win_num)); + advk_writel(pcie, 0, OB_WIN_MASK_MS(win_num)); + advk_writel(pcie, 0, OB_WIN_ACTIONS(win_num)); +} + +static void advk_pcie_setup_hw(struct advk_pcie *pcie) +{ + phys_addr_t msi_addr; + u32 reg; + int i; + + /* + * Configure PCIe Reference clock. Direction is from the PCIe + * controller to the endpoint card, so enable transmitting of + * Reference clock differential signal off-chip and disable + * receiving off-chip differential signal. + */ + reg = advk_readl(pcie, PCIE_CORE_REF_CLK_REG); + reg |= PCIE_CORE_REF_CLK_TX_ENABLE; + reg &= ~PCIE_CORE_REF_CLK_RX_ENABLE; + advk_writel(pcie, reg, PCIE_CORE_REF_CLK_REG); + + /* Set to Direct mode */ + reg = advk_readl(pcie, CTRL_CONFIG_REG); + reg &= ~(CTRL_MODE_MASK << CTRL_MODE_SHIFT); + reg |= ((PCIE_CORE_MODE_DIRECT & CTRL_MODE_MASK) << CTRL_MODE_SHIFT); + advk_writel(pcie, reg, CTRL_CONFIG_REG); + + /* Set PCI global control register to RC mode */ + reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG); + reg |= (IS_RC_MSK << IS_RC_SHIFT); + advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG); + + /* + * Replace incorrect PCI vendor id value 0x1b4b by correct value 0x11ab. + * VENDOR_ID_REG contains vendor id in low 16 bits and subsystem vendor + * id in high 16 bits. Updating this register changes readback value of + * read-only vendor id bits in PCIE_CORE_DEV_ID_REG register. Workaround + * for erratum 4.1: "The value of device and vendor ID is incorrect". + */ + reg = (PCI_VENDOR_ID_MARVELL << 16) | PCI_VENDOR_ID_MARVELL; + advk_writel(pcie, reg, VENDOR_ID_REG); + + /* + * Change Class Code of PCI Bridge device to PCI Bridge (0x600400), + * because the default value is Mass storage controller (0x010400). + * + * Note that this Aardvark PCI Bridge does not have compliant Type 1 + * Configuration Space and it even cannot be accessed via Aardvark's + * PCI config space access method. Something like config space is + * available in internal Aardvark registers starting at offset 0x0 + * and is reported as Type 0. In range 0x10 - 0x34 it has totally + * different registers. + * + * Therefore driver uses emulation of PCI Bridge which emulates + * access to configuration space via internal Aardvark registers or + * emulated configuration buffer. + */ + reg = advk_readl(pcie, PCIE_CORE_DEV_REV_REG); + reg &= ~0xffffff00; + reg |= PCI_CLASS_BRIDGE_PCI_NORMAL << 8; + advk_writel(pcie, reg, PCIE_CORE_DEV_REV_REG); + + /* Disable Root Bridge I/O space, memory space and bus mastering */ + reg = advk_readl(pcie, PCIE_CORE_CMD_STATUS_REG); + reg &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + advk_writel(pcie, reg, PCIE_CORE_CMD_STATUS_REG); + + /* Set Advanced Error Capabilities and Control PF0 register */ + reg = PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX | + PCIE_CORE_ERR_CAPCTL_ECRC_CHK_TX_EN | + PCIE_CORE_ERR_CAPCTL_ECRC_CHCK | + PCIE_CORE_ERR_CAPCTL_ECRC_CHCK_RCV; + advk_writel(pcie, reg, PCIE_CORE_ERR_CAPCTL_REG); + + /* Set PCIe Device Control register */ + reg = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + PCI_EXP_DEVCTL); + reg &= ~PCI_EXP_DEVCTL_RELAX_EN; + reg &= ~PCI_EXP_DEVCTL_NOSNOOP_EN; + reg &= ~PCI_EXP_DEVCTL_PAYLOAD; + reg &= ~PCI_EXP_DEVCTL_READRQ; + reg |= PCI_EXP_DEVCTL_PAYLOAD_512B; + reg |= PCI_EXP_DEVCTL_READRQ_512B; + advk_writel(pcie, reg, PCIE_CORE_PCIEXP_CAP + PCI_EXP_DEVCTL); + + /* Program PCIe Control 2 to disable strict ordering */ + reg = PCIE_CORE_CTRL2_RESERVED | + PCIE_CORE_CTRL2_TD_ENABLE; + advk_writel(pcie, reg, PCIE_CORE_CTRL2_REG); + + /* Set lane X1 */ + reg = advk_readl(pcie, PCIE_CORE_CTRL0_REG); + reg &= ~LANE_CNT_MSK; + reg |= LANE_COUNT_1; + advk_writel(pcie, reg, PCIE_CORE_CTRL0_REG); + + /* Set MSI address */ + msi_addr = virt_to_phys(pcie); + advk_writel(pcie, lower_32_bits(msi_addr), PCIE_MSI_ADDR_LOW_REG); + advk_writel(pcie, upper_32_bits(msi_addr), PCIE_MSI_ADDR_HIGH_REG); + + /* Enable MSI */ + reg = advk_readl(pcie, PCIE_CORE_CTRL2_REG); + reg |= PCIE_CORE_CTRL2_MSI_ENABLE; + advk_writel(pcie, reg, PCIE_CORE_CTRL2_REG); + + /* Clear all interrupts */ + advk_writel(pcie, PCIE_MSI_ALL_MASK, PCIE_MSI_STATUS_REG); + advk_writel(pcie, PCIE_ISR0_ALL_MASK, PCIE_ISR0_REG); + advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_REG); + advk_writel(pcie, PCIE_IRQ_ALL_MASK, HOST_CTRL_INT_STATUS_REG); + + /* Disable All ISR0/1 and MSI Sources */ + advk_writel(pcie, PCIE_ISR0_ALL_MASK, PCIE_ISR0_MASK_REG); + advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_MASK_REG); + advk_writel(pcie, PCIE_MSI_ALL_MASK, PCIE_MSI_MASK_REG); + + /* Unmask summary MSI interrupt */ + reg = advk_readl(pcie, PCIE_ISR0_MASK_REG); + reg &= ~PCIE_ISR0_MSI_INT_PENDING; + advk_writel(pcie, reg, PCIE_ISR0_MASK_REG); + + /* Unmask PME interrupt for processing of PME requester */ + reg = advk_readl(pcie, PCIE_ISR0_MASK_REG); + reg &= ~PCIE_MSG_PM_PME_MASK; + advk_writel(pcie, reg, PCIE_ISR0_MASK_REG); + + /* Enable summary interrupt for GIC SPI source */ + reg = PCIE_IRQ_ALL_MASK & (~PCIE_IRQ_ENABLE_INTS_MASK); + advk_writel(pcie, reg, HOST_CTRL_INT_MASK_REG); + + /* + * Enable AXI address window location generation: + * When it is enabled, the default outbound window + * configurations (Default User Field: 0xD0074CFC) + * are used to transparent address translation for + * the outbound transactions. Thus, PCIe address + * windows are not required for transparent memory + * access when default outbound window configuration + * is set for memory access. + */ + reg = advk_readl(pcie, PCIE_CORE_CTRL2_REG); + reg |= PCIE_CORE_CTRL2_OB_WIN_ENABLE; + advk_writel(pcie, reg, PCIE_CORE_CTRL2_REG); + + /* + * Set memory access in Default User Field so it + * is not required to configure PCIe address for + * transparent memory access. + */ + advk_writel(pcie, OB_WIN_TYPE_MEM, OB_WIN_DEFAULT_ACTIONS); + + /* + * Bypass the address window mapping for PIO: + * Since PIO access already contains all required + * info over AXI interface by PIO registers, the + * address window is not required. + */ + reg = advk_readl(pcie, PIO_CTRL); + reg |= PIO_CTRL_ADDR_WIN_DISABLE; + advk_writel(pcie, reg, PIO_CTRL); + + /* + * Configure PCIe address windows for non-memory or + * non-transparent access as by default PCIe uses + * transparent memory access. + */ + for (i = 0; i < pcie->wins_count; i++) + advk_pcie_set_ob_win(pcie, i, + pcie->wins[i].match, pcie->wins[i].remap, + pcie->wins[i].mask, pcie->wins[i].actions); + + /* Disable remaining PCIe outbound windows */ + for (i = pcie->wins_count; i < OB_WIN_COUNT; i++) + advk_pcie_disable_ob_win(pcie, i); + + advk_pcie_train_link(pcie); +} + +static int advk_pcie_check_pio_status(struct advk_pcie *pcie, bool allow_rrs, u32 *val) +{ + struct device *dev = &pcie->pdev->dev; + u32 reg; + unsigned int status; + char *strcomp_status, *str_posted; + int ret; + + reg = advk_readl(pcie, PIO_STAT); + status = (reg & PIO_COMPLETION_STATUS_MASK) >> + PIO_COMPLETION_STATUS_SHIFT; + + /* + * According to HW spec, the PIO status check sequence as below: + * 1) even if COMPLETION_STATUS(bit9:7) indicates successful, + * it still needs to check Error Status(bit11), only when this bit + * indicates no error happen, the operation is successful. + * 2) value Unsupported Request(1) of COMPLETION_STATUS(bit9:7) only + * means a PIO write error, and for PIO read it is successful with + * a read value of 0xFFFFFFFF. + * 3) value Config Request Retry Status(RRS) of COMPLETION_STATUS(bit9:7) + * only means a PIO write error, and for PIO read it is successful + * with a read value of 0xFFFF0001. + * 4) value Completer Abort (CA) of COMPLETION_STATUS(bit9:7) means + * error for both PIO read and PIO write operation. + * 5) other errors are indicated as 'unknown'. + */ + switch (status) { + case PIO_COMPLETION_STATUS_OK: + if (reg & PIO_ERR_STATUS) { + strcomp_status = "COMP_ERR"; + ret = -EFAULT; + break; + } + /* Get the read result */ + if (val) + *val = advk_readl(pcie, PIO_RD_DATA); + /* No error */ + strcomp_status = NULL; + ret = 0; + break; + case PIO_COMPLETION_STATUS_UR: + strcomp_status = "UR"; + ret = -EOPNOTSUPP; + break; + case PIO_COMPLETION_STATUS_RRS: + if (allow_rrs && val) { + /* PCIe r6.0, sec 2.3.2, says: + * If Configuration RRS Software Visibility is enabled: + * For a Configuration Read Request that includes both + * bytes of the Vendor ID field of a device Function's + * Configuration Space Header, the Root Complex must + * complete the Request to the host by returning a + * read-data value of 0001h for the Vendor ID field and + * all '1's for any additional bytes included in the + * request. + * + * So RRS in this case is not an error status. + */ + *val = CFG_RD_RRS_VAL; + strcomp_status = NULL; + ret = 0; + break; + } + /* PCIe r6.0, sec 2.3.2, says: + * If RRS Software Visibility is not enabled, the Root Complex + * must re-issue the Configuration Request as a new Request. + * If RRS Software Visibility is enabled: For a Configuration + * Write Request or for any other Configuration Read Request, + * the Root Complex must re-issue the Configuration Request as + * a new Request. + * A Root Complex implementation may choose to limit the number + * of Configuration Request/RRS Completion Status loops before + * determining that something is wrong with the target of the + * Request and taking appropriate action, e.g., complete the + * Request to the host as a failed transaction. + * + * So return -EAGAIN and caller (pci-aardvark.c driver) will + * re-issue request again up to the PIO_RETRY_CNT retries. + */ + strcomp_status = "RRS"; + ret = -EAGAIN; + break; + case PIO_COMPLETION_STATUS_CA: + strcomp_status = "CA"; + ret = -ECANCELED; + break; + default: + strcomp_status = "Unknown"; + ret = -EINVAL; + break; + } + + if (!strcomp_status) + return ret; + + if (reg & PIO_NON_POSTED_REQ) + str_posted = "Non-posted"; + else + str_posted = "Posted"; + + dev_dbg(dev, "%s PIO Response Status: %s, %#x @ %#x\n", + str_posted, strcomp_status, reg, advk_readl(pcie, PIO_ADDR_LS)); + + return ret; +} + +static int advk_pcie_wait_pio(struct advk_pcie *pcie) +{ + struct device *dev = &pcie->pdev->dev; + int i; + + for (i = 1; i <= PIO_RETRY_CNT; i++) { + u32 start, isr; + + start = advk_readl(pcie, PIO_START); + isr = advk_readl(pcie, PIO_ISR); + if (!start && isr) + return i; + udelay(PIO_RETRY_DELAY); + } + + dev_err(dev, "PIO read/write transfer time out\n"); + return -ETIMEDOUT; +} + +static pci_bridge_emul_read_status_t +advk_pci_bridge_emul_base_conf_read(struct pci_bridge_emul *bridge, + int reg, u32 *value) +{ + struct advk_pcie *pcie = bridge->data; + + switch (reg) { + case PCI_COMMAND: + *value = advk_readl(pcie, PCIE_CORE_CMD_STATUS_REG); + return PCI_BRIDGE_EMUL_HANDLED; + + case PCI_INTERRUPT_LINE: { + /* + * From the whole 32bit register we support reading from HW only + * two bits: PCI_BRIDGE_CTL_BUS_RESET and PCI_BRIDGE_CTL_SERR. + * Other bits are retrieved only from emulated config buffer. + */ + __le32 *cfgspace = (__le32 *)&bridge->conf; + u32 val = le32_to_cpu(cfgspace[PCI_INTERRUPT_LINE / 4]); + if (advk_readl(pcie, PCIE_ISR0_MASK_REG) & PCIE_ISR0_ERR_MASK) + val &= ~(PCI_BRIDGE_CTL_SERR << 16); + else + val |= PCI_BRIDGE_CTL_SERR << 16; + if (advk_readl(pcie, PCIE_CORE_CTRL1_REG) & HOT_RESET_GEN) + val |= PCI_BRIDGE_CTL_BUS_RESET << 16; + else + val &= ~(PCI_BRIDGE_CTL_BUS_RESET << 16); + *value = val; + return PCI_BRIDGE_EMUL_HANDLED; + } + + default: + return PCI_BRIDGE_EMUL_NOT_HANDLED; + } +} + +static void +advk_pci_bridge_emul_base_conf_write(struct pci_bridge_emul *bridge, + int reg, u32 old, u32 new, u32 mask) +{ + struct advk_pcie *pcie = bridge->data; + + switch (reg) { + case PCI_COMMAND: + advk_writel(pcie, new, PCIE_CORE_CMD_STATUS_REG); + break; + + case PCI_INTERRUPT_LINE: + /* + * According to Figure 6-3: Pseudo Logic Diagram for Error + * Message Controls in PCIe base specification, SERR# Enable bit + * in Bridge Control register enable receiving of ERR_* messages + */ + if (mask & (PCI_BRIDGE_CTL_SERR << 16)) { + u32 val = advk_readl(pcie, PCIE_ISR0_MASK_REG); + if (new & (PCI_BRIDGE_CTL_SERR << 16)) + val &= ~PCIE_ISR0_ERR_MASK; + else + val |= PCIE_ISR0_ERR_MASK; + advk_writel(pcie, val, PCIE_ISR0_MASK_REG); + } + if (mask & (PCI_BRIDGE_CTL_BUS_RESET << 16)) { + u32 val = advk_readl(pcie, PCIE_CORE_CTRL1_REG); + if (new & (PCI_BRIDGE_CTL_BUS_RESET << 16)) + val |= HOT_RESET_GEN; + else + val &= ~HOT_RESET_GEN; + advk_writel(pcie, val, PCIE_CORE_CTRL1_REG); + } + break; + + default: + break; + } +} + +static pci_bridge_emul_read_status_t +advk_pci_bridge_emul_pcie_conf_read(struct pci_bridge_emul *bridge, + int reg, u32 *value) +{ + struct advk_pcie *pcie = bridge->data; + + + switch (reg) { + /* + * PCI_EXP_SLTCAP, PCI_EXP_SLTCTL, PCI_EXP_RTCTL and PCI_EXP_RTSTA are + * also supported, but do not need to be handled here, because their + * values are stored in emulated config space buffer, and we read them + * from there when needed. + */ + + case PCI_EXP_LNKCAP: { + u32 val = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + reg); + /* + * PCI_EXP_LNKCAP_DLLLARC bit is hardwired in aardvark HW to 0. + * But support for PCI_EXP_LNKSTA_DLLLA is emulated via ltssm + * state so explicitly enable PCI_EXP_LNKCAP_DLLLARC flag. + */ + val |= PCI_EXP_LNKCAP_DLLLARC; + *value = val; + return PCI_BRIDGE_EMUL_HANDLED; + } + + case PCI_EXP_LNKCTL: { + /* u32 contains both PCI_EXP_LNKCTL and PCI_EXP_LNKSTA */ + u32 val = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + reg) & + ~(PCI_EXP_LNKSTA_LT << 16); + if (advk_pcie_link_training(pcie)) + val |= (PCI_EXP_LNKSTA_LT << 16); + if (advk_pcie_link_active(pcie)) + val |= (PCI_EXP_LNKSTA_DLLLA << 16); + *value = val; + return PCI_BRIDGE_EMUL_HANDLED; + } + + case PCI_EXP_DEVCAP: + case PCI_EXP_DEVCTL: + case PCI_EXP_DEVCAP2: + case PCI_EXP_DEVCTL2: + case PCI_EXP_LNKCAP2: + case PCI_EXP_LNKCTL2: + *value = advk_readl(pcie, PCIE_CORE_PCIEXP_CAP + reg); + return PCI_BRIDGE_EMUL_HANDLED; + + default: + return PCI_BRIDGE_EMUL_NOT_HANDLED; + } + +} + +static void +advk_pci_bridge_emul_pcie_conf_write(struct pci_bridge_emul *bridge, + int reg, u32 old, u32 new, u32 mask) +{ + struct advk_pcie *pcie = bridge->data; + + switch (reg) { + case PCI_EXP_LNKCTL: + advk_writel(pcie, new, PCIE_CORE_PCIEXP_CAP + reg); + if (new & PCI_EXP_LNKCTL_RL) + advk_pcie_wait_for_retrain(pcie); + break; + + case PCI_EXP_RTCTL: { + u16 rootctl = le16_to_cpu(bridge->pcie_conf.rootctl); + /* Only emulation of PMEIE and RRS_SVE bits is provided */ + rootctl &= PCI_EXP_RTCTL_PMEIE | PCI_EXP_RTCTL_RRS_SVE; + bridge->pcie_conf.rootctl = cpu_to_le16(rootctl); + break; + } + + /* + * PCI_EXP_RTSTA is also supported, but does not need to be handled + * here, because its value is stored in emulated config space buffer, + * and we write it there when needed. + */ + + case PCI_EXP_DEVCTL: + case PCI_EXP_DEVCTL2: + case PCI_EXP_LNKCTL2: + advk_writel(pcie, new, PCIE_CORE_PCIEXP_CAP + reg); + break; + + default: + break; + } +} + +static pci_bridge_emul_read_status_t +advk_pci_bridge_emul_ext_conf_read(struct pci_bridge_emul *bridge, + int reg, u32 *value) +{ + struct advk_pcie *pcie = bridge->data; + + switch (reg) { + case 0: + *value = advk_readl(pcie, PCIE_CORE_PCIERR_CAP + reg); + + /* + * PCI_EXT_CAP_NEXT bits are set to offset 0x150, but Armada + * 3700 Functional Specification does not document registers + * at those addresses. + * + * Thus we clear PCI_EXT_CAP_NEXT bits to make Advanced Error + * Reporting Capability header the last Extended Capability. + * If we obtain documentation for those registers in the + * future, this can be changed. + */ + *value &= 0x000fffff; + return PCI_BRIDGE_EMUL_HANDLED; + + case PCI_ERR_UNCOR_STATUS: + case PCI_ERR_UNCOR_MASK: + case PCI_ERR_UNCOR_SEVER: + case PCI_ERR_COR_STATUS: + case PCI_ERR_COR_MASK: + case PCI_ERR_CAP: + case PCI_ERR_HEADER_LOG + 0: + case PCI_ERR_HEADER_LOG + 4: + case PCI_ERR_HEADER_LOG + 8: + case PCI_ERR_HEADER_LOG + 12: + case PCI_ERR_ROOT_COMMAND: + case PCI_ERR_ROOT_STATUS: + case PCI_ERR_ROOT_ERR_SRC: + *value = advk_readl(pcie, PCIE_CORE_PCIERR_CAP + reg); + return PCI_BRIDGE_EMUL_HANDLED; + + default: + return PCI_BRIDGE_EMUL_NOT_HANDLED; + } +} + +static void +advk_pci_bridge_emul_ext_conf_write(struct pci_bridge_emul *bridge, + int reg, u32 old, u32 new, u32 mask) +{ + struct advk_pcie *pcie = bridge->data; + + switch (reg) { + /* These are W1C registers, so clear other bits */ + case PCI_ERR_UNCOR_STATUS: + case PCI_ERR_COR_STATUS: + case PCI_ERR_ROOT_STATUS: + new &= mask; + fallthrough; + + case PCI_ERR_UNCOR_MASK: + case PCI_ERR_UNCOR_SEVER: + case PCI_ERR_COR_MASK: + case PCI_ERR_CAP: + case PCI_ERR_HEADER_LOG + 0: + case PCI_ERR_HEADER_LOG + 4: + case PCI_ERR_HEADER_LOG + 8: + case PCI_ERR_HEADER_LOG + 12: + case PCI_ERR_ROOT_COMMAND: + case PCI_ERR_ROOT_ERR_SRC: + advk_writel(pcie, new, PCIE_CORE_PCIERR_CAP + reg); + break; + + default: + break; + } +} + +static const struct pci_bridge_emul_ops advk_pci_bridge_emul_ops = { + .read_base = advk_pci_bridge_emul_base_conf_read, + .write_base = advk_pci_bridge_emul_base_conf_write, + .read_pcie = advk_pci_bridge_emul_pcie_conf_read, + .write_pcie = advk_pci_bridge_emul_pcie_conf_write, + .read_ext = advk_pci_bridge_emul_ext_conf_read, + .write_ext = advk_pci_bridge_emul_ext_conf_write, +}; + +/* + * Initialize the configuration space of the PCI-to-PCI bridge + * associated with the given PCIe interface. + */ +static int advk_sw_pci_bridge_init(struct advk_pcie *pcie) +{ + struct pci_bridge_emul *bridge = &pcie->bridge; + + bridge->conf.vendor = + cpu_to_le16(advk_readl(pcie, PCIE_CORE_DEV_ID_REG) & 0xffff); + bridge->conf.device = + cpu_to_le16(advk_readl(pcie, PCIE_CORE_DEV_ID_REG) >> 16); + bridge->conf.class_revision = + cpu_to_le32(advk_readl(pcie, PCIE_CORE_DEV_REV_REG) & 0xff); + + /* Support 32 bits I/O addressing */ + bridge->conf.iobase = PCI_IO_RANGE_TYPE_32; + bridge->conf.iolimit = PCI_IO_RANGE_TYPE_32; + + /* Support 64 bits memory pref */ + bridge->conf.pref_mem_base = cpu_to_le16(PCI_PREF_RANGE_TYPE_64); + bridge->conf.pref_mem_limit = cpu_to_le16(PCI_PREF_RANGE_TYPE_64); + + /* Support interrupt A for MSI feature */ + bridge->conf.intpin = PCI_INTERRUPT_INTA; + + /* + * Aardvark HW provides PCIe Capability structure in version 2 and + * indicate slot support, which is emulated. + */ + bridge->pcie_conf.cap = cpu_to_le16(2 | PCI_EXP_FLAGS_SLOT); + + /* + * Set Presence Detect State bit permanently since there is no support + * for unplugging the card nor detecting whether it is plugged. (If a + * platform exists in the future that supports it, via a GPIO for + * example, it should be implemented via this bit.) + * + * Set physical slot number to 1 since there is only one port and zero + * value is reserved for ports within the same silicon as Root Port + * which is not our case. + */ + bridge->pcie_conf.slotcap = cpu_to_le32(FIELD_PREP(PCI_EXP_SLTCAP_PSN, + 1)); + bridge->pcie_conf.slotsta = cpu_to_le16(PCI_EXP_SLTSTA_PDS); + + /* Indicates supports for Completion Retry Status */ + bridge->pcie_conf.rootcap = cpu_to_le16(PCI_EXP_RTCAP_RRS_SV); + + bridge->subsystem_vendor_id = advk_readl(pcie, PCIE_CORE_SSDEV_ID_REG) & 0xffff; + bridge->subsystem_id = advk_readl(pcie, PCIE_CORE_SSDEV_ID_REG) >> 16; + bridge->has_pcie = true; + bridge->pcie_start = PCIE_CORE_PCIEXP_CAP; + bridge->data = pcie; + bridge->ops = &advk_pci_bridge_emul_ops; + + return pci_bridge_emul_init(bridge, 0); +} + +static bool advk_pcie_valid_device(struct advk_pcie *pcie, struct pci_bus *bus, + int devfn) +{ + if (pci_is_root_bus(bus) && PCI_SLOT(devfn) != 0) + return false; + + /* + * If the link goes down after we check for link-up, we have a problem: + * if a PIO request is executed while link-down, the whole controller + * gets stuck in a non-functional state, and even after link comes up + * again, PIO requests won't work anymore, and a reset of the whole PCIe + * controller is needed. Therefore we need to prevent sending PIO + * requests while the link is down. + */ + if (!pci_is_root_bus(bus) && !advk_pcie_link_up(pcie)) + return false; + + return true; +} + +static bool advk_pcie_pio_is_running(struct advk_pcie *pcie) +{ + struct device *dev = &pcie->pdev->dev; + + /* + * Trying to start a new PIO transfer when previous has not completed + * cause External Abort on CPU which results in kernel panic: + * + * SError Interrupt on CPU0, code 0xbf000002 -- SError + * Kernel panic - not syncing: Asynchronous SError Interrupt + * + * Functions advk_pcie_rd_conf() and advk_pcie_wr_conf() are protected + * by raw_spin_lock_irqsave() at pci_lock_config() level to prevent + * concurrent calls at the same time. But because PIO transfer may take + * about 1.5s when link is down or card is disconnected, it means that + * advk_pcie_wait_pio() does not always have to wait for completion. + * + * Some versions of ARM Trusted Firmware handles this External Abort at + * EL3 level and mask it to prevent kernel panic. Relevant TF-A commit: + * https://git.trustedfirmware.org/TF-A/trusted-firmware-a.git/commit/?id=3c7dcdac5c50 + */ + if (advk_readl(pcie, PIO_START)) { + dev_err(dev, "Previous PIO read/write transfer is still running\n"); + return true; + } + + return false; +} + +static int advk_pcie_rd_conf(struct pci_bus *bus, u32 devfn, + int where, int size, u32 *val) +{ + struct advk_pcie *pcie = bus->sysdata; + int retry_count; + bool allow_rrs; + u32 reg; + int ret; + + if (!advk_pcie_valid_device(pcie, bus, devfn)) + return PCIBIOS_DEVICE_NOT_FOUND; + + if (pci_is_root_bus(bus)) + return pci_bridge_emul_conf_read(&pcie->bridge, where, + size, val); + + /* + * Configuration Request Retry Status (RRS) is possible to return + * only when reading both bytes from PCI_VENDOR_ID at once and + * RRS_SVE flag on Root Port is enabled. + */ + allow_rrs = (where == PCI_VENDOR_ID) && (size >= 2) && + (le16_to_cpu(pcie->bridge.pcie_conf.rootctl) & + PCI_EXP_RTCTL_RRS_SVE); + + if (advk_pcie_pio_is_running(pcie)) + goto try_rrs; + + /* Program the control register */ + reg = advk_readl(pcie, PIO_CTRL); + reg &= ~PIO_CTRL_TYPE_MASK; + if (pci_is_root_bus(bus->parent)) + reg |= PCIE_CONFIG_RD_TYPE0; + else + reg |= PCIE_CONFIG_RD_TYPE1; + advk_writel(pcie, reg, PIO_CTRL); + + /* Program the address registers */ + reg = ALIGN_DOWN(PCIE_ECAM_OFFSET(bus->number, devfn, where), 4); + advk_writel(pcie, reg, PIO_ADDR_LS); + advk_writel(pcie, 0, PIO_ADDR_MS); + + /* Program the data strobe */ + advk_writel(pcie, 0xf, PIO_WR_DATA_STRB); + + retry_count = 0; + do { + /* Clear PIO DONE ISR and start the transfer */ + advk_writel(pcie, 1, PIO_ISR); + advk_writel(pcie, 1, PIO_START); + + ret = advk_pcie_wait_pio(pcie); + if (ret < 0) + goto try_rrs; + + retry_count += ret; + + /* Check PIO status and get the read result */ + ret = advk_pcie_check_pio_status(pcie, allow_rrs, val); + } while (ret == -EAGAIN && retry_count < PIO_RETRY_CNT); + + if (ret < 0) + goto fail; + + if (size == 1) + *val = (*val >> (8 * (where & 3))) & 0xff; + else if (size == 2) + *val = (*val >> (8 * (where & 3))) & 0xffff; + + return PCIBIOS_SUCCESSFUL; + +try_rrs: + /* + * If it is possible, return Configuration Request Retry Status so + * that caller tries to issue the request again instead of failing. + */ + if (allow_rrs) { + *val = CFG_RD_RRS_VAL; + return PCIBIOS_SUCCESSFUL; + } + +fail: + *val = 0xffffffff; + return PCIBIOS_SET_FAILED; +} + +static int advk_pcie_wr_conf(struct pci_bus *bus, u32 devfn, + int where, int size, u32 val) +{ + struct advk_pcie *pcie = bus->sysdata; + u32 reg; + u32 data_strobe = 0x0; + int retry_count; + int offset; + int ret; + + if (!advk_pcie_valid_device(pcie, bus, devfn)) + return PCIBIOS_DEVICE_NOT_FOUND; + + if (pci_is_root_bus(bus)) + return pci_bridge_emul_conf_write(&pcie->bridge, where, + size, val); + + if (where % size) + return PCIBIOS_SET_FAILED; + + if (advk_pcie_pio_is_running(pcie)) + return PCIBIOS_SET_FAILED; + + /* Program the control register */ + reg = advk_readl(pcie, PIO_CTRL); + reg &= ~PIO_CTRL_TYPE_MASK; + if (pci_is_root_bus(bus->parent)) + reg |= PCIE_CONFIG_WR_TYPE0; + else + reg |= PCIE_CONFIG_WR_TYPE1; + advk_writel(pcie, reg, PIO_CTRL); + + /* Program the address registers */ + reg = ALIGN_DOWN(PCIE_ECAM_OFFSET(bus->number, devfn, where), 4); + advk_writel(pcie, reg, PIO_ADDR_LS); + advk_writel(pcie, 0, PIO_ADDR_MS); + + /* Calculate the write strobe */ + offset = where & 0x3; + reg = val << (8 * offset); + data_strobe = GENMASK(size - 1, 0) << offset; + + /* Program the data register */ + advk_writel(pcie, reg, PIO_WR_DATA); + + /* Program the data strobe */ + advk_writel(pcie, data_strobe, PIO_WR_DATA_STRB); + + retry_count = 0; + do { + /* Clear PIO DONE ISR and start the transfer */ + advk_writel(pcie, 1, PIO_ISR); + advk_writel(pcie, 1, PIO_START); + + ret = advk_pcie_wait_pio(pcie); + if (ret < 0) + return PCIBIOS_SET_FAILED; + + retry_count += ret; + + ret = advk_pcie_check_pio_status(pcie, false, NULL); + } while (ret == -EAGAIN && retry_count < PIO_RETRY_CNT); + + return ret < 0 ? PCIBIOS_SET_FAILED : PCIBIOS_SUCCESSFUL; +} + +static struct pci_ops advk_pcie_ops = { + .read = advk_pcie_rd_conf, + .write = advk_pcie_wr_conf, +}; + +static void advk_msi_irq_compose_msi_msg(struct irq_data *data, + struct msi_msg *msg) +{ + struct advk_pcie *pcie = irq_data_get_irq_chip_data(data); + phys_addr_t msi_addr = virt_to_phys(pcie); + + msg->address_lo = lower_32_bits(msi_addr); + msg->address_hi = upper_32_bits(msi_addr); + msg->data = data->hwirq; +} + +static void advk_msi_irq_mask(struct irq_data *d) +{ + struct advk_pcie *pcie = d->domain->host_data; + irq_hw_number_t hwirq = irqd_to_hwirq(d); + unsigned long flags; + u32 mask; + + raw_spin_lock_irqsave(&pcie->msi_irq_lock, flags); + mask = advk_readl(pcie, PCIE_MSI_MASK_REG); + mask |= BIT(hwirq); + advk_writel(pcie, mask, PCIE_MSI_MASK_REG); + raw_spin_unlock_irqrestore(&pcie->msi_irq_lock, flags); +} + +static void advk_msi_irq_unmask(struct irq_data *d) +{ + struct advk_pcie *pcie = d->domain->host_data; + irq_hw_number_t hwirq = irqd_to_hwirq(d); + unsigned long flags; + u32 mask; + + raw_spin_lock_irqsave(&pcie->msi_irq_lock, flags); + mask = advk_readl(pcie, PCIE_MSI_MASK_REG); + mask &= ~BIT(hwirq); + advk_writel(pcie, mask, PCIE_MSI_MASK_REG); + raw_spin_unlock_irqrestore(&pcie->msi_irq_lock, flags); +} + +static struct irq_chip advk_msi_bottom_irq_chip = { + .name = "MSI", + .irq_compose_msi_msg = advk_msi_irq_compose_msi_msg, + .irq_mask = advk_msi_irq_mask, + .irq_unmask = advk_msi_irq_unmask, +}; + +static int advk_msi_irq_domain_alloc(struct irq_domain *domain, + unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct advk_pcie *pcie = domain->host_data; + int hwirq, i; + + mutex_lock(&pcie->msi_used_lock); + hwirq = bitmap_find_free_region(pcie->msi_used, MSI_IRQ_NUM, + order_base_2(nr_irqs)); + mutex_unlock(&pcie->msi_used_lock); + if (hwirq < 0) + return -ENOSPC; + + for (i = 0; i < nr_irqs; i++) + irq_domain_set_info(domain, virq + i, hwirq + i, + &advk_msi_bottom_irq_chip, + domain->host_data, handle_simple_irq, + NULL, NULL); + + return 0; +} + +static void advk_msi_irq_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct advk_pcie *pcie = domain->host_data; + + mutex_lock(&pcie->msi_used_lock); + bitmap_release_region(pcie->msi_used, d->hwirq, order_base_2(nr_irqs)); + mutex_unlock(&pcie->msi_used_lock); +} + +static const struct irq_domain_ops advk_msi_domain_ops = { + .alloc = advk_msi_irq_domain_alloc, + .free = advk_msi_irq_domain_free, +}; + +static void advk_pcie_irq_mask(struct irq_data *d) +{ + struct advk_pcie *pcie = d->domain->host_data; + irq_hw_number_t hwirq = irqd_to_hwirq(d); + unsigned long flags; + u32 mask; + + raw_spin_lock_irqsave(&pcie->irq_lock, flags); + mask = advk_readl(pcie, PCIE_ISR1_MASK_REG); + mask |= PCIE_ISR1_INTX_ASSERT(hwirq); + advk_writel(pcie, mask, PCIE_ISR1_MASK_REG); + raw_spin_unlock_irqrestore(&pcie->irq_lock, flags); +} + +static void advk_pcie_irq_unmask(struct irq_data *d) +{ + struct advk_pcie *pcie = d->domain->host_data; + irq_hw_number_t hwirq = irqd_to_hwirq(d); + unsigned long flags; + u32 mask; + + raw_spin_lock_irqsave(&pcie->irq_lock, flags); + mask = advk_readl(pcie, PCIE_ISR1_MASK_REG); + mask &= ~PCIE_ISR1_INTX_ASSERT(hwirq); + advk_writel(pcie, mask, PCIE_ISR1_MASK_REG); + raw_spin_unlock_irqrestore(&pcie->irq_lock, flags); +} + +static int advk_pcie_irq_map(struct irq_domain *h, + unsigned int virq, irq_hw_number_t hwirq) +{ + struct advk_pcie *pcie = h->host_data; + + irq_set_status_flags(virq, IRQ_LEVEL); + irq_set_chip_and_handler(virq, &pcie->irq_chip, + handle_level_irq); + irq_set_chip_data(virq, pcie); + + return 0; +} + +static const struct irq_domain_ops advk_pcie_irq_domain_ops = { + .map = advk_pcie_irq_map, + .xlate = irq_domain_xlate_onecell, +}; + +#define ADVK_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_PCI_MSI_MASK_PARENT | \ + MSI_FLAG_NO_AFFINITY) +#define ADVK_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_PCI_MSIX | \ + MSI_FLAG_MULTI_PCI_MSI) + +static const struct msi_parent_ops advk_msi_parent_ops = { + .required_flags = ADVK_MSI_FLAGS_REQUIRED, + .supported_flags = ADVK_MSI_FLAGS_SUPPORTED, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .prefix = "advk-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +static int advk_pcie_init_msi_irq_domain(struct advk_pcie *pcie) +{ + struct device *dev = &pcie->pdev->dev; + + raw_spin_lock_init(&pcie->msi_irq_lock); + mutex_init(&pcie->msi_used_lock); + + struct irq_domain_info info = { + .fwnode = dev_fwnode(dev), + .ops = &advk_msi_domain_ops, + .host_data = pcie, + .size = MSI_IRQ_NUM, + }; + + pcie->msi_inner_domain = msi_create_parent_irq_domain(&info, &advk_msi_parent_ops); + if (!pcie->msi_inner_domain) + return -ENOMEM; + + return 0; +} + +static void advk_pcie_remove_msi_irq_domain(struct advk_pcie *pcie) +{ + irq_domain_remove(pcie->msi_inner_domain); +} + +static int advk_pcie_init_irq_domain(struct advk_pcie *pcie) +{ + struct device *dev = &pcie->pdev->dev; + struct device_node *node = dev->of_node; + struct device_node *pcie_intc_node; + struct irq_chip *irq_chip; + int ret = 0; + + raw_spin_lock_init(&pcie->irq_lock); + + pcie_intc_node = of_get_next_child(node, NULL); + if (!pcie_intc_node) { + dev_err(dev, "No PCIe Intc node found\n"); + return -ENODEV; + } + + irq_chip = &pcie->irq_chip; + + irq_chip->name = devm_kasprintf(dev, GFP_KERNEL, "%s-irq", + dev_name(dev)); + if (!irq_chip->name) { + ret = -ENOMEM; + goto out_put_node; + } + + irq_chip->irq_mask = advk_pcie_irq_mask; + irq_chip->irq_unmask = advk_pcie_irq_unmask; + + pcie->irq_domain = irq_domain_create_linear(of_fwnode_handle(pcie_intc_node), PCI_NUM_INTX, + &advk_pcie_irq_domain_ops, pcie); + if (!pcie->irq_domain) { + dev_err(dev, "Failed to get a INTx IRQ domain\n"); + ret = -ENOMEM; + goto out_put_node; + } + +out_put_node: + of_node_put(pcie_intc_node); + return ret; +} + +static void advk_pcie_remove_irq_domain(struct advk_pcie *pcie) +{ + irq_domain_remove(pcie->irq_domain); +} + +static struct irq_chip advk_rp_irq_chip = { + .name = "advk-RP", +}; + +static int advk_pcie_rp_irq_map(struct irq_domain *h, + unsigned int virq, irq_hw_number_t hwirq) +{ + struct advk_pcie *pcie = h->host_data; + + irq_set_chip_and_handler(virq, &advk_rp_irq_chip, handle_simple_irq); + irq_set_chip_data(virq, pcie); + + return 0; +} + +static const struct irq_domain_ops advk_pcie_rp_irq_domain_ops = { + .map = advk_pcie_rp_irq_map, + .xlate = irq_domain_xlate_onecell, +}; + +static int advk_pcie_init_rp_irq_domain(struct advk_pcie *pcie) +{ + pcie->rp_irq_domain = irq_domain_create_linear(NULL, 1, &advk_pcie_rp_irq_domain_ops, pcie); + if (!pcie->rp_irq_domain) { + dev_err(&pcie->pdev->dev, "Failed to add Root Port IRQ domain\n"); + return -ENOMEM; + } + + return 0; +} + +static void advk_pcie_remove_rp_irq_domain(struct advk_pcie *pcie) +{ + irq_domain_remove(pcie->rp_irq_domain); +} + +static void advk_pcie_handle_pme(struct advk_pcie *pcie) +{ + u32 requester = advk_readl(pcie, PCIE_MSG_LOG_REG) >> 16; + + advk_writel(pcie, PCIE_MSG_PM_PME_MASK, PCIE_ISR0_REG); + + /* + * PCIE_MSG_LOG_REG contains the last inbound message, so store + * the requester ID only when PME was not asserted yet. + * Also do not trigger PME interrupt when PME is still asserted. + */ + if (!(le32_to_cpu(pcie->bridge.pcie_conf.rootsta) & PCI_EXP_RTSTA_PME)) { + pcie->bridge.pcie_conf.rootsta = cpu_to_le32(requester | PCI_EXP_RTSTA_PME); + + /* + * Trigger PME interrupt only if PMEIE bit in Root Control is set. + * Aardvark HW returns zero for PCI_EXP_FLAGS_IRQ, so use PCIe interrupt 0. + */ + if (!(le16_to_cpu(pcie->bridge.pcie_conf.rootctl) & PCI_EXP_RTCTL_PMEIE)) + return; + + if (generic_handle_domain_irq(pcie->rp_irq_domain, 0) == -EINVAL) + dev_err_ratelimited(&pcie->pdev->dev, "unhandled PME IRQ\n"); + } +} + +static void advk_pcie_handle_msi(struct advk_pcie *pcie) +{ + u32 msi_val, msi_mask, msi_status, msi_idx; + + msi_mask = advk_readl(pcie, PCIE_MSI_MASK_REG); + msi_val = advk_readl(pcie, PCIE_MSI_STATUS_REG); + msi_status = msi_val & ((~msi_mask) & PCIE_MSI_ALL_MASK); + + for (msi_idx = 0; msi_idx < MSI_IRQ_NUM; msi_idx++) { + if (!(BIT(msi_idx) & msi_status)) + continue; + + advk_writel(pcie, BIT(msi_idx), PCIE_MSI_STATUS_REG); + if (generic_handle_domain_irq(pcie->msi_inner_domain, msi_idx) == -EINVAL) + dev_err_ratelimited(&pcie->pdev->dev, "unexpected MSI 0x%02x\n", msi_idx); + } + + advk_writel(pcie, PCIE_ISR0_MSI_INT_PENDING, + PCIE_ISR0_REG); +} + +static void advk_pcie_handle_int(struct advk_pcie *pcie) +{ + u32 isr0_val, isr0_mask, isr0_status; + u32 isr1_val, isr1_mask, isr1_status; + int i; + + isr0_val = advk_readl(pcie, PCIE_ISR0_REG); + isr0_mask = advk_readl(pcie, PCIE_ISR0_MASK_REG); + isr0_status = isr0_val & ((~isr0_mask) & PCIE_ISR0_ALL_MASK); + + isr1_val = advk_readl(pcie, PCIE_ISR1_REG); + isr1_mask = advk_readl(pcie, PCIE_ISR1_MASK_REG); + isr1_status = isr1_val & ((~isr1_mask) & PCIE_ISR1_ALL_MASK); + + /* Process PME interrupt as the first one to do not miss PME requester id */ + if (isr0_status & PCIE_MSG_PM_PME_MASK) + advk_pcie_handle_pme(pcie); + + /* Process ERR interrupt */ + if (isr0_status & PCIE_ISR0_ERR_MASK) { + advk_writel(pcie, PCIE_ISR0_ERR_MASK, PCIE_ISR0_REG); + + /* + * Aardvark HW returns zero for PCI_ERR_ROOT_AER_IRQ, so use + * PCIe interrupt 0 + */ + if (generic_handle_domain_irq(pcie->rp_irq_domain, 0) == -EINVAL) + dev_err_ratelimited(&pcie->pdev->dev, "unhandled ERR IRQ\n"); + } + + /* Process MSI interrupts */ + if (isr0_status & PCIE_ISR0_MSI_INT_PENDING) + advk_pcie_handle_msi(pcie); + + /* Process legacy interrupts */ + for (i = 0; i < PCI_NUM_INTX; i++) { + if (!(isr1_status & PCIE_ISR1_INTX_ASSERT(i))) + continue; + + advk_writel(pcie, PCIE_ISR1_INTX_ASSERT(i), + PCIE_ISR1_REG); + + if (generic_handle_domain_irq(pcie->irq_domain, i) == -EINVAL) + dev_err_ratelimited(&pcie->pdev->dev, "unexpected INT%c IRQ\n", + (char)i + 'A'); + } +} + +static irqreturn_t advk_pcie_irq_handler(int irq, void *arg) +{ + struct advk_pcie *pcie = arg; + u32 status; + + status = advk_readl(pcie, HOST_CTRL_INT_STATUS_REG); + if (!(status & PCIE_IRQ_CORE_INT)) + return IRQ_NONE; + + advk_pcie_handle_int(pcie); + + /* Clear interrupt */ + advk_writel(pcie, PCIE_IRQ_CORE_INT, HOST_CTRL_INT_STATUS_REG); + + return IRQ_HANDLED; +} + +static int advk_pcie_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) +{ + struct advk_pcie *pcie = dev->bus->sysdata; + + /* + * Emulated root bridge has its own emulated irq chip and irq domain. + * Argument pin is the INTx pin (1=INTA, 2=INTB, 3=INTC, 4=INTD) and + * hwirq for irq_create_mapping() is indexed from zero. + */ + if (pci_is_root_bus(dev->bus)) + return irq_create_mapping(pcie->rp_irq_domain, pin - 1); + else + return of_irq_parse_and_map_pci(dev, slot, pin); +} + +static void advk_pcie_disable_phy(struct advk_pcie *pcie) +{ + phy_power_off(pcie->phy); + phy_exit(pcie->phy); +} + +static int advk_pcie_enable_phy(struct advk_pcie *pcie) +{ + int ret; + + if (!pcie->phy) + return 0; + + ret = phy_init(pcie->phy); + if (ret) + return ret; + + ret = phy_set_mode(pcie->phy, PHY_MODE_PCIE); + if (ret) { + phy_exit(pcie->phy); + return ret; + } + + ret = phy_power_on(pcie->phy); + if (ret) { + phy_exit(pcie->phy); + return ret; + } + + return 0; +} + +static int advk_pcie_setup_phy(struct advk_pcie *pcie) +{ + struct device *dev = &pcie->pdev->dev; + struct device_node *node = dev->of_node; + int ret = 0; + + pcie->phy = devm_of_phy_get(dev, node, NULL); + if (IS_ERR(pcie->phy) && (PTR_ERR(pcie->phy) == -EPROBE_DEFER)) + return PTR_ERR(pcie->phy); + + /* Old bindings miss the PHY handle */ + if (IS_ERR(pcie->phy)) { + dev_warn(dev, "PHY unavailable (%pe)\n", pcie->phy); + pcie->phy = NULL; + return 0; + } + + ret = advk_pcie_enable_phy(pcie); + if (ret) + dev_err(dev, "Failed to initialize PHY (%d)\n", ret); + + return ret; +} + +static int advk_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct advk_pcie *pcie; + struct pci_host_bridge *bridge; + struct resource_entry *entry; + int ret, irq; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(struct advk_pcie)); + if (!bridge) + return -ENOMEM; + + pcie = pci_host_bridge_priv(bridge); + pcie->pdev = pdev; + platform_set_drvdata(pdev, pcie); + + resource_list_for_each_entry(entry, &bridge->windows) { + resource_size_t start = entry->res->start; + resource_size_t size = resource_size(entry->res); + unsigned long type = resource_type(entry->res); + u64 win_size; + + /* + * Aardvark hardware allows to configure also PCIe window + * for config type 0 and type 1 mapping, but driver uses + * only PIO for issuing configuration transfers which does + * not use PCIe window configuration. + */ + if (type != IORESOURCE_MEM && type != IORESOURCE_IO) + continue; + + /* + * Skip transparent memory resources. Default outbound access + * configuration is set to transparent memory access so it + * does not need window configuration. + */ + if (type == IORESOURCE_MEM && entry->offset == 0) + continue; + + /* + * The n-th PCIe window is configured by tuple (match, remap, mask) + * and an access to address A uses this window if A matches the + * match with given mask. + * So every PCIe window size must be a power of two and every start + * address must be aligned to window size. Minimal size is 64 KiB + * because lower 16 bits of mask must be zero. Remapped address + * may have set only bits from the mask. + */ + while (pcie->wins_count < OB_WIN_COUNT && size > 0) { + /* Calculate the largest aligned window size */ + win_size = (1ULL << (fls64(size)-1)) | + (start ? (1ULL << __ffs64(start)) : 0); + win_size = 1ULL << __ffs64(win_size); + if (win_size < 0x10000) + break; + + dev_dbg(dev, + "Configuring PCIe window %d: [0x%llx-0x%llx] as %lu\n", + pcie->wins_count, (unsigned long long)start, + (unsigned long long)start + win_size, type); + + if (type == IORESOURCE_IO) { + pcie->wins[pcie->wins_count].actions = OB_WIN_TYPE_IO; + pcie->wins[pcie->wins_count].match = pci_pio_to_address(start); + } else { + pcie->wins[pcie->wins_count].actions = OB_WIN_TYPE_MEM; + pcie->wins[pcie->wins_count].match = start; + } + pcie->wins[pcie->wins_count].remap = start - entry->offset; + pcie->wins[pcie->wins_count].mask = ~(win_size - 1); + + if (pcie->wins[pcie->wins_count].remap & (win_size - 1)) + break; + + start += win_size; + size -= win_size; + pcie->wins_count++; + } + + if (size > 0) { + dev_err(&pcie->pdev->dev, + "Invalid PCIe region [0x%llx-0x%llx]\n", + (unsigned long long)entry->res->start, + (unsigned long long)entry->res->end + 1); + return -EINVAL; + } + } + + pcie->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(pcie->base)) + return PTR_ERR(pcie->base); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + ret = devm_request_irq(dev, irq, advk_pcie_irq_handler, + IRQF_SHARED | IRQF_NO_THREAD, "advk-pcie", + pcie); + if (ret) { + dev_err(dev, "Failed to register interrupt\n"); + return ret; + } + + pcie->reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW); + ret = PTR_ERR_OR_ZERO(pcie->reset_gpio); + if (ret) { + if (ret != -EPROBE_DEFER) + dev_err(dev, "Failed to get reset-gpio: %i\n", ret); + return ret; + } + + ret = gpiod_set_consumer_name(pcie->reset_gpio, "pcie1-reset"); + if (ret) { + dev_err(dev, "Failed to set reset gpio name: %d\n", ret); + return ret; + } + + ret = of_pci_get_max_link_speed(dev->of_node); + if (ret <= 0 || ret > 3) + pcie->link_gen = 3; + else + pcie->link_gen = ret; + + ret = advk_pcie_setup_phy(pcie); + if (ret) + return ret; + + advk_pcie_setup_hw(pcie); + + ret = advk_sw_pci_bridge_init(pcie); + if (ret) { + dev_err(dev, "Failed to register emulated root PCI bridge\n"); + return ret; + } + + ret = advk_pcie_init_irq_domain(pcie); + if (ret) { + dev_err(dev, "Failed to initialize irq\n"); + return ret; + } + + ret = advk_pcie_init_msi_irq_domain(pcie); + if (ret) { + dev_err(dev, "Failed to initialize irq\n"); + advk_pcie_remove_irq_domain(pcie); + return ret; + } + + ret = advk_pcie_init_rp_irq_domain(pcie); + if (ret) { + dev_err(dev, "Failed to initialize irq\n"); + advk_pcie_remove_msi_irq_domain(pcie); + advk_pcie_remove_irq_domain(pcie); + return ret; + } + + bridge->sysdata = pcie; + bridge->ops = &advk_pcie_ops; + bridge->map_irq = advk_pcie_map_irq; + + ret = pci_host_probe(bridge); + if (ret < 0) { + advk_pcie_remove_rp_irq_domain(pcie); + advk_pcie_remove_msi_irq_domain(pcie); + advk_pcie_remove_irq_domain(pcie); + return ret; + } + + return 0; +} + +static void advk_pcie_remove(struct platform_device *pdev) +{ + struct advk_pcie *pcie = platform_get_drvdata(pdev); + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + u32 val; + int i; + + /* Remove PCI bus with all devices */ + pci_lock_rescan_remove(); + pci_stop_root_bus(bridge->bus); + pci_remove_root_bus(bridge->bus); + pci_unlock_rescan_remove(); + + /* Disable Root Bridge I/O space, memory space and bus mastering */ + val = advk_readl(pcie, PCIE_CORE_CMD_STATUS_REG); + val &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + advk_writel(pcie, val, PCIE_CORE_CMD_STATUS_REG); + + /* Disable MSI */ + val = advk_readl(pcie, PCIE_CORE_CTRL2_REG); + val &= ~PCIE_CORE_CTRL2_MSI_ENABLE; + advk_writel(pcie, val, PCIE_CORE_CTRL2_REG); + + /* Clear MSI address */ + advk_writel(pcie, 0, PCIE_MSI_ADDR_LOW_REG); + advk_writel(pcie, 0, PCIE_MSI_ADDR_HIGH_REG); + + /* Mask all interrupts */ + advk_writel(pcie, PCIE_MSI_ALL_MASK, PCIE_MSI_MASK_REG); + advk_writel(pcie, PCIE_ISR0_ALL_MASK, PCIE_ISR0_MASK_REG); + advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_MASK_REG); + advk_writel(pcie, PCIE_IRQ_ALL_MASK, HOST_CTRL_INT_MASK_REG); + + /* Clear all interrupts */ + advk_writel(pcie, PCIE_MSI_ALL_MASK, PCIE_MSI_STATUS_REG); + advk_writel(pcie, PCIE_ISR0_ALL_MASK, PCIE_ISR0_REG); + advk_writel(pcie, PCIE_ISR1_ALL_MASK, PCIE_ISR1_REG); + advk_writel(pcie, PCIE_IRQ_ALL_MASK, HOST_CTRL_INT_STATUS_REG); + + /* Remove IRQ domains */ + advk_pcie_remove_rp_irq_domain(pcie); + advk_pcie_remove_msi_irq_domain(pcie); + advk_pcie_remove_irq_domain(pcie); + + /* Free config space for emulated root bridge */ + pci_bridge_emul_cleanup(&pcie->bridge); + + /* Assert PERST# signal which prepares PCIe card for power down */ + if (pcie->reset_gpio) + gpiod_set_value_cansleep(pcie->reset_gpio, 1); + + /* Disable link training */ + val = advk_readl(pcie, PCIE_CORE_CTRL0_REG); + val &= ~LINK_TRAINING_EN; + advk_writel(pcie, val, PCIE_CORE_CTRL0_REG); + + /* Disable outbound address windows mapping */ + for (i = 0; i < OB_WIN_COUNT; i++) + advk_pcie_disable_ob_win(pcie, i); + + /* Disable phy */ + advk_pcie_disable_phy(pcie); +} + +static const struct of_device_id advk_pcie_of_match_table[] = { + { .compatible = "marvell,armada-3700-pcie", }, + {}, +}; +MODULE_DEVICE_TABLE(of, advk_pcie_of_match_table); + +static struct platform_driver advk_pcie_driver = { + .driver = { + .name = "advk-pcie", + .of_match_table = advk_pcie_of_match_table, + }, + .probe = advk_pcie_probe, + .remove = advk_pcie_remove, +}; +module_platform_driver(advk_pcie_driver); + +MODULE_DESCRIPTION("Aardvark PCIe controller"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pci-ftpci100.c b/drivers/pci/controller/pci-ftpci100.c new file mode 100644 index 000000000..28e43831c --- /dev/null +++ b/drivers/pci/controller/pci-ftpci100.c @@ -0,0 +1,550 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Support for Faraday Technology FTPC100 PCI Controller + * + * Copyright (C) 2017 Linus Walleij <linus.walleij@linaro.org> + * + * Based on the out-of-tree OpenWRT patch for Cortina Gemini: + * Copyright (C) 2009 Janos Laube <janos.dev@gmail.com> + * Copyright (C) 2009 Paulius Zaleckas <paulius.zaleckas@teltonika.lt> + * Based on SL2312 PCI controller code + * Storlink (C) 2003 + */ + +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/of.h> +#include <linux/of_irq.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/irqdomain.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/bitops.h> +#include <linux/irq.h> +#include <linux/clk.h> + +#include "../pci.h" + +/* + * Special configuration registers directly in the first few words + * in I/O space. + */ +#define FTPCI_IOSIZE 0x00 +#define FTPCI_PROT 0x04 /* AHB protection */ +#define FTPCI_CTRL 0x08 /* PCI control signal */ +#define FTPCI_SOFTRST 0x10 /* Soft reset counter and response error enable */ +#define FTPCI_CONFIG 0x28 /* PCI configuration command register */ +#define FTPCI_DATA 0x2C + +#define FARADAY_PCI_STATUS_CMD 0x04 /* Status and command */ +#define FARADAY_PCI_PMC 0x40 /* Power management control */ +#define FARADAY_PCI_PMCSR 0x44 /* Power management status */ +#define FARADAY_PCI_CTRL1 0x48 /* Control register 1 */ +#define FARADAY_PCI_CTRL2 0x4C /* Control register 2 */ +#define FARADAY_PCI_MEM1_BASE_SIZE 0x50 /* Memory base and size #1 */ +#define FARADAY_PCI_MEM2_BASE_SIZE 0x54 /* Memory base and size #2 */ +#define FARADAY_PCI_MEM3_BASE_SIZE 0x58 /* Memory base and size #3 */ + +#define PCI_STATUS_66MHZ_CAPABLE BIT(21) + +/* Bits 31..28 gives INTD..INTA status */ +#define PCI_CTRL2_INTSTS_SHIFT 28 +#define PCI_CTRL2_INTMASK_CMDERR BIT(27) +#define PCI_CTRL2_INTMASK_PARERR BIT(26) +/* Bits 25..22 masks INTD..INTA */ +#define PCI_CTRL2_INTMASK_SHIFT 22 +#define PCI_CTRL2_INTMASK_MABRT_RX BIT(21) +#define PCI_CTRL2_INTMASK_TABRT_RX BIT(20) +#define PCI_CTRL2_INTMASK_TABRT_TX BIT(19) +#define PCI_CTRL2_INTMASK_RETRY4 BIT(18) +#define PCI_CTRL2_INTMASK_SERR_RX BIT(17) +#define PCI_CTRL2_INTMASK_PERR_RX BIT(16) +/* Bit 15 reserved */ +#define PCI_CTRL2_MSTPRI_REQ6 BIT(14) +#define PCI_CTRL2_MSTPRI_REQ5 BIT(13) +#define PCI_CTRL2_MSTPRI_REQ4 BIT(12) +#define PCI_CTRL2_MSTPRI_REQ3 BIT(11) +#define PCI_CTRL2_MSTPRI_REQ2 BIT(10) +#define PCI_CTRL2_MSTPRI_REQ1 BIT(9) +#define PCI_CTRL2_MSTPRI_REQ0 BIT(8) +/* Bits 7..4 reserved */ +/* Bits 3..0 TRDYW */ + +/* + * Memory configs: + * Bit 31..20 defines the PCI side memory base + * Bit 19..16 (4 bits) defines the size per below + */ +#define FARADAY_PCI_MEMBASE_MASK 0xfff00000 +#define FARADAY_PCI_MEMSIZE_1MB 0x0 +#define FARADAY_PCI_MEMSIZE_2MB 0x1 +#define FARADAY_PCI_MEMSIZE_4MB 0x2 +#define FARADAY_PCI_MEMSIZE_8MB 0x3 +#define FARADAY_PCI_MEMSIZE_16MB 0x4 +#define FARADAY_PCI_MEMSIZE_32MB 0x5 +#define FARADAY_PCI_MEMSIZE_64MB 0x6 +#define FARADAY_PCI_MEMSIZE_128MB 0x7 +#define FARADAY_PCI_MEMSIZE_256MB 0x8 +#define FARADAY_PCI_MEMSIZE_512MB 0x9 +#define FARADAY_PCI_MEMSIZE_1GB 0xa +#define FARADAY_PCI_MEMSIZE_2GB 0xb +#define FARADAY_PCI_MEMSIZE_SHIFT 16 + +/* + * The DMA base is set to 0x0 for all memory segments, it reflects the + * fact that the memory of the host system starts at 0x0. + */ +#define FARADAY_PCI_DMA_MEM1_BASE 0x00000000 +#define FARADAY_PCI_DMA_MEM2_BASE 0x00000000 +#define FARADAY_PCI_DMA_MEM3_BASE 0x00000000 + +/** + * struct faraday_pci_variant - encodes IP block differences + * @cascaded_irq: this host has cascaded IRQs from an interrupt controller + * embedded in the host bridge. + */ +struct faraday_pci_variant { + bool cascaded_irq; +}; + +struct faraday_pci { + struct device *dev; + void __iomem *base; + struct irq_domain *irqdomain; + struct pci_bus *bus; + struct clk *bus_clk; +}; + +static int faraday_res_to_memcfg(resource_size_t mem_base, + resource_size_t mem_size, u32 *val) +{ + u32 outval; + + switch (mem_size) { + case SZ_1M: + outval = FARADAY_PCI_MEMSIZE_1MB; + break; + case SZ_2M: + outval = FARADAY_PCI_MEMSIZE_2MB; + break; + case SZ_4M: + outval = FARADAY_PCI_MEMSIZE_4MB; + break; + case SZ_8M: + outval = FARADAY_PCI_MEMSIZE_8MB; + break; + case SZ_16M: + outval = FARADAY_PCI_MEMSIZE_16MB; + break; + case SZ_32M: + outval = FARADAY_PCI_MEMSIZE_32MB; + break; + case SZ_64M: + outval = FARADAY_PCI_MEMSIZE_64MB; + break; + case SZ_128M: + outval = FARADAY_PCI_MEMSIZE_128MB; + break; + case SZ_256M: + outval = FARADAY_PCI_MEMSIZE_256MB; + break; + case SZ_512M: + outval = FARADAY_PCI_MEMSIZE_512MB; + break; + case SZ_1G: + outval = FARADAY_PCI_MEMSIZE_1GB; + break; + case SZ_2G: + outval = FARADAY_PCI_MEMSIZE_2GB; + break; + default: + return -EINVAL; + } + outval <<= FARADAY_PCI_MEMSIZE_SHIFT; + + /* This is probably not good */ + if (mem_base & ~(FARADAY_PCI_MEMBASE_MASK)) + pr_warn("truncated PCI memory base\n"); + /* Translate to bridge side address space */ + outval |= (mem_base & FARADAY_PCI_MEMBASE_MASK); + pr_debug("Translated pci base @%pap, size %pap to config %08x\n", + &mem_base, &mem_size, outval); + + *val = outval; + return 0; +} + +static int faraday_raw_pci_read_config(struct faraday_pci *p, int bus_number, + unsigned int fn, int config, int size, + u32 *value) +{ + writel(PCI_CONF1_ADDRESS(bus_number, PCI_SLOT(fn), + PCI_FUNC(fn), config), + p->base + FTPCI_CONFIG); + + *value = readl(p->base + FTPCI_DATA); + + if (size == 1) + *value = (*value >> (8 * (config & 3))) & 0xFF; + else if (size == 2) + *value = (*value >> (8 * (config & 3))) & 0xFFFF; + + return PCIBIOS_SUCCESSFUL; +} + +static int faraday_pci_read_config(struct pci_bus *bus, unsigned int fn, + int config, int size, u32 *value) +{ + struct faraday_pci *p = bus->sysdata; + + dev_dbg(&bus->dev, + "[read] slt: %.2d, fnc: %d, cnf: 0x%.2X, val (%d bytes): 0x%.8X\n", + PCI_SLOT(fn), PCI_FUNC(fn), config, size, *value); + + return faraday_raw_pci_read_config(p, bus->number, fn, config, size, value); +} + +static int faraday_raw_pci_write_config(struct faraday_pci *p, int bus_number, + unsigned int fn, int config, int size, + u32 value) +{ + int ret = PCIBIOS_SUCCESSFUL; + + writel(PCI_CONF1_ADDRESS(bus_number, PCI_SLOT(fn), + PCI_FUNC(fn), config), + p->base + FTPCI_CONFIG); + + switch (size) { + case 4: + writel(value, p->base + FTPCI_DATA); + break; + case 2: + writew(value, p->base + FTPCI_DATA + (config & 3)); + break; + case 1: + writeb(value, p->base + FTPCI_DATA + (config & 3)); + break; + default: + ret = PCIBIOS_BAD_REGISTER_NUMBER; + } + + return ret; +} + +static int faraday_pci_write_config(struct pci_bus *bus, unsigned int fn, + int config, int size, u32 value) +{ + struct faraday_pci *p = bus->sysdata; + + dev_dbg(&bus->dev, + "[write] slt: %.2d, fnc: %d, cnf: 0x%.2X, val (%d bytes): 0x%.8X\n", + PCI_SLOT(fn), PCI_FUNC(fn), config, size, value); + + return faraday_raw_pci_write_config(p, bus->number, fn, config, size, + value); +} + +static struct pci_ops faraday_pci_ops = { + .read = faraday_pci_read_config, + .write = faraday_pci_write_config, +}; + +static void faraday_pci_ack_irq(struct irq_data *d) +{ + struct faraday_pci *p = irq_data_get_irq_chip_data(d); + unsigned int reg; + + faraday_raw_pci_read_config(p, 0, 0, FARADAY_PCI_CTRL2, 4, ®); + reg &= ~(0xF << PCI_CTRL2_INTSTS_SHIFT); + reg |= BIT(irqd_to_hwirq(d) + PCI_CTRL2_INTSTS_SHIFT); + faraday_raw_pci_write_config(p, 0, 0, FARADAY_PCI_CTRL2, 4, reg); +} + +static void faraday_pci_mask_irq(struct irq_data *d) +{ + struct faraday_pci *p = irq_data_get_irq_chip_data(d); + unsigned int reg; + + faraday_raw_pci_read_config(p, 0, 0, FARADAY_PCI_CTRL2, 4, ®); + reg &= ~((0xF << PCI_CTRL2_INTSTS_SHIFT) + | BIT(irqd_to_hwirq(d) + PCI_CTRL2_INTMASK_SHIFT)); + faraday_raw_pci_write_config(p, 0, 0, FARADAY_PCI_CTRL2, 4, reg); +} + +static void faraday_pci_unmask_irq(struct irq_data *d) +{ + struct faraday_pci *p = irq_data_get_irq_chip_data(d); + unsigned int reg; + + faraday_raw_pci_read_config(p, 0, 0, FARADAY_PCI_CTRL2, 4, ®); + reg &= ~(0xF << PCI_CTRL2_INTSTS_SHIFT); + reg |= BIT(irqd_to_hwirq(d) + PCI_CTRL2_INTMASK_SHIFT); + faraday_raw_pci_write_config(p, 0, 0, FARADAY_PCI_CTRL2, 4, reg); +} + +static void faraday_pci_irq_handler(struct irq_desc *desc) +{ + struct faraday_pci *p = irq_desc_get_handler_data(desc); + struct irq_chip *irqchip = irq_desc_get_chip(desc); + unsigned int irq_stat, reg, i; + + faraday_raw_pci_read_config(p, 0, 0, FARADAY_PCI_CTRL2, 4, ®); + irq_stat = reg >> PCI_CTRL2_INTSTS_SHIFT; + + chained_irq_enter(irqchip, desc); + + for (i = 0; i < 4; i++) { + if ((irq_stat & BIT(i)) == 0) + continue; + generic_handle_domain_irq(p->irqdomain, i); + } + + chained_irq_exit(irqchip, desc); +} + +static struct irq_chip faraday_pci_irq_chip = { + .name = "PCI", + .irq_ack = faraday_pci_ack_irq, + .irq_mask = faraday_pci_mask_irq, + .irq_unmask = faraday_pci_unmask_irq, +}; + +static int faraday_pci_irq_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &faraday_pci_irq_chip, handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + + return 0; +} + +static const struct irq_domain_ops faraday_pci_irqdomain_ops = { + .map = faraday_pci_irq_map, +}; + +static int faraday_pci_setup_cascaded_irq(struct faraday_pci *p) +{ + struct device_node *intc = of_get_next_child(p->dev->of_node, NULL); + int irq; + int i; + + if (!intc) { + dev_err(p->dev, "missing child interrupt-controller node\n"); + return -EINVAL; + } + + /* All PCI IRQs cascade off this one */ + irq = of_irq_get(intc, 0); + if (irq <= 0) { + dev_err(p->dev, "failed to get parent IRQ\n"); + of_node_put(intc); + return irq ?: -EINVAL; + } + + p->irqdomain = irq_domain_create_linear(of_fwnode_handle(intc), PCI_NUM_INTX, + &faraday_pci_irqdomain_ops, p); + of_node_put(intc); + if (!p->irqdomain) { + dev_err(p->dev, "failed to create Gemini PCI IRQ domain\n"); + return -EINVAL; + } + + irq_set_chained_handler_and_data(irq, faraday_pci_irq_handler, p); + + for (i = 0; i < 4; i++) + irq_create_mapping(p->irqdomain, i); + + return 0; +} + +static int faraday_pci_parse_map_dma_ranges(struct faraday_pci *p) +{ + struct device *dev = p->dev; + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(p); + struct resource_entry *entry; + u32 confreg[3] = { + FARADAY_PCI_MEM1_BASE_SIZE, + FARADAY_PCI_MEM2_BASE_SIZE, + FARADAY_PCI_MEM3_BASE_SIZE, + }; + int i = 0; + u32 val; + + resource_list_for_each_entry(entry, &bridge->dma_ranges) { + u64 pci_addr = entry->res->start - entry->offset; + u64 end = entry->res->end - entry->offset; + int ret; + + ret = faraday_res_to_memcfg(pci_addr, + resource_size(entry->res), &val); + if (ret) { + dev_err(dev, + "DMA range %d: illegal MEM resource size\n", i); + return -EINVAL; + } + + dev_info(dev, "DMA MEM%d BASE: 0x%016llx -> 0x%016llx config %08x\n", + i + 1, pci_addr, end, val); + if (i <= 2) { + faraday_raw_pci_write_config(p, 0, 0, confreg[i], + 4, val); + } else { + dev_err(dev, "ignore extraneous dma-range %d\n", i); + break; + } + + i++; + } + + return 0; +} + +static int faraday_pci_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + const struct faraday_pci_variant *variant = + of_device_get_match_data(dev); + struct resource_entry *win; + struct faraday_pci *p; + struct resource *io; + struct pci_host_bridge *host; + struct clk *clk; + unsigned char max_bus_speed = PCI_SPEED_33MHz; + unsigned char cur_bus_speed = PCI_SPEED_33MHz; + int ret; + u32 val; + + host = devm_pci_alloc_host_bridge(dev, sizeof(*p)); + if (!host) + return -ENOMEM; + + host->ops = &faraday_pci_ops; + p = pci_host_bridge_priv(host); + host->sysdata = p; + p->dev = dev; + + /* Retrieve and enable optional clocks */ + clk = devm_clk_get_enabled(dev, "PCLK"); + if (IS_ERR(clk)) + return PTR_ERR(clk); + p->bus_clk = devm_clk_get_enabled(dev, "PCICLK"); + if (IS_ERR(p->bus_clk)) + return PTR_ERR(p->bus_clk); + + p->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(p->base)) + return PTR_ERR(p->base); + + win = resource_list_first_type(&host->windows, IORESOURCE_IO); + if (win) { + io = win->res; + if (!faraday_res_to_memcfg(io->start - win->offset, + resource_size(io), &val)) { + /* setup I/O space size */ + writel(val, p->base + FTPCI_IOSIZE); + } else { + dev_err(dev, "illegal IO mem size\n"); + return -EINVAL; + } + } + + /* Setup hostbridge */ + val = readl(p->base + FTPCI_CTRL); + val |= PCI_COMMAND_IO; + val |= PCI_COMMAND_MEMORY; + val |= PCI_COMMAND_MASTER; + writel(val, p->base + FTPCI_CTRL); + /* Mask and clear all interrupts */ + faraday_raw_pci_write_config(p, 0, 0, FARADAY_PCI_CTRL2 + 2, 2, 0xF000); + if (variant->cascaded_irq) { + ret = faraday_pci_setup_cascaded_irq(p); + if (ret) { + dev_err(dev, "failed to setup cascaded IRQ\n"); + return ret; + } + } + + /* Check bus clock if we can gear up to 66 MHz */ + if (!IS_ERR(p->bus_clk)) { + unsigned long rate; + u32 val; + + faraday_raw_pci_read_config(p, 0, 0, + FARADAY_PCI_STATUS_CMD, 4, &val); + rate = clk_get_rate(p->bus_clk); + + if ((rate == 33000000) && (val & PCI_STATUS_66MHZ_CAPABLE)) { + dev_info(dev, "33MHz bus is 66MHz capable\n"); + max_bus_speed = PCI_SPEED_66MHz; + ret = clk_set_rate(p->bus_clk, 66000000); + if (ret) + dev_err(dev, "failed to set bus clock\n"); + } else { + dev_info(dev, "33MHz only bus\n"); + max_bus_speed = PCI_SPEED_33MHz; + } + + /* Bumping the clock may fail so read back the rate */ + rate = clk_get_rate(p->bus_clk); + if (rate == 33000000) + cur_bus_speed = PCI_SPEED_33MHz; + if (rate == 66000000) + cur_bus_speed = PCI_SPEED_66MHz; + } + + ret = faraday_pci_parse_map_dma_ranges(p); + if (ret) + return ret; + + ret = pci_scan_root_bus_bridge(host); + if (ret) { + dev_err(dev, "failed to scan host: %d\n", ret); + return ret; + } + p->bus = host->bus; + p->bus->max_bus_speed = max_bus_speed; + p->bus->cur_bus_speed = cur_bus_speed; + + pci_bus_assign_resources(p->bus); + pci_bus_add_devices(p->bus); + + return 0; +} + +/* + * We encode bridge variants here, we have at least two so it doesn't + * hurt to have infrastructure to encompass future variants as well. + */ +static const struct faraday_pci_variant faraday_regular = { + .cascaded_irq = true, +}; + +static const struct faraday_pci_variant faraday_dual = { + .cascaded_irq = false, +}; + +static const struct of_device_id faraday_pci_of_match[] = { + { + .compatible = "faraday,ftpci100", + .data = &faraday_regular, + }, + { + .compatible = "faraday,ftpci100-dual", + .data = &faraday_dual, + }, + {}, +}; + +static struct platform_driver faraday_pci_driver = { + .driver = { + .name = "ftpci100", + .of_match_table = faraday_pci_of_match, + .suppress_bind_attrs = true, + }, + .probe = faraday_pci_probe, +}; +builtin_platform_driver(faraday_pci_driver); diff --git a/drivers/pci/controller/pci-host-common.c b/drivers/pci/controller/pci-host-common.c new file mode 100644 index 000000000..a23907a87 --- /dev/null +++ b/drivers/pci/controller/pci-host-common.c @@ -0,0 +1,362 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Common library for PCI host controller drivers + * + * Copyright (C) 2014 ARM Limited + * + * Author: Will Deacon <will.deacon@arm.com> + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/gpio/consumer.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/pci-ecam.h> +#include <linux/platform_device.h> + +#include "../pci.h" +#include "pci-host-common.h" + +/** + * pci_host_common_delete_ports - Cleanup function for port list + * @data: Pointer to the port list head + */ +void pci_host_common_delete_ports(void *data) +{ + struct list_head *ports = data; + struct pci_host_perst *perst, *tmp_perst; + struct pci_host_port *port, *tmp_port; + + list_for_each_entry_safe(port, tmp_port, ports, list) { + list_for_each_entry_safe(perst, tmp_perst, &port->perst, list) + list_del(&perst->list); + list_del(&port->list); + } +} +EXPORT_SYMBOL_GPL(pci_host_common_delete_ports); + +/** + * pci_host_common_parse_perst - Parse PERST# from all nodes, depth first + * @dev: Device pointer + * @port: PCI host port + * @np: Device tree node to start parsing from + * + * Recursively parse PERST# GPIO from all PCIe bridge nodes starting from + * @np in a depth-first manner. + * + * Return: 0 on success, negative error code on failure. + */ +static int pci_host_common_parse_perst(struct device *dev, + struct pci_host_port *port, + struct device_node *np) +{ + struct pci_host_perst *perst; + struct gpio_desc *reset; + int ret; + + if (!of_property_present(np, "reset-gpios")) + goto parse_child_node; + + reset = devm_fwnode_gpiod_get(dev, of_fwnode_handle(np), "reset", + GPIOD_ASIS, "PERST#"); + if (IS_ERR(reset)) { + /* + * FIXME: GPIOLIB currently supports exclusive GPIO access only. + * Non exclusive access is broken. But shared PERST# requires + * non-exclusive access. So once GPIOLIB properly supports it, + * implement it here. + */ + if (PTR_ERR(reset) == -EBUSY) + dev_err(dev, "Shared PERST# is not supported\n"); + + return PTR_ERR(reset); + } + + perst = devm_kzalloc(dev, sizeof(*perst), GFP_KERNEL); + if (!perst) + return -ENOMEM; + + INIT_LIST_HEAD(&perst->list); + perst->desc = reset; + list_add_tail(&perst->list, &port->perst); + +parse_child_node: + for_each_available_child_of_node_scoped(np, child) { + ret = pci_host_common_parse_perst(dev, port, child); + if (ret) + return ret; + } + + return 0; +} + +/** + * pci_host_common_parse_port - Parse a single Root Port node + * @dev: Device pointer + * @bridge: PCI host bridge + * @node: Device tree node of the Root Port + * + * Parse Root Port properties from the device tree. Currently it only + * handles the PERST# GPIO (including PERST# GPIOs from all PCIe bridge + * nodes under this Root Port), which is optional. + * + * NOTE: This helper fetches resources (like PERST# GPIO) optionally. If a + * controller driver has a hard dependency on certain resources (PHY, + * clocks, regulators, etc.), those resources MUST be modeled correctly in + * the DT binding and validated in DTS. This helper cannot enforce such + * dependencies and the driver may fail to operate if required resources + * are missing. + * + * Return: 0 on success, negative error codes on failure. + */ +static int pci_host_common_parse_port(struct device *dev, + struct pci_host_bridge *bridge, + struct device_node *node) +{ + struct pci_host_port *port; + int ret; + + port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); + if (!port) + return -ENOMEM; + + INIT_LIST_HEAD(&port->perst); + + ret = pci_host_common_parse_perst(dev, port, node); + if (ret) + return ret; + + INIT_LIST_HEAD(&port->list); + list_add_tail(&port->list, &bridge->ports); + + return 0; +} + +/** + * pci_host_common_parse_ports - Parse Root Port nodes from device tree + * @dev: Device pointer + * @bridge: PCI host bridge + * + * Iterate through child nodes of the host bridge and parse Root Port + * properties (currently only reset GPIOs). + * + * Return: 0 on success or ports not found, negative error codes on failure. + */ +int pci_host_common_parse_ports(struct device *dev, struct pci_host_bridge *bridge) +{ + int ret = 0; + + for_each_available_child_of_node_scoped(dev->of_node, of_port) { + if (!of_node_is_type(of_port, "pci")) + continue; + ret = pci_host_common_parse_port(dev, bridge, of_port); + if (ret) + goto err_cleanup; + } + + if (list_empty(&bridge->ports)) + return 0; + + return devm_add_action_or_reset(dev, pci_host_common_delete_ports, + &bridge->ports); + +err_cleanup: + pci_host_common_delete_ports(&bridge->ports); + return ret; +} +EXPORT_SYMBOL_GPL(pci_host_common_parse_ports); + +#define PCI_HOST_D3COLD_ALLOWED BIT(0) +#define PCI_HOST_PME_D3COLD_CAPABLE BIT(1) + + +static void gen_pci_unmap_cfg(void *ptr) +{ + pci_ecam_free((struct pci_config_window *)ptr); +} + +struct pci_config_window *pci_host_common_ecam_create(struct device *dev, + struct pci_host_bridge *bridge, const struct pci_ecam_ops *ops) +{ + int err; + struct resource cfgres; + struct resource_entry *bus; + struct pci_config_window *cfg; + + err = of_address_to_resource(dev->of_node, 0, &cfgres); + if (err) { + dev_err(dev, "missing or malformed \"reg\" property\n"); + return ERR_PTR(err); + } + + bus = resource_list_first_type(&bridge->windows, IORESOURCE_BUS); + if (!bus) + return ERR_PTR(-ENODEV); + + cfg = pci_ecam_create(dev, &cfgres, bus->res, ops); + if (IS_ERR(cfg)) + return cfg; + + err = devm_add_action_or_reset(dev, gen_pci_unmap_cfg, cfg); + if (err) + return ERR_PTR(err); + + return cfg; +} +EXPORT_SYMBOL_GPL(pci_host_common_ecam_create); + +int pci_host_common_init(struct platform_device *pdev, + struct pci_host_bridge *bridge, + const struct pci_ecam_ops *ops) +{ + struct device *dev = &pdev->dev; + struct pci_config_window *cfg; + + of_pci_check_probe_only(); + + platform_set_drvdata(pdev, bridge); + + /* Parse and map our Configuration Space windows */ + cfg = pci_host_common_ecam_create(dev, bridge, ops); + if (IS_ERR(cfg)) + return PTR_ERR(cfg); + + /* Do not reassign bus numbers if probe only */ + if (!pci_has_flag(PCI_PROBE_ONLY)) + pci_add_flags(PCI_REASSIGN_ALL_BUS); + + bridge->sysdata = cfg; + bridge->ops = (struct pci_ops *)&ops->pci_ops; + bridge->enable_device = ops->enable_device; + bridge->disable_device = ops->disable_device; + bridge->msi_domain = true; + + return pci_host_probe(bridge); +} +EXPORT_SYMBOL_GPL(pci_host_common_init); + +int pci_host_common_probe(struct platform_device *pdev) +{ + const struct pci_ecam_ops *ops; + struct pci_host_bridge *bridge; + + ops = of_device_get_match_data(&pdev->dev); + if (!ops) + return -ENODEV; + + bridge = devm_pci_alloc_host_bridge(&pdev->dev, 0); + if (!bridge) + return -ENOMEM; + + return pci_host_common_init(pdev, bridge, ops); +} +EXPORT_SYMBOL_GPL(pci_host_common_probe); + +void pci_host_common_remove(struct platform_device *pdev) +{ + struct pci_host_bridge *bridge = platform_get_drvdata(pdev); + + pci_lock_rescan_remove(); + pci_stop_root_bus(bridge->bus); + pci_remove_root_bus(bridge->bus); + pci_unlock_rescan_remove(); +} +EXPORT_SYMBOL_GPL(pci_host_common_remove); + +static int __pci_host_common_d3cold_possible(struct pci_dev *pdev, + void *userdata) +{ + u32 *flags = userdata; + + if (!pdev->dev.driver && !pci_is_enabled(pdev)) + return 0; + + if (pdev->current_state != PCI_D3hot) + goto exit; + + if (device_may_wakeup(&pdev->dev)) { + if (!pci_pme_capable(pdev, PCI_D3cold)) + goto exit; + else + *flags |= PCI_HOST_PME_D3COLD_CAPABLE; + } + + return 0; + +exit: + *flags &= ~PCI_HOST_D3COLD_ALLOWED; + + return -EOPNOTSUPP; +} + +/** + * pci_host_common_d3cold_possible - Determine whether the host bridge can + * transition the devices into D3cold. + * + * @bridge: PCI host bridge to check + * @pme_capable: Pointer to update if there is any device capable of generating + * PME from D3cold. + * + * Walk downstream PCIe endpoint devices and determine whether the host bridge + * is permitted to transition the devices into D3cold. + * + * Devices under host bridge can enter D3cold only if all active PCIe + * endpoints are in PCI_D3hot and any wakeup-enabled endpoint is capable of + * generating PME from D3cold. Inactive endpoints are ignored. + * + * The @pme_capable output allows PCIe controller drivers to apply + * platform-specific handling to preserve wakeup functionality. + * + * Return: %true if the host bridge may enter D3cold, otherwise %false. + */ +bool pci_host_common_d3cold_possible(struct pci_host_bridge *bridge, + bool *pme_capable) +{ + u32 flags = PCI_HOST_D3COLD_ALLOWED; + + pci_walk_bus(bridge->bus, __pci_host_common_d3cold_possible, &flags); + + *pme_capable = !!(flags & PCI_HOST_PME_D3COLD_CAPABLE); + + return !!(flags & PCI_HOST_D3COLD_ALLOWED); +} +EXPORT_SYMBOL_GPL(pci_host_common_d3cold_possible); + +static pci_ers_result_t pci_host_reset_root_port(struct pci_dev *dev) +{ + int ret; + + pci_lock_rescan_remove(); + ret = pci_bus_error_reset(dev); + pci_unlock_rescan_remove(); + if (ret) { + pci_err(dev, "Failed to reset Root Port: %d\n", ret); + return PCI_ERS_RESULT_DISCONNECT; + } + + pci_info(dev, "Root Port has been reset\n"); + + return PCI_ERS_RESULT_RECOVERED; +} + +static void pci_host_recover_root_port(struct pci_dev *port) +{ +#if IS_ENABLED(CONFIG_PCIEAER) + pcie_do_recovery(port, pci_channel_io_frozen, pci_host_reset_root_port); +#else + pci_host_reset_root_port(port); +#endif +} + +void pci_host_handle_link_down(struct pci_dev *port) +{ + pci_info(port, "Recovering Root Port due to Link Down\n"); + pci_host_recover_root_port(port); +} +EXPORT_SYMBOL_GPL(pci_host_handle_link_down); + +MODULE_DESCRIPTION("Common library for PCI host controller drivers"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pci-host-common.h b/drivers/pci/controller/pci-host-common.h new file mode 100644 index 000000000..51afd65d3 --- /dev/null +++ b/drivers/pci/controller/pci-host-common.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Common library for PCI host controller drivers + * + * Copyright (C) 2014 ARM Limited + * + * Author: Will Deacon <will.deacon@arm.com> + */ + +#ifndef _PCI_HOST_COMMON_H +#define _PCI_HOST_COMMON_H + +#include <linux/delay.h> +#include "../pci.h" + +struct pci_ecam_ops; + +/** + * struct pci_host_perst - PERST# GPIO descriptor + * @list: List node for linking multiple PERST# GPIOs + * @desc: GPIO descriptor for PERST# signal + * + * This structure holds a single PERST# GPIO descriptor. + */ +struct pci_host_perst { + struct list_head list; + struct gpio_desc *desc; +}; + +/** + * struct pci_host_port - Generic Root Port properties + * @list: List node for linking multiple ports + * @perst: List of PERST# GPIO descriptors for this port and its children + * + * This structure contains common properties that can be parsed from + * Root Port device tree nodes. + */ +struct pci_host_port { + struct list_head list; + struct list_head perst; +}; + +void pci_host_common_delete_ports(void *data); +int pci_host_common_parse_ports(struct device *dev, + struct pci_host_bridge *bridge); +int pci_host_common_probe(struct platform_device *pdev); +int pci_host_common_init(struct platform_device *pdev, + struct pci_host_bridge *bridge, + const struct pci_ecam_ops *ops); +void pci_host_common_remove(struct platform_device *pdev); +void pci_host_handle_link_down(struct pci_dev *port); + +struct pci_config_window *pci_host_common_ecam_create(struct device *dev, + struct pci_host_bridge *bridge, const struct pci_ecam_ops *ops); + +bool pci_host_common_d3cold_possible(struct pci_host_bridge *bridge, + bool *pme_capable); + +/** + * pci_host_common_link_train_delay - Wait 100 ms if link speed > 5 GT/s + * @max_link_speed: the maximum link speed (2 = 5.0 GT/s, 3 = 8.0 GT/s, ...) + * + * Must be called after Link training completes and before the first + * Configuration Request is sent. + */ +static inline void pci_host_common_link_train_delay(int max_link_speed) +{ + if (max_link_speed > 2) + msleep(PCIE_RESET_CONFIG_WAIT_MS); +} + +#endif diff --git a/drivers/pci/controller/pci-host-generic.c b/drivers/pci/controller/pci-host-generic.c new file mode 100644 index 000000000..9e85c6e9b --- /dev/null +++ b/drivers/pci/controller/pci-host-generic.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Simple, generic PCI host controller driver targeting firmware-initialised + * systems and virtual machines (e.g. the PCI emulation provided by kvmtool). + * + * Copyright (C) 2014 ARM Limited + * + * Author: Will Deacon <will.deacon@arm.com> + */ + +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/pci-ecam.h> +#include <linux/platform_device.h> + +#include "pci-host-common.h" + +static bool pci_dw_valid_device(struct pci_bus *bus, unsigned int devfn) +{ + struct pci_config_window *cfg = bus->sysdata; + + /* + * The Synopsys DesignWare PCIe controller in ECAM mode will not filter + * type 0 config TLPs sent to devices 1 and up on its downstream port, + * resulting in devices appearing multiple times on bus 0 unless we + * filter out those accesses here. + */ + if (bus->number == cfg->busr.start && PCI_SLOT(devfn) > 0) + return false; + + return true; +} + +static void __iomem *pci_dw_ecam_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + if (!pci_dw_valid_device(bus, devfn)) + return NULL; + + return pci_ecam_map_bus(bus, devfn, where); +} + +static const struct pci_ecam_ops pci_dw_ecam_bus_ops = { + .pci_ops = { + .map_bus = pci_dw_ecam_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, + } +}; + +static const struct of_device_id gen_pci_of_match[] = { + { .compatible = "pci-host-cam-generic", + .data = &pci_generic_cam_ops }, + + { .compatible = "pci-host-ecam-generic", + .data = &pci_generic_ecam_ops }, + + { .compatible = "marvell,armada8k-pcie-ecam", + .data = &pci_dw_ecam_bus_ops }, + + { .compatible = "socionext,synquacer-pcie-ecam", + .data = &pci_dw_ecam_bus_ops }, + + { .compatible = "snps,dw-pcie-ecam", + .data = &pci_dw_ecam_bus_ops }, + + { }, +}; +MODULE_DEVICE_TABLE(of, gen_pci_of_match); + +static struct platform_driver gen_pci_driver = { + .driver = { + .name = "pci-host-generic", + .of_match_table = gen_pci_of_match, + }, + .probe = pci_host_common_probe, + .remove = pci_host_common_remove, +}; +module_platform_driver(gen_pci_driver); + +MODULE_DESCRIPTION("Generic PCI host controller driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pci-hyperv-intf.c b/drivers/pci/controller/pci-hyperv-intf.c new file mode 100644 index 000000000..18acbda86 --- /dev/null +++ b/drivers/pci/controller/pci-hyperv-intf.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) Microsoft Corporation. + * + * Author: + * Haiyang Zhang <haiyangz@microsoft.com> + * + * This small module is a helper driver allows other drivers to + * have a common interface with the Hyper-V PCI frontend driver. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/hyperv.h> +#include <linux/export.h> + +struct hyperv_pci_block_ops hvpci_block_ops; +EXPORT_SYMBOL_GPL(hvpci_block_ops); + +int hyperv_read_cfg_blk(struct pci_dev *dev, void *buf, unsigned int buf_len, + unsigned int block_id, unsigned int *bytes_returned) +{ + if (!hvpci_block_ops.read_block) + return -EOPNOTSUPP; + + return hvpci_block_ops.read_block(dev, buf, buf_len, block_id, + bytes_returned); +} +EXPORT_SYMBOL_GPL(hyperv_read_cfg_blk); + +int hyperv_write_cfg_blk(struct pci_dev *dev, void *buf, unsigned int len, + unsigned int block_id) +{ + if (!hvpci_block_ops.write_block) + return -EOPNOTSUPP; + + return hvpci_block_ops.write_block(dev, buf, len, block_id); +} +EXPORT_SYMBOL_GPL(hyperv_write_cfg_blk); + +int hyperv_reg_block_invalidate(struct pci_dev *dev, void *context, + void (*block_invalidate)(void *context, + u64 block_mask)) +{ + if (!hvpci_block_ops.reg_blk_invalidate) + return -EOPNOTSUPP; + + return hvpci_block_ops.reg_blk_invalidate(dev, context, + block_invalidate); +} +EXPORT_SYMBOL_GPL(hyperv_reg_block_invalidate); + +MODULE_DESCRIPTION("Hyper-V PCI Interface"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pci-hyperv.c b/drivers/pci/controller/pci-hyperv.c new file mode 100644 index 000000000..89816a2bd --- /dev/null +++ b/drivers/pci/controller/pci-hyperv.c @@ -0,0 +1,4195 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) Microsoft Corporation. + * + * Author: + * Jake Oshins <jakeo@microsoft.com> + * + * This driver acts as a paravirtual front-end for PCI Express root buses. + * When a PCI Express function (either an entire device or an SR-IOV + * Virtual Function) is being passed through to the VM, this driver exposes + * a new bus to the guest VM. This is modeled as a root PCI bus because + * no bridges are being exposed to the VM. In fact, with a "Generation 2" + * VM within Hyper-V, there may seem to be no PCI bus at all in the VM + * until a device as been exposed using this driver. + * + * Each root PCI bus has its own PCI domain, which is called "Segment" in + * the PCI Firmware Specifications. Thus while each device passed through + * to the VM using this front-end will appear at "device 0", the domain will + * be unique. Typically, each bus will have one PCI function on it, though + * this driver does support more than one. + * + * In order to map the interrupts from the device through to the guest VM, + * this driver also implements an IRQ Domain, which handles interrupts (either + * MSI or MSI-X) associated with the functions on the bus. As interrupts are + * set up, torn down, or reaffined, this driver communicates with the + * underlying hypervisor to adjust the mappings in the I/O MMU so that each + * interrupt will be delivered to the correct virtual processor at the right + * vector. This driver does not support level-triggered (line-based) + * interrupts, and will report that the Interrupt Line register in the + * function's configuration space is zero. + * + * The rest of this driver mostly maps PCI concepts onto underlying Hyper-V + * facilities. For instance, the configuration space of a function exposed + * by Hyper-V is mapped into a single page of memory space, and the + * read and write handlers for config space must be aware of this mechanism. + * Similarly, device setup and teardown involves messages sent to and from + * the PCI back-end driver in Hyper-V. + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/pci-ecam.h> +#include <linux/delay.h> +#include <linux/semaphore.h> +#include <linux/irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/msi.h> +#include <linux/hyperv.h> +#include <linux/refcount.h> +#include <linux/irqdomain.h> +#include <linux/acpi.h> +#include <linux/sizes.h> +#include <linux/of_irq.h> +#include <asm/mshyperv.h> + +/* + * Protocol versions. The low word is the minor version, the high word the + * major version. + */ + +#define PCI_MAKE_VERSION(major, minor) ((u32)(((major) << 16) | (minor))) +#define PCI_MAJOR_VERSION(version) ((u32)(version) >> 16) +#define PCI_MINOR_VERSION(version) ((u32)(version) & 0xff) + +enum pci_protocol_version_t { + PCI_PROTOCOL_VERSION_1_1 = PCI_MAKE_VERSION(1, 1), /* Win10 */ + PCI_PROTOCOL_VERSION_1_2 = PCI_MAKE_VERSION(1, 2), /* RS1 */ + PCI_PROTOCOL_VERSION_1_3 = PCI_MAKE_VERSION(1, 3), /* Vibranium */ + PCI_PROTOCOL_VERSION_1_4 = PCI_MAKE_VERSION(1, 4), /* WS2022 */ +}; + +#define CPU_AFFINITY_ALL -1ULL + +/* + * Supported protocol versions in the order of probing - highest go + * first. + */ +static enum pci_protocol_version_t pci_protocol_versions[] = { + PCI_PROTOCOL_VERSION_1_4, + PCI_PROTOCOL_VERSION_1_3, + PCI_PROTOCOL_VERSION_1_2, + PCI_PROTOCOL_VERSION_1_1, +}; + +#define PCI_CONFIG_MMIO_LENGTH 0x2000 +#define CFG_PAGE_OFFSET 0x1000 +#define CFG_PAGE_SIZE (PCI_CONFIG_MMIO_LENGTH - CFG_PAGE_OFFSET) + +#define MAX_SUPPORTED_MSI_MESSAGES 0x400 + +#define STATUS_REVISION_MISMATCH 0xC0000059 + +/* space for 32bit serial number as string */ +#define SLOT_NAME_SIZE 11 + +/* + * Size of requestor for VMbus; the value is based on the observation + * that having more than one request outstanding is 'rare', and so 64 + * should be generous in ensuring that we don't ever run out. + */ +#define HV_PCI_RQSTOR_SIZE 64 + +/* + * Message Types + */ + +enum pci_message_type { + /* + * Version 1.1 + */ + PCI_MESSAGE_BASE = 0x42490000, + PCI_BUS_RELATIONS = PCI_MESSAGE_BASE + 0, + PCI_QUERY_BUS_RELATIONS = PCI_MESSAGE_BASE + 1, + PCI_POWER_STATE_CHANGE = PCI_MESSAGE_BASE + 4, + PCI_QUERY_RESOURCE_REQUIREMENTS = PCI_MESSAGE_BASE + 5, + PCI_QUERY_RESOURCE_RESOURCES = PCI_MESSAGE_BASE + 6, + PCI_BUS_D0ENTRY = PCI_MESSAGE_BASE + 7, + PCI_BUS_D0EXIT = PCI_MESSAGE_BASE + 8, + PCI_READ_BLOCK = PCI_MESSAGE_BASE + 9, + PCI_WRITE_BLOCK = PCI_MESSAGE_BASE + 0xA, + PCI_EJECT = PCI_MESSAGE_BASE + 0xB, + PCI_QUERY_STOP = PCI_MESSAGE_BASE + 0xC, + PCI_REENABLE = PCI_MESSAGE_BASE + 0xD, + PCI_QUERY_STOP_FAILED = PCI_MESSAGE_BASE + 0xE, + PCI_EJECTION_COMPLETE = PCI_MESSAGE_BASE + 0xF, + PCI_RESOURCES_ASSIGNED = PCI_MESSAGE_BASE + 0x10, + PCI_RESOURCES_RELEASED = PCI_MESSAGE_BASE + 0x11, + PCI_INVALIDATE_BLOCK = PCI_MESSAGE_BASE + 0x12, + PCI_QUERY_PROTOCOL_VERSION = PCI_MESSAGE_BASE + 0x13, + PCI_CREATE_INTERRUPT_MESSAGE = PCI_MESSAGE_BASE + 0x14, + PCI_DELETE_INTERRUPT_MESSAGE = PCI_MESSAGE_BASE + 0x15, + PCI_RESOURCES_ASSIGNED2 = PCI_MESSAGE_BASE + 0x16, + PCI_CREATE_INTERRUPT_MESSAGE2 = PCI_MESSAGE_BASE + 0x17, + PCI_DELETE_INTERRUPT_MESSAGE2 = PCI_MESSAGE_BASE + 0x18, /* unused */ + PCI_BUS_RELATIONS2 = PCI_MESSAGE_BASE + 0x19, + PCI_RESOURCES_ASSIGNED3 = PCI_MESSAGE_BASE + 0x1A, + PCI_CREATE_INTERRUPT_MESSAGE3 = PCI_MESSAGE_BASE + 0x1B, + PCI_MESSAGE_MAXIMUM +}; + +/* + * Structures defining the virtual PCI Express protocol. + */ + +union pci_version { + struct { + u16 minor_version; + u16 major_version; + } parts; + u32 version; +} __packed; + +/* + * Function numbers are 8-bits wide on Express, as interpreted through ARI, + * which is all this driver does. This representation is the one used in + * Windows, which is what is expected when sending this back and forth with + * the Hyper-V parent partition. + */ +union win_slot_encoding { + struct { + u32 dev:5; + u32 func:3; + u32 reserved:24; + } bits; + u32 slot; +} __packed; + +/* + * Pretty much as defined in the PCI Specifications. + */ +struct pci_function_description { + u16 v_id; /* vendor ID */ + u16 d_id; /* device ID */ + u8 rev; + u8 prog_intf; + u8 subclass; + u8 base_class; + u32 subsystem_id; + union win_slot_encoding win_slot; + u32 ser; /* serial number */ +} __packed; + +enum pci_device_description_flags { + HV_PCI_DEVICE_FLAG_NONE = 0x0, + HV_PCI_DEVICE_FLAG_NUMA_AFFINITY = 0x1, +}; + +struct pci_function_description2 { + u16 v_id; /* vendor ID */ + u16 d_id; /* device ID */ + u8 rev; + u8 prog_intf; + u8 subclass; + u8 base_class; + u32 subsystem_id; + union win_slot_encoding win_slot; + u32 ser; /* serial number */ + u32 flags; + u16 virtual_numa_node; + u16 reserved; +} __packed; + +/** + * struct hv_msi_desc + * @vector: IDT entry + * @delivery_mode: As defined in Intel's Programmer's + * Reference Manual, Volume 3, Chapter 8. + * @vector_count: Number of contiguous entries in the + * Interrupt Descriptor Table that are + * occupied by this Message-Signaled + * Interrupt. For "MSI", as first defined + * in PCI 2.2, this can be between 1 and + * 32. For "MSI-X," as first defined in PCI + * 3.0, this must be 1, as each MSI-X table + * entry would have its own descriptor. + * @reserved: Empty space + * @cpu_mask: All the target virtual processors. + */ +struct hv_msi_desc { + u8 vector; + u8 delivery_mode; + u16 vector_count; + u32 reserved; + u64 cpu_mask; +} __packed; + +/** + * struct hv_msi_desc2 - 1.2 version of hv_msi_desc + * @vector: IDT entry + * @delivery_mode: As defined in Intel's Programmer's + * Reference Manual, Volume 3, Chapter 8. + * @vector_count: Number of contiguous entries in the + * Interrupt Descriptor Table that are + * occupied by this Message-Signaled + * Interrupt. For "MSI", as first defined + * in PCI 2.2, this can be between 1 and + * 32. For "MSI-X," as first defined in PCI + * 3.0, this must be 1, as each MSI-X table + * entry would have its own descriptor. + * @processor_count: number of bits enabled in array. + * @processor_array: All the target virtual processors. + */ +struct hv_msi_desc2 { + u8 vector; + u8 delivery_mode; + u16 vector_count; + u16 processor_count; + u16 processor_array[32]; +} __packed; + +/* + * struct hv_msi_desc3 - 1.3 version of hv_msi_desc + * Everything is the same as in 'hv_msi_desc2' except that the size of the + * 'vector' field is larger to support bigger vector values. For ex: LPI + * vectors on ARM. + */ +struct hv_msi_desc3 { + u32 vector; + u8 delivery_mode; + u8 reserved; + u16 vector_count; + u16 processor_count; + u16 processor_array[32]; +} __packed; + +/** + * struct tran_int_desc + * @reserved: unused, padding + * @vector_count: same as in hv_msi_desc + * @data: This is the "data payload" value that is + * written by the device when it generates + * a message-signaled interrupt, either MSI + * or MSI-X. + * @address: This is the address to which the data + * payload is written on interrupt + * generation. + */ +struct tran_int_desc { + u16 reserved; + u16 vector_count; + u32 data; + u64 address; +} __packed; + +/* + * A generic message format for virtual PCI. + * Specific message formats are defined later in the file. + */ + +struct pci_message { + u32 type; +} __packed; + +struct pci_child_message { + struct pci_message message_type; + union win_slot_encoding wslot; +} __packed; + +struct pci_incoming_message { + struct vmpacket_descriptor hdr; + struct pci_message message_type; +} __packed; + +struct pci_response { + struct vmpacket_descriptor hdr; + s32 status; /* negative values are failures */ +} __packed; + +struct pci_packet { + void (*completion_func)(void *context, struct pci_response *resp, + int resp_packet_size); + void *compl_ctxt; +}; + +/* + * Specific message types supporting the PCI protocol. + */ + +/* + * Version negotiation message. Sent from the guest to the host. + * The guest is free to try different versions until the host + * accepts the version. + * + * pci_version: The protocol version requested. + * is_last_attempt: If TRUE, this is the last version guest will request. + * reservedz: Reserved field, set to zero. + */ + +struct pci_version_request { + struct pci_message message_type; + u32 protocol_version; +} __packed; + +/* + * Bus D0 Entry. This is sent from the guest to the host when the virtual + * bus (PCI Express port) is ready for action. + */ + +struct pci_bus_d0_entry { + struct pci_message message_type; + u32 reserved; + u64 mmio_base; +} __packed; + +struct pci_bus_relations { + struct pci_incoming_message incoming; + u32 device_count; + struct pci_function_description func[]; +} __packed; + +struct pci_bus_relations2 { + struct pci_incoming_message incoming; + u32 device_count; + struct pci_function_description2 func[]; +} __packed; + +struct pci_q_res_req_response { + struct vmpacket_descriptor hdr; + s32 status; /* negative values are failures */ + u32 probed_bar[PCI_STD_NUM_BARS]; +} __packed; + +struct pci_set_power { + struct pci_message message_type; + union win_slot_encoding wslot; + u32 power_state; /* In Windows terms */ + u32 reserved; +} __packed; + +struct pci_set_power_response { + struct vmpacket_descriptor hdr; + s32 status; /* negative values are failures */ + union win_slot_encoding wslot; + u32 resultant_state; /* In Windows terms */ + u32 reserved; +} __packed; + +struct pci_resources_assigned { + struct pci_message message_type; + union win_slot_encoding wslot; + u8 memory_range[0x14][6]; /* not used here */ + u32 msi_descriptors; + u32 reserved[4]; +} __packed; + +struct pci_resources_assigned2 { + struct pci_message message_type; + union win_slot_encoding wslot; + u8 memory_range[0x14][6]; /* not used here */ + u32 msi_descriptor_count; + u8 reserved[70]; +} __packed; + +struct pci_create_interrupt { + struct pci_message message_type; + union win_slot_encoding wslot; + struct hv_msi_desc int_desc; +} __packed; + +struct pci_create_int_response { + struct pci_response response; + u32 reserved; + struct tran_int_desc int_desc; +} __packed; + +struct pci_create_interrupt2 { + struct pci_message message_type; + union win_slot_encoding wslot; + struct hv_msi_desc2 int_desc; +} __packed; + +struct pci_create_interrupt3 { + struct pci_message message_type; + union win_slot_encoding wslot; + struct hv_msi_desc3 int_desc; +} __packed; + +struct pci_delete_interrupt { + struct pci_message message_type; + union win_slot_encoding wslot; + struct tran_int_desc int_desc; +} __packed; + +/* + * Note: the VM must pass a valid block id, wslot and bytes_requested. + */ +struct pci_read_block { + struct pci_message message_type; + u32 block_id; + union win_slot_encoding wslot; + u32 bytes_requested; +} __packed; + +struct pci_read_block_response { + struct vmpacket_descriptor hdr; + u32 status; + u8 bytes[HV_CONFIG_BLOCK_SIZE_MAX]; +} __packed; + +/* + * Note: the VM must pass a valid block id, wslot and byte_count. + */ +struct pci_write_block { + struct pci_message message_type; + u32 block_id; + union win_slot_encoding wslot; + u32 byte_count; + u8 bytes[HV_CONFIG_BLOCK_SIZE_MAX]; +} __packed; + +struct pci_dev_inval_block { + struct pci_incoming_message incoming; + union win_slot_encoding wslot; + u64 block_mask; +} __packed; + +struct pci_dev_incoming { + struct pci_incoming_message incoming; + union win_slot_encoding wslot; +} __packed; + +struct pci_eject_response { + struct pci_message message_type; + union win_slot_encoding wslot; + u32 status; +} __packed; + +static int pci_ring_size = VMBUS_RING_SIZE(SZ_16K); + +/* + * Driver specific state. + */ + +enum hv_pcibus_state { + hv_pcibus_init = 0, + hv_pcibus_probed, + hv_pcibus_installed, + hv_pcibus_removing, + hv_pcibus_maximum +}; + +struct hv_pcibus_device { +#ifdef CONFIG_X86 + struct pci_sysdata sysdata; +#elif defined(CONFIG_ARM64) + struct pci_config_window sysdata; +#endif + struct pci_host_bridge *bridge; + struct fwnode_handle *fwnode; + /* Protocol version negotiated with the host */ + enum pci_protocol_version_t protocol_version; + + struct mutex state_lock; + enum hv_pcibus_state state; + + struct hv_device *hdev; + resource_size_t low_mmio_space; + resource_size_t high_mmio_space; + struct resource *mem_config; + struct resource *low_mmio_res; + struct resource *high_mmio_res; + struct completion *survey_event; + spinlock_t config_lock; /* Avoid two threads writing index page */ + spinlock_t device_list_lock; /* Protect lists below */ + void __iomem *cfg_addr; + + struct list_head children; + struct list_head dr_list; + + struct irq_domain *irq_domain; + + struct workqueue_struct *wq; + + /* Highest slot of child device with resources allocated */ + int wslot_res_allocated; + bool use_calls; /* Use hypercalls to access mmio cfg space */ +}; + +/* + * Tracks "Device Relations" messages from the host, which must be both + * processed in order and deferred so that they don't run in the context + * of the incoming packet callback. + */ +struct hv_dr_work { + struct work_struct wrk; + struct hv_pcibus_device *bus; +}; + +struct hv_pcidev_description { + u16 v_id; /* vendor ID */ + u16 d_id; /* device ID */ + u8 rev; + u8 prog_intf; + u8 subclass; + u8 base_class; + u32 subsystem_id; + union win_slot_encoding win_slot; + u32 ser; /* serial number */ + u32 flags; + u16 virtual_numa_node; +}; + +struct hv_dr_state { + struct list_head list_entry; + u32 device_count; + struct hv_pcidev_description func[] __counted_by(device_count); +}; + +struct hv_pci_dev { + /* List protected by pci_rescan_remove_lock */ + struct list_head list_entry; + refcount_t refs; + struct pci_slot *pci_slot; + struct hv_pcidev_description desc; + bool reported_missing; + struct hv_pcibus_device *hbus; + struct work_struct wrk; + + void (*block_invalidate)(void *context, u64 block_mask); + void *invalidate_context; + + /* + * What would be observed if one wrote 0xFFFFFFFF to a BAR and then + * read it back, for each of the BAR offsets within config space. + */ + u32 probed_bar[PCI_STD_NUM_BARS]; +}; + +struct hv_pci_compl { + struct completion host_event; + s32 completion_status; +}; + +static void hv_pci_onchannelcallback(void *context); + +#ifdef CONFIG_X86 +#define DELIVERY_MODE APIC_DELIVERY_MODE_FIXED +#define HV_MSI_CHIP_FLAGS MSI_CHIP_FLAG_SET_ACK + +static int hv_pci_irqchip_init(void) +{ + return 0; +} + +static struct irq_domain *hv_pci_get_root_domain(void) +{ + return x86_vector_domain; +} + +static unsigned int hv_msi_get_int_vector(struct irq_data *data) +{ + struct irq_cfg *cfg = irqd_cfg(data); + + return cfg->vector; +} + +#define hv_msi_prepare pci_msi_prepare + +/** + * hv_irq_retarget_interrupt() - "Unmask" the IRQ by setting its current + * affinity. + * @data: Describes the IRQ + * + * Build new a destination for the MSI and make a hypercall to + * update the Interrupt Redirection Table. "Device Logical ID" + * is built out of this PCI bus's instance GUID and the function + * number of the device. + */ +static void hv_irq_retarget_interrupt(struct irq_data *data) +{ + struct msi_desc *msi_desc = irq_data_get_msi_desc(data); + struct hv_retarget_device_interrupt *params; + struct tran_int_desc *int_desc; + struct hv_pcibus_device *hbus; + const struct cpumask *dest; + cpumask_var_t tmp; + struct pci_bus *pbus; + struct pci_dev *pdev; + unsigned long flags; + u32 var_size = 0; + int cpu, nr_bank; + u64 res; + + dest = irq_data_get_effective_affinity_mask(data); + pdev = msi_desc_to_pci_dev(msi_desc); + pbus = pdev->bus; + hbus = container_of(pbus->sysdata, struct hv_pcibus_device, sysdata); + int_desc = data->chip_data; + if (!int_desc) { + dev_warn(&hbus->hdev->device, "%s() can not unmask irq %u\n", + __func__, data->irq); + return; + } + + local_irq_save(flags); + + params = *this_cpu_ptr(hyperv_pcpu_input_arg); + memset(params, 0, sizeof(*params)); + params->partition_id = HV_PARTITION_ID_SELF; + params->int_entry.source = HV_INTERRUPT_SOURCE_MSI; + params->int_entry.msi_entry.address.as_uint32 = int_desc->address & 0xffffffff; + params->int_entry.msi_entry.data.as_uint32 = int_desc->data; + params->device_id = (hbus->hdev->dev_instance.b[5] << 24) | + (hbus->hdev->dev_instance.b[4] << 16) | + (hbus->hdev->dev_instance.b[7] << 8) | + (hbus->hdev->dev_instance.b[6] & 0xf8) | + PCI_FUNC(pdev->devfn); + params->int_target.vector = hv_msi_get_int_vector(data); + + if (hbus->protocol_version >= PCI_PROTOCOL_VERSION_1_2) { + /* + * PCI_PROTOCOL_VERSION_1_2 supports the VP_SET version of the + * HVCALL_RETARGET_INTERRUPT hypercall, which also coincides + * with >64 VP support. + * ms_hyperv.hints & HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED + * is not sufficient for this hypercall. + */ + params->int_target.flags |= + HV_DEVICE_INTERRUPT_TARGET_PROCESSOR_SET; + + if (!alloc_cpumask_var(&tmp, GFP_ATOMIC)) { + res = 1; + goto out; + } + + cpumask_and(tmp, dest, cpu_online_mask); + nr_bank = cpumask_to_vpset(¶ms->int_target.vp_set, tmp); + free_cpumask_var(tmp); + + if (nr_bank <= 0) { + res = 1; + goto out; + } + + /* + * var-sized hypercall, var-size starts after vp_mask (thus + * vp_set.format does not count, but vp_set.valid_bank_mask + * does). + */ + var_size = 1 + nr_bank; + } else { + for_each_cpu_and(cpu, dest, cpu_online_mask) { + params->int_target.vp_mask |= + (1ULL << hv_cpu_number_to_vp_number(cpu)); + } + } + + res = hv_do_hypercall(HVCALL_RETARGET_INTERRUPT | (var_size << 17), + params, NULL); + +out: + local_irq_restore(flags); + + /* + * During hibernation, when a CPU is offlined, the kernel tries + * to move the interrupt to the remaining CPUs that haven't + * been offlined yet. In this case, the below hv_do_hypercall() + * always fails since the vmbus channel has been closed: + * refer to cpu_disable_common() -> fixup_irqs() -> + * irq_migrate_all_off_this_cpu() -> migrate_one_irq(). + * + * Suppress the error message for hibernation because the failure + * during hibernation does not matter (at this time all the devices + * have been frozen). Note: the correct affinity info is still updated + * into the irqdata data structure in migrate_one_irq() -> + * irq_do_set_affinity(), so later when the VM resumes, + * hv_pci_restore_msi_state() is able to correctly restore the + * interrupt with the correct affinity. + */ + if (!hv_result_success(res) && hbus->state != hv_pcibus_removing) + dev_err(&hbus->hdev->device, + "%s() failed: %#llx", __func__, res); +} + +static void hv_arch_irq_unmask(struct irq_data *data) +{ + if (hv_root_partition()) + /* + * In case of the nested root partition, the nested hypervisor + * is taking care of interrupt remapping and thus the + * MAP_DEVICE_INTERRUPT hypercall is required instead of + * RETARGET_INTERRUPT. + */ + (void)hv_map_msi_interrupt(data, NULL); + else + hv_irq_retarget_interrupt(data); +} +#elif defined(CONFIG_ARM64) +/* + * SPI vectors to use for vPCI; arch SPIs range is [32, 1019], but leaving a bit + * of room at the start to allow for SPIs to be specified through ACPI and + * starting with a power of two to satisfy power of 2 multi-MSI requirement. + */ +#define HV_PCI_MSI_SPI_START 64 +#define HV_PCI_MSI_SPI_NR (1020 - HV_PCI_MSI_SPI_START) +#define DELIVERY_MODE 0 +#define HV_MSI_CHIP_FLAGS MSI_CHIP_FLAG_SET_EOI +#define hv_msi_prepare NULL + +struct hv_pci_chip_data { + DECLARE_BITMAP(spi_map, HV_PCI_MSI_SPI_NR); + struct mutex map_lock; +}; + +/* Hyper-V vPCI MSI GIC IRQ domain */ +static struct irq_domain *hv_msi_gic_irq_domain; + +/* Hyper-V PCI MSI IRQ chip */ +static struct irq_chip hv_arm64_msi_irq_chip = { + .name = "MSI", + .irq_set_affinity = irq_chip_set_affinity_parent, + .irq_eoi = irq_chip_eoi_parent, + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent +}; + +static unsigned int hv_msi_get_int_vector(struct irq_data *irqd) +{ + return irqd->parent_data->hwirq; +} + +/* + * @nr_bm_irqs: Indicates the number of IRQs that were allocated from + * the bitmap. + * @nr_dom_irqs: Indicates the number of IRQs that were allocated from + * the parent domain. + */ +static void hv_pci_vec_irq_free(struct irq_domain *domain, + unsigned int virq, + unsigned int nr_bm_irqs, + unsigned int nr_dom_irqs) +{ + struct hv_pci_chip_data *chip_data = domain->host_data; + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + int first = d->hwirq - HV_PCI_MSI_SPI_START; + int i; + + mutex_lock(&chip_data->map_lock); + bitmap_release_region(chip_data->spi_map, + first, + get_count_order(nr_bm_irqs)); + mutex_unlock(&chip_data->map_lock); + for (i = 0; i < nr_dom_irqs; i++) { + if (i) + d = irq_domain_get_irq_data(domain, virq + i); + irq_domain_reset_irq_data(d); + } + + irq_domain_free_irqs_parent(domain, virq, nr_dom_irqs); +} + +static void hv_pci_vec_irq_domain_free(struct irq_domain *domain, + unsigned int virq, + unsigned int nr_irqs) +{ + hv_pci_vec_irq_free(domain, virq, nr_irqs, nr_irqs); +} + +static int hv_pci_vec_alloc_device_irq(struct irq_domain *domain, + unsigned int nr_irqs, + irq_hw_number_t *hwirq) +{ + struct hv_pci_chip_data *chip_data = domain->host_data; + int index; + + /* Find and allocate region from the SPI bitmap */ + mutex_lock(&chip_data->map_lock); + index = bitmap_find_free_region(chip_data->spi_map, + HV_PCI_MSI_SPI_NR, + get_count_order(nr_irqs)); + mutex_unlock(&chip_data->map_lock); + if (index < 0) + return -ENOSPC; + + *hwirq = index + HV_PCI_MSI_SPI_START; + + return 0; +} + +static int hv_pci_vec_irq_gic_domain_alloc(struct irq_domain *domain, + unsigned int virq, + irq_hw_number_t hwirq) +{ + struct irq_fwspec fwspec; + struct irq_data *d; + int ret; + + fwspec.fwnode = domain->parent->fwnode; + if (is_of_node(fwspec.fwnode)) { + /* SPI lines for OF translations start at offset 32 */ + fwspec.param_count = 3; + fwspec.param[0] = 0; + fwspec.param[1] = hwirq - 32; + fwspec.param[2] = IRQ_TYPE_EDGE_RISING; + } else { + fwspec.param_count = 2; + fwspec.param[0] = hwirq; + fwspec.param[1] = IRQ_TYPE_EDGE_RISING; + } + + ret = irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec); + if (ret) + return ret; + + /* + * Since the interrupt specifier is not coming from ACPI or DT, the + * trigger type will need to be set explicitly. Otherwise, it will be + * set to whatever is in the GIC configuration. + */ + d = irq_domain_get_irq_data(domain->parent, virq); + + return d->chip->irq_set_type(d, IRQ_TYPE_EDGE_RISING); +} + +static int hv_pci_vec_irq_domain_alloc(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs, + void *args) +{ + irq_hw_number_t hwirq; + unsigned int i; + int ret; + + ret = hv_pci_vec_alloc_device_irq(domain, nr_irqs, &hwirq); + if (ret) + return ret; + + for (i = 0; i < nr_irqs; i++) { + ret = hv_pci_vec_irq_gic_domain_alloc(domain, virq + i, + hwirq + i); + if (ret) { + hv_pci_vec_irq_free(domain, virq, nr_irqs, i); + return ret; + } + + irq_domain_set_hwirq_and_chip(domain, virq + i, + hwirq + i, + &hv_arm64_msi_irq_chip, + domain->host_data); + pr_debug("pID:%d vID:%u\n", (int)(hwirq + i), virq + i); + } + + return 0; +} + +/* + * Pick the first cpu as the irq affinity that can be temporarily used for + * composing MSI from the hypervisor. GIC will eventually set the right + * affinity for the irq and the 'unmask' will retarget the interrupt to that + * cpu. + */ +static int hv_pci_vec_irq_domain_activate(struct irq_domain *domain, + struct irq_data *irqd, bool reserve) +{ + int cpu = cpumask_first(cpu_present_mask); + + irq_data_update_effective_affinity(irqd, cpumask_of(cpu)); + + return 0; +} + +static const struct irq_domain_ops hv_pci_domain_ops = { + .alloc = hv_pci_vec_irq_domain_alloc, + .free = hv_pci_vec_irq_domain_free, + .activate = hv_pci_vec_irq_domain_activate, +}; + +#ifdef CONFIG_OF + +static struct irq_domain *hv_pci_of_irq_domain_parent(void) +{ + struct device_node *parent; + struct irq_domain *domain; + + parent = of_irq_find_parent(hv_get_vmbus_root_device()->of_node); + if (!parent) + return NULL; + domain = irq_find_host(parent); + of_node_put(parent); + + return domain; +} + +#endif + +#ifdef CONFIG_ACPI + +static struct irq_domain *hv_pci_acpi_irq_domain_parent(void) +{ + acpi_gsi_domain_disp_fn gsi_domain_disp_fn; + + gsi_domain_disp_fn = acpi_get_gsi_dispatcher(); + if (!gsi_domain_disp_fn) + return NULL; + return irq_find_matching_fwnode(gsi_domain_disp_fn(0), + DOMAIN_BUS_ANY); +} + +#endif + +static int hv_pci_irqchip_init(void) +{ + static struct hv_pci_chip_data *chip_data; + struct fwnode_handle *fn = NULL; + struct irq_domain *irq_domain_parent = NULL; + int ret = -ENOMEM; + + chip_data = kzalloc_obj(*chip_data); + if (!chip_data) + return ret; + + mutex_init(&chip_data->map_lock); + fn = irq_domain_alloc_named_fwnode("hv_vpci_arm64"); + if (!fn) + goto free_chip; + + /* + * IRQ domain once enabled, should not be removed since there is no + * way to ensure that all the corresponding devices are also gone and + * no interrupts will be generated. + */ +#ifdef CONFIG_ACPI + if (!acpi_disabled) + irq_domain_parent = hv_pci_acpi_irq_domain_parent(); +#endif +#ifdef CONFIG_OF + if (!irq_domain_parent) + irq_domain_parent = hv_pci_of_irq_domain_parent(); +#endif + if (!irq_domain_parent) { + WARN_ONCE(1, "Invalid firmware configuration for VMBus interrupts\n"); + ret = -EINVAL; + goto free_chip; + } + + hv_msi_gic_irq_domain = irq_domain_create_hierarchy(irq_domain_parent, 0, + HV_PCI_MSI_SPI_NR, + fn, &hv_pci_domain_ops, + chip_data); + + if (!hv_msi_gic_irq_domain) { + pr_err("Failed to create Hyper-V arm64 vPCI MSI IRQ domain\n"); + goto free_chip; + } + + return 0; + +free_chip: + kfree(chip_data); + if (fn) + irq_domain_free_fwnode(fn); + + return ret; +} + +static struct irq_domain *hv_pci_get_root_domain(void) +{ + return hv_msi_gic_irq_domain; +} + +/* + * SPIs are used for interrupts of PCI devices and SPIs is managed via GICD + * registers which Hyper-V already supports, so no hypercall needed. + */ +static void hv_arch_irq_unmask(struct irq_data *data) { } +#endif /* CONFIG_ARM64 */ + +/** + * hv_pci_generic_compl() - Invoked for a completion packet + * @context: Set up by the sender of the packet. + * @resp: The response packet + * @resp_packet_size: Size in bytes of the packet + * + * This function is used to trigger an event and report status + * for any message for which the completion packet contains a + * status and nothing else. + */ +static void hv_pci_generic_compl(void *context, struct pci_response *resp, + int resp_packet_size) +{ + struct hv_pci_compl *comp_pkt = context; + + comp_pkt->completion_status = resp->status; + complete(&comp_pkt->host_event); +} + +static struct hv_pci_dev *get_pcichild_wslot(struct hv_pcibus_device *hbus, + u32 wslot); + +static void get_pcichild(struct hv_pci_dev *hpdev) +{ + refcount_inc(&hpdev->refs); +} + +static void put_pcichild(struct hv_pci_dev *hpdev) +{ + if (refcount_dec_and_test(&hpdev->refs)) + kfree(hpdev); +} + +/* + * There is no good way to get notified from vmbus_onoffer_rescind(), + * so let's use polling here, since this is not a hot path. + */ +static int wait_for_response(struct hv_device *hdev, + struct completion *comp) +{ + while (true) { + if (hdev->channel->rescind) { + dev_warn_once(&hdev->device, "The device is gone.\n"); + return -ENODEV; + } + + if (wait_for_completion_timeout(comp, HZ / 10)) + break; + } + + return 0; +} + +/** + * devfn_to_wslot() - Convert from Linux PCI slot to Windows + * @devfn: The Linux representation of PCI slot + * + * Windows uses a slightly different representation of PCI slot. + * + * Return: The Windows representation + */ +static u32 devfn_to_wslot(int devfn) +{ + union win_slot_encoding wslot; + + wslot.slot = 0; + wslot.bits.dev = PCI_SLOT(devfn); + wslot.bits.func = PCI_FUNC(devfn); + + return wslot.slot; +} + +/** + * wslot_to_devfn() - Convert from Windows PCI slot to Linux + * @wslot: The Windows representation of PCI slot + * + * Windows uses a slightly different representation of PCI slot. + * + * Return: The Linux representation + */ +static int wslot_to_devfn(u32 wslot) +{ + union win_slot_encoding slot_no; + + slot_no.slot = wslot; + return PCI_DEVFN(slot_no.bits.dev, slot_no.bits.func); +} + +static void hv_pci_read_mmio(struct device *dev, phys_addr_t gpa, int size, u32 *val) +{ + struct hv_mmio_read_input *in; + struct hv_mmio_read_output *out; + u64 ret; + + /* + * Must be called with interrupts disabled so it is safe + * to use the per-cpu input argument page. Use it for + * both input and output. + */ + in = *this_cpu_ptr(hyperv_pcpu_input_arg); + out = *this_cpu_ptr(hyperv_pcpu_input_arg) + sizeof(*in); + in->gpa = gpa; + in->size = size; + + ret = hv_do_hypercall(HVCALL_MMIO_READ, in, out); + if (hv_result_success(ret)) { + switch (size) { + case 1: + *val = *(u8 *)(out->data); + break; + case 2: + *val = *(u16 *)(out->data); + break; + default: + *val = *(u32 *)(out->data); + break; + } + } else + dev_err(dev, "MMIO read hypercall error %llx addr %llx size %d\n", + ret, gpa, size); +} + +static void hv_pci_write_mmio(struct device *dev, phys_addr_t gpa, int size, u32 val) +{ + struct hv_mmio_write_input *in; + u64 ret; + + /* + * Must be called with interrupts disabled so it is safe + * to use the per-cpu input argument memory. + */ + in = *this_cpu_ptr(hyperv_pcpu_input_arg); + in->gpa = gpa; + in->size = size; + switch (size) { + case 1: + *(u8 *)(in->data) = val; + break; + case 2: + *(u16 *)(in->data) = val; + break; + default: + *(u32 *)(in->data) = val; + break; + } + + ret = hv_do_hypercall(HVCALL_MMIO_WRITE, in, NULL); + if (!hv_result_success(ret)) + dev_err(dev, "MMIO write hypercall error %llx addr %llx size %d\n", + ret, gpa, size); +} + +/* + * PCI Configuration Space for these root PCI buses is implemented as a pair + * of pages in memory-mapped I/O space. Writing to the first page chooses + * the PCI function being written or read. Once the first page has been + * written to, the following page maps in the entire configuration space of + * the function. + */ + +/** + * _hv_pcifront_read_config() - Internal PCI config read + * @hpdev: The PCI driver's representation of the device + * @where: Offset within config space + * @size: Size of the transfer + * @val: Pointer to the buffer receiving the data + */ +static void _hv_pcifront_read_config(struct hv_pci_dev *hpdev, int where, + int size, u32 *val) +{ + struct hv_pcibus_device *hbus = hpdev->hbus; + struct device *dev = &hbus->hdev->device; + int offset = where + CFG_PAGE_OFFSET; + unsigned long flags; + + /* + * If the attempt is to read the IDs or the ROM BAR, simulate that. + */ + if (where + size <= PCI_COMMAND) { + memcpy(val, ((u8 *)&hpdev->desc.v_id) + where, size); + } else if (where >= PCI_CLASS_REVISION && where + size <= + PCI_CACHE_LINE_SIZE) { + memcpy(val, ((u8 *)&hpdev->desc.rev) + where - + PCI_CLASS_REVISION, size); + } else if (where >= PCI_SUBSYSTEM_VENDOR_ID && where + size <= + PCI_ROM_ADDRESS) { + memcpy(val, (u8 *)&hpdev->desc.subsystem_id + where - + PCI_SUBSYSTEM_VENDOR_ID, size); + } else if (where >= PCI_ROM_ADDRESS && where + size <= + PCI_CAPABILITY_LIST) { + /* ROM BARs are unimplemented */ + *val = 0; + } else if ((where >= PCI_INTERRUPT_LINE && where + size <= PCI_INTERRUPT_PIN) || + (where >= PCI_INTERRUPT_PIN && where + size <= PCI_MIN_GNT)) { + /* + * Interrupt Line and Interrupt PIN are hard-wired to zero + * because this front-end only supports message-signaled + * interrupts. + */ + *val = 0; + } else if (where + size <= CFG_PAGE_SIZE) { + + spin_lock_irqsave(&hbus->config_lock, flags); + if (hbus->use_calls) { + phys_addr_t addr = hbus->mem_config->start + offset; + + hv_pci_write_mmio(dev, hbus->mem_config->start, 4, + hpdev->desc.win_slot.slot); + hv_pci_read_mmio(dev, addr, size, val); + } else { + void __iomem *addr = hbus->cfg_addr + offset; + + /* Choose the function to be read. (See comment above) */ + writel(hpdev->desc.win_slot.slot, hbus->cfg_addr); + /* Make sure the function was chosen before reading. */ + mb(); + /* Read from that function's config space. */ + switch (size) { + case 1: + *val = readb(addr); + break; + case 2: + *val = readw(addr); + break; + default: + *val = readl(addr); + break; + } + /* + * Make sure the read was done before we release the + * spinlock allowing consecutive reads/writes. + */ + mb(); + } + spin_unlock_irqrestore(&hbus->config_lock, flags); + } else { + dev_err(dev, "Attempt to read beyond a function's config space.\n"); + } +} + +static u16 hv_pcifront_get_vendor_id(struct hv_pci_dev *hpdev) +{ + struct hv_pcibus_device *hbus = hpdev->hbus; + struct device *dev = &hbus->hdev->device; + u32 val; + u16 ret; + unsigned long flags; + + spin_lock_irqsave(&hbus->config_lock, flags); + + if (hbus->use_calls) { + phys_addr_t addr = hbus->mem_config->start + + CFG_PAGE_OFFSET + PCI_VENDOR_ID; + + hv_pci_write_mmio(dev, hbus->mem_config->start, 4, + hpdev->desc.win_slot.slot); + hv_pci_read_mmio(dev, addr, 2, &val); + ret = val; /* Truncates to 16 bits */ + } else { + void __iomem *addr = hbus->cfg_addr + CFG_PAGE_OFFSET + + PCI_VENDOR_ID; + /* Choose the function to be read. (See comment above) */ + writel(hpdev->desc.win_slot.slot, hbus->cfg_addr); + /* Make sure the function was chosen before we start reading. */ + mb(); + /* Read from that function's config space. */ + ret = readw(addr); + /* + * mb() is not required here, because the + * spin_unlock_irqrestore() is a barrier. + */ + } + + spin_unlock_irqrestore(&hbus->config_lock, flags); + + return ret; +} + +/** + * _hv_pcifront_write_config() - Internal PCI config write + * @hpdev: The PCI driver's representation of the device + * @where: Offset within config space + * @size: Size of the transfer + * @val: The data being transferred + */ +static void _hv_pcifront_write_config(struct hv_pci_dev *hpdev, int where, + int size, u32 val) +{ + struct hv_pcibus_device *hbus = hpdev->hbus; + struct device *dev = &hbus->hdev->device; + int offset = where + CFG_PAGE_OFFSET; + unsigned long flags; + + if (where >= PCI_SUBSYSTEM_VENDOR_ID && + where + size <= PCI_CAPABILITY_LIST) { + /* SSIDs and ROM BARs are read-only */ + } else if (where >= PCI_COMMAND && where + size <= CFG_PAGE_SIZE) { + spin_lock_irqsave(&hbus->config_lock, flags); + + if (hbus->use_calls) { + phys_addr_t addr = hbus->mem_config->start + offset; + + hv_pci_write_mmio(dev, hbus->mem_config->start, 4, + hpdev->desc.win_slot.slot); + hv_pci_write_mmio(dev, addr, size, val); + } else { + void __iomem *addr = hbus->cfg_addr + offset; + + /* Choose the function to write. (See comment above) */ + writel(hpdev->desc.win_slot.slot, hbus->cfg_addr); + /* Make sure the function was chosen before writing. */ + wmb(); + /* Write to that function's config space. */ + switch (size) { + case 1: + writeb(val, addr); + break; + case 2: + writew(val, addr); + break; + default: + writel(val, addr); + break; + } + /* + * Make sure the write was done before we release the + * spinlock allowing consecutive reads/writes. + */ + mb(); + } + spin_unlock_irqrestore(&hbus->config_lock, flags); + } else { + dev_err(dev, "Attempt to write beyond a function's config space.\n"); + } +} + +/** + * hv_pcifront_read_config() - Read configuration space + * @bus: PCI Bus structure + * @devfn: Device/function + * @where: Offset from base + * @size: Byte/word/dword + * @val: Value to be read + * + * Return: PCIBIOS_SUCCESSFUL on success + * PCIBIOS_DEVICE_NOT_FOUND on failure + */ +static int hv_pcifront_read_config(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct hv_pcibus_device *hbus = + container_of(bus->sysdata, struct hv_pcibus_device, sysdata); + struct hv_pci_dev *hpdev; + + hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(devfn)); + if (!hpdev) + return PCIBIOS_DEVICE_NOT_FOUND; + + _hv_pcifront_read_config(hpdev, where, size, val); + + put_pcichild(hpdev); + return PCIBIOS_SUCCESSFUL; +} + +/** + * hv_pcifront_write_config() - Write configuration space + * @bus: PCI Bus structure + * @devfn: Device/function + * @where: Offset from base + * @size: Byte/word/dword + * @val: Value to be written to device + * + * Return: PCIBIOS_SUCCESSFUL on success + * PCIBIOS_DEVICE_NOT_FOUND on failure + */ +static int hv_pcifront_write_config(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + struct hv_pcibus_device *hbus = + container_of(bus->sysdata, struct hv_pcibus_device, sysdata); + struct hv_pci_dev *hpdev; + + hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(devfn)); + if (!hpdev) + return PCIBIOS_DEVICE_NOT_FOUND; + + _hv_pcifront_write_config(hpdev, where, size, val); + + put_pcichild(hpdev); + return PCIBIOS_SUCCESSFUL; +} + +/* PCIe operations */ +static struct pci_ops hv_pcifront_ops = { + .read = hv_pcifront_read_config, + .write = hv_pcifront_write_config, +}; + +/* + * Paravirtual backchannel + * + * Hyper-V SR-IOV provides a backchannel mechanism in software for + * communication between a VF driver and a PF driver. These + * "configuration blocks" are similar in concept to PCI configuration space, + * but instead of doing reads and writes in 32-bit chunks through a very slow + * path, packets of up to 128 bytes can be sent or received asynchronously. + * + * Nearly every SR-IOV device contains just such a communications channel in + * hardware, so using this one in software is usually optional. Using the + * software channel, however, allows driver implementers to leverage software + * tools that fuzz the communications channel looking for vulnerabilities. + * + * The usage model for these packets puts the responsibility for reading or + * writing on the VF driver. The VF driver sends a read or a write packet, + * indicating which "block" is being referred to by number. + * + * If the PF driver wishes to initiate communication, it can "invalidate" one or + * more of the first 64 blocks. This invalidation is delivered via a callback + * supplied to the VF driver by this driver. + * + * No protocol is implied, except that supplied by the PF and VF drivers. + */ + +struct hv_read_config_compl { + struct hv_pci_compl comp_pkt; + void *buf; + unsigned int len; + unsigned int bytes_returned; +}; + +/** + * hv_pci_read_config_compl() - Invoked when a response packet + * for a read config block operation arrives. + * @context: Identifies the read config operation + * @resp: The response packet itself + * @resp_packet_size: Size in bytes of the response packet + */ +static void hv_pci_read_config_compl(void *context, struct pci_response *resp, + int resp_packet_size) +{ + struct hv_read_config_compl *comp = context; + struct pci_read_block_response *read_resp = + (struct pci_read_block_response *)resp; + unsigned int data_len, hdr_len; + + hdr_len = offsetof(struct pci_read_block_response, bytes); + if (resp_packet_size < hdr_len) { + comp->comp_pkt.completion_status = -1; + goto out; + } + + data_len = resp_packet_size - hdr_len; + if (data_len > 0 && read_resp->status == 0) { + comp->bytes_returned = min(comp->len, data_len); + memcpy(comp->buf, read_resp->bytes, comp->bytes_returned); + } else { + comp->bytes_returned = 0; + } + + comp->comp_pkt.completion_status = read_resp->status; +out: + complete(&comp->comp_pkt.host_event); +} + +/** + * hv_read_config_block() - Sends a read config block request to + * the back-end driver running in the Hyper-V parent partition. + * @pdev: The PCI driver's representation for this device. + * @buf: Buffer into which the config block will be copied. + * @len: Size in bytes of buf. + * @block_id: Identifies the config block which has been requested. + * @bytes_returned: Size which came back from the back-end driver. + * + * Return: 0 on success, -errno on failure + */ +static int hv_read_config_block(struct pci_dev *pdev, void *buf, + unsigned int len, unsigned int block_id, + unsigned int *bytes_returned) +{ + struct hv_pcibus_device *hbus = + container_of(pdev->bus->sysdata, struct hv_pcibus_device, + sysdata); + struct { + struct pci_packet pkt; + char buf[sizeof(struct pci_read_block)]; + } pkt; + struct hv_read_config_compl comp_pkt; + struct pci_read_block *read_blk; + int ret; + + if (len == 0 || len > HV_CONFIG_BLOCK_SIZE_MAX) + return -EINVAL; + + init_completion(&comp_pkt.comp_pkt.host_event); + comp_pkt.buf = buf; + comp_pkt.len = len; + + memset(&pkt, 0, sizeof(pkt)); + pkt.pkt.completion_func = hv_pci_read_config_compl; + pkt.pkt.compl_ctxt = &comp_pkt; + read_blk = (struct pci_read_block *)pkt.buf; + read_blk->message_type.type = PCI_READ_BLOCK; + read_blk->wslot.slot = devfn_to_wslot(pdev->devfn); + read_blk->block_id = block_id; + read_blk->bytes_requested = len; + + ret = vmbus_sendpacket(hbus->hdev->channel, read_blk, + sizeof(*read_blk), (unsigned long)&pkt.pkt, + VM_PKT_DATA_INBAND, + VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); + if (ret) + return ret; + + ret = wait_for_response(hbus->hdev, &comp_pkt.comp_pkt.host_event); + if (ret) + return ret; + + if (comp_pkt.comp_pkt.completion_status != 0 || + comp_pkt.bytes_returned == 0) { + dev_err(&hbus->hdev->device, + "Read Config Block failed: 0x%x, bytes_returned=%d\n", + comp_pkt.comp_pkt.completion_status, + comp_pkt.bytes_returned); + return -EIO; + } + + *bytes_returned = comp_pkt.bytes_returned; + return 0; +} + +/** + * hv_pci_write_config_compl() - Invoked when a response packet for a write + * config block operation arrives. + * @context: Identifies the write config operation + * @resp: The response packet itself + * @resp_packet_size: Size in bytes of the response packet + */ +static void hv_pci_write_config_compl(void *context, struct pci_response *resp, + int resp_packet_size) +{ + struct hv_pci_compl *comp_pkt = context; + + comp_pkt->completion_status = resp->status; + complete(&comp_pkt->host_event); +} + +/** + * hv_write_config_block() - Sends a write config block request to the + * back-end driver running in the Hyper-V parent partition. + * @pdev: The PCI driver's representation for this device. + * @buf: Buffer from which the config block will be copied. + * @len: Size in bytes of buf. + * @block_id: Identifies the config block which is being written. + * + * Return: 0 on success, -errno on failure + */ +static int hv_write_config_block(struct pci_dev *pdev, void *buf, + unsigned int len, unsigned int block_id) +{ + struct hv_pcibus_device *hbus = + container_of(pdev->bus->sysdata, struct hv_pcibus_device, + sysdata); + struct { + struct pci_packet pkt; + char buf[sizeof(struct pci_write_block)]; + u32 reserved; + } pkt; + struct hv_pci_compl comp_pkt; + struct pci_write_block *write_blk; + u32 pkt_size; + int ret; + + if (len == 0 || len > HV_CONFIG_BLOCK_SIZE_MAX) + return -EINVAL; + + init_completion(&comp_pkt.host_event); + + memset(&pkt, 0, sizeof(pkt)); + pkt.pkt.completion_func = hv_pci_write_config_compl; + pkt.pkt.compl_ctxt = &comp_pkt; + write_blk = (struct pci_write_block *)pkt.buf; + write_blk->message_type.type = PCI_WRITE_BLOCK; + write_blk->wslot.slot = devfn_to_wslot(pdev->devfn); + write_blk->block_id = block_id; + write_blk->byte_count = len; + memcpy(write_blk->bytes, buf, len); + pkt_size = offsetof(struct pci_write_block, bytes) + len; + /* + * This quirk is required on some hosts shipped around 2018, because + * these hosts don't check the pkt_size correctly (new hosts have been + * fixed since early 2019). The quirk is also safe on very old hosts + * and new hosts, because, on them, what really matters is the length + * specified in write_blk->byte_count. + */ + pkt_size += sizeof(pkt.reserved); + + ret = vmbus_sendpacket(hbus->hdev->channel, write_blk, pkt_size, + (unsigned long)&pkt.pkt, VM_PKT_DATA_INBAND, + VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); + if (ret) + return ret; + + ret = wait_for_response(hbus->hdev, &comp_pkt.host_event); + if (ret) + return ret; + + if (comp_pkt.completion_status != 0) { + dev_err(&hbus->hdev->device, + "Write Config Block failed: 0x%x\n", + comp_pkt.completion_status); + return -EIO; + } + + return 0; +} + +/** + * hv_register_block_invalidate() - Invoked when a config block invalidation + * arrives from the back-end driver. + * @pdev: The PCI driver's representation for this device. + * @context: Identifies the device. + * @block_invalidate: Identifies all of the blocks being invalidated. + * + * Return: 0 on success, -errno on failure + */ +static int hv_register_block_invalidate(struct pci_dev *pdev, void *context, + void (*block_invalidate)(void *context, + u64 block_mask)) +{ + struct hv_pcibus_device *hbus = + container_of(pdev->bus->sysdata, struct hv_pcibus_device, + sysdata); + struct hv_pci_dev *hpdev; + + hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(pdev->devfn)); + if (!hpdev) + return -ENODEV; + + hpdev->block_invalidate = block_invalidate; + hpdev->invalidate_context = context; + + put_pcichild(hpdev); + return 0; + +} + +/* Interrupt management hooks */ +static void hv_int_desc_free(struct hv_pci_dev *hpdev, + struct tran_int_desc *int_desc) +{ + struct pci_delete_interrupt *int_pkt; + struct { + struct pci_packet pkt; + u8 buffer[sizeof(struct pci_delete_interrupt)]; + } ctxt; + + if (!int_desc->vector_count) { + kfree(int_desc); + return; + } + memset(&ctxt, 0, sizeof(ctxt)); + int_pkt = (struct pci_delete_interrupt *)ctxt.buffer; + int_pkt->message_type.type = + PCI_DELETE_INTERRUPT_MESSAGE; + int_pkt->wslot.slot = hpdev->desc.win_slot.slot; + int_pkt->int_desc = *int_desc; + vmbus_sendpacket(hpdev->hbus->hdev->channel, int_pkt, sizeof(*int_pkt), + 0, VM_PKT_DATA_INBAND, 0); + kfree(int_desc); +} + +/** + * hv_msi_free() - Free the MSI. + * @domain: The interrupt domain pointer + * @irq: Identifies the IRQ. + * + * The Hyper-V parent partition and hypervisor are tracking the + * messages that are in use, keeping the interrupt redirection + * table up to date. This callback sends a message that frees + * the IRT entry and related tracking nonsense. + */ +static void hv_msi_free(struct irq_domain *domain, unsigned int irq) +{ + struct hv_pcibus_device *hbus; + struct hv_pci_dev *hpdev; + struct pci_dev *pdev; + struct tran_int_desc *int_desc; + struct irq_data *irq_data = irq_domain_get_irq_data(domain, irq); + struct msi_desc *msi = irq_data_get_msi_desc(irq_data); + + pdev = msi_desc_to_pci_dev(msi); + hbus = domain->host_data; + int_desc = irq_data_get_irq_chip_data(irq_data); + if (!int_desc) + return; + + irq_data->chip_data = NULL; + hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(pdev->devfn)); + if (!hpdev) { + kfree(int_desc); + return; + } + + hv_int_desc_free(hpdev, int_desc); + put_pcichild(hpdev); +} + +static void hv_irq_mask(struct irq_data *data) +{ + if (data->parent_data->chip->irq_mask) + irq_chip_mask_parent(data); +} + +static void hv_irq_unmask(struct irq_data *data) +{ + hv_arch_irq_unmask(data); + + if (data->parent_data->chip->irq_unmask) + irq_chip_unmask_parent(data); +} + +struct compose_comp_ctxt { + struct hv_pci_compl comp_pkt; + struct tran_int_desc int_desc; +}; + +static void hv_pci_compose_compl(void *context, struct pci_response *resp, + int resp_packet_size) +{ + struct compose_comp_ctxt *comp_pkt = context; + struct pci_create_int_response *int_resp = + (struct pci_create_int_response *)resp; + + if (resp_packet_size < sizeof(*int_resp)) { + comp_pkt->comp_pkt.completion_status = -1; + goto out; + } + comp_pkt->comp_pkt.completion_status = resp->status; + comp_pkt->int_desc = int_resp->int_desc; +out: + complete(&comp_pkt->comp_pkt.host_event); +} + +static u32 hv_compose_msi_req_v1( + struct pci_create_interrupt *int_pkt, + u32 slot, u8 vector, u16 vector_count) +{ + int_pkt->message_type.type = PCI_CREATE_INTERRUPT_MESSAGE; + int_pkt->wslot.slot = slot; + int_pkt->int_desc.vector = vector; + int_pkt->int_desc.vector_count = vector_count; + int_pkt->int_desc.delivery_mode = DELIVERY_MODE; + + /* + * Create MSI w/ dummy vCPU set, overwritten by subsequent retarget in + * hv_irq_unmask(). + */ + int_pkt->int_desc.cpu_mask = CPU_AFFINITY_ALL; + + return sizeof(*int_pkt); +} + +/* + * The vCPU selected by hv_compose_multi_msi_req_get_cpu() and + * hv_compose_msi_req_get_cpu() is a "dummy" vCPU because the final vCPU to be + * interrupted is specified later in hv_irq_unmask() and communicated to Hyper-V + * via the HVCALL_RETARGET_INTERRUPT hypercall. But the choice of dummy vCPU is + * not irrelevant because Hyper-V chooses the physical CPU to handle the + * interrupts based on the vCPU specified in message sent to the vPCI VSP in + * hv_compose_msi_msg(). Hyper-V's choice of pCPU is not visible to the guest, + * but assigning too many vPCI device interrupts to the same pCPU can cause a + * performance bottleneck. So we spread out the dummy vCPUs to influence Hyper-V + * to spread out the pCPUs that it selects. + * + * For the single-MSI and MSI-X cases, it's OK for hv_compose_msi_req_get_cpu() + * to always return the same dummy vCPU, because a second call to + * hv_compose_msi_msg() contains the "real" vCPU, causing Hyper-V to choose a + * new pCPU for the interrupt. But for the multi-MSI case, the second call to + * hv_compose_msi_msg() exits without sending a message to the vPCI VSP, so the + * original dummy vCPU is used. This dummy vCPU must be round-robin'ed so that + * the pCPUs are spread out. All interrupts for a multi-MSI device end up using + * the same pCPU, even though the vCPUs will be spread out by later calls + * to hv_irq_unmask(), but that is the best we can do now. + * + * With Hyper-V in Nov 2022, the HVCALL_RETARGET_INTERRUPT hypercall does *not* + * cause Hyper-V to reselect the pCPU based on the specified vCPU. Such an + * enhancement is planned for a future version. With that enhancement, the + * dummy vCPU selection won't matter, and interrupts for the same multi-MSI + * device will be spread across multiple pCPUs. + */ + +/* + * Create MSI w/ dummy vCPU set targeting just one vCPU, overwritten + * by subsequent retarget in hv_irq_unmask(). + */ +static int hv_compose_msi_req_get_cpu(const struct cpumask *affinity) +{ + return cpumask_first_and(affinity, cpu_online_mask); +} + +/* + * Make sure the dummy vCPU values for multi-MSI don't all point to vCPU0. + */ +static int hv_compose_multi_msi_req_get_cpu(void) +{ + static DEFINE_SPINLOCK(multi_msi_cpu_lock); + + /* -1 means starting with CPU 0 */ + static int cpu_next = -1; + + unsigned long flags; + int cpu; + + spin_lock_irqsave(&multi_msi_cpu_lock, flags); + + cpu_next = cpumask_next_wrap(cpu_next, cpu_online_mask); + cpu = cpu_next; + + spin_unlock_irqrestore(&multi_msi_cpu_lock, flags); + + return cpu; +} + +static u32 hv_compose_msi_req_v2( + struct pci_create_interrupt2 *int_pkt, int cpu, + u32 slot, u8 vector, u16 vector_count) +{ + int_pkt->message_type.type = PCI_CREATE_INTERRUPT_MESSAGE2; + int_pkt->wslot.slot = slot; + int_pkt->int_desc.vector = vector; + int_pkt->int_desc.vector_count = vector_count; + int_pkt->int_desc.delivery_mode = DELIVERY_MODE; + int_pkt->int_desc.processor_array[0] = + hv_cpu_number_to_vp_number(cpu); + int_pkt->int_desc.processor_count = 1; + + return sizeof(*int_pkt); +} + +static u32 hv_compose_msi_req_v3( + struct pci_create_interrupt3 *int_pkt, int cpu, + u32 slot, u32 vector, u16 vector_count) +{ + int_pkt->message_type.type = PCI_CREATE_INTERRUPT_MESSAGE3; + int_pkt->wslot.slot = slot; + int_pkt->int_desc.vector = vector; + int_pkt->int_desc.reserved = 0; + int_pkt->int_desc.vector_count = vector_count; + int_pkt->int_desc.delivery_mode = DELIVERY_MODE; + int_pkt->int_desc.processor_array[0] = + hv_cpu_number_to_vp_number(cpu); + int_pkt->int_desc.processor_count = 1; + + return sizeof(*int_pkt); +} + +/** + * hv_compose_msi_msg() - Supplies a valid MSI address/data + * @data: Everything about this MSI + * @msg: Buffer that is filled in by this function + * + * This function unpacks the IRQ looking for target CPU set, IDT + * vector and mode and sends a message to the parent partition + * asking for a mapping for that tuple in this partition. The + * response supplies a data value and address to which that data + * should be written to trigger that interrupt. + */ +static void hv_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct hv_pcibus_device *hbus; + struct vmbus_channel *channel; + struct hv_pci_dev *hpdev; + struct pci_bus *pbus; + struct pci_dev *pdev; + const struct cpumask *dest; + struct compose_comp_ctxt comp; + struct tran_int_desc *int_desc; + struct msi_desc *msi_desc; + /* + * vector_count should be u16: see hv_msi_desc, hv_msi_desc2 + * and hv_msi_desc3. vector must be u32: see hv_msi_desc3. + */ + u16 vector_count; + u32 vector; + struct { + struct pci_packet pci_pkt; + union { + struct pci_create_interrupt v1; + struct pci_create_interrupt2 v2; + struct pci_create_interrupt3 v3; + } int_pkts; + } __packed ctxt; + bool multi_msi; + u64 trans_id; + u32 size; + int ret; + int cpu; + + msi_desc = irq_data_get_msi_desc(data); + multi_msi = !msi_desc->pci.msi_attrib.is_msix && + msi_desc->nvec_used > 1; + + /* Reuse the previous allocation */ + if (data->chip_data && multi_msi) { + int_desc = data->chip_data; + msg->address_hi = int_desc->address >> 32; + msg->address_lo = int_desc->address & 0xffffffff; + msg->data = int_desc->data; + return; + } + + pdev = msi_desc_to_pci_dev(msi_desc); + dest = irq_data_get_effective_affinity_mask(data); + pbus = pdev->bus; + hbus = container_of(pbus->sysdata, struct hv_pcibus_device, sysdata); + channel = hbus->hdev->channel; + hpdev = get_pcichild_wslot(hbus, devfn_to_wslot(pdev->devfn)); + if (!hpdev) + goto return_null_message; + + /* Free any previous message that might have already been composed. */ + if (data->chip_data && !multi_msi) { + int_desc = data->chip_data; + data->chip_data = NULL; + hv_int_desc_free(hpdev, int_desc); + } + + int_desc = kzalloc_obj(*int_desc, GFP_ATOMIC); + if (!int_desc) + goto drop_reference; + + if (multi_msi) { + /* + * If this is not the first MSI of Multi MSI, we already have + * a mapping. Can exit early. + */ + if (msi_desc->irq != data->irq) { + data->chip_data = int_desc; + int_desc->address = msi_desc->msg.address_lo | + (u64)msi_desc->msg.address_hi << 32; + int_desc->data = msi_desc->msg.data + + (data->irq - msi_desc->irq); + msg->address_hi = msi_desc->msg.address_hi; + msg->address_lo = msi_desc->msg.address_lo; + msg->data = int_desc->data; + put_pcichild(hpdev); + return; + } + /* + * The vector we select here is a dummy value. The correct + * value gets sent to the hypervisor in unmask(). This needs + * to be aligned with the count, and also not zero. Multi-msi + * is powers of 2 up to 32, so 32 will always work here. + */ + vector = 32; + vector_count = msi_desc->nvec_used; + cpu = hv_compose_multi_msi_req_get_cpu(); + } else { + vector = hv_msi_get_int_vector(data); + vector_count = 1; + cpu = hv_compose_msi_req_get_cpu(dest); + } + + /* + * hv_compose_msi_req_v1 and v2 are for x86 only, meaning 'vector' + * can't exceed u8. Cast 'vector' down to u8 for v1/v2 explicitly + * for better readability. + */ + memset(&ctxt, 0, sizeof(ctxt)); + init_completion(&comp.comp_pkt.host_event); + ctxt.pci_pkt.completion_func = hv_pci_compose_compl; + ctxt.pci_pkt.compl_ctxt = ∁ + + switch (hbus->protocol_version) { + case PCI_PROTOCOL_VERSION_1_1: + size = hv_compose_msi_req_v1(&ctxt.int_pkts.v1, + hpdev->desc.win_slot.slot, + (u8)vector, + vector_count); + break; + + case PCI_PROTOCOL_VERSION_1_2: + case PCI_PROTOCOL_VERSION_1_3: + size = hv_compose_msi_req_v2(&ctxt.int_pkts.v2, + cpu, + hpdev->desc.win_slot.slot, + (u8)vector, + vector_count); + break; + + case PCI_PROTOCOL_VERSION_1_4: + size = hv_compose_msi_req_v3(&ctxt.int_pkts.v3, + cpu, + hpdev->desc.win_slot.slot, + vector, + vector_count); + break; + + default: + /* As we only negotiate protocol versions known to this driver, + * this path should never hit. However, this is it not a hot + * path so we print a message to aid future updates. + */ + dev_err(&hbus->hdev->device, + "Unexpected vPCI protocol, update driver."); + goto free_int_desc; + } + + ret = vmbus_sendpacket_getid(hpdev->hbus->hdev->channel, &ctxt.int_pkts, + size, (unsigned long)&ctxt.pci_pkt, + &trans_id, VM_PKT_DATA_INBAND, + VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); + if (ret) { + dev_err(&hbus->hdev->device, + "Sending request for interrupt failed: 0x%x", + comp.comp_pkt.completion_status); + goto free_int_desc; + } + + /* + * Prevents hv_pci_onchannelcallback() from running concurrently + * in the tasklet. + */ + tasklet_disable_in_atomic(&channel->callback_event); + + /* + * Since this function is called with IRQ locks held, can't + * do normal wait for completion; instead poll. + */ + while (!try_wait_for_completion(&comp.comp_pkt.host_event)) { + unsigned long flags; + + /* 0xFFFF means an invalid PCI VENDOR ID. */ + if (hv_pcifront_get_vendor_id(hpdev) == 0xFFFF) { + dev_err_once(&hbus->hdev->device, + "the device has gone\n"); + goto enable_tasklet; + } + + /* + * Make sure that the ring buffer data structure doesn't get + * freed while we dereference the ring buffer pointer. Test + * for the channel's onchannel_callback being NULL within a + * sched_lock critical section. See also the inline comments + * in vmbus_reset_channel_cb(). + */ + spin_lock_irqsave(&channel->sched_lock, flags); + if (unlikely(channel->onchannel_callback == NULL)) { + spin_unlock_irqrestore(&channel->sched_lock, flags); + goto enable_tasklet; + } + hv_pci_onchannelcallback(hbus); + spin_unlock_irqrestore(&channel->sched_lock, flags); + + udelay(100); + } + + tasklet_enable(&channel->callback_event); + + if (comp.comp_pkt.completion_status < 0) { + dev_err(&hbus->hdev->device, + "Request for interrupt failed: 0x%x", + comp.comp_pkt.completion_status); + goto free_int_desc; + } + + /* + * Record the assignment so that this can be unwound later. Using + * irq_set_chip_data() here would be appropriate, but the lock it takes + * is already held. + */ + *int_desc = comp.int_desc; + data->chip_data = int_desc; + + /* Pass up the result. */ + msg->address_hi = comp.int_desc.address >> 32; + msg->address_lo = comp.int_desc.address & 0xffffffff; + msg->data = comp.int_desc.data; + + put_pcichild(hpdev); + return; + +enable_tasklet: + tasklet_enable(&channel->callback_event); + /* + * The completion packet on the stack becomes invalid after 'return'; + * remove the ID from the VMbus requestor if the identifier is still + * mapped to/associated with the packet. (The identifier could have + * been 're-used', i.e., already removed and (re-)mapped.) + * + * Cf. hv_pci_onchannelcallback(). + */ + vmbus_request_addr_match(channel, trans_id, (unsigned long)&ctxt.pci_pkt); +free_int_desc: + kfree(int_desc); +drop_reference: + put_pcichild(hpdev); +return_null_message: + msg->address_hi = 0; + msg->address_lo = 0; + msg->data = 0; +} + +static bool hv_pcie_init_dev_msi_info(struct device *dev, struct irq_domain *domain, + struct irq_domain *real_parent, struct msi_domain_info *info) +{ + struct irq_chip *chip = info->chip; + + if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info)) + return false; + + info->ops->msi_prepare = hv_msi_prepare; + + chip->irq_set_affinity = irq_chip_set_affinity_parent; + chip->irq_retrigger = irq_chip_retrigger_hierarchy; + + if (IS_ENABLED(CONFIG_X86)) + chip->flags |= IRQCHIP_MOVE_DEFERRED; + + return true; +} + +#define HV_PCIE_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_PCI_MSI_MASK_PARENT) +#define HV_PCIE_MSI_FLAGS_SUPPORTED (MSI_FLAG_MULTI_PCI_MSI | \ + MSI_FLAG_PCI_MSIX | \ + MSI_FLAG_PCI_MSIX_ALLOC_DYN | \ + MSI_GENERIC_FLAGS_MASK) + +static const struct msi_parent_ops hv_pcie_msi_parent_ops = { + .required_flags = HV_PCIE_MSI_FLAGS_REQUIRED, + .supported_flags = HV_PCIE_MSI_FLAGS_SUPPORTED, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .chip_flags = HV_MSI_CHIP_FLAGS, + .prefix = "HV-", + .init_dev_msi_info = hv_pcie_init_dev_msi_info, +}; + +/* HW Interrupt Chip Descriptor */ +static struct irq_chip hv_msi_irq_chip = { + .name = "Hyper-V PCIe MSI", + .irq_compose_msi_msg = hv_compose_msi_msg, + .irq_set_affinity = irq_chip_set_affinity_parent, + .irq_ack = irq_chip_ack_parent, + .irq_eoi = irq_chip_eoi_parent, + .irq_mask = hv_irq_mask, + .irq_unmask = hv_irq_unmask, +}; + +static int hv_pcie_domain_alloc(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs, + void *arg) +{ + /* + * TODO: Allocating and populating struct tran_int_desc in hv_compose_msi_msg() + * should be moved here. + */ + int ret; + + ret = irq_domain_alloc_irqs_parent(d, virq, nr_irqs, arg); + if (ret < 0) + return ret; + + for (int i = 0; i < nr_irqs; i++) { + irq_domain_set_hwirq_and_chip(d, virq + i, 0, &hv_msi_irq_chip, NULL); + if (IS_ENABLED(CONFIG_X86)) + __irq_set_handler(virq + i, handle_edge_irq, 0, "edge"); + } + + return 0; +} + +static void hv_pcie_domain_free(struct irq_domain *d, unsigned int virq, unsigned int nr_irqs) +{ + for (int i = 0; i < nr_irqs; i++) + hv_msi_free(d, virq + i); + + irq_domain_free_irqs_top(d, virq, nr_irqs); +} + +static const struct irq_domain_ops hv_pcie_domain_ops = { + .alloc = hv_pcie_domain_alloc, + .free = hv_pcie_domain_free, +}; + +/** + * hv_pcie_init_irq_domain() - Initialize IRQ domain + * @hbus: The root PCI bus + * + * This function creates an IRQ domain which will be used for + * interrupts from devices that have been passed through. These + * devices only support MSI and MSI-X, not line-based interrupts + * or simulations of line-based interrupts through PCIe's + * fabric-layer messages. Because interrupts are remapped, we + * can support multi-message MSI here. + * + * Return: '0' on success and error value on failure + */ +static int hv_pcie_init_irq_domain(struct hv_pcibus_device *hbus) +{ + struct irq_domain_info info = { + .fwnode = hbus->fwnode, + .ops = &hv_pcie_domain_ops, + .host_data = hbus, + .parent = hv_pci_get_root_domain(), + }; + + hbus->irq_domain = msi_create_parent_irq_domain(&info, &hv_pcie_msi_parent_ops); + if (!hbus->irq_domain) { + dev_err(&hbus->hdev->device, + "Failed to build an MSI IRQ domain\n"); + return -ENODEV; + } + + dev_set_msi_domain(&hbus->bridge->dev, hbus->irq_domain); + + return 0; +} + +/** + * get_bar_size() - Get the address space consumed by a BAR + * @bar_val: Value that a BAR returned after -1 was written + * to it. + * + * This function returns the size of the BAR, rounded up to 1 + * page. It has to be rounded up because the hypervisor's page + * table entry that maps the BAR into the VM can't specify an + * offset within a page. The invariant is that the hypervisor + * must place any BARs of smaller than page length at the + * beginning of a page. + * + * Return: Size in bytes of the consumed MMIO space. + */ +static u64 get_bar_size(u64 bar_val) +{ + return round_up((1 + ~(bar_val & PCI_BASE_ADDRESS_MEM_MASK)), + PAGE_SIZE); +} + +/** + * survey_child_resources() - Total all MMIO requirements + * @hbus: Root PCI bus, as understood by this driver + */ +static void survey_child_resources(struct hv_pcibus_device *hbus) +{ + struct hv_pci_dev *hpdev; + resource_size_t bar_size = 0; + unsigned long flags; + struct completion *event; + u64 bar_val; + int i; + + /* If nobody is waiting on the answer, don't compute it. */ + event = xchg(&hbus->survey_event, NULL); + if (!event) + return; + + /* If the answer has already been computed, go with it. */ + if (hbus->low_mmio_space || hbus->high_mmio_space) { + complete(event); + return; + } + + spin_lock_irqsave(&hbus->device_list_lock, flags); + + /* + * Due to an interesting quirk of the PCI spec, all memory regions + * for a child device are a power of 2 in size and aligned in memory, + * so it's sufficient to just add them up without tracking alignment. + */ + list_for_each_entry(hpdev, &hbus->children, list_entry) { + for (i = 0; i < PCI_STD_NUM_BARS; i++) { + if (hpdev->probed_bar[i] & PCI_BASE_ADDRESS_SPACE_IO) + dev_err(&hbus->hdev->device, + "There's an I/O BAR in this list!\n"); + + if (hpdev->probed_bar[i] != 0) { + /* + * A probed BAR has all the upper bits set that + * can be changed. + */ + + bar_val = hpdev->probed_bar[i]; + if (bar_val & PCI_BASE_ADDRESS_MEM_TYPE_64) + bar_val |= + ((u64)hpdev->probed_bar[++i] << 32); + else + bar_val |= 0xffffffff00000000ULL; + + bar_size = get_bar_size(bar_val); + + if (bar_val & PCI_BASE_ADDRESS_MEM_TYPE_64) + hbus->high_mmio_space += bar_size; + else + hbus->low_mmio_space += bar_size; + } + } + } + + spin_unlock_irqrestore(&hbus->device_list_lock, flags); + complete(event); +} + +/** + * prepopulate_bars() - Fill in BARs with defaults + * @hbus: Root PCI bus, as understood by this driver + * + * The core PCI driver code seems much, much happier if the BARs + * for a device have values upon first scan. So fill them in. + * The algorithm below works down from large sizes to small, + * attempting to pack the assignments optimally. The assumption, + * enforced in other parts of the code, is that the beginning of + * the memory-mapped I/O space will be aligned on the largest + * BAR size. + */ +static void prepopulate_bars(struct hv_pcibus_device *hbus) +{ + resource_size_t high_size = 0; + resource_size_t low_size = 0; + resource_size_t high_base = 0; + resource_size_t low_base = 0; + resource_size_t bar_size; + struct hv_pci_dev *hpdev; + unsigned long flags; + u64 bar_val; + u32 command; + bool high; + int i; + + if (hbus->low_mmio_space) { + low_size = 1ULL << (63 - __builtin_clzll(hbus->low_mmio_space)); + low_base = hbus->low_mmio_res->start; + } + + if (hbus->high_mmio_space) { + high_size = 1ULL << + (63 - __builtin_clzll(hbus->high_mmio_space)); + high_base = hbus->high_mmio_res->start; + } + + spin_lock_irqsave(&hbus->device_list_lock, flags); + + /* + * Clear the memory enable bit, in case it's already set. This occurs + * in the suspend path of hibernation, where the device is suspended, + * resumed and suspended again: see hibernation_snapshot() and + * hibernation_platform_enter(). + * + * If the memory enable bit is already set, Hyper-V silently ignores + * the below BAR updates, and the related PCI device driver can not + * work, because reading from the device register(s) always returns + * 0xFFFFFFFF (PCI_ERROR_RESPONSE). + */ + list_for_each_entry(hpdev, &hbus->children, list_entry) { + _hv_pcifront_read_config(hpdev, PCI_COMMAND, 2, &command); + command &= ~PCI_COMMAND_MEMORY; + _hv_pcifront_write_config(hpdev, PCI_COMMAND, 2, command); + } + + /* Pick addresses for the BARs. */ + do { + list_for_each_entry(hpdev, &hbus->children, list_entry) { + for (i = 0; i < PCI_STD_NUM_BARS; i++) { + bar_val = hpdev->probed_bar[i]; + if (bar_val == 0) + continue; + high = bar_val & PCI_BASE_ADDRESS_MEM_TYPE_64; + if (high) { + bar_val |= + ((u64)hpdev->probed_bar[i + 1] + << 32); + } else { + bar_val |= 0xffffffffULL << 32; + } + bar_size = get_bar_size(bar_val); + if (high) { + if (high_size != bar_size) { + i++; + continue; + } + _hv_pcifront_write_config(hpdev, + PCI_BASE_ADDRESS_0 + (4 * i), + 4, + (u32)(high_base & 0xffffff00)); + i++; + _hv_pcifront_write_config(hpdev, + PCI_BASE_ADDRESS_0 + (4 * i), + 4, (u32)(high_base >> 32)); + high_base += bar_size; + } else { + if (low_size != bar_size) + continue; + _hv_pcifront_write_config(hpdev, + PCI_BASE_ADDRESS_0 + (4 * i), + 4, + (u32)(low_base & 0xffffff00)); + low_base += bar_size; + } + } + if (high_size <= 1 && low_size <= 1) { + /* + * No need to set the PCI_COMMAND_MEMORY bit as + * the core PCI driver doesn't require the bit + * to be pre-set. Actually here we intentionally + * keep the bit off so that the PCI BAR probing + * in the core PCI driver doesn't cause Hyper-V + * to unnecessarily unmap/map the virtual BARs + * from/to the physical BARs multiple times. + * This reduces the VM boot time significantly + * if the BAR sizes are huge. + */ + break; + } + } + + high_size >>= 1; + low_size >>= 1; + } while (high_size || low_size); + + spin_unlock_irqrestore(&hbus->device_list_lock, flags); +} + +/* + * Assign entries in sysfs pci slot directory. + * + * Note that this function does not need to lock the children list + * because it is called from pci_devices_present_work which + * is serialized with hv_eject_device_work because they are on the + * same ordered workqueue. Therefore hbus->children list will not change + * even when pci_create_slot sleeps. + */ +static void hv_pci_assign_slots(struct hv_pcibus_device *hbus) +{ + struct hv_pci_dev *hpdev; + char name[SLOT_NAME_SIZE]; + int slot_nr; + + list_for_each_entry(hpdev, &hbus->children, list_entry) { + if (hpdev->pci_slot) + continue; + + slot_nr = PCI_SLOT(wslot_to_devfn(hpdev->desc.win_slot.slot)); + snprintf(name, SLOT_NAME_SIZE, "%u", hpdev->desc.ser); + hpdev->pci_slot = pci_create_slot(hbus->bridge->bus, slot_nr, + name, NULL); + if (IS_ERR(hpdev->pci_slot)) { + pr_warn("pci_create slot %s failed\n", name); + hpdev->pci_slot = NULL; + } + } +} + +/* + * Remove entries in sysfs pci slot directory. + */ +static void hv_pci_remove_slots(struct hv_pcibus_device *hbus) +{ + struct hv_pci_dev *hpdev; + + list_for_each_entry(hpdev, &hbus->children, list_entry) { + if (!hpdev->pci_slot) + continue; + pci_destroy_slot(hpdev->pci_slot); + hpdev->pci_slot = NULL; + } +} + +/* + * Set NUMA node for the devices on the bus + */ +static void hv_pci_assign_numa_node(struct hv_pcibus_device *hbus) +{ + struct pci_dev *dev; + struct pci_bus *bus = hbus->bridge->bus; + struct hv_pci_dev *hv_dev; + + list_for_each_entry(dev, &bus->devices, bus_list) { + hv_dev = get_pcichild_wslot(hbus, devfn_to_wslot(dev->devfn)); + if (!hv_dev) + continue; + + /* + * If the Hyper-V host doesn't provide a NUMA node for the + * device, default to node 0. With NUMA_NO_NODE the kernel + * may spread work across NUMA nodes, which degrades + * performance on Hyper-V. + */ + set_dev_node(&dev->dev, 0); + + if (hv_dev->desc.flags & HV_PCI_DEVICE_FLAG_NUMA_AFFINITY && + hv_dev->desc.virtual_numa_node < num_possible_nodes()) + /* + * The kernel may boot with some NUMA nodes offline + * (e.g. in a KDUMP kernel) or with NUMA disabled via + * "numa=off". In those cases, adjust the host provided + * NUMA node to a valid NUMA node used by the kernel. + */ + set_dev_node(&dev->dev, + numa_map_to_online_node( + hv_dev->desc.virtual_numa_node)); + + put_pcichild(hv_dev); + } +} + +/** + * create_root_hv_pci_bus() - Expose a new root PCI bus + * @hbus: Root PCI bus, as understood by this driver + * + * Return: 0 on success, -errno on failure + */ +static int create_root_hv_pci_bus(struct hv_pcibus_device *hbus) +{ + int error; + struct pci_host_bridge *bridge = hbus->bridge; + + bridge->dev.parent = &hbus->hdev->device; + bridge->sysdata = &hbus->sysdata; + bridge->ops = &hv_pcifront_ops; + + error = pci_scan_root_bus_bridge(bridge); + if (error) + return error; + + pci_lock_rescan_remove(); + hv_pci_assign_numa_node(hbus); + pci_bus_assign_resources(bridge->bus); + hv_pci_assign_slots(hbus); + pci_bus_add_devices(bridge->bus); + pci_unlock_rescan_remove(); + hbus->state = hv_pcibus_installed; + return 0; +} + +struct q_res_req_compl { + struct completion host_event; + struct hv_pci_dev *hpdev; +}; + +/** + * q_resource_requirements() - Query Resource Requirements + * @context: The completion context. + * @resp: The response that came from the host. + * @resp_packet_size: The size in bytes of resp. + * + * This function is invoked on completion of a Query Resource + * Requirements packet. + */ +static void q_resource_requirements(void *context, struct pci_response *resp, + int resp_packet_size) +{ + struct q_res_req_compl *completion = context; + struct pci_q_res_req_response *q_res_req = + (struct pci_q_res_req_response *)resp; + s32 status; + int i; + + status = (resp_packet_size < sizeof(*q_res_req)) ? -1 : resp->status; + if (status < 0) { + dev_err(&completion->hpdev->hbus->hdev->device, + "query resource requirements failed: %x\n", + status); + } else { + for (i = 0; i < PCI_STD_NUM_BARS; i++) { + completion->hpdev->probed_bar[i] = + q_res_req->probed_bar[i]; + } + } + + complete(&completion->host_event); +} + +/** + * new_pcichild_device() - Create a new child device + * @hbus: The internal struct tracking this root PCI bus. + * @desc: The information supplied so far from the host + * about the device. + * + * This function creates the tracking structure for a new child + * device and kicks off the process of figuring out what it is. + * + * Return: Pointer to the new tracking struct + */ +static struct hv_pci_dev *new_pcichild_device(struct hv_pcibus_device *hbus, + struct hv_pcidev_description *desc) +{ + struct hv_pci_dev *hpdev; + struct pci_child_message *res_req; + struct q_res_req_compl comp_pkt; + struct { + struct pci_packet init_packet; + u8 buffer[sizeof(struct pci_child_message)]; + } pkt; + unsigned long flags; + int ret; + + hpdev = kzalloc_obj(*hpdev); + if (!hpdev) + return NULL; + + hpdev->hbus = hbus; + + memset(&pkt, 0, sizeof(pkt)); + init_completion(&comp_pkt.host_event); + comp_pkt.hpdev = hpdev; + pkt.init_packet.compl_ctxt = &comp_pkt; + pkt.init_packet.completion_func = q_resource_requirements; + res_req = (struct pci_child_message *)pkt.buffer; + res_req->message_type.type = PCI_QUERY_RESOURCE_REQUIREMENTS; + res_req->wslot.slot = desc->win_slot.slot; + + ret = vmbus_sendpacket(hbus->hdev->channel, res_req, + sizeof(struct pci_child_message), + (unsigned long)&pkt.init_packet, + VM_PKT_DATA_INBAND, + VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); + if (ret) + goto error; + + if (wait_for_response(hbus->hdev, &comp_pkt.host_event)) + goto error; + + hpdev->desc = *desc; + refcount_set(&hpdev->refs, 1); + get_pcichild(hpdev); + spin_lock_irqsave(&hbus->device_list_lock, flags); + + list_add_tail(&hpdev->list_entry, &hbus->children); + spin_unlock_irqrestore(&hbus->device_list_lock, flags); + return hpdev; + +error: + kfree(hpdev); + return NULL; +} + +/** + * get_pcichild_wslot() - Find device from slot + * @hbus: Root PCI bus, as understood by this driver + * @wslot: Location on the bus + * + * This function looks up a PCI device and returns the internal + * representation of it. It acquires a reference on it, so that + * the device won't be deleted while somebody is using it. The + * caller is responsible for calling put_pcichild() to release + * this reference. + * + * Return: Internal representation of a PCI device + */ +static struct hv_pci_dev *get_pcichild_wslot(struct hv_pcibus_device *hbus, + u32 wslot) +{ + unsigned long flags; + struct hv_pci_dev *iter, *hpdev = NULL; + + spin_lock_irqsave(&hbus->device_list_lock, flags); + list_for_each_entry(iter, &hbus->children, list_entry) { + if (iter->desc.win_slot.slot == wslot) { + hpdev = iter; + get_pcichild(hpdev); + break; + } + } + spin_unlock_irqrestore(&hbus->device_list_lock, flags); + + return hpdev; +} + +/** + * pci_devices_present_work() - Handle new list of child devices + * @work: Work struct embedded in struct hv_dr_work + * + * "Bus Relations" is the Windows term for "children of this + * bus." The terminology is preserved here for people trying to + * debug the interaction between Hyper-V and Linux. This + * function is called when the parent partition reports a list + * of functions that should be observed under this PCI Express + * port (bus). + * + * This function updates the list, and must tolerate being + * called multiple times with the same information. The typical + * number of child devices is one, with very atypical cases + * involving three or four, so the algorithms used here can be + * simple and inefficient. + * + * It must also treat the omission of a previously observed device as + * notification that the device no longer exists. + * + * Note that this function is serialized with hv_eject_device_work(), + * because both are pushed to the ordered workqueue hbus->wq. + */ +static void pci_devices_present_work(struct work_struct *work) +{ + u32 child_no; + bool found; + struct hv_pcidev_description *new_desc; + struct hv_pci_dev *hpdev; + struct hv_pcibus_device *hbus; + struct list_head removed; + struct hv_dr_work *dr_wrk; + struct hv_dr_state *dr = NULL; + unsigned long flags; + + dr_wrk = container_of(work, struct hv_dr_work, wrk); + hbus = dr_wrk->bus; + kfree(dr_wrk); + + INIT_LIST_HEAD(&removed); + + /* Pull this off the queue and process it if it was the last one. */ + spin_lock_irqsave(&hbus->device_list_lock, flags); + while (!list_empty(&hbus->dr_list)) { + dr = list_first_entry(&hbus->dr_list, struct hv_dr_state, + list_entry); + list_del(&dr->list_entry); + + /* Throw this away if the list still has stuff in it. */ + if (!list_empty(&hbus->dr_list)) { + kfree(dr); + continue; + } + } + spin_unlock_irqrestore(&hbus->device_list_lock, flags); + + if (!dr) + return; + + mutex_lock(&hbus->state_lock); + + /* First, mark all existing children as reported missing. */ + spin_lock_irqsave(&hbus->device_list_lock, flags); + list_for_each_entry(hpdev, &hbus->children, list_entry) { + hpdev->reported_missing = true; + } + spin_unlock_irqrestore(&hbus->device_list_lock, flags); + + /* Next, add back any reported devices. */ + for (child_no = 0; child_no < dr->device_count; child_no++) { + found = false; + new_desc = &dr->func[child_no]; + + spin_lock_irqsave(&hbus->device_list_lock, flags); + list_for_each_entry(hpdev, &hbus->children, list_entry) { + if ((hpdev->desc.win_slot.slot == new_desc->win_slot.slot) && + (hpdev->desc.v_id == new_desc->v_id) && + (hpdev->desc.d_id == new_desc->d_id) && + (hpdev->desc.ser == new_desc->ser)) { + hpdev->reported_missing = false; + found = true; + } + } + spin_unlock_irqrestore(&hbus->device_list_lock, flags); + + if (!found) { + hpdev = new_pcichild_device(hbus, new_desc); + if (!hpdev) + dev_err(&hbus->hdev->device, + "couldn't record a child device.\n"); + } + } + + /* Move missing children to a list on the stack. */ + spin_lock_irqsave(&hbus->device_list_lock, flags); + do { + found = false; + list_for_each_entry(hpdev, &hbus->children, list_entry) { + if (hpdev->reported_missing) { + found = true; + put_pcichild(hpdev); + list_move_tail(&hpdev->list_entry, &removed); + break; + } + } + } while (found); + spin_unlock_irqrestore(&hbus->device_list_lock, flags); + + /* Delete everything that should no longer exist. */ + while (!list_empty(&removed)) { + hpdev = list_first_entry(&removed, struct hv_pci_dev, + list_entry); + list_del(&hpdev->list_entry); + + if (hpdev->pci_slot) + pci_destroy_slot(hpdev->pci_slot); + + put_pcichild(hpdev); + } + + switch (hbus->state) { + case hv_pcibus_installed: + /* + * Tell the core to rescan bus + * because there may have been changes. + */ + pci_lock_rescan_remove(); + pci_scan_child_bus(hbus->bridge->bus); + hv_pci_assign_numa_node(hbus); + hv_pci_assign_slots(hbus); + pci_unlock_rescan_remove(); + break; + + case hv_pcibus_init: + case hv_pcibus_probed: + survey_child_resources(hbus); + break; + + default: + break; + } + + mutex_unlock(&hbus->state_lock); + + kfree(dr); +} + +/** + * hv_pci_start_relations_work() - Queue work to start device discovery + * @hbus: Root PCI bus, as understood by this driver + * @dr: The list of children returned from host + * + * Return: 0 on success, -errno on failure + */ +static int hv_pci_start_relations_work(struct hv_pcibus_device *hbus, + struct hv_dr_state *dr) +{ + struct hv_dr_work *dr_wrk; + unsigned long flags; + bool pending_dr; + + if (hbus->state == hv_pcibus_removing) { + dev_info(&hbus->hdev->device, + "PCI VMBus BUS_RELATIONS: ignored\n"); + return -ENOENT; + } + + dr_wrk = kzalloc_obj(*dr_wrk, GFP_NOWAIT); + if (!dr_wrk) + return -ENOMEM; + + INIT_WORK(&dr_wrk->wrk, pci_devices_present_work); + dr_wrk->bus = hbus; + + spin_lock_irqsave(&hbus->device_list_lock, flags); + /* + * If pending_dr is true, we have already queued a work, + * which will see the new dr. Otherwise, we need to + * queue a new work. + */ + pending_dr = !list_empty(&hbus->dr_list); + list_add_tail(&dr->list_entry, &hbus->dr_list); + spin_unlock_irqrestore(&hbus->device_list_lock, flags); + + if (pending_dr) + kfree(dr_wrk); + else + queue_work(hbus->wq, &dr_wrk->wrk); + + return 0; +} + +/** + * hv_pci_devices_present() - Handle list of new children + * @hbus: Root PCI bus, as understood by this driver + * @relations: Packet from host listing children + * + * Process a new list of devices on the bus. The list of devices is + * discovered by VSP and sent to us via VSP message PCI_BUS_RELATIONS, + * whenever a new list of devices for this bus appears. + */ +static void hv_pci_devices_present(struct hv_pcibus_device *hbus, + struct pci_bus_relations *relations) +{ + struct hv_dr_state *dr; + int i; + + dr = kzalloc_flex(*dr, func, relations->device_count, GFP_NOWAIT); + if (!dr) + return; + + dr->device_count = relations->device_count; + for (i = 0; i < dr->device_count; i++) { + dr->func[i].v_id = relations->func[i].v_id; + dr->func[i].d_id = relations->func[i].d_id; + dr->func[i].rev = relations->func[i].rev; + dr->func[i].prog_intf = relations->func[i].prog_intf; + dr->func[i].subclass = relations->func[i].subclass; + dr->func[i].base_class = relations->func[i].base_class; + dr->func[i].subsystem_id = relations->func[i].subsystem_id; + dr->func[i].win_slot = relations->func[i].win_slot; + dr->func[i].ser = relations->func[i].ser; + } + + if (hv_pci_start_relations_work(hbus, dr)) + kfree(dr); +} + +/** + * hv_pci_devices_present2() - Handle list of new children + * @hbus: Root PCI bus, as understood by this driver + * @relations: Packet from host listing children + * + * This function is the v2 version of hv_pci_devices_present() + */ +static void hv_pci_devices_present2(struct hv_pcibus_device *hbus, + struct pci_bus_relations2 *relations) +{ + struct hv_dr_state *dr; + int i; + + dr = kzalloc_flex(*dr, func, relations->device_count, GFP_NOWAIT); + if (!dr) + return; + + dr->device_count = relations->device_count; + for (i = 0; i < dr->device_count; i++) { + dr->func[i].v_id = relations->func[i].v_id; + dr->func[i].d_id = relations->func[i].d_id; + dr->func[i].rev = relations->func[i].rev; + dr->func[i].prog_intf = relations->func[i].prog_intf; + dr->func[i].subclass = relations->func[i].subclass; + dr->func[i].base_class = relations->func[i].base_class; + dr->func[i].subsystem_id = relations->func[i].subsystem_id; + dr->func[i].win_slot = relations->func[i].win_slot; + dr->func[i].ser = relations->func[i].ser; + dr->func[i].flags = relations->func[i].flags; + dr->func[i].virtual_numa_node = + relations->func[i].virtual_numa_node; + } + + if (hv_pci_start_relations_work(hbus, dr)) + kfree(dr); +} + +/** + * hv_eject_device_work() - Asynchronously handles ejection + * @work: Work struct embedded in internal device struct + * + * This function handles ejecting a device. Windows will + * attempt to gracefully eject a device, waiting 60 seconds to + * hear back from the guest OS that this completed successfully. + * If this timer expires, the device will be forcibly removed. + */ +static void hv_eject_device_work(struct work_struct *work) +{ + struct pci_eject_response *ejct_pkt; + struct hv_pcibus_device *hbus; + struct hv_pci_dev *hpdev; + struct pci_dev *pdev; + unsigned long flags; + int wslot; + struct { + struct pci_packet pkt; + u8 buffer[sizeof(struct pci_eject_response)]; + } ctxt; + + hpdev = container_of(work, struct hv_pci_dev, wrk); + hbus = hpdev->hbus; + + mutex_lock(&hbus->state_lock); + + /* + * Ejection can come before or after the PCI bus has been set up, so + * attempt to find it and tear down the bus state, if it exists. This + * must be done without constructs like pci_domain_nr(hbus->bridge->bus) + * because hbus->bridge->bus may not exist yet. + */ + wslot = wslot_to_devfn(hpdev->desc.win_slot.slot); + pdev = pci_get_domain_bus_and_slot(hbus->bridge->domain_nr, 0, wslot); + if (pdev) { + pci_lock_rescan_remove(); + pci_stop_and_remove_bus_device(pdev); + pci_dev_put(pdev); + pci_unlock_rescan_remove(); + } + + spin_lock_irqsave(&hbus->device_list_lock, flags); + list_del(&hpdev->list_entry); + spin_unlock_irqrestore(&hbus->device_list_lock, flags); + + if (hpdev->pci_slot) + pci_destroy_slot(hpdev->pci_slot); + + memset(&ctxt, 0, sizeof(ctxt)); + ejct_pkt = (struct pci_eject_response *)ctxt.buffer; + ejct_pkt->message_type.type = PCI_EJECTION_COMPLETE; + ejct_pkt->wslot.slot = hpdev->desc.win_slot.slot; + vmbus_sendpacket(hbus->hdev->channel, ejct_pkt, + sizeof(*ejct_pkt), 0, + VM_PKT_DATA_INBAND, 0); + + /* For the get_pcichild() in hv_pci_eject_device() */ + put_pcichild(hpdev); + /* For the two refs got in new_pcichild_device() */ + put_pcichild(hpdev); + put_pcichild(hpdev); + /* hpdev has been freed. Do not use it any more. */ + + mutex_unlock(&hbus->state_lock); +} + +/** + * hv_pci_eject_device() - Handles device ejection + * @hpdev: Internal device tracking struct + * + * This function is invoked when an ejection packet arrives. It + * just schedules work so that we don't re-enter the packet + * delivery code handling the ejection. + */ +static void hv_pci_eject_device(struct hv_pci_dev *hpdev) +{ + struct hv_pcibus_device *hbus = hpdev->hbus; + struct hv_device *hdev = hbus->hdev; + + if (hbus->state == hv_pcibus_removing) { + dev_info(&hdev->device, "PCI VMBus EJECT: ignored\n"); + return; + } + + get_pcichild(hpdev); + INIT_WORK(&hpdev->wrk, hv_eject_device_work); + queue_work(hbus->wq, &hpdev->wrk); +} + +/** + * hv_pci_onchannelcallback() - Handles incoming packets + * @context: Internal bus tracking struct + * + * This function is invoked whenever the host sends a packet to + * this channel (which is private to this root PCI bus). + */ +static void hv_pci_onchannelcallback(void *context) +{ + const int packet_size = 0x100; + int ret; + struct hv_pcibus_device *hbus = context; + struct vmbus_channel *chan = hbus->hdev->channel; + u32 bytes_recvd; + u64 req_id, req_addr; + struct vmpacket_descriptor *desc; + unsigned char *buffer; + int bufferlen = packet_size; + struct pci_packet *comp_packet; + struct pci_response *response; + struct pci_incoming_message *new_message; + struct pci_bus_relations *bus_rel; + struct pci_bus_relations2 *bus_rel2; + struct pci_dev_inval_block *inval; + struct pci_dev_incoming *dev_message; + struct hv_pci_dev *hpdev; + unsigned long flags; + + buffer = kmalloc(bufferlen, GFP_ATOMIC); + if (!buffer) + return; + + while (1) { + ret = vmbus_recvpacket_raw(chan, buffer, bufferlen, + &bytes_recvd, &req_id); + + if (ret == -ENOBUFS) { + kfree(buffer); + /* Handle large packet */ + bufferlen = bytes_recvd; + buffer = kmalloc(bytes_recvd, GFP_ATOMIC); + if (!buffer) + return; + continue; + } + + /* Zero length indicates there are no more packets. */ + if (ret || !bytes_recvd) + break; + + /* + * All incoming packets must be at least as large as a + * response. + */ + if (bytes_recvd <= sizeof(struct pci_response)) + continue; + desc = (struct vmpacket_descriptor *)buffer; + + switch (desc->type) { + case VM_PKT_COMP: + + lock_requestor(chan, flags); + req_addr = __vmbus_request_addr_match(chan, req_id, + VMBUS_RQST_ADDR_ANY); + if (req_addr == VMBUS_RQST_ERROR) { + unlock_requestor(chan, flags); + dev_err(&hbus->hdev->device, + "Invalid transaction ID %llx\n", + req_id); + break; + } + comp_packet = (struct pci_packet *)req_addr; + response = (struct pci_response *)buffer; + /* + * Call ->completion_func() within the critical section to make + * sure that the packet pointer is still valid during the call: + * here 'valid' means that there's a task still waiting for the + * completion, and that the packet data is still on the waiting + * task's stack. Cf. hv_compose_msi_msg(). + */ + comp_packet->completion_func(comp_packet->compl_ctxt, + response, + bytes_recvd); + unlock_requestor(chan, flags); + break; + + case VM_PKT_DATA_INBAND: + + new_message = (struct pci_incoming_message *)buffer; + switch (new_message->message_type.type) { + case PCI_BUS_RELATIONS: + + bus_rel = (struct pci_bus_relations *)buffer; + if (bytes_recvd < sizeof(*bus_rel) || + bytes_recvd < + struct_size(bus_rel, func, + bus_rel->device_count)) { + dev_err(&hbus->hdev->device, + "bus relations too small\n"); + break; + } + + hv_pci_devices_present(hbus, bus_rel); + break; + + case PCI_BUS_RELATIONS2: + + bus_rel2 = (struct pci_bus_relations2 *)buffer; + if (bytes_recvd < sizeof(*bus_rel2) || + bytes_recvd < + struct_size(bus_rel2, func, + bus_rel2->device_count)) { + dev_err(&hbus->hdev->device, + "bus relations v2 too small\n"); + break; + } + + hv_pci_devices_present2(hbus, bus_rel2); + break; + + case PCI_EJECT: + + dev_message = (struct pci_dev_incoming *)buffer; + if (bytes_recvd < sizeof(*dev_message)) { + dev_err(&hbus->hdev->device, + "eject message too small\n"); + break; + } + hpdev = get_pcichild_wslot(hbus, + dev_message->wslot.slot); + if (hpdev) { + hv_pci_eject_device(hpdev); + put_pcichild(hpdev); + } + break; + + case PCI_INVALIDATE_BLOCK: + + inval = (struct pci_dev_inval_block *)buffer; + if (bytes_recvd < sizeof(*inval)) { + dev_err(&hbus->hdev->device, + "invalidate message too small\n"); + break; + } + hpdev = get_pcichild_wslot(hbus, + inval->wslot.slot); + if (hpdev) { + if (hpdev->block_invalidate) { + hpdev->block_invalidate( + hpdev->invalidate_context, + inval->block_mask); + } + put_pcichild(hpdev); + } + break; + + default: + dev_warn(&hbus->hdev->device, + "Unimplemented protocol message %x\n", + new_message->message_type.type); + break; + } + break; + + default: + dev_err(&hbus->hdev->device, + "unhandled packet type %d, tid %llx len %d\n", + desc->type, req_id, bytes_recvd); + break; + } + } + + kfree(buffer); +} + +/** + * hv_pci_protocol_negotiation() - Set up protocol + * @hdev: VMBus's tracking struct for this root PCI bus. + * @version: Array of supported channel protocol versions in + * the order of probing - highest go first. + * @num_version: Number of elements in the version array. + * + * This driver is intended to support running on Windows 10 + * (server) and later versions. It will not run on earlier + * versions, as they assume that many of the operations which + * Linux needs accomplished with a spinlock held were done via + * asynchronous messaging via VMBus. Windows 10 increases the + * surface area of PCI emulation so that these actions can take + * place by suspending a virtual processor for their duration. + * + * This function negotiates the channel protocol version, + * failing if the host doesn't support the necessary protocol + * level. + */ +static int hv_pci_protocol_negotiation(struct hv_device *hdev, + enum pci_protocol_version_t version[], + int num_version) +{ + struct hv_pcibus_device *hbus = hv_get_drvdata(hdev); + struct pci_version_request *version_req; + struct hv_pci_compl comp_pkt; + struct pci_packet *pkt; + int ret; + int i; + + /* + * Initiate the handshake with the host and negotiate + * a version that the host can support. We start with the + * highest version number and go down if the host cannot + * support it. + */ + pkt = kzalloc(sizeof(*pkt) + sizeof(*version_req), GFP_KERNEL); + if (!pkt) + return -ENOMEM; + + init_completion(&comp_pkt.host_event); + pkt->completion_func = hv_pci_generic_compl; + pkt->compl_ctxt = &comp_pkt; + version_req = (struct pci_version_request *)(pkt + 1); + version_req->message_type.type = PCI_QUERY_PROTOCOL_VERSION; + + for (i = 0; i < num_version; i++) { + version_req->protocol_version = version[i]; + ret = vmbus_sendpacket(hdev->channel, version_req, + sizeof(struct pci_version_request), + (unsigned long)pkt, VM_PKT_DATA_INBAND, + VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); + if (!ret) + ret = wait_for_response(hdev, &comp_pkt.host_event); + + if (ret) { + dev_err(&hdev->device, + "PCI Pass-through VSP failed to request version: %d", + ret); + goto exit; + } + + if (comp_pkt.completion_status >= 0) { + hbus->protocol_version = version[i]; + dev_info(&hdev->device, + "PCI VMBus probing: Using version %#x\n", + hbus->protocol_version); + goto exit; + } + + if (comp_pkt.completion_status != STATUS_REVISION_MISMATCH) { + dev_err(&hdev->device, + "PCI Pass-through VSP failed version request: %#x", + comp_pkt.completion_status); + ret = -EPROTO; + goto exit; + } + + reinit_completion(&comp_pkt.host_event); + } + + dev_err(&hdev->device, + "PCI pass-through VSP failed to find supported version"); + ret = -EPROTO; + +exit: + kfree(pkt); + return ret; +} + +/** + * hv_pci_free_bridge_windows() - Release memory regions for the + * bus + * @hbus: Root PCI bus, as understood by this driver + */ +static void hv_pci_free_bridge_windows(struct hv_pcibus_device *hbus) +{ + /* + * Set the resources back to the way they looked when they + * were allocated by setting IORESOURCE_BUSY again. + */ + + if (hbus->low_mmio_space && hbus->low_mmio_res) { + hbus->low_mmio_res->flags |= IORESOURCE_BUSY; + vmbus_free_mmio(hbus->low_mmio_res->start, + resource_size(hbus->low_mmio_res)); + } + + if (hbus->high_mmio_space && hbus->high_mmio_res) { + hbus->high_mmio_res->flags |= IORESOURCE_BUSY; + vmbus_free_mmio(hbus->high_mmio_res->start, + resource_size(hbus->high_mmio_res)); + } +} + +/** + * hv_pci_allocate_bridge_windows() - Allocate memory regions + * for the bus + * @hbus: Root PCI bus, as understood by this driver + * + * This function calls vmbus_allocate_mmio(), which is itself a + * bit of a compromise. Ideally, we might change the pnp layer + * in the kernel such that it comprehends either PCI devices + * which are "grandchildren of ACPI," with some intermediate bus + * node (in this case, VMBus) or change it such that it + * understands VMBus. The pnp layer, however, has been declared + * deprecated, and not subject to change. + * + * The workaround, implemented here, is to ask VMBus to allocate + * MMIO space for this bus. VMBus itself knows which ranges are + * appropriate by looking at its own ACPI objects. Then, after + * these ranges are claimed, they're modified to look like they + * would have looked if the ACPI and pnp code had allocated + * bridge windows. These descriptors have to exist in this form + * in order to satisfy the code which will get invoked when the + * endpoint PCI function driver calls request_mem_region() or + * request_mem_region_exclusive(). + * + * Return: 0 on success, -errno on failure + */ +static int hv_pci_allocate_bridge_windows(struct hv_pcibus_device *hbus) +{ + resource_size_t align; + int ret; + + if (hbus->low_mmio_space) { + align = 1ULL << (63 - __builtin_clzll(hbus->low_mmio_space)); + ret = vmbus_allocate_mmio(&hbus->low_mmio_res, hbus->hdev, 0, + (u64)(u32)0xffffffff, + hbus->low_mmio_space, + align, false); + if (ret) { + dev_err(&hbus->hdev->device, + "Need %#llx of low MMIO space. Consider reconfiguring the VM.\n", + hbus->low_mmio_space); + return ret; + } + + /* Modify this resource to become a bridge window. */ + hbus->low_mmio_res->flags |= IORESOURCE_WINDOW; + hbus->low_mmio_res->flags &= ~IORESOURCE_BUSY; + pci_add_resource(&hbus->bridge->windows, hbus->low_mmio_res); + } + + if (hbus->high_mmio_space) { + align = 1ULL << (63 - __builtin_clzll(hbus->high_mmio_space)); + ret = vmbus_allocate_mmio(&hbus->high_mmio_res, hbus->hdev, + 0x100000000, -1, + hbus->high_mmio_space, align, + false); + if (ret) { + dev_err(&hbus->hdev->device, + "Need %#llx of high MMIO space. Consider reconfiguring the VM.\n", + hbus->high_mmio_space); + goto release_low_mmio; + } + + /* Modify this resource to become a bridge window. */ + hbus->high_mmio_res->flags |= IORESOURCE_WINDOW; + hbus->high_mmio_res->flags &= ~IORESOURCE_BUSY; + pci_add_resource(&hbus->bridge->windows, hbus->high_mmio_res); + } + + return 0; + +release_low_mmio: + if (hbus->low_mmio_res) { + vmbus_free_mmio(hbus->low_mmio_res->start, + resource_size(hbus->low_mmio_res)); + } + + return ret; +} + +/** + * hv_allocate_config_window() - Find MMIO space for PCI Config + * @hbus: Root PCI bus, as understood by this driver + * + * This function claims memory-mapped I/O space for accessing + * configuration space for the functions on this bus. + * + * Return: 0 on success, -errno on failure + */ +static int hv_allocate_config_window(struct hv_pcibus_device *hbus) +{ + int ret; + + /* + * Set up a region of MMIO space to use for accessing configuration + * space. + */ + ret = vmbus_allocate_mmio(&hbus->mem_config, hbus->hdev, 0, -1, + PCI_CONFIG_MMIO_LENGTH, 0x1000, false); + if (ret) + return ret; + + /* + * vmbus_allocate_mmio() gets used for allocating both device endpoint + * resource claims (those which cannot be overlapped) and the ranges + * which are valid for the children of this bus, which are intended + * to be overlapped by those children. Set the flag on this claim + * meaning that this region can't be overlapped. + */ + + hbus->mem_config->flags |= IORESOURCE_BUSY; + + return 0; +} + +static void hv_free_config_window(struct hv_pcibus_device *hbus) +{ + vmbus_free_mmio(hbus->mem_config->start, PCI_CONFIG_MMIO_LENGTH); +} + +static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs); + +/** + * hv_pci_enter_d0() - Bring the "bus" into the D0 power state + * @hdev: VMBus's tracking struct for this root PCI bus + * + * Return: 0 on success, -errno on failure + */ +static int hv_pci_enter_d0(struct hv_device *hdev) +{ + struct hv_pcibus_device *hbus = hv_get_drvdata(hdev); + struct pci_bus_d0_entry *d0_entry; + struct hv_pci_compl comp_pkt; + struct pci_packet *pkt; + bool retry = true; + int ret; + +enter_d0_retry: + /* + * Tell the host that the bus is ready to use, and moved into the + * powered-on state. This includes telling the host which region + * of memory-mapped I/O space has been chosen for configuration space + * access. + */ + pkt = kzalloc(sizeof(*pkt) + sizeof(*d0_entry), GFP_KERNEL); + if (!pkt) + return -ENOMEM; + + init_completion(&comp_pkt.host_event); + pkt->completion_func = hv_pci_generic_compl; + pkt->compl_ctxt = &comp_pkt; + d0_entry = (struct pci_bus_d0_entry *)(pkt + 1); + d0_entry->message_type.type = PCI_BUS_D0ENTRY; + d0_entry->mmio_base = hbus->mem_config->start; + + ret = vmbus_sendpacket(hdev->channel, d0_entry, sizeof(*d0_entry), + (unsigned long)pkt, VM_PKT_DATA_INBAND, + VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); + if (!ret) + ret = wait_for_response(hdev, &comp_pkt.host_event); + + if (ret) + goto exit; + + /* + * In certain case (Kdump) the pci device of interest was + * not cleanly shut down and resource is still held on host + * side, the host could return invalid device status. + * We need to explicitly request host to release the resource + * and try to enter D0 again. + */ + if (comp_pkt.completion_status < 0 && retry) { + retry = false; + + dev_err(&hdev->device, "Retrying D0 Entry\n"); + + /* + * Hv_pci_bus_exit() calls hv_send_resource_released() + * to free up resources of its child devices. + * In the kdump kernel we need to set the + * wslot_res_allocated to 255 so it scans all child + * devices to release resources allocated in the + * normal kernel before panic happened. + */ + hbus->wslot_res_allocated = 255; + + ret = hv_pci_bus_exit(hdev, true); + + if (ret == 0) { + kfree(pkt); + goto enter_d0_retry; + } + dev_err(&hdev->device, + "Retrying D0 failed with ret %d\n", ret); + } + + if (comp_pkt.completion_status < 0) { + dev_err(&hdev->device, + "PCI Pass-through VSP failed D0 Entry with status %x\n", + comp_pkt.completion_status); + ret = -EPROTO; + goto exit; + } + + ret = 0; + +exit: + kfree(pkt); + return ret; +} + +/** + * hv_pci_query_relations() - Ask host to send list of child + * devices + * @hdev: VMBus's tracking struct for this root PCI bus + * + * Return: 0 on success, -errno on failure + */ +static int hv_pci_query_relations(struct hv_device *hdev) +{ + struct hv_pcibus_device *hbus = hv_get_drvdata(hdev); + struct pci_message message; + struct completion comp; + int ret; + + /* Ask the host to send along the list of child devices */ + init_completion(&comp); + if (cmpxchg(&hbus->survey_event, NULL, &comp)) + return -ENOTEMPTY; + + memset(&message, 0, sizeof(message)); + message.type = PCI_QUERY_BUS_RELATIONS; + + ret = vmbus_sendpacket(hdev->channel, &message, sizeof(message), + 0, VM_PKT_DATA_INBAND, 0); + if (!ret) + ret = wait_for_response(hdev, &comp); + + /* + * In the case of fast device addition/removal, it's possible that + * vmbus_sendpacket() or wait_for_response() returns -ENODEV but we + * already got a PCI_BUS_RELATIONS* message from the host and the + * channel callback already scheduled a work to hbus->wq, which can be + * running pci_devices_present_work() -> survey_child_resources() -> + * complete(&hbus->survey_event), even after hv_pci_query_relations() + * exits and the stack variable 'comp' is no longer valid; as a result, + * a hang or a page fault may happen when the complete() calls + * raw_spin_lock_irqsave(). Flush hbus->wq before we exit from + * hv_pci_query_relations() to avoid the issues. Note: if 'ret' is + * -ENODEV, there can't be any more work item scheduled to hbus->wq + * after the flush_workqueue(): see vmbus_onoffer_rescind() -> + * vmbus_reset_channel_cb(), vmbus_rescind_cleanup() -> + * channel->rescind = true. + */ + flush_workqueue(hbus->wq); + + return ret; +} + +/** + * hv_send_resources_allocated() - Report local resource choices + * @hdev: VMBus's tracking struct for this root PCI bus + * + * The host OS is expecting to be sent a request as a message + * which contains all the resources that the device will use. + * The response contains those same resources, "translated" + * which is to say, the values which should be used by the + * hardware, when it delivers an interrupt. (MMIO resources are + * used in local terms.) This is nice for Windows, and lines up + * with the FDO/PDO split, which doesn't exist in Linux. Linux + * is deeply expecting to scan an emulated PCI configuration + * space. So this message is sent here only to drive the state + * machine on the host forward. + * + * Return: 0 on success, -errno on failure + */ +static int hv_send_resources_allocated(struct hv_device *hdev) +{ + struct hv_pcibus_device *hbus = hv_get_drvdata(hdev); + struct pci_resources_assigned *res_assigned; + struct pci_resources_assigned2 *res_assigned2; + struct hv_pci_compl comp_pkt; + struct hv_pci_dev *hpdev; + struct pci_packet *pkt; + size_t size_res; + int wslot; + int ret; + + size_res = (hbus->protocol_version < PCI_PROTOCOL_VERSION_1_2) + ? sizeof(*res_assigned) : sizeof(*res_assigned2); + + pkt = kmalloc(sizeof(*pkt) + size_res, GFP_KERNEL); + if (!pkt) + return -ENOMEM; + + ret = 0; + + for (wslot = 0; wslot < 256; wslot++) { + hpdev = get_pcichild_wslot(hbus, wslot); + if (!hpdev) + continue; + + memset(pkt, 0, sizeof(*pkt) + size_res); + init_completion(&comp_pkt.host_event); + pkt->completion_func = hv_pci_generic_compl; + pkt->compl_ctxt = &comp_pkt; + + if (hbus->protocol_version < PCI_PROTOCOL_VERSION_1_2) { + res_assigned = + (struct pci_resources_assigned *)(pkt + 1); + res_assigned->message_type.type = + PCI_RESOURCES_ASSIGNED; + res_assigned->wslot.slot = hpdev->desc.win_slot.slot; + } else { + res_assigned2 = + (struct pci_resources_assigned2 *)(pkt + 1); + res_assigned2->message_type.type = + PCI_RESOURCES_ASSIGNED2; + res_assigned2->wslot.slot = hpdev->desc.win_slot.slot; + } + put_pcichild(hpdev); + + ret = vmbus_sendpacket(hdev->channel, pkt + 1, + size_res, (unsigned long)pkt, + VM_PKT_DATA_INBAND, + VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); + if (!ret) + ret = wait_for_response(hdev, &comp_pkt.host_event); + if (ret) + break; + + if (comp_pkt.completion_status < 0) { + ret = -EPROTO; + dev_err(&hdev->device, + "resource allocated returned 0x%x", + comp_pkt.completion_status); + break; + } + + hbus->wslot_res_allocated = wslot; + } + + kfree(pkt); + return ret; +} + +/** + * hv_send_resources_released() - Report local resources + * released + * @hdev: VMBus's tracking struct for this root PCI bus + * + * Return: 0 on success, -errno on failure + */ +static int hv_send_resources_released(struct hv_device *hdev) +{ + struct hv_pcibus_device *hbus = hv_get_drvdata(hdev); + struct pci_child_message pkt; + struct hv_pci_dev *hpdev; + int wslot; + int ret; + + for (wslot = hbus->wslot_res_allocated; wslot >= 0; wslot--) { + hpdev = get_pcichild_wslot(hbus, wslot); + if (!hpdev) + continue; + + memset(&pkt, 0, sizeof(pkt)); + pkt.message_type.type = PCI_RESOURCES_RELEASED; + pkt.wslot.slot = hpdev->desc.win_slot.slot; + + put_pcichild(hpdev); + + ret = vmbus_sendpacket(hdev->channel, &pkt, sizeof(pkt), 0, + VM_PKT_DATA_INBAND, 0); + if (ret) + return ret; + + hbus->wslot_res_allocated = wslot - 1; + } + + hbus->wslot_res_allocated = -1; + + return 0; +} + +/** + * hv_pci_probe() - New VMBus channel probe, for a root PCI bus + * @hdev: VMBus's tracking struct for this root PCI bus + * @dev_id: Identifies the device itself + * + * Return: 0 on success, -errno on failure + */ +static int hv_pci_probe(struct hv_device *hdev, + const struct hv_vmbus_device_id *dev_id) +{ + struct pci_host_bridge *bridge; + struct hv_pcibus_device *hbus; + int ret, dom; + u16 dom_req; + char *name; + + bridge = devm_pci_alloc_host_bridge(&hdev->device, 0); + if (!bridge) + return -ENOMEM; + + hbus = kzalloc_obj(*hbus); + if (!hbus) + return -ENOMEM; + + hbus->bridge = bridge; + mutex_init(&hbus->state_lock); + hbus->state = hv_pcibus_init; + hbus->wslot_res_allocated = -1; + + /* + * The PCI bus "domain" is what is called "segment" in ACPI and other + * specs. Pull it from the instance ID, to get something usually + * unique. In rare cases of collision, we will find out another number + * not in use. + * + * Note that, since this code only runs in a Hyper-V VM, Hyper-V + * together with this guest driver can guarantee that (1) The only + * domain used by Gen1 VMs for something that looks like a physical + * PCI bus (which is actually emulated by the hypervisor) is domain 0. + * (2) There will be no overlap between domains (after fixing possible + * collisions) in the same VM. + * + * Because Gen1 VMs use domain 0, don't allow picking domain 0 here, + * even if bytes 4 and 5 of the instance GUID are both zero. For wider + * userspace compatibility, limit the domain ID to a 16-bit value. + */ + dom_req = hdev->dev_instance.b[5] << 8 | hdev->dev_instance.b[4]; + dom = pci_bus_find_emul_domain_nr(dom_req, 1, U16_MAX); + if (dom < 0) { + dev_err(&hdev->device, + "Unable to use dom# 0x%x or other numbers", dom_req); + ret = -EINVAL; + goto free_bus; + } + + if (dom != dom_req) + dev_info(&hdev->device, + "PCI dom# 0x%x has collision, using 0x%x", + dom_req, dom); + + hbus->bridge->domain_nr = dom; +#ifdef CONFIG_X86 + hbus->sysdata.domain = dom; + hbus->use_calls = !!(ms_hyperv.hints & HV_X64_USE_MMIO_HYPERCALLS); +#elif defined(CONFIG_ARM64) + /* + * Set the PCI bus parent to be the corresponding VMbus + * device. Then the VMbus device will be assigned as the + * ACPI companion in pcibios_root_bridge_prepare() and + * pci_dma_configure() will propagate device coherence + * information to devices created on the bus. + */ + hbus->sysdata.parent = hdev->device.parent; + hbus->use_calls = false; +#endif + + hbus->hdev = hdev; + INIT_LIST_HEAD(&hbus->children); + INIT_LIST_HEAD(&hbus->dr_list); + spin_lock_init(&hbus->config_lock); + spin_lock_init(&hbus->device_list_lock); + hbus->wq = alloc_ordered_workqueue("hv_pci_%x", 0, + hbus->bridge->domain_nr); + if (!hbus->wq) { + ret = -ENOMEM; + goto free_bus; + } + + hdev->channel->next_request_id_callback = vmbus_next_request_id; + hdev->channel->request_addr_callback = vmbus_request_addr; + hdev->channel->rqstor_size = HV_PCI_RQSTOR_SIZE; + + ret = vmbus_open(hdev->channel, pci_ring_size, pci_ring_size, NULL, 0, + hv_pci_onchannelcallback, hbus); + if (ret) + goto destroy_wq; + + hv_set_drvdata(hdev, hbus); + + ret = hv_pci_protocol_negotiation(hdev, pci_protocol_versions, + ARRAY_SIZE(pci_protocol_versions)); + if (ret) + goto close; + + ret = hv_allocate_config_window(hbus); + if (ret) + goto close; + + hbus->cfg_addr = ioremap(hbus->mem_config->start, + PCI_CONFIG_MMIO_LENGTH); + if (!hbus->cfg_addr) { + dev_err(&hdev->device, + "Unable to map a virtual address for config space\n"); + ret = -ENOMEM; + goto free_config; + } + + name = kasprintf(GFP_KERNEL, "%pUL", &hdev->dev_instance); + if (!name) { + ret = -ENOMEM; + goto unmap; + } + + hbus->fwnode = irq_domain_alloc_named_fwnode(name); + kfree(name); + if (!hbus->fwnode) { + ret = -ENOMEM; + goto unmap; + } + + ret = hv_pcie_init_irq_domain(hbus); + if (ret) + goto free_fwnode; + + ret = hv_pci_query_relations(hdev); + if (ret) + goto free_irq_domain; + + mutex_lock(&hbus->state_lock); + + ret = hv_pci_enter_d0(hdev); + if (ret) + goto release_state_lock; + + ret = hv_pci_allocate_bridge_windows(hbus); + if (ret) + goto exit_d0; + + ret = hv_send_resources_allocated(hdev); + if (ret) + goto free_windows; + + prepopulate_bars(hbus); + + hbus->state = hv_pcibus_probed; + + ret = create_root_hv_pci_bus(hbus); + if (ret) + goto free_windows; + + mutex_unlock(&hbus->state_lock); + return 0; + +free_windows: + hv_pci_free_bridge_windows(hbus); +exit_d0: + (void) hv_pci_bus_exit(hdev, true); +release_state_lock: + mutex_unlock(&hbus->state_lock); +free_irq_domain: + irq_domain_remove(hbus->irq_domain); +free_fwnode: + irq_domain_free_fwnode(hbus->fwnode); +unmap: + iounmap(hbus->cfg_addr); +free_config: + hv_free_config_window(hbus); +close: + vmbus_close(hdev->channel); +destroy_wq: + destroy_workqueue(hbus->wq); +free_bus: + kfree(hbus); + return ret; +} + +static int hv_pci_bus_exit(struct hv_device *hdev, bool keep_devs) +{ + struct hv_pcibus_device *hbus = hv_get_drvdata(hdev); + struct vmbus_channel *chan = hdev->channel; + struct { + struct pci_packet teardown_packet; + u8 buffer[sizeof(struct pci_message)]; + } pkt; + struct pci_message *msg; + struct hv_pci_compl comp_pkt; + struct hv_pci_dev *hpdev, *tmp; + unsigned long flags; + u64 trans_id; + int ret; + + /* + * After the host sends the RESCIND_CHANNEL message, it doesn't + * access the per-channel ringbuffer any longer. + */ + if (chan->rescind) + return 0; + + if (!keep_devs) { + struct list_head removed; + + /* Move all present children to the list on stack */ + INIT_LIST_HEAD(&removed); + spin_lock_irqsave(&hbus->device_list_lock, flags); + list_for_each_entry_safe(hpdev, tmp, &hbus->children, list_entry) + list_move_tail(&hpdev->list_entry, &removed); + spin_unlock_irqrestore(&hbus->device_list_lock, flags); + + /* Remove all children in the list */ + list_for_each_entry_safe(hpdev, tmp, &removed, list_entry) { + list_del(&hpdev->list_entry); + if (hpdev->pci_slot) + pci_destroy_slot(hpdev->pci_slot); + /* For the two refs got in new_pcichild_device() */ + put_pcichild(hpdev); + put_pcichild(hpdev); + } + } + + ret = hv_send_resources_released(hdev); + if (ret) { + dev_err(&hdev->device, + "Couldn't send resources released packet(s)\n"); + return ret; + } + + memset(&pkt.teardown_packet, 0, sizeof(pkt.teardown_packet)); + init_completion(&comp_pkt.host_event); + pkt.teardown_packet.completion_func = hv_pci_generic_compl; + pkt.teardown_packet.compl_ctxt = &comp_pkt; + msg = (struct pci_message *)pkt.buffer; + msg->type = PCI_BUS_D0EXIT; + + ret = vmbus_sendpacket_getid(chan, msg, sizeof(*msg), + (unsigned long)&pkt.teardown_packet, + &trans_id, VM_PKT_DATA_INBAND, + VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); + if (ret) + return ret; + + if (wait_for_completion_timeout(&comp_pkt.host_event, 10 * HZ) == 0) { + /* + * The completion packet on the stack becomes invalid after + * 'return'; remove the ID from the VMbus requestor if the + * identifier is still mapped to/associated with the packet. + * + * Cf. hv_pci_onchannelcallback(). + */ + vmbus_request_addr_match(chan, trans_id, + (unsigned long)&pkt.teardown_packet); + return -ETIMEDOUT; + } + + return 0; +} + +/** + * hv_pci_remove() - Remove routine for this VMBus channel + * @hdev: VMBus's tracking struct for this root PCI bus + */ +static void hv_pci_remove(struct hv_device *hdev) +{ + struct hv_pcibus_device *hbus; + + hbus = hv_get_drvdata(hdev); + if (hbus->state == hv_pcibus_installed) { + tasklet_disable(&hdev->channel->callback_event); + hbus->state = hv_pcibus_removing; + tasklet_enable(&hdev->channel->callback_event); + destroy_workqueue(hbus->wq); + hbus->wq = NULL; + /* + * At this point, no work is running or can be scheduled + * on hbus-wq. We can't race with hv_pci_devices_present() + * or hv_pci_eject_device(), it's safe to proceed. + */ + + /* Remove the bus from PCI's point of view. */ + pci_lock_rescan_remove(); + pci_stop_root_bus(hbus->bridge->bus); + hv_pci_remove_slots(hbus); + pci_remove_root_bus(hbus->bridge->bus); + pci_unlock_rescan_remove(); + } + + hv_pci_bus_exit(hdev, false); + + vmbus_close(hdev->channel); + + iounmap(hbus->cfg_addr); + hv_free_config_window(hbus); + hv_pci_free_bridge_windows(hbus); + irq_domain_remove(hbus->irq_domain); + irq_domain_free_fwnode(hbus->fwnode); + + kfree(hbus); +} + +static int hv_pci_suspend(struct hv_device *hdev) +{ + struct hv_pcibus_device *hbus = hv_get_drvdata(hdev); + enum hv_pcibus_state old_state; + int ret; + + /* + * hv_pci_suspend() must make sure there are no pending work items + * before calling vmbus_close(), since it runs in a process context + * as a callback in dpm_suspend(). When it starts to run, the channel + * callback hv_pci_onchannelcallback(), which runs in a tasklet + * context, can be still running concurrently and scheduling new work + * items onto hbus->wq in hv_pci_devices_present() and + * hv_pci_eject_device(), and the work item handlers can access the + * vmbus channel, which can be being closed by hv_pci_suspend(), e.g. + * the work item handler pci_devices_present_work() -> + * new_pcichild_device() writes to the vmbus channel. + * + * To eliminate the race, hv_pci_suspend() disables the channel + * callback tasklet, sets hbus->state to hv_pcibus_removing, and + * re-enables the tasklet. This way, when hv_pci_suspend() proceeds, + * it knows that no new work item can be scheduled, and then it flushes + * hbus->wq and safely closes the vmbus channel. + */ + tasklet_disable(&hdev->channel->callback_event); + + /* Change the hbus state to prevent new work items. */ + old_state = hbus->state; + if (hbus->state == hv_pcibus_installed) + hbus->state = hv_pcibus_removing; + + tasklet_enable(&hdev->channel->callback_event); + + if (old_state != hv_pcibus_installed) + return -EINVAL; + + flush_workqueue(hbus->wq); + + ret = hv_pci_bus_exit(hdev, true); + if (ret) + return ret; + + vmbus_close(hdev->channel); + + return 0; +} + +static int hv_pci_restore_msi_msg(struct pci_dev *pdev, void *arg) +{ + struct irq_data *irq_data; + struct msi_desc *entry; + + if (!pdev->msi_enabled && !pdev->msix_enabled) + return 0; + + guard(msi_descs_lock)(&pdev->dev); + msi_for_each_desc(entry, &pdev->dev, MSI_DESC_ASSOCIATED) { + irq_data = irq_get_irq_data(entry->irq); + if (WARN_ON_ONCE(!irq_data)) + return -EINVAL; + hv_compose_msi_msg(irq_data, &entry->msg); + } + return 0; +} + +/* + * Upon resume, pci_restore_msi_state() -> ... -> __pci_write_msi_msg() + * directly writes the MSI/MSI-X registers via MMIO, but since Hyper-V + * doesn't trap and emulate the MMIO accesses, here hv_compose_msi_msg() + * must be used to ask Hyper-V to re-create the IOMMU Interrupt Remapping + * Table entries. + */ +static void hv_pci_restore_msi_state(struct hv_pcibus_device *hbus) +{ + pci_walk_bus(hbus->bridge->bus, hv_pci_restore_msi_msg, NULL); +} + +static int hv_pci_resume(struct hv_device *hdev) +{ + struct hv_pcibus_device *hbus = hv_get_drvdata(hdev); + enum pci_protocol_version_t version[1]; + int ret; + + hbus->state = hv_pcibus_init; + + hdev->channel->next_request_id_callback = vmbus_next_request_id; + hdev->channel->request_addr_callback = vmbus_request_addr; + hdev->channel->rqstor_size = HV_PCI_RQSTOR_SIZE; + + ret = vmbus_open(hdev->channel, pci_ring_size, pci_ring_size, NULL, 0, + hv_pci_onchannelcallback, hbus); + if (ret) + return ret; + + /* Only use the version that was in use before hibernation. */ + version[0] = hbus->protocol_version; + ret = hv_pci_protocol_negotiation(hdev, version, 1); + if (ret) + goto out; + + ret = hv_pci_query_relations(hdev); + if (ret) + goto out; + + mutex_lock(&hbus->state_lock); + + ret = hv_pci_enter_d0(hdev); + if (ret) + goto release_state_lock; + + ret = hv_send_resources_allocated(hdev); + if (ret) + goto release_state_lock; + + prepopulate_bars(hbus); + + hv_pci_restore_msi_state(hbus); + + hbus->state = hv_pcibus_installed; + mutex_unlock(&hbus->state_lock); + return 0; + +release_state_lock: + mutex_unlock(&hbus->state_lock); +out: + vmbus_close(hdev->channel); + return ret; +} + +static const struct hv_vmbus_device_id hv_pci_id_table[] = { + /* PCI Pass-through Class ID */ + /* 44C4F61D-4444-4400-9D52-802E27EDE19F */ + { HV_PCIE_GUID, }, + { }, +}; + +MODULE_DEVICE_TABLE(vmbus, hv_pci_id_table); + +static struct hv_driver hv_pci_drv = { + .name = "hv_pci", + .id_table = hv_pci_id_table, + .probe = hv_pci_probe, + .remove = hv_pci_remove, + .suspend = hv_pci_suspend, + .resume = hv_pci_resume, +}; + +static void __exit exit_hv_pci_drv(void) +{ + vmbus_driver_unregister(&hv_pci_drv); + + hvpci_block_ops.read_block = NULL; + hvpci_block_ops.write_block = NULL; + hvpci_block_ops.reg_blk_invalidate = NULL; +} + +static int __init init_hv_pci_drv(void) +{ + int ret; + + if (!hv_is_hyperv_initialized()) + return -ENODEV; + + if (!hv_vmbus_exists()) + return -ENODEV; + + ret = hv_pci_irqchip_init(); + if (ret) + return ret; + + /* Initialize PCI block r/w interface */ + hvpci_block_ops.read_block = hv_read_config_block; + hvpci_block_ops.write_block = hv_write_config_block; + hvpci_block_ops.reg_blk_invalidate = hv_register_block_invalidate; + + return vmbus_driver_register(&hv_pci_drv); +} + +module_init(init_hv_pci_drv); +module_exit(exit_hv_pci_drv); + +MODULE_DESCRIPTION("Hyper-V PCI"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pci-ixp4xx.c b/drivers/pci/controller/pci-ixp4xx.c new file mode 100644 index 000000000..9fd401838 --- /dev/null +++ b/drivers/pci/controller/pci-ixp4xx.c @@ -0,0 +1,678 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Support for Intel IXP4xx PCI host controller + * + * Copyright (C) 2017 Linus Walleij <linus.walleij@linaro.org> + * + * Based on the IXP4xx arch/arm/mach-ixp4xx/common-pci.c driver + * Copyright (C) 2002 Intel Corporation + * Copyright (C) 2003 Greg Ungerer <gerg@linux-m68k.org> + * Copyright (C) 2003-2004 MontaVista Software, Inc. + * Copyright (C) 2005 Deepak Saxena <dsaxena@plexity.net> + * Copyright (C) 2005 Alessandro Zummo <a.zummo@towertech.it> + * + * TODO: + * - Test IO-space access + * - DMA support + */ + +#include <linux/init.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/of.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/bits.h> +#include "../pci.h" + +/* Register offsets */ +#define IXP4XX_PCI_NP_AD 0x00 +#define IXP4XX_PCI_NP_CBE 0x04 +#define IXP4XX_PCI_NP_WDATA 0x08 +#define IXP4XX_PCI_NP_RDATA 0x0c +#define IXP4XX_PCI_CRP_AD_CBE 0x10 +#define IXP4XX_PCI_CRP_WDATA 0x14 +#define IXP4XX_PCI_CRP_RDATA 0x18 +#define IXP4XX_PCI_CSR 0x1c +#define IXP4XX_PCI_ISR 0x20 +#define IXP4XX_PCI_INTEN 0x24 +#define IXP4XX_PCI_DMACTRL 0x28 +#define IXP4XX_PCI_AHBMEMBASE 0x2c +#define IXP4XX_PCI_AHBIOBASE 0x30 +#define IXP4XX_PCI_PCIMEMBASE 0x34 +#define IXP4XX_PCI_AHBDOORBELL 0x38 +#define IXP4XX_PCI_PCIDOORBELL 0x3c +#define IXP4XX_PCI_ATPDMA0_AHBADDR 0x40 +#define IXP4XX_PCI_ATPDMA0_PCIADDR 0x44 +#define IXP4XX_PCI_ATPDMA0_LENADDR 0x48 +#define IXP4XX_PCI_ATPDMA1_AHBADDR 0x4c +#define IXP4XX_PCI_ATPDMA1_PCIADDR 0x50 +#define IXP4XX_PCI_ATPDMA1_LENADDR 0x54 + +/* CSR bit definitions */ +#define IXP4XX_PCI_CSR_HOST BIT(0) +#define IXP4XX_PCI_CSR_ARBEN BIT(1) +#define IXP4XX_PCI_CSR_ADS BIT(2) +#define IXP4XX_PCI_CSR_PDS BIT(3) +#define IXP4XX_PCI_CSR_ABE BIT(4) +#define IXP4XX_PCI_CSR_DBT BIT(5) +#define IXP4XX_PCI_CSR_ASE BIT(8) +#define IXP4XX_PCI_CSR_IC BIT(15) +#define IXP4XX_PCI_CSR_PRST BIT(16) + +/* ISR (Interrupt status) Register bit definitions */ +#define IXP4XX_PCI_ISR_PSE BIT(0) +#define IXP4XX_PCI_ISR_PFE BIT(1) +#define IXP4XX_PCI_ISR_PPE BIT(2) +#define IXP4XX_PCI_ISR_AHBE BIT(3) +#define IXP4XX_PCI_ISR_APDC BIT(4) +#define IXP4XX_PCI_ISR_PADC BIT(5) +#define IXP4XX_PCI_ISR_ADB BIT(6) +#define IXP4XX_PCI_ISR_PDB BIT(7) + +/* INTEN (Interrupt Enable) Register bit definitions */ +#define IXP4XX_PCI_INTEN_PSE BIT(0) +#define IXP4XX_PCI_INTEN_PFE BIT(1) +#define IXP4XX_PCI_INTEN_PPE BIT(2) +#define IXP4XX_PCI_INTEN_AHBE BIT(3) +#define IXP4XX_PCI_INTEN_APDC BIT(4) +#define IXP4XX_PCI_INTEN_PADC BIT(5) +#define IXP4XX_PCI_INTEN_ADB BIT(6) +#define IXP4XX_PCI_INTEN_PDB BIT(7) + +/* Shift value for byte enable on NP cmd/byte enable register */ +#define IXP4XX_PCI_NP_CBE_BESL 4 + +/* PCI commands supported by NP access unit */ +#define NP_CMD_IOREAD 0x2 +#define NP_CMD_IOWRITE 0x3 +#define NP_CMD_CONFIGREAD 0xa +#define NP_CMD_CONFIGWRITE 0xb +#define NP_CMD_MEMREAD 0x6 +#define NP_CMD_MEMWRITE 0x7 + +/* Constants for CRP access into local config space */ +#define CRP_AD_CBE_BESL 20 +#define CRP_AD_CBE_WRITE 0x00010000 + +/* Special PCI configuration space registers for this controller */ +#define IXP4XX_PCI_RTOTTO 0x40 + +struct ixp4xx_pci { + struct device *dev; + void __iomem *base; + bool errata_hammer; + bool host_mode; +}; + +/* + * The IXP4xx has a peculiar address bus that will change the + * byte order on SoC peripherals depending on whether the device + * operates in big-endian or little-endian mode. That means that + * readl() and writel() that always use little-endian access + * will not work for SoC peripherals such as the PCI controller + * when used in big-endian mode. The accesses to the individual + * PCI devices on the other hand, are always little-endian and + * can use readl() and writel(). + * + * For local AHB bus access we need to use __raw_[readl|writel]() + * to make sure that we access the SoC devices in the CPU native + * endianness. + */ +static inline u32 ixp4xx_readl(struct ixp4xx_pci *p, u32 reg) +{ + return __raw_readl(p->base + reg); +} + +static inline void ixp4xx_writel(struct ixp4xx_pci *p, u32 reg, u32 val) +{ + __raw_writel(val, p->base + reg); +} + +static int ixp4xx_pci_check_master_abort(struct ixp4xx_pci *p) +{ + u32 isr = ixp4xx_readl(p, IXP4XX_PCI_ISR); + + if (isr & IXP4XX_PCI_ISR_PFE) { + /* Make sure the master abort bit is reset */ + ixp4xx_writel(p, IXP4XX_PCI_ISR, IXP4XX_PCI_ISR_PFE); + dev_dbg(p->dev, "master abort detected\n"); + return -EINVAL; + } + + return 0; +} + +static int ixp4xx_pci_read_indirect(struct ixp4xx_pci *p, u32 addr, u32 cmd, u32 *data) +{ + ixp4xx_writel(p, IXP4XX_PCI_NP_AD, addr); + + if (p->errata_hammer) { + int i; + + /* + * PCI workaround - only works if NP PCI space reads have + * no side effects. Hammer the register and read twice 8 + * times. last one will be good. + */ + for (i = 0; i < 8; i++) { + ixp4xx_writel(p, IXP4XX_PCI_NP_CBE, cmd); + *data = ixp4xx_readl(p, IXP4XX_PCI_NP_RDATA); + *data = ixp4xx_readl(p, IXP4XX_PCI_NP_RDATA); + } + } else { + ixp4xx_writel(p, IXP4XX_PCI_NP_CBE, cmd); + *data = ixp4xx_readl(p, IXP4XX_PCI_NP_RDATA); + } + + return ixp4xx_pci_check_master_abort(p); +} + +static int ixp4xx_pci_write_indirect(struct ixp4xx_pci *p, u32 addr, u32 cmd, u32 data) +{ + ixp4xx_writel(p, IXP4XX_PCI_NP_AD, addr); + + /* Set up the write */ + ixp4xx_writel(p, IXP4XX_PCI_NP_CBE, cmd); + + /* Execute the write by writing to NP_WDATA */ + ixp4xx_writel(p, IXP4XX_PCI_NP_WDATA, data); + + return ixp4xx_pci_check_master_abort(p); +} + +static u32 ixp4xx_config_addr(u8 bus_num, u16 devfn, int where) +{ + /* Root bus is always 0 in this hardware */ + if (bus_num == 0) { + /* type 0 */ + return (PCI_CONF1_ADDRESS(0, 0, PCI_FUNC(devfn), where) & + ~PCI_CONF1_ENABLE) | BIT(32-PCI_SLOT(devfn)); + } else { + /* type 1 */ + return (PCI_CONF1_ADDRESS(bus_num, PCI_SLOT(devfn), + PCI_FUNC(devfn), where) & + ~PCI_CONF1_ENABLE) | 1; + } +} + +/* + * CRP functions are "Controller Configuration Port" accesses + * initiated from within this driver itself to read/write PCI + * control information in the config space. + */ +static u32 ixp4xx_crp_byte_lane_enable_bits(u32 n, int size) +{ + if (size == 1) + return (0xf & ~BIT(n)) << CRP_AD_CBE_BESL; + if (size == 2) + return (0xf & ~(BIT(n) | BIT(n+1))) << CRP_AD_CBE_BESL; + if (size == 4) + return 0; + return 0xffffffff; +} + +#ifdef CONFIG_ARM +static int ixp4xx_crp_read_config(struct ixp4xx_pci *p, int where, int size, + u32 *value) +{ + u32 n, cmd, val; + + n = where % 4; + cmd = where & ~3; + + dev_dbg(p->dev, "%s from %d size %d cmd %08x\n", + __func__, where, size, cmd); + + ixp4xx_writel(p, IXP4XX_PCI_CRP_AD_CBE, cmd); + val = ixp4xx_readl(p, IXP4XX_PCI_CRP_RDATA); + + val >>= (8*n); + switch (size) { + case 1: + val &= U8_MAX; + dev_dbg(p->dev, "%s read byte %02x\n", __func__, val); + break; + case 2: + val &= U16_MAX; + dev_dbg(p->dev, "%s read word %04x\n", __func__, val); + break; + case 4: + val &= U32_MAX; + dev_dbg(p->dev, "%s read long %08x\n", __func__, val); + break; + default: + /* Should not happen */ + dev_err(p->dev, "%s illegal size\n", __func__); + return PCIBIOS_DEVICE_NOT_FOUND; + } + *value = val; + + return PCIBIOS_SUCCESSFUL; +} +#endif + +static int ixp4xx_crp_write_config(struct ixp4xx_pci *p, int where, int size, + u32 value) +{ + u32 n, cmd, val; + + n = where % 4; + cmd = ixp4xx_crp_byte_lane_enable_bits(n, size); + if (cmd == 0xffffffff) + return PCIBIOS_BAD_REGISTER_NUMBER; + cmd |= where & ~3; + cmd |= CRP_AD_CBE_WRITE; + + val = value << (8*n); + + dev_dbg(p->dev, "%s to %d size %d cmd %08x val %08x\n", + __func__, where, size, cmd, val); + + ixp4xx_writel(p, IXP4XX_PCI_CRP_AD_CBE, cmd); + ixp4xx_writel(p, IXP4XX_PCI_CRP_WDATA, val); + + return PCIBIOS_SUCCESSFUL; +} + +/* + * Then follows the functions that read and write from the common PCI + * configuration space. + */ +static u32 ixp4xx_byte_lane_enable_bits(u32 n, int size) +{ + if (size == 1) + return (0xf & ~BIT(n)) << 4; + if (size == 2) + return (0xf & ~(BIT(n) | BIT(n+1))) << 4; + if (size == 4) + return 0; + return 0xffffffff; +} + +static int ixp4xx_pci_read_config(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *value) +{ + struct ixp4xx_pci *p = bus->sysdata; + u32 n, addr, val, cmd; + u8 bus_num = bus->number; + int ret; + + *value = 0xffffffff; + n = where % 4; + cmd = ixp4xx_byte_lane_enable_bits(n, size); + if (cmd == 0xffffffff) + return PCIBIOS_BAD_REGISTER_NUMBER; + + addr = ixp4xx_config_addr(bus_num, devfn, where); + cmd |= NP_CMD_CONFIGREAD; + dev_dbg(p->dev, "read_config from %d size %d dev %d:%d:%d address: %08x cmd: %08x\n", + where, size, bus_num, PCI_SLOT(devfn), PCI_FUNC(devfn), addr, cmd); + + ret = ixp4xx_pci_read_indirect(p, addr, cmd, &val); + if (ret) + return PCIBIOS_DEVICE_NOT_FOUND; + + val >>= (8*n); + switch (size) { + case 1: + val &= U8_MAX; + dev_dbg(p->dev, "%s read byte %02x\n", __func__, val); + break; + case 2: + val &= U16_MAX; + dev_dbg(p->dev, "%s read word %04x\n", __func__, val); + break; + case 4: + val &= U32_MAX; + dev_dbg(p->dev, "%s read long %08x\n", __func__, val); + break; + default: + /* Should not happen */ + dev_err(p->dev, "%s illegal size\n", __func__); + return PCIBIOS_DEVICE_NOT_FOUND; + } + *value = val; + + return PCIBIOS_SUCCESSFUL; +} + +static int ixp4xx_pci_write_config(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 value) +{ + struct ixp4xx_pci *p = bus->sysdata; + u32 n, addr, val, cmd; + u8 bus_num = bus->number; + int ret; + + n = where % 4; + cmd = ixp4xx_byte_lane_enable_bits(n, size); + if (cmd == 0xffffffff) + return PCIBIOS_BAD_REGISTER_NUMBER; + + addr = ixp4xx_config_addr(bus_num, devfn, where); + cmd |= NP_CMD_CONFIGWRITE; + val = value << (8*n); + + dev_dbg(p->dev, "write_config_byte %#x to %d size %d dev %d:%d:%d addr: %08x cmd %08x\n", + value, where, size, bus_num, PCI_SLOT(devfn), PCI_FUNC(devfn), addr, cmd); + + ret = ixp4xx_pci_write_indirect(p, addr, cmd, val); + if (ret) + return PCIBIOS_DEVICE_NOT_FOUND; + + return PCIBIOS_SUCCESSFUL; +} + +static struct pci_ops ixp4xx_pci_ops = { + .read = ixp4xx_pci_read_config, + .write = ixp4xx_pci_write_config, +}; + +static u32 ixp4xx_pci_addr_to_64mconf(phys_addr_t addr) +{ + u8 base; + + base = ((addr & 0xff000000) >> 24); + return (base << 24) | ((base + 1) << 16) + | ((base + 2) << 8) | (base + 3); +} + +static int ixp4xx_pci_parse_map_ranges(struct ixp4xx_pci *p) +{ + struct device *dev = p->dev; + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(p); + struct resource_entry *win; + struct resource *res; + phys_addr_t addr; + + win = resource_list_first_type(&bridge->windows, IORESOURCE_MEM); + if (win) { + u32 pcimembase; + + res = win->res; + addr = res->start - win->offset; + + if (res->flags & IORESOURCE_PREFETCH) + res->name = "IXP4xx PCI PRE-MEM"; + else + res->name = "IXP4xx PCI NON-PRE-MEM"; + + dev_dbg(dev, "%s window %pR, bus addr %pa\n", + res->name, res, &addr); + if (resource_size(res) != SZ_64M) { + dev_err(dev, "memory range is not 64MB\n"); + return -EINVAL; + } + + pcimembase = ixp4xx_pci_addr_to_64mconf(addr); + /* Commit configuration */ + ixp4xx_writel(p, IXP4XX_PCI_PCIMEMBASE, pcimembase); + } else { + dev_err(dev, "no AHB memory mapping defined\n"); + } + + win = resource_list_first_type(&bridge->windows, IORESOURCE_IO); + if (win) { + res = win->res; + + addr = pci_pio_to_address(res->start); + if (addr & 0xff) { + dev_err(dev, "IO mem at uneven address: %pa\n", &addr); + return -EINVAL; + } + + res->name = "IXP4xx PCI IO MEM"; + /* + * Setup I/O space location for PCI->AHB access, the + * upper 24 bits of the address goes into the lower + * 24 bits of this register. + */ + ixp4xx_writel(p, IXP4XX_PCI_AHBIOBASE, (addr >> 8)); + } else { + dev_info(dev, "no IO space AHB memory mapping defined\n"); + } + + return 0; +} + +static int ixp4xx_pci_parse_map_dma_ranges(struct ixp4xx_pci *p) +{ + struct device *dev = p->dev; + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(p); + struct resource_entry *win; + struct resource *res; + phys_addr_t addr; + u32 ahbmembase; + + win = resource_list_first_type(&bridge->dma_ranges, IORESOURCE_MEM); + if (win) { + res = win->res; + addr = res->start - win->offset; + + if (resource_size(res) != SZ_64M) { + dev_err(dev, "DMA memory range is not 64MB\n"); + return -EINVAL; + } + + dev_dbg(dev, "DMA MEM BASE: %pa\n", &addr); + /* + * 4 PCI-to-AHB windows of 16 MB each, write the 8 high bits + * into each byte of the PCI_AHBMEMBASE register. + */ + ahbmembase = ixp4xx_pci_addr_to_64mconf(addr); + /* Commit AHB membase */ + ixp4xx_writel(p, IXP4XX_PCI_AHBMEMBASE, ahbmembase); + } else { + dev_err(dev, "no DMA memory range defined\n"); + } + + return 0; +} + +#ifdef CONFIG_ARM +/* Only used to get context for abort handling */ +static struct ixp4xx_pci *ixp4xx_pci_abort_singleton; + +static int ixp4xx_pci_abort_handler(unsigned long addr, unsigned int fsr, + struct pt_regs *regs) +{ + struct ixp4xx_pci *p = ixp4xx_pci_abort_singleton; + u32 isr, status; + int ret; + + isr = ixp4xx_readl(p, IXP4XX_PCI_ISR); + ret = ixp4xx_crp_read_config(p, PCI_STATUS, 2, &status); + if (ret) { + dev_err(p->dev, "unable to read abort status\n"); + return -EINVAL; + } + + dev_err(p->dev, + "PCI: abort_handler addr = %#lx, isr = %#x, status = %#x\n", + addr, isr, status); + + /* Make sure the Master Abort bit is reset */ + ixp4xx_writel(p, IXP4XX_PCI_ISR, IXP4XX_PCI_ISR_PFE); + status |= PCI_STATUS_REC_MASTER_ABORT; + ret = ixp4xx_crp_write_config(p, PCI_STATUS, 2, status); + if (ret) + dev_err(p->dev, "unable to clear abort status bit\n"); + + /* + * If it was an imprecise abort, then we need to correct the + * return address to be _after_ the instruction. + */ + if (fsr & (1 << 10)) { + dev_err(p->dev, "imprecise abort\n"); + regs->ARM_pc += 4; + } + + return 0; +} +#endif + +static int __init ixp4xx_pci_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct ixp4xx_pci *p; + struct pci_host_bridge *host; + int ret; + u32 val; + phys_addr_t addr; + u32 basereg[4] = { + PCI_BASE_ADDRESS_0, + PCI_BASE_ADDRESS_1, + PCI_BASE_ADDRESS_2, + PCI_BASE_ADDRESS_3, + }; + int i; + + host = devm_pci_alloc_host_bridge(dev, sizeof(*p)); + if (!host) + return -ENOMEM; + + host->ops = &ixp4xx_pci_ops; + p = pci_host_bridge_priv(host); + host->sysdata = p; + p->dev = dev; + dev_set_drvdata(dev, p); + + /* + * Set up quirk for erratic behaviour in the 42x variant + * when accessing config space. + */ + if (of_device_is_compatible(np, "intel,ixp42x-pci")) { + p->errata_hammer = true; + dev_info(dev, "activate hammering errata\n"); + } + + p->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(p->base)) + return PTR_ERR(p->base); + + val = ixp4xx_readl(p, IXP4XX_PCI_CSR); + p->host_mode = !!(val & IXP4XX_PCI_CSR_HOST); + dev_info(dev, "controller is in %s mode\n", + p->host_mode ? "host" : "option"); + +#ifdef CONFIG_ARM + /* Hook in our fault handler for PCI errors */ + ixp4xx_pci_abort_singleton = p; + hook_fault_code(16+6, ixp4xx_pci_abort_handler, SIGBUS, 0, + "imprecise external abort"); +#endif + + ret = ixp4xx_pci_parse_map_ranges(p); + if (ret) + return ret; + + ret = ixp4xx_pci_parse_map_dma_ranges(p); + if (ret) + return ret; + + /* This is only configured in host mode */ + if (p->host_mode) { + addr = __pa(PAGE_OFFSET); + /* This is a noop (0x00) but explains what is going on */ + addr |= PCI_BASE_ADDRESS_SPACE_MEMORY; + + for (i = 0; i < 4; i++) { + /* Write this directly into the config space */ + ret = ixp4xx_crp_write_config(p, basereg[i], 4, addr); + if (ret) + dev_err(dev, "failed to set up PCI_BASE_ADDRESS_%d\n", i); + else + dev_info(dev, "set PCI_BASE_ADDR_%d to %pa\n", i, &addr); + addr += SZ_16M; + } + + /* + * Enable CSR window at 64 MiB to allow PCI masters to continue + * prefetching past the 64 MiB boundary, if all AHB to PCI + * windows are consecutive. + */ + ret = ixp4xx_crp_write_config(p, PCI_BASE_ADDRESS_4, 4, addr); + if (ret) + dev_err(dev, "failed to set up PCI_BASE_ADDRESS_4\n"); + else + dev_info(dev, "set PCI_BASE_ADDR_4 to %pa\n", &addr); + + /* + * Put the IO memory window at the very end of physical memory + * at 0xfffffc00. This is when the system is trying to access IO + * memory over AHB. + */ + addr = 0xfffffc00; + addr |= PCI_BASE_ADDRESS_SPACE_IO; + ret = ixp4xx_crp_write_config(p, PCI_BASE_ADDRESS_5, 4, addr); + if (ret) + dev_err(dev, "failed to set up PCI_BASE_ADDRESS_5\n"); + else + dev_info(dev, "set PCI_BASE_ADDR_5 to %pa\n", &addr); + + /* + * Retry timeout to 0x80 + * Transfer ready timeout to 0xff + */ + ret = ixp4xx_crp_write_config(p, IXP4XX_PCI_RTOTTO, 4, + 0x000080ff); + if (ret) + dev_err(dev, "failed to set up TRDY limit\n"); + else + dev_info(dev, "set TRDY limit to 0x80ff\n"); + } + + /* Clear interrupts */ + val = IXP4XX_PCI_ISR_PSE | IXP4XX_PCI_ISR_PFE | IXP4XX_PCI_ISR_PPE | IXP4XX_PCI_ISR_AHBE; + ixp4xx_writel(p, IXP4XX_PCI_ISR, val); + + /* + * Set Initialize Complete in PCI Control Register: allow IXP4XX to + * generate PCI configuration cycles. Specify that the AHB bus is + * operating in big-endian mode. Set up byte lane swapping between + * little-endian PCI and the big-endian AHB bus. + */ + val = IXP4XX_PCI_CSR_IC | IXP4XX_PCI_CSR_ABE; + if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) + val |= (IXP4XX_PCI_CSR_PDS | IXP4XX_PCI_CSR_ADS); + ixp4xx_writel(p, IXP4XX_PCI_CSR, val); + + ret = ixp4xx_crp_write_config(p, PCI_COMMAND, 2, PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY); + if (ret) + dev_err(dev, "unable to initialize master and command memory\n"); + else + dev_info(dev, "initialized as master\n"); + + pci_host_probe(host); + + return 0; +} + +static const struct of_device_id ixp4xx_pci_of_match[] = { + { + .compatible = "intel,ixp42x-pci", + }, + { + .compatible = "intel,ixp43x-pci", + }, + {}, +}; + +/* + * This driver needs to be a builtin module with suppressed bind + * attributes since the probe() is initializing a hard exception + * handler and this can only be done from __init-tagged code + * sections. This module cannot be removed and inserted at all. + */ +static struct platform_driver ixp4xx_pci_driver = { + .driver = { + .name = "ixp4xx-pci", + .suppress_bind_attrs = true, + .of_match_table = ixp4xx_pci_of_match, + }, +}; +builtin_platform_driver_probe(ixp4xx_pci_driver, ixp4xx_pci_probe); diff --git a/drivers/pci/controller/pci-loongson.c b/drivers/pci/controller/pci-loongson.c new file mode 100644 index 000000000..d0c643996 --- /dev/null +++ b/drivers/pci/controller/pci-loongson.c @@ -0,0 +1,447 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Loongson PCI Host Controller Driver + * + * Copyright (C) 2020 Jiaxun Yang <jiaxun.yang@flygoat.com> + */ + +#include <linux/of.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/pci_ids.h> +#include <linux/pci-acpi.h> +#include <linux/pci-ecam.h> + +#include "../pci.h" + +/* Device IDs */ +#define DEV_LS2K_PCIE_PORT0 0x1a05 +#define DEV_LS7A_PCIE_PORT0 0x7a09 +#define DEV_LS7A_PCIE_PORT1 0x7a19 +#define DEV_LS7A_PCIE_PORT2 0x7a29 +#define DEV_LS7A_PCIE_PORT3 0x7a39 +#define DEV_LS7A_PCIE_PORT4 0x7a49 +#define DEV_LS7A_PCIE_PORT5 0x7a59 +#define DEV_LS7A_PCIE_PORT6 0x7a69 + +#define DEV_LS2K_APB 0x7a02 +#define DEV_LS7A_GMAC 0x7a03 +#define DEV_LS7A_DC1 0x7a06 +#define DEV_LS7A_LPC 0x7a0c +#define DEV_LS7A_AHCI 0x7a08 +#define DEV_LS7A_CONF 0x7a10 +#define DEV_LS7A_GNET 0x7a13 +#define DEV_LS7A_EHCI 0x7a14 +#define DEV_LS7A_DC2 0x7a36 +#define DEV_LS7A_HDMI 0x7a37 + +#define FLAG_CFG0 BIT(0) +#define FLAG_CFG1 BIT(1) +#define FLAG_DEV_FIX BIT(2) +#define FLAG_DEV_HIDDEN BIT(3) + +struct loongson_pci_data { + u32 flags; + struct pci_ops *ops; +}; + +struct loongson_pci { + void __iomem *cfg0_base; + void __iomem *cfg1_base; + struct platform_device *pdev; + const struct loongson_pci_data *data; +}; + +/* Fixup wrong class code in PCIe bridges */ +static void bridge_class_quirk(struct pci_dev *dev) +{ + dev->class = PCI_CLASS_BRIDGE_PCI_NORMAL; +} +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_PCIE_PORT0, bridge_class_quirk); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_PCIE_PORT1, bridge_class_quirk); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_PCIE_PORT2, bridge_class_quirk); + +static void system_bus_quirk(struct pci_dev *pdev) +{ + /* + * The address space consumed by these devices is outside the + * resources of the host bridge. + */ + pdev->mmio_always_on = 1; + pdev->non_compliant_bars = 1; +} +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, + DEV_LS2K_APB, system_bus_quirk); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_CONF, system_bus_quirk); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_LPC, system_bus_quirk); + +static const struct pci_device_id loongson_internal_bridge_devids[] = { + { PCI_VDEVICE(LOONGSON, DEV_LS2K_PCIE_PORT0) }, + { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT0) }, + { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT1) }, + { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT2) }, + { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT3) }, + { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT4) }, + { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT5) }, + { PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT6) }, + { 0, }, +}; + +/* + * Some Loongson PCIe ports have hardware limitations on their Maximum Read + * Request Size. They can't handle anything larger than this. Sane + * firmware will set proper MRRS at boot, so we only need no_inc_mrrs for + * bridges. However, some MIPS Loongson firmware doesn't set MRRS properly, + * so we have to enforce maximum safe MRRS, which is 256 bytes. + */ +#ifdef CONFIG_MIPS +static void loongson_set_min_mrrs_quirk(struct pci_dev *pdev) +{ + struct pci_bus *bus = pdev->bus; + struct pci_dev *bridge; + + /* look for the matching bridge */ + while (!pci_is_root_bus(bus)) { + bridge = bus->self; + bus = bus->parent; + + if (pci_match_id(loongson_internal_bridge_devids, bridge)) { + if (pcie_get_readrq(pdev) > 256) { + pci_info(pdev, "limiting MRRS to 256\n"); + pcie_set_readrq(pdev, 256); + } + break; + } + } +} +DECLARE_PCI_FIXUP_ENABLE(PCI_ANY_ID, PCI_ANY_ID, loongson_set_min_mrrs_quirk); +#endif + +static void loongson_mrrs_quirk(struct pci_dev *pdev) +{ + struct pci_host_bridge *bridge = pci_find_host_bridge(pdev->bus); + + bridge->no_inc_mrrs = 1; +} +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, + DEV_LS2K_PCIE_PORT0, loongson_mrrs_quirk); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_PCIE_PORT0, loongson_mrrs_quirk); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_PCIE_PORT1, loongson_mrrs_quirk); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_PCIE_PORT2, loongson_mrrs_quirk); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_PCIE_PORT3, loongson_mrrs_quirk); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_PCIE_PORT4, loongson_mrrs_quirk); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_PCIE_PORT5, loongson_mrrs_quirk); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_PCIE_PORT6, loongson_mrrs_quirk); + +static void loongson_pci_pin_quirk(struct pci_dev *pdev) +{ + pdev->pin = 1 + (PCI_FUNC(pdev->devfn) & 3); +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_DC1, loongson_pci_pin_quirk); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_DC2, loongson_pci_pin_quirk); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_GMAC, loongson_pci_pin_quirk); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_AHCI, loongson_pci_pin_quirk); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_EHCI, loongson_pci_pin_quirk); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_GNET, loongson_pci_pin_quirk); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, + DEV_LS7A_HDMI, loongson_pci_pin_quirk); + +static void loongson_pci_msi_quirk(struct pci_dev *dev) +{ + u16 val, class = dev->class >> 8; + + if (class != PCI_CLASS_BRIDGE_HOST) + return; + + pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &val); + val |= PCI_MSI_FLAGS_ENABLE; + pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, val); +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, DEV_LS7A_PCIE_PORT5, loongson_pci_msi_quirk); + +/* + * Older steppings of the Loongson-3C6000 series incorrectly report the + * supported link speeds on their PCIe bridges (device IDs 0x3c19, + * 0x3c29) as only 2.5 GT/s, despite the upstream bus supporting speeds + * from 2.5 GT/s up to 16 GT/s. + */ +static void loongson_pci_bridge_speed_quirk(struct pci_dev *pdev) +{ + u8 old_supported_speeds = pdev->supported_speeds; + + switch (pdev->bus->max_bus_speed) { + case PCIE_SPEED_16_0GT: + pdev->supported_speeds |= PCI_EXP_LNKCAP2_SLS_16_0GB; + fallthrough; + case PCIE_SPEED_8_0GT: + pdev->supported_speeds |= PCI_EXP_LNKCAP2_SLS_8_0GB; + fallthrough; + case PCIE_SPEED_5_0GT: + pdev->supported_speeds |= PCI_EXP_LNKCAP2_SLS_5_0GB; + fallthrough; + case PCIE_SPEED_2_5GT: + pdev->supported_speeds |= PCI_EXP_LNKCAP2_SLS_2_5GB; + break; + default: + pci_warn(pdev, "unexpected max bus speed"); + + return; + } + + if (pdev->supported_speeds != old_supported_speeds) + pci_info(pdev, "fixed up supported link speeds: 0x%x => 0x%x", + old_supported_speeds, pdev->supported_speeds); +} +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LOONGSON, 0x3c19, loongson_pci_bridge_speed_quirk); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LOONGSON, 0x3c29, loongson_pci_bridge_speed_quirk); + +static struct loongson_pci *pci_bus_to_loongson_pci(struct pci_bus *bus) +{ + struct pci_config_window *cfg; + + if (acpi_disabled) + return (struct loongson_pci *)(bus->sysdata); + + cfg = bus->sysdata; + return (struct loongson_pci *)(cfg->priv); +} + +static void __iomem *cfg0_map(struct loongson_pci *priv, struct pci_bus *bus, + unsigned int devfn, int where) +{ + unsigned long addroff = 0x0; + unsigned char busnum = bus->number; + + if (!pci_is_root_bus(bus)) { + addroff |= BIT(24); /* Type 1 Access */ + addroff |= (busnum << 16); + } + addroff |= (devfn << 8) | where; + return priv->cfg0_base + addroff; +} + +static void __iomem *cfg1_map(struct loongson_pci *priv, struct pci_bus *bus, + unsigned int devfn, int where) +{ + unsigned long addroff = 0x0; + unsigned char busnum = bus->number; + + if (!pci_is_root_bus(bus)) { + addroff |= BIT(28); /* Type 1 Access */ + addroff |= (busnum << 16); + } + addroff |= (devfn << 8) | (where & 0xff) | ((where & 0xf00) << 16); + return priv->cfg1_base + addroff; +} + +static bool pdev_may_exist(struct pci_bus *bus, unsigned int device, + unsigned int function) +{ + return !(pci_is_root_bus(bus) && + (device >= 9 && device <= 20) && (function > 0)); +} + +static void __iomem *pci_loongson_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + unsigned int device = PCI_SLOT(devfn); + unsigned int function = PCI_FUNC(devfn); + struct loongson_pci *priv = pci_bus_to_loongson_pci(bus); + + /* + * Do not read more than one device on the internal bridges. + */ + if ((priv->data->flags & FLAG_DEV_FIX) && bus->self) { + if (!pci_is_root_bus(bus) && (device > 0) && + pci_match_id(loongson_internal_bridge_devids, bus->self)) + return NULL; + } + + /* Don't access non-existent devices */ + if (priv->data->flags & FLAG_DEV_HIDDEN) { + if (!pdev_may_exist(bus, device, function)) + return NULL; + } + + /* CFG0 can only access standard space */ + if (where < PCI_CFG_SPACE_SIZE && priv->cfg0_base) + return cfg0_map(priv, bus, devfn, where); + + /* CFG1 can access extended space */ + if (where < PCI_CFG_SPACE_EXP_SIZE && priv->cfg1_base) + return cfg1_map(priv, bus, devfn, where); + + return NULL; +} + +#ifdef CONFIG_OF + +static int loongson_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) +{ + int irq; + u8 val; + + irq = of_irq_parse_and_map_pci(dev, slot, pin); + if (irq > 0) + return irq; + + /* Care i8259 legacy systems */ + pci_read_config_byte(dev, PCI_INTERRUPT_LINE, &val); + /* i8259 only have 15 IRQs */ + if (val > 15) + return 0; + + return val; +} + +/* LS2K/LS7A accept 8/16/32-bit PCI config operations */ +static struct pci_ops loongson_pci_ops = { + .map_bus = pci_loongson_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; + +/* RS780/SR5690 only accept 32-bit PCI config operations */ +static struct pci_ops loongson_pci_ops32 = { + .map_bus = pci_loongson_map_bus, + .read = pci_generic_config_read32, + .write = pci_generic_config_write32, +}; + +static const struct loongson_pci_data ls2k_pci_data = { + .flags = FLAG_CFG1 | FLAG_DEV_FIX | FLAG_DEV_HIDDEN, + .ops = &loongson_pci_ops, +}; + +static const struct loongson_pci_data ls7a_pci_data = { + .flags = FLAG_CFG1 | FLAG_DEV_FIX | FLAG_DEV_HIDDEN, + .ops = &loongson_pci_ops, +}; + +static const struct loongson_pci_data rs780e_pci_data = { + .flags = FLAG_CFG0, + .ops = &loongson_pci_ops32, +}; + +static const struct of_device_id loongson_pci_of_match[] = { + { .compatible = "loongson,ls2k-pci", + .data = &ls2k_pci_data, }, + { .compatible = "loongson,ls7a-pci", + .data = &ls7a_pci_data, }, + { .compatible = "loongson,rs780e-pci", + .data = &rs780e_pci_data, }, + {} +}; + +static int loongson_pci_probe(struct platform_device *pdev) +{ + struct loongson_pci *priv; + struct device *dev = &pdev->dev; + struct device_node *node = dev->of_node; + struct pci_host_bridge *bridge; + struct resource *regs; + + if (!node) + return -ENODEV; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*priv)); + if (!bridge) + return -ENODEV; + + priv = pci_host_bridge_priv(bridge); + priv->pdev = pdev; + priv->data = of_device_get_match_data(dev); + + if (priv->data->flags & FLAG_CFG0) { + regs = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!regs) + dev_err(dev, "missing mem resources for cfg0\n"); + else { + priv->cfg0_base = devm_pci_remap_cfg_resource(dev, regs); + if (IS_ERR(priv->cfg0_base)) + return PTR_ERR(priv->cfg0_base); + } + } + + if (priv->data->flags & FLAG_CFG1) { + regs = platform_get_resource(pdev, IORESOURCE_MEM, 1); + if (!regs) + dev_info(dev, "missing mem resource for cfg1\n"); + else { + priv->cfg1_base = devm_pci_remap_cfg_resource(dev, regs); + if (IS_ERR(priv->cfg1_base)) + priv->cfg1_base = NULL; + } + } + + bridge->sysdata = priv; + bridge->ops = priv->data->ops; + bridge->map_irq = loongson_map_irq; + + return pci_host_probe(bridge); +} + +static struct platform_driver loongson_pci_driver = { + .driver = { + .name = "loongson-pci", + .of_match_table = loongson_pci_of_match, + }, + .probe = loongson_pci_probe, +}; +builtin_platform_driver(loongson_pci_driver); + +#endif + +#ifdef CONFIG_ACPI + +static int loongson_pci_ecam_init(struct pci_config_window *cfg) +{ + struct device *dev = cfg->parent; + struct loongson_pci *priv; + struct loongson_pci_data *data; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + cfg->priv = priv; + data->flags = FLAG_CFG1 | FLAG_DEV_HIDDEN; + priv->data = data; + priv->cfg1_base = cfg->win - (cfg->busr.start << 16); + + return 0; +} + +const struct pci_ecam_ops loongson_pci_ecam_ops = { + .bus_shift = 16, + .init = loongson_pci_ecam_init, + .pci_ops = { + .map_bus = pci_loongson_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, + } +}; + +#endif diff --git a/drivers/pci/controller/pci-mvebu.c b/drivers/pci/controller/pci-mvebu.c new file mode 100644 index 000000000..69a9d7703 --- /dev/null +++ b/drivers/pci/controller/pci-mvebu.c @@ -0,0 +1,1714 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe driver for Marvell Armada 370 and Armada XP SoCs + * + * Author: Thomas Petazzoni <thomas.petazzoni@free-electrons.com> + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/gpio/consumer.h> +#include <linux/init.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqdomain.h> +#include <linux/mbus.h> +#include <linux/slab.h> +#include <linux/platform_device.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> + +#include "../pci.h" +#include "../pci-bridge-emul.h" + +/* + * PCIe unit register offsets. + */ +#define PCIE_DEV_ID_OFF 0x0000 +#define PCIE_CMD_OFF 0x0004 +#define PCIE_DEV_REV_OFF 0x0008 +#define PCIE_BAR_LO_OFF(n) (0x0010 + ((n) << 3)) +#define PCIE_BAR_HI_OFF(n) (0x0014 + ((n) << 3)) +#define PCIE_SSDEV_ID_OFF 0x002c +#define PCIE_CAP_PCIEXP 0x0060 +#define PCIE_CAP_PCIERR_OFF 0x0100 +#define PCIE_BAR_CTRL_OFF(n) (0x1804 + (((n) - 1) * 4)) +#define PCIE_WIN04_CTRL_OFF(n) (0x1820 + ((n) << 4)) +#define PCIE_WIN04_BASE_OFF(n) (0x1824 + ((n) << 4)) +#define PCIE_WIN04_REMAP_OFF(n) (0x182c + ((n) << 4)) +#define PCIE_WIN5_CTRL_OFF 0x1880 +#define PCIE_WIN5_BASE_OFF 0x1884 +#define PCIE_WIN5_REMAP_OFF 0x188c +#define PCIE_CONF_ADDR_OFF 0x18f8 +#define PCIE_CONF_ADDR_EN 0x80000000 +#define PCIE_CONF_REG(r) ((((r) & 0xf00) << 16) | ((r) & 0xfc)) +#define PCIE_CONF_BUS(b) (((b) & 0xff) << 16) +#define PCIE_CONF_DEV(d) (((d) & 0x1f) << 11) +#define PCIE_CONF_FUNC(f) (((f) & 0x7) << 8) +#define PCIE_CONF_ADDR(bus, devfn, where) \ + (PCIE_CONF_BUS(bus) | PCIE_CONF_DEV(PCI_SLOT(devfn)) | \ + PCIE_CONF_FUNC(PCI_FUNC(devfn)) | PCIE_CONF_REG(where) | \ + PCIE_CONF_ADDR_EN) +#define PCIE_CONF_DATA_OFF 0x18fc +#define PCIE_INT_CAUSE_OFF 0x1900 +#define PCIE_INT_UNMASK_OFF 0x1910 +#define PCIE_INT_INTX(i) BIT_U32(24 + (i)) +#define PCIE_INT_PM_PME BIT_U32(28) +#define PCIE_INT_ALL_MASK GENMASK_U32(31, 0) +#define PCIE_CTRL_OFF 0x1a00 +#define PCIE_CTRL_X1_MODE 0x0001 +#define PCIE_CTRL_RC_MODE BIT(1) +#define PCIE_CTRL_MASTER_HOT_RESET BIT(24) +#define PCIE_STAT_OFF 0x1a04 +#define PCIE_STAT_BUS 0xff00 +#define PCIE_STAT_DEV 0x1f0000 +#define PCIE_STAT_LINK_DOWN BIT(0) +#define PCIE_SSPL_OFF 0x1a0c +#define PCIE_SSPL_VALUE_SHIFT 0 +#define PCIE_SSPL_VALUE_MASK GENMASK(7, 0) +#define PCIE_SSPL_SCALE_SHIFT 8 +#define PCIE_SSPL_SCALE_MASK GENMASK(9, 8) +#define PCIE_SSPL_ENABLE BIT(16) +#define PCIE_RC_RTSTA 0x1a14 +#define PCIE_DEBUG_CTRL 0x1a60 +#define PCIE_DEBUG_SOFT_RESET BIT(20) + +struct mvebu_pcie_port; + +/* Structure representing all PCIe interfaces */ +struct mvebu_pcie { + struct platform_device *pdev; + struct mvebu_pcie_port *ports; + struct resource io; + struct resource realio; + struct resource mem; + int nports; +}; + +struct mvebu_pcie_window { + phys_addr_t base; + phys_addr_t remap; + size_t size; +}; + +/* Structure representing one PCIe interface */ +struct mvebu_pcie_port { + char *name; + void __iomem *base; + u32 port; + u32 lane; + bool is_x4; + int devfn; + unsigned int mem_target; + unsigned int mem_attr; + unsigned int io_target; + unsigned int io_attr; + struct clk *clk; + struct gpio_desc *reset_gpio; + char *reset_name; + struct pci_bridge_emul bridge; + struct device_node *dn; + struct mvebu_pcie *pcie; + struct mvebu_pcie_window memwin; + struct mvebu_pcie_window iowin; + u32 saved_pcie_stat; + struct resource regs; + u8 slot_power_limit_value; + u8 slot_power_limit_scale; + struct irq_domain *intx_irq_domain; + raw_spinlock_t irq_lock; + int intx_irq; +}; + +static inline void mvebu_writel(struct mvebu_pcie_port *port, u32 val, u32 reg) +{ + writel(val, port->base + reg); +} + +static inline u32 mvebu_readl(struct mvebu_pcie_port *port, u32 reg) +{ + return readl(port->base + reg); +} + +static inline bool mvebu_has_ioport(struct mvebu_pcie_port *port) +{ + return port->io_target != -1 && port->io_attr != -1; +} + +static bool mvebu_pcie_link_up(struct mvebu_pcie_port *port) +{ + return !(mvebu_readl(port, PCIE_STAT_OFF) & PCIE_STAT_LINK_DOWN); +} + +static u8 mvebu_pcie_get_local_bus_nr(struct mvebu_pcie_port *port) +{ + return (mvebu_readl(port, PCIE_STAT_OFF) & PCIE_STAT_BUS) >> 8; +} + +static void mvebu_pcie_set_local_bus_nr(struct mvebu_pcie_port *port, int nr) +{ + u32 stat; + + stat = mvebu_readl(port, PCIE_STAT_OFF); + stat &= ~PCIE_STAT_BUS; + stat |= nr << 8; + mvebu_writel(port, stat, PCIE_STAT_OFF); +} + +static void mvebu_pcie_set_local_dev_nr(struct mvebu_pcie_port *port, int nr) +{ + u32 stat; + + stat = mvebu_readl(port, PCIE_STAT_OFF); + stat &= ~PCIE_STAT_DEV; + stat |= nr << 16; + mvebu_writel(port, stat, PCIE_STAT_OFF); +} + +static void mvebu_pcie_disable_wins(struct mvebu_pcie_port *port) +{ + int i; + + mvebu_writel(port, 0, PCIE_BAR_LO_OFF(0)); + mvebu_writel(port, 0, PCIE_BAR_HI_OFF(0)); + + for (i = 1; i < 3; i++) { + mvebu_writel(port, 0, PCIE_BAR_CTRL_OFF(i)); + mvebu_writel(port, 0, PCIE_BAR_LO_OFF(i)); + mvebu_writel(port, 0, PCIE_BAR_HI_OFF(i)); + } + + for (i = 0; i < 5; i++) { + mvebu_writel(port, 0, PCIE_WIN04_CTRL_OFF(i)); + mvebu_writel(port, 0, PCIE_WIN04_BASE_OFF(i)); + mvebu_writel(port, 0, PCIE_WIN04_REMAP_OFF(i)); + } + + mvebu_writel(port, 0, PCIE_WIN5_CTRL_OFF); + mvebu_writel(port, 0, PCIE_WIN5_BASE_OFF); + mvebu_writel(port, 0, PCIE_WIN5_REMAP_OFF); +} + +/* + * Setup PCIE BARs and Address Decode Wins: + * BAR[0] -> internal registers (needed for MSI) + * BAR[1] -> covers all DRAM banks + * BAR[2] -> Disabled + * WIN[0-3] -> DRAM bank[0-3] + */ +static void mvebu_pcie_setup_wins(struct mvebu_pcie_port *port) +{ + const struct mbus_dram_target_info *dram; + u32 size; + int i; + + dram = mv_mbus_dram_info(); + + /* First, disable and clear BARs and windows. */ + mvebu_pcie_disable_wins(port); + + /* Setup windows for DDR banks. Count total DDR size on the fly. */ + size = 0; + for (i = 0; i < dram->num_cs; i++) { + const struct mbus_dram_window *cs = dram->cs + i; + + mvebu_writel(port, cs->base & 0xffff0000, + PCIE_WIN04_BASE_OFF(i)); + mvebu_writel(port, 0, PCIE_WIN04_REMAP_OFF(i)); + mvebu_writel(port, + ((cs->size - 1) & 0xffff0000) | + (cs->mbus_attr << 8) | + (dram->mbus_dram_target_id << 4) | 1, + PCIE_WIN04_CTRL_OFF(i)); + + size += cs->size; + } + + /* Round up 'size' to the nearest power of two. */ + if ((size & (size - 1)) != 0) + size = 1 << fls(size); + + /* Setup BAR[1] to all DRAM banks. */ + mvebu_writel(port, dram->cs[0].base, PCIE_BAR_LO_OFF(1)); + mvebu_writel(port, 0, PCIE_BAR_HI_OFF(1)); + mvebu_writel(port, ((size - 1) & 0xffff0000) | 1, + PCIE_BAR_CTRL_OFF(1)); + + /* + * Point BAR[0] to the device's internal registers. + */ + mvebu_writel(port, round_down(port->regs.start, SZ_1M), PCIE_BAR_LO_OFF(0)); + mvebu_writel(port, 0, PCIE_BAR_HI_OFF(0)); +} + +static void mvebu_pcie_setup_hw(struct mvebu_pcie_port *port) +{ + u32 ctrl, lnkcap, cmd, dev_rev, unmask, sspl; + + /* Setup PCIe controller to Root Complex mode. */ + ctrl = mvebu_readl(port, PCIE_CTRL_OFF); + ctrl |= PCIE_CTRL_RC_MODE; + mvebu_writel(port, ctrl, PCIE_CTRL_OFF); + + /* + * Set Maximum Link Width to X1 or X4 in Root Port's PCIe Link + * Capability register. This register is defined by PCIe specification + * as read-only but this mvebu controller has it as read-write and must + * be set to number of SerDes PCIe lanes (1 or 4). If this register is + * not set correctly then link with endpoint card is not established. + */ + lnkcap = mvebu_readl(port, PCIE_CAP_PCIEXP + PCI_EXP_LNKCAP); + FIELD_MODIFY(PCI_EXP_LNKCAP_MLW, &lnkcap, port->is_x4 ? 4 : 1); + mvebu_writel(port, lnkcap, PCIE_CAP_PCIEXP + PCI_EXP_LNKCAP); + + /* Disable Root Bridge I/O space, memory space and bus mastering. */ + cmd = mvebu_readl(port, PCIE_CMD_OFF); + cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + mvebu_writel(port, cmd, PCIE_CMD_OFF); + + /* + * Change Class Code of PCI Bridge device to PCI Bridge (0x6004) + * because default value is Memory controller (0x5080). + * + * Note that this mvebu PCI Bridge does not have compliant Type 1 + * Configuration Space. Header Type is reported as Type 0 and it + * has format of Type 0 config space. + * + * Moreover Type 0 BAR registers (ranges 0x10 - 0x28 and 0x30 - 0x34) + * have the same format in Marvell's specification as in PCIe + * specification, but their meaning is totally different and they do + * different things: they are aliased into internal mvebu registers + * (e.g. PCIE_BAR_LO_OFF) and these should not be changed or + * reconfigured by pci device drivers. + * + * Therefore driver uses emulation of PCI Bridge which emulates + * access to configuration space via internal mvebu registers or + * emulated configuration buffer. Driver access these PCI Bridge + * directly for simplification, but these registers can be accessed + * also via standard mvebu way for accessing PCI config space. + */ + dev_rev = mvebu_readl(port, PCIE_DEV_REV_OFF); + dev_rev &= ~0xffffff00; + dev_rev |= PCI_CLASS_BRIDGE_PCI_NORMAL << 8; + mvebu_writel(port, dev_rev, PCIE_DEV_REV_OFF); + + /* Point PCIe unit MBUS decode windows to DRAM space. */ + mvebu_pcie_setup_wins(port); + + /* + * Program Root Port to automatically send Set_Slot_Power_Limit + * PCIe Message when changing status from Dl_Down to Dl_Up and valid + * slot power limit was specified. + */ + sspl = mvebu_readl(port, PCIE_SSPL_OFF); + sspl &= ~(PCIE_SSPL_VALUE_MASK | PCIE_SSPL_SCALE_MASK | PCIE_SSPL_ENABLE); + if (port->slot_power_limit_value) { + sspl |= port->slot_power_limit_value << PCIE_SSPL_VALUE_SHIFT; + sspl |= port->slot_power_limit_scale << PCIE_SSPL_SCALE_SHIFT; + sspl |= PCIE_SSPL_ENABLE; + } + mvebu_writel(port, sspl, PCIE_SSPL_OFF); + + /* Mask all interrupt sources. */ + mvebu_writel(port, ~PCIE_INT_ALL_MASK, PCIE_INT_UNMASK_OFF); + + /* Clear all interrupt causes. */ + mvebu_writel(port, ~PCIE_INT_ALL_MASK, PCIE_INT_CAUSE_OFF); + + /* Check if "intx" interrupt was specified in DT. */ + if (port->intx_irq > 0) + return; + + /* + * Fallback code when "intx" interrupt was not specified in DT: + * Unmask all legacy INTx interrupts as driver does not provide a way + * for masking and unmasking of individual legacy INTx interrupts. + * Legacy INTx are reported via one shared GIC source and therefore + * kernel cannot distinguish which individual legacy INTx was triggered. + * These interrupts are shared, so it should not cause any issue. Just + * performance penalty as every PCIe interrupt handler needs to be + * called when some interrupt is triggered. + */ + unmask = mvebu_readl(port, PCIE_INT_UNMASK_OFF); + unmask |= PCIE_INT_INTX(0) | PCIE_INT_INTX(1) | + PCIE_INT_INTX(2) | PCIE_INT_INTX(3); + mvebu_writel(port, unmask, PCIE_INT_UNMASK_OFF); +} + +static struct mvebu_pcie_port *mvebu_pcie_find_port(struct mvebu_pcie *pcie, + struct pci_bus *bus, + int devfn); + +static int mvebu_pcie_child_rd_conf(struct pci_bus *bus, u32 devfn, int where, + int size, u32 *val) +{ + struct mvebu_pcie *pcie = bus->sysdata; + struct mvebu_pcie_port *port; + void __iomem *conf_data; + + port = mvebu_pcie_find_port(pcie, bus, devfn); + if (!port) + return PCIBIOS_DEVICE_NOT_FOUND; + + if (!mvebu_pcie_link_up(port)) + return PCIBIOS_DEVICE_NOT_FOUND; + + conf_data = port->base + PCIE_CONF_DATA_OFF; + + mvebu_writel(port, PCIE_CONF_ADDR(bus->number, devfn, where), + PCIE_CONF_ADDR_OFF); + + switch (size) { + case 1: + *val = readb_relaxed(conf_data + (where & 3)); + break; + case 2: + *val = readw_relaxed(conf_data + (where & 2)); + break; + case 4: + *val = readl_relaxed(conf_data); + break; + default: + return PCIBIOS_BAD_REGISTER_NUMBER; + } + + return PCIBIOS_SUCCESSFUL; +} + +static int mvebu_pcie_child_wr_conf(struct pci_bus *bus, u32 devfn, + int where, int size, u32 val) +{ + struct mvebu_pcie *pcie = bus->sysdata; + struct mvebu_pcie_port *port; + void __iomem *conf_data; + + port = mvebu_pcie_find_port(pcie, bus, devfn); + if (!port) + return PCIBIOS_DEVICE_NOT_FOUND; + + if (!mvebu_pcie_link_up(port)) + return PCIBIOS_DEVICE_NOT_FOUND; + + conf_data = port->base + PCIE_CONF_DATA_OFF; + + mvebu_writel(port, PCIE_CONF_ADDR(bus->number, devfn, where), + PCIE_CONF_ADDR_OFF); + + switch (size) { + case 1: + writeb(val, conf_data + (where & 3)); + break; + case 2: + writew(val, conf_data + (where & 2)); + break; + case 4: + writel(val, conf_data); + break; + default: + return PCIBIOS_BAD_REGISTER_NUMBER; + } + + return PCIBIOS_SUCCESSFUL; +} + +static struct pci_ops mvebu_pcie_child_ops = { + .read = mvebu_pcie_child_rd_conf, + .write = mvebu_pcie_child_wr_conf, +}; + +/* + * Remove windows, starting from the largest ones to the smallest + * ones. + */ +static void mvebu_pcie_del_windows(struct mvebu_pcie_port *port, + phys_addr_t base, size_t size) +{ + while (size) { + size_t sz = 1 << (fls(size) - 1); + + mvebu_mbus_del_window(base, sz); + base += sz; + size -= sz; + } +} + +/* + * MBus windows can only have a power of two size, but PCI BARs do not + * have this constraint. Therefore, we have to split the PCI BAR into + * areas each having a power of two size. We start from the largest + * one (i.e highest order bit set in the size). + */ +static int mvebu_pcie_add_windows(struct mvebu_pcie_port *port, + unsigned int target, unsigned int attribute, + phys_addr_t base, size_t size, + phys_addr_t remap) +{ + size_t size_mapped = 0; + + while (size) { + size_t sz = 1 << (fls(size) - 1); + int ret; + + ret = mvebu_mbus_add_window_remap_by_id(target, attribute, base, + sz, remap); + if (ret) { + phys_addr_t end = base + sz - 1; + + dev_err(&port->pcie->pdev->dev, + "Could not create MBus window at [mem %pa-%pa]: %d\n", + &base, &end, ret); + mvebu_pcie_del_windows(port, base - size_mapped, + size_mapped); + return ret; + } + + size -= sz; + size_mapped += sz; + base += sz; + if (remap != MVEBU_MBUS_NO_REMAP) + remap += sz; + } + + return 0; +} + +static int mvebu_pcie_set_window(struct mvebu_pcie_port *port, + unsigned int target, unsigned int attribute, + const struct mvebu_pcie_window *desired, + struct mvebu_pcie_window *cur) +{ + int ret; + + if (desired->base == cur->base && desired->remap == cur->remap && + desired->size == cur->size) + return 0; + + if (cur->size != 0) { + mvebu_pcie_del_windows(port, cur->base, cur->size); + cur->size = 0; + cur->base = 0; + + /* + * If something tries to change the window while it is enabled + * the change will not be done atomically. That would be + * difficult to do in the general case. + */ + } + + if (desired->size == 0) + return 0; + + ret = mvebu_pcie_add_windows(port, target, attribute, desired->base, + desired->size, desired->remap); + if (ret) { + cur->size = 0; + cur->base = 0; + return ret; + } + + *cur = *desired; + return 0; +} + +static int mvebu_pcie_handle_iobase_change(struct mvebu_pcie_port *port) +{ + struct mvebu_pcie_window desired = {}; + struct pci_bridge_emul_conf *conf = &port->bridge.conf; + + /* Are the new iobase/iolimit values invalid? */ + if (conf->iolimit < conf->iobase || + le16_to_cpu(conf->iolimitupper) < le16_to_cpu(conf->iobaseupper)) + return mvebu_pcie_set_window(port, port->io_target, port->io_attr, + &desired, &port->iowin); + + /* + * We read the PCI-to-PCI bridge emulated registers, and + * calculate the base address and size of the address decoding + * window to setup, according to the PCI-to-PCI bridge + * specifications. iobase is the bus address, port->iowin_base + * is the CPU address. + */ + desired.remap = ((conf->iobase & 0xF0) << 8) | + (le16_to_cpu(conf->iobaseupper) << 16); + desired.base = port->pcie->io.start + desired.remap; + desired.size = ((0xFFF | ((conf->iolimit & 0xF0) << 8) | + (le16_to_cpu(conf->iolimitupper) << 16)) - + desired.remap) + + 1; + + return mvebu_pcie_set_window(port, port->io_target, port->io_attr, &desired, + &port->iowin); +} + +static int mvebu_pcie_handle_membase_change(struct mvebu_pcie_port *port) +{ + struct mvebu_pcie_window desired = {.remap = MVEBU_MBUS_NO_REMAP}; + struct pci_bridge_emul_conf *conf = &port->bridge.conf; + + /* Are the new membase/memlimit values invalid? */ + if (le16_to_cpu(conf->memlimit) < le16_to_cpu(conf->membase)) + return mvebu_pcie_set_window(port, port->mem_target, port->mem_attr, + &desired, &port->memwin); + + /* + * We read the PCI-to-PCI bridge emulated registers, and + * calculate the base address and size of the address decoding + * window to setup, according to the PCI-to-PCI bridge + * specifications. + */ + desired.base = ((le16_to_cpu(conf->membase) & 0xFFF0) << 16); + desired.size = (((le16_to_cpu(conf->memlimit) & 0xFFF0) << 16) | 0xFFFFF) - + desired.base + 1; + + return mvebu_pcie_set_window(port, port->mem_target, port->mem_attr, &desired, + &port->memwin); +} + +static pci_bridge_emul_read_status_t +mvebu_pci_bridge_emul_base_conf_read(struct pci_bridge_emul *bridge, + int reg, u32 *value) +{ + struct mvebu_pcie_port *port = bridge->data; + + switch (reg) { + case PCI_COMMAND: + *value = mvebu_readl(port, PCIE_CMD_OFF); + break; + + case PCI_PRIMARY_BUS: { + /* + * From the whole 32bit register we support reading from HW only + * secondary bus number which is mvebu local bus number. + * Other bits are retrieved only from emulated config buffer. + */ + __le32 *cfgspace = (__le32 *)&bridge->conf; + u32 val = le32_to_cpu(cfgspace[PCI_PRIMARY_BUS / 4]); + val &= ~0xff00; + val |= mvebu_pcie_get_local_bus_nr(port) << 8; + *value = val; + break; + } + + case PCI_INTERRUPT_LINE: { + /* + * From the whole 32bit register we support reading from HW only + * one bit: PCI_BRIDGE_CTL_BUS_RESET. + * Other bits are retrieved only from emulated config buffer. + */ + __le32 *cfgspace = (__le32 *)&bridge->conf; + u32 val = le32_to_cpu(cfgspace[PCI_INTERRUPT_LINE / 4]); + if (mvebu_readl(port, PCIE_CTRL_OFF) & PCIE_CTRL_MASTER_HOT_RESET) + val |= PCI_BRIDGE_CTL_BUS_RESET << 16; + else + val &= ~(PCI_BRIDGE_CTL_BUS_RESET << 16); + *value = val; + break; + } + + default: + return PCI_BRIDGE_EMUL_NOT_HANDLED; + } + + return PCI_BRIDGE_EMUL_HANDLED; +} + +static pci_bridge_emul_read_status_t +mvebu_pci_bridge_emul_pcie_conf_read(struct pci_bridge_emul *bridge, + int reg, u32 *value) +{ + struct mvebu_pcie_port *port = bridge->data; + + switch (reg) { + case PCI_EXP_DEVCAP: + *value = mvebu_readl(port, PCIE_CAP_PCIEXP + PCI_EXP_DEVCAP); + break; + + case PCI_EXP_DEVCTL: + *value = mvebu_readl(port, PCIE_CAP_PCIEXP + PCI_EXP_DEVCTL); + break; + + case PCI_EXP_LNKCAP: + /* + * PCIe requires that the Clock Power Management capability bit + * is hard-wired to zero for downstream ports but HW returns 1. + * Additionally enable Data Link Layer Link Active Reporting + * Capable bit as DL_Active indication is provided too. + */ + *value = (mvebu_readl(port, PCIE_CAP_PCIEXP + PCI_EXP_LNKCAP) & + ~PCI_EXP_LNKCAP_CLKPM) | PCI_EXP_LNKCAP_DLLLARC; + break; + + case PCI_EXP_LNKCTL: + /* DL_Active indication is provided via PCIE_STAT_OFF */ + *value = mvebu_readl(port, PCIE_CAP_PCIEXP + PCI_EXP_LNKCTL) | + (mvebu_pcie_link_up(port) ? + (PCI_EXP_LNKSTA_DLLLA << 16) : 0); + break; + + case PCI_EXP_SLTCTL: { + u16 slotctl = le16_to_cpu(bridge->pcie_conf.slotctl); + u16 slotsta = le16_to_cpu(bridge->pcie_conf.slotsta); + u32 val = 0; + /* + * When slot power limit was not specified in DT then + * ASPL_DISABLE bit is stored only in emulated config space. + * Otherwise reflect status of PCIE_SSPL_ENABLE bit in HW. + */ + if (!port->slot_power_limit_value) + val |= slotctl & PCI_EXP_SLTCTL_ASPL_DISABLE; + else if (!(mvebu_readl(port, PCIE_SSPL_OFF) & PCIE_SSPL_ENABLE)) + val |= PCI_EXP_SLTCTL_ASPL_DISABLE; + /* This callback is 32-bit and in high bits is slot status. */ + val |= slotsta << 16; + *value = val; + break; + } + + case PCI_EXP_RTSTA: + *value = mvebu_readl(port, PCIE_RC_RTSTA); + break; + + case PCI_EXP_DEVCAP2: + *value = mvebu_readl(port, PCIE_CAP_PCIEXP + PCI_EXP_DEVCAP2); + break; + + case PCI_EXP_DEVCTL2: + *value = mvebu_readl(port, PCIE_CAP_PCIEXP + PCI_EXP_DEVCTL2); + break; + + case PCI_EXP_LNKCTL2: + *value = mvebu_readl(port, PCIE_CAP_PCIEXP + PCI_EXP_LNKCTL2); + break; + + default: + return PCI_BRIDGE_EMUL_NOT_HANDLED; + } + + return PCI_BRIDGE_EMUL_HANDLED; +} + +static pci_bridge_emul_read_status_t +mvebu_pci_bridge_emul_ext_conf_read(struct pci_bridge_emul *bridge, + int reg, u32 *value) +{ + struct mvebu_pcie_port *port = bridge->data; + + switch (reg) { + case 0: + case PCI_ERR_UNCOR_STATUS: + case PCI_ERR_UNCOR_MASK: + case PCI_ERR_UNCOR_SEVER: + case PCI_ERR_COR_STATUS: + case PCI_ERR_COR_MASK: + case PCI_ERR_CAP: + case PCI_ERR_HEADER_LOG+0: + case PCI_ERR_HEADER_LOG+4: + case PCI_ERR_HEADER_LOG+8: + case PCI_ERR_HEADER_LOG+12: + case PCI_ERR_ROOT_COMMAND: + case PCI_ERR_ROOT_STATUS: + case PCI_ERR_ROOT_ERR_SRC: + *value = mvebu_readl(port, PCIE_CAP_PCIERR_OFF + reg); + break; + + default: + return PCI_BRIDGE_EMUL_NOT_HANDLED; + } + + return PCI_BRIDGE_EMUL_HANDLED; +} + +static void +mvebu_pci_bridge_emul_base_conf_write(struct pci_bridge_emul *bridge, + int reg, u32 old, u32 new, u32 mask) +{ + struct mvebu_pcie_port *port = bridge->data; + struct pci_bridge_emul_conf *conf = &bridge->conf; + + switch (reg) { + case PCI_COMMAND: + mvebu_writel(port, new, PCIE_CMD_OFF); + break; + + case PCI_IO_BASE: + if ((mask & 0xffff) && mvebu_has_ioport(port) && + mvebu_pcie_handle_iobase_change(port)) { + /* On error disable IO range */ + conf->iobase &= ~0xf0; + conf->iolimit &= ~0xf0; + conf->iobase |= 0xf0; + conf->iobaseupper = cpu_to_le16(0x0000); + conf->iolimitupper = cpu_to_le16(0x0000); + } + break; + + case PCI_MEMORY_BASE: + if (mvebu_pcie_handle_membase_change(port)) { + /* On error disable mem range */ + conf->membase = cpu_to_le16(le16_to_cpu(conf->membase) & ~0xfff0); + conf->memlimit = cpu_to_le16(le16_to_cpu(conf->memlimit) & ~0xfff0); + conf->membase = cpu_to_le16(le16_to_cpu(conf->membase) | 0xfff0); + } + break; + + case PCI_IO_BASE_UPPER16: + if (mvebu_has_ioport(port) && + mvebu_pcie_handle_iobase_change(port)) { + /* On error disable IO range */ + conf->iobase &= ~0xf0; + conf->iolimit &= ~0xf0; + conf->iobase |= 0xf0; + conf->iobaseupper = cpu_to_le16(0x0000); + conf->iolimitupper = cpu_to_le16(0x0000); + } + break; + + case PCI_PRIMARY_BUS: + if (mask & 0xff00) + mvebu_pcie_set_local_bus_nr(port, conf->secondary_bus); + break; + + case PCI_INTERRUPT_LINE: + if (mask & (PCI_BRIDGE_CTL_BUS_RESET << 16)) { + u32 ctrl = mvebu_readl(port, PCIE_CTRL_OFF); + if (new & (PCI_BRIDGE_CTL_BUS_RESET << 16)) + ctrl |= PCIE_CTRL_MASTER_HOT_RESET; + else + ctrl &= ~PCIE_CTRL_MASTER_HOT_RESET; + mvebu_writel(port, ctrl, PCIE_CTRL_OFF); + } + break; + + default: + break; + } +} + +static void +mvebu_pci_bridge_emul_pcie_conf_write(struct pci_bridge_emul *bridge, + int reg, u32 old, u32 new, u32 mask) +{ + struct mvebu_pcie_port *port = bridge->data; + + switch (reg) { + case PCI_EXP_DEVCTL: + mvebu_writel(port, new, PCIE_CAP_PCIEXP + PCI_EXP_DEVCTL); + break; + + case PCI_EXP_LNKCTL: + /* + * PCIe requires that the Enable Clock Power Management bit + * is hard-wired to zero for downstream ports but HW allows + * to change it. + */ + new &= ~PCI_EXP_LNKCTL_CLKREQ_EN; + + mvebu_writel(port, new, PCIE_CAP_PCIEXP + PCI_EXP_LNKCTL); + break; + + case PCI_EXP_SLTCTL: + /* + * Allow to change PCIE_SSPL_ENABLE bit only when slot power + * limit was specified in DT and configured into HW. + */ + if ((mask & PCI_EXP_SLTCTL_ASPL_DISABLE) && + port->slot_power_limit_value) { + u32 sspl = mvebu_readl(port, PCIE_SSPL_OFF); + if (new & PCI_EXP_SLTCTL_ASPL_DISABLE) + sspl &= ~PCIE_SSPL_ENABLE; + else + sspl |= PCIE_SSPL_ENABLE; + mvebu_writel(port, sspl, PCIE_SSPL_OFF); + } + break; + + case PCI_EXP_RTSTA: + /* + * PME Status bit in Root Status Register (PCIE_RC_RTSTA) + * is read-only and can be cleared only by writing 0b to the + * Interrupt Cause RW0C register (PCIE_INT_CAUSE_OFF). So + * clear PME via Interrupt Cause. + */ + if (new & PCI_EXP_RTSTA_PME) + mvebu_writel(port, ~PCIE_INT_PM_PME, PCIE_INT_CAUSE_OFF); + break; + + case PCI_EXP_DEVCTL2: + mvebu_writel(port, new, PCIE_CAP_PCIEXP + PCI_EXP_DEVCTL2); + break; + + case PCI_EXP_LNKCTL2: + mvebu_writel(port, new, PCIE_CAP_PCIEXP + PCI_EXP_LNKCTL2); + break; + + default: + break; + } +} + +static void +mvebu_pci_bridge_emul_ext_conf_write(struct pci_bridge_emul *bridge, + int reg, u32 old, u32 new, u32 mask) +{ + struct mvebu_pcie_port *port = bridge->data; + + switch (reg) { + /* These are W1C registers, so clear other bits */ + case PCI_ERR_UNCOR_STATUS: + case PCI_ERR_COR_STATUS: + case PCI_ERR_ROOT_STATUS: + new &= mask; + fallthrough; + + case PCI_ERR_UNCOR_MASK: + case PCI_ERR_UNCOR_SEVER: + case PCI_ERR_COR_MASK: + case PCI_ERR_CAP: + case PCI_ERR_HEADER_LOG+0: + case PCI_ERR_HEADER_LOG+4: + case PCI_ERR_HEADER_LOG+8: + case PCI_ERR_HEADER_LOG+12: + case PCI_ERR_ROOT_COMMAND: + case PCI_ERR_ROOT_ERR_SRC: + mvebu_writel(port, new, PCIE_CAP_PCIERR_OFF + reg); + break; + + default: + break; + } +} + +static const struct pci_bridge_emul_ops mvebu_pci_bridge_emul_ops = { + .read_base = mvebu_pci_bridge_emul_base_conf_read, + .write_base = mvebu_pci_bridge_emul_base_conf_write, + .read_pcie = mvebu_pci_bridge_emul_pcie_conf_read, + .write_pcie = mvebu_pci_bridge_emul_pcie_conf_write, + .read_ext = mvebu_pci_bridge_emul_ext_conf_read, + .write_ext = mvebu_pci_bridge_emul_ext_conf_write, +}; + +/* + * Initialize the configuration space of the PCI-to-PCI bridge + * associated with the given PCIe interface. + */ +static int mvebu_pci_bridge_emul_init(struct mvebu_pcie_port *port) +{ + unsigned int bridge_flags = PCI_BRIDGE_EMUL_NO_PREFMEM_FORWARD; + struct pci_bridge_emul *bridge = &port->bridge; + u32 dev_id = mvebu_readl(port, PCIE_DEV_ID_OFF); + u32 dev_rev = mvebu_readl(port, PCIE_DEV_REV_OFF); + u32 ssdev_id = mvebu_readl(port, PCIE_SSDEV_ID_OFF); + u32 pcie_cap = mvebu_readl(port, PCIE_CAP_PCIEXP); + u8 pcie_cap_ver = ((pcie_cap >> 16) & PCI_EXP_FLAGS_VERS); + + bridge->conf.vendor = cpu_to_le16(dev_id & 0xffff); + bridge->conf.device = cpu_to_le16(dev_id >> 16); + bridge->conf.class_revision = cpu_to_le32(dev_rev & 0xff); + + if (mvebu_has_ioport(port)) { + /* We support 32 bits I/O addressing */ + bridge->conf.iobase = PCI_IO_RANGE_TYPE_32; + bridge->conf.iolimit = PCI_IO_RANGE_TYPE_32; + } else { + bridge_flags |= PCI_BRIDGE_EMUL_NO_IO_FORWARD; + } + + /* + * Older mvebu hardware provides PCIe Capability structure only in + * version 1. New hardware provides it in version 2. + * Enable slot support which is emulated. + */ + bridge->pcie_conf.cap = cpu_to_le16(pcie_cap_ver | PCI_EXP_FLAGS_SLOT); + + /* + * Set Presence Detect State bit permanently as there is no support for + * unplugging PCIe card from the slot. Assume that PCIe card is always + * connected in slot. + * + * Set physical slot number to port+1 as mvebu ports are indexed from + * zero and zero value is reserved for ports within the same silicon + * as Root Port which is not mvebu case. + * + * Also set correct slot power limit. + */ + bridge->pcie_conf.slotcap = cpu_to_le32( + FIELD_PREP(PCI_EXP_SLTCAP_SPLV, port->slot_power_limit_value) | + FIELD_PREP(PCI_EXP_SLTCAP_SPLS, port->slot_power_limit_scale) | + FIELD_PREP(PCI_EXP_SLTCAP_PSN, port->port+1)); + bridge->pcie_conf.slotsta = cpu_to_le16(PCI_EXP_SLTSTA_PDS); + + bridge->subsystem_vendor_id = ssdev_id & 0xffff; + bridge->subsystem_id = ssdev_id >> 16; + bridge->has_pcie = true; + bridge->pcie_start = PCIE_CAP_PCIEXP; + bridge->data = port; + bridge->ops = &mvebu_pci_bridge_emul_ops; + + return pci_bridge_emul_init(bridge, bridge_flags); +} + +static inline struct mvebu_pcie *sys_to_pcie(struct pci_sys_data *sys) +{ + return sys->private_data; +} + +static struct mvebu_pcie_port *mvebu_pcie_find_port(struct mvebu_pcie *pcie, + struct pci_bus *bus, + int devfn) +{ + int i; + + for (i = 0; i < pcie->nports; i++) { + struct mvebu_pcie_port *port = &pcie->ports[i]; + + if (!port->base) + continue; + + if (bus->number == 0 && port->devfn == devfn) + return port; + if (bus->number != 0 && + bus->number >= port->bridge.conf.secondary_bus && + bus->number <= port->bridge.conf.subordinate_bus) + return port; + } + + return NULL; +} + +/* PCI configuration space write function */ +static int mvebu_pcie_wr_conf(struct pci_bus *bus, u32 devfn, + int where, int size, u32 val) +{ + struct mvebu_pcie *pcie = bus->sysdata; + struct mvebu_pcie_port *port; + + port = mvebu_pcie_find_port(pcie, bus, devfn); + if (!port) + return PCIBIOS_DEVICE_NOT_FOUND; + + return pci_bridge_emul_conf_write(&port->bridge, where, size, val); +} + +/* PCI configuration space read function */ +static int mvebu_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where, + int size, u32 *val) +{ + struct mvebu_pcie *pcie = bus->sysdata; + struct mvebu_pcie_port *port; + + port = mvebu_pcie_find_port(pcie, bus, devfn); + if (!port) + return PCIBIOS_DEVICE_NOT_FOUND; + + return pci_bridge_emul_conf_read(&port->bridge, where, size, val); +} + +static struct pci_ops mvebu_pcie_ops = { + .read = mvebu_pcie_rd_conf, + .write = mvebu_pcie_wr_conf, +}; + +static void mvebu_pcie_intx_irq_mask(struct irq_data *d) +{ + struct mvebu_pcie_port *port = d->domain->host_data; + irq_hw_number_t hwirq = irqd_to_hwirq(d); + unsigned long flags; + u32 unmask; + + raw_spin_lock_irqsave(&port->irq_lock, flags); + unmask = mvebu_readl(port, PCIE_INT_UNMASK_OFF); + unmask &= ~PCIE_INT_INTX(hwirq); + mvebu_writel(port, unmask, PCIE_INT_UNMASK_OFF); + raw_spin_unlock_irqrestore(&port->irq_lock, flags); +} + +static void mvebu_pcie_intx_irq_unmask(struct irq_data *d) +{ + struct mvebu_pcie_port *port = d->domain->host_data; + irq_hw_number_t hwirq = irqd_to_hwirq(d); + unsigned long flags; + u32 unmask; + + raw_spin_lock_irqsave(&port->irq_lock, flags); + unmask = mvebu_readl(port, PCIE_INT_UNMASK_OFF); + unmask |= PCIE_INT_INTX(hwirq); + mvebu_writel(port, unmask, PCIE_INT_UNMASK_OFF); + raw_spin_unlock_irqrestore(&port->irq_lock, flags); +} + +static struct irq_chip intx_irq_chip = { + .name = "mvebu-INTx", + .irq_mask = mvebu_pcie_intx_irq_mask, + .irq_unmask = mvebu_pcie_intx_irq_unmask, +}; + +static int mvebu_pcie_intx_irq_map(struct irq_domain *h, + unsigned int virq, irq_hw_number_t hwirq) +{ + struct mvebu_pcie_port *port = h->host_data; + + irq_set_status_flags(virq, IRQ_LEVEL); + irq_set_chip_and_handler(virq, &intx_irq_chip, handle_level_irq); + irq_set_chip_data(virq, port); + + return 0; +} + +static const struct irq_domain_ops mvebu_pcie_intx_irq_domain_ops = { + .map = mvebu_pcie_intx_irq_map, + .xlate = irq_domain_xlate_onecell, +}; + +static int mvebu_pcie_init_irq_domain(struct mvebu_pcie_port *port) +{ + struct device *dev = &port->pcie->pdev->dev; + struct device_node *pcie_intc_node; + + raw_spin_lock_init(&port->irq_lock); + + pcie_intc_node = of_get_next_child(port->dn, NULL); + if (!pcie_intc_node) { + dev_err(dev, "No PCIe Intc node found for %s\n", port->name); + return -ENODEV; + } + + port->intx_irq_domain = irq_domain_create_linear(of_fwnode_handle(pcie_intc_node), + PCI_NUM_INTX, + &mvebu_pcie_intx_irq_domain_ops, port); + of_node_put(pcie_intc_node); + if (!port->intx_irq_domain) { + dev_err(dev, "Failed to get INTx IRQ domain for %s\n", port->name); + return -ENOMEM; + } + + return 0; +} + +static void mvebu_pcie_irq_handler(struct irq_desc *desc) +{ + struct mvebu_pcie_port *port = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + struct device *dev = &port->pcie->pdev->dev; + u32 cause, unmask, status; + int i; + + chained_irq_enter(chip, desc); + + cause = mvebu_readl(port, PCIE_INT_CAUSE_OFF); + unmask = mvebu_readl(port, PCIE_INT_UNMASK_OFF); + status = cause & unmask; + + /* Process legacy INTx interrupts */ + for (i = 0; i < PCI_NUM_INTX; i++) { + if (!(status & PCIE_INT_INTX(i))) + continue; + + if (generic_handle_domain_irq(port->intx_irq_domain, i) == -EINVAL) + dev_err_ratelimited(dev, "unexpected INT%c IRQ\n", (char)i+'A'); + } + + chained_irq_exit(chip, desc); +} + +static int mvebu_pcie_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) +{ + /* Interrupt support on mvebu emulated bridges is not implemented yet */ + if (dev->bus->number == 0) + return 0; /* Proper return code 0 == NO_IRQ */ + + return of_irq_parse_and_map_pci(dev, slot, pin); +} + +static resource_size_t mvebu_pcie_align_resource(struct pci_dev *dev, + const struct resource *res, + resource_size_t start, + resource_size_t size, + resource_size_t align) +{ + if (dev->bus->number != 0) + return start; + + /* + * On the PCI-to-PCI bridge side, the I/O windows must have at + * least a 64 KB size and the memory windows must have at + * least a 1 MB size. Moreover, MBus windows need to have a + * base address aligned on their size, and their size must be + * a power of two. This means that if the BAR doesn't have a + * power of two size, several MBus windows will actually be + * created. We need to ensure that the biggest MBus window + * (which will be the first one) is aligned on its size, which + * explains the rounddown_pow_of_two() being done here. + */ + if (res->flags & IORESOURCE_IO) + return round_up(start, max_t(resource_size_t, SZ_64K, + rounddown_pow_of_two(size))); + else if (res->flags & IORESOURCE_MEM) + return round_up(start, max_t(resource_size_t, SZ_1M, + rounddown_pow_of_two(size))); + else + return start; +} + +static void __iomem *mvebu_pcie_map_registers(struct platform_device *pdev, + struct device_node *np, + struct mvebu_pcie_port *port) +{ + int ret = 0; + + ret = of_address_to_resource(np, 0, &port->regs); + if (ret) + return (void __iomem *)ERR_PTR(ret); + + return devm_ioremap_resource(&pdev->dev, &port->regs); +} + +static int mvebu_get_tgt_attr(struct device_node *np, int devfn, + unsigned long type, + unsigned int *tgt, + unsigned int *attr) +{ + struct of_range range; + struct of_range_parser parser; + + *tgt = -1; + *attr = -1; + + if (of_pci_range_parser_init(&parser, np)) + return -EINVAL; + + for_each_of_range(&parser, &range) { + u32 slot = upper_32_bits(range.bus_addr); + + if (slot == PCI_SLOT(devfn) && + type == (range.flags & IORESOURCE_TYPE_BITS)) { + *tgt = (range.parent_bus_addr >> 56) & 0xFF; + *attr = (range.parent_bus_addr >> 48) & 0xFF; + return 0; + } + } + + return -ENOENT; +} + +static int mvebu_pcie_suspend(struct device *dev) +{ + struct mvebu_pcie *pcie; + int i; + + pcie = dev_get_drvdata(dev); + for (i = 0; i < pcie->nports; i++) { + struct mvebu_pcie_port *port = pcie->ports + i; + if (!port->base) + continue; + port->saved_pcie_stat = mvebu_readl(port, PCIE_STAT_OFF); + } + + return 0; +} + +static int mvebu_pcie_resume(struct device *dev) +{ + struct mvebu_pcie *pcie; + int i; + + pcie = dev_get_drvdata(dev); + for (i = 0; i < pcie->nports; i++) { + struct mvebu_pcie_port *port = pcie->ports + i; + if (!port->base) + continue; + mvebu_writel(port, port->saved_pcie_stat, PCIE_STAT_OFF); + mvebu_pcie_setup_hw(port); + } + + return 0; +} + +static void mvebu_pcie_port_clk_put(void *data) +{ + struct mvebu_pcie_port *port = data; + + clk_put(port->clk); +} + +static int mvebu_pcie_parse_port(struct mvebu_pcie *pcie, + struct mvebu_pcie_port *port, struct device_node *child) +{ + struct device *dev = &pcie->pdev->dev; + u32 slot_power_limit; + int ret; + u32 num_lanes; + + port->pcie = pcie; + + if (of_property_read_u32(child, "marvell,pcie-port", &port->port)) { + dev_warn(dev, "ignoring %pOF, missing pcie-port property\n", + child); + goto skip; + } + + if (of_property_read_u32(child, "marvell,pcie-lane", &port->lane)) + port->lane = 0; + + if (!of_property_read_u32(child, "num-lanes", &num_lanes) && num_lanes == 4) + port->is_x4 = true; + + port->name = devm_kasprintf(dev, GFP_KERNEL, "pcie%d.%d", port->port, + port->lane); + if (!port->name) { + ret = -ENOMEM; + goto err; + } + + port->devfn = of_pci_get_devfn(child); + if (port->devfn < 0) + goto skip; + if (PCI_FUNC(port->devfn) != 0) { + dev_err(dev, "%s: invalid function number, must be zero\n", + port->name); + goto skip; + } + + ret = mvebu_get_tgt_attr(dev->of_node, port->devfn, IORESOURCE_MEM, + &port->mem_target, &port->mem_attr); + if (ret < 0) { + dev_err(dev, "%s: cannot get tgt/attr for mem window\n", + port->name); + goto skip; + } + + if (resource_size(&pcie->io) != 0) { + mvebu_get_tgt_attr(dev->of_node, port->devfn, IORESOURCE_IO, + &port->io_target, &port->io_attr); + } else { + port->io_target = -1; + port->io_attr = -1; + } + + /* + * Old DT bindings do not contain "intx" interrupt + * so do not fail probing driver when interrupt does not exist. + */ + port->intx_irq = of_irq_get_byname(child, "intx"); + if (port->intx_irq == -EPROBE_DEFER) { + ret = port->intx_irq; + goto err; + } + if (port->intx_irq <= 0) { + dev_warn(dev, "%s: legacy INTx interrupts cannot be masked individually, " + "%pOF does not contain intx interrupt\n", + port->name, child); + } + + port->reset_name = devm_kasprintf(dev, GFP_KERNEL, "%s-reset", + port->name); + if (!port->reset_name) { + ret = -ENOMEM; + goto err; + } + + port->reset_gpio = devm_fwnode_gpiod_get(dev, of_fwnode_handle(child), + "reset", GPIOD_OUT_HIGH, + port->name); + ret = PTR_ERR_OR_ZERO(port->reset_gpio); + if (ret) { + if (ret != -ENOENT) + goto err; + /* reset gpio is optional */ + port->reset_gpio = NULL; + devm_kfree(dev, port->reset_name); + port->reset_name = NULL; + } + + slot_power_limit = of_pci_get_slot_power_limit(child, + &port->slot_power_limit_value, + &port->slot_power_limit_scale); + if (slot_power_limit) + dev_info(dev, "%s: Slot power limit %u.%uW\n", + port->name, + slot_power_limit / 1000, + (slot_power_limit / 100) % 10); + + port->clk = of_clk_get_by_name(child, NULL); + if (IS_ERR(port->clk)) { + dev_err(dev, "%s: cannot get clock\n", port->name); + goto skip; + } + + ret = devm_add_action_or_reset(dev, mvebu_pcie_port_clk_put, port); + if (ret < 0) + goto err; + + return 1; + +skip: + ret = 0; + + /* In the case of skipping, we need to free these */ + devm_kfree(dev, port->reset_name); + port->reset_name = NULL; + devm_kfree(dev, port->name); + port->name = NULL; + +err: + return ret; +} + +/* + * Power up a PCIe port. PCIe requires the refclk to be stable for 100µs + * prior to releasing PERST. See table 2-4 in section 2.6.2 AC Specifications + * of the PCI Express Card Electromechanical Specification, 1.1. + */ +static int mvebu_pcie_powerup(struct mvebu_pcie_port *port) +{ + int ret; + + ret = clk_prepare_enable(port->clk); + if (ret < 0) + return ret; + + if (port->reset_gpio) { + u32 reset_udelay = PCI_PM_D3COLD_WAIT * 1000; + + of_property_read_u32(port->dn, "reset-delay-us", + &reset_udelay); + + udelay(100); + + gpiod_set_value_cansleep(port->reset_gpio, 0); + msleep(reset_udelay / 1000); + } + + return 0; +} + +/* + * Power down a PCIe port. Strictly, PCIe requires us to place the card + * in D3hot state before asserting PERST#. + */ +static void mvebu_pcie_powerdown(struct mvebu_pcie_port *port) +{ + gpiod_set_value_cansleep(port->reset_gpio, 1); + + clk_disable_unprepare(port->clk); +} + +/* + * devm_of_pci_get_host_bridge_resources() only sets up translatable resources, + * so we need extra resource setup parsing our special DT properties encoding + * the MEM and IO apertures. + */ +static int mvebu_pcie_parse_request_resources(struct mvebu_pcie *pcie) +{ + struct device *dev = &pcie->pdev->dev; + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + int ret; + + /* Get the PCIe memory aperture */ + mvebu_mbus_get_pcie_mem_aperture(&pcie->mem); + if (resource_size(&pcie->mem) == 0) { + dev_err(dev, "invalid memory aperture size\n"); + return -EINVAL; + } + + pcie->mem.name = "PCI MEM"; + pci_add_resource(&bridge->windows, &pcie->mem); + ret = devm_request_resource(dev, &iomem_resource, &pcie->mem); + if (ret) + return ret; + + /* Get the PCIe IO aperture */ + mvebu_mbus_get_pcie_io_aperture(&pcie->io); + + if (resource_size(&pcie->io) != 0) { + pcie->realio.flags = pcie->io.flags; + pcie->realio.start = PCIBIOS_MIN_IO; + pcie->realio.end = min_t(resource_size_t, + IO_SPACE_LIMIT - SZ_64K, + resource_size(&pcie->io) - 1); + pcie->realio.name = "PCI I/O"; + + ret = devm_pci_remap_iospace(dev, &pcie->realio, pcie->io.start); + if (ret) + return ret; + + pci_add_resource(&bridge->windows, &pcie->realio); + ret = devm_request_resource(dev, &ioport_resource, &pcie->realio); + if (ret) + return ret; + } + + return 0; +} + +static int mvebu_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct mvebu_pcie *pcie; + struct pci_host_bridge *bridge; + struct device_node *np = dev->of_node; + struct device_node *child; + int num, i, ret; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(struct mvebu_pcie)); + if (!bridge) + return -ENOMEM; + + pcie = pci_host_bridge_priv(bridge); + pcie->pdev = pdev; + platform_set_drvdata(pdev, pcie); + + ret = mvebu_pcie_parse_request_resources(pcie); + if (ret) + return ret; + + num = of_get_available_child_count(np); + + pcie->ports = devm_kcalloc(dev, num, sizeof(*pcie->ports), GFP_KERNEL); + if (!pcie->ports) + return -ENOMEM; + + i = 0; + for_each_available_child_of_node(np, child) { + struct mvebu_pcie_port *port = &pcie->ports[i]; + + ret = mvebu_pcie_parse_port(pcie, port, child); + if (ret < 0) { + of_node_put(child); + return ret; + } else if (ret == 0) { + continue; + } + + port->dn = child; + i++; + } + pcie->nports = i; + + for (i = 0; i < pcie->nports; i++) { + struct mvebu_pcie_port *port = &pcie->ports[i]; + int irq = port->intx_irq; + + child = port->dn; + if (!child) + continue; + + ret = mvebu_pcie_powerup(port); + if (ret < 0) + continue; + + port->base = mvebu_pcie_map_registers(pdev, child, port); + if (IS_ERR(port->base)) { + dev_err(dev, "%s: cannot map registers\n", port->name); + port->base = NULL; + mvebu_pcie_powerdown(port); + continue; + } + + ret = mvebu_pci_bridge_emul_init(port); + if (ret < 0) { + dev_err(dev, "%s: cannot init emulated bridge\n", + port->name); + devm_iounmap(dev, port->base); + port->base = NULL; + mvebu_pcie_powerdown(port); + continue; + } + + if (irq > 0) { + ret = mvebu_pcie_init_irq_domain(port); + if (ret) { + dev_err(dev, "%s: cannot init irq domain\n", + port->name); + pci_bridge_emul_cleanup(&port->bridge); + devm_iounmap(dev, port->base); + port->base = NULL; + mvebu_pcie_powerdown(port); + continue; + } + irq_set_chained_handler_and_data(irq, + mvebu_pcie_irq_handler, + port); + } + + /* + * PCIe topology exported by mvebu hw is quite complicated. In + * reality has something like N fully independent host bridges + * where each host bridge has one PCIe Root Port (which acts as + * PCI Bridge device). Each host bridge has its own independent + * internal registers, independent access to PCI config space, + * independent interrupt lines, independent window and memory + * access configuration. But additionally there is some kind of + * peer-to-peer support between PCIe devices behind different + * host bridges limited just to forwarding of memory and I/O + * transactions (forwarding of error messages and config cycles + * is not supported). So we could say there are N independent + * PCIe Root Complexes. + * + * For this kind of setup DT should have been structured into + * N independent PCIe controllers / host bridges. But instead + * structure in past was defined to put PCIe Root Ports of all + * host bridges into one bus zero, like in classic multi-port + * Root Complex setup with just one host bridge. + * + * This means that pci-mvebu.c driver provides "virtual" bus 0 + * on which registers all PCIe Root Ports (PCI Bridge devices) + * specified in DT by their BDF addresses and virtually routes + * PCI config access of each PCI bridge device to specific PCIe + * host bridge. + * + * Normally PCI Bridge should choose between Type 0 and Type 1 + * config requests based on primary and secondary bus numbers + * configured on the bridge itself. But because mvebu PCI Bridge + * does not have registers for primary and secondary bus numbers + * in its config space, it determinates type of config requests + * via its own custom way. + * + * There are two options how mvebu determinate type of config + * request. + * + * 1. If Secondary Bus Number Enable bit is not set or is not + * available (applies for pre-XP PCIe controllers) then Type 0 + * is used if target bus number equals Local Bus Number (bits + * [15:8] in register 0x1a04) and target device number differs + * from Local Device Number (bits [20:16] in register 0x1a04). + * Type 1 is used if target bus number differs from Local Bus + * Number. And when target bus number equals Local Bus Number + * and target device equals Local Device Number then request is + * routed to Local PCI Bridge (PCIe Root Port). + * + * 2. If Secondary Bus Number Enable bit is set (bit 7 in + * register 0x1a2c) then mvebu hw determinate type of config + * request like compliant PCI Bridge based on primary bus number + * which is configured via Local Bus Number (bits [15:8] in + * register 0x1a04) and secondary bus number which is configured + * via Secondary Bus Number (bits [7:0] in register 0x1a2c). + * Local PCI Bridge (PCIe Root Port) is available on primary bus + * as device with Local Device Number (bits [20:16] in register + * 0x1a04). + * + * Secondary Bus Number Enable bit is disabled by default and + * option 2. is not available on pre-XP PCIe controllers. Hence + * this driver always use option 1. + * + * Basically it means that primary and secondary buses shares + * one virtual number configured via Local Bus Number bits and + * Local Device Number bits determinates if accessing primary + * or secondary bus. Set Local Device Number to 1 and redirect + * all writes of PCI Bridge Secondary Bus Number register to + * Local Bus Number (bits [15:8] in register 0x1a04). + * + * So when accessing devices on buses behind secondary bus + * number it would work correctly. And also when accessing + * device 0 at secondary bus number via config space would be + * correctly routed to secondary bus. Due to issues described + * in mvebu_pcie_setup_hw(), PCI Bridges at primary bus (zero) + * are not accessed directly via PCI config space but rarher + * indirectly via kernel emulated PCI bridge driver. + */ + mvebu_pcie_setup_hw(port); + mvebu_pcie_set_local_dev_nr(port, 1); + mvebu_pcie_set_local_bus_nr(port, 0); + } + + bridge->sysdata = pcie; + bridge->ops = &mvebu_pcie_ops; + bridge->child_ops = &mvebu_pcie_child_ops; + bridge->align_resource = mvebu_pcie_align_resource; + bridge->map_irq = mvebu_pcie_map_irq; + + return pci_host_probe(bridge); +} + +static void mvebu_pcie_remove(struct platform_device *pdev) +{ + struct mvebu_pcie *pcie = platform_get_drvdata(pdev); + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + u32 cmd, sspl; + int i; + + /* Remove PCI bus with all devices. */ + pci_lock_rescan_remove(); + pci_stop_root_bus(bridge->bus); + pci_remove_root_bus(bridge->bus); + pci_unlock_rescan_remove(); + + for (i = 0; i < pcie->nports; i++) { + struct mvebu_pcie_port *port = &pcie->ports[i]; + int irq = port->intx_irq; + + if (!port->base) + continue; + + /* Disable Root Bridge I/O space, memory space and bus mastering. */ + cmd = mvebu_readl(port, PCIE_CMD_OFF); + cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + mvebu_writel(port, cmd, PCIE_CMD_OFF); + + /* Mask all interrupt sources. */ + mvebu_writel(port, ~PCIE_INT_ALL_MASK, PCIE_INT_UNMASK_OFF); + + /* Clear all interrupt causes. */ + mvebu_writel(port, ~PCIE_INT_ALL_MASK, PCIE_INT_CAUSE_OFF); + + if (irq > 0) + irq_set_chained_handler_and_data(irq, NULL, NULL); + + /* Remove IRQ domains. */ + if (port->intx_irq_domain) + irq_domain_remove(port->intx_irq_domain); + + /* Free config space for emulated root bridge. */ + pci_bridge_emul_cleanup(&port->bridge); + + /* Disable sending Set_Slot_Power_Limit PCIe Message. */ + sspl = mvebu_readl(port, PCIE_SSPL_OFF); + sspl &= ~(PCIE_SSPL_VALUE_MASK | PCIE_SSPL_SCALE_MASK | PCIE_SSPL_ENABLE); + mvebu_writel(port, sspl, PCIE_SSPL_OFF); + + /* Disable and clear BARs and windows. */ + mvebu_pcie_disable_wins(port); + + /* Delete PCIe IO and MEM windows. */ + if (port->iowin.size) + mvebu_pcie_del_windows(port, port->iowin.base, port->iowin.size); + if (port->memwin.size) + mvebu_pcie_del_windows(port, port->memwin.base, port->memwin.size); + + /* Power down card and disable clocks. Must be the last step. */ + mvebu_pcie_powerdown(port); + } +} + +static const struct of_device_id mvebu_pcie_of_match_table[] = { + { .compatible = "marvell,armada-xp-pcie", }, + { .compatible = "marvell,armada-370-pcie", }, + { .compatible = "marvell,dove-pcie", }, + { .compatible = "marvell,kirkwood-pcie", }, + {}, +}; +MODULE_DEVICE_TABLE(of, mvebu_pcie_of_match_table); + +static const struct dev_pm_ops mvebu_pcie_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(mvebu_pcie_suspend, mvebu_pcie_resume) +}; + +static struct platform_driver mvebu_pcie_driver = { + .driver = { + .name = "mvebu-pcie", + .of_match_table = mvebu_pcie_of_match_table, + .pm = &mvebu_pcie_pm_ops, + }, + .probe = mvebu_pcie_probe, + .remove = mvebu_pcie_remove, +}; +module_platform_driver(mvebu_pcie_driver); + +MODULE_AUTHOR("Thomas Petazzoni <thomas.petazzoni@bootlin.com>"); +MODULE_AUTHOR("Pali Rohár <pali@kernel.org>"); +MODULE_DESCRIPTION("Marvell EBU PCIe controller"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pci-rcar-gen2.c b/drivers/pci/controller/pci-rcar-gen2.c new file mode 100644 index 000000000..d29866485 --- /dev/null +++ b/drivers/pci/controller/pci-rcar-gen2.c @@ -0,0 +1,342 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * pci-rcar-gen2: internal PCI bus support + * + * Copyright (C) 2013 Renesas Solutions Corp. + * Copyright (C) 2013 Cogent Embedded, Inc. + * + * Author: Valentine Barshak <valentine.barshak@cogentembedded.com> + */ + +#include <linux/delay.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/sizes.h> +#include <linux/slab.h> + +#include "../pci.h" + +/* AHB-PCI Bridge PCI communication registers */ +#define RCAR_AHBPCI_PCICOM_OFFSET 0x800 + +#define RCAR_PCIAHB_WIN1_CTR_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x00) +#define RCAR_PCIAHB_WIN2_CTR_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x04) +#define RCAR_PCIAHB_PREFETCH0 0x0 +#define RCAR_PCIAHB_PREFETCH4 0x1 +#define RCAR_PCIAHB_PREFETCH8 0x2 +#define RCAR_PCIAHB_PREFETCH16 0x3 + +#define RCAR_AHBPCI_WIN1_CTR_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x10) +#define RCAR_AHBPCI_WIN2_CTR_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x14) +#define RCAR_AHBPCI_WIN_CTR_MEM (3 << 1) +#define RCAR_AHBPCI_WIN_CTR_CFG (5 << 1) +#define RCAR_AHBPCI_WIN1_HOST (1 << 30) +#define RCAR_AHBPCI_WIN1_DEVICE (1 << 31) + +#define RCAR_PCI_INT_ENABLE_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x20) +#define RCAR_PCI_INT_STATUS_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x24) +#define RCAR_PCI_INT_SIGTABORT (1 << 0) +#define RCAR_PCI_INT_SIGRETABORT (1 << 1) +#define RCAR_PCI_INT_REMABORT (1 << 2) +#define RCAR_PCI_INT_PERR (1 << 3) +#define RCAR_PCI_INT_SIGSERR (1 << 4) +#define RCAR_PCI_INT_RESERR (1 << 5) +#define RCAR_PCI_INT_WIN1ERR (1 << 12) +#define RCAR_PCI_INT_WIN2ERR (1 << 13) +#define RCAR_PCI_INT_A (1 << 16) +#define RCAR_PCI_INT_B (1 << 17) +#define RCAR_PCI_INT_PME (1 << 19) +#define RCAR_PCI_INT_ALLERRORS (RCAR_PCI_INT_SIGTABORT | \ + RCAR_PCI_INT_SIGRETABORT | \ + RCAR_PCI_INT_REMABORT | \ + RCAR_PCI_INT_PERR | \ + RCAR_PCI_INT_SIGSERR | \ + RCAR_PCI_INT_RESERR | \ + RCAR_PCI_INT_WIN1ERR | \ + RCAR_PCI_INT_WIN2ERR) + +#define RCAR_AHB_BUS_CTR_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x30) +#define RCAR_AHB_BUS_MMODE_HTRANS (1 << 0) +#define RCAR_AHB_BUS_MMODE_BYTE_BURST (1 << 1) +#define RCAR_AHB_BUS_MMODE_WR_INCR (1 << 2) +#define RCAR_AHB_BUS_MMODE_HBUS_REQ (1 << 7) +#define RCAR_AHB_BUS_SMODE_READYCTR (1 << 17) +#define RCAR_AHB_BUS_MODE (RCAR_AHB_BUS_MMODE_HTRANS | \ + RCAR_AHB_BUS_MMODE_BYTE_BURST | \ + RCAR_AHB_BUS_MMODE_WR_INCR | \ + RCAR_AHB_BUS_MMODE_HBUS_REQ | \ + RCAR_AHB_BUS_SMODE_READYCTR) + +#define RCAR_USBCTR_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x34) +#define RCAR_USBCTR_USBH_RST (1 << 0) +#define RCAR_USBCTR_PCICLK_MASK (1 << 1) +#define RCAR_USBCTR_PLL_RST (1 << 2) +#define RCAR_USBCTR_DIRPD (1 << 8) +#define RCAR_USBCTR_PCIAHB_WIN2_EN (1 << 9) +#define RCAR_USBCTR_PCIAHB_WIN1_256M (0 << 10) +#define RCAR_USBCTR_PCIAHB_WIN1_512M (1 << 10) +#define RCAR_USBCTR_PCIAHB_WIN1_1G (2 << 10) +#define RCAR_USBCTR_PCIAHB_WIN1_2G (3 << 10) +#define RCAR_USBCTR_PCIAHB_WIN1_MASK (3 << 10) + +#define RCAR_PCI_ARBITER_CTR_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x40) +#define RCAR_PCI_ARBITER_PCIREQ0 (1 << 0) +#define RCAR_PCI_ARBITER_PCIREQ1 (1 << 1) +#define RCAR_PCI_ARBITER_PCIBP_MODE (1 << 12) + +#define RCAR_PCI_UNIT_REV_REG (RCAR_AHBPCI_PCICOM_OFFSET + 0x48) + +struct rcar_pci { + struct device *dev; + void __iomem *reg; + struct resource mem_res; + struct resource *cfg_res; + int irq; +}; + +/* PCI configuration space operations */ +static void __iomem *rcar_pci_cfg_base(struct pci_bus *bus, unsigned int devfn, + int where) +{ + struct rcar_pci *priv = bus->sysdata; + int slot, val; + + if (!pci_is_root_bus(bus) || PCI_FUNC(devfn)) + return NULL; + + /* Only one EHCI/OHCI device built-in */ + slot = PCI_SLOT(devfn); + if (slot > 2) + return NULL; + + /* bridge logic only has registers to 0x40 */ + if (slot == 0x0 && where >= 0x40) + return NULL; + + val = slot ? RCAR_AHBPCI_WIN1_DEVICE | RCAR_AHBPCI_WIN_CTR_CFG : + RCAR_AHBPCI_WIN1_HOST | RCAR_AHBPCI_WIN_CTR_CFG; + + iowrite32(val, priv->reg + RCAR_AHBPCI_WIN1_CTR_REG); + return priv->reg + (slot >> 1) * 0x100 + where; +} + +#ifdef CONFIG_PCI_DEBUG +/* if debug enabled, then attach an error handler irq to the bridge */ + +static irqreturn_t rcar_pci_err_irq(int irq, void *pw) +{ + struct rcar_pci *priv = pw; + struct device *dev = priv->dev; + u32 status = ioread32(priv->reg + RCAR_PCI_INT_STATUS_REG); + + if (status & RCAR_PCI_INT_ALLERRORS) { + dev_err(dev, "error irq: status %08x\n", status); + + /* clear the error(s) */ + iowrite32(status & RCAR_PCI_INT_ALLERRORS, + priv->reg + RCAR_PCI_INT_STATUS_REG); + return IRQ_HANDLED; + } + + return IRQ_NONE; +} + +static void rcar_pci_setup_errirq(struct rcar_pci *priv) +{ + struct device *dev = priv->dev; + int ret; + u32 val; + + ret = devm_request_irq(dev, priv->irq, rcar_pci_err_irq, + IRQF_SHARED, "error irq", priv); + if (ret) { + dev_err(dev, "cannot claim IRQ for error handling\n"); + return; + } + + val = ioread32(priv->reg + RCAR_PCI_INT_ENABLE_REG); + val |= RCAR_PCI_INT_ALLERRORS; + iowrite32(val, priv->reg + RCAR_PCI_INT_ENABLE_REG); +} +#else +static inline void rcar_pci_setup_errirq(struct rcar_pci *priv) { } +#endif + +/* PCI host controller setup */ +static void rcar_pci_setup(struct rcar_pci *priv) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(priv); + struct device *dev = priv->dev; + void __iomem *reg = priv->reg; + struct resource_entry *entry; + unsigned long window_size; + unsigned long window_addr; + unsigned long window_pci; + u32 val; + + entry = resource_list_first_type(&bridge->dma_ranges, IORESOURCE_MEM); + if (!entry) { + window_addr = 0x40000000; + window_pci = 0x40000000; + window_size = SZ_1G; + } else { + window_addr = entry->res->start; + window_pci = entry->res->start - entry->offset; + window_size = resource_size(entry->res); + } + + pm_runtime_enable(dev); + pm_runtime_get_sync(dev); + + val = ioread32(reg + RCAR_PCI_UNIT_REV_REG); + dev_info(dev, "PCI: revision %x\n", val); + + /* Disable Direct Power Down State and assert reset */ + val = ioread32(reg + RCAR_USBCTR_REG) & ~RCAR_USBCTR_DIRPD; + val |= RCAR_USBCTR_USBH_RST | RCAR_USBCTR_PLL_RST; + iowrite32(val, reg + RCAR_USBCTR_REG); + udelay(4); + + /* De-assert reset and reset PCIAHB window1 size */ + val &= ~(RCAR_USBCTR_PCIAHB_WIN1_MASK | RCAR_USBCTR_PCICLK_MASK | + RCAR_USBCTR_USBH_RST | RCAR_USBCTR_PLL_RST); + + /* Setup PCIAHB window1 size */ + switch (window_size) { + case SZ_2G: + val |= RCAR_USBCTR_PCIAHB_WIN1_2G; + break; + case SZ_1G: + val |= RCAR_USBCTR_PCIAHB_WIN1_1G; + break; + case SZ_512M: + val |= RCAR_USBCTR_PCIAHB_WIN1_512M; + break; + default: + pr_warn("unknown window size %ld - defaulting to 256M\n", + window_size); + window_size = SZ_256M; + fallthrough; + case SZ_256M: + val |= RCAR_USBCTR_PCIAHB_WIN1_256M; + break; + } + iowrite32(val, reg + RCAR_USBCTR_REG); + + /* Configure AHB master and slave modes */ + iowrite32(RCAR_AHB_BUS_MODE, reg + RCAR_AHB_BUS_CTR_REG); + + /* Configure PCI arbiter */ + val = ioread32(reg + RCAR_PCI_ARBITER_CTR_REG); + val |= RCAR_PCI_ARBITER_PCIREQ0 | RCAR_PCI_ARBITER_PCIREQ1 | + RCAR_PCI_ARBITER_PCIBP_MODE; + iowrite32(val, reg + RCAR_PCI_ARBITER_CTR_REG); + + /* PCI-AHB mapping */ + iowrite32(window_addr | RCAR_PCIAHB_PREFETCH16, + reg + RCAR_PCIAHB_WIN1_CTR_REG); + + /* AHB-PCI mapping: OHCI/EHCI registers */ + val = priv->mem_res.start | RCAR_AHBPCI_WIN_CTR_MEM; + iowrite32(val, reg + RCAR_AHBPCI_WIN2_CTR_REG); + + /* Enable AHB-PCI bridge PCI configuration access */ + iowrite32(RCAR_AHBPCI_WIN1_HOST | RCAR_AHBPCI_WIN_CTR_CFG, + reg + RCAR_AHBPCI_WIN1_CTR_REG); + /* Set PCI-AHB Window1 address */ + iowrite32(window_pci | PCI_BASE_ADDRESS_MEM_PREFETCH, + reg + PCI_BASE_ADDRESS_1); + /* Set AHB-PCI bridge PCI communication area address */ + val = priv->cfg_res->start + RCAR_AHBPCI_PCICOM_OFFSET; + iowrite32(val, reg + PCI_BASE_ADDRESS_0); + + val = ioread32(reg + PCI_COMMAND); + val |= PCI_COMMAND_SERR | PCI_COMMAND_PARITY | + PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER; + iowrite32(val, reg + PCI_COMMAND); + + /* Enable PCI interrupts */ + iowrite32(RCAR_PCI_INT_A | RCAR_PCI_INT_B | RCAR_PCI_INT_PME, + reg + RCAR_PCI_INT_ENABLE_REG); + + rcar_pci_setup_errirq(priv); +} + +static struct pci_ops rcar_pci_ops = { + .map_bus = rcar_pci_cfg_base, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; + +static int rcar_pci_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct resource *cfg_res, *mem_res; + struct rcar_pci *priv; + struct pci_host_bridge *bridge; + void __iomem *reg; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*priv)); + if (!bridge) + return -ENOMEM; + + priv = pci_host_bridge_priv(bridge); + bridge->sysdata = priv; + + reg = devm_platform_get_and_ioremap_resource(pdev, 0, &cfg_res); + if (IS_ERR(reg)) + return PTR_ERR(reg); + + mem_res = platform_get_resource(pdev, IORESOURCE_MEM, 1); + if (!mem_res || !mem_res->start) + return -ENODEV; + + if (mem_res->start & 0xFFFF) + return -EINVAL; + + priv->mem_res = *mem_res; + priv->cfg_res = cfg_res; + + priv->irq = platform_get_irq(pdev, 0); + priv->reg = reg; + priv->dev = dev; + + if (priv->irq < 0) { + dev_err(dev, "no valid irq found\n"); + return priv->irq; + } + + bridge->ops = &rcar_pci_ops; + + pci_add_flags(PCI_REASSIGN_ALL_BUS); + + rcar_pci_setup(priv); + + return pci_host_probe(bridge); +} + +static const struct of_device_id rcar_pci_of_match[] = { + { .compatible = "renesas,pci-r8a7790", }, + { .compatible = "renesas,pci-r8a7791", }, + { .compatible = "renesas,pci-r8a7794", }, + { .compatible = "renesas,pci-rcar-gen2", }, + { .compatible = "renesas,pci-rzn1", }, + { }, +}; + +static struct platform_driver rcar_pci_driver = { + .driver = { + .name = "pci-rcar-gen2", + .suppress_bind_attrs = true, + .of_match_table = rcar_pci_of_match, + }, + .probe = rcar_pci_probe, +}; +builtin_platform_driver(rcar_pci_driver); diff --git a/drivers/pci/controller/pci-tegra.c b/drivers/pci/controller/pci-tegra.c new file mode 100644 index 000000000..59b56446c --- /dev/null +++ b/drivers/pci/controller/pci-tegra.c @@ -0,0 +1,2736 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * PCIe host controller driver for Tegra SoCs + * + * Copyright (c) 2010, CompuLab, Ltd. + * Author: Mike Rapoport <mike@compulab.co.il> + * + * Based on NVIDIA PCIe driver + * Copyright (c) 2008-2009, NVIDIA Corporation. + * + * Bits taken from arch/arm/mach-dove/pcie.c + * + * Author: Thierry Reding <treding@nvidia.com> + */ + +#include <linux/clk.h> +#include <linux/cleanup.h> +#include <linux/debugfs.h> +#include <linux/delay.h> +#include <linux/export.h> +#include <linux/gpio/consumer.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/irq.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/irqdomain.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/pci.h> +#include <linux/phy/phy.h> +#include <linux/pinctrl/consumer.h> +#include <linux/platform_device.h> +#include <linux/reset.h> +#include <linux/sizes.h> +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <linux/regulator/consumer.h> + +#include <soc/tegra/cpuidle.h> +#include <soc/tegra/pmc.h> + +#include "../pci.h" + +#define INT_PCI_MSI_NR (8 * 32) + +/* register definitions */ + +#define AFI_AXI_BAR0_SZ 0x00 +#define AFI_AXI_BAR1_SZ 0x04 +#define AFI_AXI_BAR2_SZ 0x08 +#define AFI_AXI_BAR3_SZ 0x0c +#define AFI_AXI_BAR4_SZ 0x10 +#define AFI_AXI_BAR5_SZ 0x14 + +#define AFI_AXI_BAR0_START 0x18 +#define AFI_AXI_BAR1_START 0x1c +#define AFI_AXI_BAR2_START 0x20 +#define AFI_AXI_BAR3_START 0x24 +#define AFI_AXI_BAR4_START 0x28 +#define AFI_AXI_BAR5_START 0x2c + +#define AFI_FPCI_BAR0 0x30 +#define AFI_FPCI_BAR1 0x34 +#define AFI_FPCI_BAR2 0x38 +#define AFI_FPCI_BAR3 0x3c +#define AFI_FPCI_BAR4 0x40 +#define AFI_FPCI_BAR5 0x44 + +#define AFI_CACHE_BAR0_SZ 0x48 +#define AFI_CACHE_BAR0_ST 0x4c +#define AFI_CACHE_BAR1_SZ 0x50 +#define AFI_CACHE_BAR1_ST 0x54 + +#define AFI_MSI_BAR_SZ 0x60 +#define AFI_MSI_FPCI_BAR_ST 0x64 +#define AFI_MSI_AXI_BAR_ST 0x68 + +#define AFI_MSI_VEC(x) (0x6c + ((x) * 4)) +#define AFI_MSI_EN_VEC(x) (0x8c + ((x) * 4)) + +#define AFI_CONFIGURATION 0xac +#define AFI_CONFIGURATION_EN_FPCI (1 << 0) +#define AFI_CONFIGURATION_CLKEN_OVERRIDE (1 << 31) + +#define AFI_FPCI_ERROR_MASKS 0xb0 + +#define AFI_INTR_MASK 0xb4 +#define AFI_INTR_MASK_INT_MASK (1 << 0) +#define AFI_INTR_MASK_MSI_MASK (1 << 8) + +#define AFI_INTR_CODE 0xb8 +#define AFI_INTR_CODE_MASK 0xf +#define AFI_INTR_INI_SLAVE_ERROR 1 +#define AFI_INTR_INI_DECODE_ERROR 2 +#define AFI_INTR_TARGET_ABORT 3 +#define AFI_INTR_MASTER_ABORT 4 +#define AFI_INTR_INVALID_WRITE 5 +#define AFI_INTR_LEGACY 6 +#define AFI_INTR_FPCI_DECODE_ERROR 7 +#define AFI_INTR_AXI_DECODE_ERROR 8 +#define AFI_INTR_FPCI_TIMEOUT 9 +#define AFI_INTR_PE_PRSNT_SENSE 10 +#define AFI_INTR_PE_CLKREQ_SENSE 11 +#define AFI_INTR_CLKCLAMP_SENSE 12 +#define AFI_INTR_RDY4PD_SENSE 13 +#define AFI_INTR_P2P_ERROR 14 + +#define AFI_INTR_SIGNATURE 0xbc +#define AFI_UPPER_FPCI_ADDRESS 0xc0 +#define AFI_SM_INTR_ENABLE 0xc4 +#define AFI_SM_INTR_INTA_ASSERT (1 << 0) +#define AFI_SM_INTR_INTB_ASSERT (1 << 1) +#define AFI_SM_INTR_INTC_ASSERT (1 << 2) +#define AFI_SM_INTR_INTD_ASSERT (1 << 3) +#define AFI_SM_INTR_INTA_DEASSERT (1 << 4) +#define AFI_SM_INTR_INTB_DEASSERT (1 << 5) +#define AFI_SM_INTR_INTC_DEASSERT (1 << 6) +#define AFI_SM_INTR_INTD_DEASSERT (1 << 7) + +#define AFI_AFI_INTR_ENABLE 0xc8 +#define AFI_INTR_EN_INI_SLVERR (1 << 0) +#define AFI_INTR_EN_INI_DECERR (1 << 1) +#define AFI_INTR_EN_TGT_SLVERR (1 << 2) +#define AFI_INTR_EN_TGT_DECERR (1 << 3) +#define AFI_INTR_EN_TGT_WRERR (1 << 4) +#define AFI_INTR_EN_DFPCI_DECERR (1 << 5) +#define AFI_INTR_EN_AXI_DECERR (1 << 6) +#define AFI_INTR_EN_FPCI_TIMEOUT (1 << 7) +#define AFI_INTR_EN_PRSNT_SENSE (1 << 8) + +#define AFI_PCIE_PME 0xf0 + +#define AFI_PCIE_CONFIG 0x0f8 +#define AFI_PCIE_CONFIG_PCIE_DISABLE(x) (1 << ((x) + 1)) +#define AFI_PCIE_CONFIG_PCIE_DISABLE_ALL 0xe +#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK (0xf << 20) +#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_SINGLE (0x0 << 20) +#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_420 (0x0 << 20) +#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X2_X1 (0x0 << 20) +#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_401 (0x0 << 20) +#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_DUAL (0x1 << 20) +#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_222 (0x1 << 20) +#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X4_X1 (0x1 << 20) +#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_211 (0x1 << 20) +#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_411 (0x2 << 20) +#define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_111 (0x2 << 20) +#define AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(x) (1 << ((x) + 29)) +#define AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO_ALL (0x7 << 29) + +#define AFI_FUSE 0x104 +#define AFI_FUSE_PCIE_T0_GEN2_DIS (1 << 2) + +#define AFI_PEX0_CTRL 0x110 +#define AFI_PEX1_CTRL 0x118 +#define AFI_PEX_CTRL_RST (1 << 0) +#define AFI_PEX_CTRL_CLKREQ_EN (1 << 1) +#define AFI_PEX_CTRL_REFCLK_EN (1 << 3) +#define AFI_PEX_CTRL_OVERRIDE_EN (1 << 4) + +#define AFI_PLLE_CONTROL 0x160 +#define AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL (1 << 9) +#define AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN (1 << 1) + +#define AFI_PEXBIAS_CTRL_0 0x168 + +#define RP_ECTL_2_R1 0x00000e84 +#define RP_ECTL_2_R1_RX_CTLE_1C_MASK 0xffff + +#define RP_ECTL_4_R1 0x00000e8c +#define RP_ECTL_4_R1_RX_CDR_CTRL_1C_MASK (0xffff << 16) +#define RP_ECTL_4_R1_RX_CDR_CTRL_1C_SHIFT 16 + +#define RP_ECTL_5_R1 0x00000e90 +#define RP_ECTL_5_R1_RX_EQ_CTRL_L_1C_MASK 0xffffffff + +#define RP_ECTL_6_R1 0x00000e94 +#define RP_ECTL_6_R1_RX_EQ_CTRL_H_1C_MASK 0xffffffff + +#define RP_ECTL_2_R2 0x00000ea4 +#define RP_ECTL_2_R2_RX_CTLE_1C_MASK 0xffff + +#define RP_ECTL_4_R2 0x00000eac +#define RP_ECTL_4_R2_RX_CDR_CTRL_1C_MASK (0xffff << 16) +#define RP_ECTL_4_R2_RX_CDR_CTRL_1C_SHIFT 16 + +#define RP_ECTL_5_R2 0x00000eb0 +#define RP_ECTL_5_R2_RX_EQ_CTRL_L_1C_MASK 0xffffffff + +#define RP_ECTL_6_R2 0x00000eb4 +#define RP_ECTL_6_R2_RX_EQ_CTRL_H_1C_MASK 0xffffffff + +#define RP_VEND_XP 0x00000f00 +#define RP_VEND_XP_DL_UP (1 << 30) +#define RP_VEND_XP_OPPORTUNISTIC_ACK (1 << 27) +#define RP_VEND_XP_OPPORTUNISTIC_UPDATEFC (1 << 28) +#define RP_VEND_XP_UPDATE_FC_THRESHOLD_MASK (0xff << 18) + +#define RP_VEND_CTL0 0x00000f44 +#define RP_VEND_CTL0_DSK_RST_PULSE_WIDTH_MASK (0xf << 12) +#define RP_VEND_CTL0_DSK_RST_PULSE_WIDTH (0x9 << 12) + +#define RP_VEND_CTL1 0x00000f48 +#define RP_VEND_CTL1_ERPT (1 << 13) + +#define RP_VEND_XP_BIST 0x00000f4c +#define RP_VEND_XP_BIST_GOTO_L1_L2_AFTER_DLLP_DONE (1 << 28) + +#define RP_VEND_CTL2 0x00000fa8 +#define RP_VEND_CTL2_PCA_ENABLE (1 << 7) + +#define RP_PRIV_MISC 0x00000fe0 +#define RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT (0xe << 0) +#define RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT (0xf << 0) +#define RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD_MASK (0x7f << 16) +#define RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD (0xf << 16) +#define RP_PRIV_MISC_CTLR_CLK_CLAMP_ENABLE (1 << 23) +#define RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD_MASK (0x7f << 24) +#define RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD (0xf << 24) +#define RP_PRIV_MISC_TMS_CLK_CLAMP_ENABLE (1 << 31) + +#define RP_LINK_CONTROL_STATUS 0x00000090 +#define RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE 0x20000000 +#define RP_LINK_CONTROL_STATUS_LINKSTAT_MASK 0x3fff0000 + +#define RP_LINK_CONTROL_STATUS_2 0x000000b0 + +#define PADS_CTL_SEL 0x0000009c + +#define PADS_CTL 0x000000a0 +#define PADS_CTL_IDDQ_1L (1 << 0) +#define PADS_CTL_TX_DATA_EN_1L (1 << 6) +#define PADS_CTL_RX_DATA_EN_1L (1 << 10) + +#define PADS_PLL_CTL_TEGRA20 0x000000b8 +#define PADS_PLL_CTL_TEGRA30 0x000000b4 +#define PADS_PLL_CTL_RST_B4SM (1 << 1) +#define PADS_PLL_CTL_LOCKDET (1 << 8) +#define PADS_PLL_CTL_REFCLK_MASK (0x3 << 16) +#define PADS_PLL_CTL_REFCLK_INTERNAL_CML (0 << 16) +#define PADS_PLL_CTL_REFCLK_INTERNAL_CMOS (1 << 16) +#define PADS_PLL_CTL_REFCLK_EXTERNAL (2 << 16) +#define PADS_PLL_CTL_TXCLKREF_MASK (0x1 << 20) +#define PADS_PLL_CTL_TXCLKREF_DIV10 (0 << 20) +#define PADS_PLL_CTL_TXCLKREF_DIV5 (1 << 20) +#define PADS_PLL_CTL_TXCLKREF_BUF_EN (1 << 22) + +#define PADS_REFCLK_CFG0 0x000000c8 +#define PADS_REFCLK_CFG1 0x000000cc +#define PADS_REFCLK_BIAS 0x000000d0 + +/* + * Fields in PADS_REFCLK_CFG*. Those registers form an array of 16-bit + * entries, one entry per PCIe port. These field definitions and desired + * values aren't in the TRM, but do come from NVIDIA. + */ +#define PADS_REFCLK_CFG_TERM_SHIFT 2 /* 6:2 */ +#define PADS_REFCLK_CFG_E_TERM_SHIFT 7 +#define PADS_REFCLK_CFG_PREDI_SHIFT 8 /* 11:8 */ +#define PADS_REFCLK_CFG_DRVI_SHIFT 12 /* 15:12 */ + +#define PME_ACK_TIMEOUT 10000 +#define LINK_RETRAIN_TIMEOUT 100000 /* in usec */ + +struct tegra_msi { + DECLARE_BITMAP(used, INT_PCI_MSI_NR); + struct irq_domain *domain; + struct mutex map_lock; + raw_spinlock_t mask_lock; + void *virt; + dma_addr_t phys; + int irq; +}; + +/* used to differentiate between Tegra SoC generations */ +struct tegra_pcie_port_soc { + struct { + u8 turnoff_bit; + u8 ack_bit; + } pme; +}; + +struct tegra_pcie_soc { + unsigned int num_ports; + const struct tegra_pcie_port_soc *ports; + unsigned int msi_base_shift; + unsigned long afi_pex2_ctrl; + u32 pads_pll_ctl; + u32 tx_ref_sel; + u32 pads_refclk_cfg0; + u32 pads_refclk_cfg1; + u32 update_fc_threshold; + bool has_pex_clkreq_en; + bool has_pex_bias_ctrl; + bool has_intr_prsnt_sense; + bool has_cml_clk; + bool has_gen2; + bool force_pca_enable; + bool program_uphy; + bool update_clamp_threshold; + bool program_deskew_time; + bool update_fc_timer; + bool has_cache_bars; + struct { + struct { + u32 rp_ectl_2_r1; + u32 rp_ectl_4_r1; + u32 rp_ectl_5_r1; + u32 rp_ectl_6_r1; + u32 rp_ectl_2_r2; + u32 rp_ectl_4_r2; + u32 rp_ectl_5_r2; + u32 rp_ectl_6_r2; + } regs; + bool enable; + } ectl; +}; + +struct tegra_pcie { + struct device *dev; + + void __iomem *pads; + void __iomem *afi; + void __iomem *cfg; + int irq; + + struct resource cs; + + struct clk *pex_clk; + struct clk *afi_clk; + struct clk *pll_e; + struct clk *cml_clk; + + struct reset_control *pex_rst; + struct reset_control *afi_rst; + struct reset_control *pcie_xrst; + + struct tegra_pmc *pmc; + + bool legacy_phy; + struct phy *phy; + + struct tegra_msi msi; + + struct list_head ports; + u32 xbar_config; + + struct regulator_bulk_data *supplies; + unsigned int num_supplies; + + const struct tegra_pcie_soc *soc; + struct dentry *debugfs; +}; + +static inline struct tegra_pcie *msi_to_pcie(struct tegra_msi *msi) +{ + return container_of(msi, struct tegra_pcie, msi); +} + +struct tegra_pcie_port { + struct tegra_pcie *pcie; + struct device_node *np; + struct list_head list; + struct resource regs; + void __iomem *base; + unsigned int index; + unsigned int lanes; + + struct phy **phys; + + struct gpio_desc *reset_gpio; +}; + +static inline void afi_writel(struct tegra_pcie *pcie, u32 value, + unsigned long offset) +{ + writel(value, pcie->afi + offset); +} + +static inline u32 afi_readl(struct tegra_pcie *pcie, unsigned long offset) +{ + return readl(pcie->afi + offset); +} + +static inline void pads_writel(struct tegra_pcie *pcie, u32 value, + unsigned long offset) +{ + writel(value, pcie->pads + offset); +} + +static inline u32 pads_readl(struct tegra_pcie *pcie, unsigned long offset) +{ + return readl(pcie->pads + offset); +} + +/* + * The configuration space mapping on Tegra is somewhat similar to the ECAM + * defined by PCIe. However it deviates a bit in how the 4 bits for extended + * register accesses are mapped: + * + * [27:24] extended register number + * [23:16] bus number + * [15:11] device number + * [10: 8] function number + * [ 7: 0] register number + * + * Mapping the whole extended configuration space would require 256 MiB of + * virtual address space, only a small part of which will actually be used. + * + * To work around this, a 4 KiB region is used to generate the required + * configuration transaction with relevant B:D:F and register offset values. + * This is achieved by dynamically programming base address and size of + * AFI_AXI_BAR used for end point config space mapping to make sure that the + * address (access to which generates correct config transaction) falls in + * this 4 KiB region. + */ +static unsigned int tegra_pcie_conf_offset(u8 bus, unsigned int devfn, + unsigned int where) +{ + return ((where & 0xf00) << 16) | (bus << 16) | (PCI_SLOT(devfn) << 11) | + (PCI_FUNC(devfn) << 8) | (where & 0xff); +} + +static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus, + unsigned int devfn, + int where) +{ + struct tegra_pcie *pcie = bus->sysdata; + void __iomem *addr = NULL; + + if (bus->number == 0) { + unsigned int slot = PCI_SLOT(devfn); + struct tegra_pcie_port *port; + + list_for_each_entry(port, &pcie->ports, list) { + if (port->index + 1 == slot) { + addr = port->base + (where & ~3); + break; + } + } + } else { + unsigned int offset; + u32 base; + + offset = tegra_pcie_conf_offset(bus->number, devfn, where); + + /* move 4 KiB window to offset within the FPCI region */ + base = 0xfe100000 + ((offset & ~(SZ_4K - 1)) >> 8); + afi_writel(pcie, base, AFI_FPCI_BAR0); + + /* move to correct offset within the 4 KiB page */ + addr = pcie->cfg + (offset & (SZ_4K - 1)); + } + + return addr; +} + +static int tegra_pcie_config_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *value) +{ + if (bus->number == 0) + return pci_generic_config_read32(bus, devfn, where, size, + value); + + return pci_generic_config_read(bus, devfn, where, size, value); +} + +static int tegra_pcie_config_write(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 value) +{ + if (bus->number == 0) + return pci_generic_config_write32(bus, devfn, where, size, + value); + + return pci_generic_config_write(bus, devfn, where, size, value); +} + +static struct pci_ops tegra_pcie_ops = { + .map_bus = tegra_pcie_map_bus, + .read = tegra_pcie_config_read, + .write = tegra_pcie_config_write, +}; + +static unsigned long tegra_pcie_port_get_pex_ctrl(struct tegra_pcie_port *port) +{ + const struct tegra_pcie_soc *soc = port->pcie->soc; + unsigned long ret = 0; + + switch (port->index) { + case 0: + ret = AFI_PEX0_CTRL; + break; + + case 1: + ret = AFI_PEX1_CTRL; + break; + + case 2: + ret = soc->afi_pex2_ctrl; + break; + } + + return ret; +} + +static void tegra_pcie_port_reset(struct tegra_pcie_port *port) +{ + unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port); + unsigned long value; + + /* pulse reset signal */ + if (port->reset_gpio) { + gpiod_set_value(port->reset_gpio, 1); + } else { + value = afi_readl(port->pcie, ctrl); + value &= ~AFI_PEX_CTRL_RST; + afi_writel(port->pcie, value, ctrl); + } + + usleep_range(1000, 2000); + + if (port->reset_gpio) { + gpiod_set_value(port->reset_gpio, 0); + } else { + value = afi_readl(port->pcie, ctrl); + value |= AFI_PEX_CTRL_RST; + afi_writel(port->pcie, value, ctrl); + } +} + +static void tegra_pcie_enable_rp_features(struct tegra_pcie_port *port) +{ + const struct tegra_pcie_soc *soc = port->pcie->soc; + u32 value; + + /* Enable AER capability */ + value = readl(port->base + RP_VEND_CTL1); + value |= RP_VEND_CTL1_ERPT; + writel(value, port->base + RP_VEND_CTL1); + + /* Optimal settings to enhance bandwidth */ + value = readl(port->base + RP_VEND_XP); + value |= RP_VEND_XP_OPPORTUNISTIC_ACK; + value |= RP_VEND_XP_OPPORTUNISTIC_UPDATEFC; + writel(value, port->base + RP_VEND_XP); + + /* + * LTSSM will wait for DLLP to finish before entering L1 or L2, + * to avoid truncation of PM messages which results in receiver errors + */ + value = readl(port->base + RP_VEND_XP_BIST); + value |= RP_VEND_XP_BIST_GOTO_L1_L2_AFTER_DLLP_DONE; + writel(value, port->base + RP_VEND_XP_BIST); + + value = readl(port->base + RP_PRIV_MISC); + value |= RP_PRIV_MISC_CTLR_CLK_CLAMP_ENABLE; + value |= RP_PRIV_MISC_TMS_CLK_CLAMP_ENABLE; + + if (soc->update_clamp_threshold) { + value &= ~(RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD_MASK | + RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD_MASK); + value |= RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD | + RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD; + } + + writel(value, port->base + RP_PRIV_MISC); +} + +static void tegra_pcie_program_ectl_settings(struct tegra_pcie_port *port) +{ + const struct tegra_pcie_soc *soc = port->pcie->soc; + u32 value; + + value = readl(port->base + RP_ECTL_2_R1); + value &= ~RP_ECTL_2_R1_RX_CTLE_1C_MASK; + value |= soc->ectl.regs.rp_ectl_2_r1; + writel(value, port->base + RP_ECTL_2_R1); + + value = readl(port->base + RP_ECTL_4_R1); + value &= ~RP_ECTL_4_R1_RX_CDR_CTRL_1C_MASK; + value |= soc->ectl.regs.rp_ectl_4_r1 << + RP_ECTL_4_R1_RX_CDR_CTRL_1C_SHIFT; + writel(value, port->base + RP_ECTL_4_R1); + + value = readl(port->base + RP_ECTL_5_R1); + value &= ~RP_ECTL_5_R1_RX_EQ_CTRL_L_1C_MASK; + value |= soc->ectl.regs.rp_ectl_5_r1; + writel(value, port->base + RP_ECTL_5_R1); + + value = readl(port->base + RP_ECTL_6_R1); + value &= ~RP_ECTL_6_R1_RX_EQ_CTRL_H_1C_MASK; + value |= soc->ectl.regs.rp_ectl_6_r1; + writel(value, port->base + RP_ECTL_6_R1); + + value = readl(port->base + RP_ECTL_2_R2); + value &= ~RP_ECTL_2_R2_RX_CTLE_1C_MASK; + value |= soc->ectl.regs.rp_ectl_2_r2; + writel(value, port->base + RP_ECTL_2_R2); + + value = readl(port->base + RP_ECTL_4_R2); + value &= ~RP_ECTL_4_R2_RX_CDR_CTRL_1C_MASK; + value |= soc->ectl.regs.rp_ectl_4_r2 << + RP_ECTL_4_R2_RX_CDR_CTRL_1C_SHIFT; + writel(value, port->base + RP_ECTL_4_R2); + + value = readl(port->base + RP_ECTL_5_R2); + value &= ~RP_ECTL_5_R2_RX_EQ_CTRL_L_1C_MASK; + value |= soc->ectl.regs.rp_ectl_5_r2; + writel(value, port->base + RP_ECTL_5_R2); + + value = readl(port->base + RP_ECTL_6_R2); + value &= ~RP_ECTL_6_R2_RX_EQ_CTRL_H_1C_MASK; + value |= soc->ectl.regs.rp_ectl_6_r2; + writel(value, port->base + RP_ECTL_6_R2); +} + +static void tegra_pcie_apply_sw_fixup(struct tegra_pcie_port *port) +{ + const struct tegra_pcie_soc *soc = port->pcie->soc; + u32 value; + + /* + * Sometimes link speed change from Gen2 to Gen1 fails due to + * instability in deskew logic on lane-0. Increase the deskew + * retry time to resolve this issue. + */ + if (soc->program_deskew_time) { + value = readl(port->base + RP_VEND_CTL0); + value &= ~RP_VEND_CTL0_DSK_RST_PULSE_WIDTH_MASK; + value |= RP_VEND_CTL0_DSK_RST_PULSE_WIDTH; + writel(value, port->base + RP_VEND_CTL0); + } + + if (soc->update_fc_timer) { + value = readl(port->base + RP_VEND_XP); + value &= ~RP_VEND_XP_UPDATE_FC_THRESHOLD_MASK; + value |= soc->update_fc_threshold; + writel(value, port->base + RP_VEND_XP); + } + + /* + * PCIe link doesn't come up with few legacy PCIe endpoints if + * root port advertises both Gen-1 and Gen-2 speeds in Tegra. + * Hence, the strategy followed here is to initially advertise + * only Gen-1 and after link is up, retrain link to Gen-2 speed + */ + value = readl(port->base + RP_LINK_CONTROL_STATUS_2); + value &= ~PCI_EXP_LNKSTA_CLS; + value |= PCI_EXP_LNKSTA_CLS_2_5GB; + writel(value, port->base + RP_LINK_CONTROL_STATUS_2); +} + +static void tegra_pcie_port_enable(struct tegra_pcie_port *port) +{ + unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port); + const struct tegra_pcie_soc *soc = port->pcie->soc; + unsigned long value; + + /* enable reference clock */ + value = afi_readl(port->pcie, ctrl); + value |= AFI_PEX_CTRL_REFCLK_EN; + + if (soc->has_pex_clkreq_en) + value |= AFI_PEX_CTRL_CLKREQ_EN; + + value |= AFI_PEX_CTRL_OVERRIDE_EN; + + afi_writel(port->pcie, value, ctrl); + + tegra_pcie_port_reset(port); + + if (soc->force_pca_enable) { + value = readl(port->base + RP_VEND_CTL2); + value |= RP_VEND_CTL2_PCA_ENABLE; + writel(value, port->base + RP_VEND_CTL2); + } + + tegra_pcie_enable_rp_features(port); + + if (soc->ectl.enable) + tegra_pcie_program_ectl_settings(port); + + tegra_pcie_apply_sw_fixup(port); +} + +static void tegra_pcie_port_disable(struct tegra_pcie_port *port) +{ + unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port); + const struct tegra_pcie_soc *soc = port->pcie->soc; + unsigned long value; + + /* assert port reset */ + value = afi_readl(port->pcie, ctrl); + value &= ~AFI_PEX_CTRL_RST; + afi_writel(port->pcie, value, ctrl); + + /* disable reference clock */ + value = afi_readl(port->pcie, ctrl); + + if (soc->has_pex_clkreq_en) + value &= ~AFI_PEX_CTRL_CLKREQ_EN; + + value &= ~AFI_PEX_CTRL_REFCLK_EN; + afi_writel(port->pcie, value, ctrl); + + /* disable PCIe port and set CLKREQ# as GPIO to allow PLLE power down */ + value = afi_readl(port->pcie, AFI_PCIE_CONFIG); + value |= AFI_PCIE_CONFIG_PCIE_DISABLE(port->index); + value |= AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(port->index); + afi_writel(port->pcie, value, AFI_PCIE_CONFIG); +} + +static void tegra_pcie_port_free(struct tegra_pcie_port *port) +{ + struct tegra_pcie *pcie = port->pcie; + struct device *dev = pcie->dev; + + devm_iounmap(dev, port->base); + devm_release_mem_region(dev, port->regs.start, + resource_size(&port->regs)); + list_del(&port->list); + devm_kfree(dev, port); +} + +/* Tegra PCIE root complex wrongly reports device class */ +static void tegra_pcie_fixup_class(struct pci_dev *dev) +{ + dev->class = PCI_CLASS_BRIDGE_PCI_NORMAL; +} +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0bf0, tegra_pcie_fixup_class); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0bf1, tegra_pcie_fixup_class); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1c, tegra_pcie_fixup_class); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1d, tegra_pcie_fixup_class); + +/* Tegra20 and Tegra30 PCIE requires relaxed ordering */ +static void tegra_pcie_relax_enable(struct pci_dev *dev) +{ + pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN); +} +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0bf0, tegra_pcie_relax_enable); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0bf1, tegra_pcie_relax_enable); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0e1c, tegra_pcie_relax_enable); +DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0e1d, tegra_pcie_relax_enable); + +static int tegra_pcie_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin) +{ + struct tegra_pcie *pcie = pdev->bus->sysdata; + int irq; + + tegra_cpuidle_pcie_irqs_in_use(); + + irq = of_irq_parse_and_map_pci(pdev, slot, pin); + if (!irq) + irq = pcie->irq; + + return irq; +} + +static irqreturn_t tegra_pcie_isr(int irq, void *arg) +{ + static const char * const err_msg[] = { + "Unknown", + "AXI slave error", + "AXI decode error", + "Target abort", + "Master abort", + "Invalid write", + "Legacy interrupt", + "Response decoding error", + "AXI response decoding error", + "Transaction timeout", + "Slot present pin change", + "Slot clock request change", + "TMS clock ramp change", + "TMS ready for power down", + "Peer2Peer error", + }; + struct tegra_pcie *pcie = arg; + struct device *dev = pcie->dev; + u32 code, signature; + + code = afi_readl(pcie, AFI_INTR_CODE) & AFI_INTR_CODE_MASK; + signature = afi_readl(pcie, AFI_INTR_SIGNATURE); + afi_writel(pcie, 0, AFI_INTR_CODE); + + if (code == AFI_INTR_LEGACY) + return IRQ_NONE; + + if (code >= ARRAY_SIZE(err_msg)) + code = 0; + + /* + * do not pollute kernel log with master abort reports since they + * happen a lot during enumeration + */ + if (code == AFI_INTR_MASTER_ABORT || code == AFI_INTR_PE_PRSNT_SENSE) + dev_dbg(dev, "%s, signature: %08x\n", err_msg[code], signature); + else + dev_err(dev, "%s, signature: %08x\n", err_msg[code], signature); + + if (code == AFI_INTR_TARGET_ABORT || code == AFI_INTR_MASTER_ABORT || + code == AFI_INTR_FPCI_DECODE_ERROR) { + u32 fpci = afi_readl(pcie, AFI_UPPER_FPCI_ADDRESS) & 0xff; + u64 address = (u64)fpci << 32 | (signature & 0xfffffffc); + + if (code == AFI_INTR_MASTER_ABORT) + dev_dbg(dev, " FPCI address: %10llx\n", address); + else + dev_err(dev, " FPCI address: %10llx\n", address); + } + + return IRQ_HANDLED; +} + +/* + * FPCI map is as follows: + * - 0xfdfc000000: I/O space + * - 0xfdfe000000: type 0 configuration space + * - 0xfdff000000: type 1 configuration space + * - 0xfe00000000: type 0 extended configuration space + * - 0xfe10000000: type 1 extended configuration space + */ +static void tegra_pcie_setup_translations(struct tegra_pcie *pcie) +{ + u32 size; + struct resource_entry *entry; + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + + /* Bar 0: type 1 extended configuration space */ + size = resource_size(&pcie->cs); + afi_writel(pcie, pcie->cs.start, AFI_AXI_BAR0_START); + afi_writel(pcie, size >> 12, AFI_AXI_BAR0_SZ); + + resource_list_for_each_entry(entry, &bridge->windows) { + u32 fpci_bar, axi_address; + struct resource *res = entry->res; + + size = resource_size(res); + + switch (resource_type(res)) { + case IORESOURCE_IO: + /* Bar 1: downstream IO bar */ + fpci_bar = 0xfdfc0000; + axi_address = pci_pio_to_address(res->start); + afi_writel(pcie, axi_address, AFI_AXI_BAR1_START); + afi_writel(pcie, size >> 12, AFI_AXI_BAR1_SZ); + afi_writel(pcie, fpci_bar, AFI_FPCI_BAR1); + break; + case IORESOURCE_MEM: + fpci_bar = (((res->start >> 12) & 0x0fffffff) << 4) | 0x1; + axi_address = res->start; + + if (res->flags & IORESOURCE_PREFETCH) { + /* Bar 2: prefetchable memory BAR */ + afi_writel(pcie, axi_address, AFI_AXI_BAR2_START); + afi_writel(pcie, size >> 12, AFI_AXI_BAR2_SZ); + afi_writel(pcie, fpci_bar, AFI_FPCI_BAR2); + + } else { + /* Bar 3: non prefetchable memory BAR */ + afi_writel(pcie, axi_address, AFI_AXI_BAR3_START); + afi_writel(pcie, size >> 12, AFI_AXI_BAR3_SZ); + afi_writel(pcie, fpci_bar, AFI_FPCI_BAR3); + } + break; + } + } + + /* NULL out the remaining BARs as they are not used */ + afi_writel(pcie, 0, AFI_AXI_BAR4_START); + afi_writel(pcie, 0, AFI_AXI_BAR4_SZ); + afi_writel(pcie, 0, AFI_FPCI_BAR4); + + afi_writel(pcie, 0, AFI_AXI_BAR5_START); + afi_writel(pcie, 0, AFI_AXI_BAR5_SZ); + afi_writel(pcie, 0, AFI_FPCI_BAR5); + + if (pcie->soc->has_cache_bars) { + /* map all upstream transactions as uncached */ + afi_writel(pcie, 0, AFI_CACHE_BAR0_ST); + afi_writel(pcie, 0, AFI_CACHE_BAR0_SZ); + afi_writel(pcie, 0, AFI_CACHE_BAR1_ST); + afi_writel(pcie, 0, AFI_CACHE_BAR1_SZ); + } + + /* MSI translations are setup only when needed */ + afi_writel(pcie, 0, AFI_MSI_FPCI_BAR_ST); + afi_writel(pcie, 0, AFI_MSI_BAR_SZ); + afi_writel(pcie, 0, AFI_MSI_AXI_BAR_ST); + afi_writel(pcie, 0, AFI_MSI_BAR_SZ); +} + +static int tegra_pcie_pll_wait(struct tegra_pcie *pcie, unsigned long timeout) +{ + const struct tegra_pcie_soc *soc = pcie->soc; + u32 value; + + timeout = jiffies + msecs_to_jiffies(timeout); + + while (time_before(jiffies, timeout)) { + value = pads_readl(pcie, soc->pads_pll_ctl); + if (value & PADS_PLL_CTL_LOCKDET) + return 0; + } + + return -ETIMEDOUT; +} + +static int tegra_pcie_phy_enable(struct tegra_pcie *pcie) +{ + struct device *dev = pcie->dev; + const struct tegra_pcie_soc *soc = pcie->soc; + u32 value; + int err; + + /* initialize internal PHY, enable up to 16 PCIE lanes */ + pads_writel(pcie, 0x0, PADS_CTL_SEL); + + /* override IDDQ to 1 on all 4 lanes */ + value = pads_readl(pcie, PADS_CTL); + value |= PADS_CTL_IDDQ_1L; + pads_writel(pcie, value, PADS_CTL); + + /* + * Set up PHY PLL inputs select PLLE output as refclock, + * set TX ref sel to div10 (not div5). + */ + value = pads_readl(pcie, soc->pads_pll_ctl); + value &= ~(PADS_PLL_CTL_REFCLK_MASK | PADS_PLL_CTL_TXCLKREF_MASK); + value |= PADS_PLL_CTL_REFCLK_INTERNAL_CML | soc->tx_ref_sel; + pads_writel(pcie, value, soc->pads_pll_ctl); + + /* reset PLL */ + value = pads_readl(pcie, soc->pads_pll_ctl); + value &= ~PADS_PLL_CTL_RST_B4SM; + pads_writel(pcie, value, soc->pads_pll_ctl); + + usleep_range(20, 100); + + /* take PLL out of reset */ + value = pads_readl(pcie, soc->pads_pll_ctl); + value |= PADS_PLL_CTL_RST_B4SM; + pads_writel(pcie, value, soc->pads_pll_ctl); + + /* wait for the PLL to lock */ + err = tegra_pcie_pll_wait(pcie, 500); + if (err < 0) { + dev_err(dev, "PLL failed to lock: %d\n", err); + return err; + } + + /* turn off IDDQ override */ + value = pads_readl(pcie, PADS_CTL); + value &= ~PADS_CTL_IDDQ_1L; + pads_writel(pcie, value, PADS_CTL); + + /* enable TX/RX data */ + value = pads_readl(pcie, PADS_CTL); + value |= PADS_CTL_TX_DATA_EN_1L | PADS_CTL_RX_DATA_EN_1L; + pads_writel(pcie, value, PADS_CTL); + + return 0; +} + +static int tegra_pcie_phy_disable(struct tegra_pcie *pcie) +{ + const struct tegra_pcie_soc *soc = pcie->soc; + u32 value; + + /* disable TX/RX data */ + value = pads_readl(pcie, PADS_CTL); + value &= ~(PADS_CTL_TX_DATA_EN_1L | PADS_CTL_RX_DATA_EN_1L); + pads_writel(pcie, value, PADS_CTL); + + /* override IDDQ */ + value = pads_readl(pcie, PADS_CTL); + value |= PADS_CTL_IDDQ_1L; + pads_writel(pcie, value, PADS_CTL); + + /* reset PLL */ + value = pads_readl(pcie, soc->pads_pll_ctl); + value &= ~PADS_PLL_CTL_RST_B4SM; + pads_writel(pcie, value, soc->pads_pll_ctl); + + usleep_range(20, 100); + + return 0; +} + +static int tegra_pcie_port_phy_power_on(struct tegra_pcie_port *port) +{ + struct device *dev = port->pcie->dev; + unsigned int i; + int err; + + for (i = 0; i < port->lanes; i++) { + err = phy_power_on(port->phys[i]); + if (err < 0) { + dev_err(dev, "failed to power on PHY#%u: %d\n", i, err); + return err; + } + } + + return 0; +} + +static int tegra_pcie_port_phy_power_off(struct tegra_pcie_port *port) +{ + struct device *dev = port->pcie->dev; + unsigned int i; + int err; + + for (i = 0; i < port->lanes; i++) { + err = phy_power_off(port->phys[i]); + if (err < 0) { + dev_err(dev, "failed to power off PHY#%u: %d\n", i, + err); + return err; + } + } + + return 0; +} + +static int tegra_pcie_phy_power_on(struct tegra_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct tegra_pcie_port *port; + int err; + + if (pcie->legacy_phy) { + if (pcie->phy) + err = phy_power_on(pcie->phy); + else + err = tegra_pcie_phy_enable(pcie); + + if (err < 0) + dev_err(dev, "failed to power on PHY: %d\n", err); + + return err; + } + + list_for_each_entry(port, &pcie->ports, list) { + err = tegra_pcie_port_phy_power_on(port); + if (err < 0) { + dev_err(dev, + "failed to power on PCIe port %u PHY: %d\n", + port->index, err); + return err; + } + } + + return 0; +} + +static int tegra_pcie_phy_power_off(struct tegra_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct tegra_pcie_port *port; + int err; + + if (pcie->legacy_phy) { + if (pcie->phy) + err = phy_power_off(pcie->phy); + else + err = tegra_pcie_phy_disable(pcie); + + if (err < 0) + dev_err(dev, "failed to power off PHY: %d\n", err); + + return err; + } + + list_for_each_entry(port, &pcie->ports, list) { + err = tegra_pcie_port_phy_power_off(port); + if (err < 0) { + dev_err(dev, + "failed to power off PCIe port %u PHY: %d\n", + port->index, err); + return err; + } + } + + return 0; +} + +static void tegra_pcie_enable_controller(struct tegra_pcie *pcie) +{ + const struct tegra_pcie_soc *soc = pcie->soc; + struct tegra_pcie_port *port; + unsigned long value; + + /* enable PLL power down */ + if (pcie->phy) { + value = afi_readl(pcie, AFI_PLLE_CONTROL); + value &= ~AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL; + value |= AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN; + afi_writel(pcie, value, AFI_PLLE_CONTROL); + } + + /* power down PCIe slot clock bias pad */ + if (soc->has_pex_bias_ctrl) + afi_writel(pcie, 0, AFI_PEXBIAS_CTRL_0); + + /* configure mode and disable all ports */ + value = afi_readl(pcie, AFI_PCIE_CONFIG); + value &= ~AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK; + value |= AFI_PCIE_CONFIG_PCIE_DISABLE_ALL | pcie->xbar_config; + value |= AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO_ALL; + + list_for_each_entry(port, &pcie->ports, list) { + value &= ~AFI_PCIE_CONFIG_PCIE_DISABLE(port->index); + value &= ~AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(port->index); + } + + afi_writel(pcie, value, AFI_PCIE_CONFIG); + + if (soc->has_gen2) { + value = afi_readl(pcie, AFI_FUSE); + value &= ~AFI_FUSE_PCIE_T0_GEN2_DIS; + afi_writel(pcie, value, AFI_FUSE); + } else { + value = afi_readl(pcie, AFI_FUSE); + value |= AFI_FUSE_PCIE_T0_GEN2_DIS; + afi_writel(pcie, value, AFI_FUSE); + } + + /* Disable AFI dynamic clock gating and enable PCIe */ + value = afi_readl(pcie, AFI_CONFIGURATION); + value |= AFI_CONFIGURATION_EN_FPCI; + value |= AFI_CONFIGURATION_CLKEN_OVERRIDE; + afi_writel(pcie, value, AFI_CONFIGURATION); + + value = AFI_INTR_EN_INI_SLVERR | AFI_INTR_EN_INI_DECERR | + AFI_INTR_EN_TGT_SLVERR | AFI_INTR_EN_TGT_DECERR | + AFI_INTR_EN_TGT_WRERR | AFI_INTR_EN_DFPCI_DECERR; + + if (soc->has_intr_prsnt_sense) + value |= AFI_INTR_EN_PRSNT_SENSE; + + afi_writel(pcie, value, AFI_AFI_INTR_ENABLE); + afi_writel(pcie, 0xffffffff, AFI_SM_INTR_ENABLE); + + /* don't enable MSI for now, only when needed */ + afi_writel(pcie, AFI_INTR_MASK_INT_MASK, AFI_INTR_MASK); + + /* disable all exceptions */ + afi_writel(pcie, 0, AFI_FPCI_ERROR_MASKS); +} + +static void tegra_pcie_power_off(struct tegra_pcie *pcie) +{ + struct device *dev = pcie->dev; + const struct tegra_pcie_soc *soc = pcie->soc; + int err; + + reset_control_assert(pcie->afi_rst); + + clk_disable_unprepare(pcie->pll_e); + if (soc->has_cml_clk) + clk_disable_unprepare(pcie->cml_clk); + clk_disable_unprepare(pcie->afi_clk); + + if (!dev->pm_domain) + tegra_pmc_powergate_power_off(pcie->pmc, TEGRA_POWERGATE_PCIE); + + err = regulator_bulk_disable(pcie->num_supplies, pcie->supplies); + if (err < 0) + dev_warn(dev, "failed to disable regulators: %d\n", err); +} + +static int tegra_pcie_power_on(struct tegra_pcie *pcie) +{ + struct device *dev = pcie->dev; + const struct tegra_pcie_soc *soc = pcie->soc; + int err; + + reset_control_assert(pcie->pcie_xrst); + reset_control_assert(pcie->afi_rst); + reset_control_assert(pcie->pex_rst); + + if (!dev->pm_domain) + tegra_pmc_powergate_power_off(pcie->pmc, TEGRA_POWERGATE_PCIE); + + /* enable regulators */ + err = regulator_bulk_enable(pcie->num_supplies, pcie->supplies); + if (err < 0) + dev_err(dev, "failed to enable regulators: %d\n", err); + + if (!dev->pm_domain) { + err = tegra_pmc_powergate_power_on(pcie->pmc, + TEGRA_POWERGATE_PCIE); + if (err) { + dev_err(dev, "failed to power ungate: %d\n", err); + goto regulator_disable; + } + err = tegra_pmc_powergate_remove_clamping(pcie->pmc, + TEGRA_POWERGATE_PCIE); + if (err) { + dev_err(dev, "failed to remove clamp: %d\n", err); + goto powergate; + } + } + + err = clk_prepare_enable(pcie->afi_clk); + if (err < 0) { + dev_err(dev, "failed to enable AFI clock: %d\n", err); + goto powergate; + } + + if (soc->has_cml_clk) { + err = clk_prepare_enable(pcie->cml_clk); + if (err < 0) { + dev_err(dev, "failed to enable CML clock: %d\n", err); + goto disable_afi_clk; + } + } + + err = clk_prepare_enable(pcie->pll_e); + if (err < 0) { + dev_err(dev, "failed to enable PLLE clock: %d\n", err); + goto disable_cml_clk; + } + + reset_control_deassert(pcie->afi_rst); + + return 0; + +disable_cml_clk: + if (soc->has_cml_clk) + clk_disable_unprepare(pcie->cml_clk); +disable_afi_clk: + clk_disable_unprepare(pcie->afi_clk); +powergate: + if (!dev->pm_domain) + tegra_pmc_powergate_power_off(pcie->pmc, TEGRA_POWERGATE_PCIE); +regulator_disable: + regulator_bulk_disable(pcie->num_supplies, pcie->supplies); + + return err; +} + +static void tegra_pcie_apply_pad_settings(struct tegra_pcie *pcie) +{ + const struct tegra_pcie_soc *soc = pcie->soc; + + /* Configure the reference clock driver */ + pads_writel(pcie, soc->pads_refclk_cfg0, PADS_REFCLK_CFG0); + + if (soc->num_ports > 2) + pads_writel(pcie, soc->pads_refclk_cfg1, PADS_REFCLK_CFG1); +} + +static int tegra_pcie_clocks_get(struct tegra_pcie *pcie) +{ + struct device *dev = pcie->dev; + const struct tegra_pcie_soc *soc = pcie->soc; + + pcie->pex_clk = devm_clk_get(dev, "pex"); + if (IS_ERR(pcie->pex_clk)) + return PTR_ERR(pcie->pex_clk); + + pcie->afi_clk = devm_clk_get(dev, "afi"); + if (IS_ERR(pcie->afi_clk)) + return PTR_ERR(pcie->afi_clk); + + pcie->pll_e = devm_clk_get(dev, "pll_e"); + if (IS_ERR(pcie->pll_e)) + return PTR_ERR(pcie->pll_e); + + if (soc->has_cml_clk) { + pcie->cml_clk = devm_clk_get(dev, "cml"); + if (IS_ERR(pcie->cml_clk)) + return PTR_ERR(pcie->cml_clk); + } + + return 0; +} + +static int tegra_pcie_resets_get(struct tegra_pcie *pcie) +{ + struct device *dev = pcie->dev; + + pcie->pex_rst = devm_reset_control_get_exclusive(dev, "pex"); + if (IS_ERR(pcie->pex_rst)) + return PTR_ERR(pcie->pex_rst); + + pcie->afi_rst = devm_reset_control_get_exclusive(dev, "afi"); + if (IS_ERR(pcie->afi_rst)) + return PTR_ERR(pcie->afi_rst); + + pcie->pcie_xrst = devm_reset_control_get_exclusive(dev, "pcie_x"); + if (IS_ERR(pcie->pcie_xrst)) + return PTR_ERR(pcie->pcie_xrst); + + return 0; +} + +static int tegra_pcie_phys_get_legacy(struct tegra_pcie *pcie) +{ + struct device *dev = pcie->dev; + int err; + + pcie->phy = devm_phy_optional_get(dev, "pcie"); + if (IS_ERR(pcie->phy)) { + err = PTR_ERR(pcie->phy); + dev_err(dev, "failed to get PHY: %d\n", err); + return err; + } + + err = phy_init(pcie->phy); + if (err < 0) { + dev_err(dev, "failed to initialize PHY: %d\n", err); + return err; + } + + pcie->legacy_phy = true; + + return 0; +} + +static struct phy *devm_of_phy_optional_get_index(struct device *dev, + struct device_node *np, + const char *consumer, + unsigned int index) +{ + struct phy *phy; + char *name; + + name = kasprintf(GFP_KERNEL, "%s-%u", consumer, index); + if (!name) + return ERR_PTR(-ENOMEM); + + phy = devm_of_phy_optional_get(dev, np, name); + kfree(name); + + return phy; +} + +static int tegra_pcie_port_get_phys(struct tegra_pcie_port *port) +{ + struct device *dev = port->pcie->dev; + struct phy *phy; + unsigned int i; + int err; + + port->phys = devm_kcalloc(dev, port->lanes, sizeof(phy), GFP_KERNEL); + if (!port->phys) + return -ENOMEM; + + for (i = 0; i < port->lanes; i++) { + phy = devm_of_phy_optional_get_index(dev, port->np, "pcie", i); + if (IS_ERR(phy)) { + dev_err(dev, "failed to get PHY#%u: %pe\n", i, phy); + return PTR_ERR(phy); + } + + err = phy_init(phy); + if (err < 0) { + dev_err(dev, "failed to initialize PHY#%u: %d\n", i, + err); + return err; + } + + port->phys[i] = phy; + } + + return 0; +} + +static int tegra_pcie_phys_get(struct tegra_pcie *pcie) +{ + const struct tegra_pcie_soc *soc = pcie->soc; + struct device_node *np = pcie->dev->of_node; + struct tegra_pcie_port *port; + int err; + + if (!soc->has_gen2 || of_property_present(np, "phys")) + return tegra_pcie_phys_get_legacy(pcie); + + list_for_each_entry(port, &pcie->ports, list) { + err = tegra_pcie_port_get_phys(port); + if (err < 0) + return err; + } + + return 0; +} + +static void tegra_pcie_phys_put(struct tegra_pcie *pcie) +{ + struct tegra_pcie_port *port; + struct device *dev = pcie->dev; + int err, i; + + if (pcie->legacy_phy) { + err = phy_exit(pcie->phy); + if (err < 0) + dev_err(dev, "failed to teardown PHY: %d\n", err); + return; + } + + list_for_each_entry(port, &pcie->ports, list) { + for (i = 0; i < port->lanes; i++) { + err = phy_exit(port->phys[i]); + if (err < 0) + dev_err(dev, "failed to teardown PHY#%u: %d\n", + i, err); + } + } +} + +static int tegra_pcie_get_resources(struct tegra_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct platform_device *pdev = to_platform_device(dev); + struct resource *res; + const struct tegra_pcie_soc *soc = pcie->soc; + int err; + + err = tegra_pcie_clocks_get(pcie); + if (err) { + dev_err(dev, "failed to get clocks: %d\n", err); + return err; + } + + err = tegra_pcie_resets_get(pcie); + if (err) { + dev_err(dev, "failed to get resets: %d\n", err); + return err; + } + + pcie->pmc = devm_tegra_pmc_get(dev); + if (IS_ERR(pcie->pmc)) + return dev_err_probe(dev, PTR_ERR(pcie->pmc), + "failed to get PMC\n"); + + if (soc->program_uphy) { + err = tegra_pcie_phys_get(pcie); + if (err < 0) { + dev_err(dev, "failed to get PHYs: %d\n", err); + return err; + } + } + + pcie->pads = devm_platform_ioremap_resource_byname(pdev, "pads"); + if (IS_ERR(pcie->pads)) { + err = PTR_ERR(pcie->pads); + goto phys_put; + } + + pcie->afi = devm_platform_ioremap_resource_byname(pdev, "afi"); + if (IS_ERR(pcie->afi)) { + err = PTR_ERR(pcie->afi); + goto phys_put; + } + + /* request configuration space, but remap later, on demand */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cs"); + if (!res) { + err = -EADDRNOTAVAIL; + goto phys_put; + } + + pcie->cs = *res; + + /* constrain configuration space to 4 KiB */ + resource_set_size(&pcie->cs, SZ_4K); + + pcie->cfg = devm_ioremap_resource(dev, &pcie->cs); + if (IS_ERR(pcie->cfg)) { + err = PTR_ERR(pcie->cfg); + goto phys_put; + } + + /* request interrupt */ + err = platform_get_irq_byname(pdev, "intr"); + if (err < 0) + goto phys_put; + + pcie->irq = err; + + err = request_irq(pcie->irq, tegra_pcie_isr, IRQF_SHARED, "PCIE", pcie); + if (err) { + dev_err(dev, "failed to register IRQ: %d\n", err); + goto phys_put; + } + + return 0; + +phys_put: + if (soc->program_uphy) + tegra_pcie_phys_put(pcie); + + return err; +} + +static int tegra_pcie_put_resources(struct tegra_pcie *pcie) +{ + const struct tegra_pcie_soc *soc = pcie->soc; + + if (pcie->irq > 0) + free_irq(pcie->irq, pcie); + + if (soc->program_uphy) + tegra_pcie_phys_put(pcie); + + return 0; +} + +static void tegra_pcie_pme_turnoff(struct tegra_pcie_port *port) +{ + struct tegra_pcie *pcie = port->pcie; + const struct tegra_pcie_soc *soc = pcie->soc; + int err; + u32 val; + u8 ack_bit; + + val = afi_readl(pcie, AFI_PCIE_PME); + val |= (0x1 << soc->ports[port->index].pme.turnoff_bit); + afi_writel(pcie, val, AFI_PCIE_PME); + + ack_bit = soc->ports[port->index].pme.ack_bit; + err = readl_poll_timeout(pcie->afi + AFI_PCIE_PME, val, + val & (0x1 << ack_bit), 1, PME_ACK_TIMEOUT); + if (err) + dev_err(pcie->dev, "PME Ack is not received on port: %d\n", + port->index); + + usleep_range(10000, 11000); + + val = afi_readl(pcie, AFI_PCIE_PME); + val &= ~(0x1 << soc->ports[port->index].pme.turnoff_bit); + afi_writel(pcie, val, AFI_PCIE_PME); +} + +static void tegra_pcie_msi_irq(struct irq_desc *desc) +{ + struct tegra_pcie *pcie = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + struct tegra_msi *msi = &pcie->msi; + struct device *dev = pcie->dev; + unsigned int i; + + chained_irq_enter(chip, desc); + + for (i = 0; i < 8; i++) { + unsigned long reg = afi_readl(pcie, AFI_MSI_VEC(i)); + + while (reg) { + unsigned int offset = find_first_bit(®, 32); + unsigned int index = i * 32 + offset; + int ret; + + ret = generic_handle_domain_irq(msi->domain, index); + if (ret) { + /* + * that's weird who triggered this? + * just clear it + */ + dev_info(dev, "unexpected MSI\n"); + afi_writel(pcie, BIT(index % 32), AFI_MSI_VEC(index)); + } + + /* see if there's any more pending in this vector */ + reg = afi_readl(pcie, AFI_MSI_VEC(i)); + } + } + + chained_irq_exit(chip, desc); +} + +static void tegra_msi_irq_ack(struct irq_data *d) +{ + struct tegra_msi *msi = irq_data_get_irq_chip_data(d); + struct tegra_pcie *pcie = msi_to_pcie(msi); + unsigned int index = d->hwirq / 32; + + /* clear the interrupt */ + afi_writel(pcie, BIT(d->hwirq % 32), AFI_MSI_VEC(index)); +} + +static void tegra_msi_irq_mask(struct irq_data *d) +{ + struct tegra_msi *msi = irq_data_get_irq_chip_data(d); + struct tegra_pcie *pcie = msi_to_pcie(msi); + unsigned int index = d->hwirq / 32; + u32 value; + + scoped_guard(raw_spinlock_irqsave, &msi->mask_lock) { + value = afi_readl(pcie, AFI_MSI_EN_VEC(index)); + value &= ~BIT(d->hwirq % 32); + afi_writel(pcie, value, AFI_MSI_EN_VEC(index)); + } +} + +static void tegra_msi_irq_unmask(struct irq_data *d) +{ + struct tegra_msi *msi = irq_data_get_irq_chip_data(d); + struct tegra_pcie *pcie = msi_to_pcie(msi); + unsigned int index = d->hwirq / 32; + u32 value; + + scoped_guard(raw_spinlock_irqsave, &msi->mask_lock) { + value = afi_readl(pcie, AFI_MSI_EN_VEC(index)); + value |= BIT(d->hwirq % 32); + afi_writel(pcie, value, AFI_MSI_EN_VEC(index)); + } +} + +static void tegra_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct tegra_msi *msi = irq_data_get_irq_chip_data(data); + + msg->address_lo = lower_32_bits(msi->phys); + msg->address_hi = upper_32_bits(msi->phys); + msg->data = data->hwirq; +} + +static struct irq_chip tegra_msi_bottom_chip = { + .name = "Tegra MSI", + .irq_ack = tegra_msi_irq_ack, + .irq_mask = tegra_msi_irq_mask, + .irq_unmask = tegra_msi_irq_unmask, + .irq_compose_msi_msg = tegra_compose_msi_msg, +}; + +static int tegra_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct tegra_msi *msi = domain->host_data; + unsigned int i; + int hwirq; + + mutex_lock(&msi->map_lock); + + hwirq = bitmap_find_free_region(msi->used, INT_PCI_MSI_NR, order_base_2(nr_irqs)); + + mutex_unlock(&msi->map_lock); + + if (hwirq < 0) + return -ENOSPC; + + for (i = 0; i < nr_irqs; i++) + irq_domain_set_info(domain, virq + i, hwirq + i, + &tegra_msi_bottom_chip, domain->host_data, + handle_edge_irq, NULL, NULL); + + tegra_cpuidle_pcie_irqs_in_use(); + + return 0; +} + +static void tegra_msi_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct tegra_msi *msi = domain->host_data; + + mutex_lock(&msi->map_lock); + + bitmap_release_region(msi->used, d->hwirq, order_base_2(nr_irqs)); + + mutex_unlock(&msi->map_lock); +} + +static const struct irq_domain_ops tegra_msi_domain_ops = { + .alloc = tegra_msi_domain_alloc, + .free = tegra_msi_domain_free, +}; + +static const struct msi_parent_ops tegra_msi_parent_ops = { + .supported_flags = (MSI_GENERIC_FLAGS_MASK | + MSI_FLAG_PCI_MSIX), + .required_flags = (MSI_FLAG_USE_DEF_DOM_OPS | + MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_PCI_MSI_MASK_PARENT | + MSI_FLAG_NO_AFFINITY), + .chip_flags = MSI_CHIP_FLAG_SET_ACK, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +static int tegra_allocate_domains(struct tegra_msi *msi) +{ + struct tegra_pcie *pcie = msi_to_pcie(msi); + struct fwnode_handle *fwnode = dev_fwnode(pcie->dev); + struct irq_domain_info info = { + .fwnode = fwnode, + .ops = &tegra_msi_domain_ops, + .size = INT_PCI_MSI_NR, + .host_data = msi, + }; + + msi->domain = msi_create_parent_irq_domain(&info, &tegra_msi_parent_ops); + if (!msi->domain) { + dev_err(pcie->dev, "failed to create MSI domain\n"); + return -ENOMEM; + } + return 0; +} + +static void tegra_free_domains(struct tegra_msi *msi) +{ + irq_domain_remove(msi->domain); +} + +static int tegra_pcie_msi_setup(struct tegra_pcie *pcie) +{ + struct platform_device *pdev = to_platform_device(pcie->dev); + struct tegra_msi *msi = &pcie->msi; + struct device *dev = pcie->dev; + int err; + + mutex_init(&msi->map_lock); + raw_spin_lock_init(&msi->mask_lock); + + if (IS_ENABLED(CONFIG_PCI_MSI)) { + err = tegra_allocate_domains(msi); + if (err) + return err; + } + + err = platform_get_irq_byname(pdev, "msi"); + if (err < 0) + goto free_irq_domain; + + msi->irq = err; + + irq_set_chained_handler_and_data(msi->irq, tegra_pcie_msi_irq, pcie); + + /* Though the PCIe controller can address >32-bit address space, to + * facilitate endpoints that support only 32-bit MSI target address, + * the mask is set to 32-bit to make sure that MSI target address is + * always a 32-bit address + */ + err = dma_set_coherent_mask(dev, DMA_BIT_MASK(32)); + if (err < 0) { + dev_err(dev, "failed to set DMA coherent mask: %d\n", err); + goto free_irq; + } + + msi->virt = dma_alloc_attrs(dev, PAGE_SIZE, &msi->phys, GFP_KERNEL, + DMA_ATTR_NO_KERNEL_MAPPING); + if (!msi->virt) { + dev_err(dev, "failed to allocate DMA memory for MSI\n"); + err = -ENOMEM; + goto free_irq; + } + + return 0; + +free_irq: + irq_set_chained_handler_and_data(msi->irq, NULL, NULL); +free_irq_domain: + if (IS_ENABLED(CONFIG_PCI_MSI)) + tegra_free_domains(msi); + + return err; +} + +static void tegra_pcie_enable_msi(struct tegra_pcie *pcie) +{ + const struct tegra_pcie_soc *soc = pcie->soc; + struct tegra_msi *msi = &pcie->msi; + u32 reg, msi_state[INT_PCI_MSI_NR / 32]; + int i; + + afi_writel(pcie, msi->phys >> soc->msi_base_shift, AFI_MSI_FPCI_BAR_ST); + afi_writel(pcie, msi->phys, AFI_MSI_AXI_BAR_ST); + /* this register is in 4K increments */ + afi_writel(pcie, 1, AFI_MSI_BAR_SZ); + + /* Restore the MSI allocation state */ + bitmap_to_arr32(msi_state, msi->used, INT_PCI_MSI_NR); + for (i = 0; i < ARRAY_SIZE(msi_state); i++) + afi_writel(pcie, msi_state[i], AFI_MSI_EN_VEC(i)); + + /* and unmask the MSI interrupt */ + reg = afi_readl(pcie, AFI_INTR_MASK); + reg |= AFI_INTR_MASK_MSI_MASK; + afi_writel(pcie, reg, AFI_INTR_MASK); +} + +static void tegra_pcie_msi_teardown(struct tegra_pcie *pcie) +{ + struct tegra_msi *msi = &pcie->msi; + unsigned int i, irq; + + dma_free_attrs(pcie->dev, PAGE_SIZE, msi->virt, msi->phys, + DMA_ATTR_NO_KERNEL_MAPPING); + + for (i = 0; i < INT_PCI_MSI_NR; i++) { + irq = irq_find_mapping(msi->domain, i); + if (irq > 0) + irq_domain_free_irqs(irq, 1); + } + + irq_set_chained_handler_and_data(msi->irq, NULL, NULL); + + if (IS_ENABLED(CONFIG_PCI_MSI)) + tegra_free_domains(msi); +} + +static int tegra_pcie_disable_msi(struct tegra_pcie *pcie) +{ + u32 value; + + /* mask the MSI interrupt */ + value = afi_readl(pcie, AFI_INTR_MASK); + value &= ~AFI_INTR_MASK_MSI_MASK; + afi_writel(pcie, value, AFI_INTR_MASK); + + return 0; +} + +static void tegra_pcie_disable_interrupts(struct tegra_pcie *pcie) +{ + u32 value; + + value = afi_readl(pcie, AFI_INTR_MASK); + value &= ~AFI_INTR_MASK_INT_MASK; + afi_writel(pcie, value, AFI_INTR_MASK); +} + +static int tegra_pcie_get_xbar_config(struct tegra_pcie *pcie, u32 lanes, + u32 *xbar) +{ + struct device *dev = pcie->dev; + struct device_node *np = dev->of_node; + + if (of_device_is_compatible(np, "nvidia,tegra186-pcie")) { + switch (lanes) { + case 0x010004: + dev_info(dev, "4x1, 1x1 configuration\n"); + *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_401; + return 0; + + case 0x010102: + dev_info(dev, "2x1, 1X1, 1x1 configuration\n"); + *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_211; + return 0; + + case 0x010101: + dev_info(dev, "1x1, 1x1, 1x1 configuration\n"); + *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_111; + return 0; + + default: + dev_info(dev, "wrong configuration updated in DT, " + "switching to default 2x1, 1x1, 1x1 " + "configuration\n"); + *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_211; + return 0; + } + } else if (of_device_is_compatible(np, "nvidia,tegra124-pcie") || + of_device_is_compatible(np, "nvidia,tegra210-pcie")) { + switch (lanes) { + case 0x0000104: + dev_info(dev, "4x1, 1x1 configuration\n"); + *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X4_X1; + return 0; + + case 0x0000102: + dev_info(dev, "2x1, 1x1 configuration\n"); + *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X2_X1; + return 0; + } + } else if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) { + switch (lanes) { + case 0x00000204: + dev_info(dev, "4x1, 2x1 configuration\n"); + *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_420; + return 0; + + case 0x00020202: + dev_info(dev, "2x3 configuration\n"); + *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_222; + return 0; + + case 0x00010104: + dev_info(dev, "4x1, 1x2 configuration\n"); + *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_411; + return 0; + } + } else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) { + switch (lanes) { + case 0x00000004: + dev_info(dev, "single-mode configuration\n"); + *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_SINGLE; + return 0; + + case 0x00000202: + dev_info(dev, "dual-mode configuration\n"); + *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_DUAL; + return 0; + } + } + + return -EINVAL; +} + +/* + * Check whether a given set of supplies is available in a device tree node. + * This is used to check whether the new or the legacy device tree bindings + * should be used. + */ +static bool of_regulator_bulk_available(struct device_node *np, + struct regulator_bulk_data *supplies, + unsigned int num_supplies) +{ + char property[32]; + unsigned int i; + + for (i = 0; i < num_supplies; i++) { + snprintf(property, 32, "%s-supply", supplies[i].supply); + + if (!of_property_present(np, property)) + return false; + } + + return true; +} + +/* + * Old versions of the device tree binding for this device used a set of power + * supplies that didn't match the hardware inputs. This happened to work for a + * number of cases but is not future proof. However to preserve backwards- + * compatibility with old device trees, this function will try to use the old + * set of supplies. + */ +static int tegra_pcie_get_legacy_regulators(struct tegra_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct device_node *np = dev->of_node; + + if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) + pcie->num_supplies = 3; + else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) + pcie->num_supplies = 2; + + if (pcie->num_supplies == 0) { + dev_err(dev, "device %pOF not supported in legacy mode\n", np); + return -ENODEV; + } + + pcie->supplies = devm_kcalloc(dev, pcie->num_supplies, + sizeof(*pcie->supplies), + GFP_KERNEL); + if (!pcie->supplies) + return -ENOMEM; + + pcie->supplies[0].supply = "pex-clk"; + pcie->supplies[1].supply = "vdd"; + + if (pcie->num_supplies > 2) + pcie->supplies[2].supply = "avdd"; + + return devm_regulator_bulk_get(dev, pcie->num_supplies, pcie->supplies); +} + +/* + * Obtains the list of regulators required for a particular generation of the + * IP block. + * + * This would've been nice to do simply by providing static tables for use + * with the regulator_bulk_*() API, but unfortunately Tegra30 is a bit quirky + * in that it has two pairs or AVDD_PEX and VDD_PEX supplies (PEXA and PEXB) + * and either seems to be optional depending on which ports are being used. + */ +static int tegra_pcie_get_regulators(struct tegra_pcie *pcie, u32 lane_mask) +{ + struct device *dev = pcie->dev; + struct device_node *np = dev->of_node; + unsigned int i = 0; + + if (of_device_is_compatible(np, "nvidia,tegra186-pcie")) { + pcie->num_supplies = 4; + + pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies, + sizeof(*pcie->supplies), + GFP_KERNEL); + if (!pcie->supplies) + return -ENOMEM; + + pcie->supplies[i++].supply = "dvdd-pex"; + pcie->supplies[i++].supply = "hvdd-pex-pll"; + pcie->supplies[i++].supply = "hvdd-pex"; + pcie->supplies[i++].supply = "vddio-pexctl-aud"; + } else if (of_device_is_compatible(np, "nvidia,tegra210-pcie")) { + pcie->num_supplies = 3; + + pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies, + sizeof(*pcie->supplies), + GFP_KERNEL); + if (!pcie->supplies) + return -ENOMEM; + + pcie->supplies[i++].supply = "hvddio-pex"; + pcie->supplies[i++].supply = "dvddio-pex"; + pcie->supplies[i++].supply = "vddio-pex-ctl"; + } else if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) { + pcie->num_supplies = 4; + + pcie->supplies = devm_kcalloc(dev, pcie->num_supplies, + sizeof(*pcie->supplies), + GFP_KERNEL); + if (!pcie->supplies) + return -ENOMEM; + + pcie->supplies[i++].supply = "avddio-pex"; + pcie->supplies[i++].supply = "dvddio-pex"; + pcie->supplies[i++].supply = "hvdd-pex"; + pcie->supplies[i++].supply = "vddio-pex-ctl"; + } else if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) { + bool need_pexa = false, need_pexb = false; + + /* VDD_PEXA and AVDD_PEXA supply lanes 0 to 3 */ + if (lane_mask & 0x0f) + need_pexa = true; + + /* VDD_PEXB and AVDD_PEXB supply lanes 4 to 5 */ + if (lane_mask & 0x30) + need_pexb = true; + + pcie->num_supplies = 4 + (need_pexa ? 2 : 0) + + (need_pexb ? 2 : 0); + + pcie->supplies = devm_kcalloc(dev, pcie->num_supplies, + sizeof(*pcie->supplies), + GFP_KERNEL); + if (!pcie->supplies) + return -ENOMEM; + + pcie->supplies[i++].supply = "avdd-pex-pll"; + pcie->supplies[i++].supply = "hvdd-pex"; + pcie->supplies[i++].supply = "vddio-pex-ctl"; + pcie->supplies[i++].supply = "avdd-plle"; + + if (need_pexa) { + pcie->supplies[i++].supply = "avdd-pexa"; + pcie->supplies[i++].supply = "vdd-pexa"; + } + + if (need_pexb) { + pcie->supplies[i++].supply = "avdd-pexb"; + pcie->supplies[i++].supply = "vdd-pexb"; + } + } else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) { + pcie->num_supplies = 5; + + pcie->supplies = devm_kcalloc(dev, pcie->num_supplies, + sizeof(*pcie->supplies), + GFP_KERNEL); + if (!pcie->supplies) + return -ENOMEM; + + pcie->supplies[0].supply = "avdd-pex"; + pcie->supplies[1].supply = "vdd-pex"; + pcie->supplies[2].supply = "avdd-pex-pll"; + pcie->supplies[3].supply = "avdd-plle"; + pcie->supplies[4].supply = "vddio-pex-clk"; + } + + if (of_regulator_bulk_available(dev->of_node, pcie->supplies, + pcie->num_supplies)) + return devm_regulator_bulk_get(dev, pcie->num_supplies, + pcie->supplies); + + /* + * If not all regulators are available for this new scheme, assume + * that the device tree complies with an older version of the device + * tree binding. + */ + dev_info(dev, "using legacy DT binding for power supplies\n"); + + devm_kfree(dev, pcie->supplies); + pcie->num_supplies = 0; + + return tegra_pcie_get_legacy_regulators(pcie); +} + +static int tegra_pcie_parse_dt(struct tegra_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct device_node *np = dev->of_node; + const struct tegra_pcie_soc *soc = pcie->soc; + u32 lanes = 0, mask = 0; + unsigned int lane = 0; + int err; + + /* parse root ports */ + for_each_child_of_node_scoped(np, port) { + struct tegra_pcie_port *rp; + unsigned int index; + u32 value; + char *label; + + err = of_pci_get_devfn(port); + if (err < 0) + return dev_err_probe(dev, err, "failed to parse address\n"); + + index = PCI_SLOT(err); + + if (index < 1 || index > soc->num_ports) + return dev_err_probe(dev, -EINVAL, + "invalid port number: %d\n", index); + + index--; + + err = of_property_read_u32(port, "nvidia,num-lanes", &value); + if (err < 0) + return dev_err_probe(dev, err, + "failed to parse # of lanes\n"); + + if (value > 16) + return dev_err_probe(dev, -EINVAL, + "invalid # of lanes: %u\n", value); + + lanes |= value << (index << 3); + + if (!of_device_is_available(port)) { + lane += value; + continue; + } + + mask |= ((1 << value) - 1) << lane; + lane += value; + + rp = devm_kzalloc(dev, sizeof(*rp), GFP_KERNEL); + if (!rp) + return -ENOMEM; + + err = of_address_to_resource(port, 0, &rp->regs); + if (err < 0) + return dev_err_probe(dev, err, "failed to parse address\n"); + + INIT_LIST_HEAD(&rp->list); + rp->index = index; + rp->lanes = value; + rp->pcie = pcie; + rp->np = port; + + rp->base = devm_pci_remap_cfg_resource(dev, &rp->regs); + if (IS_ERR(rp->base)) + return PTR_ERR(rp->base); + + label = devm_kasprintf(dev, GFP_KERNEL, "pex-reset-%u", index); + if (!label) + return -ENOMEM; + + /* + * Returns -ENOENT if reset-gpios property is not populated + * and in this case fall back to using AFI per port register + * to toggle PERST# SFIO line. + */ + rp->reset_gpio = devm_fwnode_gpiod_get(dev, + of_fwnode_handle(port), + "reset", + GPIOD_OUT_LOW, + label); + if (IS_ERR(rp->reset_gpio)) { + if (PTR_ERR(rp->reset_gpio) == -ENOENT) + rp->reset_gpio = NULL; + else + return dev_err_probe(dev, PTR_ERR(rp->reset_gpio), + "failed to get reset GPIO\n"); + } + + list_add_tail(&rp->list, &pcie->ports); + } + + err = tegra_pcie_get_xbar_config(pcie, lanes, &pcie->xbar_config); + if (err < 0) + return dev_err_probe(dev, err, + "invalid lane configuration\n"); + + err = tegra_pcie_get_regulators(pcie, mask); + if (err < 0) + return err; + + return 0; +} + +/* + * FIXME: If there are no PCIe cards attached, then calling this function + * can result in the increase of the bootup time as there are big timeout + * loops. + */ +#define TEGRA_PCIE_LINKUP_TIMEOUT 200 /* up to 1.2 seconds */ +static bool tegra_pcie_port_check_link(struct tegra_pcie_port *port) +{ + struct device *dev = port->pcie->dev; + unsigned int retries = 3; + unsigned long value; + + /* override presence detection */ + value = readl(port->base + RP_PRIV_MISC); + value &= ~RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT; + value |= RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT; + writel(value, port->base + RP_PRIV_MISC); + + do { + unsigned int timeout = TEGRA_PCIE_LINKUP_TIMEOUT; + + do { + value = readl(port->base + RP_VEND_XP); + + if (value & RP_VEND_XP_DL_UP) + break; + + usleep_range(1000, 2000); + } while (--timeout); + + if (!timeout) { + dev_dbg(dev, "link %u down, retrying\n", port->index); + goto retry; + } + + timeout = TEGRA_PCIE_LINKUP_TIMEOUT; + + do { + value = readl(port->base + RP_LINK_CONTROL_STATUS); + + if (value & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE) + return true; + + usleep_range(1000, 2000); + } while (--timeout); + +retry: + tegra_pcie_port_reset(port); + } while (--retries); + + return false; +} + +static void tegra_pcie_change_link_speed(struct tegra_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct tegra_pcie_port *port; + ktime_t deadline; + u32 value; + + list_for_each_entry(port, &pcie->ports, list) { + /* + * "Supported Link Speeds Vector" in "Link Capabilities 2" + * is not supported by Tegra. tegra_pcie_change_link_speed() + * is called only for Tegra chips which support Gen2. + * So there no harm if supported link speed is not verified. + */ + value = readl(port->base + RP_LINK_CONTROL_STATUS_2); + value &= ~PCI_EXP_LNKSTA_CLS; + value |= PCI_EXP_LNKSTA_CLS_5_0GB; + writel(value, port->base + RP_LINK_CONTROL_STATUS_2); + + /* + * Poll until link comes back from recovery to avoid race + * condition. + */ + deadline = ktime_add_us(ktime_get(), LINK_RETRAIN_TIMEOUT); + + while (ktime_before(ktime_get(), deadline)) { + value = readl(port->base + RP_LINK_CONTROL_STATUS); + if ((value & PCI_EXP_LNKSTA_LT) == 0) + break; + + usleep_range(2000, 3000); + } + + if (value & PCI_EXP_LNKSTA_LT) + dev_warn(dev, "PCIe port %u link is in recovery\n", + port->index); + + /* Retrain the link */ + value = readl(port->base + RP_LINK_CONTROL_STATUS); + value |= PCI_EXP_LNKCTL_RL; + writel(value, port->base + RP_LINK_CONTROL_STATUS); + + deadline = ktime_add_us(ktime_get(), LINK_RETRAIN_TIMEOUT); + + while (ktime_before(ktime_get(), deadline)) { + value = readl(port->base + RP_LINK_CONTROL_STATUS); + if ((value & PCI_EXP_LNKSTA_LT) == 0) + break; + + usleep_range(2000, 3000); + } + + if (value & PCI_EXP_LNKSTA_LT) + dev_err(dev, "failed to retrain link of port %u\n", + port->index); + } +} + +static void tegra_pcie_enable_ports(struct tegra_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct tegra_pcie_port *port, *tmp; + + list_for_each_entry_safe(port, tmp, &pcie->ports, list) { + dev_info(dev, "probing port %u, using %u lanes\n", + port->index, port->lanes); + + tegra_pcie_port_enable(port); + } + + /* Start LTSSM from Tegra side */ + reset_control_deassert(pcie->pcie_xrst); + + list_for_each_entry_safe(port, tmp, &pcie->ports, list) { + if (tegra_pcie_port_check_link(port)) + continue; + + dev_info(dev, "link %u down, ignoring\n", port->index); + + tegra_pcie_port_disable(port); + tegra_pcie_port_free(port); + } + + if (pcie->soc->has_gen2) + tegra_pcie_change_link_speed(pcie); +} + +static void tegra_pcie_disable_ports(struct tegra_pcie *pcie) +{ + struct tegra_pcie_port *port, *tmp; + + reset_control_assert(pcie->pcie_xrst); + + list_for_each_entry_safe(port, tmp, &pcie->ports, list) + tegra_pcie_port_disable(port); +} + +static const struct tegra_pcie_port_soc tegra20_pcie_ports[] = { + { .pme.turnoff_bit = 0, .pme.ack_bit = 5 }, + { .pme.turnoff_bit = 8, .pme.ack_bit = 10 }, +}; + +static const struct tegra_pcie_soc tegra20_pcie = { + .num_ports = 2, + .ports = tegra20_pcie_ports, + .msi_base_shift = 0, + .pads_pll_ctl = PADS_PLL_CTL_TEGRA20, + .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_DIV10, + .pads_refclk_cfg0 = 0xfa5cfa5c, + .has_pex_clkreq_en = false, + .has_pex_bias_ctrl = false, + .has_intr_prsnt_sense = false, + .has_cml_clk = false, + .has_gen2 = false, + .force_pca_enable = false, + .program_uphy = true, + .update_clamp_threshold = false, + .program_deskew_time = false, + .update_fc_timer = false, + .has_cache_bars = true, + .ectl.enable = false, +}; + +static const struct tegra_pcie_port_soc tegra30_pcie_ports[] = { + { .pme.turnoff_bit = 0, .pme.ack_bit = 5 }, + { .pme.turnoff_bit = 8, .pme.ack_bit = 10 }, + { .pme.turnoff_bit = 16, .pme.ack_bit = 18 }, +}; + +static const struct tegra_pcie_soc tegra30_pcie = { + .num_ports = 3, + .ports = tegra30_pcie_ports, + .msi_base_shift = 8, + .afi_pex2_ctrl = 0x128, + .pads_pll_ctl = PADS_PLL_CTL_TEGRA30, + .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN, + .pads_refclk_cfg0 = 0xfa5cfa5c, + .pads_refclk_cfg1 = 0xfa5cfa5c, + .has_pex_clkreq_en = true, + .has_pex_bias_ctrl = true, + .has_intr_prsnt_sense = true, + .has_cml_clk = true, + .has_gen2 = false, + .force_pca_enable = false, + .program_uphy = true, + .update_clamp_threshold = false, + .program_deskew_time = false, + .update_fc_timer = false, + .has_cache_bars = false, + .ectl.enable = false, +}; + +static const struct tegra_pcie_soc tegra124_pcie = { + .num_ports = 2, + .ports = tegra20_pcie_ports, + .msi_base_shift = 8, + .pads_pll_ctl = PADS_PLL_CTL_TEGRA30, + .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN, + .pads_refclk_cfg0 = 0x44ac44ac, + .has_pex_clkreq_en = true, + .has_pex_bias_ctrl = true, + .has_intr_prsnt_sense = true, + .has_cml_clk = true, + .has_gen2 = true, + .force_pca_enable = false, + .program_uphy = true, + .update_clamp_threshold = true, + .program_deskew_time = false, + .update_fc_timer = false, + .has_cache_bars = false, + .ectl.enable = false, +}; + +static const struct tegra_pcie_soc tegra210_pcie = { + .num_ports = 2, + .ports = tegra20_pcie_ports, + .msi_base_shift = 8, + .pads_pll_ctl = PADS_PLL_CTL_TEGRA30, + .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN, + .pads_refclk_cfg0 = 0x90b890b8, + /* FC threshold is bit[25:18] */ + .update_fc_threshold = 0x01800000, + .has_pex_clkreq_en = true, + .has_pex_bias_ctrl = true, + .has_intr_prsnt_sense = true, + .has_cml_clk = true, + .has_gen2 = true, + .force_pca_enable = true, + .program_uphy = true, + .update_clamp_threshold = true, + .program_deskew_time = true, + .update_fc_timer = true, + .has_cache_bars = false, + .ectl = { + .regs = { + .rp_ectl_2_r1 = 0x0000000f, + .rp_ectl_4_r1 = 0x00000067, + .rp_ectl_5_r1 = 0x55010000, + .rp_ectl_6_r1 = 0x00000001, + .rp_ectl_2_r2 = 0x0000008f, + .rp_ectl_4_r2 = 0x000000c7, + .rp_ectl_5_r2 = 0x55010000, + .rp_ectl_6_r2 = 0x00000001, + }, + .enable = true, + }, +}; + +static const struct tegra_pcie_port_soc tegra186_pcie_ports[] = { + { .pme.turnoff_bit = 0, .pme.ack_bit = 5 }, + { .pme.turnoff_bit = 8, .pme.ack_bit = 10 }, + { .pme.turnoff_bit = 12, .pme.ack_bit = 14 }, +}; + +static const struct tegra_pcie_soc tegra186_pcie = { + .num_ports = 3, + .ports = tegra186_pcie_ports, + .msi_base_shift = 8, + .afi_pex2_ctrl = 0x19c, + .pads_pll_ctl = PADS_PLL_CTL_TEGRA30, + .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN, + .pads_refclk_cfg0 = 0x80b880b8, + .pads_refclk_cfg1 = 0x000480b8, + .has_pex_clkreq_en = true, + .has_pex_bias_ctrl = true, + .has_intr_prsnt_sense = true, + .has_cml_clk = false, + .has_gen2 = true, + .force_pca_enable = false, + .program_uphy = false, + .update_clamp_threshold = false, + .program_deskew_time = false, + .update_fc_timer = false, + .has_cache_bars = false, + .ectl.enable = false, +}; + +static const struct of_device_id tegra_pcie_of_match[] = { + { .compatible = "nvidia,tegra186-pcie", .data = &tegra186_pcie }, + { .compatible = "nvidia,tegra210-pcie", .data = &tegra210_pcie }, + { .compatible = "nvidia,tegra124-pcie", .data = &tegra124_pcie }, + { .compatible = "nvidia,tegra30-pcie", .data = &tegra30_pcie }, + { .compatible = "nvidia,tegra20-pcie", .data = &tegra20_pcie }, + { }, +}; +MODULE_DEVICE_TABLE(of, tegra_pcie_of_match); + +static void *tegra_pcie_ports_seq_start(struct seq_file *s, loff_t *pos) +{ + struct tegra_pcie *pcie = s->private; + + if (list_empty(&pcie->ports)) + return NULL; + + seq_puts(s, "Index Status\n"); + + return seq_list_start(&pcie->ports, *pos); +} + +static void *tegra_pcie_ports_seq_next(struct seq_file *s, void *v, loff_t *pos) +{ + struct tegra_pcie *pcie = s->private; + + return seq_list_next(v, &pcie->ports, pos); +} + +static void tegra_pcie_ports_seq_stop(struct seq_file *s, void *v) +{ +} + +static int tegra_pcie_ports_seq_show(struct seq_file *s, void *v) +{ + bool up = false, active = false; + struct tegra_pcie_port *port; + unsigned int value; + + port = list_entry(v, struct tegra_pcie_port, list); + + value = readl(port->base + RP_VEND_XP); + + if (value & RP_VEND_XP_DL_UP) + up = true; + + value = readl(port->base + RP_LINK_CONTROL_STATUS); + + if (value & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE) + active = true; + + seq_printf(s, "%2u ", port->index); + + if (up) + seq_puts(s, "up"); + + if (active) { + if (up) + seq_puts(s, ", "); + + seq_puts(s, "active"); + } + + seq_puts(s, "\n"); + return 0; +} + +static const struct seq_operations tegra_pcie_ports_sops = { + .start = tegra_pcie_ports_seq_start, + .next = tegra_pcie_ports_seq_next, + .stop = tegra_pcie_ports_seq_stop, + .show = tegra_pcie_ports_seq_show, +}; + +DEFINE_SEQ_ATTRIBUTE(tegra_pcie_ports); + +static void tegra_pcie_debugfs_init(struct tegra_pcie *pcie) +{ + pcie->debugfs = debugfs_create_dir("pcie", NULL); + + debugfs_create_file("ports", S_IFREG | S_IRUGO, pcie->debugfs, pcie, + &tegra_pcie_ports_fops); +} + +static int tegra_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct pci_host_bridge *host; + struct tegra_pcie *pcie; + int err; + + host = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!host) + return -ENOMEM; + + pcie = pci_host_bridge_priv(host); + host->sysdata = pcie; + platform_set_drvdata(pdev, pcie); + + pcie->soc = of_device_get_match_data(dev); + INIT_LIST_HEAD(&pcie->ports); + pcie->dev = dev; + + err = tegra_pcie_parse_dt(pcie); + if (err < 0) + return err; + + err = tegra_pcie_get_resources(pcie); + if (err < 0) { + dev_err(dev, "failed to request resources: %d\n", err); + return err; + } + + err = tegra_pcie_msi_setup(pcie); + if (err < 0) { + dev_err(dev, "failed to enable MSI support: %d\n", err); + goto put_resources; + } + + pm_runtime_enable(pcie->dev); + err = pm_runtime_get_sync(pcie->dev); + if (err < 0) { + dev_err(dev, "fail to enable pcie controller: %d\n", err); + goto pm_runtime_put; + } + + host->ops = &tegra_pcie_ops; + host->map_irq = tegra_pcie_map_irq; + + err = pci_host_probe(host); + if (err < 0) { + dev_err(dev, "failed to register host: %d\n", err); + goto pm_runtime_put; + } + + if (IS_ENABLED(CONFIG_DEBUG_FS)) + tegra_pcie_debugfs_init(pcie); + + return 0; + +pm_runtime_put: + pm_runtime_put_sync(pcie->dev); + pm_runtime_disable(pcie->dev); + tegra_pcie_msi_teardown(pcie); +put_resources: + tegra_pcie_put_resources(pcie); + return err; +} + +static int tegra_pcie_pm_suspend(struct device *dev) +{ + struct tegra_pcie *pcie = dev_get_drvdata(dev); + struct tegra_pcie_port *port; + int err; + + list_for_each_entry(port, &pcie->ports, list) + tegra_pcie_pme_turnoff(port); + + tegra_pcie_disable_ports(pcie); + + /* + * AFI_INTR is unmasked in tegra_pcie_enable_controller(), mask it to + * avoid unwanted interrupts raised by AFI after pex_rst is asserted. + */ + tegra_pcie_disable_interrupts(pcie); + + if (pcie->soc->program_uphy) { + err = tegra_pcie_phy_power_off(pcie); + if (err < 0) + dev_err(dev, "failed to power off PHY(s): %d\n", err); + } + + reset_control_assert(pcie->pex_rst); + clk_disable_unprepare(pcie->pex_clk); + + if (IS_ENABLED(CONFIG_PCI_MSI)) + tegra_pcie_disable_msi(pcie); + + pinctrl_pm_select_idle_state(dev); + tegra_pcie_power_off(pcie); + + return 0; +} + +static int tegra_pcie_pm_resume(struct device *dev) +{ + struct tegra_pcie *pcie = dev_get_drvdata(dev); + int err; + + err = tegra_pcie_power_on(pcie); + if (err) { + dev_err(dev, "tegra pcie power on fail: %d\n", err); + return err; + } + + err = pinctrl_pm_select_default_state(dev); + if (err < 0) { + dev_err(dev, "failed to disable PCIe IO DPD: %d\n", err); + goto poweroff; + } + + tegra_pcie_enable_controller(pcie); + tegra_pcie_setup_translations(pcie); + + if (IS_ENABLED(CONFIG_PCI_MSI)) + tegra_pcie_enable_msi(pcie); + + err = clk_prepare_enable(pcie->pex_clk); + if (err) { + dev_err(dev, "failed to enable PEX clock: %d\n", err); + goto pex_dpd_enable; + } + + reset_control_deassert(pcie->pex_rst); + + if (pcie->soc->program_uphy) { + err = tegra_pcie_phy_power_on(pcie); + if (err < 0) { + dev_err(dev, "failed to power on PHY(s): %d\n", err); + goto disable_pex_clk; + } + } + + tegra_pcie_apply_pad_settings(pcie); + tegra_pcie_enable_ports(pcie); + + return 0; + +disable_pex_clk: + reset_control_assert(pcie->pex_rst); + clk_disable_unprepare(pcie->pex_clk); +pex_dpd_enable: + pinctrl_pm_select_idle_state(dev); +poweroff: + tegra_pcie_power_off(pcie); + + return err; +} + +static const struct dev_pm_ops tegra_pcie_pm_ops = { + RUNTIME_PM_OPS(tegra_pcie_pm_suspend, tegra_pcie_pm_resume, NULL) + NOIRQ_SYSTEM_SLEEP_PM_OPS(tegra_pcie_pm_suspend, tegra_pcie_pm_resume) +}; + +static struct platform_driver tegra_pcie_driver = { + .driver = { + .name = "tegra-pcie", + .of_match_table = tegra_pcie_of_match, + .suppress_bind_attrs = true, + .pm = &tegra_pcie_pm_ops, + }, + .probe = tegra_pcie_probe, +}; +builtin_platform_driver(tegra_pcie_driver); +MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>"); +MODULE_DESCRIPTION("NVIDIA PCI host controller driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pci/controller/pci-thunder-ecam.c b/drivers/pci/controller/pci-thunder-ecam.c new file mode 100644 index 000000000..b5b4a958e --- /dev/null +++ b/drivers/pci/controller/pci-thunder-ecam.c @@ -0,0 +1,365 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015, 2016 Cavium, Inc. + */ + +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/ioport.h> +#include <linux/of_pci.h> +#include <linux/of.h> +#include <linux/pci-ecam.h> +#include <linux/platform_device.h> + +#include "pci-host-common.h" + +#if defined(CONFIG_PCI_HOST_THUNDER_ECAM) || (defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS)) + +static void set_val(u32 v, int where, int size, u32 *val) +{ + int shift = (where & 3) * 8; + + pr_debug("set_val %04x: %08x\n", (unsigned int)(where & ~3), v); + v >>= shift; + if (size == 1) + v &= 0xff; + else if (size == 2) + v &= 0xffff; + *val = v; +} + +static int handle_ea_bar(u32 e0, int bar, struct pci_bus *bus, + unsigned int devfn, int where, int size, u32 *val) +{ + void __iomem *addr; + u32 v; + + /* Entries are 16-byte aligned; bits[2,3] select word in entry */ + int where_a = where & 0xc; + + if (where_a == 0) { + set_val(e0, where, size, val); + return PCIBIOS_SUCCESSFUL; + } + if (where_a == 0x4) { + addr = bus->ops->map_bus(bus, devfn, bar); /* BAR 0 */ + if (!addr) + return PCIBIOS_DEVICE_NOT_FOUND; + + v = readl(addr); + v &= ~0xf; + v |= 2; /* EA entry-1. Base-L */ + set_val(v, where, size, val); + return PCIBIOS_SUCCESSFUL; + } + if (where_a == 0x8) { + u32 barl_orig; + u32 barl_rb; + + addr = bus->ops->map_bus(bus, devfn, bar); /* BAR 0 */ + if (!addr) + return PCIBIOS_DEVICE_NOT_FOUND; + + barl_orig = readl(addr + 0); + writel(0xffffffff, addr + 0); + barl_rb = readl(addr + 0); + writel(barl_orig, addr + 0); + /* zeros in unsettable bits */ + v = ~barl_rb & ~3; + v |= 0xc; /* EA entry-2. Offset-L */ + set_val(v, where, size, val); + return PCIBIOS_SUCCESSFUL; + } + if (where_a == 0xc) { + addr = bus->ops->map_bus(bus, devfn, bar + 4); /* BAR 1 */ + if (!addr) + return PCIBIOS_DEVICE_NOT_FOUND; + + v = readl(addr); /* EA entry-3. Base-H */ + set_val(v, where, size, val); + return PCIBIOS_SUCCESSFUL; + } + return PCIBIOS_DEVICE_NOT_FOUND; +} + +static int thunder_ecam_p2_config_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct pci_config_window *cfg = bus->sysdata; + int where_a = where & ~3; + void __iomem *addr; + u32 node_bits; + u32 v; + + /* EA Base[63:32] may be missing some bits ... */ + switch (where_a) { + case 0xa8: + case 0xbc: + case 0xd0: + case 0xe4: + break; + default: + return pci_generic_config_read(bus, devfn, where, size, val); + } + + addr = bus->ops->map_bus(bus, devfn, where_a); + if (!addr) + return PCIBIOS_DEVICE_NOT_FOUND; + + v = readl(addr); + + /* + * Bit 44 of the 64-bit Base must match the same bit in + * the config space access window. Since we are working with + * the high-order 32 bits, shift everything down by 32 bits. + */ + node_bits = upper_32_bits(cfg->res.start) & (1 << 12); + + v |= node_bits; + set_val(v, where, size, val); + + return PCIBIOS_SUCCESSFUL; +} + +static int thunder_ecam_config_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + u32 v; + u32 vendor_device; + u32 class_rev; + void __iomem *addr; + int cfg_type; + int where_a = where & ~3; + + addr = bus->ops->map_bus(bus, devfn, 0xc); + if (!addr) + return PCIBIOS_DEVICE_NOT_FOUND; + + v = readl(addr); + + /* Check for non type-00 header */ + cfg_type = (v >> 16) & 0x7f; + + addr = bus->ops->map_bus(bus, devfn, 8); + if (!addr) + return PCIBIOS_DEVICE_NOT_FOUND; + + class_rev = readl(addr); + if (class_rev == 0xffffffff) + goto no_emulation; + + if ((class_rev & 0xff) >= 8) { + /* Pass-2 handling */ + if (cfg_type) + goto no_emulation; + return thunder_ecam_p2_config_read(bus, devfn, where, + size, val); + } + + /* + * All BARs have fixed addresses specified by the EA + * capability; they must return zero on read. + */ + if (cfg_type == 0 && + ((where >= 0x10 && where < 0x2c) || + (where >= 0x1a4 && where < 0x1bc))) { + /* BAR or SR-IOV BAR */ + *val = 0; + return PCIBIOS_SUCCESSFUL; + } + + addr = bus->ops->map_bus(bus, devfn, 0); + if (!addr) + return PCIBIOS_DEVICE_NOT_FOUND; + + vendor_device = readl(addr); + if (vendor_device == 0xffffffff) + goto no_emulation; + + pr_debug("%04x:%04x - Fix pass#: %08x, where: %03x, devfn: %03x\n", + vendor_device & 0xffff, vendor_device >> 16, class_rev, + (unsigned int)where, devfn); + + /* Check for non type-00 header */ + if (cfg_type == 0) { + bool has_msix; + bool is_nic = (vendor_device == 0xa01e177d); + bool is_tns = (vendor_device == 0xa01f177d); + + addr = bus->ops->map_bus(bus, devfn, 0x70); + if (!addr) + return PCIBIOS_DEVICE_NOT_FOUND; + + /* E_CAP */ + v = readl(addr); + has_msix = (v & 0xff00) != 0; + + if (!has_msix && where_a == 0x70) { + v |= 0xbc00; /* next capability is EA at 0xbc */ + set_val(v, where, size, val); + return PCIBIOS_SUCCESSFUL; + } + if (where_a == 0xb0) { + addr = bus->ops->map_bus(bus, devfn, where_a); + if (!addr) + return PCIBIOS_DEVICE_NOT_FOUND; + + v = readl(addr); + if (v & 0xff00) + pr_err("Bad MSI-X cap header: %08x\n", v); + v |= 0xbc00; /* next capability is EA at 0xbc */ + set_val(v, where, size, val); + return PCIBIOS_SUCCESSFUL; + } + if (where_a == 0xbc) { + if (is_nic) + v = 0x40014; /* EA last in chain, 4 entries */ + else if (is_tns) + v = 0x30014; /* EA last in chain, 3 entries */ + else if (has_msix) + v = 0x20014; /* EA last in chain, 2 entries */ + else + v = 0x10014; /* EA last in chain, 1 entry */ + set_val(v, where, size, val); + return PCIBIOS_SUCCESSFUL; + } + if (where_a >= 0xc0 && where_a < 0xd0) + /* EA entry-0. PP=0, BAR0 Size:3 */ + return handle_ea_bar(0x80ff0003, + 0x10, bus, devfn, where, + size, val); + if (where_a >= 0xd0 && where_a < 0xe0 && has_msix) + /* EA entry-1. PP=0, BAR4 Size:3 */ + return handle_ea_bar(0x80ff0043, + 0x20, bus, devfn, where, + size, val); + if (where_a >= 0xe0 && where_a < 0xf0 && is_tns) + /* EA entry-2. PP=0, BAR2, Size:3 */ + return handle_ea_bar(0x80ff0023, + 0x18, bus, devfn, where, + size, val); + if (where_a >= 0xe0 && where_a < 0xf0 && is_nic) + /* EA entry-2. PP=4, VF_BAR0 (9), Size:3 */ + return handle_ea_bar(0x80ff0493, + 0x1a4, bus, devfn, where, + size, val); + if (where_a >= 0xf0 && where_a < 0x100 && is_nic) + /* EA entry-3. PP=4, VF_BAR4 (d), Size:3 */ + return handle_ea_bar(0x80ff04d3, + 0x1b4, bus, devfn, where, + size, val); + } else if (cfg_type == 1) { + bool is_rsl_bridge = devfn == 0x08; + bool is_rad_bridge = devfn == 0xa0; + bool is_zip_bridge = devfn == 0xa8; + bool is_dfa_bridge = devfn == 0xb0; + bool is_nic_bridge = devfn == 0x10; + + if (where_a == 0x70) { + addr = bus->ops->map_bus(bus, devfn, where_a); + if (!addr) + return PCIBIOS_DEVICE_NOT_FOUND; + + v = readl(addr); + if (v & 0xff00) + pr_err("Bad PCIe cap header: %08x\n", v); + v |= 0xbc00; /* next capability is EA at 0xbc */ + set_val(v, where, size, val); + return PCIBIOS_SUCCESSFUL; + } + if (where_a == 0xbc) { + if (is_nic_bridge) + v = 0x10014; /* EA last in chain, 1 entry */ + else + v = 0x00014; /* EA last in chain, no entries */ + set_val(v, where, size, val); + return PCIBIOS_SUCCESSFUL; + } + if (where_a == 0xc0) { + if (is_rsl_bridge || is_nic_bridge) + v = 0x0101; /* subordinate:secondary = 1:1 */ + else if (is_rad_bridge) + v = 0x0202; /* subordinate:secondary = 2:2 */ + else if (is_zip_bridge) + v = 0x0303; /* subordinate:secondary = 3:3 */ + else if (is_dfa_bridge) + v = 0x0404; /* subordinate:secondary = 4:4 */ + set_val(v, where, size, val); + return PCIBIOS_SUCCESSFUL; + } + if (where_a == 0xc4 && is_nic_bridge) { + /* Enabled, not-Write, SP=ff, PP=05, BEI=6, ES=4 */ + v = 0x80ff0564; + set_val(v, where, size, val); + return PCIBIOS_SUCCESSFUL; + } + if (where_a == 0xc8 && is_nic_bridge) { + v = 0x00000002; /* Base-L 64-bit */ + set_val(v, where, size, val); + return PCIBIOS_SUCCESSFUL; + } + if (where_a == 0xcc && is_nic_bridge) { + v = 0xfffffffe; /* MaxOffset-L 64-bit */ + set_val(v, where, size, val); + return PCIBIOS_SUCCESSFUL; + } + if (where_a == 0xd0 && is_nic_bridge) { + v = 0x00008430; /* NIC Base-H */ + set_val(v, where, size, val); + return PCIBIOS_SUCCESSFUL; + } + if (where_a == 0xd4 && is_nic_bridge) { + v = 0x0000000f; /* MaxOffset-H */ + set_val(v, where, size, val); + return PCIBIOS_SUCCESSFUL; + } + } +no_emulation: + return pci_generic_config_read(bus, devfn, where, size, val); +} + +static int thunder_ecam_config_write(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + /* + * All BARs have fixed addresses; ignore BAR writes so they + * don't get corrupted. + */ + if ((where >= 0x10 && where < 0x2c) || + (where >= 0x1a4 && where < 0x1bc)) + /* BAR or SR-IOV BAR */ + return PCIBIOS_SUCCESSFUL; + + return pci_generic_config_write(bus, devfn, where, size, val); +} + +const struct pci_ecam_ops pci_thunder_ecam_ops = { + .pci_ops = { + .map_bus = pci_ecam_map_bus, + .read = thunder_ecam_config_read, + .write = thunder_ecam_config_write, + } +}; + +#ifdef CONFIG_PCI_HOST_THUNDER_ECAM + +static const struct of_device_id thunder_ecam_of_match[] = { + { + .compatible = "cavium,pci-host-thunder-ecam", + .data = &pci_thunder_ecam_ops, + }, + { }, +}; + +static struct platform_driver thunder_ecam_driver = { + .driver = { + .name = KBUILD_MODNAME, + .of_match_table = thunder_ecam_of_match, + .suppress_bind_attrs = true, + }, + .probe = pci_host_common_probe, +}; +builtin_platform_driver(thunder_ecam_driver); + +#endif +#endif diff --git a/drivers/pci/controller/pci-thunder-pem.c b/drivers/pci/controller/pci-thunder-pem.c new file mode 100644 index 000000000..5fa037fb6 --- /dev/null +++ b/drivers/pci/controller/pci-thunder-pem.c @@ -0,0 +1,481 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015 - 2016 Cavium, Inc. + */ + +#include <linux/bitfield.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/pci.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/pci-acpi.h> +#include <linux/pci-ecam.h> +#include <linux/platform_device.h> +#include <linux/io-64-nonatomic-lo-hi.h> +#include "../pci.h" +#include "pci-host-common.h" + +#if defined(CONFIG_PCI_HOST_THUNDER_PEM) || (defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS)) + +#define PEM_CFG_WR 0x28 +#define PEM_CFG_RD 0x30 + +/* + * Enhanced Configuration Access Mechanism (ECAM) + * + * N.B. This is a non-standard platform-specific ECAM bus shift value. For + * standard values defined in the PCI Express Base Specification see + * include/linux/pci-ecam.h. + */ +#define THUNDER_PCIE_ECAM_BUS_SHIFT 24 + +struct thunder_pem_pci { + u32 ea_entry[3]; + void __iomem *pem_reg_base; +}; + +static int thunder_pem_bridge_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + u64 read_val, tmp_val; + struct pci_config_window *cfg = bus->sysdata; + struct thunder_pem_pci *pem_pci = (struct thunder_pem_pci *)cfg->priv; + + if (devfn != 0 || where >= 2048) + return PCIBIOS_DEVICE_NOT_FOUND; + + /* + * 32-bit accesses only. Write the address to the low order + * bits of PEM_CFG_RD, then trigger the read by reading back. + * The config data lands in the upper 32-bits of PEM_CFG_RD. + */ + read_val = where & ~3ull; + writeq(read_val, pem_pci->pem_reg_base + PEM_CFG_RD); + read_val = readq(pem_pci->pem_reg_base + PEM_CFG_RD); + read_val >>= 32; + + /* + * The config space contains some garbage, fix it up. Also + * synthesize an EA capability for the BAR used by MSI-X. + */ + switch (where & ~3) { + case 0x40: + read_val &= 0xffff00ff; + read_val |= 0x00007000; /* Skip MSI CAP */ + break; + case 0x70: /* Express Cap */ + /* + * Change PME interrupt to vector 2 on T88 where it + * reads as 0, else leave it alone. + */ + if (!(read_val & (0x1f << 25))) + read_val |= (2u << 25); + break; + case 0xb0: /* MSI-X Cap */ + /* TableSize=2 or 4, Next Cap is EA */ + read_val &= 0xc00000ff; + /* + * If Express Cap(0x70) raw PME vector reads as 0 we are on + * T88 and TableSize is reported as 4, else TableSize + * is 2. + */ + writeq(0x70, pem_pci->pem_reg_base + PEM_CFG_RD); + tmp_val = readq(pem_pci->pem_reg_base + PEM_CFG_RD); + tmp_val >>= 32; + if (!(tmp_val & (0x1f << 25))) + read_val |= 0x0003bc00; + else + read_val |= 0x0001bc00; + break; + case 0xb4: + /* Table offset=0, BIR=0 */ + read_val = 0x00000000; + break; + case 0xb8: + /* BPA offset=0xf0000, BIR=0 */ + read_val = 0x000f0000; + break; + case 0xbc: + /* EA, 1 entry, no next Cap */ + read_val = 0x00010014; + break; + case 0xc0: + /* DW2 for type-1 */ + read_val = 0x00000000; + break; + case 0xc4: + /* Entry BEI=0, PP=0x00, SP=0xff, ES=3 */ + read_val = 0x80ff0003; + break; + case 0xc8: + read_val = pem_pci->ea_entry[0]; + break; + case 0xcc: + read_val = pem_pci->ea_entry[1]; + break; + case 0xd0: + read_val = pem_pci->ea_entry[2]; + break; + default: + break; + } + read_val >>= (8 * (where & 3)); + switch (size) { + case 1: + read_val &= 0xff; + break; + case 2: + read_val &= 0xffff; + break; + default: + break; + } + *val = read_val; + return PCIBIOS_SUCCESSFUL; +} + +static int thunder_pem_config_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct pci_config_window *cfg = bus->sysdata; + + if (bus->number < cfg->busr.start || + bus->number > cfg->busr.end) + return PCIBIOS_DEVICE_NOT_FOUND; + + /* + * The first device on the bus is the PEM PCIe bridge. + * Special case its config access. + */ + if (bus->number == cfg->busr.start) + return thunder_pem_bridge_read(bus, devfn, where, size, val); + + return pci_generic_config_read(bus, devfn, where, size, val); +} + +/* + * Some of the w1c_bits below also include read-only or non-writable + * reserved bits, this makes the code simpler and is OK as the bits + * are not affected by writing zeros to them. + */ +static u32 thunder_pem_bridge_w1c_bits(u64 where_aligned) +{ + u32 w1c_bits = 0; + + switch (where_aligned) { + case 0x04: /* Command/Status */ + case 0x1c: /* Base and I/O Limit/Secondary Status */ + w1c_bits = 0xff000000; + break; + case 0x44: /* Power Management Control and Status */ + w1c_bits = 0xfffffe00; + break; + case 0x78: /* Device Control/Device Status */ + case 0x80: /* Link Control/Link Status */ + case 0x88: /* Slot Control/Slot Status */ + case 0x90: /* Root Status */ + case 0xa0: /* Link Control 2 Registers/Link Status 2 */ + w1c_bits = 0xffff0000; + break; + case 0x104: /* Uncorrectable Error Status */ + case 0x110: /* Correctable Error Status */ + case 0x130: /* Error Status */ + case 0x160: /* Link Control 4 */ + w1c_bits = 0xffffffff; + break; + default: + break; + } + return w1c_bits; +} + +/* Some bits must be written to one so they appear to be read-only. */ +static u32 thunder_pem_bridge_w1_bits(u64 where_aligned) +{ + u32 w1_bits; + + switch (where_aligned) { + case 0x1c: /* I/O Base / I/O Limit, Secondary Status */ + /* Force 32-bit I/O addressing. */ + w1_bits = 0x0101; + break; + case 0x24: /* Prefetchable Memory Base / Prefetchable Memory Limit */ + /* Force 64-bit addressing */ + w1_bits = 0x00010001; + break; + default: + w1_bits = 0; + break; + } + return w1_bits; +} + +static int thunder_pem_bridge_write(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + struct pci_config_window *cfg = bus->sysdata; + struct thunder_pem_pci *pem_pci = (struct thunder_pem_pci *)cfg->priv; + u64 write_val, read_val; + u64 where_aligned = where & ~3ull; + u32 mask = 0; + + + if (devfn != 0 || where >= 2048) + return PCIBIOS_DEVICE_NOT_FOUND; + + /* + * 32-bit accesses only. If the write is for a size smaller + * than 32-bits, we must first read the 32-bit value and merge + * in the desired bits and then write the whole 32-bits back + * out. + */ + switch (size) { + case 1: + writeq(where_aligned, pem_pci->pem_reg_base + PEM_CFG_RD); + read_val = readq(pem_pci->pem_reg_base + PEM_CFG_RD); + read_val >>= 32; + mask = ~(0xff << (8 * (where & 3))); + read_val &= mask; + val = (val & 0xff) << (8 * (where & 3)); + val |= (u32)read_val; + break; + case 2: + writeq(where_aligned, pem_pci->pem_reg_base + PEM_CFG_RD); + read_val = readq(pem_pci->pem_reg_base + PEM_CFG_RD); + read_val >>= 32; + mask = ~(0xffff << (8 * (where & 3))); + read_val &= mask; + val = (val & 0xffff) << (8 * (where & 3)); + val |= (u32)read_val; + break; + default: + break; + } + + /* + * By expanding the write width to 32 bits, we may + * inadvertently hit some W1C bits that were not intended to + * be written. Calculate the mask that must be applied to the + * data to be written to avoid these cases. + */ + if (mask) { + u32 w1c_bits = thunder_pem_bridge_w1c_bits(where); + + if (w1c_bits) { + mask &= w1c_bits; + val &= ~mask; + } + } + + /* + * Some bits must be read-only with value of one. Since the + * access method allows these to be cleared if a zero is + * written, force them to one before writing. + */ + val |= thunder_pem_bridge_w1_bits(where_aligned); + + /* + * Low order bits are the config address, the high order 32 + * bits are the data to be written. + */ + write_val = (((u64)val) << 32) | where_aligned; + writeq(write_val, pem_pci->pem_reg_base + PEM_CFG_WR); + return PCIBIOS_SUCCESSFUL; +} + +static int thunder_pem_config_write(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + struct pci_config_window *cfg = bus->sysdata; + + if (bus->number < cfg->busr.start || + bus->number > cfg->busr.end) + return PCIBIOS_DEVICE_NOT_FOUND; + /* + * The first device on the bus is the PEM PCIe bridge. + * Special case its config access. + */ + if (bus->number == cfg->busr.start) + return thunder_pem_bridge_write(bus, devfn, where, size, val); + + + return pci_generic_config_write(bus, devfn, where, size, val); +} + +static int thunder_pem_init(struct device *dev, struct pci_config_window *cfg, + struct resource *res_pem) +{ + struct thunder_pem_pci *pem_pci; + resource_size_t bar4_start; + + pem_pci = devm_kzalloc(dev, sizeof(*pem_pci), GFP_KERNEL); + if (!pem_pci) + return -ENOMEM; + + pem_pci->pem_reg_base = devm_ioremap(dev, res_pem->start, 0x10000); + if (!pem_pci->pem_reg_base) + return -ENOMEM; + + /* + * The MSI-X BAR for the PEM and AER interrupts is located at + * a fixed offset from the PEM register base. Generate a + * fragment of the synthesized Enhanced Allocation capability + * structure here for the BAR. + */ + bar4_start = res_pem->start + 0xf00000; + pem_pci->ea_entry[0] = lower_32_bits(bar4_start) | 2; + pem_pci->ea_entry[1] = lower_32_bits(res_pem->end - bar4_start) & ~3u; + pem_pci->ea_entry[2] = upper_32_bits(bar4_start); + + cfg->priv = pem_pci; + return 0; +} + +#if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS) + +#define PEM_RES_BASE 0x87e0c0000000ULL +#define PEM_NODE_MASK GENMASK_ULL(45, 44) +#define PEM_INDX_MASK GENMASK_ULL(26, 24) +#define PEM_MIN_DOM_IN_NODE 4 +#define PEM_MAX_DOM_IN_NODE 10 + +static void thunder_pem_reserve_range(struct device *dev, int seg, + struct resource *r) +{ + resource_size_t start = r->start, end = r->end; + struct resource *res; + const char *regionid; + + regionid = kasprintf(GFP_KERNEL, "PEM RC:%d", seg); + if (!regionid) + return; + + res = request_mem_region(start, end - start + 1, regionid); + if (res) + res->flags &= ~IORESOURCE_BUSY; + else + kfree(regionid); + + dev_info(dev, "%pR %s reserved\n", r, + res ? "has been" : "could not be"); +} + +static void thunder_pem_legacy_fw(struct acpi_pci_root *root, + struct resource *res_pem) +{ + int node = acpi_get_node(root->device->handle); + int index; + + if (node == NUMA_NO_NODE) + node = 0; + + index = root->segment - PEM_MIN_DOM_IN_NODE; + index -= node * PEM_MAX_DOM_IN_NODE; + res_pem->start = PEM_RES_BASE | FIELD_PREP(PEM_NODE_MASK, node) | + FIELD_PREP(PEM_INDX_MASK, index); + res_pem->flags = IORESOURCE_MEM; +} + +static int thunder_pem_acpi_init(struct pci_config_window *cfg) +{ + struct device *dev = cfg->parent; + struct acpi_device *adev = to_acpi_device(dev); + struct acpi_pci_root *root = acpi_driver_data(adev); + struct resource *res_pem; + int ret; + + res_pem = devm_kzalloc(&adev->dev, sizeof(*res_pem), GFP_KERNEL); + if (!res_pem) + return -ENOMEM; + + ret = acpi_get_rc_resources(dev, "CAVA02B", root->segment, res_pem); + + /* + * If we fail to gather resources it means that we run with old + * FW where we need to calculate PEM-specific resources manually. + */ + if (ret) { + thunder_pem_legacy_fw(root, res_pem); + /* + * Reserve 64K size PEM specific resources. The full 16M range + * size is required for thunder_pem_init() call. + */ + resource_set_size(res_pem, SZ_64K); + thunder_pem_reserve_range(dev, root->segment, res_pem); + resource_set_size(res_pem, SZ_16M); + + /* Reserve PCI configuration space as well. */ + thunder_pem_reserve_range(dev, root->segment, &cfg->res); + } + + return thunder_pem_init(dev, cfg, res_pem); +} + +const struct pci_ecam_ops thunder_pem_ecam_ops = { + .bus_shift = THUNDER_PCIE_ECAM_BUS_SHIFT, + .init = thunder_pem_acpi_init, + .pci_ops = { + .map_bus = pci_ecam_map_bus, + .read = thunder_pem_config_read, + .write = thunder_pem_config_write, + } +}; + +#endif + +#ifdef CONFIG_PCI_HOST_THUNDER_PEM + +static int thunder_pem_platform_init(struct pci_config_window *cfg) +{ + struct device *dev = cfg->parent; + struct platform_device *pdev = to_platform_device(dev); + struct resource *res_pem; + + if (!dev->of_node) + return -EINVAL; + + /* + * The second register range is the PEM bridge to the PCIe + * bus. It has a different config access method than those + * devices behind the bridge. + */ + res_pem = platform_get_resource(pdev, IORESOURCE_MEM, 1); + if (!res_pem) { + dev_err(dev, "missing \"reg[1]\"property\n"); + return -EINVAL; + } + + return thunder_pem_init(dev, cfg, res_pem); +} + +static const struct pci_ecam_ops pci_thunder_pem_ops = { + .bus_shift = THUNDER_PCIE_ECAM_BUS_SHIFT, + .init = thunder_pem_platform_init, + .pci_ops = { + .map_bus = pci_ecam_map_bus, + .read = thunder_pem_config_read, + .write = thunder_pem_config_write, + } +}; + +static const struct of_device_id thunder_pem_of_match[] = { + { + .compatible = "cavium,pci-host-thunder-pem", + .data = &pci_thunder_pem_ops, + }, + { }, +}; + +static struct platform_driver thunder_pem_driver = { + .driver = { + .name = KBUILD_MODNAME, + .of_match_table = thunder_pem_of_match, + .suppress_bind_attrs = true, + }, + .probe = pci_host_common_probe, +}; +builtin_platform_driver(thunder_pem_driver); + +#endif +#endif diff --git a/drivers/pci/controller/pci-v3-semi.c b/drivers/pci/controller/pci-v3-semi.c new file mode 100644 index 000000000..460a82532 --- /dev/null +++ b/drivers/pci/controller/pci-v3-semi.c @@ -0,0 +1,907 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Support for V3 Semiconductor PCI Local Bus to PCI Bridge + * Copyright (C) 2017 Linus Walleij <linus.walleij@linaro.org> + * + * Based on the code from arch/arm/mach-integrator/pci_v3.c + * Copyright (C) 1999 ARM Limited + * Copyright (C) 2000-2001 Deep Blue Solutions Ltd + * + * Contributors to the old driver include: + * Russell King <linux@armlinux.org.uk> + * David A. Rusling <david.rusling@linaro.org> (uHAL, ARM Firmware suite) + * Rob Herring <robh@kernel.org> + * Liviu Dudau <Liviu.Dudau@arm.com> + * Grant Likely <grant.likely@secretlab.ca> + * Arnd Bergmann <arnd@arndb.de> + * Bjorn Helgaas <bhelgaas@google.com> + */ +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/of.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/bitops.h> +#include <linux/irq.h> +#include <linux/mfd/syscon.h> +#include <linux/regmap.h> +#include <linux/clk.h> + +#include "../pci.h" + +#define V3_PCI_VENDOR 0x00000000 +#define V3_PCI_DEVICE 0x00000002 +#define V3_PCI_CMD 0x00000004 +#define V3_PCI_STAT 0x00000006 +#define V3_PCI_CC_REV 0x00000008 +#define V3_PCI_HDR_CFG 0x0000000C +#define V3_PCI_IO_BASE 0x00000010 +#define V3_PCI_BASE0 0x00000014 +#define V3_PCI_BASE1 0x00000018 +#define V3_PCI_SUB_VENDOR 0x0000002C +#define V3_PCI_SUB_ID 0x0000002E +#define V3_PCI_ROM 0x00000030 +#define V3_PCI_BPARAM 0x0000003C +#define V3_PCI_MAP0 0x00000040 +#define V3_PCI_MAP1 0x00000044 +#define V3_PCI_INT_STAT 0x00000048 +#define V3_PCI_INT_CFG 0x0000004C +#define V3_LB_BASE0 0x00000054 +#define V3_LB_BASE1 0x00000058 +#define V3_LB_MAP0 0x0000005E +#define V3_LB_MAP1 0x00000062 +#define V3_LB_BASE2 0x00000064 +#define V3_LB_MAP2 0x00000066 +#define V3_LB_SIZE 0x00000068 +#define V3_LB_IO_BASE 0x0000006E +#define V3_FIFO_CFG 0x00000070 +#define V3_FIFO_PRIORITY 0x00000072 +#define V3_FIFO_STAT 0x00000074 +#define V3_LB_ISTAT 0x00000076 +#define V3_LB_IMASK 0x00000077 +#define V3_SYSTEM 0x00000078 +#define V3_LB_CFG 0x0000007A +#define V3_PCI_CFG 0x0000007C +#define V3_DMA_PCI_ADR0 0x00000080 +#define V3_DMA_PCI_ADR1 0x00000090 +#define V3_DMA_LOCAL_ADR0 0x00000084 +#define V3_DMA_LOCAL_ADR1 0x00000094 +#define V3_DMA_LENGTH0 0x00000088 +#define V3_DMA_LENGTH1 0x00000098 +#define V3_DMA_CSR0 0x0000008B +#define V3_DMA_CSR1 0x0000009B +#define V3_DMA_CTLB_ADR0 0x0000008C +#define V3_DMA_CTLB_ADR1 0x0000009C +#define V3_DMA_DELAY 0x000000E0 +#define V3_MAIL_DATA 0x000000C0 +#define V3_PCI_MAIL_IEWR 0x000000D0 +#define V3_PCI_MAIL_IERD 0x000000D2 +#define V3_LB_MAIL_IEWR 0x000000D4 +#define V3_LB_MAIL_IERD 0x000000D6 +#define V3_MAIL_WR_STAT 0x000000D8 +#define V3_MAIL_RD_STAT 0x000000DA +#define V3_QBA_MAP 0x000000DC + +/* PCI STATUS bits */ +#define V3_PCI_STAT_PAR_ERR BIT(15) +#define V3_PCI_STAT_SYS_ERR BIT(14) +#define V3_PCI_STAT_M_ABORT_ERR BIT(13) +#define V3_PCI_STAT_T_ABORT_ERR BIT(12) + +/* LB ISTAT bits */ +#define V3_LB_ISTAT_MAILBOX BIT(7) +#define V3_LB_ISTAT_PCI_RD BIT(6) +#define V3_LB_ISTAT_PCI_WR BIT(5) +#define V3_LB_ISTAT_PCI_INT BIT(4) +#define V3_LB_ISTAT_PCI_PERR BIT(3) +#define V3_LB_ISTAT_I2O_QWR BIT(2) +#define V3_LB_ISTAT_DMA1 BIT(1) +#define V3_LB_ISTAT_DMA0 BIT(0) + +/* PCI COMMAND bits */ +#define V3_COMMAND_M_FBB_EN BIT(9) +#define V3_COMMAND_M_SERR_EN BIT(8) +#define V3_COMMAND_M_PAR_EN BIT(6) +#define V3_COMMAND_M_MASTER_EN BIT(2) +#define V3_COMMAND_M_MEM_EN BIT(1) +#define V3_COMMAND_M_IO_EN BIT(0) + +/* SYSTEM bits */ +#define V3_SYSTEM_M_RST_OUT BIT(15) +#define V3_SYSTEM_M_LOCK BIT(14) +#define V3_SYSTEM_UNLOCK 0xa05f + +/* PCI CFG bits */ +#define V3_PCI_CFG_M_I2O_EN BIT(15) +#define V3_PCI_CFG_M_IO_REG_DIS BIT(14) +#define V3_PCI_CFG_M_IO_DIS BIT(13) +#define V3_PCI_CFG_M_EN3V BIT(12) +#define V3_PCI_CFG_M_RETRY_EN BIT(10) +#define V3_PCI_CFG_M_AD_LOW1 BIT(9) +#define V3_PCI_CFG_M_AD_LOW0 BIT(8) +/* + * This is the value applied to C/BE[3:1], with bit 0 always held 0 + * during DMA access. + */ +#define V3_PCI_CFG_M_RTYPE_SHIFT 5 +#define V3_PCI_CFG_M_WTYPE_SHIFT 1 +#define V3_PCI_CFG_TYPE_DEFAULT 0x3 + +/* PCI BASE bits (PCI -> Local Bus) */ +#define V3_PCI_BASE_M_ADR_BASE 0xFFF00000U +#define V3_PCI_BASE_M_ADR_BASEL 0x000FFF00U +#define V3_PCI_BASE_M_PREFETCH BIT(3) +#define V3_PCI_BASE_M_TYPE (3 << 1) +#define V3_PCI_BASE_M_IO BIT(0) + +/* PCI MAP bits (PCI -> Local bus) */ +#define V3_PCI_MAP_M_MAP_ADR 0xFFF00000U +#define V3_PCI_MAP_M_RD_POST_INH BIT(15) +#define V3_PCI_MAP_M_ROM_SIZE (3 << 10) +#define V3_PCI_MAP_M_SWAP (3 << 8) +#define V3_PCI_MAP_M_ADR_SIZE 0x000000F0U +#define V3_PCI_MAP_M_REG_EN BIT(1) +#define V3_PCI_MAP_M_ENABLE BIT(0) + +/* LB_BASE0,1 bits (Local bus -> PCI) */ +#define V3_LB_BASE_ADR_BASE 0xfff00000U +#define V3_LB_BASE_SWAP (3 << 8) +#define V3_LB_BASE_ADR_SIZE (15 << 4) +#define V3_LB_BASE_PREFETCH BIT(3) +#define V3_LB_BASE_ENABLE BIT(0) + +#define V3_LB_BASE_ADR_SIZE_1MB (0 << 4) +#define V3_LB_BASE_ADR_SIZE_2MB (1 << 4) +#define V3_LB_BASE_ADR_SIZE_4MB (2 << 4) +#define V3_LB_BASE_ADR_SIZE_8MB (3 << 4) +#define V3_LB_BASE_ADR_SIZE_16MB (4 << 4) +#define V3_LB_BASE_ADR_SIZE_32MB (5 << 4) +#define V3_LB_BASE_ADR_SIZE_64MB (6 << 4) +#define V3_LB_BASE_ADR_SIZE_128MB (7 << 4) +#define V3_LB_BASE_ADR_SIZE_256MB (8 << 4) +#define V3_LB_BASE_ADR_SIZE_512MB (9 << 4) +#define V3_LB_BASE_ADR_SIZE_1GB (10 << 4) +#define V3_LB_BASE_ADR_SIZE_2GB (11 << 4) + +#define v3_addr_to_lb_base(a) ((a) & V3_LB_BASE_ADR_BASE) + +/* LB_MAP0,1 bits (Local bus -> PCI) */ +#define V3_LB_MAP_MAP_ADR 0xfff0U +#define V3_LB_MAP_TYPE (7 << 1) +#define V3_LB_MAP_AD_LOW_EN BIT(0) + +#define V3_LB_MAP_TYPE_IACK (0 << 1) +#define V3_LB_MAP_TYPE_IO (1 << 1) +#define V3_LB_MAP_TYPE_MEM (3 << 1) +#define V3_LB_MAP_TYPE_CONFIG (5 << 1) +#define V3_LB_MAP_TYPE_MEM_MULTIPLE (6 << 1) + +#define v3_addr_to_lb_map(a) (((a) >> 16) & V3_LB_MAP_MAP_ADR) + +/* LB_BASE2 bits (Local bus -> PCI IO) */ +#define V3_LB_BASE2_ADR_BASE 0xff00U +#define V3_LB_BASE2_SWAP_AUTO (3 << 6) +#define V3_LB_BASE2_ENABLE BIT(0) + +#define v3_addr_to_lb_base2(a) (((a) >> 16) & V3_LB_BASE2_ADR_BASE) + +/* LB_MAP2 bits (Local bus -> PCI IO) */ +#define V3_LB_MAP2_MAP_ADR 0xff00U + +#define v3_addr_to_lb_map2(a) (((a) >> 16) & V3_LB_MAP2_MAP_ADR) + +/* FIFO priority bits */ +#define V3_FIFO_PRIO_LOCAL BIT(12) +#define V3_FIFO_PRIO_LB_RD1_FLUSH_EOB BIT(10) +#define V3_FIFO_PRIO_LB_RD1_FLUSH_AP1 BIT(11) +#define V3_FIFO_PRIO_LB_RD1_FLUSH_ANY (BIT(10)|BIT(11)) +#define V3_FIFO_PRIO_LB_RD0_FLUSH_EOB BIT(8) +#define V3_FIFO_PRIO_LB_RD0_FLUSH_AP1 BIT(9) +#define V3_FIFO_PRIO_LB_RD0_FLUSH_ANY (BIT(8)|BIT(9)) +#define V3_FIFO_PRIO_PCI BIT(4) +#define V3_FIFO_PRIO_PCI_RD1_FLUSH_EOB BIT(2) +#define V3_FIFO_PRIO_PCI_RD1_FLUSH_AP1 BIT(3) +#define V3_FIFO_PRIO_PCI_RD1_FLUSH_ANY (BIT(2)|BIT(3)) +#define V3_FIFO_PRIO_PCI_RD0_FLUSH_EOB BIT(0) +#define V3_FIFO_PRIO_PCI_RD0_FLUSH_AP1 BIT(1) +#define V3_FIFO_PRIO_PCI_RD0_FLUSH_ANY (BIT(0)|BIT(1)) + +/* Local bus configuration bits */ +#define V3_LB_CFG_LB_TO_64_CYCLES 0x0000 +#define V3_LB_CFG_LB_TO_256_CYCLES BIT(13) +#define V3_LB_CFG_LB_TO_512_CYCLES BIT(14) +#define V3_LB_CFG_LB_TO_1024_CYCLES (BIT(13)|BIT(14)) +#define V3_LB_CFG_LB_RST BIT(12) +#define V3_LB_CFG_LB_PPC_RDY BIT(11) +#define V3_LB_CFG_LB_LB_INT BIT(10) +#define V3_LB_CFG_LB_ERR_EN BIT(9) +#define V3_LB_CFG_LB_RDY_EN BIT(8) +#define V3_LB_CFG_LB_BE_IMODE BIT(7) +#define V3_LB_CFG_LB_BE_OMODE BIT(6) +#define V3_LB_CFG_LB_ENDIAN BIT(5) +#define V3_LB_CFG_LB_PARK_EN BIT(4) +#define V3_LB_CFG_LB_FBB_DIS BIT(2) + +/* ARM Integrator-specific extended control registers */ +#define INTEGRATOR_SC_PCI_OFFSET 0x18 +#define INTEGRATOR_SC_PCI_ENABLE BIT(0) +#define INTEGRATOR_SC_PCI_INTCLR BIT(1) +#define INTEGRATOR_SC_LBFADDR_OFFSET 0x20 +#define INTEGRATOR_SC_LBFCODE_OFFSET 0x24 + +struct v3_pci { + struct device *dev; + void __iomem *base; + void __iomem *config_base; + u32 config_mem; + u32 non_pre_mem; + u32 pre_mem; + phys_addr_t non_pre_bus_addr; + phys_addr_t pre_bus_addr; + struct regmap *map; +}; + +/* + * The V3 PCI interface chip in Integrator provides several windows from + * local bus memory into the PCI memory areas. Unfortunately, there + * are not really enough windows for our usage, therefore we reuse + * one of the windows for access to PCI configuration space. On the + * Integrator/AP, the memory map is as follows: + * + * Local Bus Memory Usage + * + * 40000000 - 4FFFFFFF PCI memory. 256M non-prefetchable + * 50000000 - 5FFFFFFF PCI memory. 256M prefetchable + * 60000000 - 60FFFFFF PCI IO. 16M + * 61000000 - 61FFFFFF PCI Configuration. 16M + * + * There are three V3 windows, each described by a pair of V3 registers. + * These are LB_BASE0/LB_MAP0, LB_BASE1/LB_MAP1 and LB_BASE2/LB_MAP2. + * Base0 and Base1 can be used for any type of PCI memory access. Base2 + * can be used either for PCI I/O or for I20 accesses. By default, uHAL + * uses this only for PCI IO space. + * + * Normally these spaces are mapped using the following base registers: + * + * Usage Local Bus Memory Base/Map registers used + * + * Mem 40000000 - 4FFFFFFF LB_BASE0/LB_MAP0 + * Mem 50000000 - 5FFFFFFF LB_BASE1/LB_MAP1 + * IO 60000000 - 60FFFFFF LB_BASE2/LB_MAP2 + * Cfg 61000000 - 61FFFFFF + * + * This means that I20 and PCI configuration space accesses will fail. + * When PCI configuration accesses are needed (via the uHAL PCI + * configuration space primitives) we must remap the spaces as follows: + * + * Usage Local Bus Memory Base/Map registers used + * + * Mem 40000000 - 4FFFFFFF LB_BASE0/LB_MAP0 + * Mem 50000000 - 5FFFFFFF LB_BASE0/LB_MAP0 + * IO 60000000 - 60FFFFFF LB_BASE2/LB_MAP2 + * Cfg 61000000 - 61FFFFFF LB_BASE1/LB_MAP1 + * + * To make this work, the code depends on overlapping windows working. + * The V3 chip translates an address by checking its range within + * each of the BASE/MAP pairs in turn (in ascending register number + * order). It will use the first matching pair. So, for example, + * if the same address is mapped by both LB_BASE0/LB_MAP0 and + * LB_BASE1/LB_MAP1, the V3 will use the translation from + * LB_BASE0/LB_MAP0. + * + * To allow PCI Configuration space access, the code enlarges the + * window mapped by LB_BASE0/LB_MAP0 from 256M to 512M. This occludes + * the windows currently mapped by LB_BASE1/LB_MAP1 so that it can + * be remapped for use by configuration cycles. + * + * At the end of the PCI Configuration space accesses, + * LB_BASE1/LB_MAP1 is reset to map PCI Memory. Finally the window + * mapped by LB_BASE0/LB_MAP0 is reduced in size from 512M to 256M to + * reveal the now restored LB_BASE1/LB_MAP1 window. + * + * NOTE: We do not set up I2O mapping. I suspect that this is only + * for an intelligent (target) device. Using I2O disables most of + * the mappings into PCI memory. + */ +static void __iomem *v3_map_bus(struct pci_bus *bus, + unsigned int devfn, int offset) +{ + struct v3_pci *v3 = bus->sysdata; + unsigned int address, mapaddress, busnr; + + busnr = bus->number; + if (busnr == 0) { + int slot = PCI_SLOT(devfn); + + /* + * local bus segment so need a type 0 config cycle + * + * build the PCI configuration "address" with one-hot in + * A31-A11 + * + * mapaddress: + * 3:1 = config cycle (101) + * 0 = PCI A1 & A0 are 0 (0) + */ + address = PCI_FUNC(devfn) << 8; + mapaddress = V3_LB_MAP_TYPE_CONFIG; + + if (slot > 12) + /* + * high order bits are handled by the MAP register + */ + mapaddress |= BIT(slot - 5); + else + /* + * low order bits handled directly in the address + */ + address |= BIT(slot + 11); + } else { + /* + * not the local bus segment so need a type 1 config cycle + * + * address: + * 23:16 = bus number + * 15:11 = slot number (7:3 of devfn) + * 10:8 = func number (2:0 of devfn) + * + * mapaddress: + * 3:1 = config cycle (101) + * 0 = PCI A1 & A0 from host bus (1) + */ + mapaddress = V3_LB_MAP_TYPE_CONFIG | V3_LB_MAP_AD_LOW_EN; + address = (busnr << 16) | (devfn << 8); + } + + /* + * Set up base0 to see all 512Mbytes of memory space (not + * prefetchable), this frees up base1 for re-use by + * configuration memory + */ + writel(v3_addr_to_lb_base(v3->non_pre_mem) | + V3_LB_BASE_ADR_SIZE_512MB | V3_LB_BASE_ENABLE, + v3->base + V3_LB_BASE0); + + /* + * Set up base1/map1 to point into configuration space. + * The config mem is always 16MB. + */ + writel(v3_addr_to_lb_base(v3->config_mem) | + V3_LB_BASE_ADR_SIZE_16MB | V3_LB_BASE_ENABLE, + v3->base + V3_LB_BASE1); + writew(mapaddress, v3->base + V3_LB_MAP1); + + return v3->config_base + address + offset; +} + +static void v3_unmap_bus(struct v3_pci *v3) +{ + /* + * Reassign base1 for use by prefetchable PCI memory + */ + writel(v3_addr_to_lb_base(v3->pre_mem) | + V3_LB_BASE_ADR_SIZE_256MB | V3_LB_BASE_PREFETCH | + V3_LB_BASE_ENABLE, + v3->base + V3_LB_BASE1); + writew(v3_addr_to_lb_map(v3->pre_bus_addr) | + V3_LB_MAP_TYPE_MEM, /* was V3_LB_MAP_TYPE_MEM_MULTIPLE */ + v3->base + V3_LB_MAP1); + + /* + * And shrink base0 back to a 256M window (NOTE: MAP0 already correct) + */ + writel(v3_addr_to_lb_base(v3->non_pre_mem) | + V3_LB_BASE_ADR_SIZE_256MB | V3_LB_BASE_ENABLE, + v3->base + V3_LB_BASE0); +} + +static int v3_pci_read_config(struct pci_bus *bus, unsigned int fn, + int config, int size, u32 *value) +{ + struct v3_pci *v3 = bus->sysdata; + int ret; + + dev_dbg(&bus->dev, + "[read] slt: %.2d, fnc: %d, cnf: 0x%.2X, val (%d bytes): 0x%.8X\n", + PCI_SLOT(fn), PCI_FUNC(fn), config, size, *value); + ret = pci_generic_config_read(bus, fn, config, size, value); + v3_unmap_bus(v3); + return ret; +} + +static int v3_pci_write_config(struct pci_bus *bus, unsigned int fn, + int config, int size, u32 value) +{ + struct v3_pci *v3 = bus->sysdata; + int ret; + + dev_dbg(&bus->dev, + "[write] slt: %.2d, fnc: %d, cnf: 0x%.2X, val (%d bytes): 0x%.8X\n", + PCI_SLOT(fn), PCI_FUNC(fn), config, size, value); + ret = pci_generic_config_write(bus, fn, config, size, value); + v3_unmap_bus(v3); + return ret; +} + +static struct pci_ops v3_pci_ops = { + .map_bus = v3_map_bus, + .read = v3_pci_read_config, + .write = v3_pci_write_config, +}; + +static irqreturn_t v3_irq(int irq, void *data) +{ + struct v3_pci *v3 = data; + struct device *dev = v3->dev; + u32 status; + + status = readw(v3->base + V3_PCI_STAT); + if (status & V3_PCI_STAT_PAR_ERR) + dev_err(dev, "parity error interrupt\n"); + if (status & V3_PCI_STAT_SYS_ERR) + dev_err(dev, "system error interrupt\n"); + if (status & V3_PCI_STAT_M_ABORT_ERR) + dev_err(dev, "master abort error interrupt\n"); + if (status & V3_PCI_STAT_T_ABORT_ERR) + dev_err(dev, "target abort error interrupt\n"); + writew(status, v3->base + V3_PCI_STAT); + + status = readb(v3->base + V3_LB_ISTAT); + if (status & V3_LB_ISTAT_MAILBOX) + dev_info(dev, "PCI mailbox interrupt\n"); + if (status & V3_LB_ISTAT_PCI_RD) + dev_err(dev, "PCI target LB->PCI READ abort interrupt\n"); + if (status & V3_LB_ISTAT_PCI_WR) + dev_err(dev, "PCI target LB->PCI WRITE abort interrupt\n"); + if (status & V3_LB_ISTAT_PCI_INT) + dev_info(dev, "PCI pin interrupt\n"); + if (status & V3_LB_ISTAT_PCI_PERR) + dev_err(dev, "PCI parity error interrupt\n"); + if (status & V3_LB_ISTAT_I2O_QWR) + dev_info(dev, "I2O inbound post queue interrupt\n"); + if (status & V3_LB_ISTAT_DMA1) + dev_info(dev, "DMA channel 1 interrupt\n"); + if (status & V3_LB_ISTAT_DMA0) + dev_info(dev, "DMA channel 0 interrupt\n"); + /* Clear all possible interrupts on the local bus */ + writeb(0, v3->base + V3_LB_ISTAT); + if (v3->map) + regmap_write(v3->map, INTEGRATOR_SC_PCI_OFFSET, + INTEGRATOR_SC_PCI_ENABLE | + INTEGRATOR_SC_PCI_INTCLR); + + return IRQ_HANDLED; +} + +static int v3_integrator_init(struct v3_pci *v3) +{ + unsigned int val; + + v3->map = + syscon_regmap_lookup_by_compatible("arm,integrator-ap-syscon"); + if (IS_ERR(v3->map)) { + dev_err(v3->dev, "no syscon\n"); + return -ENODEV; + } + + regmap_read(v3->map, INTEGRATOR_SC_PCI_OFFSET, &val); + /* Take the PCI bridge out of reset, clear IRQs */ + regmap_write(v3->map, INTEGRATOR_SC_PCI_OFFSET, + INTEGRATOR_SC_PCI_ENABLE | + INTEGRATOR_SC_PCI_INTCLR); + + if (!(val & INTEGRATOR_SC_PCI_ENABLE)) { + /* If we were in reset we need to sleep a bit */ + msleep(230); + + /* Set the physical base for the controller itself */ + writel(0x6200, v3->base + V3_LB_IO_BASE); + + /* Wait for the mailbox to settle after reset */ + do { + writeb(0xaa, v3->base + V3_MAIL_DATA); + writeb(0x55, v3->base + V3_MAIL_DATA + 4); + } while (readb(v3->base + V3_MAIL_DATA) != 0xaa && + readb(v3->base + V3_MAIL_DATA) != 0x55); + } + + dev_info(v3->dev, "initialized PCI V3 Integrator/AP integration\n"); + + return 0; +} + +static int v3_pci_setup_resource(struct v3_pci *v3, + struct pci_host_bridge *host, + struct resource_entry *win) +{ + struct device *dev = v3->dev; + struct resource *mem; + struct resource *io; + + switch (resource_type(win->res)) { + case IORESOURCE_IO: + io = win->res; + + /* Setup window 2 - PCI I/O */ + writel(v3_addr_to_lb_base2(pci_pio_to_address(io->start)) | + V3_LB_BASE2_ENABLE, + v3->base + V3_LB_BASE2); + writew(v3_addr_to_lb_map2(io->start - win->offset), + v3->base + V3_LB_MAP2); + break; + case IORESOURCE_MEM: + mem = win->res; + if (mem->flags & IORESOURCE_PREFETCH) { + mem->name = "V3 PCI PRE-MEM"; + v3->pre_mem = mem->start; + v3->pre_bus_addr = mem->start - win->offset; + dev_dbg(dev, "PREFETCHABLE MEM window %pR, bus addr %pap\n", + mem, &v3->pre_bus_addr); + if (resource_size(mem) != SZ_256M) { + dev_err(dev, "prefetchable memory range is not 256MB\n"); + return -EINVAL; + } + if (v3->non_pre_mem && + (mem->start != v3->non_pre_mem + SZ_256M)) { + dev_err(dev, + "prefetchable memory is not adjacent to non-prefetchable memory\n"); + return -EINVAL; + } + /* Setup window 1 - PCI prefetchable memory */ + writel(v3_addr_to_lb_base(v3->pre_mem) | + V3_LB_BASE_ADR_SIZE_256MB | + V3_LB_BASE_PREFETCH | + V3_LB_BASE_ENABLE, + v3->base + V3_LB_BASE1); + writew(v3_addr_to_lb_map(v3->pre_bus_addr) | + V3_LB_MAP_TYPE_MEM, /* Was V3_LB_MAP_TYPE_MEM_MULTIPLE */ + v3->base + V3_LB_MAP1); + } else { + mem->name = "V3 PCI NON-PRE-MEM"; + v3->non_pre_mem = mem->start; + v3->non_pre_bus_addr = mem->start - win->offset; + dev_dbg(dev, "NON-PREFETCHABLE MEM window %pR, bus addr %pap\n", + mem, &v3->non_pre_bus_addr); + if (resource_size(mem) != SZ_256M) { + dev_err(dev, + "non-prefetchable memory range is not 256MB\n"); + return -EINVAL; + } + /* Setup window 0 - PCI non-prefetchable memory */ + writel(v3_addr_to_lb_base(v3->non_pre_mem) | + V3_LB_BASE_ADR_SIZE_256MB | + V3_LB_BASE_ENABLE, + v3->base + V3_LB_BASE0); + writew(v3_addr_to_lb_map(v3->non_pre_bus_addr) | + V3_LB_MAP_TYPE_MEM, + v3->base + V3_LB_MAP0); + } + break; + case IORESOURCE_BUS: + break; + default: + dev_info(dev, "Unknown resource type %lu\n", + resource_type(win->res)); + break; + } + + return 0; +} + +static int v3_get_dma_range_config(struct v3_pci *v3, + struct resource_entry *entry, + u32 *pci_base, u32 *pci_map) +{ + struct device *dev = v3->dev; + u64 cpu_addr = entry->res->start; + u64 cpu_end = entry->res->end; + u64 pci_end = cpu_end - entry->offset; + u64 pci_addr = entry->res->start - entry->offset; + u32 val; + + if (pci_addr & ~V3_PCI_BASE_M_ADR_BASE) { + dev_err(dev, "illegal range, only PCI bits 31..20 allowed\n"); + return -EINVAL; + } + val = ((u32)pci_addr) & V3_PCI_BASE_M_ADR_BASE; + *pci_base = val; + + if (cpu_addr & ~V3_PCI_MAP_M_MAP_ADR) { + dev_err(dev, "illegal range, only CPU bits 31..20 allowed\n"); + return -EINVAL; + } + val = ((u32)cpu_addr) & V3_PCI_MAP_M_MAP_ADR; + + switch (resource_size(entry->res)) { + case SZ_1M: + val |= V3_LB_BASE_ADR_SIZE_1MB; + break; + case SZ_2M: + val |= V3_LB_BASE_ADR_SIZE_2MB; + break; + case SZ_4M: + val |= V3_LB_BASE_ADR_SIZE_4MB; + break; + case SZ_8M: + val |= V3_LB_BASE_ADR_SIZE_8MB; + break; + case SZ_16M: + val |= V3_LB_BASE_ADR_SIZE_16MB; + break; + case SZ_32M: + val |= V3_LB_BASE_ADR_SIZE_32MB; + break; + case SZ_64M: + val |= V3_LB_BASE_ADR_SIZE_64MB; + break; + case SZ_128M: + val |= V3_LB_BASE_ADR_SIZE_128MB; + break; + case SZ_256M: + val |= V3_LB_BASE_ADR_SIZE_256MB; + break; + case SZ_512M: + val |= V3_LB_BASE_ADR_SIZE_512MB; + break; + case SZ_1G: + val |= V3_LB_BASE_ADR_SIZE_1GB; + break; + case SZ_2G: + val |= V3_LB_BASE_ADR_SIZE_2GB; + break; + default: + dev_err(v3->dev, "illegal dma memory chunk size\n"); + return -EINVAL; + } + val |= V3_PCI_MAP_M_REG_EN | V3_PCI_MAP_M_ENABLE; + *pci_map = val; + + dev_dbg(dev, + "DMA MEM CPU: 0x%016llx -> 0x%016llx => " + "PCI: 0x%016llx -> 0x%016llx base %08x map %08x\n", + cpu_addr, cpu_end, + pci_addr, pci_end, + *pci_base, *pci_map); + + return 0; +} + +static int v3_pci_parse_map_dma_ranges(struct v3_pci *v3, + struct device_node *np) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(v3); + struct device *dev = v3->dev; + struct resource_entry *entry; + int i = 0; + + resource_list_for_each_entry(entry, &bridge->dma_ranges) { + int ret; + u32 pci_base, pci_map; + + ret = v3_get_dma_range_config(v3, entry, &pci_base, &pci_map); + if (ret) + return ret; + + if (i == 0) { + writel(pci_base, v3->base + V3_PCI_BASE0); + writel(pci_map, v3->base + V3_PCI_MAP0); + } else if (i == 1) { + writel(pci_base, v3->base + V3_PCI_BASE1); + writel(pci_map, v3->base + V3_PCI_MAP1); + } else { + dev_err(dev, "too many ranges, only two supported\n"); + dev_err(dev, "range %d ignored\n", i); + } + i++; + } + return 0; +} + +static int v3_pci_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct resource *regs; + struct resource_entry *win; + struct v3_pci *v3; + struct pci_host_bridge *host; + struct clk *clk; + u16 val; + int irq; + int ret; + + host = devm_pci_alloc_host_bridge(dev, sizeof(*v3)); + if (!host) + return -ENOMEM; + + host->ops = &v3_pci_ops; + v3 = pci_host_bridge_priv(host); + host->sysdata = v3; + v3->dev = dev; + + /* Get and enable host clock */ + clk = devm_clk_get(dev, NULL); + if (IS_ERR(clk)) { + dev_err(dev, "clock not found\n"); + return PTR_ERR(clk); + } + ret = clk_prepare_enable(clk); + if (ret) { + dev_err(dev, "unable to enable clock\n"); + return ret; + } + + v3->base = devm_platform_get_and_ioremap_resource(pdev, 0, ®s); + if (IS_ERR(v3->base)) + return PTR_ERR(v3->base); + /* + * The hardware has a register with the physical base address + * of the V3 controller itself, verify that this is the same + * as the physical memory we've remapped it from. + */ + if (readl(v3->base + V3_LB_IO_BASE) != (regs->start >> 16)) + dev_err(dev, "V3_LB_IO_BASE = %08x but device is @%pR\n", + readl(v3->base + V3_LB_IO_BASE), regs); + + /* Configuration space is 16MB directly mapped */ + regs = platform_get_resource(pdev, IORESOURCE_MEM, 1); + if (resource_size(regs) != SZ_16M) { + dev_err(dev, "config mem is not 16MB!\n"); + return -EINVAL; + } + v3->config_mem = regs->start; + v3->config_base = devm_ioremap_resource(dev, regs); + if (IS_ERR(v3->config_base)) + return PTR_ERR(v3->config_base); + + /* Get and request error IRQ resource */ + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + ret = devm_request_irq(dev, irq, v3_irq, 0, + "PCIv3 error", v3); + if (ret < 0) { + dev_err(dev, + "unable to request PCIv3 error IRQ %d (%d)\n", + irq, ret); + return ret; + } + + /* + * Unlock V3 registers, but only if they were previously locked. + */ + if (readw(v3->base + V3_SYSTEM) & V3_SYSTEM_M_LOCK) + writew(V3_SYSTEM_UNLOCK, v3->base + V3_SYSTEM); + + /* Disable all slave access while we set up the windows */ + val = readw(v3->base + V3_PCI_CMD); + val &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + writew(val, v3->base + V3_PCI_CMD); + + /* Put the PCI bus into reset */ + val = readw(v3->base + V3_SYSTEM); + val &= ~V3_SYSTEM_M_RST_OUT; + writew(val, v3->base + V3_SYSTEM); + + /* Retry until we're ready */ + val = readw(v3->base + V3_PCI_CFG); + val |= V3_PCI_CFG_M_RETRY_EN; + writew(val, v3->base + V3_PCI_CFG); + + /* Set up the local bus protocol */ + val = readw(v3->base + V3_LB_CFG); + val |= V3_LB_CFG_LB_BE_IMODE; /* Byte enable input */ + val |= V3_LB_CFG_LB_BE_OMODE; /* Byte enable output */ + val &= ~V3_LB_CFG_LB_ENDIAN; /* Little endian */ + val &= ~V3_LB_CFG_LB_PPC_RDY; /* TODO: when using on PPC403Gx, set to 1 */ + writew(val, v3->base + V3_LB_CFG); + + /* Enable the PCI bus master */ + val = readw(v3->base + V3_PCI_CMD); + val |= PCI_COMMAND_MASTER; + writew(val, v3->base + V3_PCI_CMD); + + /* Get the I/O and memory ranges from DT */ + resource_list_for_each_entry(win, &host->windows) { + ret = v3_pci_setup_resource(v3, host, win); + if (ret) { + dev_err(dev, "error setting up resources\n"); + return ret; + } + } + ret = v3_pci_parse_map_dma_ranges(v3, np); + if (ret) + return ret; + + /* + * Disable PCI to host IO cycles, enable I/O buffers @3.3V, + * set AD_LOW0 to 1 if one of the LB_MAP registers choose + * to use this (should be unused). + */ + writel(0x00000000, v3->base + V3_PCI_IO_BASE); + val = V3_PCI_CFG_M_IO_REG_DIS | V3_PCI_CFG_M_IO_DIS | + V3_PCI_CFG_M_EN3V | V3_PCI_CFG_M_AD_LOW0; + /* + * DMA read and write from PCI bus commands types + */ + val |= V3_PCI_CFG_TYPE_DEFAULT << V3_PCI_CFG_M_RTYPE_SHIFT; + val |= V3_PCI_CFG_TYPE_DEFAULT << V3_PCI_CFG_M_WTYPE_SHIFT; + writew(val, v3->base + V3_PCI_CFG); + + /* + * Set the V3 FIFO such that writes have higher priority than + * reads, and local bus write causes local bus read fifo flush + * on aperture 1. Same for PCI. + */ + writew(V3_FIFO_PRIO_LB_RD1_FLUSH_AP1 | + V3_FIFO_PRIO_LB_RD0_FLUSH_AP1 | + V3_FIFO_PRIO_PCI_RD1_FLUSH_AP1 | + V3_FIFO_PRIO_PCI_RD0_FLUSH_AP1, + v3->base + V3_FIFO_PRIORITY); + + + /* + * Clear any error interrupts, and enable parity and write error + * interrupts + */ + writeb(0, v3->base + V3_LB_ISTAT); + val = readw(v3->base + V3_LB_CFG); + val |= V3_LB_CFG_LB_LB_INT; + writew(val, v3->base + V3_LB_CFG); + writeb(V3_LB_ISTAT_PCI_WR | V3_LB_ISTAT_PCI_PERR, + v3->base + V3_LB_IMASK); + + /* Special Integrator initialization */ + if (of_device_is_compatible(np, "arm,integrator-ap-pci")) { + ret = v3_integrator_init(v3); + if (ret) + return ret; + } + + /* Post-init: enable PCI memory and invalidate (master already on) */ + val = readw(v3->base + V3_PCI_CMD); + val |= PCI_COMMAND_MEMORY | PCI_COMMAND_INVALIDATE; + writew(val, v3->base + V3_PCI_CMD); + + /* Clear pending interrupts */ + writeb(0, v3->base + V3_LB_ISTAT); + /* Read or write errors and parity errors cause interrupts */ + writeb(V3_LB_ISTAT_PCI_RD | V3_LB_ISTAT_PCI_WR | V3_LB_ISTAT_PCI_PERR, + v3->base + V3_LB_IMASK); + + /* Take the PCI bus out of reset so devices can initialize */ + val = readw(v3->base + V3_SYSTEM); + val |= V3_SYSTEM_M_RST_OUT; + writew(val, v3->base + V3_SYSTEM); + + /* + * Re-lock the system register. + */ + val = readw(v3->base + V3_SYSTEM); + val |= V3_SYSTEM_M_LOCK; + writew(val, v3->base + V3_SYSTEM); + + return pci_host_probe(host); +} + +static const struct of_device_id v3_pci_of_match[] = { + { + .compatible = "v3,v360epc-pci", + }, + {}, +}; + +static struct platform_driver v3_pci_driver = { + .driver = { + .name = "pci-v3-semi", + .of_match_table = v3_pci_of_match, + .suppress_bind_attrs = true, + }, + .probe = v3_pci_probe, +}; +builtin_platform_driver(v3_pci_driver); diff --git a/drivers/pci/controller/pci-versatile.c b/drivers/pci/controller/pci-versatile.c new file mode 100644 index 000000000..e9a6758fe --- /dev/null +++ b/drivers/pci/controller/pci-versatile.c @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2004 Koninklijke Philips Electronics NV + * + * Conversion to platform driver and DT: + * Copyright 2014 Linaro Ltd. + * + * 14/04/2005 Initial version, colin.king@philips.com + */ +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/pci.h> +#include <linux/platform_device.h> + +#include "../pci.h" + +static void __iomem *versatile_pci_base; +static void __iomem *versatile_cfg_base[2]; + +#define PCI_IMAP(m) (versatile_pci_base + ((m) * 4)) +#define PCI_SMAP(m) (versatile_pci_base + 0x14 + ((m) * 4)) +#define PCI_SELFID (versatile_pci_base + 0xc) + +#define VP_PCI_DEVICE_ID 0x030010ee +#define VP_PCI_CLASS_ID 0x0b400000 + +static u32 pci_slot_ignore; + +static int __init versatile_pci_slot_ignore(char *str) +{ + int slot; + + while (get_option(&str, &slot)) { + if ((slot < 0) || (slot > 31)) + pr_err("Illegal slot value: %d\n", slot); + else + pci_slot_ignore |= (1 << slot); + } + return 1; +} +__setup("pci_slot_ignore=", versatile_pci_slot_ignore); + + +static void __iomem *versatile_map_bus(struct pci_bus *bus, + unsigned int devfn, int offset) +{ + unsigned int busnr = bus->number; + + if (pci_slot_ignore & (1 << PCI_SLOT(devfn))) + return NULL; + + return versatile_cfg_base[1] + ((busnr << 16) | (devfn << 8) | offset); +} + +static struct pci_ops pci_versatile_ops = { + .map_bus = versatile_map_bus, + .read = pci_generic_config_read32, + .write = pci_generic_config_write, +}; + +static int versatile_pci_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct resource *res; + struct resource_entry *entry; + int i, myslot = -1, mem = 1; + u32 val; + void __iomem *local_pci_cfg_base; + struct pci_host_bridge *bridge; + + bridge = devm_pci_alloc_host_bridge(dev, 0); + if (!bridge) + return -ENOMEM; + + versatile_pci_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(versatile_pci_base)) + return PTR_ERR(versatile_pci_base); + + versatile_cfg_base[0] = devm_platform_ioremap_resource(pdev, 1); + if (IS_ERR(versatile_cfg_base[0])) + return PTR_ERR(versatile_cfg_base[0]); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 2); + versatile_cfg_base[1] = devm_pci_remap_cfg_resource(dev, res); + if (IS_ERR(versatile_cfg_base[1])) + return PTR_ERR(versatile_cfg_base[1]); + + resource_list_for_each_entry(entry, &bridge->windows) { + if (resource_type(entry->res) == IORESOURCE_MEM) { + writel(entry->res->start >> 28, PCI_IMAP(mem)); + writel(__pa(PAGE_OFFSET) >> 28, PCI_SMAP(mem)); + mem++; + } + } + + /* + * We need to discover the PCI core first to configure itself + * before the main PCI probing is performed + */ + for (i = 0; i < 32; i++) { + if ((readl(versatile_cfg_base[0] + (i << 11) + PCI_VENDOR_ID) == VP_PCI_DEVICE_ID) && + (readl(versatile_cfg_base[0] + (i << 11) + PCI_CLASS_REVISION) == VP_PCI_CLASS_ID)) { + myslot = i; + break; + } + } + if (myslot == -1) { + dev_err(dev, "Cannot find PCI core!\n"); + return -EIO; + } + /* + * Do not to map Versatile FPGA PCI device into memory space + */ + pci_slot_ignore |= (1 << myslot); + + dev_info(dev, "PCI core found (slot %d)\n", myslot); + + writel(myslot, PCI_SELFID); + local_pci_cfg_base = versatile_cfg_base[1] + (myslot << 11); + + val = readl(local_pci_cfg_base + PCI_COMMAND); + val |= PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER | PCI_COMMAND_INVALIDATE; + writel(val, local_pci_cfg_base + PCI_COMMAND); + + /* + * Configure the PCI inbound memory windows to be 1:1 mapped to SDRAM + */ + writel(__pa(PAGE_OFFSET), local_pci_cfg_base + PCI_BASE_ADDRESS_0); + writel(__pa(PAGE_OFFSET), local_pci_cfg_base + PCI_BASE_ADDRESS_1); + writel(__pa(PAGE_OFFSET), local_pci_cfg_base + PCI_BASE_ADDRESS_2); + + /* + * For many years the kernel and QEMU were symbiotically buggy + * in that they both assumed the same broken IRQ mapping. + * QEMU therefore attempts to auto-detect old broken kernels + * so that they still work on newer QEMU as they did on old + * QEMU. Since we now use the correct (ie matching-hardware) + * IRQ mapping we write a definitely different value to a + * PCI_INTERRUPT_LINE register to tell QEMU that we expect + * real hardware behaviour and it need not be backwards + * compatible for us. This write is harmless on real hardware. + */ + writel(0, versatile_cfg_base[0] + PCI_INTERRUPT_LINE); + + pci_add_flags(PCI_REASSIGN_ALL_BUS); + + bridge->ops = &pci_versatile_ops; + + return pci_host_probe(bridge); +} + +static const struct of_device_id versatile_pci_of_match[] = { + { .compatible = "arm,versatile-pci", }, + { }, +}; +MODULE_DEVICE_TABLE(of, versatile_pci_of_match); + +static struct platform_driver versatile_pci_driver = { + .driver = { + .name = "versatile-pci", + .of_match_table = versatile_pci_of_match, + .suppress_bind_attrs = true, + }, + .probe = versatile_pci_probe, +}; +module_platform_driver(versatile_pci_driver); + +MODULE_DESCRIPTION("Versatile PCI driver"); diff --git a/drivers/pci/controller/pci-xgene-msi.c b/drivers/pci/controller/pci-xgene-msi.c new file mode 100644 index 000000000..654639bcc --- /dev/null +++ b/drivers/pci/controller/pci-xgene-msi.c @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * APM X-Gene MSI Driver + * + * Copyright (c) 2014, Applied Micro Circuits Corporation + * Author: Tanmay Inamdar <tinamdar@apm.com> + * Duc Dang <dhdang@apm.com> + */ +#include <linux/bitfield.h> +#include <linux/cpu.h> +#include <linux/interrupt.h> +#include <linux/irqdomain.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/of_pci.h> + +#define MSI_IR0 0x000000 +#define MSI_INT0 0x800000 +#define IDX_PER_GROUP 8 +#define IRQS_PER_IDX 16 +#define NR_HW_IRQS 16 +#define NR_MSI_BITS (IDX_PER_GROUP * IRQS_PER_IDX * NR_HW_IRQS) +#define NR_MSI_VEC (NR_MSI_BITS / num_possible_cpus()) + +#define MSI_GROUP_MASK GENMASK(22, 19) +#define MSI_INDEX_MASK GENMASK(18, 16) +#define MSI_INTR_MASK GENMASK(19, 16) + +#define MSInRx_HWIRQ_MASK GENMASK(6, 4) +#define DATA_HWIRQ_MASK GENMASK(3, 0) + +struct xgene_msi { + struct irq_domain *inner_domain; + u64 msi_addr; + void __iomem *msi_regs; + unsigned long *bitmap; + struct mutex bitmap_lock; + unsigned int gic_irq[NR_HW_IRQS]; +}; + +/* Global data */ +static struct xgene_msi *xgene_msi_ctrl; + +/* + * X-Gene v1 has 16 frames of MSI termination registers MSInIRx, where n is + * frame number (0..15), x is index of registers in each frame (0..7). Each + * 32b register is at the beginning of a 64kB region, each frame occupying + * 512kB (and the whole thing 8MB of PA space). + * + * Each register supports 16 MSI vectors (0..15) to generate interrupts. A + * write to the MSInIRx from the PCI side generates an interrupt. A read + * from the MSInRx on the CPU side returns a bitmap of the pending MSIs in + * the lower 16 bits. A side effect of this read is that all pending + * interrupts are acknowledged and cleared). + * + * Additionally, each MSI termination frame has 1 MSIINTn register (n is + * 0..15) to indicate the MSI pending status caused by any of its 8 + * termination registers, reported as a bitmap in the lower 8 bits. Each 32b + * register is at the beginning of a 64kB region (and overall occupying an + * extra 1MB). + * + * There is one GIC IRQ assigned for each MSI termination frame, 16 in + * total. + * + * The register layout is as follows: + * MSI0IR0 base_addr + * MSI0IR1 base_addr + 0x10000 + * ... ... + * MSI0IR6 base_addr + 0x60000 + * MSI0IR7 base_addr + 0x70000 + * MSI1IR0 base_addr + 0x80000 + * MSI1IR1 base_addr + 0x90000 + * ... ... + * MSI1IR7 base_addr + 0xF0000 + * MSI2IR0 base_addr + 0x100000 + * ... ... + * MSIFIR0 base_addr + 0x780000 + * MSIFIR1 base_addr + 0x790000 + * ... ... + * MSIFIR7 base_addr + 0x7F0000 + * MSIINT0 base_addr + 0x800000 + * MSIINT1 base_addr + 0x810000 + * ... ... + * MSIINTF base_addr + 0x8F0000 + */ + +/* MSInIRx read helper */ +static u32 xgene_msi_ir_read(struct xgene_msi *msi, u32 msi_grp, u32 msir_idx) +{ + return readl_relaxed(msi->msi_regs + MSI_IR0 + + (FIELD_PREP(MSI_GROUP_MASK, msi_grp) | + FIELD_PREP(MSI_INDEX_MASK, msir_idx))); +} + +/* MSIINTn read helper */ +static u32 xgene_msi_int_read(struct xgene_msi *msi, u32 msi_grp) +{ + return readl_relaxed(msi->msi_regs + MSI_INT0 + + FIELD_PREP(MSI_INTR_MASK, msi_grp)); +} + +/* + * In order to allow an MSI to be moved from one CPU to another without + * having to repaint both the address and the data (which cannot be done + * atomically), we statically partitions the MSI frames between CPUs. Given + * that XGene-1 has 8 CPUs, each CPU gets two frames assigned to it + * + * We adopt the convention that when an MSI is moved, it is configured to + * target the same register number in the congruent frame assigned to the + * new target CPU. This reserves a given MSI across all CPUs, and reduces + * the MSI capacity from 2048 to 256. + * + * Effectively, this amounts to: + * - hwirq[7]::cpu[2:0] is the target frame number (n in MSInIRx) + * - hwirq[6:4] is the register index in any given frame (x in MSInIRx) + * - hwirq[3:0] is the MSI data + */ +static irq_hw_number_t compute_hwirq(u8 frame, u8 index, u8 data) +{ + return (FIELD_PREP(BIT(7), FIELD_GET(BIT(3), frame)) | + FIELD_PREP(MSInRx_HWIRQ_MASK, index) | + FIELD_PREP(DATA_HWIRQ_MASK, data)); +} + +static void xgene_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct xgene_msi *msi = irq_data_get_irq_chip_data(data); + u64 target_addr; + u32 frame, msir; + int cpu; + + cpu = cpumask_first(irq_data_get_effective_affinity_mask(data)); + msir = FIELD_GET(MSInRx_HWIRQ_MASK, data->hwirq); + frame = FIELD_PREP(BIT(3), FIELD_GET(BIT(7), data->hwirq)) | cpu; + + target_addr = msi->msi_addr; + target_addr += (FIELD_PREP(MSI_GROUP_MASK, frame) | + FIELD_PREP(MSI_INTR_MASK, msir)); + + msg->address_hi = upper_32_bits(target_addr); + msg->address_lo = lower_32_bits(target_addr); + msg->data = FIELD_GET(DATA_HWIRQ_MASK, data->hwirq); +} + +static int xgene_msi_set_affinity(struct irq_data *irqdata, + const struct cpumask *mask, bool force) +{ + int target_cpu = cpumask_first(mask); + + irq_data_update_effective_affinity(irqdata, cpumask_of(target_cpu)); + + /* Force the core code to regenerate the message */ + return IRQ_SET_MASK_OK; +} + +static struct irq_chip xgene_msi_bottom_irq_chip = { + .name = "MSI", + .irq_set_affinity = xgene_msi_set_affinity, + .irq_compose_msi_msg = xgene_compose_msi_msg, +}; + +static int xgene_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct xgene_msi *msi = domain->host_data; + irq_hw_number_t hwirq; + + mutex_lock(&msi->bitmap_lock); + + hwirq = find_first_zero_bit(msi->bitmap, NR_MSI_VEC); + if (hwirq < NR_MSI_VEC) + set_bit(hwirq, msi->bitmap); + + mutex_unlock(&msi->bitmap_lock); + + if (hwirq >= NR_MSI_VEC) + return -ENOSPC; + + irq_domain_set_info(domain, virq, hwirq, + &xgene_msi_bottom_irq_chip, domain->host_data, + handle_simple_irq, NULL, NULL); + irqd_set_resend_when_in_progress(irq_get_irq_data(virq)); + + return 0; +} + +static void xgene_irq_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct xgene_msi *msi = irq_data_get_irq_chip_data(d); + + mutex_lock(&msi->bitmap_lock); + + clear_bit(d->hwirq, msi->bitmap); + + mutex_unlock(&msi->bitmap_lock); + + irq_domain_free_irqs_parent(domain, virq, nr_irqs); +} + +static const struct irq_domain_ops xgene_msi_domain_ops = { + .alloc = xgene_irq_domain_alloc, + .free = xgene_irq_domain_free, +}; + +static const struct msi_parent_ops xgene_msi_parent_ops = { + .supported_flags = (MSI_GENERIC_FLAGS_MASK | + MSI_FLAG_PCI_MSIX), + .required_flags = (MSI_FLAG_USE_DEF_DOM_OPS | + MSI_FLAG_USE_DEF_CHIP_OPS), + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +static int xgene_allocate_domains(struct device_node *node, + struct xgene_msi *msi) +{ + struct irq_domain_info info = { + .fwnode = of_fwnode_handle(node), + .ops = &xgene_msi_domain_ops, + .size = NR_MSI_VEC, + .host_data = msi, + }; + + msi->inner_domain = msi_create_parent_irq_domain(&info, &xgene_msi_parent_ops); + return msi->inner_domain ? 0 : -ENOMEM; +} + +static int xgene_msi_init_allocator(struct device *dev) +{ + xgene_msi_ctrl->bitmap = devm_bitmap_zalloc(dev, NR_MSI_VEC, GFP_KERNEL); + if (!xgene_msi_ctrl->bitmap) + return -ENOMEM; + + mutex_init(&xgene_msi_ctrl->bitmap_lock); + + return 0; +} + +static void xgene_msi_isr(struct irq_desc *desc) +{ + unsigned int *irqp = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + struct xgene_msi *xgene_msi = xgene_msi_ctrl; + unsigned long grp_pending; + int msir_idx; + u32 msi_grp; + + chained_irq_enter(chip, desc); + + msi_grp = irqp - xgene_msi->gic_irq; + + grp_pending = xgene_msi_int_read(xgene_msi, msi_grp); + + for_each_set_bit(msir_idx, &grp_pending, IDX_PER_GROUP) { + unsigned long msir; + int intr_idx; + + msir = xgene_msi_ir_read(xgene_msi, msi_grp, msir_idx); + + for_each_set_bit(intr_idx, &msir, IRQS_PER_IDX) { + irq_hw_number_t hwirq; + int ret; + + hwirq = compute_hwirq(msi_grp, msir_idx, intr_idx); + ret = generic_handle_domain_irq(xgene_msi->inner_domain, + hwirq); + WARN_ON_ONCE(ret); + } + } + + chained_irq_exit(chip, desc); +} + +static void xgene_msi_remove(struct platform_device *pdev) +{ + for (int i = 0; i < NR_HW_IRQS; i++) { + unsigned int irq = xgene_msi_ctrl->gic_irq[i]; + if (!irq) + continue; + irq_set_chained_handler_and_data(irq, NULL, NULL); + } + + if (xgene_msi_ctrl->inner_domain) + irq_domain_remove(xgene_msi_ctrl->inner_domain); +} + +static int xgene_msi_handler_setup(struct platform_device *pdev) +{ + struct xgene_msi *xgene_msi = xgene_msi_ctrl; + int i; + + for (i = 0; i < NR_HW_IRQS; i++) { + u32 msi_val; + int irq, err; + + /* + * MSInIRx registers are read-to-clear; before registering + * interrupt handlers, read all of them to clear spurious + * interrupts that may occur before the driver is probed. + */ + for (int msi_idx = 0; msi_idx < IDX_PER_GROUP; msi_idx++) + xgene_msi_ir_read(xgene_msi, i, msi_idx); + + /* Read MSIINTn to confirm */ + msi_val = xgene_msi_int_read(xgene_msi, i); + if (msi_val) { + dev_err(&pdev->dev, "Failed to clear spurious IRQ\n"); + return -EINVAL; + } + + irq = platform_get_irq(pdev, i); + if (irq < 0) + return irq; + + xgene_msi->gic_irq[i] = irq; + + /* + * Statically allocate MSI GIC IRQs to each CPU core. + * With 8-core X-Gene v1, 2 MSI GIC IRQs are allocated + * to each core. + */ + irq_set_status_flags(irq, IRQ_NO_BALANCING); + err = irq_set_affinity(irq, cpumask_of(i % num_possible_cpus())); + if (err) { + pr_err("failed to set affinity for GIC IRQ"); + return err; + } + + irq_set_chained_handler_and_data(irq, xgene_msi_isr, + &xgene_msi_ctrl->gic_irq[i]); + } + + return 0; +} + +static const struct of_device_id xgene_msi_match_table[] = { + {.compatible = "apm,xgene1-msi"}, + {}, +}; + +static int xgene_msi_probe(struct platform_device *pdev) +{ + struct resource *res; + struct xgene_msi *xgene_msi; + int rc; + + xgene_msi_ctrl = devm_kzalloc(&pdev->dev, sizeof(*xgene_msi_ctrl), + GFP_KERNEL); + if (!xgene_msi_ctrl) + return -ENOMEM; + + xgene_msi = xgene_msi_ctrl; + + xgene_msi->msi_regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res); + if (IS_ERR(xgene_msi->msi_regs)) { + rc = PTR_ERR(xgene_msi->msi_regs); + goto error; + } + xgene_msi->msi_addr = res->start; + + rc = xgene_msi_init_allocator(&pdev->dev); + if (rc) { + dev_err(&pdev->dev, "Error allocating MSI bitmap\n"); + goto error; + } + + rc = xgene_allocate_domains(dev_of_node(&pdev->dev), xgene_msi); + if (rc) { + dev_err(&pdev->dev, "Failed to allocate MSI domain\n"); + goto error; + } + + rc = xgene_msi_handler_setup(pdev); + if (rc) + goto error; + + dev_info(&pdev->dev, "APM X-Gene PCIe MSI driver loaded\n"); + + return 0; +error: + xgene_msi_remove(pdev); + return rc; +} + +static struct platform_driver xgene_msi_driver = { + .driver = { + .name = "xgene-msi", + .of_match_table = xgene_msi_match_table, + }, + .probe = xgene_msi_probe, + .remove = xgene_msi_remove, +}; +builtin_platform_driver(xgene_msi_driver); diff --git a/drivers/pci/controller/pci-xgene.c b/drivers/pci/controller/pci-xgene.c new file mode 100644 index 000000000..83c9a2930 --- /dev/null +++ b/drivers/pci/controller/pci-xgene.c @@ -0,0 +1,662 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * APM X-Gene PCIe Driver + * + * Copyright (c) 2014 Applied Micro Circuits Corporation. + * + * Author: Tanmay Inamdar <tinamdar@apm.com>. + */ +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/io.h> +#include <linux/jiffies.h> +#include <linux/memblock.h> +#include <linux/init.h> +#include <linux/irqdomain.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/pci-acpi.h> +#include <linux/pci-ecam.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +#include "../pci.h" + +#define PCIECORE_CTLANDSTATUS 0x50 +#define PIM1_1L 0x80 +#define IBAR2 0x98 +#define IR2MSK 0x9c +#define PIM2_1L 0xa0 +#define IBAR3L 0xb4 +#define IR3MSKL 0xbc +#define PIM3_1L 0xc4 +#define OMR1BARL 0x100 +#define OMR2BARL 0x118 +#define OMR3BARL 0x130 +#define CFGBARL 0x154 +#define CFGBARH 0x158 +#define CFGCTL 0x15c +#define RTDID 0x160 +#define BRIDGE_CFG_0 0x2000 +#define BRIDGE_CFG_4 0x2010 +#define BRIDGE_STATUS_0 0x2600 + +#define LINK_UP_MASK 0x00000100 +#define AXI_EP_CFG_ACCESS 0x10000 +#define EN_COHERENCY 0xF0000000 +#define EN_REG 0x00000001 +#define OB_LO_IO 0x00000002 +#define XGENE_PCIE_DEVICEID 0xE004 +#define PIPE_PHY_RATE_RD(src) ((0xc000 & (u32)(src)) >> 0xe) + +#define XGENE_V1_PCI_EXP_CAP 0x40 + +/* PCIe IP version */ +#define XGENE_PCIE_IP_VER_1 1 +#define XGENE_PCIE_IP_VER_2 2 + +struct xgene_pcie { + struct device *dev; + struct clk *clk; + void __iomem *csr_base; + void __iomem *cfg_base; + unsigned long cfg_addr; + bool link_up; + u32 version; +}; + +static u32 xgene_pcie_readl(struct xgene_pcie *port, u32 reg) +{ + return readl(port->csr_base + reg); +} + +static void xgene_pcie_writel(struct xgene_pcie *port, u32 reg, u32 val) +{ + writel(val, port->csr_base + reg); +} + +static inline u32 pcie_bar_low_val(u32 addr, u32 flags) +{ + return (addr & PCI_BASE_ADDRESS_MEM_MASK) | flags; +} + +static inline struct xgene_pcie *pcie_bus_to_port(struct pci_bus *bus) +{ + struct pci_config_window *cfg; + + if (acpi_disabled) + return (struct xgene_pcie *)(bus->sysdata); + + cfg = bus->sysdata; + return (struct xgene_pcie *)(cfg->priv); +} + +/* + * When the address bit [17:16] is 2'b01, the Configuration access will be + * treated as Type 1 and it will be forwarded to external PCIe device. + */ +static void __iomem *xgene_pcie_get_cfg_base(struct pci_bus *bus) +{ + struct xgene_pcie *port = pcie_bus_to_port(bus); + + if (bus->number >= (bus->primary + 1)) + return port->cfg_base + AXI_EP_CFG_ACCESS; + + return port->cfg_base; +} + +/* + * For Configuration request, RTDID register is used as Bus Number, + * Device Number and Function number of the header fields. + */ +static void xgene_pcie_set_rtdid_reg(struct pci_bus *bus, uint devfn) +{ + struct xgene_pcie *port = pcie_bus_to_port(bus); + unsigned int b, d, f; + u32 rtdid_val = 0; + + b = bus->number; + d = PCI_SLOT(devfn); + f = PCI_FUNC(devfn); + + if (!pci_is_root_bus(bus)) + rtdid_val = (b << 8) | (d << 3) | f; + + xgene_pcie_writel(port, RTDID, rtdid_val); + /* read the register back to ensure flush */ + xgene_pcie_readl(port, RTDID); +} + +/* + * X-Gene PCIe port uses BAR0-BAR1 of RC's configuration space as + * the translation from PCI bus to native BUS. Entire DDR region + * is mapped into PCIe space using these registers, so it can be + * reached by DMA from EP devices. The BAR0/1 of bridge should be + * hidden during enumeration to avoid the sizing and resource allocation + * by PCIe core. + */ +static bool xgene_pcie_hide_rc_bars(struct pci_bus *bus, int offset) +{ + if (pci_is_root_bus(bus) && ((offset == PCI_BASE_ADDRESS_0) || + (offset == PCI_BASE_ADDRESS_1))) + return true; + + return false; +} + +static void __iomem *xgene_pcie_map_bus(struct pci_bus *bus, unsigned int devfn, + int offset) +{ + if ((pci_is_root_bus(bus) && devfn != 0) || + xgene_pcie_hide_rc_bars(bus, offset)) + return NULL; + + xgene_pcie_set_rtdid_reg(bus, devfn); + return xgene_pcie_get_cfg_base(bus) + offset; +} + +static int xgene_pcie_config_read32(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct xgene_pcie *port = pcie_bus_to_port(bus); + int ret; + + ret = pci_generic_config_read32(bus, devfn, where & ~0x3, 4, val); + if (ret != PCIBIOS_SUCCESSFUL) + return ret; + + /* + * The v1 controller has a bug in its Configuration Request Retry + * Status (RRS) logic: when RRS Software Visibility is enabled and + * we read the Vendor and Device ID of a non-existent device, the + * controller fabricates return data of 0xFFFF0001 ("device exists + * but is not ready") instead of 0xFFFFFFFF (PCI_ERROR_RESPONSE) + * ("device does not exist"). This causes the PCI core to retry + * the read until it times out. Avoid this by not claiming to + * support RRS SV. + */ + if (pci_is_root_bus(bus) && (port->version == XGENE_PCIE_IP_VER_1) && + ((where & ~0x3) == XGENE_V1_PCI_EXP_CAP + PCI_EXP_RTCTL)) + *val &= ~(PCI_EXP_RTCAP_RRS_SV << 16); + + if (size <= 2) + *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); + + return PCIBIOS_SUCCESSFUL; +} + +#if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS) +static int xgene_get_csr_resource(struct acpi_device *adev, + struct resource *res) +{ + struct device *dev = &adev->dev; + struct resource_entry *entry; + struct list_head list; + unsigned long flags; + int ret; + + INIT_LIST_HEAD(&list); + flags = IORESOURCE_MEM; + ret = acpi_dev_get_resources(adev, &list, + acpi_dev_filter_resource_type_cb, + (void *) flags); + if (ret < 0) { + dev_err(dev, "failed to parse _CRS method, error code %d\n", + ret); + return ret; + } + + if (ret == 0) { + dev_err(dev, "no IO and memory resources present in _CRS\n"); + return -EINVAL; + } + + entry = list_first_entry(&list, struct resource_entry, node); + *res = *entry->res; + acpi_dev_free_resource_list(&list); + return 0; +} + +static int xgene_pcie_ecam_init(struct pci_config_window *cfg, u32 ipversion) +{ + struct device *dev = cfg->parent; + struct acpi_device *adev = to_acpi_device(dev); + struct xgene_pcie *port; + struct resource csr; + int ret; + + port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); + if (!port) + return -ENOMEM; + + ret = xgene_get_csr_resource(adev, &csr); + if (ret) { + dev_err(dev, "can't get CSR resource\n"); + return ret; + } + port->csr_base = devm_pci_remap_cfg_resource(dev, &csr); + if (IS_ERR(port->csr_base)) + return PTR_ERR(port->csr_base); + + port->cfg_base = cfg->win; + port->version = ipversion; + + cfg->priv = port; + return 0; +} + +static int xgene_v1_pcie_ecam_init(struct pci_config_window *cfg) +{ + return xgene_pcie_ecam_init(cfg, XGENE_PCIE_IP_VER_1); +} + +const struct pci_ecam_ops xgene_v1_pcie_ecam_ops = { + .init = xgene_v1_pcie_ecam_init, + .pci_ops = { + .map_bus = xgene_pcie_map_bus, + .read = xgene_pcie_config_read32, + .write = pci_generic_config_write, + } +}; + +static int xgene_v2_pcie_ecam_init(struct pci_config_window *cfg) +{ + return xgene_pcie_ecam_init(cfg, XGENE_PCIE_IP_VER_2); +} + +const struct pci_ecam_ops xgene_v2_pcie_ecam_ops = { + .init = xgene_v2_pcie_ecam_init, + .pci_ops = { + .map_bus = xgene_pcie_map_bus, + .read = xgene_pcie_config_read32, + .write = pci_generic_config_write, + } +}; +#endif + +static u64 xgene_pcie_set_ib_mask(struct xgene_pcie *port, u32 addr, + u32 flags, u64 size) +{ + u64 mask = (~(size - 1) & PCI_BASE_ADDRESS_MEM_MASK) | flags; + u32 val32 = 0; + u32 val; + + val32 = xgene_pcie_readl(port, addr); + val = (val32 & 0x0000ffff) | (lower_32_bits(mask) << 16); + xgene_pcie_writel(port, addr, val); + + val32 = xgene_pcie_readl(port, addr + 0x04); + val = (val32 & 0xffff0000) | (lower_32_bits(mask) >> 16); + xgene_pcie_writel(port, addr + 0x04, val); + + val32 = xgene_pcie_readl(port, addr + 0x04); + val = (val32 & 0x0000ffff) | (upper_32_bits(mask) << 16); + xgene_pcie_writel(port, addr + 0x04, val); + + val32 = xgene_pcie_readl(port, addr + 0x08); + val = (val32 & 0xffff0000) | (upper_32_bits(mask) >> 16); + xgene_pcie_writel(port, addr + 0x08, val); + + return mask; +} + +static void xgene_pcie_linkup(struct xgene_pcie *port, + u32 *lanes, u32 *speed) +{ + u32 val32; + + port->link_up = false; + val32 = xgene_pcie_readl(port, PCIECORE_CTLANDSTATUS); + if (val32 & LINK_UP_MASK) { + port->link_up = true; + *speed = PIPE_PHY_RATE_RD(val32); + val32 = xgene_pcie_readl(port, BRIDGE_STATUS_0); + *lanes = val32 >> 26; + } +} + +static int xgene_pcie_init_port(struct xgene_pcie *port) +{ + struct device *dev = port->dev; + int rc; + + port->clk = clk_get(dev, NULL); + if (IS_ERR(port->clk)) { + dev_err(dev, "clock not available\n"); + return -ENODEV; + } + + rc = clk_prepare_enable(port->clk); + if (rc) { + dev_err(dev, "clock enable failed\n"); + return rc; + } + + return 0; +} + +static int xgene_pcie_map_reg(struct xgene_pcie *port, + struct platform_device *pdev) +{ + struct device *dev = port->dev; + struct resource *res; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "csr"); + port->csr_base = devm_pci_remap_cfg_resource(dev, res); + if (IS_ERR(port->csr_base)) + return PTR_ERR(port->csr_base); + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg"); + port->cfg_base = devm_ioremap_resource(dev, res); + if (IS_ERR(port->cfg_base)) + return PTR_ERR(port->cfg_base); + port->cfg_addr = res->start; + + return 0; +} + +static void xgene_pcie_setup_ob_reg(struct xgene_pcie *port, + struct resource *res, u32 offset, + u64 cpu_addr, u64 pci_addr) +{ + struct device *dev = port->dev; + resource_size_t size = resource_size(res); + u64 restype = resource_type(res); + u64 mask = 0; + u32 min_size; + u32 flag = EN_REG; + + if (restype == IORESOURCE_MEM) { + min_size = SZ_128M; + } else { + min_size = 128; + flag |= OB_LO_IO; + } + + if (size >= min_size) + mask = ~(size - 1) | flag; + else + dev_warn(dev, "res size 0x%llx less than minimum 0x%x\n", + (u64)size, min_size); + + xgene_pcie_writel(port, offset, lower_32_bits(cpu_addr)); + xgene_pcie_writel(port, offset + 0x04, upper_32_bits(cpu_addr)); + xgene_pcie_writel(port, offset + 0x08, lower_32_bits(mask)); + xgene_pcie_writel(port, offset + 0x0c, upper_32_bits(mask)); + xgene_pcie_writel(port, offset + 0x10, lower_32_bits(pci_addr)); + xgene_pcie_writel(port, offset + 0x14, upper_32_bits(pci_addr)); +} + +static void xgene_pcie_setup_cfg_reg(struct xgene_pcie *port) +{ + u64 addr = port->cfg_addr; + + xgene_pcie_writel(port, CFGBARL, lower_32_bits(addr)); + xgene_pcie_writel(port, CFGBARH, upper_32_bits(addr)); + xgene_pcie_writel(port, CFGCTL, EN_REG); +} + +static int xgene_pcie_map_ranges(struct xgene_pcie *port) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(port); + struct resource_entry *window; + struct device *dev = port->dev; + + resource_list_for_each_entry(window, &bridge->windows) { + struct resource *res = window->res; + u64 restype = resource_type(res); + + dev_dbg(dev, "%pR\n", res); + + switch (restype) { + case IORESOURCE_IO: + xgene_pcie_setup_ob_reg(port, res, OMR3BARL, + pci_pio_to_address(res->start), + res->start - window->offset); + break; + case IORESOURCE_MEM: + if (res->flags & IORESOURCE_PREFETCH) + xgene_pcie_setup_ob_reg(port, res, OMR2BARL, + res->start, + res->start - + window->offset); + else + xgene_pcie_setup_ob_reg(port, res, OMR1BARL, + res->start, + res->start - + window->offset); + break; + case IORESOURCE_BUS: + break; + default: + dev_err(dev, "invalid resource %pR\n", res); + return -EINVAL; + } + } + xgene_pcie_setup_cfg_reg(port); + return 0; +} + +static void xgene_pcie_setup_pims(struct xgene_pcie *port, u32 pim_reg, + u64 pim, u64 size) +{ + xgene_pcie_writel(port, pim_reg, lower_32_bits(pim)); + xgene_pcie_writel(port, pim_reg + 0x04, + upper_32_bits(pim) | EN_COHERENCY); + xgene_pcie_writel(port, pim_reg + 0x10, lower_32_bits(size)); + xgene_pcie_writel(port, pim_reg + 0x14, upper_32_bits(size)); +} + +/* + * X-Gene PCIe support maximum 3 inbound memory regions + * This function helps to select a region based on size of region + */ +static int xgene_pcie_select_ib_reg(u8 *ib_reg_mask, u64 size) +{ + if ((size > 4) && (size < SZ_16M) && !(*ib_reg_mask & (1 << 1))) { + *ib_reg_mask |= (1 << 1); + return 1; + } + + if ((size > SZ_1K) && (size < SZ_1T) && !(*ib_reg_mask & (1 << 0))) { + *ib_reg_mask |= (1 << 0); + return 0; + } + + if ((size > SZ_1M) && (size < SZ_1T) && !(*ib_reg_mask & (1 << 2))) { + *ib_reg_mask |= (1 << 2); + return 2; + } + + return -EINVAL; +} + +static void xgene_pcie_setup_ib_reg(struct xgene_pcie *port, + struct of_pci_range *range, u8 *ib_reg_mask) +{ + void __iomem *cfg_base = port->cfg_base; + struct device *dev = port->dev; + void __iomem *bar_addr; + u32 pim_reg; + u64 cpu_addr = range->cpu_addr; + u64 pci_addr = range->pci_addr; + u64 size = range->size; + u64 mask = ~(size - 1) | EN_REG; + u32 flags = PCI_BASE_ADDRESS_MEM_TYPE_64; + u32 bar_low; + int region; + + region = xgene_pcie_select_ib_reg(ib_reg_mask, range->size); + if (region < 0) { + dev_warn(dev, "invalid pcie dma-range config\n"); + return; + } + + if (range->flags & IORESOURCE_PREFETCH) + flags |= PCI_BASE_ADDRESS_MEM_PREFETCH; + + bar_low = pcie_bar_low_val((u32)cpu_addr, flags); + switch (region) { + case 0: + xgene_pcie_set_ib_mask(port, BRIDGE_CFG_4, flags, size); + bar_addr = cfg_base + PCI_BASE_ADDRESS_0; + writel(bar_low, bar_addr); + writel(upper_32_bits(cpu_addr), bar_addr + 0x4); + pim_reg = PIM1_1L; + break; + case 1: + xgene_pcie_writel(port, IBAR2, bar_low); + xgene_pcie_writel(port, IR2MSK, lower_32_bits(mask)); + pim_reg = PIM2_1L; + break; + case 2: + xgene_pcie_writel(port, IBAR3L, bar_low); + xgene_pcie_writel(port, IBAR3L + 0x4, upper_32_bits(cpu_addr)); + xgene_pcie_writel(port, IR3MSKL, lower_32_bits(mask)); + xgene_pcie_writel(port, IR3MSKL + 0x4, upper_32_bits(mask)); + pim_reg = PIM3_1L; + break; + } + + xgene_pcie_setup_pims(port, pim_reg, pci_addr, ~(size - 1)); +} + +static int xgene_pcie_parse_map_dma_ranges(struct xgene_pcie *port) +{ + struct device_node *np = port->dev->of_node; + struct of_pci_range range; + struct of_pci_range_parser parser; + struct device *dev = port->dev; + u8 ib_reg_mask = 0; + + if (of_pci_dma_range_parser_init(&parser, np)) { + dev_err(dev, "missing dma-ranges property\n"); + return -EINVAL; + } + + /* Get the dma-ranges from DT */ + for_each_of_pci_range(&parser, &range) { + u64 end = range.cpu_addr + range.size - 1; + + dev_dbg(dev, "0x%08x 0x%016llx..0x%016llx -> 0x%016llx\n", + range.flags, range.cpu_addr, end, range.pci_addr); + xgene_pcie_setup_ib_reg(port, &range, &ib_reg_mask); + } + return 0; +} + +/* clear BAR configuration which was done by firmware */ +static void xgene_pcie_clear_config(struct xgene_pcie *port) +{ + int i; + + for (i = PIM1_1L; i <= CFGCTL; i += 4) + xgene_pcie_writel(port, i, 0); +} + +static int xgene_pcie_setup(struct xgene_pcie *port) +{ + struct device *dev = port->dev; + u32 val, lanes = 0, speed = 0; + int ret; + + xgene_pcie_clear_config(port); + + /* setup the vendor and device IDs correctly */ + val = (XGENE_PCIE_DEVICEID << 16) | PCI_VENDOR_ID_AMCC; + xgene_pcie_writel(port, BRIDGE_CFG_0, val); + + ret = xgene_pcie_map_ranges(port); + if (ret) + return ret; + + ret = xgene_pcie_parse_map_dma_ranges(port); + if (ret) + return ret; + + xgene_pcie_linkup(port, &lanes, &speed); + if (!port->link_up) + dev_info(dev, "(rc) link down\n"); + else + dev_info(dev, "(rc) x%d gen-%d link up\n", lanes, speed + 1); + return 0; +} + +static struct pci_ops xgene_pcie_ops = { + .map_bus = xgene_pcie_map_bus, + .read = xgene_pcie_config_read32, + .write = pci_generic_config_write32, +}; + +static bool xgene_check_pcie_msi_ready(void) +{ + struct device_node *np; + struct irq_domain *d; + + if (!IS_ENABLED(CONFIG_PCI_XGENE_MSI)) + return true; + + np = of_find_compatible_node(NULL, NULL, "apm,xgene1-msi"); + if (!np) + return true; + + d = irq_find_matching_host(np, DOMAIN_BUS_PCI_MSI); + of_node_put(np); + + return d && irq_domain_is_msi_parent(d); +} + +static int xgene_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct xgene_pcie *port; + struct pci_host_bridge *bridge; + int ret; + + if (!xgene_check_pcie_msi_ready()) + return dev_err_probe(&pdev->dev, -EPROBE_DEFER, + "MSI driver not ready\n"); + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*port)); + if (!bridge) + return -ENOMEM; + + port = pci_host_bridge_priv(bridge); + + port->dev = dev; + port->version = XGENE_PCIE_IP_VER_1; + + ret = xgene_pcie_map_reg(port, pdev); + if (ret) + return ret; + + ret = xgene_pcie_init_port(port); + if (ret) + return ret; + + ret = xgene_pcie_setup(port); + if (ret) + return ret; + + bridge->sysdata = port; + bridge->ops = &xgene_pcie_ops; + + return pci_host_probe(bridge); +} + +static const struct of_device_id xgene_pcie_match_table[] = { + {.compatible = "apm,xgene-pcie",}, + {}, +}; + +static struct platform_driver xgene_pcie_driver = { + .driver = { + .name = "xgene-pcie", + .of_match_table = xgene_pcie_match_table, + .suppress_bind_attrs = true, + }, + .probe = xgene_pcie_probe, +}; +builtin_platform_driver(xgene_pcie_driver); diff --git a/drivers/pci/controller/pcie-altera-msi.c b/drivers/pci/controller/pcie-altera-msi.c new file mode 100644 index 000000000..ea2ca2e70 --- /dev/null +++ b/drivers/pci/controller/pcie-altera-msi.c @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Altera PCIe MSI support + * + * Author: Ley Foon Tan <lftan@altera.com> + * + * Copyright Altera Corporation (C) 2013-2015. All rights reserved + */ + +#include <linux/interrupt.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/irqdomain.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +#define MSI_STATUS 0x0 +#define MSI_ERROR 0x4 +#define MSI_INTMASK 0x8 + +#define MAX_MSI_VECTORS 32 + +struct altera_msi { + DECLARE_BITMAP(used, MAX_MSI_VECTORS); + struct mutex lock; /* protect "used" bitmap */ + struct platform_device *pdev; + struct irq_domain *inner_domain; + void __iomem *csr_base; + void __iomem *vector_base; + phys_addr_t vector_phy; + u32 num_of_vectors; + int irq; +}; + +static inline void msi_writel(struct altera_msi *msi, const u32 value, + const u32 reg) +{ + writel_relaxed(value, msi->csr_base + reg); +} + +static inline u32 msi_readl(struct altera_msi *msi, const u32 reg) +{ + return readl_relaxed(msi->csr_base + reg); +} + +static void altera_msi_isr(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + struct altera_msi *msi; + unsigned long status; + u32 bit; + int ret; + + chained_irq_enter(chip, desc); + msi = irq_desc_get_handler_data(desc); + + while ((status = msi_readl(msi, MSI_STATUS)) != 0) { + for_each_set_bit(bit, &status, msi->num_of_vectors) { + /* Dummy read from vector to clear the interrupt */ + readl_relaxed(msi->vector_base + (bit * sizeof(u32))); + + ret = generic_handle_domain_irq(msi->inner_domain, bit); + if (ret) + dev_err_ratelimited(&msi->pdev->dev, "unexpected MSI\n"); + } + } + + chained_irq_exit(chip, desc); +} + +#define ALTERA_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_NO_AFFINITY) + +#define ALTERA_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_PCI_MSIX) + +static const struct msi_parent_ops altera_msi_parent_ops = { + .required_flags = ALTERA_MSI_FLAGS_REQUIRED, + .supported_flags = ALTERA_MSI_FLAGS_SUPPORTED, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .prefix = "Altera-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; +static void altera_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct altera_msi *msi = irq_data_get_irq_chip_data(data); + phys_addr_t addr = msi->vector_phy + (data->hwirq * sizeof(u32)); + + msg->address_lo = lower_32_bits(addr); + msg->address_hi = upper_32_bits(addr); + msg->data = data->hwirq; + + dev_dbg(&msi->pdev->dev, "msi#%d address_hi %#x address_lo %#x\n", + (int)data->hwirq, msg->address_hi, msg->address_lo); +} + +static struct irq_chip altera_msi_bottom_irq_chip = { + .name = "Altera MSI", + .irq_compose_msi_msg = altera_compose_msi_msg, +}; + +static int altera_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct altera_msi *msi = domain->host_data; + unsigned long bit; + u32 mask; + + WARN_ON(nr_irqs != 1); + mutex_lock(&msi->lock); + + bit = find_first_zero_bit(msi->used, msi->num_of_vectors); + if (bit >= msi->num_of_vectors) { + mutex_unlock(&msi->lock); + return -ENOSPC; + } + + set_bit(bit, msi->used); + + mutex_unlock(&msi->lock); + + irq_domain_set_info(domain, virq, bit, &altera_msi_bottom_irq_chip, + domain->host_data, handle_simple_irq, + NULL, NULL); + + mask = msi_readl(msi, MSI_INTMASK); + mask |= 1 << bit; + msi_writel(msi, mask, MSI_INTMASK); + + return 0; +} + +static void altera_irq_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct altera_msi *msi = irq_data_get_irq_chip_data(d); + u32 mask; + + mutex_lock(&msi->lock); + + if (!test_bit(d->hwirq, msi->used)) { + dev_err(&msi->pdev->dev, "trying to free unused MSI#%lu\n", + d->hwirq); + } else { + __clear_bit(d->hwirq, msi->used); + mask = msi_readl(msi, MSI_INTMASK); + mask &= ~(1 << d->hwirq); + msi_writel(msi, mask, MSI_INTMASK); + } + + mutex_unlock(&msi->lock); +} + +static const struct irq_domain_ops msi_domain_ops = { + .alloc = altera_irq_domain_alloc, + .free = altera_irq_domain_free, +}; + +static int altera_allocate_domains(struct altera_msi *msi) +{ + struct irq_domain_info info = { + .fwnode = dev_fwnode(&msi->pdev->dev), + .ops = &msi_domain_ops, + .host_data = msi, + .size = msi->num_of_vectors, + }; + + msi->inner_domain = msi_create_parent_irq_domain(&info, &altera_msi_parent_ops); + if (!msi->inner_domain) { + dev_err(&msi->pdev->dev, "failed to create MSI domain\n"); + return -ENOMEM; + } + + return 0; +} + +static void altera_free_domains(struct altera_msi *msi) +{ + irq_domain_remove(msi->inner_domain); +} + +static void altera_msi_remove(struct platform_device *pdev) +{ + struct altera_msi *msi = platform_get_drvdata(pdev); + + msi_writel(msi, 0, MSI_INTMASK); + irq_set_chained_handler_and_data(msi->irq, NULL, NULL); + + altera_free_domains(msi); + + platform_set_drvdata(pdev, NULL); +} + +static int altera_msi_probe(struct platform_device *pdev) +{ + struct altera_msi *msi; + struct device_node *np = pdev->dev.of_node; + struct resource *res; + int ret; + + msi = devm_kzalloc(&pdev->dev, sizeof(struct altera_msi), + GFP_KERNEL); + if (!msi) + return -ENOMEM; + + mutex_init(&msi->lock); + msi->pdev = pdev; + + msi->csr_base = devm_platform_ioremap_resource_byname(pdev, "csr"); + if (IS_ERR(msi->csr_base)) { + dev_err(&pdev->dev, "failed to map csr memory\n"); + return PTR_ERR(msi->csr_base); + } + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, + "vector_slave"); + msi->vector_base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(msi->vector_base)) + return PTR_ERR(msi->vector_base); + + msi->vector_phy = res->start; + + if (of_property_read_u32(np, "num-vectors", &msi->num_of_vectors)) { + dev_err(&pdev->dev, "failed to parse the number of vectors\n"); + return -EINVAL; + } + + ret = altera_allocate_domains(msi); + if (ret) + return ret; + + msi->irq = platform_get_irq(pdev, 0); + if (msi->irq < 0) { + ret = msi->irq; + goto err; + } + + irq_set_chained_handler_and_data(msi->irq, altera_msi_isr, msi); + platform_set_drvdata(pdev, msi); + + return 0; + +err: + altera_msi_remove(pdev); + return ret; +} + +static const struct of_device_id altera_msi_of_match[] = { + { .compatible = "altr,msi-1.0", NULL }, + { }, +}; + +static struct platform_driver altera_msi_driver = { + .driver = { + .name = "altera-msi", + .of_match_table = altera_msi_of_match, + }, + .probe = altera_msi_probe, + .remove = altera_msi_remove, +}; + +static int __init altera_msi_init(void) +{ + return platform_driver_register(&altera_msi_driver); +} + +static void __exit altera_msi_exit(void) +{ + platform_driver_unregister(&altera_msi_driver); +} + +subsys_initcall(altera_msi_init); +MODULE_DEVICE_TABLE(of, altera_msi_of_match); +module_exit(altera_msi_exit); +MODULE_DESCRIPTION("Altera PCIe MSI support driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pcie-altera.c b/drivers/pci/controller/pcie-altera.c new file mode 100644 index 000000000..8ef56e2fa --- /dev/null +++ b/drivers/pci/controller/pcie-altera.c @@ -0,0 +1,1097 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright Altera Corporation (C) 2013-2015. All rights reserved + * + * Author: Ley Foon Tan <lftan@altera.com> + * Description: Altera PCIe host controller driver + */ + +#include <linux/bitfield.h> +#include <linux/delay.h> +#include <linux/interrupt.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqdomain.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +#include "../pci.h" + +#define RP_TX_REG0 0x2000 +#define RP_TX_REG1 0x2004 +#define RP_TX_CNTRL 0x2008 +#define RP_TX_EOP 0x2 +#define RP_TX_SOP 0x1 +#define RP_RXCPL_STATUS 0x2010 +#define RP_RXCPL_EOP 0x2 +#define RP_RXCPL_SOP 0x1 +#define RP_RXCPL_REG0 0x2014 +#define RP_RXCPL_REG1 0x2018 +#define P2A_INT_STATUS 0x3060 +#define P2A_INT_STS_ALL 0xf +#define P2A_INT_ENABLE 0x3070 +#define P2A_INT_ENA_ALL 0xf +#define RP_LTSSM 0x3c64 +#define RP_LTSSM_MASK 0x1f +#define LTSSM_L0 0xf + +#define S10_RP_TX_CNTRL 0x2004 +#define S10_RP_RXCPL_REG 0x2008 +#define S10_RP_RXCPL_STATUS 0x200C +#define S10_RP_CFG_ADDR(pcie, reg) \ + (((pcie)->hip_base) + (reg) + (1 << 20)) +#define S10_RP_SECONDARY(pcie) \ + readb(S10_RP_CFG_ADDR(pcie, PCI_SECONDARY_BUS)) + +/* TLP configuration type 0 and 1 */ +#define TLP_FMTTYPE_CFGRD0 0x04 /* Configuration Read Type 0 */ +#define TLP_FMTTYPE_CFGWR0 0x44 /* Configuration Write Type 0 */ +#define TLP_FMTTYPE_CFGRD1 0x05 /* Configuration Read Type 1 */ +#define TLP_FMTTYPE_CFGWR1 0x45 /* Configuration Write Type 1 */ +#define TLP_PAYLOAD_SIZE 0x01 +#define TLP_READ_TAG 0x1d +#define TLP_WRITE_TAG 0x10 +#define RP_DEVFN 0 +#define TLP_CFG_DW0(pcie, cfg) \ + (((cfg) << 24) | \ + TLP_PAYLOAD_SIZE) +#define TLP_CFG_DW1(pcie, tag, be) \ + (((PCI_DEVID(pcie->root_bus_nr, RP_DEVFN)) << 16) | (tag << 8) | (be)) +#define TLP_CFG_DW2(bus, devfn, offset) \ + (((bus) << 24) | ((devfn) << 16) | (offset)) +#define TLP_COMP_STATUS(s) (((s) >> 13) & 7) +#define TLP_BYTE_COUNT(s) (((s) >> 0) & 0xfff) +#define TLP_HDR_SIZE 3 +#define TLP_LOOP 500 + +#define LINK_UP_TIMEOUT HZ +#define LINK_RETRAIN_TIMEOUT HZ + +#define DWORD_MASK 3 + +#define S10_TLP_FMTTYPE_CFGRD0 0x05 +#define S10_TLP_FMTTYPE_CFGRD1 0x04 +#define S10_TLP_FMTTYPE_CFGWR0 0x45 +#define S10_TLP_FMTTYPE_CFGWR1 0x44 + +#define AGLX_RP_CFG_ADDR(pcie, reg) (((pcie)->hip_base) + (reg)) +#define AGLX_RP_SECONDARY(pcie) \ + readb(AGLX_RP_CFG_ADDR(pcie, PCI_SECONDARY_BUS)) + +#define AGLX_BDF_REG 0x00002004 +#define AGLX_ROOT_PORT_IRQ_STATUS 0x14c +#define AGLX_ROOT_PORT_IRQ_ENABLE 0x150 +#define CFG_AER BIT(4) + +#define AGLX_CFG_TARGET GENMASK(13, 12) +#define AGLX_CFG_TARGET_TYPE0 0 +#define AGLX_CFG_TARGET_TYPE1 1 +#define AGLX_CFG_TARGET_LOCAL_2000 2 +#define AGLX_CFG_TARGET_LOCAL_3000 3 + +enum altera_pcie_version { + ALTERA_PCIE_V1 = 0, + ALTERA_PCIE_V2, + ALTERA_PCIE_V3, +}; + +struct altera_pcie { + struct platform_device *pdev; + void __iomem *cra_base; + void __iomem *hip_base; + int irq; + u8 root_bus_nr; + struct irq_domain *irq_domain; + struct resource bus_range; + const struct altera_pcie_data *pcie_data; +}; + +struct altera_pcie_ops { + int (*tlp_read_pkt)(struct altera_pcie *pcie, u32 *value); + void (*tlp_write_pkt)(struct altera_pcie *pcie, u32 *headers, + u32 data, bool align); + bool (*get_link_status)(struct altera_pcie *pcie); + int (*rp_read_cfg)(struct altera_pcie *pcie, int where, + int size, u32 *value); + int (*rp_write_cfg)(struct altera_pcie *pcie, u8 busno, + int where, int size, u32 value); + int (*ep_read_cfg)(struct altera_pcie *pcie, u8 busno, + unsigned int devfn, int where, int size, u32 *value); + int (*ep_write_cfg)(struct altera_pcie *pcie, u8 busno, + unsigned int devfn, int where, int size, u32 value); + void (*rp_isr)(struct irq_desc *desc); +}; + +struct altera_pcie_data { + const struct altera_pcie_ops *ops; + enum altera_pcie_version version; + u32 cap_offset; /* PCIe capability structure register offset */ + u32 cfgrd0; + u32 cfgrd1; + u32 cfgwr0; + u32 cfgwr1; + u32 port_conf_offset; + u32 port_irq_status_offset; + u32 port_irq_enable_offset; +}; + +struct tlp_rp_regpair_t { + u32 ctrl; + u32 reg0; + u32 reg1; +}; + +static inline void cra_writel(struct altera_pcie *pcie, const u32 value, + const u32 reg) +{ + writel_relaxed(value, pcie->cra_base + reg); +} + +static inline u32 cra_readl(struct altera_pcie *pcie, const u32 reg) +{ + return readl_relaxed(pcie->cra_base + reg); +} + +static inline void cra_writew(struct altera_pcie *pcie, const u32 value, + const u32 reg) +{ + writew_relaxed(value, pcie->cra_base + reg); +} + +static inline u32 cra_readw(struct altera_pcie *pcie, const u32 reg) +{ + return readw_relaxed(pcie->cra_base + reg); +} + +static inline void cra_writeb(struct altera_pcie *pcie, const u32 value, + const u32 reg) +{ + writeb_relaxed(value, pcie->cra_base + reg); +} + +static inline u32 cra_readb(struct altera_pcie *pcie, const u32 reg) +{ + return readb_relaxed(pcie->cra_base + reg); +} + +static bool altera_pcie_link_up(struct altera_pcie *pcie) +{ + return !!((cra_readl(pcie, RP_LTSSM) & RP_LTSSM_MASK) == LTSSM_L0); +} + +static bool s10_altera_pcie_link_up(struct altera_pcie *pcie) +{ + void __iomem *addr = S10_RP_CFG_ADDR(pcie, + pcie->pcie_data->cap_offset + + PCI_EXP_LNKSTA); + + return !!(readw(addr) & PCI_EXP_LNKSTA_DLLLA); +} + +static bool aglx_altera_pcie_link_up(struct altera_pcie *pcie) +{ + void __iomem *addr = AGLX_RP_CFG_ADDR(pcie, + pcie->pcie_data->cap_offset + + PCI_EXP_LNKSTA); + + return (readw_relaxed(addr) & PCI_EXP_LNKSTA_DLLLA); +} + +/* + * Altera PCIe port uses BAR0 of RC's configuration space as the translation + * from PCI bus to native BUS. Entire DDR region is mapped into PCIe space + * using these registers, so it can be reached by DMA from EP devices. + * This BAR0 will also access to MSI vector when receiving MSI/MSI-X interrupt + * from EP devices, eventually trigger interrupt to GIC. The BAR0 of bridge + * should be hidden during enumeration to avoid the sizing and resource + * allocation by PCIe core. + */ +static bool altera_pcie_hide_rc_bar(struct pci_bus *bus, unsigned int devfn, + int offset) +{ + if (pci_is_root_bus(bus) && (devfn == 0) && + (offset == PCI_BASE_ADDRESS_0)) + return true; + + return false; +} + +static void tlp_write_tx(struct altera_pcie *pcie, + struct tlp_rp_regpair_t *tlp_rp_regdata) +{ + cra_writel(pcie, tlp_rp_regdata->reg0, RP_TX_REG0); + cra_writel(pcie, tlp_rp_regdata->reg1, RP_TX_REG1); + cra_writel(pcie, tlp_rp_regdata->ctrl, RP_TX_CNTRL); +} + +static void s10_tlp_write_tx(struct altera_pcie *pcie, u32 reg0, u32 ctrl) +{ + cra_writel(pcie, reg0, RP_TX_REG0); + cra_writel(pcie, ctrl, S10_RP_TX_CNTRL); +} + +static bool altera_pcie_valid_device(struct altera_pcie *pcie, + struct pci_bus *bus, int dev) +{ + /* If there is no link, then there is no device */ + if (bus->number != pcie->root_bus_nr) { + if (!pcie->pcie_data->ops->get_link_status(pcie)) + return false; + } + + /* access only one slot on each root port */ + if (bus->number == pcie->root_bus_nr && dev > 0) + return false; + + return true; +} + +static int tlp_read_packet(struct altera_pcie *pcie, u32 *value) +{ + int i; + bool sop = false; + u32 ctrl; + u32 reg0, reg1; + u32 comp_status = 1; + + /* + * Minimum 2 loops to read TLP headers and 1 loop to read data + * payload. + */ + for (i = 0; i < TLP_LOOP; i++) { + ctrl = cra_readl(pcie, RP_RXCPL_STATUS); + if ((ctrl & RP_RXCPL_SOP) || (ctrl & RP_RXCPL_EOP) || sop) { + reg0 = cra_readl(pcie, RP_RXCPL_REG0); + reg1 = cra_readl(pcie, RP_RXCPL_REG1); + + if (ctrl & RP_RXCPL_SOP) { + sop = true; + comp_status = TLP_COMP_STATUS(reg1); + } + + if (ctrl & RP_RXCPL_EOP) { + if (comp_status) + return PCIBIOS_DEVICE_NOT_FOUND; + + if (value) + *value = reg0; + + return PCIBIOS_SUCCESSFUL; + } + } + udelay(5); + } + + return PCIBIOS_DEVICE_NOT_FOUND; +} + +static int s10_tlp_read_packet(struct altera_pcie *pcie, u32 *value) +{ + u32 ctrl; + u32 comp_status; + u32 dw[4]; + u32 count; + struct device *dev = &pcie->pdev->dev; + + for (count = 0; count < TLP_LOOP; count++) { + ctrl = cra_readl(pcie, S10_RP_RXCPL_STATUS); + if (ctrl & RP_RXCPL_SOP) { + /* Read first DW */ + dw[0] = cra_readl(pcie, S10_RP_RXCPL_REG); + break; + } + + udelay(5); + } + + /* SOP detection failed, return error */ + if (count == TLP_LOOP) + return PCIBIOS_DEVICE_NOT_FOUND; + + count = 1; + + /* Poll for EOP */ + while (count < ARRAY_SIZE(dw)) { + ctrl = cra_readl(pcie, S10_RP_RXCPL_STATUS); + dw[count++] = cra_readl(pcie, S10_RP_RXCPL_REG); + if (ctrl & RP_RXCPL_EOP) { + comp_status = TLP_COMP_STATUS(dw[1]); + if (comp_status) + return PCIBIOS_DEVICE_NOT_FOUND; + + if (value && TLP_BYTE_COUNT(dw[1]) == sizeof(u32) && + count == 4) + *value = dw[3]; + + return PCIBIOS_SUCCESSFUL; + } + } + + dev_warn(dev, "Malformed TLP packet\n"); + + return PCIBIOS_DEVICE_NOT_FOUND; +} + +static void tlp_write_packet(struct altera_pcie *pcie, u32 *headers, + u32 data, bool align) +{ + struct tlp_rp_regpair_t tlp_rp_regdata; + + tlp_rp_regdata.reg0 = headers[0]; + tlp_rp_regdata.reg1 = headers[1]; + tlp_rp_regdata.ctrl = RP_TX_SOP; + tlp_write_tx(pcie, &tlp_rp_regdata); + + if (align) { + tlp_rp_regdata.reg0 = headers[2]; + tlp_rp_regdata.reg1 = 0; + tlp_rp_regdata.ctrl = 0; + tlp_write_tx(pcie, &tlp_rp_regdata); + + tlp_rp_regdata.reg0 = data; + tlp_rp_regdata.reg1 = 0; + } else { + tlp_rp_regdata.reg0 = headers[2]; + tlp_rp_regdata.reg1 = data; + } + + tlp_rp_regdata.ctrl = RP_TX_EOP; + tlp_write_tx(pcie, &tlp_rp_regdata); +} + +static void s10_tlp_write_packet(struct altera_pcie *pcie, u32 *headers, + u32 data, bool dummy) +{ + s10_tlp_write_tx(pcie, headers[0], RP_TX_SOP); + s10_tlp_write_tx(pcie, headers[1], 0); + s10_tlp_write_tx(pcie, headers[2], 0); + s10_tlp_write_tx(pcie, data, RP_TX_EOP); +} + +static void get_tlp_header(struct altera_pcie *pcie, u8 bus, u32 devfn, + int where, u8 byte_en, bool read, u32 *headers) +{ + u8 cfg; + u8 cfg0 = read ? pcie->pcie_data->cfgrd0 : pcie->pcie_data->cfgwr0; + u8 cfg1 = read ? pcie->pcie_data->cfgrd1 : pcie->pcie_data->cfgwr1; + u8 tag = read ? TLP_READ_TAG : TLP_WRITE_TAG; + + if (pcie->pcie_data->version == ALTERA_PCIE_V1) + cfg = (bus == pcie->root_bus_nr) ? cfg0 : cfg1; + else + cfg = (bus > S10_RP_SECONDARY(pcie)) ? cfg0 : cfg1; + + headers[0] = TLP_CFG_DW0(pcie, cfg); + headers[1] = TLP_CFG_DW1(pcie, tag, byte_en); + headers[2] = TLP_CFG_DW2(bus, devfn, where); +} + +static int tlp_cfg_dword_read(struct altera_pcie *pcie, u8 bus, u32 devfn, + int where, u8 byte_en, u32 *value) +{ + u32 headers[TLP_HDR_SIZE]; + + get_tlp_header(pcie, bus, devfn, where, byte_en, true, + headers); + + pcie->pcie_data->ops->tlp_write_pkt(pcie, headers, 0, false); + + return pcie->pcie_data->ops->tlp_read_pkt(pcie, value); +} + +static int tlp_cfg_dword_write(struct altera_pcie *pcie, u8 bus, u32 devfn, + int where, u8 byte_en, u32 value) +{ + u32 headers[TLP_HDR_SIZE]; + int ret; + + get_tlp_header(pcie, bus, devfn, where, byte_en, false, + headers); + + /* check alignment to Qword */ + if ((where & 0x7) == 0) + pcie->pcie_data->ops->tlp_write_pkt(pcie, headers, + value, true); + else + pcie->pcie_data->ops->tlp_write_pkt(pcie, headers, + value, false); + + ret = pcie->pcie_data->ops->tlp_read_pkt(pcie, NULL); + if (ret != PCIBIOS_SUCCESSFUL) + return ret; + + /* + * Monitor changes to PCI_PRIMARY_BUS register on root port + * and update local copy of root bus number accordingly. + */ + if ((bus == pcie->root_bus_nr) && (where == PCI_PRIMARY_BUS)) + pcie->root_bus_nr = (u8)(value); + + return PCIBIOS_SUCCESSFUL; +} + +static int s10_rp_read_cfg(struct altera_pcie *pcie, int where, + int size, u32 *value) +{ + void __iomem *addr = S10_RP_CFG_ADDR(pcie, where); + + switch (size) { + case 1: + *value = readb(addr); + break; + case 2: + *value = readw(addr); + break; + default: + *value = readl(addr); + break; + } + + return PCIBIOS_SUCCESSFUL; +} + +static int s10_rp_write_cfg(struct altera_pcie *pcie, u8 busno, + int where, int size, u32 value) +{ + void __iomem *addr = S10_RP_CFG_ADDR(pcie, where); + + switch (size) { + case 1: + writeb(value, addr); + break; + case 2: + writew(value, addr); + break; + default: + writel(value, addr); + break; + } + + /* + * Monitor changes to PCI_PRIMARY_BUS register on root port + * and update local copy of root bus number accordingly. + */ + if (busno == pcie->root_bus_nr && where == PCI_PRIMARY_BUS) + pcie->root_bus_nr = value & 0xff; + + return PCIBIOS_SUCCESSFUL; +} + +static int aglx_rp_read_cfg(struct altera_pcie *pcie, int where, + int size, u32 *value) +{ + void __iomem *addr = AGLX_RP_CFG_ADDR(pcie, where); + + switch (size) { + case 1: + *value = readb_relaxed(addr); + break; + case 2: + *value = readw_relaxed(addr); + break; + default: + *value = readl_relaxed(addr); + break; + } + + /* Interrupt PIN not programmed in hardware, set to INTA. */ + if (where == PCI_INTERRUPT_PIN && size == 1 && !(*value)) + *value = 0x01; + else if (where == PCI_INTERRUPT_LINE && !(*value & 0xff00)) + *value |= 0x0100; + + return PCIBIOS_SUCCESSFUL; +} + +static int aglx_rp_write_cfg(struct altera_pcie *pcie, u8 busno, + int where, int size, u32 value) +{ + void __iomem *addr = AGLX_RP_CFG_ADDR(pcie, where); + + switch (size) { + case 1: + writeb_relaxed(value, addr); + break; + case 2: + writew_relaxed(value, addr); + break; + default: + writel_relaxed(value, addr); + break; + } + + /* + * Monitor changes to PCI_PRIMARY_BUS register on Root Port + * and update local copy of root bus number accordingly. + */ + if (busno == pcie->root_bus_nr && where == PCI_PRIMARY_BUS) + pcie->root_bus_nr = value & 0xff; + + return PCIBIOS_SUCCESSFUL; +} + +static int aglx_ep_write_cfg(struct altera_pcie *pcie, u8 busno, + unsigned int devfn, int where, int size, u32 value) +{ + cra_writel(pcie, ((busno << 8) | devfn), AGLX_BDF_REG); + if (busno > AGLX_RP_SECONDARY(pcie)) + where |= FIELD_PREP(AGLX_CFG_TARGET, AGLX_CFG_TARGET_TYPE1); + + switch (size) { + case 1: + cra_writeb(pcie, value, where); + break; + case 2: + cra_writew(pcie, value, where); + break; + default: + cra_writel(pcie, value, where); + break; + } + + return PCIBIOS_SUCCESSFUL; +} + +static int aglx_ep_read_cfg(struct altera_pcie *pcie, u8 busno, + unsigned int devfn, int where, int size, u32 *value) +{ + cra_writel(pcie, ((busno << 8) | devfn), AGLX_BDF_REG); + if (busno > AGLX_RP_SECONDARY(pcie)) + where |= FIELD_PREP(AGLX_CFG_TARGET, AGLX_CFG_TARGET_TYPE1); + + switch (size) { + case 1: + *value = cra_readb(pcie, where); + break; + case 2: + *value = cra_readw(pcie, where); + break; + default: + *value = cra_readl(pcie, where); + break; + } + + return PCIBIOS_SUCCESSFUL; +} + +static int _altera_pcie_cfg_read(struct altera_pcie *pcie, u8 busno, + unsigned int devfn, int where, int size, + u32 *value) +{ + int ret; + u32 data; + u8 byte_en; + + if (busno == pcie->root_bus_nr && pcie->pcie_data->ops->rp_read_cfg) + return pcie->pcie_data->ops->rp_read_cfg(pcie, where, + size, value); + + if (pcie->pcie_data->ops->ep_read_cfg) + return pcie->pcie_data->ops->ep_read_cfg(pcie, busno, devfn, + where, size, value); + + switch (size) { + case 1: + byte_en = 1 << (where & 3); + break; + case 2: + byte_en = 3 << (where & 3); + break; + default: + byte_en = 0xf; + break; + } + + ret = tlp_cfg_dword_read(pcie, busno, devfn, + (where & ~DWORD_MASK), byte_en, &data); + if (ret != PCIBIOS_SUCCESSFUL) + return ret; + + switch (size) { + case 1: + *value = (data >> (8 * (where & 0x3))) & 0xff; + break; + case 2: + *value = (data >> (8 * (where & 0x2))) & 0xffff; + break; + default: + *value = data; + break; + } + + return PCIBIOS_SUCCESSFUL; +} + +static int _altera_pcie_cfg_write(struct altera_pcie *pcie, u8 busno, + unsigned int devfn, int where, int size, + u32 value) +{ + u32 data32; + u32 shift = 8 * (where & 3); + u8 byte_en; + + if (busno == pcie->root_bus_nr && pcie->pcie_data->ops->rp_write_cfg) + return pcie->pcie_data->ops->rp_write_cfg(pcie, busno, + where, size, value); + + if (pcie->pcie_data->ops->ep_write_cfg) + return pcie->pcie_data->ops->ep_write_cfg(pcie, busno, devfn, + where, size, value); + + switch (size) { + case 1: + data32 = (value & 0xff) << shift; + byte_en = 1 << (where & 3); + break; + case 2: + data32 = (value & 0xffff) << shift; + byte_en = 3 << (where & 3); + break; + default: + data32 = value; + byte_en = 0xf; + break; + } + + return tlp_cfg_dword_write(pcie, busno, devfn, (where & ~DWORD_MASK), + byte_en, data32); +} + +static int altera_pcie_cfg_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *value) +{ + struct altera_pcie *pcie = bus->sysdata; + + if (altera_pcie_hide_rc_bar(bus, devfn, where)) + return PCIBIOS_BAD_REGISTER_NUMBER; + + if (!altera_pcie_valid_device(pcie, bus, PCI_SLOT(devfn))) + return PCIBIOS_DEVICE_NOT_FOUND; + + return _altera_pcie_cfg_read(pcie, bus->number, devfn, where, size, + value); +} + +static int altera_pcie_cfg_write(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 value) +{ + struct altera_pcie *pcie = bus->sysdata; + + if (altera_pcie_hide_rc_bar(bus, devfn, where)) + return PCIBIOS_BAD_REGISTER_NUMBER; + + if (!altera_pcie_valid_device(pcie, bus, PCI_SLOT(devfn))) + return PCIBIOS_DEVICE_NOT_FOUND; + + return _altera_pcie_cfg_write(pcie, bus->number, devfn, where, size, + value); +} + +static struct pci_ops altera_pcie_ops = { + .read = altera_pcie_cfg_read, + .write = altera_pcie_cfg_write, +}; + +static int altera_read_cap_word(struct altera_pcie *pcie, u8 busno, + unsigned int devfn, int offset, u16 *value) +{ + u32 data; + int ret; + + ret = _altera_pcie_cfg_read(pcie, busno, devfn, + pcie->pcie_data->cap_offset + offset, + sizeof(*value), + &data); + *value = data; + return ret; +} + +static int altera_write_cap_word(struct altera_pcie *pcie, u8 busno, + unsigned int devfn, int offset, u16 value) +{ + return _altera_pcie_cfg_write(pcie, busno, devfn, + pcie->pcie_data->cap_offset + offset, + sizeof(value), + value); +} + +static void altera_wait_link_retrain(struct altera_pcie *pcie) +{ + struct device *dev = &pcie->pdev->dev; + u16 reg16; + unsigned long start_jiffies; + + /* Wait for link training end. */ + start_jiffies = jiffies; + for (;;) { + altera_read_cap_word(pcie, pcie->root_bus_nr, RP_DEVFN, + PCI_EXP_LNKSTA, ®16); + if (!(reg16 & PCI_EXP_LNKSTA_LT)) + break; + + if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT)) { + dev_err(dev, "link retrain timeout\n"); + break; + } + udelay(100); + } + + /* Wait for link is up */ + start_jiffies = jiffies; + for (;;) { + if (pcie->pcie_data->ops->get_link_status(pcie)) + break; + + if (time_after(jiffies, start_jiffies + LINK_UP_TIMEOUT)) { + dev_err(dev, "link up timeout\n"); + break; + } + udelay(100); + } +} + +static void altera_pcie_retrain(struct altera_pcie *pcie) +{ + u16 linkcap, linkstat, linkctl; + + if (!pcie->pcie_data->ops->get_link_status(pcie)) + return; + + /* + * Set the retrain bit if the PCIe rootport support > 2.5GB/s, but + * current speed is 2.5 GB/s. + */ + altera_read_cap_word(pcie, pcie->root_bus_nr, RP_DEVFN, PCI_EXP_LNKCAP, + &linkcap); + if ((linkcap & PCI_EXP_LNKCAP_SLS) <= PCI_EXP_LNKCAP_SLS_2_5GB) + return; + + altera_read_cap_word(pcie, pcie->root_bus_nr, RP_DEVFN, PCI_EXP_LNKSTA, + &linkstat); + if ((linkstat & PCI_EXP_LNKSTA_CLS) == PCI_EXP_LNKSTA_CLS_2_5GB) { + altera_read_cap_word(pcie, pcie->root_bus_nr, RP_DEVFN, + PCI_EXP_LNKCTL, &linkctl); + linkctl |= PCI_EXP_LNKCTL_RL; + altera_write_cap_word(pcie, pcie->root_bus_nr, RP_DEVFN, + PCI_EXP_LNKCTL, linkctl); + + altera_wait_link_retrain(pcie); + } +} + +static int altera_pcie_intx_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq); + irq_set_chip_data(irq, domain->host_data); + return 0; +} + +static const struct irq_domain_ops intx_domain_ops = { + .map = altera_pcie_intx_map, + .xlate = pci_irqd_intx_xlate, +}; + +static void altera_pcie_isr(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + struct altera_pcie *pcie; + struct device *dev; + unsigned long status; + u32 bit; + int ret; + + chained_irq_enter(chip, desc); + pcie = irq_desc_get_handler_data(desc); + dev = &pcie->pdev->dev; + + while ((status = cra_readl(pcie, P2A_INT_STATUS) + & P2A_INT_STS_ALL) != 0) { + for_each_set_bit(bit, &status, PCI_NUM_INTX) { + /* clear interrupts */ + cra_writel(pcie, 1 << bit, P2A_INT_STATUS); + + ret = generic_handle_domain_irq(pcie->irq_domain, bit); + if (ret) + dev_err_ratelimited(dev, "unexpected IRQ, INT%d\n", bit); + } + } + chained_irq_exit(chip, desc); +} + +static void aglx_isr(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + struct altera_pcie *pcie; + struct device *dev; + u32 status; + int ret; + + chained_irq_enter(chip, desc); + pcie = irq_desc_get_handler_data(desc); + dev = &pcie->pdev->dev; + + status = readl(pcie->hip_base + pcie->pcie_data->port_conf_offset + + pcie->pcie_data->port_irq_status_offset); + + if (status & CFG_AER) { + writel(CFG_AER, (pcie->hip_base + pcie->pcie_data->port_conf_offset + + pcie->pcie_data->port_irq_status_offset)); + + ret = generic_handle_domain_irq(pcie->irq_domain, 0); + if (ret) + dev_err_ratelimited(dev, "unexpected IRQ %d\n", pcie->irq); + } + chained_irq_exit(chip, desc); +} + +static int altera_pcie_init_irq_domain(struct altera_pcie *pcie) +{ + struct device *dev = &pcie->pdev->dev; + + /* Setup INTx */ + pcie->irq_domain = irq_domain_create_linear(dev_fwnode(dev), PCI_NUM_INTX, + &intx_domain_ops, pcie); + if (!pcie->irq_domain) { + dev_err(dev, "Failed to get a INTx IRQ domain\n"); + return -ENOMEM; + } + + return 0; +} + +static void altera_pcie_disable_irq(struct altera_pcie *pcie) +{ + if (pcie->pcie_data->version == ALTERA_PCIE_V1 || + pcie->pcie_data->version == ALTERA_PCIE_V2) { + /* Disable all P2A interrupts */ + cra_writel(pcie, 0, P2A_INT_ENABLE); + } else if (pcie->pcie_data->version == ALTERA_PCIE_V3) { + /* Disable port-level interrupts (CFG_AER, etc.) */ + writel(0, pcie->hip_base + + pcie->pcie_data->port_conf_offset + + pcie->pcie_data->port_irq_enable_offset); + } +} + +static void altera_pcie_irq_teardown(struct altera_pcie *pcie) +{ + altera_pcie_disable_irq(pcie); + irq_set_chained_handler_and_data(pcie->irq, NULL, NULL); + irq_domain_remove(pcie->irq_domain); +} + +static int altera_pcie_parse_dt(struct altera_pcie *pcie) +{ + struct platform_device *pdev = pcie->pdev; + + pcie->cra_base = devm_platform_ioremap_resource_byname(pdev, "Cra"); + if (IS_ERR(pcie->cra_base)) + return PTR_ERR(pcie->cra_base); + + if (pcie->pcie_data->version == ALTERA_PCIE_V2 || + pcie->pcie_data->version == ALTERA_PCIE_V3) { + pcie->hip_base = devm_platform_ioremap_resource_byname(pdev, "Hip"); + if (IS_ERR(pcie->hip_base)) + return PTR_ERR(pcie->hip_base); + } + + /* setup IRQ */ + pcie->irq = platform_get_irq(pdev, 0); + if (pcie->irq < 0) + return pcie->irq; + + return 0; +} + +static void altera_pcie_host_init(struct altera_pcie *pcie) +{ + altera_pcie_retrain(pcie); +} + +static const struct altera_pcie_ops altera_pcie_ops_1_0 = { + .tlp_read_pkt = tlp_read_packet, + .tlp_write_pkt = tlp_write_packet, + .get_link_status = altera_pcie_link_up, + .rp_isr = altera_pcie_isr, +}; + +static const struct altera_pcie_ops altera_pcie_ops_2_0 = { + .tlp_read_pkt = s10_tlp_read_packet, + .tlp_write_pkt = s10_tlp_write_packet, + .get_link_status = s10_altera_pcie_link_up, + .rp_read_cfg = s10_rp_read_cfg, + .rp_write_cfg = s10_rp_write_cfg, + .rp_isr = altera_pcie_isr, +}; + +static const struct altera_pcie_ops altera_pcie_ops_3_0 = { + .rp_read_cfg = aglx_rp_read_cfg, + .rp_write_cfg = aglx_rp_write_cfg, + .get_link_status = aglx_altera_pcie_link_up, + .ep_read_cfg = aglx_ep_read_cfg, + .ep_write_cfg = aglx_ep_write_cfg, + .rp_isr = aglx_isr, +}; + +static const struct altera_pcie_data altera_pcie_1_0_data = { + .ops = &altera_pcie_ops_1_0, + .cap_offset = 0x80, + .version = ALTERA_PCIE_V1, + .cfgrd0 = TLP_FMTTYPE_CFGRD0, + .cfgrd1 = TLP_FMTTYPE_CFGRD1, + .cfgwr0 = TLP_FMTTYPE_CFGWR0, + .cfgwr1 = TLP_FMTTYPE_CFGWR1, +}; + +static const struct altera_pcie_data altera_pcie_2_0_data = { + .ops = &altera_pcie_ops_2_0, + .version = ALTERA_PCIE_V2, + .cap_offset = 0x70, + .cfgrd0 = S10_TLP_FMTTYPE_CFGRD0, + .cfgrd1 = S10_TLP_FMTTYPE_CFGRD1, + .cfgwr0 = S10_TLP_FMTTYPE_CFGWR0, + .cfgwr1 = S10_TLP_FMTTYPE_CFGWR1, +}; + +static const struct altera_pcie_data altera_pcie_3_0_f_tile_data = { + .ops = &altera_pcie_ops_3_0, + .version = ALTERA_PCIE_V3, + .cap_offset = 0x70, + .port_conf_offset = 0x14000, + .port_irq_status_offset = AGLX_ROOT_PORT_IRQ_STATUS, + .port_irq_enable_offset = AGLX_ROOT_PORT_IRQ_ENABLE, +}; + +static const struct altera_pcie_data altera_pcie_3_0_p_tile_data = { + .ops = &altera_pcie_ops_3_0, + .version = ALTERA_PCIE_V3, + .cap_offset = 0x70, + .port_conf_offset = 0x104000, + .port_irq_status_offset = AGLX_ROOT_PORT_IRQ_STATUS, + .port_irq_enable_offset = AGLX_ROOT_PORT_IRQ_ENABLE, +}; + +static const struct altera_pcie_data altera_pcie_3_0_r_tile_data = { + .ops = &altera_pcie_ops_3_0, + .version = ALTERA_PCIE_V3, + .cap_offset = 0x70, + .port_conf_offset = 0x1300, + .port_irq_status_offset = 0x0, + .port_irq_enable_offset = 0x4, +}; + +static const struct of_device_id altera_pcie_of_match[] = { + {.compatible = "altr,pcie-root-port-1.0", + .data = &altera_pcie_1_0_data }, + {.compatible = "altr,pcie-root-port-2.0", + .data = &altera_pcie_2_0_data }, + {.compatible = "altr,pcie-root-port-3.0-f-tile", + .data = &altera_pcie_3_0_f_tile_data }, + {.compatible = "altr,pcie-root-port-3.0-p-tile", + .data = &altera_pcie_3_0_p_tile_data }, + {.compatible = "altr,pcie-root-port-3.0-r-tile", + .data = &altera_pcie_3_0_r_tile_data }, + {}, +}; + +static int altera_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct altera_pcie *pcie; + struct pci_host_bridge *bridge; + int ret; + const struct altera_pcie_data *data; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!bridge) + return -ENOMEM; + + pcie = pci_host_bridge_priv(bridge); + pcie->pdev = pdev; + platform_set_drvdata(pdev, pcie); + + data = of_device_get_match_data(&pdev->dev); + if (!data) + return -ENODEV; + + pcie->pcie_data = data; + + ret = altera_pcie_parse_dt(pcie); + if (ret) { + dev_err(dev, "Parsing DT failed\n"); + return ret; + } + + ret = altera_pcie_init_irq_domain(pcie); + if (ret) { + dev_err(dev, "Failed creating IRQ Domain\n"); + return ret; + } + + /* + * The chained handler uses pcie->irq_domain, so set it only after the + * INTx domain has been created. + */ + irq_set_chained_handler_and_data(pcie->irq, + pcie->pcie_data->ops->rp_isr, + pcie); + + if (pcie->pcie_data->version == ALTERA_PCIE_V1 || + pcie->pcie_data->version == ALTERA_PCIE_V2) { + /* clear all interrupts */ + cra_writel(pcie, P2A_INT_STS_ALL, P2A_INT_STATUS); + /* enable all interrupts */ + cra_writel(pcie, P2A_INT_ENA_ALL, P2A_INT_ENABLE); + altera_pcie_host_init(pcie); + } else if (pcie->pcie_data->version == ALTERA_PCIE_V3) { + writel(CFG_AER, + pcie->hip_base + pcie->pcie_data->port_conf_offset + + pcie->pcie_data->port_irq_enable_offset); + } + + bridge->sysdata = pcie; + bridge->busnr = pcie->root_bus_nr; + bridge->ops = &altera_pcie_ops; + + ret = pci_host_probe(bridge); + if (ret) + goto err_teardown_irq; + + return 0; + +err_teardown_irq: + altera_pcie_irq_teardown(pcie); + + return ret; +} + +static void altera_pcie_remove(struct platform_device *pdev) +{ + struct altera_pcie *pcie = platform_get_drvdata(pdev); + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + + pci_lock_rescan_remove(); + pci_stop_root_bus(bridge->bus); + pci_remove_root_bus(bridge->bus); + pci_unlock_rescan_remove(); + altera_pcie_irq_teardown(pcie); +} + +static struct platform_driver altera_pcie_driver = { + .probe = altera_pcie_probe, + .remove = altera_pcie_remove, + .driver = { + .name = "altera-pcie", + .of_match_table = altera_pcie_of_match, + }, +}; + +MODULE_DEVICE_TABLE(of, altera_pcie_of_match); +module_platform_driver(altera_pcie_driver); +MODULE_DESCRIPTION("Altera PCIe host controller driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pcie-apple.c b/drivers/pci/controller/pcie-apple.c new file mode 100644 index 000000000..c2cffc065 --- /dev/null +++ b/drivers/pci/controller/pcie-apple.c @@ -0,0 +1,898 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe host bridge driver for Apple system-on-chips. + * + * The HW is ECAM compliant, so once the controller is initialized, + * the driver mostly deals MSI mapping and handling of per-port + * interrupts (INTx, management and error signals). + * + * Initialization requires enabling power and clocks, along with a + * number of register pokes. + * + * Copyright (C) 2021 Alyssa Rosenzweig <alyssa@rosenzweig.io> + * Copyright (C) 2021 Google LLC + * Copyright (C) 2021 Corellium LLC + * Copyright (C) 2021 Mark Kettenis <kettenis@openbsd.org> + * + * Author: Alyssa Rosenzweig <alyssa@rosenzweig.io> + * Author: Marc Zyngier <maz@kernel.org> + */ + +#include <linux/bitfield.h> +#include <linux/gpio/consumer.h> +#include <linux/kernel.h> +#include <linux/iopoll.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/irqdomain.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/of_irq.h> +#include <linux/pci-ecam.h> + +#include "pci-host-common.h" + +/* T8103 (original M1) and related SoCs */ +#define CORE_RC_PHYIF_CTL 0x00024 +#define CORE_RC_PHYIF_CTL_RUN BIT(0) +#define CORE_RC_PHYIF_STAT 0x00028 +#define CORE_RC_PHYIF_STAT_REFCLK BIT(4) +#define CORE_RC_CTL 0x00050 +#define CORE_RC_CTL_RUN BIT(0) +#define CORE_RC_STAT 0x00058 +#define CORE_RC_STAT_READY BIT(0) +#define CORE_FABRIC_STAT 0x04000 +#define CORE_FABRIC_STAT_MASK 0x001F001F + +#define CORE_PHY_DEFAULT_BASE(port) (0x84000 + 0x4000 * (port)) + +#define PHY_LANE_CFG 0x00000 +#define PHY_LANE_CFG_REFCLK0REQ BIT(0) +#define PHY_LANE_CFG_REFCLK1REQ BIT(1) +#define PHY_LANE_CFG_REFCLK0ACK BIT(2) +#define PHY_LANE_CFG_REFCLK1ACK BIT(3) +#define PHY_LANE_CFG_REFCLKEN (BIT(9) | BIT(10)) +#define PHY_LANE_CFG_REFCLKCGEN (BIT(30) | BIT(31)) +#define PHY_LANE_CTL 0x00004 +#define PHY_LANE_CTL_CFGACC BIT(15) + +#define PORT_LTSSMCTL 0x00080 +#define PORT_LTSSMCTL_START BIT(0) +#define PORT_INTSTAT 0x00100 +#define PORT_INT_TUNNEL_ERR 31 +#define PORT_INT_CPL_TIMEOUT 23 +#define PORT_INT_RID2SID_MAPERR 22 +#define PORT_INT_CPL_ABORT 21 +#define PORT_INT_MSI_BAD_DATA 19 +#define PORT_INT_MSI_ERR 18 +#define PORT_INT_REQADDR_GT32 17 +#define PORT_INT_AF_TIMEOUT 15 +#define PORT_INT_LINK_DOWN 14 +#define PORT_INT_LINK_UP 12 +#define PORT_INT_LINK_BWMGMT 11 +#define PORT_INT_AER_MASK (15 << 4) +#define PORT_INT_PORT_ERR 4 +#define PORT_INT_INTx(i) i +#define PORT_INT_INTx_MASK 15 +#define PORT_INTMSK 0x00104 +#define PORT_INTMSKSET 0x00108 +#define PORT_INTMSKCLR 0x0010c +#define PORT_MSICFG 0x00124 +#define PORT_MSICFG_EN BIT(0) +#define PORT_MSICFG_L2MSINUM_SHIFT 4 +#define PORT_MSIBASE 0x00128 +#define PORT_MSIBASE_1_SHIFT 16 +#define PORT_MSIADDR 0x00168 +#define PORT_LINKSTS 0x00208 +#define PORT_LINKSTS_UP BIT(0) +#define PORT_LINKSTS_BUSY BIT(2) +#define PORT_LINKCMDSTS 0x00210 +#define PORT_OUTS_NPREQS 0x00284 +#define PORT_OUTS_NPREQS_REQ BIT(24) +#define PORT_OUTS_NPREQS_CPL BIT(16) +#define PORT_RXWR_FIFO 0x00288 +#define PORT_RXWR_FIFO_HDR GENMASK(15, 10) +#define PORT_RXWR_FIFO_DATA GENMASK(9, 0) +#define PORT_RXRD_FIFO 0x0028C +#define PORT_RXRD_FIFO_REQ GENMASK(6, 0) +#define PORT_OUTS_CPLS 0x00290 +#define PORT_OUTS_CPLS_SHRD GENMASK(14, 8) +#define PORT_OUTS_CPLS_WAIT GENMASK(6, 0) +#define PORT_APPCLK 0x00800 +#define PORT_APPCLK_EN BIT(0) +#define PORT_APPCLK_CGDIS BIT(8) +#define PORT_STATUS 0x00804 +#define PORT_STATUS_READY BIT(0) +#define PORT_REFCLK 0x00810 +#define PORT_REFCLK_EN BIT(0) +#define PORT_REFCLK_CGDIS BIT(8) +#define PORT_PERST 0x00814 +#define PORT_PERST_OFF BIT(0) +#define PORT_RID2SID 0x00828 +#define PORT_RID2SID_VALID BIT(31) +#define PORT_RID2SID_SID_SHIFT 16 +#define PORT_RID2SID_BUS_SHIFT 8 +#define PORT_RID2SID_DEV_SHIFT 3 +#define PORT_RID2SID_FUNC_SHIFT 0 +#define PORT_OUTS_PREQS_HDR 0x00980 +#define PORT_OUTS_PREQS_HDR_MASK GENMASK(9, 0) +#define PORT_OUTS_PREQS_DATA 0x00984 +#define PORT_OUTS_PREQS_DATA_MASK GENMASK(15, 0) +#define PORT_TUNCTRL 0x00988 +#define PORT_TUNCTRL_PERST_ON BIT(0) +#define PORT_TUNCTRL_PERST_ACK_REQ BIT(1) +#define PORT_TUNSTAT 0x0098c +#define PORT_TUNSTAT_PERST_ON BIT(0) +#define PORT_TUNSTAT_PERST_ACK_PEND BIT(1) +#define PORT_PREFMEM_ENABLE 0x00994 + +/* T602x (M2-pro and co) */ +#define PORT_T602X_MSIADDR 0x016c +#define PORT_T602X_MSIADDR_HI 0x0170 +#define PORT_T602X_PERST 0x082c +#define PORT_T602X_RID2SID 0x3000 +#define PORT_T602X_MSIMAP 0x3800 + +#define PORT_MSIMAP_ENABLE BIT(31) +#define PORT_MSIMAP_TARGET GENMASK(7, 0) + +/* + * The doorbell address is set to 0xfffff000, which by convention + * matches what MacOS does, and it is possible to use any other + * address (in the bottom 4GB, as the base register is only 32bit). + * However, it has to be excluded from the IOVA range, and the DART + * driver has to know about it. + */ +#define DOORBELL_ADDR CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR + +struct hw_info { + u32 phy_lane_ctl; + u32 port_msiaddr; + u32 port_msiaddr_hi; + u32 port_refclk; + u32 port_perst; + u32 port_rid2sid; + u32 port_msimap; + u32 max_rid2sid; +}; + +static const struct hw_info t8103_hw = { + .phy_lane_ctl = PHY_LANE_CTL, + .port_msiaddr = PORT_MSIADDR, + .port_msiaddr_hi = 0, + .port_refclk = PORT_REFCLK, + .port_perst = PORT_PERST, + .port_rid2sid = PORT_RID2SID, + .port_msimap = 0, + .max_rid2sid = 64, +}; + +static const struct hw_info t602x_hw = { + .phy_lane_ctl = 0, + .port_msiaddr = PORT_T602X_MSIADDR, + .port_msiaddr_hi = PORT_T602X_MSIADDR_HI, + .port_refclk = 0, + .port_perst = PORT_T602X_PERST, + .port_rid2sid = PORT_T602X_RID2SID, + .port_msimap = PORT_T602X_MSIMAP, + /* 16 on t602x, guess for autodetect on future HW */ + .max_rid2sid = 512, +}; + +struct apple_pcie { + struct mutex lock; + struct device *dev; + void __iomem *base; + const struct hw_info *hw; + unsigned long *bitmap; + struct list_head ports; + struct completion event; + struct irq_fwspec fwspec; + u32 nvecs; +}; + +struct apple_pcie_port { + raw_spinlock_t lock; + struct apple_pcie *pcie; + struct device_node *np; + void __iomem *base; + void __iomem *phy; + struct irq_domain *domain; + struct list_head entry; + unsigned long *sid_map; + int sid_map_sz; + int idx; +}; + +static void rmw_set(u32 set, void __iomem *addr) +{ + writel_relaxed(readl_relaxed(addr) | set, addr); +} + +static void rmw_clear(u32 clr, void __iomem *addr) +{ + writel_relaxed(readl_relaxed(addr) & ~clr, addr); +} + +static void apple_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) +{ + msg->address_hi = upper_32_bits(DOORBELL_ADDR); + msg->address_lo = lower_32_bits(DOORBELL_ADDR); + msg->data = data->hwirq; +} + +static struct irq_chip apple_msi_bottom_chip = { + .name = "MSI", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_affinity = irq_chip_set_affinity_parent, + .irq_set_type = irq_chip_set_type_parent, + .irq_compose_msi_msg = apple_msi_compose_msg, +}; + +static int apple_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct apple_pcie *pcie = domain->host_data; + struct irq_fwspec fwspec = pcie->fwspec; + unsigned int i; + int ret, hwirq; + + mutex_lock(&pcie->lock); + + hwirq = bitmap_find_free_region(pcie->bitmap, pcie->nvecs, + order_base_2(nr_irqs)); + + mutex_unlock(&pcie->lock); + + if (hwirq < 0) + return -ENOSPC; + + fwspec.param[fwspec.param_count - 2] += hwirq; + + ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &fwspec); + if (ret) + return ret; + + for (i = 0; i < nr_irqs; i++) { + irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, + &apple_msi_bottom_chip, pcie); + } + + return 0; +} + +static void apple_msi_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct apple_pcie *pcie = domain->host_data; + + mutex_lock(&pcie->lock); + + bitmap_release_region(pcie->bitmap, d->hwirq, order_base_2(nr_irqs)); + + mutex_unlock(&pcie->lock); +} + +static const struct irq_domain_ops apple_msi_domain_ops = { + .alloc = apple_msi_domain_alloc, + .free = apple_msi_domain_free, +}; + +static void apple_port_irq_mask(struct irq_data *data) +{ + struct apple_pcie_port *port = irq_data_get_irq_chip_data(data); + + guard(raw_spinlock_irqsave)(&port->lock); + rmw_set(BIT(data->hwirq), port->base + PORT_INTMSK); +} + +static void apple_port_irq_unmask(struct irq_data *data) +{ + struct apple_pcie_port *port = irq_data_get_irq_chip_data(data); + + guard(raw_spinlock_irqsave)(&port->lock); + rmw_clear(BIT(data->hwirq), port->base + PORT_INTMSK); +} + +static bool hwirq_is_intx(unsigned int hwirq) +{ + return BIT(hwirq) & PORT_INT_INTx_MASK; +} + +static void apple_port_irq_ack(struct irq_data *data) +{ + struct apple_pcie_port *port = irq_data_get_irq_chip_data(data); + + if (!hwirq_is_intx(data->hwirq)) + writel_relaxed(BIT(data->hwirq), port->base + PORT_INTSTAT); +} + +static int apple_port_irq_set_type(struct irq_data *data, unsigned int type) +{ + /* + * It doesn't seem that there is any way to configure the + * trigger, so assume INTx have to be level (as per the spec), + * and the rest is edge (which looks likely). + */ + if (hwirq_is_intx(data->hwirq) ^ !!(type & IRQ_TYPE_LEVEL_MASK)) + return -EINVAL; + + irqd_set_trigger_type(data, type); + return 0; +} + +static struct irq_chip apple_port_irqchip = { + .name = "PCIe", + .irq_ack = apple_port_irq_ack, + .irq_mask = apple_port_irq_mask, + .irq_unmask = apple_port_irq_unmask, + .irq_set_type = apple_port_irq_set_type, +}; + +static int apple_port_irq_domain_alloc(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs, + void *args) +{ + struct apple_pcie_port *port = domain->host_data; + struct irq_fwspec *fwspec = args; + int i; + + for (i = 0; i < nr_irqs; i++) { + irq_flow_handler_t flow = handle_edge_irq; + unsigned int type = IRQ_TYPE_EDGE_RISING; + + if (hwirq_is_intx(fwspec->param[0] + i)) { + flow = handle_level_irq; + type = IRQ_TYPE_LEVEL_HIGH; + } + + irq_domain_set_info(domain, virq + i, fwspec->param[0] + i, + &apple_port_irqchip, port, flow, + NULL, NULL); + + irq_set_irq_type(virq + i, type); + } + + return 0; +} + +static void apple_port_irq_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + int i; + + for (i = 0; i < nr_irqs; i++) { + struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); + + irq_set_handler(virq + i, NULL); + irq_domain_reset_irq_data(d); + } +} + +static const struct irq_domain_ops apple_port_irq_domain_ops = { + .translate = irq_domain_translate_onecell, + .alloc = apple_port_irq_domain_alloc, + .free = apple_port_irq_domain_free, +}; + +static void apple_port_irq_handler(struct irq_desc *desc) +{ + struct apple_pcie_port *port = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + unsigned long stat; + int i; + + chained_irq_enter(chip, desc); + + stat = readl_relaxed(port->base + PORT_INTSTAT); + + for_each_set_bit(i, &stat, 32) + generic_handle_domain_irq(port->domain, i); + + chained_irq_exit(chip, desc); +} + +static int apple_pcie_port_setup_irq(struct apple_pcie_port *port) +{ + struct fwnode_handle *fwnode = &port->np->fwnode; + struct apple_pcie *pcie = port->pcie; + unsigned int irq; + u32 val = 0; + + /* FIXME: consider moving each interrupt under each port */ + irq = irq_of_parse_and_map(to_of_node(dev_fwnode(port->pcie->dev)), + port->idx); + if (!irq) + return -ENXIO; + + port->domain = irq_domain_create_linear(fwnode, 32, + &apple_port_irq_domain_ops, + port); + if (!port->domain) + return -ENOMEM; + + /* Disable all interrupts */ + writel_relaxed(~0, port->base + PORT_INTMSK); + writel_relaxed(~0, port->base + PORT_INTSTAT); + writel_relaxed(~0, port->base + PORT_LINKCMDSTS); + + irq_set_chained_handler_and_data(irq, apple_port_irq_handler, port); + + /* Configure MSI base address */ + BUILD_BUG_ON(upper_32_bits(DOORBELL_ADDR)); + writel_relaxed(lower_32_bits(DOORBELL_ADDR), + port->base + pcie->hw->port_msiaddr); + if (pcie->hw->port_msiaddr_hi) + writel_relaxed(0, port->base + pcie->hw->port_msiaddr_hi); + + /* Enable MSIs, shared between all ports */ + if (pcie->hw->port_msimap) { + for (int i = 0; i < pcie->nvecs; i++) + writel_relaxed(FIELD_PREP(PORT_MSIMAP_TARGET, i) | + PORT_MSIMAP_ENABLE, + port->base + pcie->hw->port_msimap + 4 * i); + } else { + writel_relaxed(0, port->base + PORT_MSIBASE); + val = ilog2(pcie->nvecs) << PORT_MSICFG_L2MSINUM_SHIFT; + } + + writel_relaxed(val | PORT_MSICFG_EN, port->base + PORT_MSICFG); + return 0; +} + +static irqreturn_t apple_pcie_port_irq(int irq, void *data) +{ + struct apple_pcie_port *port = data; + unsigned int hwirq = irq_domain_get_irq_data(port->domain, irq)->hwirq; + + switch (hwirq) { + case PORT_INT_LINK_UP: + dev_info_ratelimited(port->pcie->dev, "Link up on %pOF\n", + port->np); + complete_all(&port->pcie->event); + break; + case PORT_INT_LINK_DOWN: + dev_info_ratelimited(port->pcie->dev, "Link down on %pOF\n", + port->np); + break; + default: + return IRQ_NONE; + } + + return IRQ_HANDLED; +} + +static int apple_pcie_port_register_irqs(struct apple_pcie_port *port) +{ + static struct { + unsigned int hwirq; + const char *name; + } port_irqs[] = { + { PORT_INT_LINK_UP, "Link up", }, + { PORT_INT_LINK_DOWN, "Link down", }, + }; + int i; + + for (i = 0; i < ARRAY_SIZE(port_irqs); i++) { + struct irq_fwspec fwspec = { + .fwnode = &port->np->fwnode, + .param_count = 1, + .param = { + [0] = port_irqs[i].hwirq, + }, + }; + unsigned int irq; + int ret; + + irq = irq_domain_alloc_irqs(port->domain, 1, NUMA_NO_NODE, + &fwspec); + if (WARN_ON(!irq)) + continue; + + ret = request_irq(irq, apple_pcie_port_irq, 0, + port_irqs[i].name, port); + WARN_ON(ret); + } + + return 0; +} + +static int apple_pcie_setup_refclk(struct apple_pcie *pcie, + struct apple_pcie_port *port) +{ + u32 stat; + int res; + + if (pcie->hw->phy_lane_ctl) + rmw_set(PHY_LANE_CTL_CFGACC, port->phy + pcie->hw->phy_lane_ctl); + + rmw_set(PHY_LANE_CFG_REFCLK0REQ, port->phy + PHY_LANE_CFG); + + res = readl_relaxed_poll_timeout(port->phy + PHY_LANE_CFG, + stat, stat & PHY_LANE_CFG_REFCLK0ACK, + 100, 50000); + if (res < 0) + return res; + + rmw_set(PHY_LANE_CFG_REFCLK1REQ, port->phy + PHY_LANE_CFG); + res = readl_relaxed_poll_timeout(port->phy + PHY_LANE_CFG, + stat, stat & PHY_LANE_CFG_REFCLK1ACK, + 100, 50000); + + if (res < 0) + return res; + + if (pcie->hw->phy_lane_ctl) + rmw_clear(PHY_LANE_CTL_CFGACC, port->phy + pcie->hw->phy_lane_ctl); + + rmw_set(PHY_LANE_CFG_REFCLKEN, port->phy + PHY_LANE_CFG); + + if (pcie->hw->port_refclk) + rmw_set(PORT_REFCLK_EN, port->base + pcie->hw->port_refclk); + + return 0; +} + +static void __iomem *port_rid2sid_addr(struct apple_pcie_port *port, int idx) +{ + return port->base + port->pcie->hw->port_rid2sid + 4 * idx; +} + +static u32 apple_pcie_rid2sid_write(struct apple_pcie_port *port, + int idx, u32 val) +{ + writel_relaxed(val, port_rid2sid_addr(port, idx)); + /* Read back to ensure completion of the write */ + return readl_relaxed(port_rid2sid_addr(port, idx)); +} + +static int apple_pcie_setup_port(struct apple_pcie *pcie, + struct device_node *np) +{ + struct platform_device *platform = to_platform_device(pcie->dev); + struct apple_pcie_port *port; + struct gpio_desc *reset; + struct resource *res; + char name[16]; + u32 stat, idx; + int ret, i; + + reset = devm_fwnode_gpiod_get(pcie->dev, of_fwnode_handle(np), "reset", + GPIOD_OUT_LOW, "PERST#"); + if (IS_ERR(reset)) + return PTR_ERR(reset); + + port = devm_kzalloc(pcie->dev, sizeof(*port), GFP_KERNEL); + if (!port) + return -ENOMEM; + + port->sid_map = devm_bitmap_zalloc(pcie->dev, pcie->hw->max_rid2sid, GFP_KERNEL); + if (!port->sid_map) + return -ENOMEM; + + ret = of_property_read_u32_index(np, "reg", 0, &idx); + if (ret) + return ret; + + /* Use the first reg entry to work out the port index */ + port->idx = idx >> 11; + port->pcie = pcie; + port->np = np; + + raw_spin_lock_init(&port->lock); + + snprintf(name, sizeof(name), "port%d", port->idx); + res = platform_get_resource_byname(platform, IORESOURCE_MEM, name); + if (!res) + res = platform_get_resource(platform, IORESOURCE_MEM, port->idx + 2); + + port->base = devm_ioremap_resource(&platform->dev, res); + if (IS_ERR(port->base)) + return PTR_ERR(port->base); + + snprintf(name, sizeof(name), "phy%d", port->idx); + res = platform_get_resource_byname(platform, IORESOURCE_MEM, name); + if (res) + port->phy = devm_ioremap_resource(&platform->dev, res); + else + port->phy = pcie->base + CORE_PHY_DEFAULT_BASE(port->idx); + + rmw_set(PORT_APPCLK_EN, port->base + PORT_APPCLK); + + /* Assert PERST# before setting up the clock */ + gpiod_set_value_cansleep(reset, 1); + + ret = apple_pcie_setup_refclk(pcie, port); + if (ret < 0) + return ret; + + /* The minimal Tperst-clk value is 100us (PCIe CEM r5.0, 2.9.2) */ + usleep_range(100, 200); + + /* Deassert PERST# */ + rmw_set(PORT_PERST_OFF, port->base + pcie->hw->port_perst); + gpiod_set_value_cansleep(reset, 0); + + /* Wait for 100ms after PERST# deassertion (PCIe r5.0, 6.6.1) */ + msleep(100); + + ret = readl_relaxed_poll_timeout(port->base + PORT_STATUS, stat, + stat & PORT_STATUS_READY, 100, 250000); + if (ret < 0) { + dev_err(pcie->dev, "port %pOF ready wait timeout\n", np); + return ret; + } + + if (pcie->hw->port_refclk) + rmw_clear(PORT_REFCLK_CGDIS, port->base + pcie->hw->port_refclk); + else + rmw_set(PHY_LANE_CFG_REFCLKCGEN, port->phy + PHY_LANE_CFG); + + rmw_clear(PORT_APPCLK_CGDIS, port->base + PORT_APPCLK); + + ret = apple_pcie_port_setup_irq(port); + if (ret) + return ret; + + /* Reset all RID/SID mappings, and check for RAZ/WI registers */ + for (i = 0; i < pcie->hw->max_rid2sid; i++) { + if (apple_pcie_rid2sid_write(port, i, 0xbad1d) != 0xbad1d) + break; + apple_pcie_rid2sid_write(port, i, 0); + } + + dev_dbg(pcie->dev, "%pOF: %d RID/SID mapping entries\n", np, i); + + port->sid_map_sz = i; + + list_add_tail(&port->entry, &pcie->ports); + init_completion(&pcie->event); + + /* In the success path, we keep a reference to np around */ + of_node_get(np); + + ret = apple_pcie_port_register_irqs(port); + WARN_ON(ret); + + writel_relaxed(PORT_LTSSMCTL_START, port->base + PORT_LTSSMCTL); + + if (!wait_for_completion_timeout(&pcie->event, HZ / 10)) + dev_warn(pcie->dev, "%pOF link didn't come up\n", np); + + return 0; +} + +static const struct msi_parent_ops apple_msi_parent_ops = { + .supported_flags = (MSI_GENERIC_FLAGS_MASK | + MSI_FLAG_PCI_MSIX | + MSI_FLAG_MULTI_PCI_MSI), + .required_flags = (MSI_FLAG_USE_DEF_DOM_OPS | + MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_PCI_MSI_MASK_PARENT), + .chip_flags = MSI_CHIP_FLAG_SET_EOI, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +static int apple_msi_init(struct apple_pcie *pcie) +{ + struct fwnode_handle *fwnode = dev_fwnode(pcie->dev); + struct irq_domain_info info = { + .fwnode = fwnode, + .ops = &apple_msi_domain_ops, + .size = pcie->nvecs, + .host_data = pcie, + }; + struct of_phandle_args args = {}; + int ret; + + ret = of_parse_phandle_with_args(to_of_node(fwnode), "msi-ranges", + "#interrupt-cells", 0, &args); + if (ret) + return ret; + + ret = of_property_read_u32_index(to_of_node(fwnode), "msi-ranges", + args.args_count + 1, &pcie->nvecs); + if (ret) + return ret; + + of_phandle_args_to_fwspec(args.np, args.args, args.args_count, + &pcie->fwspec); + + pcie->bitmap = devm_bitmap_zalloc(pcie->dev, pcie->nvecs, GFP_KERNEL); + if (!pcie->bitmap) + return -ENOMEM; + + info.parent = irq_find_matching_fwspec(&pcie->fwspec, DOMAIN_BUS_WIRED); + if (!info.parent) { + dev_err(pcie->dev, "failed to find parent domain\n"); + return -ENXIO; + } + + if (!msi_create_parent_irq_domain(&info, &apple_msi_parent_ops)) { + dev_err(pcie->dev, "failed to create IRQ domain\n"); + return -ENOMEM; + } + return 0; +} + +static struct apple_pcie *apple_pcie_lookup(struct device *dev) +{ + return pci_host_bridge_priv(dev_get_drvdata(dev)); +} + +static struct apple_pcie_port *apple_pcie_get_port(struct pci_dev *pdev) +{ + struct pci_config_window *cfg = pdev->sysdata; + struct apple_pcie *pcie; + struct pci_dev *port_pdev; + struct apple_pcie_port *port; + + pcie = apple_pcie_lookup(cfg->parent); + if (WARN_ON(!pcie)) + return NULL; + + /* Find the root port this device is on */ + port_pdev = pcie_find_root_port(pdev); + + /* If finding the port itself, nothing to do */ + if (WARN_ON(!port_pdev) || pdev == port_pdev) + return NULL; + + list_for_each_entry(port, &pcie->ports, entry) { + if (port->idx == PCI_SLOT(port_pdev->devfn)) + return port; + } + + return NULL; +} + +static int apple_pcie_enable_device(struct pci_host_bridge *bridge, struct pci_dev *pdev) +{ + u32 sid, rid = pci_dev_id(pdev); + struct apple_pcie_port *port; + struct of_phandle_args iommu_spec = {}; + int idx, err; + + port = apple_pcie_get_port(pdev); + if (!port) + return 0; + + dev_dbg(&pdev->dev, "added to bus %s, index %d\n", + pci_name(pdev->bus->self), port->idx); + + err = of_map_iommu_id(port->pcie->dev->of_node, rid, &iommu_spec); + if (err) + return err; + + of_node_put(iommu_spec.np); + sid = iommu_spec.args[0]; + mutex_lock(&port->pcie->lock); + + idx = bitmap_find_free_region(port->sid_map, port->sid_map_sz, 0); + if (idx >= 0) { + apple_pcie_rid2sid_write(port, idx, + PORT_RID2SID_VALID | + (sid << PORT_RID2SID_SID_SHIFT) | rid); + + dev_dbg(&pdev->dev, "mapping RID%x to SID%x (index %d)\n", + rid, sid, idx); + } + + mutex_unlock(&port->pcie->lock); + + return idx >= 0 ? 0 : -ENOSPC; +} + +static void apple_pcie_disable_device(struct pci_host_bridge *bridge, struct pci_dev *pdev) +{ + struct apple_pcie_port *port; + u32 rid = pci_dev_id(pdev); + int idx; + + port = apple_pcie_get_port(pdev); + if (!port) + return; + + mutex_lock(&port->pcie->lock); + + for_each_set_bit(idx, port->sid_map, port->sid_map_sz) { + u32 val; + + val = readl_relaxed(port_rid2sid_addr(port, idx)); + if ((val & 0xffff) == rid) { + apple_pcie_rid2sid_write(port, idx, 0); + bitmap_release_region(port->sid_map, idx, 0); + dev_dbg(&pdev->dev, "Released %x (%d)\n", val, idx); + break; + } + } + + mutex_unlock(&port->pcie->lock); +} + +static int apple_pcie_init(struct pci_config_window *cfg) +{ + struct device *dev = cfg->parent; + struct apple_pcie *pcie; + int ret; + + pcie = apple_pcie_lookup(dev); + if (WARN_ON(!pcie)) + return -ENOENT; + + for_each_available_child_of_node_scoped(dev->of_node, of_port) { + ret = apple_pcie_setup_port(pcie, of_port); + if (ret) { + dev_err(dev, "Port %pOF setup fail: %d\n", of_port, ret); + return ret; + } + } + + return 0; +} + +static const struct pci_ecam_ops apple_pcie_cfg_ecam_ops = { + .init = apple_pcie_init, + .enable_device = apple_pcie_enable_device, + .disable_device = apple_pcie_disable_device, + .pci_ops = { + .map_bus = pci_ecam_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, + } +}; + +static int apple_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct pci_host_bridge *bridge; + struct apple_pcie *pcie; + int ret; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!bridge) + return -ENOMEM; + + pcie = pci_host_bridge_priv(bridge); + pcie->dev = dev; + pcie->hw = of_device_get_match_data(dev); + if (!pcie->hw) + return -ENODEV; + pcie->base = devm_platform_ioremap_resource(pdev, 1); + if (IS_ERR(pcie->base)) + return PTR_ERR(pcie->base); + + mutex_init(&pcie->lock); + INIT_LIST_HEAD(&pcie->ports); + + ret = apple_msi_init(pcie); + if (ret) + return ret; + + return pci_host_common_init(pdev, bridge, &apple_pcie_cfg_ecam_ops); +} + +static const struct of_device_id apple_pcie_of_match[] = { + { .compatible = "apple,t6020-pcie", .data = &t602x_hw }, + { .compatible = "apple,pcie", .data = &t8103_hw }, + { } +}; +MODULE_DEVICE_TABLE(of, apple_pcie_of_match); + +static struct platform_driver apple_pcie_driver = { + .probe = apple_pcie_probe, + .driver = { + .name = "pcie-apple", + .of_match_table = apple_pcie_of_match, + .suppress_bind_attrs = true, + }, +}; +module_platform_driver(apple_pcie_driver); + +MODULE_DESCRIPTION("Apple PCIe host bridge driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pcie-aspeed.c b/drivers/pci/controller/pcie-aspeed.c new file mode 100644 index 000000000..74b03a556 --- /dev/null +++ b/drivers/pci/controller/pcie-aspeed.c @@ -0,0 +1,1111 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Copyright 2025 Aspeed Technology Inc. + */ +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/irqdomain.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/kernel.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/mutex.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/phy/pcie.h> +#include <linux/phy/phy.h> +#include <linux/regmap.h> +#include <linux/reset.h> + +#include "../pci.h" + +#define MAX_MSI_HOST_IRQS 64 +#define ASPEED_RESET_RC_WAIT_MS 10 + +/* AST2600 AHBC Registers */ +#define ASPEED_AHBC_KEY 0x00 +#define ASPEED_AHBC_UNLOCK_KEY 0xaeed1a03 +#define ASPEED_AHBC_UNLOCK 0x01 +#define ASPEED_AHBC_ADDR_MAPPING 0x8c +#define ASPEED_PCIE_RC_MEMORY_EN BIT(5) + +/* AST2600 H2X Controller Registers */ +#define ASPEED_H2X_INT_STS 0x08 +#define ASPEED_PCIE_TX_IDLE_CLEAR BIT(0) +#define ASPEED_PCIE_INTX_STS GENMASK(3, 0) +#define ASPEED_H2X_HOST_RX_DESC_DATA 0x0c +#define ASPEED_H2X_TX_DESC0 0x10 +#define ASPEED_H2X_TX_DESC1 0x14 +#define ASPEED_H2X_TX_DESC2 0x18 +#define ASPEED_H2X_TX_DESC3 0x1c +#define ASPEED_H2X_TX_DESC_DATA 0x20 +#define ASPEED_H2X_STS 0x24 +#define ASPEED_PCIE_TX_IDLE BIT(31) +#define ASPEED_PCIE_STATUS_OF_TX GENMASK(25, 24) +#define ASPEED_PCIE_RC_H_TX_COMPLETE BIT(25) +#define ASPEED_PCIE_TRIGGER_TX BIT(0) +#define ASPEED_H2X_AHB_ADDR_CONFIG0 0x60 +#define ASPEED_AHB_REMAP_LO_ADDR(x) (x & GENMASK(15, 4)) +#define ASPEED_AHB_MASK_LO_ADDR(x) FIELD_PREP(GENMASK(31, 20), x) +#define ASPEED_H2X_AHB_ADDR_CONFIG1 0x64 +#define ASPEED_AHB_REMAP_HI_ADDR(x) (x) +#define ASPEED_H2X_AHB_ADDR_CONFIG2 0x68 +#define ASPEED_AHB_MASK_HI_ADDR(x) (x) +#define ASPEED_H2X_DEV_CTRL 0xc0 +#define ASPEED_PCIE_RX_DMA_EN BIT(9) +#define ASPEED_PCIE_RX_LINEAR BIT(8) +#define ASPEED_PCIE_RX_MSI_SEL BIT(7) +#define ASPEED_PCIE_RX_MSI_EN BIT(6) +#define ASPEED_PCIE_UNLOCK_RX_BUFF BIT(4) +#define ASPEED_PCIE_WAIT_RX_TLP_CLR BIT(2) +#define ASPEED_PCIE_RC_RX_ENABLE BIT(1) +#define ASPEED_PCIE_RC_ENABLE BIT(0) +#define ASPEED_H2X_DEV_STS 0xc8 +#define ASPEED_PCIE_RC_RX_DONE_ISR BIT(4) +#define ASPEED_H2X_DEV_RX_DESC_DATA 0xcc +#define ASPEED_H2X_DEV_RX_DESC1 0xd4 +#define ASPEED_H2X_DEV_TX_TAG 0xfc +#define ASPEED_RC_TLP_TX_TAG_NUM 0x28 + +/* AST2700 H2X */ +#define ASPEED_H2X_CTRL 0x00 +#define ASPEED_H2X_BRIDGE_EN BIT(0) +#define ASPEED_H2X_BRIDGE_DIRECT_EN BIT(1) +#define ASPEED_H2X_CFGE_INT_STS 0x08 +#define ASPEED_CFGE_TX_IDLE BIT(0) +#define ASPEED_CFGE_RX_BUSY BIT(1) +#define ASPEED_H2X_CFGI_TLP 0x20 +#define ASPEED_CFGI_BYTE_EN_MASK GENMASK(19, 16) +#define ASPEED_CFGI_BYTE_EN(x) \ + FIELD_PREP(ASPEED_CFGI_BYTE_EN_MASK, (x)) +#define ASPEED_H2X_CFGI_WR_DATA 0x24 +#define ASPEED_CFGI_WRITE BIT(20) +#define ASPEED_H2X_CFGI_CTRL 0x28 +#define ASPEED_CFGI_TLP_FIRE BIT(0) +#define ASPEED_H2X_CFGI_RET_DATA 0x2c +#define ASPEED_H2X_CFGE_TLP_1ST 0x30 +#define ASPEED_H2X_CFGE_TLP_NEXT 0x34 +#define ASPEED_H2X_CFGE_CTRL 0x38 +#define ASPEED_CFGE_TLP_FIRE BIT(0) +#define ASPEED_H2X_CFGE_RET_DATA 0x3c +#define ASPEED_H2X_REMAP_PREF_ADDR 0x70 +#define ASPEED_REMAP_PREF_ADDR_63_32(x) (x) +#define ASPEED_H2X_REMAP_PCI_ADDR_HI 0x74 +#define ASPEED_REMAP_PCI_ADDR_63_32(x) (((x) >> 32) & GENMASK(31, 0)) +#define ASPEED_H2X_REMAP_PCI_ADDR_LO 0x78 +#define ASPEED_REMAP_PCI_ADDR_31_12(x) ((x) & GENMASK(31, 12)) + +/* AST2700 SCU */ +#define ASPEED_SCU_60 0x60 +#define ASPEED_RC_E2M_PATH_EN BIT(0) +#define ASPEED_RC_H2XS_PATH_EN BIT(16) +#define ASPEED_RC_H2XD_PATH_EN BIT(17) +#define ASPEED_RC_H2XX_PATH_EN BIT(18) +#define ASPEED_RC_UPSTREAM_MEM_EN BIT(19) +#define ASPEED_SCU_64 0x64 +#define ASPEED_RC0_DECODE_DMA_BASE(x) FIELD_PREP(GENMASK(7, 0), x) +#define ASPEED_RC0_DECODE_DMA_LIMIT(x) FIELD_PREP(GENMASK(15, 8), x) +#define ASPEED_RC1_DECODE_DMA_BASE(x) FIELD_PREP(GENMASK(23, 16), x) +#define ASPEED_RC1_DECODE_DMA_LIMIT(x) FIELD_PREP(GENMASK(31, 24), x) +#define ASPEED_SCU_70 0x70 +#define ASPEED_DISABLE_EP_FUNC 0 + +/* Macro to combine Fmt and Type into the 8-bit field */ +#define ASPEED_TLP_FMT_TYPE(fmt, type) ((((fmt) & 0x7) << 5) | ((type) & 0x1f)) +#define ASPEED_TLP_COMMON_FIELDS GENMASK(31, 24) + +/* Completion status */ +#define CPL_STS(x) FIELD_GET(GENMASK(15, 13), (x)) +/* TLP configuration type 0 and type 1 */ +#define CFG0_READ_FMTTYPE \ + FIELD_PREP(ASPEED_TLP_COMMON_FIELDS, \ + ASPEED_TLP_FMT_TYPE(PCIE_TLP_FMT_3DW_NO_DATA, \ + PCIE_TLP_TYPE_CFG0_RDWR)) +#define CFG0_WRITE_FMTTYPE \ + FIELD_PREP(ASPEED_TLP_COMMON_FIELDS, \ + ASPEED_TLP_FMT_TYPE(PCIE_TLP_FMT_3DW_DATA, \ + PCIE_TLP_TYPE_CFG0_RDWR)) +#define CFG1_READ_FMTTYPE \ + FIELD_PREP(ASPEED_TLP_COMMON_FIELDS, \ + ASPEED_TLP_FMT_TYPE(PCIE_TLP_FMT_3DW_NO_DATA, \ + PCIE_TLP_TYPE_CFG1_RDWR)) +#define CFG1_WRITE_FMTTYPE \ + FIELD_PREP(ASPEED_TLP_COMMON_FIELDS, \ + ASPEED_TLP_FMT_TYPE(PCIE_TLP_FMT_3DW_DATA, \ + PCIE_TLP_TYPE_CFG1_RDWR)) +#define CFG_PAYLOAD_SIZE 0x01 /* 1 DWORD */ +#define TLP_HEADER_BYTE_EN(x, y) ((GENMASK((x) - 1, 0) << ((y) % 4))) +#define TLP_GET_VALUE(x, y, z) \ + (((x) >> ((((z) % 4)) * 8)) & GENMASK((8 * (y)) - 1, 0)) +#define TLP_SET_VALUE(x, y, z) \ + ((((x) & GENMASK((8 * (y)) - 1, 0)) << ((((z) % 4)) * 8))) +#define AST2600_TX_DESC1_VALUE 0x00002000 +#define AST2700_TX_DESC1_VALUE 0x00401000 + +/** + * struct aspeed_pcie_port - PCIe port information + * @list: port list + * @pcie: pointer to PCIe host info + * @clk: pointer to the port clock gate + * @phy: pointer to PCIe PHY + * @perst: pointer to port reset control + * @slot: port slot + */ +struct aspeed_pcie_port { + struct list_head list; + struct aspeed_pcie *pcie; + struct clk *clk; + struct phy *phy; + struct reset_control *perst; + u32 slot; +}; + +/** + * struct aspeed_pcie - PCIe RC information + * @host: pointer to PCIe host bridge + * @dev: pointer to device structure + * @reg: PCIe host register base address + * @ahbc: pointer to AHHC register map + * @cfg: pointer to Aspeed PCIe configuration register map + * @platform: platform specific information + * @ports: list of PCIe ports + * @tx_tag: current TX tag for the port + * @root_bus_nr: bus number of the host bridge + * @h2xrst: pointer to H2X reset control + * @intx_domain: IRQ domain for INTx interrupts + * @msi_domain: IRQ domain for MSI interrupts + * @lock: mutex to protect MSI bitmap variable + * @msi_irq_in_use: bitmap to track used MSI host IRQs + * @clear_msi_twice: AST2700 workaround to clear MSI status twice + */ +struct aspeed_pcie { + struct pci_host_bridge *host; + struct device *dev; + void __iomem *reg; + struct regmap *ahbc; + struct regmap *cfg; + const struct aspeed_pcie_rc_platform *platform; + struct list_head ports; + + u8 tx_tag; + u8 root_bus_nr; + + struct reset_control *h2xrst; + + struct irq_domain *intx_domain; + struct irq_domain *msi_domain; + struct mutex lock; + DECLARE_BITMAP(msi_irq_in_use, MAX_MSI_HOST_IRQS); + + bool clear_msi_twice; /* AST2700 workaround */ +}; + +/** + * struct aspeed_pcie_rc_platform - Platform information + * @setup: initialization function + * @pcie_map_ranges: function to map PCIe address ranges + * @reg_intx_en: INTx enable register offset + * @reg_intx_sts: INTx status register offset + * @reg_msi_en: MSI enable register offset + * @reg_msi_sts: MSI enable register offset + * @msi_address: HW fixed MSI address + */ +struct aspeed_pcie_rc_platform { + int (*setup)(struct platform_device *pdev); + void (*pcie_map_ranges)(struct aspeed_pcie *pcie, u64 pci_addr); + int reg_intx_en; + int reg_intx_sts; + int reg_msi_en; + int reg_msi_sts; + u32 msi_address; +}; + +static void aspeed_pcie_intx_irq_ack(struct irq_data *d) +{ + struct aspeed_pcie *pcie = irq_data_get_irq_chip_data(d); + int intx_en = pcie->platform->reg_intx_en; + u32 en; + + en = readl(pcie->reg + intx_en); + en |= BIT(d->hwirq); + writel(en, pcie->reg + intx_en); +} + +static void aspeed_pcie_intx_irq_mask(struct irq_data *d) +{ + struct aspeed_pcie *pcie = irq_data_get_irq_chip_data(d); + int intx_en = pcie->platform->reg_intx_en; + u32 en; + + en = readl(pcie->reg + intx_en); + en &= ~BIT(d->hwirq); + writel(en, pcie->reg + intx_en); +} + +static void aspeed_pcie_intx_irq_unmask(struct irq_data *d) +{ + struct aspeed_pcie *pcie = irq_data_get_irq_chip_data(d); + int intx_en = pcie->platform->reg_intx_en; + u32 en; + + en = readl(pcie->reg + intx_en); + en |= BIT(d->hwirq); + writel(en, pcie->reg + intx_en); +} + +static struct irq_chip aspeed_intx_irq_chip = { + .name = "INTx", + .irq_ack = aspeed_pcie_intx_irq_ack, + .irq_mask = aspeed_pcie_intx_irq_mask, + .irq_unmask = aspeed_pcie_intx_irq_unmask, +}; + +static int aspeed_pcie_intx_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &aspeed_intx_irq_chip, handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + irq_set_status_flags(irq, IRQ_LEVEL); + + return 0; +} + +static const struct irq_domain_ops aspeed_intx_domain_ops = { + .map = aspeed_pcie_intx_map, +}; + +static irqreturn_t aspeed_pcie_intr_handler(int irq, void *dev_id) +{ + struct aspeed_pcie *pcie = dev_id; + const struct aspeed_pcie_rc_platform *platform = pcie->platform; + unsigned long status; + unsigned long intx; + u32 bit; + int i; + + intx = FIELD_GET(ASPEED_PCIE_INTX_STS, + readl(pcie->reg + platform->reg_intx_sts)); + for_each_set_bit(bit, &intx, PCI_NUM_INTX) + generic_handle_domain_irq(pcie->intx_domain, bit); + + for (i = 0; i < 2; i++) { + int msi_sts_reg = platform->reg_msi_sts + (i * 4); + + status = readl(pcie->reg + msi_sts_reg); + writel(status, pcie->reg + msi_sts_reg); + + /* + * AST2700 workaround: + * The MSI status needs to clear one more time. + */ + if (pcie->clear_msi_twice) + writel(status, pcie->reg + msi_sts_reg); + + for_each_set_bit(bit, &status, 32) { + bit += (i * 32); + generic_handle_domain_irq(pcie->msi_domain, bit); + } + } + + return IRQ_HANDLED; +} + +static u32 aspeed_pcie_get_bdf_offset(struct pci_bus *bus, unsigned int devfn, + int where) +{ + return ((bus->number) << 24) | (PCI_SLOT(devfn) << 19) | + (PCI_FUNC(devfn) << 16) | (where & ~3); +} + +static int aspeed_ast2600_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val, u32 fmt_type, + bool write) +{ + struct aspeed_pcie *pcie = bus->sysdata; + u32 bdf_offset, cfg_val, isr; + int ret; + + bdf_offset = aspeed_pcie_get_bdf_offset(bus, devfn, where); + + /* Driver may set unlock RX buffer before triggering next TX config */ + cfg_val = readl(pcie->reg + ASPEED_H2X_DEV_CTRL); + writel(ASPEED_PCIE_UNLOCK_RX_BUFF | cfg_val, + pcie->reg + ASPEED_H2X_DEV_CTRL); + + cfg_val = fmt_type | CFG_PAYLOAD_SIZE; + writel(cfg_val, pcie->reg + ASPEED_H2X_TX_DESC0); + + cfg_val = AST2600_TX_DESC1_VALUE | + FIELD_PREP(GENMASK(11, 8), pcie->tx_tag) | + TLP_HEADER_BYTE_EN(size, where); + writel(cfg_val, pcie->reg + ASPEED_H2X_TX_DESC1); + + writel(bdf_offset, pcie->reg + ASPEED_H2X_TX_DESC2); + writel(0, pcie->reg + ASPEED_H2X_TX_DESC3); + if (write) + writel(TLP_SET_VALUE(*val, size, where), + pcie->reg + ASPEED_H2X_TX_DESC_DATA); + + cfg_val = readl(pcie->reg + ASPEED_H2X_STS); + cfg_val |= ASPEED_PCIE_TRIGGER_TX; + writel(cfg_val, pcie->reg + ASPEED_H2X_STS); + + ret = readl_poll_timeout(pcie->reg + ASPEED_H2X_STS, cfg_val, + (cfg_val & ASPEED_PCIE_TX_IDLE), 0, 50); + if (ret) { + dev_err(pcie->dev, + "%02x:%02x.%d CR tx timeout sts: 0x%08x\n", + bus->number, PCI_SLOT(devfn), PCI_FUNC(devfn), cfg_val); + ret = PCIBIOS_SET_FAILED; + PCI_SET_ERROR_RESPONSE(val); + goto out; + } + + cfg_val = readl(pcie->reg + ASPEED_H2X_INT_STS); + cfg_val |= ASPEED_PCIE_TX_IDLE_CLEAR; + writel(cfg_val, pcie->reg + ASPEED_H2X_INT_STS); + + cfg_val = readl(pcie->reg + ASPEED_H2X_STS); + switch (cfg_val & ASPEED_PCIE_STATUS_OF_TX) { + case ASPEED_PCIE_RC_H_TX_COMPLETE: + ret = readl_poll_timeout(pcie->reg + ASPEED_H2X_DEV_STS, isr, + (isr & ASPEED_PCIE_RC_RX_DONE_ISR), 0, + 50); + if (ret) { + dev_err(pcie->dev, + "%02x:%02x.%d CR rx timeout sts: 0x%08x\n", + bus->number, PCI_SLOT(devfn), + PCI_FUNC(devfn), isr); + ret = PCIBIOS_SET_FAILED; + PCI_SET_ERROR_RESPONSE(val); + goto out; + } + if (!write) { + cfg_val = readl(pcie->reg + ASPEED_H2X_DEV_RX_DESC1); + if (CPL_STS(cfg_val) != PCIE_CPL_STS_SUCCESS) { + ret = PCIBIOS_SET_FAILED; + PCI_SET_ERROR_RESPONSE(val); + goto out; + } else { + *val = readl(pcie->reg + + ASPEED_H2X_DEV_RX_DESC_DATA); + } + } + break; + case ASPEED_PCIE_STATUS_OF_TX: + ret = PCIBIOS_SET_FAILED; + PCI_SET_ERROR_RESPONSE(val); + goto out; + default: + *val = readl(pcie->reg + ASPEED_H2X_HOST_RX_DESC_DATA); + break; + } + + cfg_val = readl(pcie->reg + ASPEED_H2X_DEV_CTRL); + cfg_val |= ASPEED_PCIE_UNLOCK_RX_BUFF; + writel(cfg_val, pcie->reg + ASPEED_H2X_DEV_CTRL); + + *val = TLP_GET_VALUE(*val, size, where); + + ret = PCIBIOS_SUCCESSFUL; +out: + cfg_val = readl(pcie->reg + ASPEED_H2X_DEV_STS); + writel(cfg_val, pcie->reg + ASPEED_H2X_DEV_STS); + pcie->tx_tag = (pcie->tx_tag + 1) % 0x8; + return ret; +} + +static int aspeed_ast2600_rd_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + /* + * AST2600 has only one Root Port on the root bus. + */ + if (PCI_SLOT(devfn) != 8) + return PCIBIOS_DEVICE_NOT_FOUND; + + return aspeed_ast2600_conf(bus, devfn, where, size, val, + CFG0_READ_FMTTYPE, false); +} + +static int aspeed_ast2600_child_rd_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + return aspeed_ast2600_conf(bus, devfn, where, size, val, + CFG1_READ_FMTTYPE, false); +} + +static int aspeed_ast2600_wr_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + /* + * AST2600 has only one Root Port on the root bus. + */ + if (PCI_SLOT(devfn) != 8) + return PCIBIOS_DEVICE_NOT_FOUND; + + return aspeed_ast2600_conf(bus, devfn, where, size, &val, + CFG0_WRITE_FMTTYPE, true); +} + +static int aspeed_ast2600_child_wr_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + return aspeed_ast2600_conf(bus, devfn, where, size, &val, + CFG1_WRITE_FMTTYPE, true); +} + +static int aspeed_ast2700_config(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val, bool write) +{ + struct aspeed_pcie *pcie = bus->sysdata; + u32 cfg_val; + + cfg_val = ASPEED_CFGI_BYTE_EN(TLP_HEADER_BYTE_EN(size, where)) | + (where & ~3); + if (write) + cfg_val |= ASPEED_CFGI_WRITE; + writel(cfg_val, pcie->reg + ASPEED_H2X_CFGI_TLP); + + writel(TLP_SET_VALUE(*val, size, where), + pcie->reg + ASPEED_H2X_CFGI_WR_DATA); + writel(ASPEED_CFGI_TLP_FIRE, pcie->reg + ASPEED_H2X_CFGI_CTRL); + *val = readl(pcie->reg + ASPEED_H2X_CFGI_RET_DATA); + *val = TLP_GET_VALUE(*val, size, where); + + return PCIBIOS_SUCCESSFUL; +} + +static int aspeed_ast2700_child_config(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val, + bool write) +{ + struct aspeed_pcie *pcie = bus->sysdata; + u32 bdf_offset, status, cfg_val; + int ret; + + bdf_offset = aspeed_pcie_get_bdf_offset(bus, devfn, where); + + cfg_val = CFG_PAYLOAD_SIZE; + if (write) + cfg_val |= (bus->number == (pcie->root_bus_nr + 1)) ? + CFG0_WRITE_FMTTYPE : + CFG1_WRITE_FMTTYPE; + else + cfg_val |= (bus->number == (pcie->root_bus_nr + 1)) ? + CFG0_READ_FMTTYPE : + CFG1_READ_FMTTYPE; + writel(cfg_val, pcie->reg + ASPEED_H2X_CFGE_TLP_1ST); + + cfg_val = AST2700_TX_DESC1_VALUE | + FIELD_PREP(GENMASK(11, 8), pcie->tx_tag) | + TLP_HEADER_BYTE_EN(size, where); + writel(cfg_val, pcie->reg + ASPEED_H2X_CFGE_TLP_NEXT); + + writel(bdf_offset, pcie->reg + ASPEED_H2X_CFGE_TLP_NEXT); + if (write) + writel(TLP_SET_VALUE(*val, size, where), + pcie->reg + ASPEED_H2X_CFGE_TLP_NEXT); + writel(ASPEED_CFGE_TX_IDLE | ASPEED_CFGE_RX_BUSY, + pcie->reg + ASPEED_H2X_CFGE_INT_STS); + writel(ASPEED_CFGE_TLP_FIRE, pcie->reg + ASPEED_H2X_CFGE_CTRL); + + ret = readl_poll_timeout(pcie->reg + ASPEED_H2X_CFGE_INT_STS, status, + (status & ASPEED_CFGE_TX_IDLE), 0, 50); + if (ret) { + dev_err(pcie->dev, + "%02x:%02x.%d CR tx timeout sts: 0x%08x\n", + bus->number, PCI_SLOT(devfn), PCI_FUNC(devfn), status); + ret = PCIBIOS_SET_FAILED; + PCI_SET_ERROR_RESPONSE(val); + goto out; + } + + ret = readl_poll_timeout(pcie->reg + ASPEED_H2X_CFGE_INT_STS, status, + (status & ASPEED_CFGE_RX_BUSY), 0, 50); + if (ret) { + dev_err(pcie->dev, + "%02x:%02x.%d CR rx timeout sts: 0x%08x\n", + bus->number, PCI_SLOT(devfn), PCI_FUNC(devfn), status); + ret = PCIBIOS_SET_FAILED; + PCI_SET_ERROR_RESPONSE(val); + goto out; + } + *val = readl(pcie->reg + ASPEED_H2X_CFGE_RET_DATA); + *val = TLP_GET_VALUE(*val, size, where); + + ret = PCIBIOS_SUCCESSFUL; +out: + writel(status, pcie->reg + ASPEED_H2X_CFGE_INT_STS); + pcie->tx_tag = (pcie->tx_tag + 1) % 0xf; + return ret; +} + +static int aspeed_ast2700_rd_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + /* + * AST2700 has only one Root Port on the root bus. + */ + if (devfn != 0) + return PCIBIOS_DEVICE_NOT_FOUND; + + return aspeed_ast2700_config(bus, devfn, where, size, val, false); +} + +static int aspeed_ast2700_child_rd_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + return aspeed_ast2700_child_config(bus, devfn, where, size, val, false); +} + +static int aspeed_ast2700_wr_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + /* + * AST2700 has only one Root Port on the root bus. + */ + if (devfn != 0) + return PCIBIOS_DEVICE_NOT_FOUND; + + return aspeed_ast2700_config(bus, devfn, where, size, &val, true); +} + +static int aspeed_ast2700_child_wr_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + return aspeed_ast2700_child_config(bus, devfn, where, size, &val, true); +} + +static struct pci_ops aspeed_ast2600_pcie_ops = { + .read = aspeed_ast2600_rd_conf, + .write = aspeed_ast2600_wr_conf, +}; + +static struct pci_ops aspeed_ast2600_pcie_child_ops = { + .read = aspeed_ast2600_child_rd_conf, + .write = aspeed_ast2600_child_wr_conf, +}; + +static struct pci_ops aspeed_ast2700_pcie_ops = { + .read = aspeed_ast2700_rd_conf, + .write = aspeed_ast2700_wr_conf, +}; + +static struct pci_ops aspeed_ast2700_pcie_child_ops = { + .read = aspeed_ast2700_child_rd_conf, + .write = aspeed_ast2700_child_wr_conf, +}; + +static void aspeed_irq_compose_msi_msg(struct irq_data *data, + struct msi_msg *msg) +{ + struct aspeed_pcie *pcie = irq_data_get_irq_chip_data(data); + + msg->address_hi = 0; + msg->address_lo = pcie->platform->msi_address; + msg->data = data->hwirq; +} + +static struct irq_chip aspeed_msi_bottom_irq_chip = { + .name = "ASPEED MSI", + .irq_compose_msi_msg = aspeed_irq_compose_msi_msg, +}; + +static int aspeed_irq_msi_domain_alloc(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs, + void *args) +{ + struct aspeed_pcie *pcie = domain->host_data; + int bit; + int i; + + guard(mutex)(&pcie->lock); + + bit = bitmap_find_free_region(pcie->msi_irq_in_use, MAX_MSI_HOST_IRQS, + get_count_order(nr_irqs)); + + if (bit < 0) + return -ENOSPC; + + for (i = 0; i < nr_irqs; i++) { + irq_domain_set_info(domain, virq + i, bit + i, + &aspeed_msi_bottom_irq_chip, + domain->host_data, handle_simple_irq, NULL, + NULL); + } + + return 0; +} + +static void aspeed_irq_msi_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + struct irq_data *data = irq_domain_get_irq_data(domain, virq); + struct aspeed_pcie *pcie = irq_data_get_irq_chip_data(data); + + guard(mutex)(&pcie->lock); + + bitmap_release_region(pcie->msi_irq_in_use, data->hwirq, + get_count_order(nr_irqs)); +} + +static const struct irq_domain_ops aspeed_msi_domain_ops = { + .alloc = aspeed_irq_msi_domain_alloc, + .free = aspeed_irq_msi_domain_free, +}; + +#define ASPEED_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_NO_AFFINITY) + +#define ASPEED_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_MULTI_PCI_MSI | \ + MSI_FLAG_PCI_MSIX) + +static const struct msi_parent_ops aspeed_msi_parent_ops = { + .required_flags = ASPEED_MSI_FLAGS_REQUIRED, + .supported_flags = ASPEED_MSI_FLAGS_SUPPORTED, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .chip_flags = MSI_CHIP_FLAG_SET_ACK, + .prefix = "ASPEED-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +static int aspeed_pcie_msi_init(struct aspeed_pcie *pcie) +{ + writel(~0, pcie->reg + pcie->platform->reg_msi_en); + writel(~0, pcie->reg + pcie->platform->reg_msi_en + 0x04); + writel(~0, pcie->reg + pcie->platform->reg_msi_sts); + writel(~0, pcie->reg + pcie->platform->reg_msi_sts + 0x04); + + struct irq_domain_info info = { + .fwnode = dev_fwnode(pcie->dev), + .ops = &aspeed_msi_domain_ops, + .host_data = pcie, + .size = MAX_MSI_HOST_IRQS, + }; + + pcie->msi_domain = msi_create_parent_irq_domain(&info, + &aspeed_msi_parent_ops); + if (!pcie->msi_domain) + return dev_err_probe(pcie->dev, -ENOMEM, + "failed to create MSI domain\n"); + + return 0; +} + +static void aspeed_pcie_msi_free(struct aspeed_pcie *pcie) +{ + if (pcie->msi_domain) { + irq_domain_remove(pcie->msi_domain); + pcie->msi_domain = NULL; + } +} + +static void aspeed_pcie_irq_domain_free(void *d) +{ + struct aspeed_pcie *pcie = d; + + if (pcie->intx_domain) { + irq_domain_remove(pcie->intx_domain); + pcie->intx_domain = NULL; + } + aspeed_pcie_msi_free(pcie); +} + +static int aspeed_pcie_init_irq_domain(struct aspeed_pcie *pcie) +{ + int ret; + + pcie->intx_domain = irq_domain_create_linear(dev_fwnode(pcie->dev), + PCI_NUM_INTX, + &aspeed_intx_domain_ops, + pcie); + if (!pcie->intx_domain) { + ret = dev_err_probe(pcie->dev, -ENOMEM, + "failed to get INTx IRQ domain\n"); + goto err; + } + + writel(0, pcie->reg + pcie->platform->reg_intx_en); + writel(~0, pcie->reg + pcie->platform->reg_intx_sts); + + ret = aspeed_pcie_msi_init(pcie); + if (ret) + goto err; + + return 0; +err: + aspeed_pcie_irq_domain_free(pcie); + return ret; +} + +static int aspeed_pcie_port_init(struct aspeed_pcie_port *port) +{ + struct aspeed_pcie *pcie = port->pcie; + struct device *dev = pcie->dev; + int ret; + + ret = clk_prepare_enable(port->clk); + if (ret) + return dev_err_probe(dev, ret, + "failed to set clock for slot (%d)\n", + port->slot); + + ret = phy_init(port->phy); + if (ret) + return dev_err_probe(dev, ret, + "failed to init phy pcie for slot (%d)\n", + port->slot); + + ret = phy_set_mode_ext(port->phy, PHY_MODE_PCIE, PHY_MODE_PCIE_RC); + if (ret) + return dev_err_probe(dev, ret, + "failed to set phy mode for slot (%d)\n", + port->slot); + + reset_control_deassert(port->perst); + msleep(PCIE_RESET_CONFIG_WAIT_MS); + + return 0; +} + +static void aspeed_host_reset(struct aspeed_pcie *pcie) +{ + reset_control_assert(pcie->h2xrst); + mdelay(ASPEED_RESET_RC_WAIT_MS); + reset_control_deassert(pcie->h2xrst); +} + +static void aspeed_pcie_map_ranges(struct aspeed_pcie *pcie) +{ + struct pci_host_bridge *bridge = pcie->host; + struct resource_entry *window; + + resource_list_for_each_entry(window, &bridge->windows) { + u64 pci_addr; + + if (resource_type(window->res) != IORESOURCE_MEM) + continue; + + pci_addr = window->res->start - window->offset; + pcie->platform->pcie_map_ranges(pcie, pci_addr); + break; + } +} + +static void aspeed_ast2600_pcie_map_ranges(struct aspeed_pcie *pcie, + u64 pci_addr) +{ + u32 pci_addr_lo = pci_addr & GENMASK(31, 0); + u32 pci_addr_hi = (pci_addr >> 32) & GENMASK(31, 0); + + pci_addr_lo >>= 16; + writel(ASPEED_AHB_REMAP_LO_ADDR(pci_addr_lo) | + ASPEED_AHB_MASK_LO_ADDR(0xe00), + pcie->reg + ASPEED_H2X_AHB_ADDR_CONFIG0); + writel(ASPEED_AHB_REMAP_HI_ADDR(pci_addr_hi), + pcie->reg + ASPEED_H2X_AHB_ADDR_CONFIG1); + writel(ASPEED_AHB_MASK_HI_ADDR(~0), + pcie->reg + ASPEED_H2X_AHB_ADDR_CONFIG2); +} + +static int aspeed_ast2600_setup(struct platform_device *pdev) +{ + struct aspeed_pcie *pcie = platform_get_drvdata(pdev); + struct device *dev = pcie->dev; + + pcie->ahbc = syscon_regmap_lookup_by_phandle(dev->of_node, + "aspeed,ahbc"); + if (IS_ERR(pcie->ahbc)) + return dev_err_probe(dev, PTR_ERR(pcie->ahbc), + "failed to map ahbc base\n"); + + aspeed_host_reset(pcie); + + regmap_write(pcie->ahbc, ASPEED_AHBC_KEY, ASPEED_AHBC_UNLOCK_KEY); + regmap_update_bits(pcie->ahbc, ASPEED_AHBC_ADDR_MAPPING, + ASPEED_PCIE_RC_MEMORY_EN, ASPEED_PCIE_RC_MEMORY_EN); + regmap_write(pcie->ahbc, ASPEED_AHBC_KEY, ASPEED_AHBC_UNLOCK); + + writel(ASPEED_H2X_BRIDGE_EN, pcie->reg + ASPEED_H2X_CTRL); + + writel(ASPEED_PCIE_RX_DMA_EN | ASPEED_PCIE_RX_LINEAR | + ASPEED_PCIE_RX_MSI_SEL | ASPEED_PCIE_RX_MSI_EN | + ASPEED_PCIE_WAIT_RX_TLP_CLR | ASPEED_PCIE_RC_RX_ENABLE | + ASPEED_PCIE_RC_ENABLE, + pcie->reg + ASPEED_H2X_DEV_CTRL); + + writel(ASPEED_RC_TLP_TX_TAG_NUM, pcie->reg + ASPEED_H2X_DEV_TX_TAG); + + pcie->host->ops = &aspeed_ast2600_pcie_ops; + pcie->host->child_ops = &aspeed_ast2600_pcie_child_ops; + + return 0; +} + +static void aspeed_ast2700_pcie_map_ranges(struct aspeed_pcie *pcie, + u64 pci_addr) +{ + writel(ASPEED_REMAP_PCI_ADDR_31_12(pci_addr), + pcie->reg + ASPEED_H2X_REMAP_PCI_ADDR_LO); + writel(ASPEED_REMAP_PCI_ADDR_63_32(pci_addr), + pcie->reg + ASPEED_H2X_REMAP_PCI_ADDR_HI); +} + +static int aspeed_ast2700_setup(struct platform_device *pdev) +{ + struct aspeed_pcie *pcie = platform_get_drvdata(pdev); + struct device *dev = pcie->dev; + + pcie->cfg = syscon_regmap_lookup_by_phandle(dev->of_node, + "aspeed,pciecfg"); + if (IS_ERR(pcie->cfg)) + return dev_err_probe(dev, PTR_ERR(pcie->cfg), + "failed to map pciecfg base\n"); + + regmap_update_bits(pcie->cfg, ASPEED_SCU_60, + ASPEED_RC_E2M_PATH_EN | ASPEED_RC_H2XS_PATH_EN | + ASPEED_RC_H2XD_PATH_EN | ASPEED_RC_H2XX_PATH_EN | + ASPEED_RC_UPSTREAM_MEM_EN, + ASPEED_RC_E2M_PATH_EN | ASPEED_RC_H2XS_PATH_EN | + ASPEED_RC_H2XD_PATH_EN | ASPEED_RC_H2XX_PATH_EN | + ASPEED_RC_UPSTREAM_MEM_EN); + regmap_write(pcie->cfg, ASPEED_SCU_64, + ASPEED_RC0_DECODE_DMA_BASE(0) | + ASPEED_RC0_DECODE_DMA_LIMIT(0xff) | + ASPEED_RC1_DECODE_DMA_BASE(0) | + ASPEED_RC1_DECODE_DMA_LIMIT(0xff)); + regmap_write(pcie->cfg, ASPEED_SCU_70, ASPEED_DISABLE_EP_FUNC); + + aspeed_host_reset(pcie); + + writel(0, pcie->reg + ASPEED_H2X_CTRL); + writel(ASPEED_H2X_BRIDGE_EN | ASPEED_H2X_BRIDGE_DIRECT_EN, + pcie->reg + ASPEED_H2X_CTRL); + + /* Prepare for 64-bit BAR pref */ + writel(ASPEED_REMAP_PREF_ADDR_63_32(0x3), + pcie->reg + ASPEED_H2X_REMAP_PREF_ADDR); + + pcie->host->ops = &aspeed_ast2700_pcie_ops; + pcie->host->child_ops = &aspeed_ast2700_pcie_child_ops; + pcie->clear_msi_twice = true; + + return 0; +} + +static void aspeed_pcie_reset_release(void *d) +{ + struct reset_control *perst = d; + + if (!perst) + return; + + reset_control_put(perst); +} + +static int aspeed_pcie_parse_port(struct aspeed_pcie *pcie, + struct device_node *node, + int slot) +{ + struct aspeed_pcie_port *port; + struct device *dev = pcie->dev; + int ret; + + port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); + if (!port) + return -ENOMEM; + + port->clk = devm_get_clk_from_child(dev, node, NULL); + if (IS_ERR(port->clk)) + return dev_err_probe(dev, PTR_ERR(port->clk), + "failed to get pcie%d clock\n", slot); + + port->phy = devm_of_phy_get(dev, node, NULL); + if (IS_ERR(port->phy)) + return dev_err_probe(dev, PTR_ERR(port->phy), + "failed to get phy pcie%d\n", slot); + + port->perst = of_reset_control_get_exclusive(node, "perst"); + if (IS_ERR(port->perst)) + return dev_err_probe(dev, PTR_ERR(port->perst), + "failed to get pcie%d reset control\n", + slot); + ret = devm_add_action_or_reset(dev, aspeed_pcie_reset_release, + port->perst); + if (ret) + return ret; + reset_control_assert(port->perst); + + port->slot = slot; + port->pcie = pcie; + + INIT_LIST_HEAD(&port->list); + list_add_tail(&port->list, &pcie->ports); + + ret = aspeed_pcie_port_init(port); + if (ret) + return ret; + + return 0; +} + +static int aspeed_pcie_parse_dt(struct aspeed_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct device_node *node = dev->of_node; + int ret; + + for_each_available_child_of_node_scoped(node, child) { + int slot; + const char *type; + + ret = of_property_read_string(child, "device_type", &type); + if (ret || strcmp(type, "pci")) + continue; + + ret = of_pci_get_devfn(child); + if (ret < 0) + return dev_err_probe(dev, ret, + "failed to parse devfn\n"); + + slot = PCI_SLOT(ret); + + ret = aspeed_pcie_parse_port(pcie, child, slot); + if (ret) + return ret; + } + + if (list_empty(&pcie->ports)) + return dev_err_probe(dev, -ENODEV, + "No PCIe port found in DT\n"); + + return 0; +} + +static int aspeed_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct pci_host_bridge *host; + struct aspeed_pcie *pcie; + struct resource_entry *entry; + const struct aspeed_pcie_rc_platform *md; + int irq, ret; + + md = of_device_get_match_data(dev); + if (!md) + return -ENODEV; + + host = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!host) + return -ENOMEM; + + pcie = pci_host_bridge_priv(host); + pcie->dev = dev; + pcie->tx_tag = 0; + platform_set_drvdata(pdev, pcie); + + pcie->platform = md; + pcie->host = host; + INIT_LIST_HEAD(&pcie->ports); + + /* Get root bus num for cfg command to decide tlp type 0 or type 1 */ + entry = resource_list_first_type(&host->windows, IORESOURCE_BUS); + if (entry) + pcie->root_bus_nr = entry->res->start; + + pcie->reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(pcie->reg)) + return PTR_ERR(pcie->reg); + + pcie->h2xrst = devm_reset_control_get_exclusive(dev, "h2x"); + if (IS_ERR(pcie->h2xrst)) + return dev_err_probe(dev, PTR_ERR(pcie->h2xrst), + "failed to get h2x reset\n"); + + ret = devm_mutex_init(dev, &pcie->lock); + if (ret) + return dev_err_probe(dev, ret, "failed to init mutex\n"); + + ret = pcie->platform->setup(pdev); + if (ret) + return dev_err_probe(dev, ret, "failed to setup PCIe RC\n"); + + aspeed_pcie_map_ranges(pcie); + + ret = aspeed_pcie_parse_dt(pcie); + if (ret) + return ret; + + host->sysdata = pcie; + + ret = aspeed_pcie_init_irq_domain(pcie); + if (ret) + return ret; + + ret = devm_add_action_or_reset(dev, aspeed_pcie_irq_domain_free, pcie); + if (ret) + return ret; + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + ret = devm_request_irq(dev, irq, aspeed_pcie_intr_handler, IRQF_SHARED, + dev_name(dev), pcie); + if (ret) + return ret; + + return pci_host_probe(host); +} + +static const struct aspeed_pcie_rc_platform pcie_rc_ast2600 = { + .setup = aspeed_ast2600_setup, + .pcie_map_ranges = aspeed_ast2600_pcie_map_ranges, + .reg_intx_en = 0xc4, + .reg_intx_sts = 0xc8, + .reg_msi_en = 0xe0, + .reg_msi_sts = 0xe8, + .msi_address = 0x1e77005c, +}; + +static const struct aspeed_pcie_rc_platform pcie_rc_ast2700 = { + .setup = aspeed_ast2700_setup, + .pcie_map_ranges = aspeed_ast2700_pcie_map_ranges, + .reg_intx_en = 0x40, + .reg_intx_sts = 0x48, + .reg_msi_en = 0x50, + .reg_msi_sts = 0x58, + .msi_address = 0x000000f0, +}; + +static const struct of_device_id aspeed_pcie_of_match[] = { + { .compatible = "aspeed,ast2600-pcie", .data = &pcie_rc_ast2600 }, + { .compatible = "aspeed,ast2700-pcie", .data = &pcie_rc_ast2700 }, + {} +}; + +static struct platform_driver aspeed_pcie_driver = { + .driver = { + .name = "aspeed-pcie", + .of_match_table = aspeed_pcie_of_match, + .suppress_bind_attrs = true, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, + .probe = aspeed_pcie_probe, +}; + +builtin_platform_driver(aspeed_pcie_driver); + +MODULE_AUTHOR("Jacky Chou <jacky_chou@aspeedtech.com>"); +MODULE_DESCRIPTION("ASPEED PCIe Root Complex"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c new file mode 100644 index 000000000..8a0c353d2 --- /dev/null +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -0,0 +1,2216 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* Copyright (C) 2009 - 2019 Broadcom */ + +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/clk.h> +#include <linux/compiler.h> +#include <linux/delay.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/ioport.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/irqdomain.h> +#include <linux/kdebug.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/log2.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/notifier.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/panic_notifier.h> +#include <linux/pci.h> +#include <linux/pci-ecam.h> +#include <linux/printk.h> +#include <linux/regulator/consumer.h> +#include <linux/reset.h> +#include <linux/sizes.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/string.h> +#include <linux/string_choices.h> +#include <linux/types.h> + +#include "../pci.h" + +/* BRCM_PCIE_CAP_REGS - Offset for the mandatory capability config regs */ +#define BRCM_PCIE_CAP_REGS 0x00ac + +/* Broadcom STB PCIe Register Offsets */ +#define PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1 0x0188 +#define PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1_ENDIAN_MODE_BAR2_MASK 0xc +#define PCIE_RC_CFG_VENDOR_SPECIFIC_REG1_LITTLE_ENDIAN 0x0 + +#define PCIE_RC_CFG_PRIV1_ID_VAL3 0x043c +#define PCIE_RC_CFG_PRIV1_ID_VAL3_CLASS_CODE_MASK 0xffffff + +#define PCIE_RC_CFG_PRIV1_LINK_CAPABILITY 0x04dc +#define PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_MAX_LINK_WIDTH_MASK 0x1f0 + +#define PCIE_RC_CFG_PRIV1_ROOT_CAP 0x4f8 +#define PCIE_RC_CFG_PRIV1_ROOT_CAP_L1SS_MODE_MASK 0xf8 + +#define PCIE_RC_DL_MDIO_ADDR 0x1100 +#define PCIE_RC_DL_MDIO_WR_DATA 0x1104 +#define PCIE_RC_DL_MDIO_RD_DATA 0x1108 + +#define PCIE_RC_PL_REG_PHY_CTL_1 0x1804 +#define PCIE_RC_PL_REG_PHY_CTL_1_REG_P2_POWERDOWN_ENA_NOSYNC_MASK 0x8 + +#define PCIE_RC_PL_PHY_CTL_15 0x184c +#define PCIE_RC_PL_PHY_CTL_15_DIS_PLL_PD_MASK 0x400000 +#define PCIE_RC_PL_PHY_CTL_15_PM_CLK_PERIOD_MASK 0xff + +#define PCIE_MISC_MISC_CTRL 0x4008 +#define PCIE_MISC_MISC_CTRL_PCIE_RCB_64B_MODE_MASK 0x80 +#define PCIE_MISC_MISC_CTRL_PCIE_RCB_MPS_MODE_MASK 0x400 +#define PCIE_MISC_MISC_CTRL_SCB_ACCESS_EN_MASK 0x1000 +#define PCIE_MISC_MISC_CTRL_CFG_READ_UR_MODE_MASK 0x2000 +#define PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK 0x300000 + +#define PCIE_MISC_MISC_CTRL_SCB0_SIZE_MASK 0xf8000000 +#define PCIE_MISC_MISC_CTRL_SCB1_SIZE_MASK 0x07c00000 +#define PCIE_MISC_MISC_CTRL_SCB2_SIZE_MASK 0x0000001f +#define SCB_SIZE_MASK(x) PCIE_MISC_MISC_CTRL_SCB ## x ## _SIZE_MASK + +#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO 0x400c +#define PCIE_MEM_WIN0_LO(win) \ + PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LO + ((win) * 8) + +#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI 0x4010 +#define PCIE_MEM_WIN0_HI(win) \ + PCIE_MISC_CPU_2_PCIE_MEM_WIN0_HI + ((win) * 8) + +/* + * NOTE: You may see the term "BAR" in a number of register names used by + * this driver. The term is an artifact of when the HW core was an + * endpoint device (EP). Now it is a root complex (RC) and anywhere a + * register has the term "BAR" it is related to an inbound window. + */ + +#define PCIE_BRCM_MAX_INBOUND_WINS 16 +#define PCIE_MISC_RC_BAR1_CONFIG_LO 0x402c +#define PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK 0x1f + +#define PCIE_MISC_RC_BAR4_CONFIG_LO 0x40d4 + + +#define PCIE_MISC_MSI_BAR_CONFIG_LO 0x4044 +#define PCIE_MISC_MSI_BAR_CONFIG_HI 0x4048 + +#define PCIE_MISC_MSI_DATA_CONFIG 0x404c +#define PCIE_MISC_MSI_DATA_CONFIG_VAL_32 0xffe06540 +#define PCIE_MISC_MSI_DATA_CONFIG_VAL_8 0xfff86540 + +#define PCIE_MISC_PCIE_CTRL 0x4064 +#define PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK 0x1 +#define PCIE_MISC_PCIE_CTRL_PCIE_PERSTB_MASK 0x4 + +#define PCIE_MISC_PCIE_STATUS 0x4068 +#define PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK 0x80 +#define PCIE_MISC_PCIE_STATUS_PCIE_DL_ACTIVE_MASK 0x20 +#define PCIE_MISC_PCIE_STATUS_PCIE_PHYLINKUP_MASK 0x10 +#define PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK 0x40 + +#define PCIE_MISC_REVISION 0x406c +#define BRCM_PCIE_HW_REV_33 0x0303 +#define BRCM_PCIE_HW_REV_3_20 0x0320 + +#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT 0x4070 +#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_LIMIT_MASK 0xfff00000 +#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK 0xfff0 +#define PCIE_MEM_WIN0_BASE_LIMIT(win) \ + PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT + ((win) * 4) + +#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI 0x4080 +#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI_BASE_MASK 0xff +#define PCIE_MEM_WIN0_BASE_HI(win) \ + PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI + ((win) * 8) + +#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI 0x4084 +#define PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI_LIMIT_MASK 0xff +#define PCIE_MEM_WIN0_LIMIT_HI(win) \ + PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI + ((win) * 8) + +#define PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK 0x2 +#define PCIE_MISC_HARD_PCIE_HARD_DEBUG_L1SS_ENABLE_MASK 0x200000 +#define PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK 0x08000000 +#define PCIE_BMIPS_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK 0x00800000 +#define PCIE_CLKREQ_MASK \ + (PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK | \ + PCIE_MISC_HARD_PCIE_HARD_DEBUG_L1SS_ENABLE_MASK) + +#define PCIE_MISC_UBUS_BAR1_CONFIG_REMAP 0x40ac +#define PCIE_MISC_UBUS_BAR1_CONFIG_REMAP_ACCESS_EN_MASK BIT(0) +#define PCIE_MISC_UBUS_BAR4_CONFIG_REMAP 0x410c + +#define PCIE_MSI_INTR2_BASE 0x4500 + +/* Offsets from INTR2_CPU and MSI_INTR2 BASE offsets */ +#define MSI_INT_STATUS 0x0 +#define MSI_INT_CLR 0x8 +#define MSI_INT_MASK_SET 0x10 +#define MSI_INT_MASK_CLR 0x14 + +/* Error report registers */ +#define PCIE_OUTB_ERR_TREAT 0x6000 +#define PCIE_OUTB_ERR_TREAT_CONFIG 0x1 +#define PCIE_OUTB_ERR_TREAT_MEM 0x2 +#define PCIE_OUTB_ERR_VALID 0x6004 +#define PCIE_OUTB_ERR_CLEAR 0x6008 +#define PCIE_OUTB_ERR_ACC_INFO 0x600c +#define PCIE_OUTB_ERR_ACC_INFO_CFG_ERR BIT(0) +#define PCIE_OUTB_ERR_ACC_INFO_MEM_ERR BIT(1) +#define PCIE_OUTB_ERR_ACC_INFO_TYPE_64 BIT(2) +#define PCIE_OUTB_ERR_ACC_INFO_DIR_WRITE BIT(4) +#define PCIE_OUTB_ERR_ACC_INFO_BYTE_LANES 0xff00 +#define PCIE_OUTB_ERR_ACC_ADDR 0x6010 +#define PCIE_OUTB_ERR_ACC_ADDR_BUS 0xff00000 +#define PCIE_OUTB_ERR_ACC_ADDR_DEV 0xf8000 +#define PCIE_OUTB_ERR_ACC_ADDR_FUNC 0x7000 +#define PCIE_OUTB_ERR_ACC_ADDR_REG 0xfff +#define PCIE_OUTB_ERR_CFG_CAUSE 0x6014 +#define PCIE_OUTB_ERR_CFG_CAUSE_TIMEOUT BIT(6) +#define PCIE_OUTB_ERR_CFG_CAUSE_ABORT BIT(5) +#define PCIE_OUTB_ERR_CFG_CAUSE_UNSUPP_REQ BIT(4) +#define PCIE_OUTB_ERR_CFG_CAUSE_ACC_TIMEOUT BIT(2) +#define PCIE_OUTB_ERR_CFG_CAUSE_ACC_DISABLED BIT(1) +#define PCIE_OUTB_ERR_CFG_CAUSE_ACC_64BIT BIT(0) +#define PCIE_OUTB_ERR_MEM_ADDR_LO 0x6018 +#define PCIE_OUTB_ERR_MEM_ADDR_HI 0x601c +#define PCIE_OUTB_ERR_MEM_CAUSE 0x6020 +#define PCIE_OUTB_ERR_MEM_CAUSE_TIMEOUT BIT(6) +#define PCIE_OUTB_ERR_MEM_CAUSE_ABORT BIT(5) +#define PCIE_OUTB_ERR_MEM_CAUSE_UNSUPP_REQ BIT(4) +#define PCIE_OUTB_ERR_MEM_CAUSE_ACC_DISABLED BIT(1) +#define PCIE_OUTB_ERR_MEM_CAUSE_BAD_ADDR BIT(0) + +#define PCIE_RGR1_SW_INIT_1_PERST_MASK 0x1 + +#define RGR1_SW_INIT_1_INIT_GENERIC_MASK 0x2 +#define RGR1_SW_INIT_1_INIT_GENERIC_SHIFT 0x1 +#define RGR1_SW_INIT_1_INIT_7278_MASK 0x1 +#define RGR1_SW_INIT_1_INIT_7278_SHIFT 0x0 + +/* PCIe parameters */ +#define BRCM_NUM_PCIE_OUT_WINS 0x4 +#define BRCM_INT_PCI_MSI_NR 32 +#define BRCM_INT_PCI_MSI_LEGACY_NR 8 +#define BRCM_INT_PCI_MSI_SHIFT 0 +#define BRCM_INT_PCI_MSI_MASK GENMASK(BRCM_INT_PCI_MSI_NR - 1, 0) +#define BRCM_INT_PCI_MSI_LEGACY_MASK GENMASK(31, \ + 32 - BRCM_INT_PCI_MSI_LEGACY_NR) + +/* MSI target addresses */ +#define BRCM_MSI_TARGET_ADDR_LT_4GB 0x0fffffffcULL +#define BRCM_MSI_TARGET_ADDR_GT_4GB 0xffffffffcULL + +/* MDIO registers */ +#define MDIO_PORT0 0x0 +#define MDIO_DATA_MASK 0x7fffffff +#define MDIO_PORT_MASK 0xf0000 +#define MDIO_PORT_EXT_MASK 0x200000 +#define MDIO_REGAD_MASK 0xffff +#define MDIO_CMD_MASK 0x00100000 +#define MDIO_CMD_READ 0x1 +#define MDIO_CMD_WRITE 0x0 +#define MDIO_DATA_DONE_MASK 0x80000000 +#define MDIO_RD_DONE(x) (((x) & MDIO_DATA_DONE_MASK) ? 1 : 0) +#define MDIO_WT_DONE(x) (((x) & MDIO_DATA_DONE_MASK) ? 0 : 1) +#define SSC_REGS_ADDR 0x1100 +#define SET_ADDR_OFFSET 0x1f +#define SSC_CNTL_OFFSET 0x2 +#define SSC_CNTL_OVRD_EN_MASK 0x8000 +#define SSC_CNTL_OVRD_VAL_MASK 0x4000 +#define SSC_STATUS_OFFSET 0x1 +#define SSC_STATUS_SSC_MASK 0x400 +#define SSC_STATUS_PLL_LOCK_MASK 0x800 +#define PCIE_BRCM_MAX_MEMC 3 + +#define IDX_ADDR(pcie) ((pcie)->cfg->offsets[EXT_CFG_INDEX]) +#define DATA_ADDR(pcie) ((pcie)->cfg->offsets[EXT_CFG_DATA]) +#define PCIE_RGR1_SW_INIT_1(pcie) ((pcie)->cfg->offsets[RGR1_SW_INIT_1]) +#define HARD_DEBUG(pcie) ((pcie)->cfg->offsets[PCIE_HARD_DEBUG]) +#define INTR2_CPU_BASE(pcie) ((pcie)->cfg->offsets[PCIE_INTR2_CPU_BASE]) + +/* Rescal registers */ +#define PCIE_DVT_PMU_PCIE_PHY_CTRL 0xc700 +#define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS 0x3 +#define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_MASK 0x4 +#define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_SHIFT 0x2 +#define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_MASK 0x2 +#define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_SHIFT 0x1 +#define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_MASK 0x1 +#define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_SHIFT 0x0 + +/* Forward declarations */ +struct brcm_pcie; + +enum { + RGR1_SW_INIT_1, + EXT_CFG_INDEX, + EXT_CFG_DATA, + PCIE_HARD_DEBUG, + PCIE_INTR2_CPU_BASE, +}; + +enum pcie_soc_base { + GENERIC, + BCM2711, + BCM4908, + BCM7278, + BCM7425, + BCM7435, + BCM7712, +}; + +struct inbound_win { + u64 size; + u64 pci_offset; + u64 cpu_addr; +}; + +/* + * The RESCAL block is tied to PCIe controller #1, regardless of the number of + * controllers, and turning off PCIe controller #1 prevents access to the RESCAL + * register blocks, therefore no other controller can access this register + * space, and depending upon the bus fabric we may get a timeout (UBUS/GISB), + * or a hang (AXI). + */ +#define CFG_QUIRK_AVOID_BRIDGE_SHUTDOWN BIT(0) + +struct pcie_cfg_data { + const int *offsets; + const enum pcie_soc_base soc_base; + const bool has_phy; + const u32 quirks; + u8 num_inbound_wins; + int (*perst_set)(struct brcm_pcie *pcie, u32 val); + int (*bridge_sw_init_set)(struct brcm_pcie *pcie, u32 val); + int (*post_setup)(struct brcm_pcie *pcie); + bool has_err_report; +}; + +struct subdev_regulators { + unsigned int num_supplies; + struct regulator_bulk_data supplies[]; +}; + +struct brcm_msi { + struct device *dev; + void __iomem *base; + struct device_node *np; + struct irq_domain *inner_domain; + struct mutex lock; /* guards the alloc/free operations */ + u64 target_addr; + int irq; + DECLARE_BITMAP(used, BRCM_INT_PCI_MSI_NR); + bool legacy; + /* Some chips have MSIs in bits [31..24] of a shared register. */ + int legacy_shift; + int nr; /* No. of MSI available, depends on chip */ + /* This is the base pointer for interrupt status/set/clr regs */ + void __iomem *intr_base; +}; + +/* Internal PCIe Host Controller Information.*/ +struct brcm_pcie { + struct device *dev; + void __iomem *base; + struct clk *clk; + struct device_node *np; + bool ssc; + int gen; + u64 msi_target_addr; + struct brcm_msi *msi; + struct reset_control *rescal; + struct reset_control *perst_reset; + struct reset_control *bridge_reset; + struct reset_control *swinit_reset; + int num_memc; + u64 memc_size[PCIE_BRCM_MAX_MEMC]; + u32 hw_rev; + struct subdev_regulators *sr; + bool ep_wakeup_capable; + const struct pcie_cfg_data *cfg; + bool bridge_in_reset; + struct notifier_block die_notifier; + struct notifier_block panic_notifier; + spinlock_t bridge_lock; +}; + +static inline bool is_bmips(const struct brcm_pcie *pcie) +{ + return pcie->cfg->soc_base == BCM7435 || pcie->cfg->soc_base == BCM7425; +} + +static int brcm_pcie_bridge_sw_init_set(struct brcm_pcie *pcie, u32 val) +{ + unsigned long flags; + int ret; + + if (pcie->cfg->has_err_report) + spin_lock_irqsave(&pcie->bridge_lock, flags); + + ret = pcie->cfg->bridge_sw_init_set(pcie, val); + /* If we fail, assume the bridge is in reset (off) */ + pcie->bridge_in_reset = ret ? true : val; + + if (pcie->cfg->has_err_report) + spin_unlock_irqrestore(&pcie->bridge_lock, flags); + + return ret; +} + +/* + * This is to convert the size of the inbound "BAR" region to the + * non-linear values of PCIE_X_MISC_RC_BAR[123]_CONFIG_LO.SIZE + */ +static int brcm_pcie_encode_ibar_size(u64 size) +{ + int log2_in = ilog2(size); + + if (log2_in >= 12 && log2_in <= 15) + /* Covers 4KB to 32KB (inclusive) */ + return (log2_in - 12) + 0x1c; + else if (log2_in >= 16 && log2_in <= 36) + /* Covers 64KB to 64GB, (inclusive) */ + return log2_in - 15; + /* Something is awry so disable */ + return 0; +} + +static u32 brcm_pcie_mdio_form_pkt(int port, int regad, int cmd) +{ + u32 pkt = 0; + + pkt |= FIELD_PREP(MDIO_PORT_EXT_MASK, port >> 4); + pkt |= FIELD_PREP(MDIO_PORT_MASK, port); + pkt |= FIELD_PREP(MDIO_REGAD_MASK, regad); + pkt |= FIELD_PREP(MDIO_CMD_MASK, cmd); + + return pkt; +} + +/* negative return value indicates error */ +static int brcm_pcie_mdio_read(void __iomem *base, u8 port, u8 regad, u32 *val) +{ + u32 data; + int err; + + writel(brcm_pcie_mdio_form_pkt(port, regad, MDIO_CMD_READ), + base + PCIE_RC_DL_MDIO_ADDR); + readl(base + PCIE_RC_DL_MDIO_ADDR); + err = readl_poll_timeout_atomic(base + PCIE_RC_DL_MDIO_RD_DATA, data, + MDIO_RD_DONE(data), 10, 100); + *val = FIELD_GET(MDIO_DATA_MASK, data); + + return err; +} + +/* negative return value indicates error */ +static int brcm_pcie_mdio_write(void __iomem *base, u8 port, + u8 regad, u16 wrdata) +{ + u32 data; + int err; + + writel(brcm_pcie_mdio_form_pkt(port, regad, MDIO_CMD_WRITE), + base + PCIE_RC_DL_MDIO_ADDR); + readl(base + PCIE_RC_DL_MDIO_ADDR); + writel(MDIO_DATA_DONE_MASK | wrdata, base + PCIE_RC_DL_MDIO_WR_DATA); + + err = readl_poll_timeout_atomic(base + PCIE_RC_DL_MDIO_WR_DATA, data, + MDIO_WT_DONE(data), 10, 100); + return err; +} + +/* + * Configures device for Spread Spectrum Clocking (SSC) mode; a negative + * return value indicates error. + */ +static int brcm_pcie_set_ssc(struct brcm_pcie *pcie) +{ + int pll, ssc; + int ret; + u32 tmp; + + ret = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0, SET_ADDR_OFFSET, + SSC_REGS_ADDR); + if (ret < 0) + return ret; + + ret = brcm_pcie_mdio_read(pcie->base, MDIO_PORT0, + SSC_CNTL_OFFSET, &tmp); + if (ret < 0) + return ret; + + u32p_replace_bits(&tmp, 1, SSC_CNTL_OVRD_EN_MASK); + u32p_replace_bits(&tmp, 1, SSC_CNTL_OVRD_VAL_MASK); + ret = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0, + SSC_CNTL_OFFSET, tmp); + if (ret < 0) + return ret; + + usleep_range(1000, 2000); + ret = brcm_pcie_mdio_read(pcie->base, MDIO_PORT0, + SSC_STATUS_OFFSET, &tmp); + if (ret < 0) + return ret; + + ssc = FIELD_GET(SSC_STATUS_SSC_MASK, tmp); + pll = FIELD_GET(SSC_STATUS_PLL_LOCK_MASK, tmp); + + return ssc && pll ? 0 : -EIO; +} + +/* Limits operation to a specific generation (1, 2, or 3) */ +static void brcm_pcie_set_gen(struct brcm_pcie *pcie, int gen) +{ + u16 lnkctl2 = readw(pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCTL2); + u32 lnkcap = readl(pcie->base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY); + + u32p_replace_bits(&lnkcap, gen, PCI_EXP_LNKCAP_SLS); + writel(lnkcap, pcie->base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY); + + u16p_replace_bits(&lnkctl2, gen, PCI_EXP_LNKCTL2_TLS); + writew(lnkctl2, pcie->base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKCTL2); +} + +static void brcm_pcie_set_outbound_win(struct brcm_pcie *pcie, + u8 win, u64 cpu_addr, + u64 pcie_addr, u64 size) +{ + u32 cpu_addr_mb_high, limit_addr_mb_high; + phys_addr_t cpu_addr_mb, limit_addr_mb; + int high_addr_shift; + u32 tmp; + + /* Set the base of the pcie_addr window */ + writel(lower_32_bits(pcie_addr), pcie->base + PCIE_MEM_WIN0_LO(win)); + writel(upper_32_bits(pcie_addr), pcie->base + PCIE_MEM_WIN0_HI(win)); + + /* Write the addr base & limit lower bits (in MBs) */ + cpu_addr_mb = cpu_addr / SZ_1M; + limit_addr_mb = (cpu_addr + size - 1) / SZ_1M; + + tmp = readl(pcie->base + PCIE_MEM_WIN0_BASE_LIMIT(win)); + u32p_replace_bits(&tmp, cpu_addr_mb, + PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK); + u32p_replace_bits(&tmp, limit_addr_mb, + PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_LIMIT_MASK); + writel(tmp, pcie->base + PCIE_MEM_WIN0_BASE_LIMIT(win)); + + if (is_bmips(pcie)) + return; + + /* Write the cpu & limit addr upper bits */ + high_addr_shift = + HWEIGHT32(PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT_BASE_MASK); + + cpu_addr_mb_high = cpu_addr_mb >> high_addr_shift; + tmp = readl(pcie->base + PCIE_MEM_WIN0_BASE_HI(win)); + u32p_replace_bits(&tmp, cpu_addr_mb_high, + PCIE_MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI_BASE_MASK); + writel(tmp, pcie->base + PCIE_MEM_WIN0_BASE_HI(win)); + + limit_addr_mb_high = limit_addr_mb >> high_addr_shift; + tmp = readl(pcie->base + PCIE_MEM_WIN0_LIMIT_HI(win)); + u32p_replace_bits(&tmp, limit_addr_mb_high, + PCIE_MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI_LIMIT_MASK); + writel(tmp, pcie->base + PCIE_MEM_WIN0_LIMIT_HI(win)); +} + +#define BRCM_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_NO_AFFINITY) + +#define BRCM_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_MULTI_PCI_MSI) + +static const struct msi_parent_ops brcm_msi_parent_ops = { + .required_flags = BRCM_MSI_FLAGS_REQUIRED, + .supported_flags = BRCM_MSI_FLAGS_SUPPORTED, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .chip_flags = MSI_CHIP_FLAG_SET_ACK, + .prefix = "BRCM-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +static void brcm_pcie_msi_isr(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + unsigned long status; + struct brcm_msi *msi; + struct device *dev; + u32 bit; + + chained_irq_enter(chip, desc); + msi = irq_desc_get_handler_data(desc); + dev = msi->dev; + + status = readl(msi->intr_base + MSI_INT_STATUS); + status >>= msi->legacy_shift; + + for_each_set_bit(bit, &status, msi->nr) { + int ret; + ret = generic_handle_domain_irq(msi->inner_domain, bit); + if (ret) + dev_dbg(dev, "unexpected MSI\n"); + } + + chained_irq_exit(chip, desc); +} + +static void brcm_msi_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct brcm_msi *msi = irq_data_get_irq_chip_data(data); + + msg->address_lo = lower_32_bits(msi->target_addr); + msg->address_hi = upper_32_bits(msi->target_addr); + msg->data = (0xffff & PCIE_MISC_MSI_DATA_CONFIG_VAL_32) | data->hwirq; +} + +static void brcm_msi_ack_irq(struct irq_data *data) +{ + struct brcm_msi *msi = irq_data_get_irq_chip_data(data); + const int shift_amt = data->hwirq + msi->legacy_shift; + + writel(1 << shift_amt, msi->intr_base + MSI_INT_CLR); +} + + +static struct irq_chip brcm_msi_bottom_irq_chip = { + .name = "BRCM STB MSI", + .irq_compose_msi_msg = brcm_msi_compose_msi_msg, + .irq_ack = brcm_msi_ack_irq, +}; + +static int brcm_msi_alloc(struct brcm_msi *msi, unsigned int nr_irqs) +{ + int hwirq; + + mutex_lock(&msi->lock); + hwirq = bitmap_find_free_region(msi->used, msi->nr, + order_base_2(nr_irqs)); + mutex_unlock(&msi->lock); + + return hwirq; +} + +static void brcm_msi_free(struct brcm_msi *msi, unsigned long hwirq, + unsigned int nr_irqs) +{ + mutex_lock(&msi->lock); + bitmap_release_region(msi->used, hwirq, order_base_2(nr_irqs)); + mutex_unlock(&msi->lock); +} + +static int brcm_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct brcm_msi *msi = domain->host_data; + int hwirq, i; + + hwirq = brcm_msi_alloc(msi, nr_irqs); + + if (hwirq < 0) + return hwirq; + + for (i = 0; i < nr_irqs; i++) + irq_domain_set_info(domain, virq + i, (irq_hw_number_t)hwirq + i, + &brcm_msi_bottom_irq_chip, domain->host_data, + handle_edge_irq, NULL, NULL); + return 0; +} + +static void brcm_irq_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct brcm_msi *msi = irq_data_get_irq_chip_data(d); + + brcm_msi_free(msi, d->hwirq, nr_irqs); +} + +static const struct irq_domain_ops msi_domain_ops = { + .alloc = brcm_irq_domain_alloc, + .free = brcm_irq_domain_free, +}; + +static int brcm_allocate_domains(struct brcm_msi *msi) +{ + struct device *dev = msi->dev; + + struct irq_domain_info info = { + .fwnode = of_fwnode_handle(msi->np), + .ops = &msi_domain_ops, + .host_data = msi, + .size = msi->nr, + }; + + msi->inner_domain = msi_create_parent_irq_domain(&info, &brcm_msi_parent_ops); + if (!msi->inner_domain) { + dev_err(dev, "failed to create MSI domain\n"); + return -ENOMEM; + } + + return 0; +} + +static void brcm_free_domains(struct brcm_msi *msi) +{ + irq_domain_remove(msi->inner_domain); +} + +static void brcm_msi_remove(struct brcm_pcie *pcie) +{ + struct brcm_msi *msi = pcie->msi; + + if (!msi) + return; + irq_set_chained_handler_and_data(msi->irq, NULL, NULL); + brcm_free_domains(msi); +} + +static void brcm_msi_set_regs(struct brcm_msi *msi) +{ + u32 val = msi->legacy ? BRCM_INT_PCI_MSI_LEGACY_MASK : + BRCM_INT_PCI_MSI_MASK; + + writel(val, msi->intr_base + MSI_INT_MASK_CLR); + writel(val, msi->intr_base + MSI_INT_CLR); + + /* + * The 0 bit of PCIE_MISC_MSI_BAR_CONFIG_LO is repurposed to MSI + * enable, which we set to 1. + */ + writel(lower_32_bits(msi->target_addr) | 0x1, + msi->base + PCIE_MISC_MSI_BAR_CONFIG_LO); + writel(upper_32_bits(msi->target_addr), + msi->base + PCIE_MISC_MSI_BAR_CONFIG_HI); + + val = msi->legacy ? PCIE_MISC_MSI_DATA_CONFIG_VAL_8 : PCIE_MISC_MSI_DATA_CONFIG_VAL_32; + writel(val, msi->base + PCIE_MISC_MSI_DATA_CONFIG); +} + +static int brcm_pcie_enable_msi(struct brcm_pcie *pcie) +{ + struct brcm_msi *msi; + int irq, ret; + struct device *dev = pcie->dev; + + irq = irq_of_parse_and_map(dev->of_node, 1); + if (irq <= 0) { + dev_err(dev, "cannot map MSI interrupt\n"); + return -ENODEV; + } + + msi = devm_kzalloc(dev, sizeof(struct brcm_msi), GFP_KERNEL); + if (!msi) + return -ENOMEM; + + mutex_init(&msi->lock); + msi->dev = dev; + msi->base = pcie->base; + msi->np = pcie->np; + msi->target_addr = pcie->msi_target_addr; + msi->irq = irq; + msi->legacy = pcie->hw_rev < BRCM_PCIE_HW_REV_33; + + /* + * Sanity check to make sure that the 'used' bitmap in struct brcm_msi + * is large enough. + */ + BUILD_BUG_ON(BRCM_INT_PCI_MSI_LEGACY_NR > BRCM_INT_PCI_MSI_NR); + + if (msi->legacy) { + msi->intr_base = msi->base + INTR2_CPU_BASE(pcie); + msi->nr = BRCM_INT_PCI_MSI_LEGACY_NR; + msi->legacy_shift = 24; + } else { + msi->intr_base = msi->base + PCIE_MSI_INTR2_BASE; + msi->nr = BRCM_INT_PCI_MSI_NR; + msi->legacy_shift = 0; + } + + ret = brcm_allocate_domains(msi); + if (ret) + return ret; + + irq_set_chained_handler_and_data(msi->irq, brcm_pcie_msi_isr, msi); + + brcm_msi_set_regs(msi); + pcie->msi = msi; + + return 0; +} + +/* The controller is capable of serving in both RC and EP roles */ +static bool brcm_pcie_rc_mode(struct brcm_pcie *pcie) +{ + void __iomem *base = pcie->base; + u32 val = readl(base + PCIE_MISC_PCIE_STATUS); + + return !!FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK, val); +} + +static bool brcm_pcie_link_up(struct brcm_pcie *pcie) +{ + u32 val = readl(pcie->base + PCIE_MISC_PCIE_STATUS); + u32 dla = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_DL_ACTIVE_MASK, val); + u32 plu = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_PHYLINKUP_MASK, val); + + return dla && plu; +} + +static void __iomem *brcm_pcie_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + struct brcm_pcie *pcie = bus->sysdata; + void __iomem *base = pcie->base; + int idx; + + /* Accesses to the RC go right to the RC registers if !devfn */ + if (pci_is_root_bus(bus)) + return devfn ? NULL : base + PCIE_ECAM_REG(where); + + /* An access to our HW w/o link-up will cause a CPU Abort */ + if (!brcm_pcie_link_up(pcie)) + return NULL; + + /* For devices, write to the config space index register */ + idx = PCIE_ECAM_OFFSET(bus->number, devfn, 0); + writel(idx, base + IDX_ADDR(pcie)); + return base + DATA_ADDR(pcie) + PCIE_ECAM_REG(where); +} + +static void __iomem *brcm7425_pcie_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + struct brcm_pcie *pcie = bus->sysdata; + void __iomem *base = pcie->base; + int idx; + + /* Accesses to the RC go right to the RC registers if !devfn */ + if (pci_is_root_bus(bus)) + return devfn ? NULL : base + PCIE_ECAM_REG(where); + + /* An access to our HW w/o link-up will cause a CPU Abort */ + if (!brcm_pcie_link_up(pcie)) + return NULL; + + /* For devices, write to the config space index register */ + idx = PCIE_ECAM_OFFSET(bus->number, devfn, where); + writel(idx, base + IDX_ADDR(pcie)); + return base + DATA_ADDR(pcie); +} + +static int brcm_pcie_bridge_sw_init_set_generic(struct brcm_pcie *pcie, u32 val) +{ + u32 tmp, mask = RGR1_SW_INIT_1_INIT_GENERIC_MASK; + u32 shift = RGR1_SW_INIT_1_INIT_GENERIC_SHIFT; + int ret = 0; + + if (pcie->bridge_reset) { + if (val) + ret = reset_control_assert(pcie->bridge_reset); + else + ret = reset_control_deassert(pcie->bridge_reset); + + if (ret) + dev_err(pcie->dev, "failed to %s 'bridge' reset, err=%d\n", + val ? "assert" : "deassert", ret); + + return ret; + } + + tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); + tmp = (tmp & ~mask) | ((val << shift) & mask); + writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); + + return ret; +} + +static int brcm_pcie_bridge_sw_init_set_7278(struct brcm_pcie *pcie, u32 val) +{ + u32 tmp, mask = RGR1_SW_INIT_1_INIT_7278_MASK; + u32 shift = RGR1_SW_INIT_1_INIT_7278_SHIFT; + + tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); + tmp = (tmp & ~mask) | ((val << shift) & mask); + writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); + + return 0; +} + +static int brcm_pcie_perst_set_4908(struct brcm_pcie *pcie, u32 val) +{ + int ret; + + if (WARN_ONCE(!pcie->perst_reset, "missing PERST# reset controller\n")) + return -EINVAL; + + if (val) + ret = reset_control_assert(pcie->perst_reset); + else + ret = reset_control_deassert(pcie->perst_reset); + + if (ret) + dev_err(pcie->dev, "failed to %s 'perst' reset, err=%d\n", + val ? "assert" : "deassert", ret); + return ret; +} + +static int brcm_pcie_perst_set_7278(struct brcm_pcie *pcie, u32 val) +{ + u32 tmp; + + /* Perst bit has moved and assert value is 0 */ + tmp = readl(pcie->base + PCIE_MISC_PCIE_CTRL); + u32p_replace_bits(&tmp, !val, PCIE_MISC_PCIE_CTRL_PCIE_PERSTB_MASK); + writel(tmp, pcie->base + PCIE_MISC_PCIE_CTRL); + + return 0; +} + +static int brcm_pcie_perst_set_generic(struct brcm_pcie *pcie, u32 val) +{ + u32 tmp; + + tmp = readl(pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); + u32p_replace_bits(&tmp, val, PCIE_RGR1_SW_INIT_1_PERST_MASK); + writel(tmp, pcie->base + PCIE_RGR1_SW_INIT_1(pcie)); + + return 0; +} + +static int brcm_pcie_post_setup_bcm2712(struct brcm_pcie *pcie) +{ + static const u16 data[] = { 0x50b9, 0xbda1, 0x0094, 0x97b4, 0x5030, + 0x5030, 0x0007 }; + static const u8 regs[] = { 0x16, 0x17, 0x18, 0x19, 0x1b, 0x1c, 0x1e }; + int ret, i; + u32 tmp; + + /* Allow a 54MHz (xosc) refclk source */ + ret = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0, SET_ADDR_OFFSET, 0x1600); + if (ret < 0) + return ret; + + for (i = 0; i < ARRAY_SIZE(regs); i++) { + ret = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0, regs[i], data[i]); + if (ret < 0) + return ret; + } + + usleep_range(100, 200); + + /* + * Set L1SS sub-state timers to avoid lengthy state transitions, + * PM clock period is 18.52ns (1/54MHz, round down). + */ + tmp = readl(pcie->base + PCIE_RC_PL_PHY_CTL_15); + tmp &= ~PCIE_RC_PL_PHY_CTL_15_PM_CLK_PERIOD_MASK; + tmp |= 0x12; + writel(tmp, pcie->base + PCIE_RC_PL_PHY_CTL_15); + + return 0; +} + +static void add_inbound_win(struct inbound_win *b, u8 *count, u64 size, + u64 cpu_addr, u64 pci_offset) +{ + b->size = size; + b->cpu_addr = cpu_addr; + b->pci_offset = pci_offset; + (*count)++; +} + +static int brcm_pcie_get_inbound_wins(struct brcm_pcie *pcie, + struct inbound_win inbound_wins[]) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + u64 pci_offset, cpu_addr, size = 0, tot_size = 0; + struct resource_entry *entry; + struct device *dev = pcie->dev; + u64 lowest_pcie_addr = ~(u64)0; + int ret, i = 0; + u8 n = 0; + + /* + * The HW registers (and PCIe) use order-1 numbering for BARs. As such, + * we have inbound_wins[0] unused and BAR1 starts at inbound_wins[1]. + */ + struct inbound_win *b_begin = &inbound_wins[1]; + struct inbound_win *b = b_begin; + + /* + * STB chips beside 7712 disable the first inbound window default. + * Rather being mapped to system memory it is mapped to the + * internal registers of the SoC. This feature is deprecated, has + * security considerations, and is not implemented in our modern + * SoCs. + */ + if (pcie->cfg->soc_base != BCM7712) + add_inbound_win(b++, &n, 0, 0, 0); + + resource_list_for_each_entry(entry, &bridge->dma_ranges) { + u64 pcie_start = entry->res->start - entry->offset; + u64 cpu_start = entry->res->start; + + size = resource_size(entry->res); + tot_size += size; + if (pcie_start < lowest_pcie_addr) + lowest_pcie_addr = pcie_start; + /* + * 7712 and newer chips may have many BARs, with each + * offering a non-overlapping viewport to system memory. + * That being said, each BARs size must still be a power of + * two. + */ + if (pcie->cfg->soc_base == BCM7712) + add_inbound_win(b++, &n, size, cpu_start, pcie_start); + + if (n > pcie->cfg->num_inbound_wins) + break; + } + + if (lowest_pcie_addr == ~(u64)0) { + dev_err(dev, "DT node has no dma-ranges\n"); + return -EINVAL; + } + + /* + * 7712 and newer chips do not have an internal memory mapping system + * that enables multiple memory controllers. As such, it can return + * now w/o doing special configuration. + */ + if (pcie->cfg->soc_base == BCM7712) + return n; + + ret = of_property_read_variable_u64_array(pcie->np, "brcm,scb-sizes", pcie->memc_size, 1, + PCIE_BRCM_MAX_MEMC); + if (ret <= 0) { + /* Make an educated guess */ + pcie->num_memc = 1; + pcie->memc_size[0] = 1ULL << fls64(tot_size - 1); + } else { + pcie->num_memc = ret; + } + + /* Each memc is viewed through a "port" that is a power of 2 */ + for (i = 0, size = 0; i < pcie->num_memc; i++) + size += pcie->memc_size[i]; + + /* Our HW mandates that the window size must be a power of 2 */ + size = 1ULL << fls64(size - 1); + + /* + * For STB chips, the BAR2 cpu_addr is hardwired to the start + * of system memory, so we set it to 0. + */ + cpu_addr = 0; + pci_offset = lowest_pcie_addr; + + /* + * We validate the inbound memory view even though we should trust + * whatever the device-tree provides. This is because of an HW issue on + * early Raspberry Pi 4's revisions (bcm2711). It turns out its + * firmware has to dynamically edit dma-ranges due to a bug on the + * PCIe controller integration, which prohibits any access above the + * lower 3GB of memory. Given this, we decided to keep the dma-ranges + * in check, avoiding hard to debug device-tree related issues in the + * future: + * + * The PCIe host controller by design must set the inbound viewport to + * be a contiguous arrangement of all of the system's memory. In + * addition, its size must be a power of two. To further complicate + * matters, the viewport must start on a pcie-address that is aligned + * on a multiple of its size. If a portion of the viewport does not + * represent system memory -- e.g. 3GB of memory requires a 4GB + * viewport -- we can map the outbound memory in or after 3GB and even + * though the viewport will overlap the outbound memory the controller + * will know to send outbound memory downstream and everything else + * upstream. + * + * For example: + * + * - The best-case scenario, memory up to 3GB, is to place the inbound + * region in the first 4GB of pcie-space, as some legacy devices can + * only address 32bits. We would also like to put the MSI under 4GB + * as well, since some devices require a 32bit MSI target address. + * + * - If the system memory is 4GB or larger we cannot start the inbound + * region at location 0 (since we have to allow some space for + * outbound memory @ 3GB). So instead it will start at the 1x + * multiple of its size + */ + if (!size || (pci_offset & (size - 1)) || + (pci_offset < SZ_4G && pci_offset > SZ_2G)) { + dev_err(dev, "Invalid inbound_win2_offset/size: size 0x%llx, off 0x%llx\n", + size, pci_offset); + return -EINVAL; + } + + /* Enable inbound window 2, the main inbound window for STB chips */ + add_inbound_win(b++, &n, size, cpu_addr, pci_offset); + + /* + * Disable inbound window 3. On some chips presents the same + * window as #2 but the data appears in a settable endianness. + */ + add_inbound_win(b++, &n, 0, 0, 0); + + return n; +} + +static u32 brcm_bar_reg_offset(int bar) +{ + if (bar <= 3) + return PCIE_MISC_RC_BAR1_CONFIG_LO + 8 * (bar - 1); + else + return PCIE_MISC_RC_BAR4_CONFIG_LO + 8 * (bar - 4); +} + +static u32 brcm_ubus_reg_offset(int bar) +{ + if (bar <= 3) + return PCIE_MISC_UBUS_BAR1_CONFIG_REMAP + 8 * (bar - 1); + else + return PCIE_MISC_UBUS_BAR4_CONFIG_REMAP + 8 * (bar - 4); +} + +static void set_inbound_win_registers(struct brcm_pcie *pcie, + const struct inbound_win *inbound_wins, + u8 num_inbound_wins) +{ + void __iomem *base = pcie->base; + int i; + + for (i = 1; i <= num_inbound_wins; i++) { + u64 pci_offset = inbound_wins[i].pci_offset; + u64 cpu_addr = inbound_wins[i].cpu_addr; + u64 size = inbound_wins[i].size; + u32 reg_offset = brcm_bar_reg_offset(i); + u32 tmp = lower_32_bits(pci_offset); + + u32p_replace_bits(&tmp, brcm_pcie_encode_ibar_size(size), + PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK); + + /* Write low */ + writel_relaxed(tmp, base + reg_offset); + /* Write high */ + writel_relaxed(upper_32_bits(pci_offset), base + reg_offset + 4); + + /* + * Most STB chips: + * Do nothing. + * 7712: + * All of their BARs need to be set. + */ + if (pcie->cfg->soc_base == BCM7712) { + /* BUS remap register settings */ + reg_offset = brcm_ubus_reg_offset(i); + tmp = lower_32_bits(cpu_addr) & ~0xfff; + tmp |= PCIE_MISC_UBUS_BAR1_CONFIG_REMAP_ACCESS_EN_MASK; + writel_relaxed(tmp, base + reg_offset); + tmp = upper_32_bits(cpu_addr); + writel_relaxed(tmp, base + reg_offset + 4); + } + } +} + +static int brcm_pcie_setup(struct brcm_pcie *pcie) +{ + struct inbound_win inbound_wins[PCIE_BRCM_MAX_INBOUND_WINS]; + void __iomem *base = pcie->base; + struct pci_host_bridge *bridge; + struct resource_entry *entry; + u32 tmp, burst, num_lanes, num_lanes_cap; + u8 num_out_wins = 0; + int num_inbound_wins = 0; + int memc, ret; + + /* Reset the bridge */ + ret = brcm_pcie_bridge_sw_init_set(pcie, 1); + if (ret) + return ret; + + /* Ensure that PERST# is asserted; some bootloaders may deassert it. */ + if (pcie->cfg->soc_base == BCM2711) { + ret = pcie->cfg->perst_set(pcie, 1); + if (ret) { + pcie->cfg->bridge_sw_init_set(pcie, 0); + return ret; + } + } + + usleep_range(100, 200); + + /* Take the bridge out of reset */ + ret = brcm_pcie_bridge_sw_init_set(pcie, 0); + if (ret) + return ret; + + tmp = readl(base + HARD_DEBUG(pcie)); + if (is_bmips(pcie)) + tmp &= ~PCIE_BMIPS_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK; + else + tmp &= ~PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK; + writel(tmp, base + HARD_DEBUG(pcie)); + /* Wait for SerDes to be stable */ + usleep_range(100, 200); + + /* + * SCB_MAX_BURST_SIZE is a two bit field. For GENERIC chips it + * is encoded as 0=128, 1=256, 2=512, 3=Rsvd, for BCM7278 it + * is encoded as 0=Rsvd, 1=128, 2=256, 3=512. + */ + if (is_bmips(pcie)) + burst = 0x1; /* 256 bytes */ + else if (pcie->cfg->soc_base == BCM2711) + burst = 0x0; /* 128 bytes */ + else if (pcie->cfg->soc_base == BCM7278) + burst = 0x3; /* 512 bytes */ + else + burst = 0x2; /* 512 bytes */ + + /* + * Set SCB_MAX_BURST_SIZE, CFG_READ_UR_MODE, SCB_ACCESS_EN, + * RCB_MPS_MODE, RCB_64B_MODE + */ + tmp = readl(base + PCIE_MISC_MISC_CTRL); + u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_SCB_ACCESS_EN_MASK); + u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_CFG_READ_UR_MODE_MASK); + u32p_replace_bits(&tmp, burst, PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK); + u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_PCIE_RCB_MPS_MODE_MASK); + u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_PCIE_RCB_64B_MODE_MASK); + writel(tmp, base + PCIE_MISC_MISC_CTRL); + + num_inbound_wins = brcm_pcie_get_inbound_wins(pcie, inbound_wins); + if (num_inbound_wins < 0) + return num_inbound_wins; + + set_inbound_win_registers(pcie, inbound_wins, num_inbound_wins); + + if (!brcm_pcie_rc_mode(pcie)) { + dev_err(pcie->dev, "PCIe RC controller misconfigured as Endpoint\n"); + return -EINVAL; + } + + tmp = readl(base + PCIE_MISC_MISC_CTRL); + for (memc = 0; memc < pcie->num_memc; memc++) { + u32 scb_size_val = ilog2(pcie->memc_size[memc]) - 15; + + if (memc == 0) + u32p_replace_bits(&tmp, scb_size_val, SCB_SIZE_MASK(0)); + else if (memc == 1) + u32p_replace_bits(&tmp, scb_size_val, SCB_SIZE_MASK(1)); + else if (memc == 2) + u32p_replace_bits(&tmp, scb_size_val, SCB_SIZE_MASK(2)); + } + writel(tmp, base + PCIE_MISC_MISC_CTRL); + + /* + * We ideally want the MSI target address to be located in the 32bit + * addressable memory area. Some devices might depend on it. This is + * possible either when the inbound window is located above the lower + * 4GB or when the inbound area is smaller than 4GB (taking into + * account the rounding-up we're forced to perform). + */ + if (inbound_wins[2].pci_offset >= SZ_4G || + (inbound_wins[2].size + inbound_wins[2].pci_offset) < SZ_4G) + pcie->msi_target_addr = BRCM_MSI_TARGET_ADDR_LT_4GB; + else + pcie->msi_target_addr = BRCM_MSI_TARGET_ADDR_GT_4GB; + + + /* Don't advertise L0s capability if 'aspm-no-l0s' */ + tmp = readl(base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY); + if (of_property_read_bool(pcie->np, "aspm-no-l0s")) + tmp &= ~PCI_EXP_LNKCAP_ASPM_L0S; + writel(tmp, base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY); + + /* 'tmp' still holds the contents of PRIV1_LINK_CAPABILITY */ + num_lanes_cap = u32_get_bits(tmp, PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_MAX_LINK_WIDTH_MASK); + num_lanes = 0; + + /* + * Use hardware negotiated Max Link Width value by default. If the + * "num-lanes" DT property is present, assume that the chip's default + * link width capability information is incorrect/undesired and use the + * specified value instead. + */ + if (!of_property_read_u32(pcie->np, "num-lanes", &num_lanes) && + num_lanes && num_lanes <= 4 && num_lanes_cap != num_lanes) { + u32p_replace_bits(&tmp, num_lanes, + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_MAX_LINK_WIDTH_MASK); + writel(tmp, base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY); + tmp = readl(base + PCIE_RC_PL_REG_PHY_CTL_1); + u32p_replace_bits(&tmp, 1, + PCIE_RC_PL_REG_PHY_CTL_1_REG_P2_POWERDOWN_ENA_NOSYNC_MASK); + writel(tmp, base + PCIE_RC_PL_REG_PHY_CTL_1); + } + + /* + * For config space accesses on the RC, show the right class for + * a PCIe-PCIe bridge (the default setting is to be EP mode). + */ + tmp = readl(base + PCIE_RC_CFG_PRIV1_ID_VAL3); + u32p_replace_bits(&tmp, 0x060400, + PCIE_RC_CFG_PRIV1_ID_VAL3_CLASS_CODE_MASK); + writel(tmp, base + PCIE_RC_CFG_PRIV1_ID_VAL3); + + bridge = pci_host_bridge_from_priv(pcie); + resource_list_for_each_entry(entry, &bridge->windows) { + struct resource *res = entry->res; + + if (resource_type(res) != IORESOURCE_MEM) + continue; + + if (num_out_wins >= BRCM_NUM_PCIE_OUT_WINS) { + dev_err(pcie->dev, "too many outbound wins\n"); + return -EINVAL; + } + + if (is_bmips(pcie)) { + u64 start = res->start; + unsigned int j, nwins = resource_size(res) / SZ_128M; + + /* bmips PCIe outbound windows have a 128MB max size */ + if (nwins > BRCM_NUM_PCIE_OUT_WINS) + nwins = BRCM_NUM_PCIE_OUT_WINS; + for (j = 0; j < nwins; j++, start += SZ_128M) + brcm_pcie_set_outbound_win(pcie, j, start, + start - entry->offset, + SZ_128M); + break; + } + brcm_pcie_set_outbound_win(pcie, num_out_wins, res->start, + res->start - entry->offset, + resource_size(res)); + num_out_wins++; + } + + /* PCIe->SCB endian mode for inbound window */ + tmp = readl(base + PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1); + u32p_replace_bits(&tmp, PCIE_RC_CFG_VENDOR_SPECIFIC_REG1_LITTLE_ENDIAN, + PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1_ENDIAN_MODE_BAR2_MASK); + writel(tmp, base + PCIE_RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1); + + if (pcie->cfg->post_setup) { + ret = pcie->cfg->post_setup(pcie); + if (ret < 0) + return ret; + } + + return 0; +} + +/* + * This extends the timeout period for an access to an internal bus. This + * access timeout may occur during L1SS sleep periods, even without the + * presence of a PCIe access. + */ +static void brcm_extend_rbus_timeout(struct brcm_pcie *pcie) +{ + /* TIMEOUT register is two registers before RGR1_SW_INIT_1 */ + const unsigned int REG_OFFSET = PCIE_RGR1_SW_INIT_1(pcie) - 8; + u32 timeout_us = 4000000; /* 4 seconds, our setting for L1SS */ + + /* 7712 does not have this (RGR1) timer */ + if (pcie->cfg->soc_base == BCM7712) + return; + + /* Each unit in timeout register is 1/216,000,000 seconds */ + writel(216 * timeout_us, pcie->base + REG_OFFSET); +} + +static void brcm_config_clkreq(struct brcm_pcie *pcie) +{ + static const char err_msg[] = "invalid 'brcm,clkreq-mode' DT string\n"; + const char *mode = "default"; + u32 clkreq_cntl; + int ret, tmp; + + ret = of_property_read_string(pcie->np, "brcm,clkreq-mode", &mode); + if (ret && ret != -EINVAL) { + dev_err(pcie->dev, err_msg); + mode = "safe"; + } + + /* Start out assuming safe mode (both mode bits cleared) */ + clkreq_cntl = readl(pcie->base + HARD_DEBUG(pcie)); + clkreq_cntl &= ~PCIE_CLKREQ_MASK; + + if (strcmp(mode, "no-l1ss") == 0) { + /* + * "no-l1ss" -- Provides Clock Power Management, L0s, and + * L1, but cannot provide L1 substate (L1SS) power + * savings. If the downstream device connected to the RC is + * L1SS capable AND the OS enables L1SS, all PCIe traffic + * may abruptly halt, potentially hanging the system. + */ + clkreq_cntl |= PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK; + /* + * We want to un-advertise L1 substates because if the OS + * tries to configure the controller into using L1 substate + * power savings it may fail or hang when the RC HW is in + * "no-l1ss" mode. + */ + tmp = readl(pcie->base + PCIE_RC_CFG_PRIV1_ROOT_CAP); + u32p_replace_bits(&tmp, 2, PCIE_RC_CFG_PRIV1_ROOT_CAP_L1SS_MODE_MASK); + writel(tmp, pcie->base + PCIE_RC_CFG_PRIV1_ROOT_CAP); + + } else if (strcmp(mode, "default") == 0) { + /* + * "default" -- Provides L0s, L1, and L1SS, but not + * compliant to provide Clock Power Management; + * specifically, may not be able to meet the Tclron max + * timing of 400ns as specified in "Dynamic Clock Control", + * section 3.2.5.2.2 of the PCIe spec. This situation is + * atypical and should happen only with older devices. + */ + clkreq_cntl |= PCIE_MISC_HARD_PCIE_HARD_DEBUG_L1SS_ENABLE_MASK; + brcm_extend_rbus_timeout(pcie); + + } else { + /* + * "safe" -- No power savings; refclk is driven by RC + * unconditionally. + */ + if (strcmp(mode, "safe") != 0) + dev_err(pcie->dev, err_msg); + mode = "safe"; + } + writel(clkreq_cntl, pcie->base + HARD_DEBUG(pcie)); + + dev_info(pcie->dev, "clkreq-mode set to %s\n", mode); +} + +static int brcm_pcie_start_link(struct brcm_pcie *pcie) +{ + struct device *dev = pcie->dev; + void __iomem *base = pcie->base; + u16 nlw, cls, lnksta; + bool ssc_good = false; + int ret, i; + + /* Limit the generation if specified */ + if (pcie->gen) + brcm_pcie_set_gen(pcie, pcie->gen); + + /* Unassert the fundamental reset */ + ret = pcie->cfg->perst_set(pcie, 0); + if (ret) + return ret; + + msleep(PCIE_RESET_CONFIG_WAIT_MS); + + /* + * Give the RC/EP even more time to wake up, before trying to + * configure RC. Intermittently check status for link-up, up to a + * total of 100ms. + */ + for (i = 0; i < 100 && !brcm_pcie_link_up(pcie); i += 5) + msleep(5); + + if (!brcm_pcie_link_up(pcie)) { + dev_err(dev, "link down\n"); + return -ENODEV; + } + + brcm_config_clkreq(pcie); + + if (pcie->ssc) { + ret = brcm_pcie_set_ssc(pcie); + if (ret == 0) + ssc_good = true; + else + dev_err(dev, "failed attempt to enter ssc mode\n"); + } + + lnksta = readw(base + BRCM_PCIE_CAP_REGS + PCI_EXP_LNKSTA); + cls = FIELD_GET(PCI_EXP_LNKSTA_CLS, lnksta); + nlw = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta); + dev_info(dev, "link up, %s x%u %s\n", + pci_speed_string(pcie_get_link_speed(cls)), nlw, + ssc_good ? "(SSC)" : "(!SSC)"); + + return 0; +} + +static const char * const supplies[] = { + "vpcie3v3", + "vpcie3v3aux", + "vpcie12v", +}; + +static void *alloc_subdev_regulators(struct device *dev) +{ + const size_t size = sizeof(struct subdev_regulators) + + sizeof(struct regulator_bulk_data) * ARRAY_SIZE(supplies); + struct subdev_regulators *sr; + int i; + + sr = devm_kzalloc(dev, size, GFP_KERNEL); + if (sr) { + sr->num_supplies = ARRAY_SIZE(supplies); + for (i = 0; i < ARRAY_SIZE(supplies); i++) + sr->supplies[i].supply = supplies[i]; + } + + return sr; +} + +static int brcm_pcie_add_bus(struct pci_bus *bus) +{ + struct brcm_pcie *pcie = bus->sysdata; + struct device *dev = &bus->dev; + struct subdev_regulators *sr; + int ret; + + if (!bus->parent || !pci_is_root_bus(bus->parent)) + return 0; + + if (dev->of_node) { + sr = alloc_subdev_regulators(dev); + if (!sr) { + dev_info(dev, "Can't allocate regulators for downstream device\n"); + goto no_regulators; + } + + pcie->sr = sr; + + ret = regulator_bulk_get(dev, sr->num_supplies, sr->supplies); + if (ret) { + dev_info(dev, "Did not get regulators, err=%d\n", ret); + pcie->sr = NULL; + goto no_regulators; + } + + ret = regulator_bulk_enable(sr->num_supplies, sr->supplies); + if (ret) { + dev_err(dev, "Can't enable regulators for downstream device\n"); + regulator_bulk_free(sr->num_supplies, sr->supplies); + pcie->sr = NULL; + } + } + +no_regulators: + brcm_pcie_start_link(pcie); + return 0; +} + +static void brcm_pcie_remove_bus(struct pci_bus *bus) +{ + struct brcm_pcie *pcie = bus->sysdata; + struct subdev_regulators *sr = pcie->sr; + struct device *dev = &bus->dev; + + if (!sr || !bus->parent || !pci_is_root_bus(bus->parent)) + return; + + if (regulator_bulk_disable(sr->num_supplies, sr->supplies)) + dev_err(dev, "Failed to disable regulators for downstream device\n"); + regulator_bulk_free(sr->num_supplies, sr->supplies); + pcie->sr = NULL; +} + +/* L23 is a low-power PCIe link state */ +static void brcm_pcie_enter_l23(struct brcm_pcie *pcie) +{ + void __iomem *base = pcie->base; + int l23, i; + u32 tmp; + + /* Assert request for L23 */ + tmp = readl(base + PCIE_MISC_PCIE_CTRL); + u32p_replace_bits(&tmp, 1, PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK); + writel(tmp, base + PCIE_MISC_PCIE_CTRL); + + /* Wait up to 36 msec for L23 */ + tmp = readl(base + PCIE_MISC_PCIE_STATUS); + l23 = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK, tmp); + for (i = 0; i < 15 && !l23; i++) { + usleep_range(2000, 2400); + tmp = readl(base + PCIE_MISC_PCIE_STATUS); + l23 = FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK, + tmp); + } + + if (!l23) + dev_err(pcie->dev, "failed to enter low-power link state\n"); +} + +static int brcm_phy_cntl(struct brcm_pcie *pcie, const int start) +{ + static const u32 shifts[PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS] = { + PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_SHIFT, + PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_SHIFT, + PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_SHIFT,}; + static const u32 masks[PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS] = { + PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_MASK, + PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_RESET_MASK, + PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_DIG_RESET_MASK,}; + const int beg = start ? 0 : PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS - 1; + const int end = start ? PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_NFLDS : -1; + u32 tmp, combined_mask = 0; + u32 val; + void __iomem *base = pcie->base; + int i, ret; + + for (i = beg; i != end; start ? i++ : i--) { + val = start ? BIT_MASK(shifts[i]) : 0; + tmp = readl(base + PCIE_DVT_PMU_PCIE_PHY_CTRL); + tmp = (tmp & ~masks[i]) | (val & masks[i]); + writel(tmp, base + PCIE_DVT_PMU_PCIE_PHY_CTRL); + usleep_range(50, 200); + combined_mask |= masks[i]; + } + + tmp = readl(base + PCIE_DVT_PMU_PCIE_PHY_CTRL); + val = start ? combined_mask : 0; + + ret = (tmp & combined_mask) == val ? 0 : -EIO; + if (ret) + dev_err(pcie->dev, "failed to %s phy\n", (start ? "start" : "stop")); + + return ret; +} + +static inline int brcm_phy_start(struct brcm_pcie *pcie) +{ + return pcie->cfg->has_phy ? brcm_phy_cntl(pcie, 1) : 0; +} + +static inline int brcm_phy_stop(struct brcm_pcie *pcie) +{ + return pcie->cfg->has_phy ? brcm_phy_cntl(pcie, 0) : 0; +} + +static int brcm_pcie_turn_off(struct brcm_pcie *pcie) +{ + void __iomem *base = pcie->base; + int tmp, ret; + + if (brcm_pcie_link_up(pcie)) + brcm_pcie_enter_l23(pcie); + /* Assert fundamental reset */ + ret = pcie->cfg->perst_set(pcie, 1); + if (ret) + return ret; + + /* Deassert request for L23 in case it was asserted */ + tmp = readl(base + PCIE_MISC_PCIE_CTRL); + u32p_replace_bits(&tmp, 0, PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK); + writel(tmp, base + PCIE_MISC_PCIE_CTRL); + + /* Turn off SerDes */ + tmp = readl(base + HARD_DEBUG(pcie)); + u32p_replace_bits(&tmp, 1, PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK); + writel(tmp, base + HARD_DEBUG(pcie)); + + if (!(pcie->cfg->quirks & CFG_QUIRK_AVOID_BRIDGE_SHUTDOWN)) + /* Shutdown PCIe bridge */ + ret = brcm_pcie_bridge_sw_init_set(pcie, 1); + + return ret; +} + +static int pci_dev_may_wakeup(struct pci_dev *dev, void *data) +{ + bool *ret = data; + + if (device_may_wakeup(&dev->dev)) { + *ret = true; + dev_info(&dev->dev, "Possible wake-up device; regulators will not be disabled\n"); + } + return (int) *ret; +} + +static int brcm_pcie_suspend_noirq(struct device *dev) +{ + struct brcm_pcie *pcie = dev_get_drvdata(dev); + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + int ret, rret; + + ret = brcm_pcie_turn_off(pcie); + if (ret) + return ret; + + /* + * If brcm_phy_stop() returns an error, just dev_err(). If we + * return the error it will cause the suspend to fail and this is a + * forgivable offense that will probably be erased on resume. + */ + if (brcm_phy_stop(pcie)) + dev_err(dev, "Could not stop phy for suspend\n"); + + ret = reset_control_rearm(pcie->rescal); + if (ret) { + dev_err(dev, "Could not rearm rescal reset\n"); + return ret; + } + + if (pcie->sr) { + /* + * Now turn off the regulators, but if at least one + * downstream device is enabled as a wake-up source, do not + * turn off regulators. + */ + pcie->ep_wakeup_capable = false; + pci_walk_bus(bridge->bus, pci_dev_may_wakeup, + &pcie->ep_wakeup_capable); + if (!pcie->ep_wakeup_capable) { + ret = regulator_bulk_disable(pcie->sr->num_supplies, + pcie->sr->supplies); + if (ret) { + dev_err(dev, "Could not turn off regulators\n"); + rret = reset_control_reset(pcie->rescal); + if (rret) + dev_err(dev, "failed to reset 'rascal' controller ret=%d\n", + rret); + return ret; + } + } + } + clk_disable_unprepare(pcie->clk); + + return 0; +} + +static int brcm_pcie_resume_noirq(struct device *dev) +{ + struct brcm_pcie *pcie = dev_get_drvdata(dev); + void __iomem *base; + u32 tmp; + int ret, rret; + + base = pcie->base; + ret = clk_prepare_enable(pcie->clk); + if (ret) + return ret; + + ret = reset_control_reset(pcie->rescal); + if (ret) + goto err_disable_clk; + + ret = brcm_phy_start(pcie); + if (ret) + goto err_reset; + + /* Take bridge out of reset so we can access the SERDES reg */ + ret = brcm_pcie_bridge_sw_init_set(pcie, 0); + if (ret) + goto err_reset; + + /* SERDES_IDDQ = 0 */ + tmp = readl(base + HARD_DEBUG(pcie)); + u32p_replace_bits(&tmp, 0, PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK); + writel(tmp, base + HARD_DEBUG(pcie)); + + /* wait for serdes to be stable */ + udelay(100); + + ret = brcm_pcie_setup(pcie); + if (ret) + goto err_reset; + + if (pcie->sr) { + if (pcie->ep_wakeup_capable) { + /* + * We are resuming from a suspend. In the suspend we + * did not disable the power supplies, so there is + * no need to enable them (and falsely increase their + * usage count). + */ + pcie->ep_wakeup_capable = false; + } else { + ret = regulator_bulk_enable(pcie->sr->num_supplies, + pcie->sr->supplies); + if (ret) { + dev_err(dev, "Could not turn on regulators\n"); + goto err_reset; + } + } + } + + ret = brcm_pcie_start_link(pcie); + if (ret) + goto err_regulator; + + if (pcie->msi) + brcm_msi_set_regs(pcie->msi); + + return 0; + +err_regulator: + if (pcie->sr) + regulator_bulk_disable(pcie->sr->num_supplies, pcie->sr->supplies); +err_reset: + rret = reset_control_rearm(pcie->rescal); + if (rret) + dev_err(pcie->dev, "failed to rearm 'rescal' reset, err=%d\n", rret); +err_disable_clk: + clk_disable_unprepare(pcie->clk); + return ret; +} + +/* Dump out PCIe errors on die or panic */ +static int brcm_pcie_dump_err(struct brcm_pcie *pcie, + const char *type) +{ + void __iomem *base = pcie->base; + int i, is_cfg_err, is_mem_err, lanes; + const char *width_str, *direction_str; + u32 info, cfg_addr, cfg_cause, mem_cause, lo, hi; + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + unsigned long flags; + char lanes_str[9]; + + spin_lock_irqsave(&pcie->bridge_lock, flags); + /* Don't access registers when the bridge is off */ + if (pcie->bridge_in_reset || readl(base + PCIE_OUTB_ERR_VALID) == 0) { + spin_unlock_irqrestore(&pcie->bridge_lock, flags); + return NOTIFY_DONE; + } + + /* Read all necessary registers so we can release the spinlock ASAP */ + info = readl(base + PCIE_OUTB_ERR_ACC_INFO); + is_cfg_err = !!(info & PCIE_OUTB_ERR_ACC_INFO_CFG_ERR); + is_mem_err = !!(info & PCIE_OUTB_ERR_ACC_INFO_MEM_ERR); + if (is_cfg_err) { + cfg_addr = readl(base + PCIE_OUTB_ERR_ACC_ADDR); + cfg_cause = readl(base + PCIE_OUTB_ERR_CFG_CAUSE); + } + if (is_mem_err) { + mem_cause = readl(base + PCIE_OUTB_ERR_MEM_CAUSE); + lo = readl(base + PCIE_OUTB_ERR_MEM_ADDR_LO); + hi = readl(base + PCIE_OUTB_ERR_MEM_ADDR_HI); + } + /* We've got all of the info, clear the error */ + writel(1, base + PCIE_OUTB_ERR_CLEAR); + spin_unlock_irqrestore(&pcie->bridge_lock, flags); + + dev_err(pcie->dev, "reporting PCIe info which may be related to %s error\n", + type); + width_str = (info & PCIE_OUTB_ERR_ACC_INFO_TYPE_64) ? "64bit" : "32bit"; + direction_str = str_read_write(!(info & PCIE_OUTB_ERR_ACC_INFO_DIR_WRITE)); + lanes = FIELD_GET(PCIE_OUTB_ERR_ACC_INFO_BYTE_LANES, info); + for (i = 0, lanes_str[8] = 0; i < 8; i++) + lanes_str[i] = (lanes & (1 << i)) ? '1' : '0'; + + if (is_cfg_err) { + int bus = FIELD_GET(PCIE_OUTB_ERR_ACC_ADDR_BUS, cfg_addr); + int dev = FIELD_GET(PCIE_OUTB_ERR_ACC_ADDR_DEV, cfg_addr); + int func = FIELD_GET(PCIE_OUTB_ERR_ACC_ADDR_FUNC, cfg_addr); + int reg = FIELD_GET(PCIE_OUTB_ERR_ACC_ADDR_REG, cfg_addr); + + dev_err(pcie->dev, "Error: CFG Acc, %s, %s (%04x:%02x:%02x.%d) reg=0x%x, lanes=%s\n", + width_str, direction_str, bridge->domain_nr, bus, dev, + func, reg, lanes_str); + dev_err(pcie->dev, " Type: TO=%d Abt=%d UnsupReq=%d AccTO=%d AccDsbld=%d Acc64bit=%d\n", + !!(cfg_cause & PCIE_OUTB_ERR_CFG_CAUSE_TIMEOUT), + !!(cfg_cause & PCIE_OUTB_ERR_CFG_CAUSE_ABORT), + !!(cfg_cause & PCIE_OUTB_ERR_CFG_CAUSE_UNSUPP_REQ), + !!(cfg_cause & PCIE_OUTB_ERR_CFG_CAUSE_ACC_TIMEOUT), + !!(cfg_cause & PCIE_OUTB_ERR_CFG_CAUSE_ACC_DISABLED), + !!(cfg_cause & PCIE_OUTB_ERR_CFG_CAUSE_ACC_64BIT)); + } + + if (is_mem_err) { + u64 addr = ((u64)hi << 32) | (u64)lo; + + dev_err(pcie->dev, "Error: Mem Acc, %s, %s, @0x%llx, lanes=%s\n", + width_str, direction_str, addr, lanes_str); + dev_err(pcie->dev, " Type: TO=%d Abt=%d UnsupReq=%d AccDsble=%d BadAddr=%d\n", + !!(mem_cause & PCIE_OUTB_ERR_MEM_CAUSE_TIMEOUT), + !!(mem_cause & PCIE_OUTB_ERR_MEM_CAUSE_ABORT), + !!(mem_cause & PCIE_OUTB_ERR_MEM_CAUSE_UNSUPP_REQ), + !!(mem_cause & PCIE_OUTB_ERR_MEM_CAUSE_ACC_DISABLED), + !!(mem_cause & PCIE_OUTB_ERR_MEM_CAUSE_BAD_ADDR)); + } + + return NOTIFY_DONE; +} + +static int brcm_pcie_die_notify_cb(struct notifier_block *self, + unsigned long v, void *p) +{ + struct brcm_pcie *pcie = + container_of(self, struct brcm_pcie, die_notifier); + + return brcm_pcie_dump_err(pcie, "Die"); +} + +static int brcm_pcie_panic_notify_cb(struct notifier_block *self, + unsigned long v, void *p) +{ + struct brcm_pcie *pcie = + container_of(self, struct brcm_pcie, panic_notifier); + + return brcm_pcie_dump_err(pcie, "Panic"); +} + +static void brcm_register_die_notifiers(struct brcm_pcie *pcie) +{ + pcie->panic_notifier.notifier_call = brcm_pcie_panic_notify_cb; + atomic_notifier_chain_register(&panic_notifier_list, + &pcie->panic_notifier); + + pcie->die_notifier.notifier_call = brcm_pcie_die_notify_cb; + register_die_notifier(&pcie->die_notifier); +} + +static void brcm_unregister_die_notifiers(struct brcm_pcie *pcie) +{ + unregister_die_notifier(&pcie->die_notifier); + atomic_notifier_chain_unregister(&panic_notifier_list, + &pcie->panic_notifier); +} + +static void __brcm_pcie_remove(struct brcm_pcie *pcie) +{ + brcm_msi_remove(pcie); + brcm_pcie_turn_off(pcie); + if (brcm_phy_stop(pcie)) + dev_err(pcie->dev, "Could not stop phy\n"); + if (reset_control_rearm(pcie->rescal)) + dev_err(pcie->dev, "Could not rearm rescal reset\n"); + clk_disable_unprepare(pcie->clk); +} + +static void brcm_pcie_remove(struct platform_device *pdev) +{ + struct brcm_pcie *pcie = platform_get_drvdata(pdev); + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); + + pci_lock_rescan_remove(); + pci_stop_root_bus(bridge->bus); + pci_remove_root_bus(bridge->bus); + pci_unlock_rescan_remove(); + if (pcie->cfg->has_err_report) + brcm_unregister_die_notifiers(pcie); + + __brcm_pcie_remove(pcie); +} + +static const int pcie_offsets[] = { + [RGR1_SW_INIT_1] = 0x9210, + [EXT_CFG_INDEX] = 0x9000, + [EXT_CFG_DATA] = 0x8000, + [PCIE_HARD_DEBUG] = 0x4204, + [PCIE_INTR2_CPU_BASE] = 0x4300, +}; + +static const int pcie_offsets_bcm7278[] = { + [RGR1_SW_INIT_1] = 0xc010, + [EXT_CFG_INDEX] = 0x9000, + [EXT_CFG_DATA] = 0x8000, + [PCIE_HARD_DEBUG] = 0x4204, + [PCIE_INTR2_CPU_BASE] = 0x4300, +}; + +static const int pcie_offsets_bcm7425[] = { + [RGR1_SW_INIT_1] = 0x8010, + [EXT_CFG_INDEX] = 0x8300, + [EXT_CFG_DATA] = 0x8304, + [PCIE_HARD_DEBUG] = 0x4204, + [PCIE_INTR2_CPU_BASE] = 0x4300, +}; + +static const int pcie_offsets_bcm7712[] = { + [RGR1_SW_INIT_1] = 0x9210, + [EXT_CFG_INDEX] = 0x9000, + [EXT_CFG_DATA] = 0x8000, + [PCIE_HARD_DEBUG] = 0x4304, + [PCIE_INTR2_CPU_BASE] = 0x4400, +}; + +static const struct pcie_cfg_data generic_cfg = { + .offsets = pcie_offsets, + .soc_base = GENERIC, + .perst_set = brcm_pcie_perst_set_generic, + .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, + .num_inbound_wins = 3, +}; + +static const struct pcie_cfg_data bcm2711_cfg = { + .offsets = pcie_offsets, + .soc_base = BCM2711, + .perst_set = brcm_pcie_perst_set_generic, + .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, + .num_inbound_wins = 3, +}; + +static const struct pcie_cfg_data bcm2712_cfg = { + .offsets = pcie_offsets_bcm7712, + .soc_base = BCM7712, + .perst_set = brcm_pcie_perst_set_7278, + .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, + .post_setup = brcm_pcie_post_setup_bcm2712, + .quirks = CFG_QUIRK_AVOID_BRIDGE_SHUTDOWN, + .num_inbound_wins = 10, +}; + +static const struct pcie_cfg_data bcm4908_cfg = { + .offsets = pcie_offsets, + .soc_base = BCM4908, + .perst_set = brcm_pcie_perst_set_4908, + .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, + .num_inbound_wins = 3, +}; + +static const struct pcie_cfg_data bcm7278_cfg = { + .offsets = pcie_offsets_bcm7278, + .soc_base = BCM7278, + .perst_set = brcm_pcie_perst_set_7278, + .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_7278, + .num_inbound_wins = 3, +}; + +static const struct pcie_cfg_data bcm7425_cfg = { + .offsets = pcie_offsets_bcm7425, + .soc_base = BCM7425, + .perst_set = brcm_pcie_perst_set_generic, + .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, + .num_inbound_wins = 3, +}; + +static const struct pcie_cfg_data bcm7435_cfg = { + .offsets = pcie_offsets, + .soc_base = BCM7435, + .perst_set = brcm_pcie_perst_set_generic, + .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, + .num_inbound_wins = 3, +}; + +static const struct pcie_cfg_data bcm7216_cfg = { + .offsets = pcie_offsets_bcm7278, + .soc_base = BCM7278, + .perst_set = brcm_pcie_perst_set_7278, + .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_7278, + .has_phy = true, + .num_inbound_wins = 3, + .has_err_report = true, +}; + +static const struct pcie_cfg_data bcm7712_cfg = { + .offsets = pcie_offsets_bcm7712, + .perst_set = brcm_pcie_perst_set_7278, + .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, + .soc_base = BCM7712, + .num_inbound_wins = 10, +}; + +static const struct of_device_id brcm_pcie_match[] = { + { .compatible = "brcm,bcm2711-pcie", .data = &bcm2711_cfg }, + { .compatible = "brcm,bcm2712-pcie", .data = &bcm2712_cfg }, + { .compatible = "brcm,bcm4908-pcie", .data = &bcm4908_cfg }, + { .compatible = "brcm,bcm7211-pcie", .data = &generic_cfg }, + { .compatible = "brcm,bcm7216-pcie", .data = &bcm7216_cfg }, + { .compatible = "brcm,bcm7278-pcie", .data = &bcm7278_cfg }, + { .compatible = "brcm,bcm7425-pcie", .data = &bcm7425_cfg }, + { .compatible = "brcm,bcm7435-pcie", .data = &bcm7435_cfg }, + { .compatible = "brcm,bcm7445-pcie", .data = &generic_cfg }, + { .compatible = "brcm,bcm7712-pcie", .data = &bcm7712_cfg }, + {}, +}; + +static struct pci_ops brcm_pcie_ops = { + .map_bus = brcm_pcie_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, + .add_bus = brcm_pcie_add_bus, + .remove_bus = brcm_pcie_remove_bus, +}; + +static struct pci_ops brcm7425_pcie_ops = { + .map_bus = brcm7425_pcie_map_bus, + .read = pci_generic_config_read32, + .write = pci_generic_config_write32, + .add_bus = brcm_pcie_add_bus, + .remove_bus = brcm_pcie_remove_bus, +}; + +static int brcm_pcie_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct pci_host_bridge *bridge; + const struct pcie_cfg_data *data; + struct brcm_pcie *pcie; + int ret; + + bridge = devm_pci_alloc_host_bridge(&pdev->dev, sizeof(*pcie)); + if (!bridge) + return -ENOMEM; + + data = of_device_get_match_data(&pdev->dev); + if (!data) { + pr_err("failed to look up compatible string\n"); + return -EINVAL; + } + + pcie = pci_host_bridge_priv(bridge); + pcie->dev = &pdev->dev; + pcie->np = np; + pcie->cfg = data; + + pcie->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(pcie->base)) + return PTR_ERR(pcie->base); + + pcie->clk = devm_clk_get_optional(&pdev->dev, "sw_pcie"); + if (IS_ERR(pcie->clk)) + return PTR_ERR(pcie->clk); + + ret = of_pci_get_max_link_speed(np); + if (ret < 0 || ret > 3) + pcie->gen = 0; + else + pcie->gen = ret; + + pcie->ssc = of_property_read_bool(np, "brcm,enable-ssc"); + + pcie->rescal = devm_reset_control_get_optional_shared(&pdev->dev, "rescal"); + if (IS_ERR(pcie->rescal)) + return PTR_ERR(pcie->rescal); + + pcie->perst_reset = devm_reset_control_get_optional_exclusive(&pdev->dev, "perst"); + if (IS_ERR(pcie->perst_reset)) + return PTR_ERR(pcie->perst_reset); + + pcie->bridge_reset = devm_reset_control_get_optional_exclusive(&pdev->dev, "bridge"); + if (IS_ERR(pcie->bridge_reset)) + return PTR_ERR(pcie->bridge_reset); + + pcie->swinit_reset = devm_reset_control_get_optional_exclusive(&pdev->dev, "swinit"); + if (IS_ERR(pcie->swinit_reset)) + return PTR_ERR(pcie->swinit_reset); + + ret = clk_prepare_enable(pcie->clk); + if (ret) + return dev_err_probe(&pdev->dev, ret, "could not enable clock\n"); + + ret = brcm_pcie_bridge_sw_init_set(pcie, 0); + if (ret) + return dev_err_probe(&pdev->dev, ret, + "could not de-assert bridge reset\n"); + + if (pcie->swinit_reset) { + ret = reset_control_assert(pcie->swinit_reset); + if (ret) { + clk_disable_unprepare(pcie->clk); + return dev_err_probe(&pdev->dev, ret, + "could not assert reset 'swinit'\n"); + } + + /* HW team recommends 1us for proper sync and propagation of reset */ + udelay(1); + + ret = reset_control_deassert(pcie->swinit_reset); + if (ret) { + clk_disable_unprepare(pcie->clk); + return dev_err_probe(&pdev->dev, ret, + "could not de-assert reset 'swinit'\n"); + } + } + + ret = reset_control_reset(pcie->rescal); + if (ret) { + clk_disable_unprepare(pcie->clk); + return dev_err_probe(&pdev->dev, ret, "failed to deassert 'rescal'\n"); + } + + ret = brcm_phy_start(pcie); + if (ret) { + reset_control_rearm(pcie->rescal); + clk_disable_unprepare(pcie->clk); + return ret; + } + + ret = brcm_pcie_setup(pcie); + if (ret) + goto fail; + + pcie->hw_rev = readl(pcie->base + PCIE_MISC_REVISION); + if (pcie->cfg->soc_base == BCM4908 && + pcie->hw_rev >= BRCM_PCIE_HW_REV_3_20) { + dev_err(pcie->dev, "hardware revision with unsupported PERST# setup\n"); + ret = -ENODEV; + goto fail; + } + + if (pci_msi_enabled()) { + struct device_node *msi_np = of_parse_phandle(pcie->np, "msi-parent", 0); + + if (msi_np == pcie->np) + ret = brcm_pcie_enable_msi(pcie); + + of_node_put(msi_np); + + if (ret) { + dev_err(pcie->dev, "probe of internal MSI failed"); + goto fail; + } + } + + bridge->ops = pcie->cfg->soc_base == BCM7425 ? + &brcm7425_pcie_ops : &brcm_pcie_ops; + bridge->sysdata = pcie; + + platform_set_drvdata(pdev, pcie); + + ret = pci_host_probe(bridge); + if (!ret && !brcm_pcie_link_up(pcie)) + ret = -ENODEV; + + if (ret) { + brcm_pcie_remove(pdev); + return ret; + } + + if (pcie->cfg->has_err_report) { + spin_lock_init(&pcie->bridge_lock); + brcm_register_die_notifiers(pcie); + } + + return 0; + +fail: + __brcm_pcie_remove(pcie); + + return ret; +} + +MODULE_DEVICE_TABLE(of, brcm_pcie_match); + +static const struct dev_pm_ops brcm_pcie_pm_ops = { + .suspend_noirq = brcm_pcie_suspend_noirq, + .resume_noirq = brcm_pcie_resume_noirq, +}; + +static struct platform_driver brcm_pcie_driver = { + .probe = brcm_pcie_probe, + .remove = brcm_pcie_remove, + .driver = { + .name = "brcm-pcie", + .of_match_table = brcm_pcie_match, + .pm = &brcm_pcie_pm_ops, + }, +}; +module_platform_driver(brcm_pcie_driver); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Broadcom STB PCIe RC driver"); +MODULE_AUTHOR("Broadcom"); +MODULE_SOFTDEP("pre: irq_bcm2712_mip"); diff --git a/drivers/pci/controller/pcie-hisi-error.c b/drivers/pci/controller/pcie-hisi-error.c new file mode 100644 index 000000000..36be86d82 --- /dev/null +++ b/drivers/pci/controller/pcie-hisi-error.c @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Driver for handling the PCIe controller errors on + * HiSilicon HIP SoCs. + * + * Copyright (c) 2020 HiSilicon Limited. + */ + +#include <linux/acpi.h> +#include <acpi/ghes.h> +#include <linux/bitops.h> +#include <linux/delay.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/kfifo.h> +#include <linux/spinlock.h> + +/* HISI PCIe controller error definitions */ +#define HISI_PCIE_ERR_MISC_REGS 33 + +#define HISI_PCIE_LOCAL_VALID_VERSION BIT(0) +#define HISI_PCIE_LOCAL_VALID_SOC_ID BIT(1) +#define HISI_PCIE_LOCAL_VALID_SOCKET_ID BIT(2) +#define HISI_PCIE_LOCAL_VALID_NIMBUS_ID BIT(3) +#define HISI_PCIE_LOCAL_VALID_SUB_MODULE_ID BIT(4) +#define HISI_PCIE_LOCAL_VALID_CORE_ID BIT(5) +#define HISI_PCIE_LOCAL_VALID_PORT_ID BIT(6) +#define HISI_PCIE_LOCAL_VALID_ERR_TYPE BIT(7) +#define HISI_PCIE_LOCAL_VALID_ERR_SEVERITY BIT(8) +#define HISI_PCIE_LOCAL_VALID_ERR_MISC 9 + +static guid_t hisi_pcie_sec_guid = + GUID_INIT(0xB2889FC9, 0xE7D7, 0x4F9D, + 0xA8, 0x67, 0xAF, 0x42, 0xE9, 0x8B, 0xE7, 0x72); + +/* + * Firmware reports the socket port ID where the error occurred. These + * macros convert that to the core ID and core port ID required by the + * ACPI reset method. + */ +#define HISI_PCIE_PORT_ID(core, v) (((v) >> 1) + ((core) << 3)) +#define HISI_PCIE_CORE_ID(v) ((v) >> 3) +#define HISI_PCIE_CORE_PORT_ID(v) (((v) & 7) << 1) + +struct hisi_pcie_error_data { + u64 val_bits; + u8 version; + u8 soc_id; + u8 socket_id; + u8 nimbus_id; + u8 sub_module_id; + u8 core_id; + u8 port_id; + u8 err_severity; + u16 err_type; + u8 reserv[2]; + u32 err_misc[HISI_PCIE_ERR_MISC_REGS]; +}; + +struct hisi_pcie_error_private { + struct notifier_block nb; + struct device *dev; +}; + +enum hisi_pcie_submodule_id { + HISI_PCIE_SUB_MODULE_ID_AP, + HISI_PCIE_SUB_MODULE_ID_TL, + HISI_PCIE_SUB_MODULE_ID_MAC, + HISI_PCIE_SUB_MODULE_ID_DL, + HISI_PCIE_SUB_MODULE_ID_SDI, +}; + +static const char * const hisi_pcie_sub_module[] = { + [HISI_PCIE_SUB_MODULE_ID_AP] = "AP Layer", + [HISI_PCIE_SUB_MODULE_ID_TL] = "TL Layer", + [HISI_PCIE_SUB_MODULE_ID_MAC] = "MAC Layer", + [HISI_PCIE_SUB_MODULE_ID_DL] = "DL Layer", + [HISI_PCIE_SUB_MODULE_ID_SDI] = "SDI Layer", +}; + +enum hisi_pcie_err_severity { + HISI_PCIE_ERR_SEV_RECOVERABLE, + HISI_PCIE_ERR_SEV_FATAL, + HISI_PCIE_ERR_SEV_CORRECTED, + HISI_PCIE_ERR_SEV_NONE, +}; + +static const char * const hisi_pcie_error_sev[] = { + [HISI_PCIE_ERR_SEV_RECOVERABLE] = "recoverable", + [HISI_PCIE_ERR_SEV_FATAL] = "fatal", + [HISI_PCIE_ERR_SEV_CORRECTED] = "corrected", + [HISI_PCIE_ERR_SEV_NONE] = "none", +}; + +static const char *hisi_pcie_get_string(const char * const *array, + size_t n, u32 id) +{ + u32 index; + + for (index = 0; index < n; index++) { + if (index == id && array[index]) + return array[index]; + } + + return "unknown"; +} + +static int hisi_pcie_port_reset(struct platform_device *pdev, + u32 chip_id, u32 port_id) +{ + struct device *dev = &pdev->dev; + acpi_handle handle = ACPI_HANDLE(dev); + union acpi_object arg[3]; + struct acpi_object_list arg_list; + acpi_status s; + unsigned long long data = 0; + + arg[0].type = ACPI_TYPE_INTEGER; + arg[0].integer.value = chip_id; + arg[1].type = ACPI_TYPE_INTEGER; + arg[1].integer.value = HISI_PCIE_CORE_ID(port_id); + arg[2].type = ACPI_TYPE_INTEGER; + arg[2].integer.value = HISI_PCIE_CORE_PORT_ID(port_id); + + arg_list.count = 3; + arg_list.pointer = arg; + + s = acpi_evaluate_integer(handle, "RST", &arg_list, &data); + if (ACPI_FAILURE(s)) { + dev_err(dev, "No RST method\n"); + return -EIO; + } + + if (data) { + dev_err(dev, "Failed to Reset\n"); + return -EIO; + } + + return 0; +} + +static int hisi_pcie_port_do_recovery(struct platform_device *dev, + u32 chip_id, u32 port_id) +{ + acpi_status s; + struct device *device = &dev->dev; + acpi_handle root_handle = ACPI_HANDLE(device); + struct acpi_pci_root *pci_root; + struct pci_bus *root_bus; + struct pci_dev *pdev; + u32 domain, busnr, devfn; + + s = acpi_get_parent(root_handle, &root_handle); + if (ACPI_FAILURE(s)) + return -ENODEV; + pci_root = acpi_pci_find_root(root_handle); + if (!pci_root) + return -ENODEV; + root_bus = pci_root->bus; + domain = pci_root->segment; + + busnr = root_bus->number; + devfn = PCI_DEVFN(port_id, 0); + pdev = pci_get_domain_bus_and_slot(domain, busnr, devfn); + if (!pdev) { + dev_info(device, "Fail to get root port %04x:%02x:%02x.%d device\n", + domain, busnr, PCI_SLOT(devfn), PCI_FUNC(devfn)); + return -ENODEV; + } + + pci_stop_and_remove_bus_device_locked(pdev); + pci_dev_put(pdev); + + if (hisi_pcie_port_reset(dev, chip_id, port_id)) + return -EIO; + + /* + * The initialization time of subordinate devices after + * hot reset is no more than 1s, which is required by + * the PCI spec v5.0 sec 6.6.1. The time will shorten + * if Readiness Notifications mechanisms are used. But + * wait 1s here to adapt any conditions. + */ + ssleep(1UL); + + /* add root port and downstream devices */ + pci_lock_rescan_remove(); + pci_rescan_bus(root_bus); + pci_unlock_rescan_remove(); + + return 0; +} + +static void hisi_pcie_handle_error(struct platform_device *pdev, + const struct hisi_pcie_error_data *edata) +{ + struct device *dev = &pdev->dev; + int idx, rc; + const unsigned long valid_bits[] = {BITMAP_FROM_U64(edata->val_bits)}; + + if (edata->val_bits == 0) { + dev_warn(dev, "%s: no valid error information\n", __func__); + return; + } + + dev_info(dev, "\nHISI : HIP : PCIe controller error\n"); + if (edata->val_bits & HISI_PCIE_LOCAL_VALID_SOC_ID) + dev_info(dev, "Table version = %d\n", edata->version); + if (edata->val_bits & HISI_PCIE_LOCAL_VALID_SOCKET_ID) + dev_info(dev, "Socket ID = %d\n", edata->socket_id); + if (edata->val_bits & HISI_PCIE_LOCAL_VALID_NIMBUS_ID) + dev_info(dev, "Nimbus ID = %d\n", edata->nimbus_id); + if (edata->val_bits & HISI_PCIE_LOCAL_VALID_SUB_MODULE_ID) + dev_info(dev, "Sub Module = %s\n", + hisi_pcie_get_string(hisi_pcie_sub_module, + ARRAY_SIZE(hisi_pcie_sub_module), + edata->sub_module_id)); + if (edata->val_bits & HISI_PCIE_LOCAL_VALID_CORE_ID) + dev_info(dev, "Core ID = core%d\n", edata->core_id); + if (edata->val_bits & HISI_PCIE_LOCAL_VALID_PORT_ID) + dev_info(dev, "Port ID = port%d\n", edata->port_id); + if (edata->val_bits & HISI_PCIE_LOCAL_VALID_ERR_SEVERITY) + dev_info(dev, "Error severity = %s\n", + hisi_pcie_get_string(hisi_pcie_error_sev, + ARRAY_SIZE(hisi_pcie_error_sev), + edata->err_severity)); + if (edata->val_bits & HISI_PCIE_LOCAL_VALID_ERR_TYPE) + dev_info(dev, "Error type = 0x%x\n", edata->err_type); + + dev_info(dev, "Reg Dump:\n"); + idx = HISI_PCIE_LOCAL_VALID_ERR_MISC; + for_each_set_bit_from(idx, valid_bits, + HISI_PCIE_LOCAL_VALID_ERR_MISC + HISI_PCIE_ERR_MISC_REGS) + dev_info(dev, "ERR_MISC_%d = 0x%x\n", idx - HISI_PCIE_LOCAL_VALID_ERR_MISC, + edata->err_misc[idx - HISI_PCIE_LOCAL_VALID_ERR_MISC]); + + if (edata->err_severity != HISI_PCIE_ERR_SEV_RECOVERABLE) + return; + + /* Recovery for the PCIe controller errors, try reset + * PCI port for the error recovery + */ + rc = hisi_pcie_port_do_recovery(pdev, edata->socket_id, + HISI_PCIE_PORT_ID(edata->core_id, edata->port_id)); + if (rc) + dev_info(dev, "fail to do hisi pcie port reset\n"); +} + +static int hisi_pcie_notify_error(struct notifier_block *nb, + unsigned long event, void *data) +{ + struct acpi_hest_generic_data *gdata = data; + const struct hisi_pcie_error_data *error_data = acpi_hest_get_payload(gdata); + struct hisi_pcie_error_private *priv; + struct device *dev; + struct platform_device *pdev; + guid_t err_sec_guid; + u8 socket; + + import_guid(&err_sec_guid, gdata->section_type); + if (!guid_equal(&err_sec_guid, &hisi_pcie_sec_guid)) + return NOTIFY_DONE; + + priv = container_of(nb, struct hisi_pcie_error_private, nb); + dev = priv->dev; + + if (device_property_read_u8(dev, "socket", &socket)) + return NOTIFY_DONE; + + if (error_data->socket_id != socket) + return NOTIFY_DONE; + + pdev = container_of(dev, struct platform_device, dev); + hisi_pcie_handle_error(pdev, error_data); + + return NOTIFY_OK; +} + +static int hisi_pcie_error_handler_probe(struct platform_device *pdev) +{ + struct hisi_pcie_error_private *priv; + int ret; + + priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->nb.notifier_call = hisi_pcie_notify_error; + priv->dev = &pdev->dev; + ret = devm_ghes_register_vendor_record_notifier(&pdev->dev, &priv->nb); + if (ret) { + dev_err(&pdev->dev, + "Failed to register hisi pcie controller error handler with apei\n"); + return ret; + } + + return 0; +} + +static const struct acpi_device_id hisi_pcie_acpi_match[] = { + { "HISI0361", 0 }, + { } +}; + +static struct platform_driver hisi_pcie_error_handler_driver = { + .driver = { + .name = "hisi-pcie-error-handler", + .acpi_match_table = hisi_pcie_acpi_match, + }, + .probe = hisi_pcie_error_handler_probe, +}; +module_platform_driver(hisi_pcie_error_handler_driver); + +MODULE_DESCRIPTION("HiSilicon HIP PCIe controller error handling driver"); diff --git a/drivers/pci/controller/pcie-iproc-bcma.c b/drivers/pci/controller/pcie-iproc-bcma.c new file mode 100644 index 000000000..593418c2b --- /dev/null +++ b/drivers/pci/controller/pcie-iproc-bcma.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015 Broadcom Corporation + * Copyright (C) 2015 Hauke Mehrtens <hauke@hauke-m.de> + */ + +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/phy/phy.h> +#include <linux/bcma/bcma.h> +#include <linux/ioport.h> + +#include "pcie-iproc.h" + + +/* NS: CLASS field is R/O, and set to wrong 0x200 value */ +static void bcma_pcie2_fixup_class(struct pci_dev *dev) +{ + dev->class = PCI_CLASS_BRIDGE_PCI_NORMAL; +} +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x8011, bcma_pcie2_fixup_class); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x8012, bcma_pcie2_fixup_class); + +static int iproc_bcma_pcie_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) +{ + struct iproc_pcie *pcie = dev->sysdata; + struct bcma_device *bdev = container_of(pcie->dev, struct bcma_device, dev); + + return bcma_core_irq(bdev, 5); +} + +static int iproc_bcma_pcie_probe(struct bcma_device *bdev) +{ + struct device *dev = &bdev->dev; + struct iproc_pcie *pcie; + struct pci_host_bridge *bridge; + int ret; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!bridge) + return -ENOMEM; + + pcie = pci_host_bridge_priv(bridge); + + pcie->dev = dev; + + pcie->type = IPROC_PCIE_PAXB_BCMA; + pcie->base = bdev->io_addr; + if (!pcie->base) { + dev_err(dev, "no controller registers\n"); + return -ENOMEM; + } + + pcie->base_addr = bdev->addr; + + pcie->mem.start = bdev->addr_s[0]; + pcie->mem.end = bdev->addr_s[0] + SZ_128M - 1; + pcie->mem.name = "PCIe MEM space"; + pcie->mem.flags = IORESOURCE_MEM; + pci_add_resource(&bridge->windows, &pcie->mem); + ret = devm_request_pci_bus_resources(dev, &bridge->windows); + if (ret) + return ret; + + bridge->map_irq = iproc_bcma_pcie_map_irq; + + bcma_set_drvdata(bdev, pcie); + + return iproc_pcie_setup(pcie, &bridge->windows); +} + +static void iproc_bcma_pcie_remove(struct bcma_device *bdev) +{ + struct iproc_pcie *pcie = bcma_get_drvdata(bdev); + + iproc_pcie_remove(pcie); +} + +static const struct bcma_device_id iproc_bcma_pcie_table[] = { + BCMA_CORE(BCMA_MANUF_BCM, BCMA_CORE_NS_PCIEG2, BCMA_ANY_REV, BCMA_ANY_CLASS), + {}, +}; +MODULE_DEVICE_TABLE(bcma, iproc_bcma_pcie_table); + +static struct bcma_driver iproc_bcma_pcie_driver = { + .name = KBUILD_MODNAME, + .id_table = iproc_bcma_pcie_table, + .probe = iproc_bcma_pcie_probe, + .remove = iproc_bcma_pcie_remove, +}; +module_bcma_driver(iproc_bcma_pcie_driver); + +MODULE_AUTHOR("Hauke Mehrtens"); +MODULE_DESCRIPTION("Broadcom iProc PCIe BCMA driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pcie-iproc-msi.c b/drivers/pci/controller/pcie-iproc-msi.c new file mode 100644 index 000000000..9ba242ab9 --- /dev/null +++ b/drivers/pci/controller/pcie-iproc-msi.c @@ -0,0 +1,677 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015 Broadcom Corporation + */ + +#include <linux/interrupt.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/irqdomain.h> +#include <linux/msi.h> +#include <linux/of_irq.h> +#include <linux/of_pci.h> +#include <linux/pci.h> + +#include "pcie-iproc.h" + +#define IPROC_MSI_INTR_EN_SHIFT 11 +#define IPROC_MSI_INTR_EN BIT(IPROC_MSI_INTR_EN_SHIFT) +#define IPROC_MSI_INT_N_EVENT_SHIFT 1 +#define IPROC_MSI_INT_N_EVENT BIT(IPROC_MSI_INT_N_EVENT_SHIFT) +#define IPROC_MSI_EQ_EN_SHIFT 0 +#define IPROC_MSI_EQ_EN BIT(IPROC_MSI_EQ_EN_SHIFT) + +#define IPROC_MSI_EQ_MASK 0x3f + +/* Max number of GIC interrupts */ +#define NR_HW_IRQS 6 + +/* Number of entries in each event queue */ +#define EQ_LEN 64 + +/* Size of each event queue memory region */ +#define EQ_MEM_REGION_SIZE SZ_4K + +/* Size of each MSI address region */ +#define MSI_MEM_REGION_SIZE SZ_4K + +enum iproc_msi_reg { + IPROC_MSI_EQ_PAGE = 0, + IPROC_MSI_EQ_PAGE_UPPER, + IPROC_MSI_PAGE, + IPROC_MSI_PAGE_UPPER, + IPROC_MSI_CTRL, + IPROC_MSI_EQ_HEAD, + IPROC_MSI_EQ_TAIL, + IPROC_MSI_INTS_EN, + IPROC_MSI_REG_SIZE, +}; + +struct iproc_msi; + +/** + * struct iproc_msi_grp - iProc MSI group + * + * One MSI group is allocated per GIC interrupt, serviced by one iProc MSI + * event queue. + * + * @msi: pointer to iProc MSI data + * @gic_irq: GIC interrupt + * @eq: Event queue number + */ +struct iproc_msi_grp { + struct iproc_msi *msi; + int gic_irq; + unsigned int eq; +}; + +/** + * struct iproc_msi - iProc event queue based MSI + * + * Only meant to be used on platforms without MSI support integrated into the + * GIC. + * + * @pcie: pointer to iProc PCIe data + * @reg_offsets: MSI register offsets + * @grps: MSI groups + * @nr_irqs: number of total interrupts connected to GIC + * @nr_cpus: number of toal CPUs + * @has_inten_reg: indicates the MSI interrupt enable register needs to be + * set explicitly (required for some legacy platforms) + * @bitmap: MSI vector bitmap + * @bitmap_lock: lock to protect access to the MSI bitmap + * @nr_msi_vecs: total number of MSI vectors + * @inner_domain: inner IRQ domain + * @nr_eq_region: required number of 4K aligned memory region for MSI event + * queues + * @nr_msi_region: required number of 4K aligned address region for MSI posted + * writes + * @eq_cpu: pointer to allocated memory region for MSI event queues + * @eq_dma: DMA address of MSI event queues + * @msi_addr: MSI address + */ +struct iproc_msi { + struct iproc_pcie *pcie; + const u16 (*reg_offsets)[IPROC_MSI_REG_SIZE]; + struct iproc_msi_grp *grps; + int nr_irqs; + int nr_cpus; + bool has_inten_reg; + unsigned long *bitmap; + struct mutex bitmap_lock; + unsigned int nr_msi_vecs; + struct irq_domain *inner_domain; + unsigned int nr_eq_region; + unsigned int nr_msi_region; + void *eq_cpu; + dma_addr_t eq_dma; + phys_addr_t msi_addr; +}; + +static const u16 iproc_msi_reg_paxb[NR_HW_IRQS][IPROC_MSI_REG_SIZE] = { + { 0x200, 0x2c0, 0x204, 0x2c4, 0x210, 0x250, 0x254, 0x208 }, + { 0x200, 0x2c0, 0x204, 0x2c4, 0x214, 0x258, 0x25c, 0x208 }, + { 0x200, 0x2c0, 0x204, 0x2c4, 0x218, 0x260, 0x264, 0x208 }, + { 0x200, 0x2c0, 0x204, 0x2c4, 0x21c, 0x268, 0x26c, 0x208 }, + { 0x200, 0x2c0, 0x204, 0x2c4, 0x220, 0x270, 0x274, 0x208 }, + { 0x200, 0x2c0, 0x204, 0x2c4, 0x224, 0x278, 0x27c, 0x208 }, +}; + +static const u16 iproc_msi_reg_paxc[NR_HW_IRQS][IPROC_MSI_REG_SIZE] = { + { 0xc00, 0xc04, 0xc08, 0xc0c, 0xc40, 0xc50, 0xc60 }, + { 0xc10, 0xc14, 0xc18, 0xc1c, 0xc44, 0xc54, 0xc64 }, + { 0xc20, 0xc24, 0xc28, 0xc2c, 0xc48, 0xc58, 0xc68 }, + { 0xc30, 0xc34, 0xc38, 0xc3c, 0xc4c, 0xc5c, 0xc6c }, +}; + +static inline u32 iproc_msi_read_reg(struct iproc_msi *msi, + enum iproc_msi_reg reg, + unsigned int eq) +{ + struct iproc_pcie *pcie = msi->pcie; + + return readl_relaxed(pcie->base + msi->reg_offsets[eq][reg]); +} + +static inline void iproc_msi_write_reg(struct iproc_msi *msi, + enum iproc_msi_reg reg, + int eq, u32 val) +{ + struct iproc_pcie *pcie = msi->pcie; + + writel_relaxed(val, pcie->base + msi->reg_offsets[eq][reg]); +} + +static inline u32 hwirq_to_group(struct iproc_msi *msi, unsigned long hwirq) +{ + return (hwirq % msi->nr_irqs); +} + +static inline unsigned int iproc_msi_addr_offset(struct iproc_msi *msi, + unsigned long hwirq) +{ + if (msi->nr_msi_region > 1) + return hwirq_to_group(msi, hwirq) * MSI_MEM_REGION_SIZE; + else + return hwirq_to_group(msi, hwirq) * sizeof(u32); +} + +static inline unsigned int iproc_msi_eq_offset(struct iproc_msi *msi, u32 eq) +{ + if (msi->nr_eq_region > 1) + return eq * EQ_MEM_REGION_SIZE; + else + return eq * EQ_LEN * sizeof(u32); +} + +#define IPROC_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS) +#define IPROC_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_PCI_MSIX) + +static struct msi_parent_ops iproc_msi_parent_ops = { + .required_flags = IPROC_MSI_FLAGS_REQUIRED, + .supported_flags = IPROC_MSI_FLAGS_SUPPORTED, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .prefix = "iProc-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; +/* + * In iProc PCIe core, each MSI group is serviced by a GIC interrupt and a + * dedicated event queue. Each MSI group can support up to 64 MSI vectors. + * + * The number of MSI groups varies between different iProc SoCs. The total + * number of CPU cores also varies. To support MSI IRQ affinity, we + * distribute GIC interrupts across all available CPUs. MSI vector is moved + * from one GIC interrupt to another to steer to the target CPU. + * + * Assuming: + * - the number of MSI groups is M + * - the number of CPU cores is N + * - M is always a multiple of N + * + * Total number of raw MSI vectors = M * 64 + * Total number of supported MSI vectors = (M * 64) / N + */ +static inline int hwirq_to_cpu(struct iproc_msi *msi, unsigned long hwirq) +{ + return (hwirq % msi->nr_cpus); +} + +static inline unsigned long hwirq_to_canonical_hwirq(struct iproc_msi *msi, + unsigned long hwirq) +{ + return (hwirq - hwirq_to_cpu(msi, hwirq)); +} + +static int iproc_msi_irq_set_affinity(struct irq_data *data, + const struct cpumask *mask, bool force) +{ + struct iproc_msi *msi = irq_data_get_irq_chip_data(data); + int target_cpu = cpumask_first(mask); + int curr_cpu; + int ret; + + curr_cpu = hwirq_to_cpu(msi, data->hwirq); + if (curr_cpu == target_cpu) + ret = IRQ_SET_MASK_OK_DONE; + else { + /* steer MSI to the target CPU */ + data->hwirq = hwirq_to_canonical_hwirq(msi, data->hwirq) + target_cpu; + ret = IRQ_SET_MASK_OK; + } + + irq_data_update_effective_affinity(data, cpumask_of(target_cpu)); + + return ret; +} + +static void iproc_msi_irq_compose_msi_msg(struct irq_data *data, + struct msi_msg *msg) +{ + struct iproc_msi *msi = irq_data_get_irq_chip_data(data); + dma_addr_t addr; + + addr = msi->msi_addr + iproc_msi_addr_offset(msi, data->hwirq); + msg->address_lo = lower_32_bits(addr); + msg->address_hi = upper_32_bits(addr); + msg->data = data->hwirq << 5; +} + +static struct irq_chip iproc_msi_bottom_irq_chip = { + .name = "MSI", + .irq_set_affinity = iproc_msi_irq_set_affinity, + .irq_compose_msi_msg = iproc_msi_irq_compose_msi_msg, +}; + +static int iproc_msi_irq_domain_alloc(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs, + void *args) +{ + struct iproc_msi *msi = domain->host_data; + int hwirq, i; + + if (msi->nr_cpus > 1 && nr_irqs > 1) + return -EINVAL; + + mutex_lock(&msi->bitmap_lock); + + /* + * Allocate 'nr_irqs' multiplied by 'nr_cpus' number of MSI vectors + * each time + */ + hwirq = bitmap_find_free_region(msi->bitmap, msi->nr_msi_vecs, + order_base_2(msi->nr_cpus * nr_irqs)); + + mutex_unlock(&msi->bitmap_lock); + + if (hwirq < 0) + return -ENOSPC; + + for (i = 0; i < nr_irqs; i++) { + irq_domain_set_info(domain, virq + i, hwirq + i, + &iproc_msi_bottom_irq_chip, + domain->host_data, handle_simple_irq, + NULL, NULL); + } + + return 0; +} + +static void iproc_msi_irq_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + struct irq_data *data = irq_domain_get_irq_data(domain, virq); + struct iproc_msi *msi = irq_data_get_irq_chip_data(data); + unsigned int hwirq; + + mutex_lock(&msi->bitmap_lock); + + hwirq = hwirq_to_canonical_hwirq(msi, data->hwirq); + bitmap_release_region(msi->bitmap, hwirq, + order_base_2(msi->nr_cpus * nr_irqs)); + + mutex_unlock(&msi->bitmap_lock); + + irq_domain_free_irqs_parent(domain, virq, nr_irqs); +} + +static const struct irq_domain_ops msi_domain_ops = { + .alloc = iproc_msi_irq_domain_alloc, + .free = iproc_msi_irq_domain_free, +}; + +static inline u32 decode_msi_hwirq(struct iproc_msi *msi, u32 eq, u32 head) +{ + u32 __iomem *msg; + u32 hwirq; + unsigned int offs; + + offs = iproc_msi_eq_offset(msi, eq) + head * sizeof(u32); + msg = (u32 __iomem *)(msi->eq_cpu + offs); + hwirq = readl(msg); + hwirq = (hwirq >> 5) + (hwirq & 0x1f); + + /* + * Since we have multiple hwirq mapped to a single MSI vector, + * now we need to derive the hwirq at CPU0. It can then be used to + * mapped back to virq. + */ + return hwirq_to_canonical_hwirq(msi, hwirq); +} + +static void iproc_msi_handler(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + struct iproc_msi_grp *grp; + struct iproc_msi *msi; + u32 eq, head, tail, nr_events; + unsigned long hwirq; + + chained_irq_enter(chip, desc); + + grp = irq_desc_get_handler_data(desc); + msi = grp->msi; + eq = grp->eq; + + /* + * iProc MSI event queue is tracked by head and tail pointers. Head + * pointer indicates the next entry (MSI data) to be consumed by SW in + * the queue and needs to be updated by SW. iProc MSI core uses the + * tail pointer as the next data insertion point. + * + * Entries between head and tail pointers contain valid MSI data. MSI + * data is guaranteed to be in the event queue memory before the tail + * pointer is updated by the iProc MSI core. + */ + head = iproc_msi_read_reg(msi, IPROC_MSI_EQ_HEAD, + eq) & IPROC_MSI_EQ_MASK; + do { + tail = iproc_msi_read_reg(msi, IPROC_MSI_EQ_TAIL, + eq) & IPROC_MSI_EQ_MASK; + + /* + * Figure out total number of events (MSI data) to be + * processed. + */ + nr_events = (tail < head) ? + (EQ_LEN - (head - tail)) : (tail - head); + if (!nr_events) + break; + + /* process all outstanding events */ + while (nr_events--) { + hwirq = decode_msi_hwirq(msi, eq, head); + generic_handle_domain_irq(msi->inner_domain, hwirq); + + head++; + head %= EQ_LEN; + } + + /* + * Now all outstanding events have been processed. Update the + * head pointer. + */ + iproc_msi_write_reg(msi, IPROC_MSI_EQ_HEAD, eq, head); + + /* + * Now go read the tail pointer again to see if there are new + * outstanding events that came in during the above window. + */ + } while (true); + + chained_irq_exit(chip, desc); +} + +static void iproc_msi_enable(struct iproc_msi *msi) +{ + int i, eq; + u32 val; + + /* Program memory region for each event queue */ + for (i = 0; i < msi->nr_eq_region; i++) { + dma_addr_t addr = msi->eq_dma + (i * EQ_MEM_REGION_SIZE); + + iproc_msi_write_reg(msi, IPROC_MSI_EQ_PAGE, i, + lower_32_bits(addr)); + iproc_msi_write_reg(msi, IPROC_MSI_EQ_PAGE_UPPER, i, + upper_32_bits(addr)); + } + + /* Program address region for MSI posted writes */ + for (i = 0; i < msi->nr_msi_region; i++) { + phys_addr_t addr = msi->msi_addr + (i * MSI_MEM_REGION_SIZE); + + iproc_msi_write_reg(msi, IPROC_MSI_PAGE, i, + lower_32_bits(addr)); + iproc_msi_write_reg(msi, IPROC_MSI_PAGE_UPPER, i, + upper_32_bits(addr)); + } + + for (eq = 0; eq < msi->nr_irqs; eq++) { + /* Enable MSI event queue */ + val = IPROC_MSI_INTR_EN | IPROC_MSI_INT_N_EVENT | + IPROC_MSI_EQ_EN; + iproc_msi_write_reg(msi, IPROC_MSI_CTRL, eq, val); + + /* + * Some legacy platforms require the MSI interrupt enable + * register to be set explicitly. + */ + if (msi->has_inten_reg) { + val = iproc_msi_read_reg(msi, IPROC_MSI_INTS_EN, eq); + val |= BIT(eq); + iproc_msi_write_reg(msi, IPROC_MSI_INTS_EN, eq, val); + } + } +} + +static void iproc_msi_disable(struct iproc_msi *msi) +{ + u32 eq, val; + + for (eq = 0; eq < msi->nr_irqs; eq++) { + if (msi->has_inten_reg) { + val = iproc_msi_read_reg(msi, IPROC_MSI_INTS_EN, eq); + val &= ~BIT(eq); + iproc_msi_write_reg(msi, IPROC_MSI_INTS_EN, eq, val); + } + + val = iproc_msi_read_reg(msi, IPROC_MSI_CTRL, eq); + val &= ~(IPROC_MSI_INTR_EN | IPROC_MSI_INT_N_EVENT | + IPROC_MSI_EQ_EN); + iproc_msi_write_reg(msi, IPROC_MSI_CTRL, eq, val); + } +} + +static int iproc_msi_alloc_domains(struct device_node *node, + struct iproc_msi *msi) +{ + struct irq_domain_info info = { + .fwnode = of_fwnode_handle(node), + .ops = &msi_domain_ops, + .host_data = msi, + .size = msi->nr_msi_vecs, + }; + + msi->inner_domain = msi_create_parent_irq_domain(&info, &iproc_msi_parent_ops); + if (!msi->inner_domain) + return -ENOMEM; + + return 0; +} + +static void iproc_msi_free_domains(struct iproc_msi *msi) +{ + if (msi->inner_domain) + irq_domain_remove(msi->inner_domain); +} + +static void iproc_msi_irq_free(struct iproc_msi *msi, unsigned int cpu) +{ + int i; + + for (i = cpu; i < msi->nr_irqs; i += msi->nr_cpus) { + irq_set_chained_handler_and_data(msi->grps[i].gic_irq, + NULL, NULL); + } +} + +static int iproc_msi_irq_setup(struct iproc_msi *msi, unsigned int cpu) +{ + int i, ret; + cpumask_var_t mask; + struct iproc_pcie *pcie = msi->pcie; + + for (i = cpu; i < msi->nr_irqs; i += msi->nr_cpus) { + irq_set_chained_handler_and_data(msi->grps[i].gic_irq, + iproc_msi_handler, + &msi->grps[i]); + /* Dedicate GIC interrupt to each CPU core */ + if (alloc_cpumask_var(&mask, GFP_KERNEL)) { + cpumask_clear(mask); + cpumask_set_cpu(cpu, mask); + ret = irq_set_affinity(msi->grps[i].gic_irq, mask); + if (ret) + dev_err(pcie->dev, + "failed to set affinity for IRQ%d\n", + msi->grps[i].gic_irq); + free_cpumask_var(mask); + } else { + dev_err(pcie->dev, "failed to alloc CPU mask\n"); + ret = -EINVAL; + } + + if (ret) { + /* Free all configured/unconfigured IRQs */ + iproc_msi_irq_free(msi, cpu); + return ret; + } + } + + return 0; +} + +int iproc_msi_init(struct iproc_pcie *pcie, struct device_node *node) +{ + struct iproc_msi *msi; + int i, ret; + unsigned int cpu; + + if (!of_device_is_compatible(node, "brcm,iproc-msi")) + return -ENODEV; + + if (!of_property_read_bool(node, "msi-controller")) + return -ENODEV; + + if (pcie->msi) + return -EBUSY; + + msi = devm_kzalloc(pcie->dev, sizeof(*msi), GFP_KERNEL); + if (!msi) + return -ENOMEM; + + msi->pcie = pcie; + pcie->msi = msi; + msi->msi_addr = pcie->base_addr; + mutex_init(&msi->bitmap_lock); + msi->nr_cpus = num_possible_cpus(); + + if (msi->nr_cpus == 1) + iproc_msi_parent_ops.supported_flags |= MSI_FLAG_MULTI_PCI_MSI; + + msi->nr_irqs = of_irq_count(node); + if (!msi->nr_irqs) { + dev_err(pcie->dev, "found no MSI GIC interrupt\n"); + return -ENODEV; + } + + if (msi->nr_irqs > NR_HW_IRQS) { + dev_warn(pcie->dev, "too many MSI GIC interrupts defined %d\n", + msi->nr_irqs); + msi->nr_irqs = NR_HW_IRQS; + } + + if (msi->nr_irqs < msi->nr_cpus) { + dev_err(pcie->dev, + "not enough GIC interrupts for MSI affinity\n"); + return -EINVAL; + } + + if (msi->nr_irqs % msi->nr_cpus != 0) { + msi->nr_irqs -= msi->nr_irqs % msi->nr_cpus; + dev_warn(pcie->dev, "Reducing number of interrupts to %d\n", + msi->nr_irqs); + } + + switch (pcie->type) { + case IPROC_PCIE_PAXB_BCMA: + case IPROC_PCIE_PAXB: + msi->reg_offsets = iproc_msi_reg_paxb; + msi->nr_eq_region = 1; + msi->nr_msi_region = 1; + break; + case IPROC_PCIE_PAXC: + msi->reg_offsets = iproc_msi_reg_paxc; + msi->nr_eq_region = msi->nr_irqs; + msi->nr_msi_region = msi->nr_irqs; + break; + default: + dev_err(pcie->dev, "incompatible iProc PCIe interface\n"); + return -EINVAL; + } + + msi->has_inten_reg = of_property_read_bool(node, "brcm,pcie-msi-inten"); + + msi->nr_msi_vecs = msi->nr_irqs * EQ_LEN; + msi->bitmap = devm_bitmap_zalloc(pcie->dev, msi->nr_msi_vecs, + GFP_KERNEL); + if (!msi->bitmap) + return -ENOMEM; + + msi->grps = devm_kcalloc(pcie->dev, msi->nr_irqs, sizeof(*msi->grps), + GFP_KERNEL); + if (!msi->grps) + return -ENOMEM; + + for (i = 0; i < msi->nr_irqs; i++) { + unsigned int irq = irq_of_parse_and_map(node, i); + + if (!irq) { + dev_err(pcie->dev, "unable to parse/map interrupt\n"); + ret = -ENODEV; + goto free_irqs; + } + msi->grps[i].gic_irq = irq; + msi->grps[i].msi = msi; + msi->grps[i].eq = i; + } + + /* Reserve memory for event queue and make sure memories are zeroed */ + msi->eq_cpu = dma_alloc_coherent(pcie->dev, + msi->nr_eq_region * EQ_MEM_REGION_SIZE, + &msi->eq_dma, GFP_KERNEL); + if (!msi->eq_cpu) { + ret = -ENOMEM; + goto free_irqs; + } + + ret = iproc_msi_alloc_domains(node, msi); + if (ret) { + dev_err(pcie->dev, "failed to create MSI domains\n"); + goto free_eq_dma; + } + + for_each_online_cpu(cpu) { + ret = iproc_msi_irq_setup(msi, cpu); + if (ret) + goto free_msi_irq; + } + + iproc_msi_enable(msi); + + return 0; + +free_msi_irq: + for_each_online_cpu(cpu) + iproc_msi_irq_free(msi, cpu); + iproc_msi_free_domains(msi); + +free_eq_dma: + dma_free_coherent(pcie->dev, msi->nr_eq_region * EQ_MEM_REGION_SIZE, + msi->eq_cpu, msi->eq_dma); + +free_irqs: + for (i = 0; i < msi->nr_irqs; i++) { + if (msi->grps[i].gic_irq) + irq_dispose_mapping(msi->grps[i].gic_irq); + } + pcie->msi = NULL; + return ret; +} +EXPORT_SYMBOL(iproc_msi_init); + +void iproc_msi_exit(struct iproc_pcie *pcie) +{ + struct iproc_msi *msi = pcie->msi; + unsigned int i, cpu; + + if (!msi) + return; + + iproc_msi_disable(msi); + + for_each_online_cpu(cpu) + iproc_msi_irq_free(msi, cpu); + + iproc_msi_free_domains(msi); + + dma_free_coherent(pcie->dev, msi->nr_eq_region * EQ_MEM_REGION_SIZE, + msi->eq_cpu, msi->eq_dma); + + for (i = 0; i < msi->nr_irqs; i++) { + if (msi->grps[i].gic_irq) + irq_dispose_mapping(msi->grps[i].gic_irq); + } +} +EXPORT_SYMBOL(iproc_msi_exit); diff --git a/drivers/pci/controller/pcie-iproc-platform.c b/drivers/pci/controller/pcie-iproc-platform.c new file mode 100644 index 000000000..4c9a0c4bb --- /dev/null +++ b/drivers/pci/controller/pcie-iproc-platform.c @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2015 Broadcom Corporation + */ + +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/clk.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/interrupt.h> +#include <linux/platform_device.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/phy/phy.h> + +#include "../pci.h" +#include "pcie-iproc.h" + +static const struct of_device_id iproc_pcie_of_match_table[] = { + { + .compatible = "brcm,iproc-pcie", + .data = (int *)IPROC_PCIE_PAXB, + }, { + .compatible = "brcm,iproc-pcie-paxb-v2", + .data = (int *)IPROC_PCIE_PAXB_V2, + }, { + .compatible = "brcm,iproc-pcie-paxc", + .data = (int *)IPROC_PCIE_PAXC, + }, { + .compatible = "brcm,iproc-pcie-paxc-v2", + .data = (int *)IPROC_PCIE_PAXC_V2, + }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, iproc_pcie_of_match_table); + +static int iproc_pltfm_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct iproc_pcie *pcie; + struct device_node *np = dev->of_node; + struct resource reg; + struct pci_host_bridge *bridge; + int ret; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!bridge) + return -ENOMEM; + + pcie = pci_host_bridge_priv(bridge); + + pcie->dev = dev; + pcie->type = (uintptr_t)of_device_get_match_data(dev); + + ret = of_address_to_resource(np, 0, ®); + if (ret < 0) { + dev_err(dev, "unable to obtain controller resources\n"); + return ret; + } + + pcie->base = devm_pci_remap_cfgspace(dev, reg.start, + resource_size(®)); + if (!pcie->base) { + dev_err(dev, "unable to map controller registers\n"); + return -ENOMEM; + } + pcie->base_addr = reg.start; + + if (of_property_read_bool(np, "brcm,pcie-ob")) { + u32 val; + + ret = of_property_read_u32(np, "brcm,pcie-ob-axi-offset", + &val); + if (ret) { + dev_err(dev, + "missing brcm,pcie-ob-axi-offset property\n"); + return ret; + } + pcie->ob.axi_offset = val; + pcie->need_ob_cfg = true; + } + + /* + * DT nodes are not used by all platforms that use the iProc PCIe + * core driver. For platforms that require explicit inbound mapping + * configuration, "dma-ranges" would have been present in DT + */ + pcie->need_ib_cfg = of_property_read_bool(np, "dma-ranges"); + + /* PHY use is optional */ + pcie->phy = devm_phy_optional_get(dev, "pcie-phy"); + if (IS_ERR(pcie->phy)) + return PTR_ERR(pcie->phy); + + /* PAXC doesn't support legacy IRQs, skip mapping */ + switch (pcie->type) { + case IPROC_PCIE_PAXC: + case IPROC_PCIE_PAXC_V2: + bridge->map_irq = NULL; + break; + default: + break; + } + + ret = iproc_pcie_setup(pcie, &bridge->windows); + if (ret) { + dev_err(dev, "PCIe controller setup failed\n"); + return ret; + } + + platform_set_drvdata(pdev, pcie); + return 0; +} + +static void iproc_pltfm_pcie_remove(struct platform_device *pdev) +{ + struct iproc_pcie *pcie = platform_get_drvdata(pdev); + + iproc_pcie_remove(pcie); +} + +static void iproc_pltfm_pcie_shutdown(struct platform_device *pdev) +{ + struct iproc_pcie *pcie = platform_get_drvdata(pdev); + + iproc_pcie_shutdown(pcie); +} + +static struct platform_driver iproc_pltfm_pcie_driver = { + .driver = { + .name = "iproc-pcie", + .of_match_table = of_match_ptr(iproc_pcie_of_match_table), + }, + .probe = iproc_pltfm_pcie_probe, + .remove = iproc_pltfm_pcie_remove, + .shutdown = iproc_pltfm_pcie_shutdown, +}; +module_platform_driver(iproc_pltfm_pcie_driver); + +MODULE_AUTHOR("Ray Jui <rjui@broadcom.com>"); +MODULE_DESCRIPTION("Broadcom iPROC PCIe platform driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pcie-iproc.c b/drivers/pci/controller/pcie-iproc.c new file mode 100644 index 000000000..afb3aa38a --- /dev/null +++ b/drivers/pci/controller/pcie-iproc.c @@ -0,0 +1,1587 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2014 Hauke Mehrtens <hauke@hauke-m.de> + * Copyright (C) 2015 Broadcom Corporation + */ + +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/pci-ecam.h> +#include <linux/msi.h> +#include <linux/clk.h> +#include <linux/module.h> +#include <linux/mbus.h> +#include <linux/slab.h> +#include <linux/delay.h> +#include <linux/interrupt.h> +#include <linux/irqchip/arm-gic-v3.h> +#include <linux/platform_device.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/phy/phy.h> + +#include "pcie-iproc.h" + +#define EP_PERST_SOURCE_SELECT_SHIFT 2 +#define EP_PERST_SOURCE_SELECT BIT(EP_PERST_SOURCE_SELECT_SHIFT) +#define EP_MODE_SURVIVE_PERST_SHIFT 1 +#define EP_MODE_SURVIVE_PERST BIT(EP_MODE_SURVIVE_PERST_SHIFT) +#define RC_PCIE_RST_OUTPUT_SHIFT 0 +#define RC_PCIE_RST_OUTPUT BIT(RC_PCIE_RST_OUTPUT_SHIFT) +#define PAXC_RESET_MASK 0x7f + +#define GIC_V3_CFG_SHIFT 0 +#define GIC_V3_CFG BIT(GIC_V3_CFG_SHIFT) + +#define MSI_ENABLE_CFG_SHIFT 0 +#define MSI_ENABLE_CFG BIT(MSI_ENABLE_CFG_SHIFT) + +#define CFG_IND_ADDR_MASK 0x00001ffc + +#define CFG_ADDR_REG_NUM_MASK 0x00000ffc +#define CFG_ADDR_CFG_TYPE_1 1 + +#define SYS_RC_INTX_MASK 0xf + +#define PCIE_PHYLINKUP_SHIFT 3 +#define PCIE_PHYLINKUP BIT(PCIE_PHYLINKUP_SHIFT) +#define PCIE_DL_ACTIVE_SHIFT 2 +#define PCIE_DL_ACTIVE BIT(PCIE_DL_ACTIVE_SHIFT) + +#define APB_ERR_EN_SHIFT 0 +#define APB_ERR_EN BIT(APB_ERR_EN_SHIFT) + +#define CFG_RD_SUCCESS 0 +#define CFG_RD_UR 1 +#define CFG_RD_RRS 2 +#define CFG_RD_CA 3 +#define CFG_RETRY_STATUS 0xffff0001 +#define CFG_RETRY_STATUS_TIMEOUT_US 500000 /* 500 milliseconds */ + +/* derive the enum index of the outbound/inbound mapping registers */ +#define MAP_REG(base_reg, index) ((base_reg) + (index) * 2) + +/* + * Maximum number of outbound mapping window sizes that can be supported by any + * OARR/OMAP mapping pair + */ +#define MAX_NUM_OB_WINDOW_SIZES 4 + +#define OARR_VALID_SHIFT 0 +#define OARR_VALID BIT(OARR_VALID_SHIFT) +#define OARR_SIZE_CFG_SHIFT 1 + +/* + * Maximum number of inbound mapping region sizes that can be supported by an + * IARR + */ +#define MAX_NUM_IB_REGION_SIZES 9 + +#define IMAP_VALID_SHIFT 0 +#define IMAP_VALID BIT(IMAP_VALID_SHIFT) + +#define IPROC_PCI_PM_CAP 0x48 +#define IPROC_PCI_PM_CAP_MASK 0xffff +#define IPROC_PCI_EXP_CAP 0xac + +#define IPROC_PCIE_REG_INVALID 0xffff + +/** + * struct iproc_pcie_ob_map - iProc PCIe outbound mapping controller-specific + * parameters + * @window_sizes: list of supported outbound mapping window sizes in MB + * @nr_sizes: number of supported outbound mapping window sizes + */ +struct iproc_pcie_ob_map { + resource_size_t window_sizes[MAX_NUM_OB_WINDOW_SIZES]; + unsigned int nr_sizes; +}; + +static const struct iproc_pcie_ob_map paxb_ob_map[] = { + { + /* OARR0/OMAP0 */ + .window_sizes = { 128, 256 }, + .nr_sizes = 2, + }, + { + /* OARR1/OMAP1 */ + .window_sizes = { 128, 256 }, + .nr_sizes = 2, + }, +}; + +static const struct iproc_pcie_ob_map paxb_v2_ob_map[] = { + { + /* OARR0/OMAP0 */ + .window_sizes = { 128, 256 }, + .nr_sizes = 2, + }, + { + /* OARR1/OMAP1 */ + .window_sizes = { 128, 256 }, + .nr_sizes = 2, + }, + { + /* OARR2/OMAP2 */ + .window_sizes = { 128, 256, 512, 1024 }, + .nr_sizes = 4, + }, + { + /* OARR3/OMAP3 */ + .window_sizes = { 128, 256, 512, 1024 }, + .nr_sizes = 4, + }, +}; + +/** + * enum iproc_pcie_ib_map_type - iProc PCIe inbound mapping type + * @IPROC_PCIE_IB_MAP_MEM: DDR memory + * @IPROC_PCIE_IB_MAP_IO: device I/O memory + * @IPROC_PCIE_IB_MAP_INVALID: invalid or unused + */ +enum iproc_pcie_ib_map_type { + IPROC_PCIE_IB_MAP_MEM = 0, + IPROC_PCIE_IB_MAP_IO, + IPROC_PCIE_IB_MAP_INVALID +}; + +/** + * struct iproc_pcie_ib_map - iProc PCIe inbound mapping controller-specific + * parameters + * @type: inbound mapping region type + * @size_unit: inbound mapping region size unit, could be SZ_1K, SZ_1M, or + * SZ_1G + * @region_sizes: list of supported inbound mapping region sizes in KB, MB, or + * GB, depending on the size unit + * @nr_sizes: number of supported inbound mapping region sizes + * @nr_windows: number of supported inbound mapping windows for the region + * @imap_addr_offset: register offset between the upper and lower 32-bit + * IMAP address registers + * @imap_window_offset: register offset between each IMAP window + */ +struct iproc_pcie_ib_map { + enum iproc_pcie_ib_map_type type; + unsigned int size_unit; + resource_size_t region_sizes[MAX_NUM_IB_REGION_SIZES]; + unsigned int nr_sizes; + unsigned int nr_windows; + u16 imap_addr_offset; + u16 imap_window_offset; +}; + +static const struct iproc_pcie_ib_map paxb_v2_ib_map[] = { + { + /* IARR0/IMAP0 */ + .type = IPROC_PCIE_IB_MAP_IO, + .size_unit = SZ_1K, + .region_sizes = { 32 }, + .nr_sizes = 1, + .nr_windows = 8, + .imap_addr_offset = 0x40, + .imap_window_offset = 0x4, + }, + { + /* IARR1/IMAP1 */ + .type = IPROC_PCIE_IB_MAP_MEM, + .size_unit = SZ_1M, + .region_sizes = { 8 }, + .nr_sizes = 1, + .nr_windows = 8, + .imap_addr_offset = 0x4, + .imap_window_offset = 0x8, + + }, + { + /* IARR2/IMAP2 */ + .type = IPROC_PCIE_IB_MAP_MEM, + .size_unit = SZ_1M, + .region_sizes = { 64, 128, 256, 512, 1024, 2048, 4096, 8192, + 16384 }, + .nr_sizes = 9, + .nr_windows = 1, + .imap_addr_offset = 0x4, + .imap_window_offset = 0x8, + }, + { + /* IARR3/IMAP3 */ + .type = IPROC_PCIE_IB_MAP_MEM, + .size_unit = SZ_1G, + .region_sizes = { 1, 2, 4, 8, 16, 32 }, + .nr_sizes = 6, + .nr_windows = 8, + .imap_addr_offset = 0x4, + .imap_window_offset = 0x8, + }, + { + /* IARR4/IMAP4 */ + .type = IPROC_PCIE_IB_MAP_MEM, + .size_unit = SZ_1G, + .region_sizes = { 32, 64, 128, 256, 512 }, + .nr_sizes = 5, + .nr_windows = 8, + .imap_addr_offset = 0x4, + .imap_window_offset = 0x8, + }, +}; + +/* + * iProc PCIe host registers + */ +enum iproc_pcie_reg { + /* clock/reset signal control */ + IPROC_PCIE_CLK_CTRL = 0, + + /* + * To allow MSI to be steered to an external MSI controller (e.g., ARM + * GICv3 ITS) + */ + IPROC_PCIE_MSI_GIC_MODE, + + /* + * IPROC_PCIE_MSI_BASE_ADDR and IPROC_PCIE_MSI_WINDOW_SIZE define the + * window where the MSI posted writes are written, for the writes to be + * interpreted as MSI writes. + */ + IPROC_PCIE_MSI_BASE_ADDR, + IPROC_PCIE_MSI_WINDOW_SIZE, + + /* + * To hold the address of the register where the MSI writes are + * programmed. When ARM GICv3 ITS is used, this should be programmed + * with the address of the GITS_TRANSLATER register. + */ + IPROC_PCIE_MSI_ADDR_LO, + IPROC_PCIE_MSI_ADDR_HI, + + /* enable MSI */ + IPROC_PCIE_MSI_EN_CFG, + + /* allow access to root complex configuration space */ + IPROC_PCIE_CFG_IND_ADDR, + IPROC_PCIE_CFG_IND_DATA, + + /* allow access to device configuration space */ + IPROC_PCIE_CFG_ADDR, + IPROC_PCIE_CFG_DATA, + + /* enable INTx */ + IPROC_PCIE_INTX_EN, + + /* outbound address mapping */ + IPROC_PCIE_OARR0, + IPROC_PCIE_OMAP0, + IPROC_PCIE_OARR1, + IPROC_PCIE_OMAP1, + IPROC_PCIE_OARR2, + IPROC_PCIE_OMAP2, + IPROC_PCIE_OARR3, + IPROC_PCIE_OMAP3, + + /* inbound address mapping */ + IPROC_PCIE_IARR0, + IPROC_PCIE_IMAP0, + IPROC_PCIE_IARR1, + IPROC_PCIE_IMAP1, + IPROC_PCIE_IARR2, + IPROC_PCIE_IMAP2, + IPROC_PCIE_IARR3, + IPROC_PCIE_IMAP3, + IPROC_PCIE_IARR4, + IPROC_PCIE_IMAP4, + + /* config read status */ + IPROC_PCIE_CFG_RD_STATUS, + + /* link status */ + IPROC_PCIE_LINK_STATUS, + + /* enable APB error for unsupported requests */ + IPROC_PCIE_APB_ERR_EN, + + /* total number of core registers */ + IPROC_PCIE_MAX_NUM_REG, +}; + +/* iProc PCIe PAXB BCMA registers */ +static const u16 iproc_pcie_reg_paxb_bcma[IPROC_PCIE_MAX_NUM_REG] = { + [IPROC_PCIE_CLK_CTRL] = 0x000, + [IPROC_PCIE_CFG_IND_ADDR] = 0x120, + [IPROC_PCIE_CFG_IND_DATA] = 0x124, + [IPROC_PCIE_CFG_ADDR] = 0x1f8, + [IPROC_PCIE_CFG_DATA] = 0x1fc, + [IPROC_PCIE_INTX_EN] = 0x330, + [IPROC_PCIE_LINK_STATUS] = 0xf0c, +}; + +/* iProc PCIe PAXB registers */ +static const u16 iproc_pcie_reg_paxb[IPROC_PCIE_MAX_NUM_REG] = { + [IPROC_PCIE_CLK_CTRL] = 0x000, + [IPROC_PCIE_CFG_IND_ADDR] = 0x120, + [IPROC_PCIE_CFG_IND_DATA] = 0x124, + [IPROC_PCIE_CFG_ADDR] = 0x1f8, + [IPROC_PCIE_CFG_DATA] = 0x1fc, + [IPROC_PCIE_INTX_EN] = 0x330, + [IPROC_PCIE_OARR0] = 0xd20, + [IPROC_PCIE_OMAP0] = 0xd40, + [IPROC_PCIE_OARR1] = 0xd28, + [IPROC_PCIE_OMAP1] = 0xd48, + [IPROC_PCIE_LINK_STATUS] = 0xf0c, + [IPROC_PCIE_APB_ERR_EN] = 0xf40, +}; + +/* iProc PCIe PAXB v2 registers */ +static const u16 iproc_pcie_reg_paxb_v2[IPROC_PCIE_MAX_NUM_REG] = { + [IPROC_PCIE_CLK_CTRL] = 0x000, + [IPROC_PCIE_CFG_IND_ADDR] = 0x120, + [IPROC_PCIE_CFG_IND_DATA] = 0x124, + [IPROC_PCIE_CFG_ADDR] = 0x1f8, + [IPROC_PCIE_CFG_DATA] = 0x1fc, + [IPROC_PCIE_INTX_EN] = 0x330, + [IPROC_PCIE_OARR0] = 0xd20, + [IPROC_PCIE_OMAP0] = 0xd40, + [IPROC_PCIE_OARR1] = 0xd28, + [IPROC_PCIE_OMAP1] = 0xd48, + [IPROC_PCIE_OARR2] = 0xd60, + [IPROC_PCIE_OMAP2] = 0xd68, + [IPROC_PCIE_OARR3] = 0xdf0, + [IPROC_PCIE_OMAP3] = 0xdf8, + [IPROC_PCIE_IARR0] = 0xd00, + [IPROC_PCIE_IMAP0] = 0xc00, + [IPROC_PCIE_IARR1] = 0xd08, + [IPROC_PCIE_IMAP1] = 0xd70, + [IPROC_PCIE_IARR2] = 0xd10, + [IPROC_PCIE_IMAP2] = 0xcc0, + [IPROC_PCIE_IARR3] = 0xe00, + [IPROC_PCIE_IMAP3] = 0xe08, + [IPROC_PCIE_IARR4] = 0xe68, + [IPROC_PCIE_IMAP4] = 0xe70, + [IPROC_PCIE_CFG_RD_STATUS] = 0xee0, + [IPROC_PCIE_LINK_STATUS] = 0xf0c, + [IPROC_PCIE_APB_ERR_EN] = 0xf40, +}; + +/* iProc PCIe PAXC v1 registers */ +static const u16 iproc_pcie_reg_paxc[IPROC_PCIE_MAX_NUM_REG] = { + [IPROC_PCIE_CLK_CTRL] = 0x000, + [IPROC_PCIE_CFG_IND_ADDR] = 0x1f0, + [IPROC_PCIE_CFG_IND_DATA] = 0x1f4, + [IPROC_PCIE_CFG_ADDR] = 0x1f8, + [IPROC_PCIE_CFG_DATA] = 0x1fc, +}; + +/* iProc PCIe PAXC v2 registers */ +static const u16 iproc_pcie_reg_paxc_v2[IPROC_PCIE_MAX_NUM_REG] = { + [IPROC_PCIE_MSI_GIC_MODE] = 0x050, + [IPROC_PCIE_MSI_BASE_ADDR] = 0x074, + [IPROC_PCIE_MSI_WINDOW_SIZE] = 0x078, + [IPROC_PCIE_MSI_ADDR_LO] = 0x07c, + [IPROC_PCIE_MSI_ADDR_HI] = 0x080, + [IPROC_PCIE_MSI_EN_CFG] = 0x09c, + [IPROC_PCIE_CFG_IND_ADDR] = 0x1f0, + [IPROC_PCIE_CFG_IND_DATA] = 0x1f4, + [IPROC_PCIE_CFG_ADDR] = 0x1f8, + [IPROC_PCIE_CFG_DATA] = 0x1fc, +}; + +/* + * List of device IDs of controllers that have corrupted capability list that + * require SW fixup + */ +static const u16 iproc_pcie_corrupt_cap_did[] = { + 0x16cd, + 0x16f0, + 0xd802, + 0xd804 +}; + +static inline struct iproc_pcie *iproc_data(struct pci_bus *bus) +{ + struct iproc_pcie *pcie = bus->sysdata; + return pcie; +} + +static inline bool iproc_pcie_reg_is_invalid(u16 reg_offset) +{ + return !!(reg_offset == IPROC_PCIE_REG_INVALID); +} + +static inline u16 iproc_pcie_reg_offset(struct iproc_pcie *pcie, + enum iproc_pcie_reg reg) +{ + return pcie->reg_offsets[reg]; +} + +static inline u32 iproc_pcie_read_reg(struct iproc_pcie *pcie, + enum iproc_pcie_reg reg) +{ + u16 offset = iproc_pcie_reg_offset(pcie, reg); + + if (iproc_pcie_reg_is_invalid(offset)) + return 0; + + return readl(pcie->base + offset); +} + +static inline void iproc_pcie_write_reg(struct iproc_pcie *pcie, + enum iproc_pcie_reg reg, u32 val) +{ + u16 offset = iproc_pcie_reg_offset(pcie, reg); + + if (iproc_pcie_reg_is_invalid(offset)) + return; + + writel(val, pcie->base + offset); +} + +/* + * APB error forwarding can be disabled during access of configuration + * registers of the endpoint device, to prevent unsupported requests + * (typically seen during enumeration with multi-function devices) from + * triggering a system exception. + */ +static inline void iproc_pcie_apb_err_disable(struct pci_bus *bus, + bool disable) +{ + struct iproc_pcie *pcie = iproc_data(bus); + u32 val; + + if (bus->number && pcie->has_apb_err_disable) { + val = iproc_pcie_read_reg(pcie, IPROC_PCIE_APB_ERR_EN); + if (disable) + val &= ~APB_ERR_EN; + else + val |= APB_ERR_EN; + iproc_pcie_write_reg(pcie, IPROC_PCIE_APB_ERR_EN, val); + } +} + +static void __iomem *iproc_pcie_map_ep_cfg_reg(struct iproc_pcie *pcie, + unsigned int busno, + unsigned int devfn, + int where) +{ + u16 offset; + u32 val; + + /* EP device access */ + val = ALIGN_DOWN(PCIE_ECAM_OFFSET(busno, devfn, where), 4) | + CFG_ADDR_CFG_TYPE_1; + + iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_ADDR, val); + offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_DATA); + + if (iproc_pcie_reg_is_invalid(offset)) + return NULL; + + return (pcie->base + offset); +} + +static unsigned int iproc_pcie_cfg_retry(struct iproc_pcie *pcie, + void __iomem *cfg_data_p) +{ + int timeout = CFG_RETRY_STATUS_TIMEOUT_US; + unsigned int data; + u32 status; + + /* + * As per PCIe r6.0, sec 2.3.2, Config RRS Software Visibility only + * affects config reads of the Vendor ID. For config writes or any + * other config reads, the Root may automatically reissue the + * configuration request again as a new request. + * + * For config reads, this hardware returns CFG_RETRY_STATUS data + * when it receives a RRS completion, regardless of the address of + * the read or the RRS Software Visibility Enable bit. As a + * partial workaround for this, we retry in software any read that + * returns CFG_RETRY_STATUS. + * + * Note that a non-Vendor ID config register may have a value of + * CFG_RETRY_STATUS. If we read that, we can't distinguish it from + * a RRS completion, so we will incorrectly retry the read and + * eventually return the wrong data (0xffffffff). + */ + data = readl(cfg_data_p); + while (data == CFG_RETRY_STATUS && timeout--) { + /* + * RRS state is set in CFG_RD status register + * This will handle the case where CFG_RETRY_STATUS is + * valid config data. + */ + status = iproc_pcie_read_reg(pcie, IPROC_PCIE_CFG_RD_STATUS); + if (status != CFG_RD_RRS) + return data; + + udelay(1); + data = readl(cfg_data_p); + } + + if (data == CFG_RETRY_STATUS) + data = 0xffffffff; + + return data; +} + +static void iproc_pcie_fix_cap(struct iproc_pcie *pcie, int where, u32 *val) +{ + u32 i, dev_id; + + switch (where & ~0x3) { + case PCI_VENDOR_ID: + dev_id = *val >> 16; + + /* + * Activate fixup for those controllers that have corrupted + * capability list registers + */ + for (i = 0; i < ARRAY_SIZE(iproc_pcie_corrupt_cap_did); i++) + if (dev_id == iproc_pcie_corrupt_cap_did[i]) + pcie->fix_paxc_cap = true; + break; + + case IPROC_PCI_PM_CAP: + if (pcie->fix_paxc_cap) { + /* advertise PM, force next capability to PCIe */ + *val &= ~IPROC_PCI_PM_CAP_MASK; + *val |= IPROC_PCI_EXP_CAP << 8 | PCI_CAP_ID_PM; + } + break; + + case IPROC_PCI_EXP_CAP: + if (pcie->fix_paxc_cap) { + /* advertise root port, version 2, terminate here */ + *val = (PCI_EXP_TYPE_ROOT_PORT << 4 | 2) << 16 | + PCI_CAP_ID_EXP; + } + break; + + case IPROC_PCI_EXP_CAP + PCI_EXP_RTCTL: + /* Don't advertise RRS SV support */ + *val &= ~(PCI_EXP_RTCAP_RRS_SV << 16); + break; + + default: + break; + } +} + +static int iproc_pcie_config_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct iproc_pcie *pcie = iproc_data(bus); + unsigned int busno = bus->number; + void __iomem *cfg_data_p; + unsigned int data; + int ret; + + /* root complex access */ + if (busno == 0) { + ret = pci_generic_config_read32(bus, devfn, where, size, val); + if (ret == PCIBIOS_SUCCESSFUL) + iproc_pcie_fix_cap(pcie, where, val); + + return ret; + } + + cfg_data_p = iproc_pcie_map_ep_cfg_reg(pcie, busno, devfn, where); + + if (!cfg_data_p) + return PCIBIOS_DEVICE_NOT_FOUND; + + data = iproc_pcie_cfg_retry(pcie, cfg_data_p); + + *val = data; + if (size <= 2) + *val = (data >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); + + /* + * For PAXC and PAXCv2, the total number of PFs that one can enumerate + * depends on the firmware configuration. Unfortunately, due to an ASIC + * bug, unconfigured PFs cannot be properly hidden from the root + * complex. As a result, write access to these PFs will cause bus lock + * up on the embedded processor + * + * Since all unconfigured PFs are left with an incorrect, staled device + * ID of 0x168e (PCI_DEVICE_ID_NX2_57810), we try to catch those access + * early here and reject them all + */ +#define DEVICE_ID_MASK 0xffff0000 +#define DEVICE_ID_SHIFT 16 + if (pcie->rej_unconfig_pf && + (where & CFG_ADDR_REG_NUM_MASK) == PCI_VENDOR_ID) + if ((*val & DEVICE_ID_MASK) == + (PCI_DEVICE_ID_NX2_57810 << DEVICE_ID_SHIFT)) + return PCIBIOS_FUNC_NOT_SUPPORTED; + + return PCIBIOS_SUCCESSFUL; +} + +/* + * Note access to the configuration registers are protected at the higher layer + * by 'pci_lock' in drivers/pci/access.c + */ +static void __iomem *iproc_pcie_map_cfg_bus(struct iproc_pcie *pcie, + int busno, unsigned int devfn, + int where) +{ + u16 offset; + + /* root complex access */ + if (busno == 0) { + if (PCIE_ECAM_DEVFN(devfn) > 0) + return NULL; + + iproc_pcie_write_reg(pcie, IPROC_PCIE_CFG_IND_ADDR, + where & CFG_IND_ADDR_MASK); + offset = iproc_pcie_reg_offset(pcie, IPROC_PCIE_CFG_IND_DATA); + if (iproc_pcie_reg_is_invalid(offset)) + return NULL; + else + return (pcie->base + offset); + } + + return iproc_pcie_map_ep_cfg_reg(pcie, busno, devfn, where); +} + +static void __iomem *iproc_pcie_bus_map_cfg_bus(struct pci_bus *bus, + unsigned int devfn, + int where) +{ + return iproc_pcie_map_cfg_bus(iproc_data(bus), bus->number, devfn, + where); +} + +static int iproc_pci_raw_config_read32(struct iproc_pcie *pcie, + unsigned int devfn, int where, + int size, u32 *val) +{ + void __iomem *addr; + + addr = iproc_pcie_map_cfg_bus(pcie, 0, devfn, where & ~0x3); + if (!addr) + return PCIBIOS_DEVICE_NOT_FOUND; + + *val = readl(addr); + + if (size <= 2) + *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); + + return PCIBIOS_SUCCESSFUL; +} + +static int iproc_pci_raw_config_write32(struct iproc_pcie *pcie, + unsigned int devfn, int where, + int size, u32 val) +{ + void __iomem *addr; + u32 mask, tmp; + + addr = iproc_pcie_map_cfg_bus(pcie, 0, devfn, where & ~0x3); + if (!addr) + return PCIBIOS_DEVICE_NOT_FOUND; + + if (size == 4) { + writel(val, addr); + return PCIBIOS_SUCCESSFUL; + } + + mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); + tmp = readl(addr) & mask; + tmp |= val << ((where & 0x3) * 8); + writel(tmp, addr); + + return PCIBIOS_SUCCESSFUL; +} + +static int iproc_pcie_config_read32(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + int ret; + struct iproc_pcie *pcie = iproc_data(bus); + + iproc_pcie_apb_err_disable(bus, true); + if (pcie->iproc_cfg_read) + ret = iproc_pcie_config_read(bus, devfn, where, size, val); + else + ret = pci_generic_config_read32(bus, devfn, where, size, val); + iproc_pcie_apb_err_disable(bus, false); + + return ret; +} + +static int iproc_pcie_config_write32(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + int ret; + + iproc_pcie_apb_err_disable(bus, true); + ret = pci_generic_config_write32(bus, devfn, where, size, val); + iproc_pcie_apb_err_disable(bus, false); + + return ret; +} + +static struct pci_ops iproc_pcie_ops = { + .map_bus = iproc_pcie_bus_map_cfg_bus, + .read = iproc_pcie_config_read32, + .write = iproc_pcie_config_write32, +}; + +static void iproc_pcie_perst_ctrl(struct iproc_pcie *pcie, bool assert) +{ + u32 val; + + /* + * PAXC and the internal emulated endpoint device downstream should not + * be reset. If firmware has been loaded on the endpoint device at an + * earlier boot stage, reset here causes issues. + */ + if (pcie->ep_is_internal) + return; + + if (assert) { + val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL); + val &= ~EP_PERST_SOURCE_SELECT & ~EP_MODE_SURVIVE_PERST & + ~RC_PCIE_RST_OUTPUT; + iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val); + udelay(250); + } else { + val = iproc_pcie_read_reg(pcie, IPROC_PCIE_CLK_CTRL); + val |= RC_PCIE_RST_OUTPUT; + iproc_pcie_write_reg(pcie, IPROC_PCIE_CLK_CTRL, val); + msleep(100); + } +} + +int iproc_pcie_shutdown(struct iproc_pcie *pcie) +{ + iproc_pcie_perst_ctrl(pcie, true); + msleep(500); + + return 0; +} +EXPORT_SYMBOL_GPL(iproc_pcie_shutdown); + +static int iproc_pcie_check_link(struct iproc_pcie *pcie) +{ + struct device *dev = pcie->dev; + u32 hdr_type, link_ctrl, link_status, class, val; + bool link_is_active = false; + + /* + * PAXC connects to emulated endpoint devices directly and does not + * have a Serdes. Therefore skip the link detection logic here. + */ + if (pcie->ep_is_internal) + return 0; + + val = iproc_pcie_read_reg(pcie, IPROC_PCIE_LINK_STATUS); + if (!(val & PCIE_PHYLINKUP) || !(val & PCIE_DL_ACTIVE)) { + dev_err(dev, "PHY or data link is INACTIVE!\n"); + return -ENODEV; + } + + /* make sure we are not in EP mode */ + iproc_pci_raw_config_read32(pcie, 0, PCI_HEADER_TYPE, 1, &hdr_type); + if ((hdr_type & PCI_HEADER_TYPE_MASK) != PCI_HEADER_TYPE_BRIDGE) { + dev_err(dev, "in EP mode, hdr=%#02x\n", hdr_type); + return -EFAULT; + } + + /* force class to PCI_CLASS_BRIDGE_PCI_NORMAL (0x060400) */ +#define PCI_BRIDGE_CTRL_REG_OFFSET 0x43c +#define PCI_BRIDGE_CTRL_REG_CLASS_MASK 0xffffff + iproc_pci_raw_config_read32(pcie, 0, PCI_BRIDGE_CTRL_REG_OFFSET, + 4, &class); + class &= ~PCI_BRIDGE_CTRL_REG_CLASS_MASK; + class |= PCI_CLASS_BRIDGE_PCI_NORMAL; + iproc_pci_raw_config_write32(pcie, 0, PCI_BRIDGE_CTRL_REG_OFFSET, + 4, class); + + /* check link status to see if link is active */ + iproc_pci_raw_config_read32(pcie, 0, IPROC_PCI_EXP_CAP + PCI_EXP_LNKSTA, + 2, &link_status); + if (link_status & PCI_EXP_LNKSTA_NLW) + link_is_active = true; + + if (!link_is_active) { + /* try GEN 1 link speed */ +#define PCI_TARGET_LINK_SPEED_MASK 0xf +#define PCI_TARGET_LINK_SPEED_GEN2 0x2 +#define PCI_TARGET_LINK_SPEED_GEN1 0x1 + iproc_pci_raw_config_read32(pcie, 0, + IPROC_PCI_EXP_CAP + PCI_EXP_LNKCTL2, + 4, &link_ctrl); + if ((link_ctrl & PCI_TARGET_LINK_SPEED_MASK) == + PCI_TARGET_LINK_SPEED_GEN2) { + link_ctrl &= ~PCI_TARGET_LINK_SPEED_MASK; + link_ctrl |= PCI_TARGET_LINK_SPEED_GEN1; + iproc_pci_raw_config_write32(pcie, 0, + IPROC_PCI_EXP_CAP + PCI_EXP_LNKCTL2, + 4, link_ctrl); + msleep(100); + + iproc_pci_raw_config_read32(pcie, 0, + IPROC_PCI_EXP_CAP + PCI_EXP_LNKSTA, + 2, &link_status); + if (link_status & PCI_EXP_LNKSTA_NLW) + link_is_active = true; + } + } + + dev_info(dev, "link: %s\n", link_is_active ? "UP" : "DOWN"); + + return link_is_active ? 0 : -ENODEV; +} + +static void iproc_pcie_enable(struct iproc_pcie *pcie) +{ + iproc_pcie_write_reg(pcie, IPROC_PCIE_INTX_EN, SYS_RC_INTX_MASK); +} + +static inline bool iproc_pcie_ob_is_valid(struct iproc_pcie *pcie, + int window_idx) +{ + u32 val; + + val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_OARR0, window_idx)); + + return !!(val & OARR_VALID); +} + +static inline int iproc_pcie_ob_write(struct iproc_pcie *pcie, int window_idx, + int size_idx, u64 axi_addr, u64 pci_addr) +{ + struct device *dev = pcie->dev; + u16 oarr_offset, omap_offset; + + /* + * Derive the OARR/OMAP offset from the first pair (OARR0/OMAP0) based + * on window index. + */ + oarr_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OARR0, + window_idx)); + omap_offset = iproc_pcie_reg_offset(pcie, MAP_REG(IPROC_PCIE_OMAP0, + window_idx)); + if (iproc_pcie_reg_is_invalid(oarr_offset) || + iproc_pcie_reg_is_invalid(omap_offset)) + return -EINVAL; + + /* + * Program the OARR registers. The upper 32-bit OARR register is + * always right after the lower 32-bit OARR register. + */ + writel(lower_32_bits(axi_addr) | (size_idx << OARR_SIZE_CFG_SHIFT) | + OARR_VALID, pcie->base + oarr_offset); + writel(upper_32_bits(axi_addr), pcie->base + oarr_offset + 4); + + /* now program the OMAP registers */ + writel(lower_32_bits(pci_addr), pcie->base + omap_offset); + writel(upper_32_bits(pci_addr), pcie->base + omap_offset + 4); + + dev_dbg(dev, "ob window [%d]: offset 0x%x axi %pap pci %pap\n", + window_idx, oarr_offset, &axi_addr, &pci_addr); + dev_dbg(dev, "oarr lo 0x%x oarr hi 0x%x\n", + readl(pcie->base + oarr_offset), + readl(pcie->base + oarr_offset + 4)); + dev_dbg(dev, "omap lo 0x%x omap hi 0x%x\n", + readl(pcie->base + omap_offset), + readl(pcie->base + omap_offset + 4)); + + return 0; +} + +/* + * Some iProc SoCs require the SW to configure the outbound address mapping + * + * Outbound address translation: + * + * iproc_pcie_address = axi_address - axi_offset + * OARR = iproc_pcie_address + * OMAP = pci_addr + * + * axi_addr -> iproc_pcie_address -> OARR -> OMAP -> pci_address + */ +static int iproc_pcie_setup_ob(struct iproc_pcie *pcie, u64 axi_addr, + u64 pci_addr, resource_size_t size) +{ + struct iproc_pcie_ob *ob = &pcie->ob; + struct device *dev = pcie->dev; + int ret = -EINVAL, window_idx, size_idx; + + if (axi_addr < ob->axi_offset) { + dev_err(dev, "axi address %pap less than offset %pap\n", + &axi_addr, &ob->axi_offset); + return -EINVAL; + } + + /* + * Translate the AXI address to the internal address used by the iProc + * PCIe core before programming the OARR + */ + axi_addr -= ob->axi_offset; + + /* iterate through all OARR/OMAP mapping windows */ + for (window_idx = ob->nr_windows - 1; window_idx >= 0; window_idx--) { + const struct iproc_pcie_ob_map *ob_map = + &pcie->ob_map[window_idx]; + + /* + * If current outbound window is already in use, move on to the + * next one. + */ + if (iproc_pcie_ob_is_valid(pcie, window_idx)) + continue; + + /* + * Iterate through all supported window sizes within the + * OARR/OMAP pair to find a match. Go through the window sizes + * in a descending order. + */ + for (size_idx = ob_map->nr_sizes - 1; size_idx >= 0; + size_idx--) { + resource_size_t window_size = + ob_map->window_sizes[size_idx] * SZ_1M; + + /* + * Keep iterating until we reach the last window and + * with the minimal window size at index zero. In this + * case, we take a compromise by mapping it using the + * minimum window size that can be supported + */ + if (size < window_size) { + if (size_idx > 0 || window_idx > 0) + continue; + + /* + * For the corner case of reaching the minimal + * window size that can be supported on the + * last window + */ + axi_addr = ALIGN_DOWN(axi_addr, window_size); + pci_addr = ALIGN_DOWN(pci_addr, window_size); + size = window_size; + } + + if (!IS_ALIGNED(axi_addr, window_size) || + !IS_ALIGNED(pci_addr, window_size)) { + dev_err(dev, + "axi %pap or pci %pap not aligned\n", + &axi_addr, &pci_addr); + return -EINVAL; + } + + /* + * Match found! Program both OARR and OMAP and mark + * them as a valid entry. + */ + ret = iproc_pcie_ob_write(pcie, window_idx, size_idx, + axi_addr, pci_addr); + if (ret) + goto err_ob; + + size -= window_size; + if (size == 0) + return 0; + + /* + * If we are here, we are done with the current window, + * but not yet finished all mappings. Need to move on + * to the next window. + */ + axi_addr += window_size; + pci_addr += window_size; + break; + } + } + +err_ob: + dev_err(dev, "unable to configure outbound mapping\n"); + dev_err(dev, + "axi %pap, axi offset %pap, pci %pap, res size %pap\n", + &axi_addr, &ob->axi_offset, &pci_addr, &size); + + return ret; +} + +static int iproc_pcie_map_ranges(struct iproc_pcie *pcie, + struct list_head *resources) +{ + struct device *dev = pcie->dev; + struct resource_entry *window; + int ret; + + resource_list_for_each_entry(window, resources) { + struct resource *res = window->res; + u64 res_type = resource_type(res); + + switch (res_type) { + case IORESOURCE_IO: + case IORESOURCE_BUS: + break; + case IORESOURCE_MEM: + ret = iproc_pcie_setup_ob(pcie, res->start, + res->start - window->offset, + resource_size(res)); + if (ret) + return ret; + break; + default: + dev_err(dev, "invalid resource %pR\n", res); + return -EINVAL; + } + } + + return 0; +} + +static inline bool iproc_pcie_ib_is_in_use(struct iproc_pcie *pcie, + int region_idx) +{ + const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx]; + u32 val; + + val = iproc_pcie_read_reg(pcie, MAP_REG(IPROC_PCIE_IARR0, region_idx)); + + return !!(val & (BIT(ib_map->nr_sizes) - 1)); +} + +static inline bool iproc_pcie_ib_check_type(const struct iproc_pcie_ib_map *ib_map, + enum iproc_pcie_ib_map_type type) +{ + return !!(ib_map->type == type); +} + +static int iproc_pcie_ib_write(struct iproc_pcie *pcie, int region_idx, + int size_idx, int nr_windows, u64 axi_addr, + u64 pci_addr, resource_size_t size) +{ + struct device *dev = pcie->dev; + const struct iproc_pcie_ib_map *ib_map = &pcie->ib_map[region_idx]; + u16 iarr_offset, imap_offset; + u32 val; + int window_idx; + + iarr_offset = iproc_pcie_reg_offset(pcie, + MAP_REG(IPROC_PCIE_IARR0, region_idx)); + imap_offset = iproc_pcie_reg_offset(pcie, + MAP_REG(IPROC_PCIE_IMAP0, region_idx)); + if (iproc_pcie_reg_is_invalid(iarr_offset) || + iproc_pcie_reg_is_invalid(imap_offset)) + return -EINVAL; + + dev_dbg(dev, "ib region [%d]: offset 0x%x axi %pap pci %pap\n", + region_idx, iarr_offset, &axi_addr, &pci_addr); + + /* + * Program the IARR registers. The upper 32-bit IARR register is + * always right after the lower 32-bit IARR register. + */ + writel(lower_32_bits(pci_addr) | BIT(size_idx), + pcie->base + iarr_offset); + writel(upper_32_bits(pci_addr), pcie->base + iarr_offset + 4); + + dev_dbg(dev, "iarr lo 0x%x iarr hi 0x%x\n", + readl(pcie->base + iarr_offset), + readl(pcie->base + iarr_offset + 4)); + + /* + * Now program the IMAP registers. Each IARR region may have one or + * more IMAP windows. + */ + size >>= ilog2(nr_windows); + for (window_idx = 0; window_idx < nr_windows; window_idx++) { + val = readl(pcie->base + imap_offset); + val |= lower_32_bits(axi_addr) | IMAP_VALID; + writel(val, pcie->base + imap_offset); + writel(upper_32_bits(axi_addr), + pcie->base + imap_offset + ib_map->imap_addr_offset); + + dev_dbg(dev, "imap window [%d] lo 0x%x hi 0x%x\n", + window_idx, readl(pcie->base + imap_offset), + readl(pcie->base + imap_offset + + ib_map->imap_addr_offset)); + + imap_offset += ib_map->imap_window_offset; + axi_addr += size; + } + + return 0; +} + +static int iproc_pcie_setup_ib(struct iproc_pcie *pcie, + struct resource_entry *entry, + enum iproc_pcie_ib_map_type type) +{ + struct device *dev = pcie->dev; + struct iproc_pcie_ib *ib = &pcie->ib; + int ret; + unsigned int region_idx, size_idx; + u64 axi_addr = entry->res->start; + u64 pci_addr = entry->res->start - entry->offset; + resource_size_t size = resource_size(entry->res); + + /* iterate through all IARR mapping regions */ + for (region_idx = 0; region_idx < ib->nr_regions; region_idx++) { + const struct iproc_pcie_ib_map *ib_map = + &pcie->ib_map[region_idx]; + + /* + * If current inbound region is already in use or not a + * compatible type, move on to the next. + */ + if (iproc_pcie_ib_is_in_use(pcie, region_idx) || + !iproc_pcie_ib_check_type(ib_map, type)) + continue; + + /* iterate through all supported region sizes to find a match */ + for (size_idx = 0; size_idx < ib_map->nr_sizes; size_idx++) { + resource_size_t region_size = + ib_map->region_sizes[size_idx] * ib_map->size_unit; + + if (size != region_size) + continue; + + if (!IS_ALIGNED(axi_addr, region_size) || + !IS_ALIGNED(pci_addr, region_size)) { + dev_err(dev, + "axi %pap or pci %pap not aligned\n", + &axi_addr, &pci_addr); + return -EINVAL; + } + + /* Match found! Program IARR and all IMAP windows. */ + ret = iproc_pcie_ib_write(pcie, region_idx, size_idx, + ib_map->nr_windows, axi_addr, + pci_addr, size); + if (ret) + goto err_ib; + else + return 0; + + } + } + ret = -EINVAL; + +err_ib: + dev_err(dev, "unable to configure inbound mapping\n"); + dev_err(dev, "axi %pap, pci %pap, res size %pap\n", + &axi_addr, &pci_addr, &size); + + return ret; +} + +static int iproc_pcie_map_dma_ranges(struct iproc_pcie *pcie) +{ + struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); + struct resource_entry *entry; + int ret = 0; + + resource_list_for_each_entry(entry, &host->dma_ranges) { + /* Each range entry corresponds to an inbound mapping region */ + ret = iproc_pcie_setup_ib(pcie, entry, IPROC_PCIE_IB_MAP_MEM); + if (ret) + break; + } + + return ret; +} + +static void iproc_pcie_invalidate_mapping(struct iproc_pcie *pcie) +{ + struct iproc_pcie_ib *ib = &pcie->ib; + struct iproc_pcie_ob *ob = &pcie->ob; + int idx; + + if (pcie->ep_is_internal) + return; + + if (pcie->need_ob_cfg) { + /* iterate through all OARR mapping regions */ + for (idx = ob->nr_windows - 1; idx >= 0; idx--) { + iproc_pcie_write_reg(pcie, + MAP_REG(IPROC_PCIE_OARR0, idx), 0); + } + } + + if (pcie->need_ib_cfg) { + /* iterate through all IARR mapping regions */ + for (idx = 0; idx < ib->nr_regions; idx++) { + iproc_pcie_write_reg(pcie, + MAP_REG(IPROC_PCIE_IARR0, idx), 0); + } + } +} + +static int iproce_pcie_get_msi(struct iproc_pcie *pcie, + struct device_node *msi_node, + u64 *msi_addr) +{ + struct device *dev = pcie->dev; + int ret; + struct resource res; + + /* + * Check if 'msi-map' points to ARM GICv3 ITS, which is the only + * supported external MSI controller that requires steering. + */ + if (!of_device_is_compatible(msi_node, "arm,gic-v3-its")) { + dev_err(dev, "unable to find compatible MSI controller\n"); + return -ENODEV; + } + + /* derive GITS_TRANSLATER address from GICv3 */ + ret = of_address_to_resource(msi_node, 0, &res); + if (ret < 0) { + dev_err(dev, "unable to obtain MSI controller resources\n"); + return ret; + } + + *msi_addr = res.start + GITS_TRANSLATER; + return 0; +} + +static int iproc_pcie_paxb_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr) +{ + int ret; + struct resource_entry entry; + + memset(&entry, 0, sizeof(entry)); + entry.res = &entry.__res; + + msi_addr &= ~(SZ_32K - 1); + entry.res->start = msi_addr; + entry.res->end = msi_addr + SZ_32K - 1; + + ret = iproc_pcie_setup_ib(pcie, &entry, IPROC_PCIE_IB_MAP_IO); + return ret; +} + +static void iproc_pcie_paxc_v2_msi_steer(struct iproc_pcie *pcie, u64 msi_addr, + bool enable) +{ + u32 val; + + if (!enable) { + /* + * Disable PAXC MSI steering. All write transfers will be + * treated as non-MSI transfers + */ + val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_EN_CFG); + val &= ~MSI_ENABLE_CFG; + iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_EN_CFG, val); + return; + } + + /* + * Program bits [43:13] of address of GITS_TRANSLATER register into + * bits [30:0] of the MSI base address register. In fact, in all iProc + * based SoCs, all I/O register bases are well below the 32-bit + * boundary, so we can safely assume bits [43:32] are always zeros. + */ + iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_BASE_ADDR, + (u32)(msi_addr >> 13)); + + /* use a default 8K window size */ + iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_WINDOW_SIZE, 0); + + /* steering MSI to GICv3 ITS */ + val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_GIC_MODE); + val |= GIC_V3_CFG; + iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_GIC_MODE, val); + + /* + * Program bits [43:2] of address of GITS_TRANSLATER register into the + * iProc MSI address registers. + */ + msi_addr >>= 2; + iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_HI, + upper_32_bits(msi_addr)); + iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_ADDR_LO, + lower_32_bits(msi_addr)); + + /* enable MSI */ + val = iproc_pcie_read_reg(pcie, IPROC_PCIE_MSI_EN_CFG); + val |= MSI_ENABLE_CFG; + iproc_pcie_write_reg(pcie, IPROC_PCIE_MSI_EN_CFG, val); +} + +static int iproc_pcie_msi_steer(struct iproc_pcie *pcie, + struct device_node *msi_node) +{ + struct device *dev = pcie->dev; + int ret; + u64 msi_addr; + + ret = iproce_pcie_get_msi(pcie, msi_node, &msi_addr); + if (ret < 0) { + dev_err(dev, "msi steering failed\n"); + return ret; + } + + switch (pcie->type) { + case IPROC_PCIE_PAXB_V2: + ret = iproc_pcie_paxb_v2_msi_steer(pcie, msi_addr); + if (ret) + return ret; + break; + case IPROC_PCIE_PAXC_V2: + iproc_pcie_paxc_v2_msi_steer(pcie, msi_addr, true); + break; + default: + return -EINVAL; + } + + return 0; +} + +static int iproc_pcie_msi_enable(struct iproc_pcie *pcie) +{ + struct device_node *msi_node = NULL; + int ret; + + /* + * Either the "msi-parent" or the "msi-map" phandle needs to exist + * for us to obtain the MSI node. + */ + of_msi_xlate(pcie->dev, &msi_node, 0); + if (!msi_node) + return -ENODEV; + + /* + * Certain revisions of the iProc PCIe controller require additional + * configurations to steer the MSI writes towards an external MSI + * controller. + */ + if (pcie->need_msi_steer) { + ret = iproc_pcie_msi_steer(pcie, msi_node); + if (ret) + goto out_put_node; + } + + /* + * If another MSI controller is being used, the call below should fail + * but that is okay + */ + ret = iproc_msi_init(pcie, msi_node); + +out_put_node: + of_node_put(msi_node); + return ret; +} + +static void iproc_pcie_msi_disable(struct iproc_pcie *pcie) +{ + iproc_msi_exit(pcie); +} + +static int iproc_pcie_rev_init(struct iproc_pcie *pcie) +{ + struct device *dev = pcie->dev; + unsigned int reg_idx; + const u16 *regs; + + switch (pcie->type) { + case IPROC_PCIE_PAXB_BCMA: + regs = iproc_pcie_reg_paxb_bcma; + break; + case IPROC_PCIE_PAXB: + regs = iproc_pcie_reg_paxb; + pcie->has_apb_err_disable = true; + if (pcie->need_ob_cfg) { + pcie->ob_map = paxb_ob_map; + pcie->ob.nr_windows = ARRAY_SIZE(paxb_ob_map); + } + break; + case IPROC_PCIE_PAXB_V2: + regs = iproc_pcie_reg_paxb_v2; + pcie->iproc_cfg_read = true; + pcie->has_apb_err_disable = true; + if (pcie->need_ob_cfg) { + pcie->ob_map = paxb_v2_ob_map; + pcie->ob.nr_windows = ARRAY_SIZE(paxb_v2_ob_map); + } + pcie->ib.nr_regions = ARRAY_SIZE(paxb_v2_ib_map); + pcie->ib_map = paxb_v2_ib_map; + pcie->need_msi_steer = true; + dev_warn(dev, "reads of config registers that contain %#x return incorrect data\n", + CFG_RETRY_STATUS); + break; + case IPROC_PCIE_PAXC: + regs = iproc_pcie_reg_paxc; + pcie->ep_is_internal = true; + pcie->iproc_cfg_read = true; + pcie->rej_unconfig_pf = true; + break; + case IPROC_PCIE_PAXC_V2: + regs = iproc_pcie_reg_paxc_v2; + pcie->ep_is_internal = true; + pcie->iproc_cfg_read = true; + pcie->rej_unconfig_pf = true; + pcie->need_msi_steer = true; + break; + default: + dev_err(dev, "incompatible iProc PCIe interface\n"); + return -EINVAL; + } + + pcie->reg_offsets = devm_kcalloc(dev, IPROC_PCIE_MAX_NUM_REG, + sizeof(*pcie->reg_offsets), + GFP_KERNEL); + if (!pcie->reg_offsets) + return -ENOMEM; + + /* go through the register table and populate all valid registers */ + pcie->reg_offsets[0] = (pcie->type == IPROC_PCIE_PAXC_V2) ? + IPROC_PCIE_REG_INVALID : regs[0]; + for (reg_idx = 1; reg_idx < IPROC_PCIE_MAX_NUM_REG; reg_idx++) + pcie->reg_offsets[reg_idx] = regs[reg_idx] ? + regs[reg_idx] : IPROC_PCIE_REG_INVALID; + + return 0; +} + +int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res) +{ + struct device *dev; + int ret; + struct pci_dev *pdev; + struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); + + dev = pcie->dev; + + ret = iproc_pcie_rev_init(pcie); + if (ret) { + dev_err(dev, "unable to initialize controller parameters\n"); + return ret; + } + + ret = phy_init(pcie->phy); + if (ret) { + dev_err(dev, "unable to initialize PCIe PHY\n"); + return ret; + } + + ret = phy_power_on(pcie->phy); + if (ret) { + dev_err(dev, "unable to power on PCIe PHY\n"); + goto err_exit_phy; + } + + iproc_pcie_perst_ctrl(pcie, true); + iproc_pcie_perst_ctrl(pcie, false); + + iproc_pcie_invalidate_mapping(pcie); + + if (pcie->need_ob_cfg) { + ret = iproc_pcie_map_ranges(pcie, res); + if (ret) { + dev_err(dev, "map failed\n"); + goto err_power_off_phy; + } + } + + if (pcie->need_ib_cfg) { + ret = iproc_pcie_map_dma_ranges(pcie); + if (ret && ret != -ENOENT) + goto err_power_off_phy; + } + + ret = iproc_pcie_check_link(pcie); + if (ret) { + dev_err(dev, "no PCIe EP device detected\n"); + goto err_power_off_phy; + } + + iproc_pcie_enable(pcie); + + if (IS_ENABLED(CONFIG_PCI_MSI)) + if (iproc_pcie_msi_enable(pcie)) + dev_info(dev, "not using iProc MSI\n"); + + host->ops = &iproc_pcie_ops; + host->sysdata = pcie; + + ret = pci_host_probe(host); + if (ret < 0) { + dev_err(dev, "failed to scan host: %d\n", ret); + goto err_power_off_phy; + } + + for_each_pci_bridge(pdev, host->bus) { + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ROOT_PORT) + pcie_print_link_status(pdev); + } + + return 0; + +err_power_off_phy: + phy_power_off(pcie->phy); +err_exit_phy: + phy_exit(pcie->phy); + return ret; +} +EXPORT_SYMBOL(iproc_pcie_setup); + +void iproc_pcie_remove(struct iproc_pcie *pcie) +{ + struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); + + pci_lock_rescan_remove(); + pci_stop_root_bus(host->bus); + pci_remove_root_bus(host->bus); + pci_unlock_rescan_remove(); + + iproc_pcie_msi_disable(pcie); + + phy_power_off(pcie->phy); + phy_exit(pcie->phy); +} +EXPORT_SYMBOL(iproc_pcie_remove); + +/* + * The MSI parsing logic in certain revisions of Broadcom PAXC based root + * complex does not work and needs to be disabled + */ +static void quirk_paxc_disable_msi_parsing(struct pci_dev *pdev) +{ + struct iproc_pcie *pcie = iproc_data(pdev->bus); + + if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) + iproc_pcie_paxc_v2_msi_steer(pcie, 0, false); +} +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16f0, + quirk_paxc_disable_msi_parsing); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd802, + quirk_paxc_disable_msi_parsing); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd804, + quirk_paxc_disable_msi_parsing); + +static void quirk_paxc_bridge(struct pci_dev *pdev) +{ + /* + * The PCI config space is shared with the PAXC root port and the first + * Ethernet device. So, we need to workaround this by telling the PCI + * code that the bridge is not an Ethernet device. + */ + if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) + pdev->class = PCI_CLASS_BRIDGE_PCI_NORMAL; + + /* + * MPSS is not being set properly (as it is currently 0). This is + * because that area of the PCI config space is hard coded to zero, and + * is not modifiable by firmware. Set this to 2 (e.g., 512 byte MPS) + * so that the MPS can be set to the real max value. + */ + pdev->pcie_mpss = 2; +} +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16cd, quirk_paxc_bridge); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0x16f0, quirk_paxc_bridge); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd750, quirk_paxc_bridge); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd802, quirk_paxc_bridge); +DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_BROADCOM, 0xd804, quirk_paxc_bridge); + +MODULE_AUTHOR("Ray Jui <rjui@broadcom.com>"); +MODULE_DESCRIPTION("Broadcom iPROC PCIe common driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pcie-iproc.h b/drivers/pci/controller/pcie-iproc.h new file mode 100644 index 000000000..c4443f236 --- /dev/null +++ b/drivers/pci/controller/pcie-iproc.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2014-2015 Broadcom Corporation + */ + +#ifndef _PCIE_IPROC_H +#define _PCIE_IPROC_H + +/** + * enum iproc_pcie_type - iProc PCIe interface type + * @IPROC_PCIE_PAXB_BCMA: BCMA-based host controllers + * @IPROC_PCIE_PAXB: PAXB-based host controllers for + * NS, NSP, Cygnus, NS2, and Pegasus SOCs + * @IPROC_PCIE_PAXB_V2: PAXB-based host controllers for Stingray SoCs + * @IPROC_PCIE_PAXC: PAXC-based host controllers + * @IPROC_PCIE_PAXC_V2: PAXC-based host controllers (second generation) + * + * PAXB is the wrapper used in root complex that can be connected to an + * external endpoint device. + * + * PAXC is the wrapper used in root complex dedicated for internal emulated + * endpoint devices. + */ +enum iproc_pcie_type { + IPROC_PCIE_PAXB_BCMA = 0, + IPROC_PCIE_PAXB, + IPROC_PCIE_PAXB_V2, + IPROC_PCIE_PAXC, + IPROC_PCIE_PAXC_V2, +}; + +/** + * struct iproc_pcie_ob - iProc PCIe outbound mapping + * @axi_offset: offset from the AXI address to the internal address used by + * the iProc PCIe core + * @nr_windows: total number of supported outbound mapping windows + */ +struct iproc_pcie_ob { + resource_size_t axi_offset; + unsigned int nr_windows; +}; + +/** + * struct iproc_pcie_ib - iProc PCIe inbound mapping + * @nr_regions: total number of supported inbound mapping regions + */ +struct iproc_pcie_ib { + unsigned int nr_regions; +}; + +struct iproc_pcie_ob_map; +struct iproc_pcie_ib_map; +struct iproc_msi; + +/** + * struct iproc_pcie - iProc PCIe device + * @dev: pointer to device data structure + * @type: iProc PCIe interface type + * @reg_offsets: register offsets + * @base: PCIe host controller I/O register base + * @base_addr: PCIe host controller register base physical address + * @mem: host bridge memory window resource + * @phy: optional PHY device that controls the Serdes + * @ep_is_internal: indicates an internal emulated endpoint device is connected + * @iproc_cfg_read: indicates the iProc config read function should be used + * @rej_unconfig_pf: indicates the root complex needs to detect and reject + * enumeration against unconfigured physical functions emulated in the ASIC + * @has_apb_err_disable: indicates the controller can be configured to prevent + * unsupported request from being forwarded as an APB bus error + * @fix_paxc_cap: indicates the controller has corrupted capability list in its + * config space registers and requires SW based fixup + * + * @need_ob_cfg: indicates SW needs to configure the outbound mapping window + * @ob: outbound mapping related parameters + * @ob_map: outbound mapping related parameters specific to the controller + * + * @need_ib_cfg: indicates SW needs to configure the inbound mapping window + * @ib: inbound mapping related parameters + * @ib_map: outbound mapping region related parameters + * + * @need_msi_steer: indicates additional configuration of the iProc PCIe + * controller is required to steer MSI writes to external interrupt controller + * @msi: MSI data + */ +struct iproc_pcie { + struct device *dev; + enum iproc_pcie_type type; + u16 *reg_offsets; + void __iomem *base; + phys_addr_t base_addr; + struct resource mem; + struct phy *phy; + bool ep_is_internal; + bool iproc_cfg_read; + bool rej_unconfig_pf; + bool has_apb_err_disable; + bool fix_paxc_cap; + + bool need_ob_cfg; + struct iproc_pcie_ob ob; + const struct iproc_pcie_ob_map *ob_map; + + bool need_ib_cfg; + struct iproc_pcie_ib ib; + const struct iproc_pcie_ib_map *ib_map; + + bool need_msi_steer; + struct iproc_msi *msi; +}; + +int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res); +void iproc_pcie_remove(struct iproc_pcie *pcie); +int iproc_pcie_shutdown(struct iproc_pcie *pcie); + +#ifdef CONFIG_PCIE_IPROC_MSI +int iproc_msi_init(struct iproc_pcie *pcie, struct device_node *node); +void iproc_msi_exit(struct iproc_pcie *pcie); +#else +static inline int iproc_msi_init(struct iproc_pcie *pcie, + struct device_node *node) +{ + return -ENODEV; +} +static inline void iproc_msi_exit(struct iproc_pcie *pcie) +{ +} +#endif + +#endif /* _PCIE_IPROC_H */ diff --git a/drivers/pci/controller/pcie-mediatek-gen3.c b/drivers/pci/controller/pcie-mediatek-gen3.c new file mode 100644 index 000000000..bcbd08c82 --- /dev/null +++ b/drivers/pci/controller/pcie-mediatek-gen3.c @@ -0,0 +1,1448 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * MediaTek PCIe host controller driver. + * + * Copyright (c) 2020 MediaTek Inc. + * Author: Jianjun Wang <jianjun.wang@mediatek.com> + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/clk-provider.h> +#include <linux/delay.h> +#include <linux/iopoll.h> +#include <linux/irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqdomain.h> +#include <linux/kernel.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/of_device.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/pci-pwrctrl.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/pm_domain.h> +#include <linux/pm_runtime.h> +#include <linux/regmap.h> +#include <linux/reset.h> + +#include "pci-host-common.h" +#include "../pci.h" + +#define PCIE_BASE_CFG_REG 0x14 +#define PCIE_BASE_CFG_SPEED GENMASK(15, 8) + +#define PCIE_SETTING_REG 0x80 +#define PCIE_SETTING_LINK_WIDTH GENMASK(11, 8) +#define PCIE_SETTING_GEN_SUPPORT GENMASK(14, 12) +#define PCIE_PCI_IDS_1 0x9c +#define PCI_CLASS(class) (class << 8) +#define PCIE_RC_MODE BIT(0) + +#define PCIE_EQ_PRESET_01_REG 0x100 +#define PCIE_VAL_LN0_DOWNSTREAM GENMASK(6, 0) +#define PCIE_VAL_LN0_UPSTREAM GENMASK(14, 8) +#define PCIE_VAL_LN1_DOWNSTREAM GENMASK(22, 16) +#define PCIE_VAL_LN1_UPSTREAM GENMASK(30, 24) + +#define PCIE_CFGNUM_REG 0x140 +#define PCIE_CFG_DEVFN(devfn) ((devfn) & GENMASK(7, 0)) +#define PCIE_CFG_BUS(bus) (((bus) << 8) & GENMASK(15, 8)) +#define PCIE_CFG_BYTE_EN(bytes) (((bytes) << 16) & GENMASK(19, 16)) +#define PCIE_CFG_FORCE_BYTE_EN BIT(20) +#define PCIE_CFG_OFFSET_ADDR 0x1000 +#define PCIE_CFG_HEADER(bus, devfn) \ + (PCIE_CFG_BUS(bus) | PCIE_CFG_DEVFN(devfn)) + +#define PCIE_RST_CTRL_REG 0x148 +#define PCIE_MAC_RSTB BIT(0) +#define PCIE_PHY_RSTB BIT(1) +#define PCIE_BRG_RSTB BIT(2) +#define PCIE_PE_RSTB BIT(3) + +/* + * As described in the datasheet of MediaTek PCIe Gen3 controller, wait 10ms + * after setting PCIE_BRG_RSTB, and before accessing PCIe internal registers. + */ +#define PCIE_BRG_RST_RDY_MS 10 + +#define PCIE_LTSSM_STATUS_REG 0x150 +#define PCIE_LTSSM_STATE_MASK GENMASK(28, 24) +#define PCIE_LTSSM_STATE(val) ((val & PCIE_LTSSM_STATE_MASK) >> 24) +#define PCIE_LTSSM_STATE_L2_IDLE 0x14 + +#define PCIE_LINK_STATUS_REG 0x154 +#define PCIE_PORT_LINKUP BIT(8) + +#define PCIE_MSI_SET_NUM 8 +#define PCIE_MSI_IRQS_PER_SET 32 +#define PCIE_MSI_IRQS_NUM \ + (PCIE_MSI_IRQS_PER_SET * PCIE_MSI_SET_NUM) + +#define PCIE_INT_ENABLE_REG 0x180 +#define PCIE_MSI_ENABLE GENMASK(PCIE_MSI_SET_NUM + 8 - 1, 8) +#define PCIE_MSI_SHIFT 8 +#define PCIE_INTX_SHIFT 24 +#define PCIE_INTX_ENABLE \ + GENMASK(PCIE_INTX_SHIFT + PCI_NUM_INTX - 1, PCIE_INTX_SHIFT) + +#define PCIE_INT_STATUS_REG 0x184 +#define PCIE_MSI_SET_ENABLE_REG 0x190 +#define PCIE_MSI_SET_ENABLE GENMASK(PCIE_MSI_SET_NUM - 1, 0) + +#define PCIE_PIPE4_PIE8_REG 0x338 +#define PCIE_K_FINETUNE_MAX GENMASK(5, 0) +#define PCIE_K_FINETUNE_ERR GENMASK(7, 6) +#define PCIE_K_PRESET_TO_USE GENMASK(18, 8) +#define PCIE_K_PHYPARAM_QUERY BIT(19) +#define PCIE_K_QUERY_TIMEOUT BIT(20) +#define PCIE_K_PRESET_TO_USE_16G GENMASK(31, 21) + +#define PCIE_MSI_SET_BASE_REG 0xc00 +#define PCIE_MSI_SET_OFFSET 0x10 +#define PCIE_MSI_SET_STATUS_OFFSET 0x04 +#define PCIE_MSI_SET_ENABLE_OFFSET 0x08 + +#define PCIE_MSI_SET_ADDR_HI_BASE 0xc80 +#define PCIE_MSI_SET_ADDR_HI_OFFSET 0x04 + +#define PCIE_RESOURCE_CTRL_REG 0xd2c +#define PCIE_RSRC_SYS_CLK_RDY_TIME_MASK GENMASK(7, 0) + +#define PCIE_ICMD_PM_REG 0x198 +#define PCIE_TURN_OFF_LINK BIT(4) + +#define PCIE_MISC_CTRL_REG 0x348 +#define PCIE_DISABLE_DVFSRC_VLT_REQ BIT(1) + +#define PCIE_TRANS_TABLE_BASE_REG 0x800 +#define PCIE_ATR_SRC_ADDR_MSB_OFFSET 0x4 +#define PCIE_ATR_TRSL_ADDR_LSB_OFFSET 0x8 +#define PCIE_ATR_TRSL_ADDR_MSB_OFFSET 0xc +#define PCIE_ATR_TRSL_PARAM_OFFSET 0x10 +#define PCIE_ATR_TLB_SET_OFFSET 0x20 + +#define PCIE_MAX_TRANS_TABLES 8 +#define PCIE_ATR_EN BIT(0) +#define PCIE_ATR_SIZE(size) \ + (((((size) - 1) << 1) & GENMASK(6, 1)) | PCIE_ATR_EN) +#define PCIE_ATR_ID(id) ((id) & GENMASK(3, 0)) +#define PCIE_ATR_TYPE_MEM PCIE_ATR_ID(0) +#define PCIE_ATR_TYPE_IO PCIE_ATR_ID(1) +#define PCIE_ATR_TLP_TYPE(type) (((type) << 16) & GENMASK(18, 16)) +#define PCIE_ATR_TLP_TYPE_MEM PCIE_ATR_TLP_TYPE(0) +#define PCIE_ATR_TLP_TYPE_IO PCIE_ATR_TLP_TYPE(2) + +#define MAX_NUM_PHY_RESETS 3 + +#define PCIE_MTK_RESET_TIME_US 10 + +/* Time in ms needed to complete PCIe reset on EN7581 SoC */ +#define PCIE_EN7581_RESET_TIME_MS 100 + +struct mtk_gen3_pcie; + +#define PCIE_CONF_LINK2_CTL_STS (PCIE_CFG_OFFSET_ADDR + 0xb0) +#define PCIE_CONF_LINK2_LCR2_LINK_SPEED GENMASK(3, 0) + +enum mtk_gen3_pcie_flags { + SKIP_PCIE_RSTB = BIT(0), /* Skip PERST# assertion during device + * probing or suspend/resume phase to + * avoid hw bugs/issues. + */ +}; + +/** + * struct mtk_gen3_pcie_pdata - differentiate between host generations + * @power_up: pcie power_up callback + * @phy_resets: phy reset lines SoC data. + * @sys_clk_rdy_time_us: System clock ready time override (microseconds) + * @flags: pcie device flags. + */ +struct mtk_gen3_pcie_pdata { + int (*power_up)(struct mtk_gen3_pcie *pcie); + struct { + const char *id[MAX_NUM_PHY_RESETS]; + int num_resets; + } phy_resets; + u8 sys_clk_rdy_time_us; + u32 flags; +}; + +/** + * struct mtk_msi_set - MSI information for each set + * @base: IO mapped register base + * @msg_addr: MSI message address + * @saved_irq_state: IRQ enable state saved at suspend time + */ +struct mtk_msi_set { + void __iomem *base; + phys_addr_t msg_addr; + u32 saved_irq_state; +}; + +/** + * struct mtk_gen3_pcie - PCIe port information + * @dev: pointer to PCIe device + * @base: IO mapped register base + * @reg_base: physical register base + * @mac_reset: MAC reset control + * @phy_resets: PHY reset controllers + * @phy: PHY controller block + * @clks: PCIe clocks + * @num_clks: PCIe clocks count for this port + * @max_link_speed: Maximum link speed (PCIe Gen) for this port + * @num_lanes: Number of PCIe lanes for this port + * @irq: PCIe controller interrupt number + * @saved_irq_state: IRQ enable state saved at suspend time + * @irq_lock: lock protecting IRQ register access + * @intx_domain: legacy INTx IRQ domain + * @msi_bottom_domain: MSI IRQ bottom domain + * @msi_sets: MSI sets information + * @lock: lock protecting IRQ bit map + * @msi_irq_in_use: bit map for assigned MSI IRQ + * @soc: pointer to SoC-dependent operations + */ +struct mtk_gen3_pcie { + struct device *dev; + void __iomem *base; + phys_addr_t reg_base; + struct reset_control *mac_reset; + struct reset_control_bulk_data phy_resets[MAX_NUM_PHY_RESETS]; + struct phy *phy; + struct clk_bulk_data *clks; + int num_clks; + u8 max_link_speed; + u8 num_lanes; + + int irq; + u32 saved_irq_state; + raw_spinlock_t irq_lock; + struct irq_domain *intx_domain; + struct irq_domain *msi_bottom_domain; + struct mtk_msi_set msi_sets[PCIE_MSI_SET_NUM]; + struct mutex lock; + DECLARE_BITMAP(msi_irq_in_use, PCIE_MSI_IRQS_NUM); + + const struct mtk_gen3_pcie_pdata *soc; +}; + +/* LTSSM state in PCIE_LTSSM_STATUS_REG bit[28:24] */ +static const char *const ltssm_str[] = { + "detect.quiet", /* 0x00 */ + "detect.active", /* 0x01 */ + "polling.active", /* 0x02 */ + "polling.compliance", /* 0x03 */ + "polling.configuration", /* 0x04 */ + "config.linkwidthstart", /* 0x05 */ + "config.linkwidthaccept", /* 0x06 */ + "config.lanenumwait", /* 0x07 */ + "config.lanenumaccept", /* 0x08 */ + "config.complete", /* 0x09 */ + "config.idle", /* 0x0A */ + "recovery.receiverlock", /* 0x0B */ + "recovery.equalization", /* 0x0C */ + "recovery.speed", /* 0x0D */ + "recovery.receiverconfig", /* 0x0E */ + "recovery.idle", /* 0x0F */ + "L0", /* 0x10 */ + "L0s", /* 0x11 */ + "L1.entry", /* 0x12 */ + "L1.idle", /* 0x13 */ + "L2.idle", /* 0x14 */ + "L2.transmitwake", /* 0x15 */ + "disable", /* 0x16 */ + "loopback.entry", /* 0x17 */ + "loopback.active", /* 0x18 */ + "loopback.exit", /* 0x19 */ + "hotreset", /* 0x1A */ +}; + +/** + * mtk_pcie_config_tlp_header() - Configure a configuration TLP header + * @bus: PCI bus to query + * @devfn: device/function number + * @where: offset in config space + * @size: data size in TLP header + * + * Set byte enable field and device information in configuration TLP header. + */ +static void mtk_pcie_config_tlp_header(struct pci_bus *bus, unsigned int devfn, + int where, int size) +{ + struct mtk_gen3_pcie *pcie = bus->sysdata; + int bytes; + u32 val; + + bytes = (GENMASK(size - 1, 0) & 0xf) << (where & 0x3); + + val = PCIE_CFG_FORCE_BYTE_EN | PCIE_CFG_BYTE_EN(bytes) | + PCIE_CFG_HEADER(bus->number, devfn); + + writel_relaxed(val, pcie->base + PCIE_CFGNUM_REG); +} + +static void __iomem *mtk_pcie_map_bus(struct pci_bus *bus, unsigned int devfn, + int where) +{ + struct mtk_gen3_pcie *pcie = bus->sysdata; + + return pcie->base + PCIE_CFG_OFFSET_ADDR + where; +} + +static int mtk_pcie_config_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + mtk_pcie_config_tlp_header(bus, devfn, where, size); + + return pci_generic_config_read32(bus, devfn, where, size, val); +} + +static int mtk_pcie_config_write(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + mtk_pcie_config_tlp_header(bus, devfn, where, size); + + if (size <= 2) + val <<= (where & 0x3) * 8; + + return pci_generic_config_write32(bus, devfn, where, 4, val); +} + +static struct pci_ops mtk_pcie_ops = { + .map_bus = mtk_pcie_map_bus, + .read = mtk_pcie_config_read, + .write = mtk_pcie_config_write, +}; + +static int mtk_pcie_set_trans_table(struct mtk_gen3_pcie *pcie, + resource_size_t cpu_addr, + resource_size_t pci_addr, + resource_size_t size, + unsigned long type, int *num) +{ + resource_size_t remaining = size; + resource_size_t table_size; + resource_size_t addr_align; + const char *range_type; + void __iomem *table; + u32 val; + + while (remaining && (*num < PCIE_MAX_TRANS_TABLES)) { + /* Table size needs to be a power of 2 */ + table_size = BIT(fls(remaining) - 1); + + if (cpu_addr > 0) { + addr_align = BIT(ffs(cpu_addr) - 1); + table_size = min(table_size, addr_align); + } + + /* Minimum size of translate table is 4KiB */ + if (table_size < 0x1000) { + dev_err(pcie->dev, "illegal table size %#llx\n", + (unsigned long long)table_size); + return -EINVAL; + } + + table = pcie->base + PCIE_TRANS_TABLE_BASE_REG + *num * PCIE_ATR_TLB_SET_OFFSET; + writel_relaxed(lower_32_bits(cpu_addr) | PCIE_ATR_SIZE(fls(table_size) - 1), table); + writel_relaxed(upper_32_bits(cpu_addr), table + PCIE_ATR_SRC_ADDR_MSB_OFFSET); + writel_relaxed(lower_32_bits(pci_addr), table + PCIE_ATR_TRSL_ADDR_LSB_OFFSET); + writel_relaxed(upper_32_bits(pci_addr), table + PCIE_ATR_TRSL_ADDR_MSB_OFFSET); + + if (type == IORESOURCE_IO) { + val = PCIE_ATR_TYPE_IO | PCIE_ATR_TLP_TYPE_IO; + range_type = "IO"; + } else { + val = PCIE_ATR_TYPE_MEM | PCIE_ATR_TLP_TYPE_MEM; + range_type = "MEM"; + } + + writel_relaxed(val, table + PCIE_ATR_TRSL_PARAM_OFFSET); + + dev_dbg(pcie->dev, "set %s trans window[%d]: cpu_addr = %#llx, pci_addr = %#llx, size = %#llx\n", + range_type, *num, (unsigned long long)cpu_addr, + (unsigned long long)pci_addr, + (unsigned long long)table_size); + + cpu_addr += table_size; + pci_addr += table_size; + remaining -= table_size; + (*num)++; + } + + if (remaining) + dev_warn(pcie->dev, "not enough translate table for addr: %#llx, limited to [%d]\n", + (unsigned long long)cpu_addr, PCIE_MAX_TRANS_TABLES); + + return 0; +} + +static void mtk_pcie_enable_msi(struct mtk_gen3_pcie *pcie) +{ + int i; + u32 val; + + for (i = 0; i < PCIE_MSI_SET_NUM; i++) { + struct mtk_msi_set *msi_set = &pcie->msi_sets[i]; + + msi_set->base = pcie->base + PCIE_MSI_SET_BASE_REG + + i * PCIE_MSI_SET_OFFSET; + msi_set->msg_addr = pcie->reg_base + PCIE_MSI_SET_BASE_REG + + i * PCIE_MSI_SET_OFFSET; + + /* Configure the MSI capture address */ + writel_relaxed(lower_32_bits(msi_set->msg_addr), msi_set->base); + writel_relaxed(upper_32_bits(msi_set->msg_addr), + pcie->base + PCIE_MSI_SET_ADDR_HI_BASE + + i * PCIE_MSI_SET_ADDR_HI_OFFSET); + } + + val = readl_relaxed(pcie->base + PCIE_MSI_SET_ENABLE_REG); + val |= PCIE_MSI_SET_ENABLE; + writel_relaxed(val, pcie->base + PCIE_MSI_SET_ENABLE_REG); + + val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG); + val |= PCIE_MSI_ENABLE; + writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG); +} + +static int mtk_pcie_devices_power_up(struct mtk_gen3_pcie *pcie) +{ + int err; + u32 val; + + /* + * Airoha EN7581 has a hw bug asserting/releasing PCIE_PE_RSTB signal + * causing occasional PCIe link down. In order to overcome the issue, + * PCIE_RSTB signals are not asserted/released at this stage and the + * PCIe block is reset using en7523_reset_assert() and + * en7581_pci_enable(). + */ + if (!(pcie->soc->flags & SKIP_PCIE_RSTB)) { + /* Assert all reset signals */ + val = readl_relaxed(pcie->base + PCIE_RST_CTRL_REG); + val |= PCIE_MAC_RSTB | PCIE_PHY_RSTB | PCIE_BRG_RSTB | + PCIE_PE_RSTB; + writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG); + } + + err = pci_pwrctrl_power_on_devices(pcie->dev); + if (err) { + dev_err(pcie->dev, "Failed to power on devices: %pe\n", ERR_PTR(err)); + return err; + } + + /* + * Some of MediaTek's chips won't output REFCLK when PCIE_PHY_RSTB is + * asserted, we have to de-assert MAC & PHY & BRG reset signals first + * to allow the REFCLK to be stable. While PCIE_BRG_RSTB is asserted, + * there is a short period during which the PCIe internal register + * cannot be accessed, so we need to wait 10ms here. + */ + msleep(PCIE_BRG_RST_RDY_MS); + + if (!(pcie->soc->flags & SKIP_PCIE_RSTB)) { + /* De-assert MAC, PHY and BRG reset signals */ + val &= ~(PCIE_MAC_RSTB | PCIE_PHY_RSTB | PCIE_BRG_RSTB); + writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG); + } + + /* + * Described in PCIe CEM specification revision 6.0. + * + * The deassertion of PERST# should be delayed 100ms (TPVPERL) + * for the power and clock to become stable. + */ + msleep(PCIE_T_PVPERL_MS); + + if (!(pcie->soc->flags & SKIP_PCIE_RSTB)) { + /* De-assert PERST# signal */ + val &= ~PCIE_PE_RSTB; + writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG); + } + + return 0; +} + +static void mtk_pcie_devices_power_down(struct mtk_gen3_pcie *pcie) +{ + u32 val; + + if (!(pcie->soc->flags & SKIP_PCIE_RSTB)) { + /* Assert the PERST# pin */ + val = readl_relaxed(pcie->base + PCIE_RST_CTRL_REG); + val |= PCIE_PE_RSTB; + writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG); + } + + pci_pwrctrl_power_off_devices(pcie->dev); +} + +static int mtk_pcie_startup_port(struct mtk_gen3_pcie *pcie) +{ + struct resource_entry *entry; + struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); + unsigned int table_index = 0; + int err; + u32 val; + + /* Set as RC mode and set controller PCIe Gen speed restriction, if any */ + val = readl_relaxed(pcie->base + PCIE_SETTING_REG); + val |= PCIE_RC_MODE; + if (pcie->max_link_speed) { + val &= ~PCIE_SETTING_GEN_SUPPORT; + + /* Can enable link speed support only from Gen2 onwards */ + if (pcie->max_link_speed >= 2) + val |= FIELD_PREP(PCIE_SETTING_GEN_SUPPORT, + GENMASK(pcie->max_link_speed - 2, 0)); + } + if (pcie->num_lanes) { + val &= ~PCIE_SETTING_LINK_WIDTH; + + /* Zero means one lane, each bit activates x2/x4/x8/x16 */ + if (pcie->num_lanes > 1) + val |= FIELD_PREP(PCIE_SETTING_LINK_WIDTH, + GENMASK(fls(pcie->num_lanes >> 2), 0)); + } + writel_relaxed(val, pcie->base + PCIE_SETTING_REG); + + /* Set Link Control 2 (LNKCTL2) speed restriction, if any */ + if (pcie->max_link_speed) { + val = readl_relaxed(pcie->base + PCIE_CONF_LINK2_CTL_STS); + FIELD_MODIFY(PCIE_CONF_LINK2_LCR2_LINK_SPEED, &val, pcie->max_link_speed); + writel_relaxed(val, pcie->base + PCIE_CONF_LINK2_CTL_STS); + } + + /* If parameter is present, adjust SYS_CLK_RDY_TIME to avoid glitching */ + if (pcie->soc->sys_clk_rdy_time_us) { + val = readl_relaxed(pcie->base + PCIE_RESOURCE_CTRL_REG); + FIELD_MODIFY(PCIE_RSRC_SYS_CLK_RDY_TIME_MASK, &val, + pcie->soc->sys_clk_rdy_time_us); + writel_relaxed(val, pcie->base + PCIE_RESOURCE_CTRL_REG); + } + + /* Set class code */ + val = readl_relaxed(pcie->base + PCIE_PCI_IDS_1); + val &= ~GENMASK(31, 8); + val |= PCI_CLASS(PCI_CLASS_BRIDGE_PCI_NORMAL); + writel_relaxed(val, pcie->base + PCIE_PCI_IDS_1); + + /* Mask all INTx interrupts */ + val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG); + val &= ~PCIE_INTX_ENABLE; + writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG); + + /* Disable DVFSRC voltage request */ + val = readl_relaxed(pcie->base + PCIE_MISC_CTRL_REG); + val |= PCIE_DISABLE_DVFSRC_VLT_REQ; + writel_relaxed(val, pcie->base + PCIE_MISC_CTRL_REG); + + mtk_pcie_enable_msi(pcie); + + /* Set PCIe translation windows */ + resource_list_for_each_entry(entry, &host->windows) { + struct resource *res = entry->res; + unsigned long type = resource_type(res); + resource_size_t cpu_addr; + resource_size_t pci_addr; + resource_size_t size; + + if (type == IORESOURCE_IO) + cpu_addr = pci_pio_to_address(res->start); + else if (type == IORESOURCE_MEM) + cpu_addr = res->start; + else + continue; + + pci_addr = res->start - entry->offset; + size = resource_size(res); + err = mtk_pcie_set_trans_table(pcie, cpu_addr, pci_addr, size, + type, &table_index); + if (err) + return err; + } + + err = mtk_pcie_devices_power_up(pcie); + if (err) + return err; + + /* Check if the link is up or not */ + err = readl_poll_timeout(pcie->base + PCIE_LINK_STATUS_REG, val, + !!(val & PCIE_PORT_LINKUP), 20, + PCI_PM_D3COLD_WAIT * USEC_PER_MSEC); + if (err) { + const char *ltssm_state; + int ltssm_index; + + val = readl_relaxed(pcie->base + PCIE_LTSSM_STATUS_REG); + ltssm_index = PCIE_LTSSM_STATE(val); + ltssm_state = ltssm_index >= ARRAY_SIZE(ltssm_str) ? + "Unknown state" : ltssm_str[ltssm_index]; + dev_err(pcie->dev, + "PCIe link down, current LTSSM state: %s (%#x)\n", + ltssm_state, val); + goto err_power_down_device; + } + + pci_host_common_link_train_delay(pcie->max_link_speed); + + return 0; + +err_power_down_device: + mtk_pcie_devices_power_down(pcie); + return err; +} + +#define MTK_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_NO_AFFINITY | \ + MSI_FLAG_PCI_MSI_MASK_PARENT) + +#define MTK_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_PCI_MSIX | \ + MSI_FLAG_MULTI_PCI_MSI) + +static const struct msi_parent_ops mtk_msi_parent_ops = { + .required_flags = MTK_MSI_FLAGS_REQUIRED, + .supported_flags = MTK_MSI_FLAGS_SUPPORTED, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .chip_flags = MSI_CHIP_FLAG_SET_ACK, + .prefix = "MTK3-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +static void mtk_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct mtk_msi_set *msi_set = irq_data_get_irq_chip_data(data); + struct mtk_gen3_pcie *pcie = data->domain->host_data; + unsigned long hwirq; + + hwirq = data->hwirq % PCIE_MSI_IRQS_PER_SET; + + msg->address_hi = upper_32_bits(msi_set->msg_addr); + msg->address_lo = lower_32_bits(msi_set->msg_addr); + msg->data = hwirq; + dev_dbg(pcie->dev, "msi#%#lx address_hi %#x address_lo %#x data %d\n", + hwirq, msg->address_hi, msg->address_lo, msg->data); +} + +static void mtk_msi_bottom_irq_ack(struct irq_data *data) +{ + struct mtk_msi_set *msi_set = irq_data_get_irq_chip_data(data); + unsigned long hwirq; + + hwirq = data->hwirq % PCIE_MSI_IRQS_PER_SET; + + writel_relaxed(BIT(hwirq), msi_set->base + PCIE_MSI_SET_STATUS_OFFSET); +} + +static void mtk_msi_bottom_irq_mask(struct irq_data *data) +{ + struct mtk_msi_set *msi_set = irq_data_get_irq_chip_data(data); + struct mtk_gen3_pcie *pcie = data->domain->host_data; + unsigned long hwirq, flags; + u32 val; + + hwirq = data->hwirq % PCIE_MSI_IRQS_PER_SET; + + raw_spin_lock_irqsave(&pcie->irq_lock, flags); + val = readl_relaxed(msi_set->base + PCIE_MSI_SET_ENABLE_OFFSET); + val &= ~BIT(hwirq); + writel_relaxed(val, msi_set->base + PCIE_MSI_SET_ENABLE_OFFSET); + raw_spin_unlock_irqrestore(&pcie->irq_lock, flags); +} + +static void mtk_msi_bottom_irq_unmask(struct irq_data *data) +{ + struct mtk_msi_set *msi_set = irq_data_get_irq_chip_data(data); + struct mtk_gen3_pcie *pcie = data->domain->host_data; + unsigned long hwirq, flags; + u32 val; + + hwirq = data->hwirq % PCIE_MSI_IRQS_PER_SET; + + raw_spin_lock_irqsave(&pcie->irq_lock, flags); + val = readl_relaxed(msi_set->base + PCIE_MSI_SET_ENABLE_OFFSET); + val |= BIT(hwirq); + writel_relaxed(val, msi_set->base + PCIE_MSI_SET_ENABLE_OFFSET); + raw_spin_unlock_irqrestore(&pcie->irq_lock, flags); +} + +static struct irq_chip mtk_msi_bottom_irq_chip = { + .irq_ack = mtk_msi_bottom_irq_ack, + .irq_mask = mtk_msi_bottom_irq_mask, + .irq_unmask = mtk_msi_bottom_irq_unmask, + .irq_compose_msi_msg = mtk_compose_msi_msg, + .name = "MSI", +}; + +static int mtk_msi_bottom_domain_alloc(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs, + void *arg) +{ + struct mtk_gen3_pcie *pcie = domain->host_data; + struct mtk_msi_set *msi_set; + int i, hwirq, set_idx; + + mutex_lock(&pcie->lock); + + hwirq = bitmap_find_free_region(pcie->msi_irq_in_use, PCIE_MSI_IRQS_NUM, + order_base_2(nr_irqs)); + + mutex_unlock(&pcie->lock); + + if (hwirq < 0) + return -ENOSPC; + + set_idx = hwirq / PCIE_MSI_IRQS_PER_SET; + msi_set = &pcie->msi_sets[set_idx]; + + for (i = 0; i < nr_irqs; i++) + irq_domain_set_info(domain, virq + i, hwirq + i, + &mtk_msi_bottom_irq_chip, msi_set, + handle_edge_irq, NULL, NULL); + + return 0; +} + +static void mtk_msi_bottom_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + struct mtk_gen3_pcie *pcie = domain->host_data; + struct irq_data *data = irq_domain_get_irq_data(domain, virq); + + mutex_lock(&pcie->lock); + + bitmap_release_region(pcie->msi_irq_in_use, data->hwirq, + order_base_2(nr_irqs)); + + mutex_unlock(&pcie->lock); + + irq_domain_free_irqs_common(domain, virq, nr_irqs); +} + +static const struct irq_domain_ops mtk_msi_bottom_domain_ops = { + .alloc = mtk_msi_bottom_domain_alloc, + .free = mtk_msi_bottom_domain_free, +}; + +static void mtk_intx_mask(struct irq_data *data) +{ + struct mtk_gen3_pcie *pcie = irq_data_get_irq_chip_data(data); + unsigned long flags; + u32 val; + + raw_spin_lock_irqsave(&pcie->irq_lock, flags); + val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG); + val &= ~BIT(data->hwirq + PCIE_INTX_SHIFT); + writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG); + raw_spin_unlock_irqrestore(&pcie->irq_lock, flags); +} + +static void mtk_intx_unmask(struct irq_data *data) +{ + struct mtk_gen3_pcie *pcie = irq_data_get_irq_chip_data(data); + unsigned long flags; + u32 val; + + raw_spin_lock_irqsave(&pcie->irq_lock, flags); + val = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG); + val |= BIT(data->hwirq + PCIE_INTX_SHIFT); + writel_relaxed(val, pcie->base + PCIE_INT_ENABLE_REG); + raw_spin_unlock_irqrestore(&pcie->irq_lock, flags); +} + +/** + * mtk_intx_eoi() - Clear INTx IRQ status at the end of interrupt + * @data: pointer to chip specific data + * + * As an emulated level IRQ, its interrupt status will remain + * until the corresponding de-assert message is received; hence that + * the status can only be cleared when the interrupt has been serviced. + */ +static void mtk_intx_eoi(struct irq_data *data) +{ + struct mtk_gen3_pcie *pcie = irq_data_get_irq_chip_data(data); + unsigned long hwirq; + + hwirq = data->hwirq + PCIE_INTX_SHIFT; + writel_relaxed(BIT(hwirq), pcie->base + PCIE_INT_STATUS_REG); +} + +static struct irq_chip mtk_intx_irq_chip = { + .irq_mask = mtk_intx_mask, + .irq_unmask = mtk_intx_unmask, + .irq_eoi = mtk_intx_eoi, + .name = "INTx", +}; + +static int mtk_pcie_intx_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_data(irq, domain->host_data); + irq_set_chip_and_handler_name(irq, &mtk_intx_irq_chip, + handle_fasteoi_irq, "INTx"); + return 0; +} + +static const struct irq_domain_ops intx_domain_ops = { + .map = mtk_pcie_intx_map, +}; + +static int mtk_pcie_init_irq_domains(struct mtk_gen3_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct device_node *intc_node, *node = dev->of_node; + int ret; + + raw_spin_lock_init(&pcie->irq_lock); + + /* Setup INTx */ + intc_node = of_get_child_by_name(node, "interrupt-controller"); + if (!intc_node) { + dev_err(dev, "missing interrupt-controller node\n"); + return -ENODEV; + } + + pcie->intx_domain = irq_domain_create_linear(of_fwnode_handle(intc_node), PCI_NUM_INTX, + &intx_domain_ops, pcie); + if (!pcie->intx_domain) { + dev_err(dev, "failed to create INTx IRQ domain\n"); + ret = -ENODEV; + goto out_put_node; + } + + /* Setup MSI */ + mutex_init(&pcie->lock); + + struct irq_domain_info info = { + .fwnode = dev_fwnode(dev), + .ops = &mtk_msi_bottom_domain_ops, + .host_data = pcie, + .size = PCIE_MSI_IRQS_NUM, + }; + + pcie->msi_bottom_domain = msi_create_parent_irq_domain(&info, &mtk_msi_parent_ops); + if (!pcie->msi_bottom_domain) { + dev_err(dev, "failed to create MSI bottom domain\n"); + ret = -ENODEV; + goto err_msi_bottom_domain; + } + + of_node_put(intc_node); + return 0; + +err_msi_bottom_domain: + irq_domain_remove(pcie->intx_domain); +out_put_node: + of_node_put(intc_node); + return ret; +} + +static void mtk_pcie_irq_teardown(struct mtk_gen3_pcie *pcie) +{ + irq_set_chained_handler_and_data(pcie->irq, NULL, NULL); + + if (pcie->intx_domain) + irq_domain_remove(pcie->intx_domain); + + if (pcie->msi_bottom_domain) + irq_domain_remove(pcie->msi_bottom_domain); + + irq_dispose_mapping(pcie->irq); +} + +static void mtk_pcie_msi_handler(struct mtk_gen3_pcie *pcie, int set_idx) +{ + struct mtk_msi_set *msi_set = &pcie->msi_sets[set_idx]; + unsigned long msi_enable, msi_status; + irq_hw_number_t bit, hwirq; + + msi_enable = readl_relaxed(msi_set->base + PCIE_MSI_SET_ENABLE_OFFSET); + + do { + msi_status = readl_relaxed(msi_set->base + + PCIE_MSI_SET_STATUS_OFFSET); + msi_status &= msi_enable; + if (!msi_status) + break; + + for_each_set_bit(bit, &msi_status, PCIE_MSI_IRQS_PER_SET) { + hwirq = bit + set_idx * PCIE_MSI_IRQS_PER_SET; + generic_handle_domain_irq(pcie->msi_bottom_domain, hwirq); + } + } while (true); +} + +static void mtk_pcie_irq_handler(struct irq_desc *desc) +{ + struct mtk_gen3_pcie *pcie = irq_desc_get_handler_data(desc); + struct irq_chip *irqchip = irq_desc_get_chip(desc); + unsigned long status; + irq_hw_number_t irq_bit = PCIE_INTX_SHIFT; + + chained_irq_enter(irqchip, desc); + + status = readl_relaxed(pcie->base + PCIE_INT_STATUS_REG); + for_each_set_bit_from(irq_bit, &status, PCI_NUM_INTX + + PCIE_INTX_SHIFT) + generic_handle_domain_irq(pcie->intx_domain, + irq_bit - PCIE_INTX_SHIFT); + + irq_bit = PCIE_MSI_SHIFT; + for_each_set_bit_from(irq_bit, &status, PCIE_MSI_SET_NUM + + PCIE_MSI_SHIFT) { + mtk_pcie_msi_handler(pcie, irq_bit - PCIE_MSI_SHIFT); + + writel_relaxed(BIT(irq_bit), pcie->base + PCIE_INT_STATUS_REG); + } + + chained_irq_exit(irqchip, desc); +} + +static int mtk_pcie_setup_irq(struct mtk_gen3_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct platform_device *pdev = to_platform_device(dev); + int err; + + pcie->irq = platform_get_irq(pdev, 0); + if (pcie->irq < 0) + return pcie->irq; + + err = mtk_pcie_init_irq_domains(pcie); + if (err) + return err; + + irq_set_chained_handler_and_data(pcie->irq, mtk_pcie_irq_handler, pcie); + + return 0; +} + +static int mtk_pcie_parse_port(struct mtk_gen3_pcie *pcie) +{ + int i, ret, num_resets = pcie->soc->phy_resets.num_resets; + struct device *dev = pcie->dev; + struct platform_device *pdev = to_platform_device(dev); + struct resource *regs; + u32 num_lanes; + + regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pcie-mac"); + if (!regs) + return -EINVAL; + pcie->base = devm_ioremap_resource(dev, regs); + if (IS_ERR(pcie->base)) + return dev_err_probe(dev, PTR_ERR(pcie->base), "failed to map register base\n"); + + pcie->reg_base = regs->start; + + for (i = 0; i < num_resets; i++) + pcie->phy_resets[i].id = pcie->soc->phy_resets.id[i]; + + ret = devm_reset_control_bulk_get_optional_shared(dev, num_resets, + pcie->phy_resets); + if (ret) + return dev_err_probe(dev, ret, "failed to get PHY bulk reset\n"); + + pcie->mac_reset = devm_reset_control_get_optional_exclusive(dev, "mac"); + if (IS_ERR(pcie->mac_reset)) + return dev_err_probe(dev, PTR_ERR(pcie->mac_reset), "failed to get MAC reset\n"); + + pcie->phy = devm_phy_optional_get(dev, "pcie-phy"); + if (IS_ERR(pcie->phy)) + return dev_err_probe(dev, PTR_ERR(pcie->phy), "failed to get PHY\n"); + + pcie->num_clks = devm_clk_bulk_get_all(dev, &pcie->clks); + if (pcie->num_clks < 0) + return dev_err_probe(dev, pcie->num_clks, "failed to get clocks\n"); + + ret = of_property_read_u32(dev->of_node, "num-lanes", &num_lanes); + if (ret == 0) { + if (num_lanes == 0 || num_lanes > 16 || + (num_lanes != 1 && num_lanes % 2)) + dev_warn(dev, "invalid num-lanes, using controller defaults\n"); + else + pcie->num_lanes = num_lanes; + } + + return 0; +} + +static int mtk_pcie_en7581_power_up(struct mtk_gen3_pcie *pcie) +{ + struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); + struct device *dev = pcie->dev; + struct resource_entry *entry; + struct regmap *pbus_regmap; + u32 val, args[2], size; + resource_size_t addr; + int err; + + /* + * The controller may have been left out of reset by the bootloader + * so make sure that we get a clean start by asserting resets here. + */ + reset_control_bulk_assert(pcie->soc->phy_resets.num_resets, + pcie->phy_resets); + + /* Wait for the time needed to complete the reset lines assert. */ + msleep(PCIE_EN7581_RESET_TIME_MS); + + /* + * Configure PBus base address and base address mask to allow the + * hw to detect if a given address is accessible on PCIe controller. + */ + pbus_regmap = syscon_regmap_lookup_by_phandle_args(dev->of_node, + "mediatek,pbus-csr", + ARRAY_SIZE(args), + args); + if (IS_ERR(pbus_regmap)) + return PTR_ERR(pbus_regmap); + + entry = resource_list_first_type(&host->windows, IORESOURCE_MEM); + if (!entry) + return -ENODEV; + + addr = entry->res->start - entry->offset; + regmap_write(pbus_regmap, args[0], lower_32_bits(addr)); + size = lower_32_bits(resource_size(entry->res)); + regmap_write(pbus_regmap, args[1], GENMASK(31, __fls(size))); + + /* + * Unlike the other MediaTek Gen3 controllers, the Airoha EN7581 + * requires PHY initialization and power-on before PHY reset deassert. + */ + err = phy_init(pcie->phy); + if (err) { + dev_err(dev, "failed to initialize PHY\n"); + return err; + } + + err = phy_power_on(pcie->phy); + if (err) { + dev_err(dev, "failed to power on PHY\n"); + goto err_phy_on; + } + + err = reset_control_bulk_deassert(pcie->soc->phy_resets.num_resets, + pcie->phy_resets); + if (err) { + dev_err(dev, "failed to deassert PHYs\n"); + goto err_phy_deassert; + } + + /* + * Wait for the time needed to complete the bulk de-assert above. + * This time is specific for EN7581 SoC. + */ + msleep(PCIE_EN7581_RESET_TIME_MS); + + pm_runtime_enable(dev); + pm_runtime_get_sync(dev); + + val = FIELD_PREP(PCIE_VAL_LN0_DOWNSTREAM, 0x47) | + FIELD_PREP(PCIE_VAL_LN1_DOWNSTREAM, 0x47) | + FIELD_PREP(PCIE_VAL_LN0_UPSTREAM, 0x41) | + FIELD_PREP(PCIE_VAL_LN1_UPSTREAM, 0x41); + writel_relaxed(val, pcie->base + PCIE_EQ_PRESET_01_REG); + + val = PCIE_K_PHYPARAM_QUERY | PCIE_K_QUERY_TIMEOUT | + FIELD_PREP(PCIE_K_PRESET_TO_USE_16G, 0x80) | + FIELD_PREP(PCIE_K_PRESET_TO_USE, 0x2) | + FIELD_PREP(PCIE_K_FINETUNE_MAX, 0xf); + writel_relaxed(val, pcie->base + PCIE_PIPE4_PIE8_REG); + + err = clk_bulk_prepare_enable(pcie->num_clks, pcie->clks); + if (err) { + dev_err(dev, "failed to prepare clock\n"); + goto err_clk_prepare_enable; + } + + /* + * Airoha EN7581 performs PCIe reset via clk callbacks since it has a + * hw issue with PCIE_PE_RSTB signal. Add wait for the time needed to + * complete the PCIe reset. + */ + msleep(PCIE_T_PVPERL_MS); + + return 0; + +err_clk_prepare_enable: + pm_runtime_put_sync(dev); + pm_runtime_disable(dev); + reset_control_bulk_assert(pcie->soc->phy_resets.num_resets, + pcie->phy_resets); +err_phy_deassert: + phy_power_off(pcie->phy); +err_phy_on: + phy_exit(pcie->phy); + + return err; +} + +static int mtk_pcie_power_up(struct mtk_gen3_pcie *pcie) +{ + struct device *dev = pcie->dev; + int err; + + /* + * The controller may have been left out of reset by the bootloader + * so make sure that we get a clean start by asserting resets here. + */ + reset_control_bulk_assert(pcie->soc->phy_resets.num_resets, + pcie->phy_resets); + reset_control_assert(pcie->mac_reset); + usleep_range(PCIE_MTK_RESET_TIME_US, 2 * PCIE_MTK_RESET_TIME_US); + + /* PHY power on and enable pipe clock */ + err = reset_control_bulk_deassert(pcie->soc->phy_resets.num_resets, + pcie->phy_resets); + if (err) { + dev_err(dev, "failed to deassert PHYs\n"); + return err; + } + + err = phy_init(pcie->phy); + if (err) { + dev_err(dev, "failed to initialize PHY\n"); + goto err_phy_init; + } + + err = phy_power_on(pcie->phy); + if (err) { + dev_err(dev, "failed to power on PHY\n"); + goto err_phy_on; + } + + /* MAC power on and enable transaction layer clocks */ + reset_control_deassert(pcie->mac_reset); + + pm_runtime_enable(dev); + pm_runtime_get_sync(dev); + + err = clk_bulk_prepare_enable(pcie->num_clks, pcie->clks); + if (err) { + dev_err(dev, "failed to enable clocks\n"); + goto err_clk_init; + } + + return 0; + +err_clk_init: + pm_runtime_put_sync(dev); + pm_runtime_disable(dev); + reset_control_assert(pcie->mac_reset); + phy_power_off(pcie->phy); +err_phy_on: + phy_exit(pcie->phy); +err_phy_init: + reset_control_bulk_assert(pcie->soc->phy_resets.num_resets, + pcie->phy_resets); + + return err; +} + +static void mtk_pcie_power_down(struct mtk_gen3_pcie *pcie) +{ + clk_bulk_disable_unprepare(pcie->num_clks, pcie->clks); + + pm_runtime_put_sync(pcie->dev); + pm_runtime_disable(pcie->dev); + reset_control_assert(pcie->mac_reset); + + phy_power_off(pcie->phy); + phy_exit(pcie->phy); + reset_control_bulk_assert(pcie->soc->phy_resets.num_resets, + pcie->phy_resets); +} + +static int mtk_pcie_get_controller_max_link_speed(struct mtk_gen3_pcie *pcie) +{ + u32 val; + int ret; + + val = readl_relaxed(pcie->base + PCIE_BASE_CFG_REG); + val = FIELD_GET(PCIE_BASE_CFG_SPEED, val); + ret = fls(val); + + return ret > 0 ? ret : -EINVAL; +} + +static int mtk_pcie_setup(struct mtk_gen3_pcie *pcie) +{ + int err, max_speed; + + err = mtk_pcie_parse_port(pcie); + if (err) + return err; + + /* + * Deassert the line in order to avoid unbalance in deassert_count + * counter since the bulk is shared. + */ + reset_control_bulk_deassert(pcie->soc->phy_resets.num_resets, + pcie->phy_resets); + + /* Don't touch the hardware registers before power up */ + err = pcie->soc->power_up(pcie); + if (err) + return err; + + err = of_pci_get_max_link_speed(pcie->dev->of_node); + if (pcie_get_link_speed(err) != PCI_SPEED_UNKNOWN) { + /* Get the maximum speed supported by the controller */ + max_speed = mtk_pcie_get_controller_max_link_speed(pcie); + + /* Set max_link_speed only if the controller supports it */ + if (max_speed >= 0 && max_speed <= err) { + pcie->max_link_speed = err; + dev_info(pcie->dev, + "maximum controller link speed Gen%d, overriding to Gen%u", + max_speed, pcie->max_link_speed); + } + } + + /* Try link up */ + err = mtk_pcie_startup_port(pcie); + if (err) + goto err_setup; + + return 0; + +err_setup: + mtk_pcie_power_down(pcie); + + return err; +} + +static int mtk_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct mtk_gen3_pcie *pcie; + struct pci_host_bridge *host; + int err; + + host = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!host) + return -ENOMEM; + + pcie = pci_host_bridge_priv(host); + + pcie->dev = dev; + pcie->soc = device_get_match_data(dev); + platform_set_drvdata(pdev, pcie); + + err = mtk_pcie_setup_irq(pcie); + if (err) + return dev_err_probe(dev, err, "Failed to setup IRQ domains\n"); + + err = pci_pwrctrl_create_devices(pcie->dev); + if (err) { + dev_err_probe(dev, err, "failed to create pwrctrl devices\n"); + goto err_tear_down_irq; + } + + err = mtk_pcie_setup(pcie); + if (err) + goto err_destroy_pwrctrl; + + host->ops = &mtk_pcie_ops; + host->sysdata = pcie; + + err = pci_host_probe(host); + if (err) + goto err_power_down_pcie; + + return 0; + +err_power_down_pcie: + mtk_pcie_devices_power_down(pcie); + mtk_pcie_power_down(pcie); +err_destroy_pwrctrl: + if (err != -EPROBE_DEFER) + pci_pwrctrl_destroy_devices(pcie->dev); +err_tear_down_irq: + mtk_pcie_irq_teardown(pcie); + return err; +} + +static void mtk_pcie_remove(struct platform_device *pdev) +{ + struct mtk_gen3_pcie *pcie = platform_get_drvdata(pdev); + struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); + + pci_lock_rescan_remove(); + pci_stop_root_bus(host->bus); + pci_remove_root_bus(host->bus); + pci_unlock_rescan_remove(); + + mtk_pcie_devices_power_down(pcie); + mtk_pcie_power_down(pcie); + pci_pwrctrl_destroy_devices(pcie->dev); + mtk_pcie_irq_teardown(pcie); +} + +static void mtk_pcie_shutdown(struct platform_device *pdev) +{ + struct mtk_gen3_pcie *pcie = platform_get_drvdata(pdev); + + mtk_pcie_devices_power_down(pcie); + mtk_pcie_power_down(pcie); +} + +static void mtk_pcie_irq_save(struct mtk_gen3_pcie *pcie) +{ + int i; + + raw_spin_lock(&pcie->irq_lock); + + pcie->saved_irq_state = readl_relaxed(pcie->base + PCIE_INT_ENABLE_REG); + + for (i = 0; i < PCIE_MSI_SET_NUM; i++) { + struct mtk_msi_set *msi_set = &pcie->msi_sets[i]; + + msi_set->saved_irq_state = readl_relaxed(msi_set->base + + PCIE_MSI_SET_ENABLE_OFFSET); + } + + raw_spin_unlock(&pcie->irq_lock); +} + +static void mtk_pcie_irq_restore(struct mtk_gen3_pcie *pcie) +{ + int i; + + raw_spin_lock(&pcie->irq_lock); + + writel_relaxed(pcie->saved_irq_state, pcie->base + PCIE_INT_ENABLE_REG); + + for (i = 0; i < PCIE_MSI_SET_NUM; i++) { + struct mtk_msi_set *msi_set = &pcie->msi_sets[i]; + + writel_relaxed(msi_set->saved_irq_state, + msi_set->base + PCIE_MSI_SET_ENABLE_OFFSET); + } + + raw_spin_unlock(&pcie->irq_lock); +} + +static int mtk_pcie_turn_off_link(struct mtk_gen3_pcie *pcie) +{ + u32 val; + + val = readl_relaxed(pcie->base + PCIE_ICMD_PM_REG); + val |= PCIE_TURN_OFF_LINK; + writel_relaxed(val, pcie->base + PCIE_ICMD_PM_REG); + + /* Check the link is L2 */ + return readl_poll_timeout(pcie->base + PCIE_LTSSM_STATUS_REG, val, + (PCIE_LTSSM_STATE(val) == + PCIE_LTSSM_STATE_L2_IDLE), 20, + 50 * USEC_PER_MSEC); +} + +static int mtk_pcie_suspend_noirq(struct device *dev) +{ + struct mtk_gen3_pcie *pcie = dev_get_drvdata(dev); + int err; + + /* Trigger link to L2 state */ + err = mtk_pcie_turn_off_link(pcie); + if (err) { + dev_err(pcie->dev, "cannot enter L2 state\n"); + return err; + } + + mtk_pcie_devices_power_down(pcie); + dev_dbg(pcie->dev, "entered L2 states successfully"); + + mtk_pcie_irq_save(pcie); + mtk_pcie_power_down(pcie); + + return 0; +} + +static int mtk_pcie_resume_noirq(struct device *dev) +{ + struct mtk_gen3_pcie *pcie = dev_get_drvdata(dev); + int err; + + err = pcie->soc->power_up(pcie); + if (err) + return err; + + err = mtk_pcie_startup_port(pcie); + if (err) + goto err_power_down; + + mtk_pcie_irq_restore(pcie); + + return 0; + +err_power_down: + mtk_pcie_power_down(pcie); + return err; +} + +static const struct dev_pm_ops mtk_pcie_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_pcie_suspend_noirq, + mtk_pcie_resume_noirq) +}; + +static const struct mtk_gen3_pcie_pdata mtk_pcie_soc_mt8192 = { + .power_up = mtk_pcie_power_up, + .phy_resets = { + .id[0] = "phy", + .num_resets = 1, + }, +}; + +static const struct mtk_gen3_pcie_pdata mtk_pcie_soc_mt8196 = { + .power_up = mtk_pcie_power_up, + .phy_resets = { + .id[0] = "phy", + .num_resets = 1, + }, + .sys_clk_rdy_time_us = 10, +}; + +static const struct mtk_gen3_pcie_pdata mtk_pcie_soc_en7581 = { + .power_up = mtk_pcie_en7581_power_up, + .phy_resets = { + .id[0] = "phy-lane0", + .id[1] = "phy-lane1", + .id[2] = "phy-lane2", + .num_resets = 3, + }, + .flags = SKIP_PCIE_RSTB, +}; + +static const struct of_device_id mtk_pcie_of_match[] = { + { .compatible = "airoha,en7581-pcie", .data = &mtk_pcie_soc_en7581 }, + { .compatible = "mediatek,mt8192-pcie", .data = &mtk_pcie_soc_mt8192 }, + { .compatible = "mediatek,mt8196-pcie", .data = &mtk_pcie_soc_mt8196 }, + {}, +}; +MODULE_DEVICE_TABLE(of, mtk_pcie_of_match); + +static struct platform_driver mtk_pcie_driver = { + .probe = mtk_pcie_probe, + .remove = mtk_pcie_remove, + .shutdown = mtk_pcie_shutdown, + .driver = { + .name = "mtk-pcie-gen3", + .of_match_table = mtk_pcie_of_match, + .pm = &mtk_pcie_pm_ops, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, +}; + +module_platform_driver(mtk_pcie_driver); +MODULE_DESCRIPTION("MediaTek Gen3 PCIe host controller driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pcie-mediatek.c b/drivers/pci/controller/pcie-mediatek.c new file mode 100644 index 000000000..a60d1ae07 --- /dev/null +++ b/drivers/pci/controller/pcie-mediatek.c @@ -0,0 +1,1470 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * MediaTek PCIe host controller driver. + * + * Copyright (c) 2017 MediaTek Inc. + * Author: Ryder Lee <ryder.lee@mediatek.com> + * Honghui Zhang <honghui.zhang@mediatek.com> + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/errno.h> +#include <linux/iopoll.h> +#include <linux/irq.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/irqdomain.h> +#include <linux/kconfig.h> +#include <linux/kernel.h> +#include <linux/mfd/syscon.h> +#include <linux/msi.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/pci.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/regmap.h> +#include <linux/reset.h> + +#include "../pci.h" + +/* PCIe shared registers */ +#define PCIE_SYS_CFG 0x00 +#define PCIE_INT_ENABLE 0x0c +#define PCIE_CFG_ADDR 0x20 +#define PCIE_CFG_DATA 0x24 + +/* PCIe per port registers */ +#define PCIE_BAR0_SETUP 0x10 +#define PCIE_CLASS 0x34 +#define PCIE_LINK_STATUS 0x50 + +#define PCIE_PORT_INT_EN(x) BIT(20 + (x)) +#define PCIE_PORT_PERST(x) BIT(1 + (x)) +#define PCIE_PORT_LINKUP BIT(0) +#define PCIE_BAR_MAP_MAX GENMASK(31, 16) + +#define PCIE_BAR_ENABLE BIT(0) +#define PCIE_REVISION_ID BIT(0) +#define PCIE_CLASS_CODE (0x60400 << 8) +#define PCIE_CONF_REG(regn) (((regn) & GENMASK(7, 2)) | \ + ((((regn) >> 8) & GENMASK(3, 0)) << 24)) +#define PCIE_CONF_FUN(fun) (((fun) << 8) & GENMASK(10, 8)) +#define PCIE_CONF_DEV(dev) (((dev) << 11) & GENMASK(15, 11)) +#define PCIE_CONF_BUS(bus) (((bus) << 16) & GENMASK(23, 16)) +#define PCIE_CONF_ADDR(regn, fun, dev, bus) \ + (PCIE_CONF_REG(regn) | PCIE_CONF_FUN(fun) | \ + PCIE_CONF_DEV(dev) | PCIE_CONF_BUS(bus)) + +/* MediaTek specific configuration registers */ +#define PCIE_FTS_NUM 0x70c +#define PCIE_FTS_NUM_MASK GENMASK(15, 8) +#define PCIE_FTS_NUM_L0(x) FIELD_PREP(PCIE_FTS_NUM_MASK, x) + +#define PCIE_FC_CREDIT 0x73c +#define PCIE_FC_CREDIT_MASK (GENMASK(31, 31) | GENMASK(28, 16)) +#define PCIE_FC_CREDIT_VAL(x) ((x) << 16) + +/* PCIe V2 share registers */ +#define PCIE_SYS_CFG_V2 0x0 +#define PCIE_CSR_LTSSM_EN(x) BIT(0 + (x) * 8) +#define PCIE_CSR_ASPM_L1_EN(x) BIT(1 + (x) * 8) + +/* PCIe V2 per-port registers */ +#define PCIE_MSI_VECTOR 0x0c0 + +#define PCIE_CONF_VEND_ID 0x100 +#define PCIE_CONF_DEVICE_ID 0x102 +#define PCIE_CONF_REV_CLASS 0x104 +#define PCIE_CONF_CLASS_ID 0x106 + +#define PCIE_INT_MASK 0x420 +#define INTX_MASK GENMASK(19, 16) +#define INTX_SHIFT 16 +#define PCIE_INT_STATUS 0x424 +#define MSI_STATUS BIT(23) +#define PCIE_IMSI_STATUS 0x42c +#define PCIE_IMSI_ADDR 0x430 +#define MSI_MASK BIT(23) +#define MTK_MSI_IRQS_NUM 32 + +#define EN7528_HOST_MODE 0x00804201 +#define EN7528_LINKUP_REG 0x50 +#define EN7528_RC0_LINKUP BIT(1) +#define EN7528_RC1_LINKUP BIT(2) + +#define PCIE_AHB_TRANS_BASE0_L 0x438 +#define PCIE_AHB_TRANS_BASE0_H 0x43c +#define AHB2PCIE_SIZE(x) ((x) & GENMASK(4, 0)) +#define PCIE_AXI_WINDOW0 0x448 +#define WIN_ENABLE BIT(7) +/* + * Define PCIe to AHB window size as 2^33 to support max 8GB address space + * translate, support least 4GB DRAM size access from EP DMA(physical DRAM + * start from 0x40000000). + */ +#define PCIE2AHB_SIZE 0x21 + +/* PCIe V2 configuration transaction header */ +#define PCIE_CFG_HEADER0 0x460 +#define PCIE_CFG_HEADER1 0x464 +#define PCIE_CFG_HEADER2 0x468 +#define PCIE_CFG_WDATA 0x470 +#define PCIE_APP_TLP_REQ 0x488 +#define PCIE_CFG_RDATA 0x48c +#define APP_CFG_REQ BIT(0) +#define APP_CPL_STATUS GENMASK(7, 5) + +#define CFG_DW0_LENGTH(length) ((length) & GENMASK(9, 0)) +#define CFG_DW0_TYPE(type) (((type) << 24) & GENMASK(28, 24)) +#define CFG_DW0_FMT(fmt) (((fmt) << 29) & GENMASK(31, 29)) +#define CFG_DW2_REGN(regn) ((regn) & GENMASK(11, 2)) +#define CFG_DW2_FUN(fun) (((fun) << 16) & GENMASK(18, 16)) +#define CFG_DW2_DEV(dev) (((dev) << 19) & GENMASK(23, 19)) +#define CFG_DW2_BUS(bus) (((bus) << 24) & GENMASK(31, 24)) +#define CFG_HEADER_DW0(type, fmt) \ + (CFG_DW0_LENGTH(1) | CFG_DW0_TYPE(type) | CFG_DW0_FMT(fmt)) +#define CFG_HEADER_DW1(where, size) \ + (GENMASK(((size) - 1), 0) << ((where) & 0x3)) +#define CFG_HEADER_DW2(regn, fun, dev, bus) \ + (CFG_DW2_REGN(regn) | CFG_DW2_FUN(fun) | \ + CFG_DW2_DEV(dev) | CFG_DW2_BUS(bus)) + +#define PCIE_RST_CTRL 0x510 +#define PCIE_PHY_RSTB BIT(0) +#define PCIE_PIPE_SRSTB BIT(1) +#define PCIE_MAC_SRSTB BIT(2) +#define PCIE_CRSTB BIT(3) +#define PCIE_PERSTB BIT(8) +#define PCIE_LINKDOWN_RST_EN GENMASK(15, 13) +#define PCIE_LINK_STATUS_V2 0x804 +#define PCIE_PORT_LINKUP_V2 BIT(10) + +struct mtk_pcie_port; + +/** + * enum mtk_pcie_quirks - MTK PCIe quirks + * @MTK_PCIE_FIX_CLASS_ID: host's class ID needed to be fixed + * @MTK_PCIE_FIX_DEVICE_ID: host's device ID needed to be fixed + * @MTK_PCIE_NO_MSI: Bridge has no MSI support, and relies on an external block + * @MTK_PCIE_SKIP_RSTB: Skip calling RSTB bits on PCIe probe + * @MTK_PCIE_RETRAIN: Retrain link to bridge after startup because some + * Gen2-capable devices start as Gen1. + */ +enum mtk_pcie_quirks { + MTK_PCIE_FIX_CLASS_ID = BIT(0), + MTK_PCIE_FIX_DEVICE_ID = BIT(1), + MTK_PCIE_NO_MSI = BIT(2), + MTK_PCIE_SKIP_RSTB = BIT(3), + MTK_PCIE_RETRAIN = BIT(4), +}; + +/** + * struct mtk_pcie_soc - differentiate between host generations + * @device_id: device ID which this host need to be fixed + * @ops: pointer to configuration access functions + * @startup: pointer to controller setting functions + * @setup_irq: pointer to initialize IRQ functions + * @quirks: PCIe device quirks. + */ +struct mtk_pcie_soc { + unsigned int device_id; + struct pci_ops *ops; + int (*startup)(struct mtk_pcie_port *port); + int (*setup_irq)(struct mtk_pcie_port *port, struct device_node *node); + enum mtk_pcie_quirks quirks; +}; + +/** + * struct mtk_pcie_port - PCIe port information + * @base: IO mapped register base + * @phys_base: Physical address of the I/O register base region + * @list: port list + * @pcie: pointer to PCIe host info + * @reset: pointer to port reset control + * @sys_ck: pointer to transaction/data link layer clock + * @ahb_ck: pointer to AHB slave interface operating clock for CSR access + * and RC initiated MMIO access + * @axi_ck: pointer to application layer MMIO channel operating clock + * @aux_ck: pointer to pe2_mac_bridge and pe2_mac_core operating clock + * when pcie_mac_ck/pcie_pipe_ck is turned off + * @obff_ck: pointer to OBFF functional block operating clock + * @pipe_ck: pointer to LTSSM and PHY/MAC layer operating clock + * @phy: pointer to PHY control block + * @slot: port slot + * @irq: GIC irq + * @irq_domain: legacy INTx IRQ domain + * @inner_domain: inner IRQ domain + * @lock: protect the msi_irq_in_use bitmap + * @msi_irq_in_use: bit map for assigned MSI IRQ + */ +struct mtk_pcie_port { + void __iomem *base; + phys_addr_t phys_base; + struct list_head list; + struct mtk_pcie *pcie; + struct reset_control *reset; + struct clk *sys_ck; + struct clk *ahb_ck; + struct clk *axi_ck; + struct clk *aux_ck; + struct clk *obff_ck; + struct clk *pipe_ck; + struct phy *phy; + u32 slot; + int irq; + struct irq_domain *irq_domain; + struct irq_domain *inner_domain; + struct mutex lock; + DECLARE_BITMAP(msi_irq_in_use, MTK_MSI_IRQS_NUM); +}; + +/** + * struct mtk_pcie - PCIe host information + * @dev: pointer to PCIe device + * @base: IO mapped register base + * @cfg: IO mapped register map for PCIe config + * @free_ck: free-run reference clock + * @ports: pointer to PCIe port information + * @soc: pointer to SoC-dependent operations + */ +struct mtk_pcie { + struct device *dev; + void __iomem *base; + struct regmap *cfg; + struct clk *free_ck; + + struct list_head ports; + const struct mtk_pcie_soc *soc; +}; + +static void mtk_pcie_subsys_powerdown(struct mtk_pcie *pcie) +{ + struct device *dev = pcie->dev; + + clk_disable_unprepare(pcie->free_ck); + + pm_runtime_put_sync(dev); + pm_runtime_disable(dev); +} + +static void mtk_pcie_port_free(struct mtk_pcie_port *port) +{ + struct mtk_pcie *pcie = port->pcie; + struct device *dev = pcie->dev; + + devm_iounmap(dev, port->base); + list_del(&port->list); + devm_kfree(dev, port); +} + +static void mtk_pcie_put_resources(struct mtk_pcie *pcie) +{ + struct mtk_pcie_port *port, *tmp; + + list_for_each_entry_safe(port, tmp, &pcie->ports, list) { + phy_power_off(port->phy); + phy_exit(port->phy); + clk_disable_unprepare(port->pipe_ck); + clk_disable_unprepare(port->obff_ck); + clk_disable_unprepare(port->axi_ck); + clk_disable_unprepare(port->aux_ck); + clk_disable_unprepare(port->ahb_ck); + clk_disable_unprepare(port->sys_ck); + mtk_pcie_port_free(port); + } + + mtk_pcie_subsys_powerdown(pcie); +} + +static int mtk_pcie_check_cfg_cpld(struct mtk_pcie_port *port) +{ + u32 val; + int err; + + err = readl_poll_timeout_atomic(port->base + PCIE_APP_TLP_REQ, val, + !(val & APP_CFG_REQ), 10, + 100 * USEC_PER_MSEC); + if (err) + return PCIBIOS_SET_FAILED; + + if (readl(port->base + PCIE_APP_TLP_REQ) & APP_CPL_STATUS) + return PCIBIOS_SET_FAILED; + + return PCIBIOS_SUCCESSFUL; +} + +static int mtk_pcie_hw_rd_cfg(struct mtk_pcie_port *port, u32 bus, u32 devfn, + int where, int size, u32 *val) +{ + u32 tmp; + + /* Write PCIe configuration transaction header for Cfgrd */ + writel(CFG_HEADER_DW0(PCIE_TLP_TYPE_CFG0_RDWR, PCIE_TLP_FMT_3DW_NO_DATA), + port->base + PCIE_CFG_HEADER0); + writel(CFG_HEADER_DW1(where, size), port->base + PCIE_CFG_HEADER1); + writel(CFG_HEADER_DW2(where, PCI_FUNC(devfn), PCI_SLOT(devfn), bus), + port->base + PCIE_CFG_HEADER2); + + /* Trigger h/w to transmit Cfgrd TLP */ + tmp = readl(port->base + PCIE_APP_TLP_REQ); + tmp |= APP_CFG_REQ; + writel(tmp, port->base + PCIE_APP_TLP_REQ); + + /* Check completion status */ + if (mtk_pcie_check_cfg_cpld(port)) + return PCIBIOS_SET_FAILED; + + /* Read cpld payload of Cfgrd */ + *val = readl(port->base + PCIE_CFG_RDATA); + + if (size == 1) + *val = (*val >> (8 * (where & 3))) & 0xff; + else if (size == 2) + *val = (*val >> (8 * (where & 3))) & 0xffff; + + return PCIBIOS_SUCCESSFUL; +} + +static int mtk_pcie_hw_wr_cfg(struct mtk_pcie_port *port, u32 bus, u32 devfn, + int where, int size, u32 val) +{ + /* Write PCIe configuration transaction header for Cfgwr */ + writel(CFG_HEADER_DW0(PCIE_TLP_TYPE_CFG0_RDWR, PCIE_TLP_FMT_3DW_DATA), + port->base + PCIE_CFG_HEADER0); + writel(CFG_HEADER_DW1(where, size), port->base + PCIE_CFG_HEADER1); + writel(CFG_HEADER_DW2(where, PCI_FUNC(devfn), PCI_SLOT(devfn), bus), + port->base + PCIE_CFG_HEADER2); + + /* Write Cfgwr data */ + val = val << 8 * (where & 3); + writel(val, port->base + PCIE_CFG_WDATA); + + /* Trigger h/w to transmit Cfgwr TLP */ + val = readl(port->base + PCIE_APP_TLP_REQ); + val |= APP_CFG_REQ; + writel(val, port->base + PCIE_APP_TLP_REQ); + + /* Check completion status */ + return mtk_pcie_check_cfg_cpld(port); +} + +static struct mtk_pcie_port *mtk_pcie_find_port(struct pci_bus *bus, + unsigned int devfn) +{ + struct mtk_pcie *pcie = bus->sysdata; + struct mtk_pcie_port *port; + struct pci_dev *dev = NULL; + + /* + * Walk the bus hierarchy to get the devfn value + * of the port in the root bus. + */ + while (bus && bus->number) { + dev = bus->self; + bus = dev->bus; + devfn = dev->devfn; + } + + list_for_each_entry(port, &pcie->ports, list) + if (port->slot == PCI_SLOT(devfn)) + return port; + + return NULL; +} + +static int mtk_pcie_config_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct mtk_pcie_port *port; + u32 bn = bus->number; + + port = mtk_pcie_find_port(bus, devfn); + if (!port) + return PCIBIOS_DEVICE_NOT_FOUND; + + return mtk_pcie_hw_rd_cfg(port, bn, devfn, where, size, val); +} + +static int mtk_pcie_config_write(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + struct mtk_pcie_port *port; + u32 bn = bus->number; + + port = mtk_pcie_find_port(bus, devfn); + if (!port) + return PCIBIOS_DEVICE_NOT_FOUND; + + return mtk_pcie_hw_wr_cfg(port, bn, devfn, where, size, val); +} + +static struct pci_ops mtk_pcie_ops_v2 = { + .read = mtk_pcie_config_read, + .write = mtk_pcie_config_write, +}; + +static void mtk_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct mtk_pcie_port *port = irq_data_get_irq_chip_data(data); + phys_addr_t addr; + + /* MT2712/MT7622 only support 32-bit MSI addresses */ + addr = port->phys_base + PCIE_MSI_VECTOR; + msg->address_hi = 0; + msg->address_lo = lower_32_bits(addr); + + msg->data = data->hwirq; + + dev_dbg(port->pcie->dev, "msi#%d address_hi %#x address_lo %#x\n", + (int)data->hwirq, msg->address_hi, msg->address_lo); +} + +static void mtk_msi_ack_irq(struct irq_data *data) +{ + struct mtk_pcie_port *port = irq_data_get_irq_chip_data(data); + u32 hwirq = data->hwirq; + + writel(1 << hwirq, port->base + PCIE_IMSI_STATUS); +} + +static struct irq_chip mtk_msi_bottom_irq_chip = { + .name = "MTK MSI", + .irq_compose_msi_msg = mtk_compose_msi_msg, + .irq_ack = mtk_msi_ack_irq, +}; + +static int mtk_pcie_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct mtk_pcie_port *port = domain->host_data; + unsigned long bit; + + WARN_ON(nr_irqs != 1); + mutex_lock(&port->lock); + + bit = find_first_zero_bit(port->msi_irq_in_use, MTK_MSI_IRQS_NUM); + if (bit >= MTK_MSI_IRQS_NUM) { + mutex_unlock(&port->lock); + return -ENOSPC; + } + + __set_bit(bit, port->msi_irq_in_use); + + mutex_unlock(&port->lock); + + irq_domain_set_info(domain, virq, bit, &mtk_msi_bottom_irq_chip, + domain->host_data, handle_edge_irq, + NULL, NULL); + + return 0; +} + +static void mtk_pcie_irq_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct mtk_pcie_port *port = irq_data_get_irq_chip_data(d); + + mutex_lock(&port->lock); + + if (!test_bit(d->hwirq, port->msi_irq_in_use)) + dev_err(port->pcie->dev, "trying to free unused MSI#%lu\n", + d->hwirq); + else + __clear_bit(d->hwirq, port->msi_irq_in_use); + + mutex_unlock(&port->lock); + + irq_domain_free_irqs_parent(domain, virq, nr_irqs); +} + +static const struct irq_domain_ops msi_domain_ops = { + .alloc = mtk_pcie_irq_domain_alloc, + .free = mtk_pcie_irq_domain_free, +}; + +#define MTK_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_NO_AFFINITY) + +#define MTK_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_PCI_MSIX) + +static const struct msi_parent_ops mtk_msi_parent_ops = { + .required_flags = MTK_MSI_FLAGS_REQUIRED, + .supported_flags = MTK_MSI_FLAGS_SUPPORTED, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .chip_flags = MSI_CHIP_FLAG_SET_ACK, + .prefix = "MTK-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +static int mtk_pcie_allocate_msi_domains(struct mtk_pcie_port *port) +{ + mutex_init(&port->lock); + + struct irq_domain_info info = { + .fwnode = dev_fwnode(port->pcie->dev), + .ops = &msi_domain_ops, + .host_data = port, + .size = MTK_MSI_IRQS_NUM, + }; + + port->inner_domain = msi_create_parent_irq_domain(&info, &mtk_msi_parent_ops); + if (!port->inner_domain) { + dev_err(port->pcie->dev, "failed to create IRQ domain\n"); + return -ENOMEM; + } + + return 0; +} + +static void mtk_pcie_enable_msi(struct mtk_pcie_port *port) +{ + u32 val; + phys_addr_t msg_addr; + + msg_addr = port->phys_base + PCIE_MSI_VECTOR; + val = lower_32_bits(msg_addr); + writel(val, port->base + PCIE_IMSI_ADDR); + + val = readl(port->base + PCIE_INT_MASK); + val &= ~MSI_MASK; + writel(val, port->base + PCIE_INT_MASK); +} + +static void mtk_pcie_irq_teardown_port(struct mtk_pcie_port *port) +{ + irq_set_chained_handler_and_data(port->irq, NULL, NULL); + + if (port->irq_domain) + irq_domain_remove(port->irq_domain); + + if (IS_ENABLED(CONFIG_PCI_MSI)) { + if (port->inner_domain) + irq_domain_remove(port->inner_domain); + } + + irq_dispose_mapping(port->irq); +} + +static void mtk_pcie_irq_teardown(struct mtk_pcie *pcie) +{ + struct mtk_pcie_port *port, *tmp; + + list_for_each_entry_safe(port, tmp, &pcie->ports, list) + mtk_pcie_irq_teardown_port(port); +} + +static int mtk_pcie_intx_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq); + irq_set_chip_data(irq, domain->host_data); + + return 0; +} + +static const struct irq_domain_ops intx_domain_ops = { + .map = mtk_pcie_intx_map, +}; + +static int mtk_pcie_init_irq_domain(struct mtk_pcie_port *port, + struct device_node *node) +{ + struct device *dev = port->pcie->dev; + struct device_node *pcie_intc_node; + int ret; + + /* Setup INTx */ + pcie_intc_node = of_get_next_child(node, NULL); + if (!pcie_intc_node) { + dev_err(dev, "no PCIe Intc node found\n"); + return -ENODEV; + } + + port->irq_domain = irq_domain_create_linear(of_fwnode_handle(pcie_intc_node), PCI_NUM_INTX, + &intx_domain_ops, port); + of_node_put(pcie_intc_node); + if (!port->irq_domain) { + dev_err(dev, "failed to get INTx IRQ domain\n"); + return -ENODEV; + } + + if (IS_ENABLED(CONFIG_PCI_MSI)) { + ret = mtk_pcie_allocate_msi_domains(port); + if (ret) { + irq_domain_remove(port->irq_domain); + return ret; + } + } + + return 0; +} + +static void mtk_pcie_intr_handler(struct irq_desc *desc) +{ + struct mtk_pcie_port *port = irq_desc_get_handler_data(desc); + struct irq_chip *irqchip = irq_desc_get_chip(desc); + unsigned long status; + u32 bit = INTX_SHIFT; + + chained_irq_enter(irqchip, desc); + + status = readl(port->base + PCIE_INT_STATUS); + if (status & INTX_MASK) { + for_each_set_bit_from(bit, &status, PCI_NUM_INTX + INTX_SHIFT) { + /* Clear the INTx */ + writel(1 << bit, port->base + PCIE_INT_STATUS); + generic_handle_domain_irq(port->irq_domain, + bit - INTX_SHIFT); + } + } + + if (IS_ENABLED(CONFIG_PCI_MSI)) { + if (status & MSI_STATUS){ + unsigned long imsi_status; + + /* + * The interrupt status can be cleared even if the + * MSI status remains pending. As such, given the + * edge-triggered interrupt type, its status should + * be cleared before being dispatched to the + * handler of the underlying device. + */ + writel(MSI_STATUS, port->base + PCIE_INT_STATUS); + while ((imsi_status = readl(port->base + PCIE_IMSI_STATUS))) { + for_each_set_bit(bit, &imsi_status, MTK_MSI_IRQS_NUM) + generic_handle_domain_irq(port->inner_domain, bit); + } + } + } + + chained_irq_exit(irqchip, desc); +} + +static int mtk_pcie_setup_irq(struct mtk_pcie_port *port, + struct device_node *node) +{ + struct mtk_pcie *pcie = port->pcie; + struct device *dev = pcie->dev; + struct platform_device *pdev = to_platform_device(dev); + int err; + + err = mtk_pcie_init_irq_domain(port, node); + if (err) { + dev_err(dev, "failed to init PCIe IRQ domain\n"); + return err; + } + + if (of_property_present(dev->of_node, "interrupt-names")) + port->irq = platform_get_irq_byname(pdev, "pcie_irq"); + else + port->irq = platform_get_irq(pdev, port->slot); + + if (port->irq < 0) + return port->irq; + + irq_set_chained_handler_and_data(port->irq, + mtk_pcie_intr_handler, port); + + return 0; +} + +static int mtk_pcie_startup_port_v2(struct mtk_pcie_port *port) +{ + struct mtk_pcie *pcie = port->pcie; + struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); + struct resource *mem = NULL; + struct resource_entry *entry; + const struct mtk_pcie_soc *soc = port->pcie->soc; + u32 val; + int err; + + entry = resource_list_first_type(&host->windows, IORESOURCE_MEM); + if (entry) + mem = entry->res; + if (!mem) + return -EINVAL; + + /* MT7622 platforms need to enable LTSSM and ASPM from PCIe subsys */ + if (pcie->base) { + val = readl(pcie->base + PCIE_SYS_CFG_V2); + val |= PCIE_CSR_LTSSM_EN(port->slot) | + PCIE_CSR_ASPM_L1_EN(port->slot); + writel(val, pcie->base + PCIE_SYS_CFG_V2); + } else if (pcie->cfg) { + val = PCIE_CSR_LTSSM_EN(port->slot) | + PCIE_CSR_ASPM_L1_EN(port->slot); + regmap_update_bits(pcie->cfg, PCIE_SYS_CFG_V2, val, val); + } + + if (!(soc->quirks & MTK_PCIE_SKIP_RSTB)) { + /* Assert all reset signals */ + writel(0, port->base + PCIE_RST_CTRL); + + /* + * Enable PCIe link down reset, if link status changed from + * link up to link down, this will reset MAC control registers + * and configuration space. + */ + writel(PCIE_LINKDOWN_RST_EN, port->base + PCIE_RST_CTRL); + + msleep(PCIE_T_PVPERL_MS); + + /* De-assert PHY, PE, PIPE, MAC and configuration reset */ + val = readl(port->base + PCIE_RST_CTRL); + val |= PCIE_PHY_RSTB | PCIE_PERSTB | PCIE_PIPE_SRSTB | + PCIE_MAC_SRSTB | PCIE_CRSTB; + writel(val, port->base + PCIE_RST_CTRL); + } + + /* Set up vendor ID and class code */ + if (soc->quirks & MTK_PCIE_FIX_CLASS_ID) { + val = PCI_VENDOR_ID_MEDIATEK; + writew(val, port->base + PCIE_CONF_VEND_ID); + + val = PCI_CLASS_BRIDGE_PCI; + writew(val, port->base + PCIE_CONF_CLASS_ID); + } + + if (soc->quirks & MTK_PCIE_FIX_DEVICE_ID) + writew(soc->device_id, port->base + PCIE_CONF_DEVICE_ID); + + /* 100ms timeout value should be enough for Gen1/2 training */ + err = readl_poll_timeout(port->base + PCIE_LINK_STATUS_V2, val, + !!(val & PCIE_PORT_LINKUP_V2), 20, + 100 * USEC_PER_MSEC); + if (err) + return -ETIMEDOUT; + + /* Set INTx mask */ + val = readl(port->base + PCIE_INT_MASK); + val &= ~INTX_MASK; + writel(val, port->base + PCIE_INT_MASK); + + if (IS_ENABLED(CONFIG_PCI_MSI)) + mtk_pcie_enable_msi(port); + + /* Set AHB to PCIe translation windows */ + val = lower_32_bits(mem->start) | + AHB2PCIE_SIZE(fls(resource_size(mem))); + writel(val, port->base + PCIE_AHB_TRANS_BASE0_L); + + val = upper_32_bits(mem->start); + writel(val, port->base + PCIE_AHB_TRANS_BASE0_H); + + /* Set PCIe to AXI translation memory space.*/ + val = PCIE2AHB_SIZE | WIN_ENABLE; + writel(val, port->base + PCIE_AXI_WINDOW0); + + return 0; +} + +static int mtk_pcie_startup_port_en7528(struct mtk_pcie_port *port) +{ + struct mtk_pcie *pcie = port->pcie; + struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); + struct resource *mem = NULL; + struct resource_entry *entry; + u32 val, link_mask; + int err; + + entry = resource_list_first_type(&host->windows, IORESOURCE_MEM); + if (entry) + mem = entry->res; + if (!mem) + return -EINVAL; + + if (!pcie->cfg) { + dev_err(pcie->dev, "EN7528: pciecfg syscon not available\n"); + return -EINVAL; + } + + /* Assert all reset signals */ + writel(0, port->base + PCIE_RST_CTRL); + + /* + * Enable PCIe link down reset, if link status changed from link up to + * link down, this will reset MAC control registers and configuration + * space. + */ + writel(PCIE_LINKDOWN_RST_EN, port->base + PCIE_RST_CTRL); + + msleep(PCIE_T_PVPERL_MS); + + /* De-assert PHY, PE, PIPE, MAC and configuration reset */ + val = readl(port->base + PCIE_RST_CTRL); + val |= PCIE_PHY_RSTB | PCIE_PERSTB | PCIE_PIPE_SRSTB | + PCIE_MAC_SRSTB | PCIE_CRSTB; + writel(val, port->base + PCIE_RST_CTRL); + + writel(PCIE_CLASS_CODE | PCIE_REVISION_ID, + port->base + PCIE_CONF_REV_CLASS); + writel(EN7528_HOST_MODE, port->base); + + link_mask = (port->slot == 0) ? EN7528_RC0_LINKUP : EN7528_RC1_LINKUP; + + /* 100ms timeout value should be enough for Gen1/2 training */ + err = regmap_read_poll_timeout(pcie->cfg, EN7528_LINKUP_REG, val, + !!(val & link_mask), 20, + PCI_PM_D3COLD_WAIT * USEC_PER_MSEC); + if (err) { + dev_err(pcie->dev, "EN7528: port%d link timeout\n", port->slot); + return -ETIMEDOUT; + } + + /* Activate INTx interrupts */ + val = readl(port->base + PCIE_INT_MASK); + val &= ~INTX_MASK; + writel(val, port->base + PCIE_INT_MASK); + + if (IS_ENABLED(CONFIG_PCI_MSI)) + mtk_pcie_enable_msi(port); + + /* Set AHB to PCIe translation windows */ + val = lower_32_bits(mem->start) | + AHB2PCIE_SIZE(fls(resource_size(mem))); + writel(val, port->base + PCIE_AHB_TRANS_BASE0_L); + + val = upper_32_bits(mem->start); + writel(val, port->base + PCIE_AHB_TRANS_BASE0_H); + + writel(WIN_ENABLE, port->base + PCIE_AXI_WINDOW0); + + if (!IS_BUILTIN(CONFIG_PCIE_MEDIATEK)) + dev_info(pcie->dev, + "module not built-in, Gen2 unavailable even if supported\n"); + + return 0; +} + +/** + * mtk_pcie_retrain - retrain the root bridge link if needed + * @dev: The device, for use in logging + * @host: The host bridge which contains the link + * + * Due to what is likely a hardware bug, some devices (notably EcoNet) start up + * as Gen1, and must be retrained once after initial configuration in order to + * reach Gen2. + * + * These devices always self-identify as Gen2 capable, but sometimes the PHY is + * only capable of Gen1 operation, and sometimes the PCIe card (e.g. wifi) is + * only Gen1 capable. Therefore it is most convenient to retrain every port + * after startup. + */ +static int mtk_pcie_retrain(struct device *dev, struct pci_host_bridge *host) +{ + struct pci_dev *rp; + int ret = -ENOENT; + u16 lnksta = 0; + u32 speed; + + /* Should already have been warned about during startup_port */ + if (!IS_BUILTIN(CONFIG_PCIE_MEDIATEK)) + return 0; + + guard(rwsem_read)(&pci_bus_sem); + + for_each_pci_bridge(rp, host->bus) { + if (pci_pcie_type(rp) != PCI_EXP_TYPE_ROOT_PORT) + continue; + +#if IS_BUILTIN(CONFIG_PCIE_MEDIATEK) + ret = pcie_retrain_link(rp, true); +#endif + + if (ret) + return dev_err_probe(&rp->dev, ret, + "failed to retrain port\n"); + + pcie_capability_read_word(rp, PCI_EXP_LNKSTA, &lnksta); + speed = lnksta & PCI_EXP_LNKSTA_CLS; + + pci_info(rp, "link retrained, speed %s\n", + pci_speed_string(pcie_link_speed[speed])); + + } + + return 0; +} + +static void __iomem *mtk_pcie_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + struct mtk_pcie *pcie = bus->sysdata; + + writel(PCIE_CONF_ADDR(where, PCI_FUNC(devfn), PCI_SLOT(devfn), + bus->number), pcie->base + PCIE_CFG_ADDR); + + return pcie->base + PCIE_CFG_DATA + (where & 3); +} + +static struct pci_ops mtk_pcie_ops = { + .map_bus = mtk_pcie_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; + +static int mtk_pcie_startup_port(struct mtk_pcie_port *port) +{ + struct mtk_pcie *pcie = port->pcie; + u32 func = PCI_FUNC(port->slot); + u32 slot = PCI_SLOT(port->slot << 3); + u32 val; + int err; + + /* assert port PERST_N */ + val = readl(pcie->base + PCIE_SYS_CFG); + val |= PCIE_PORT_PERST(port->slot); + writel(val, pcie->base + PCIE_SYS_CFG); + + /* de-assert port PERST_N */ + val = readl(pcie->base + PCIE_SYS_CFG); + val &= ~PCIE_PORT_PERST(port->slot); + writel(val, pcie->base + PCIE_SYS_CFG); + + /* 100ms timeout value should be enough for Gen1/2 training */ + err = readl_poll_timeout(port->base + PCIE_LINK_STATUS, val, + !!(val & PCIE_PORT_LINKUP), 20, + 100 * USEC_PER_MSEC); + if (err) + return -ETIMEDOUT; + + /* enable interrupt */ + val = readl(pcie->base + PCIE_INT_ENABLE); + val |= PCIE_PORT_INT_EN(port->slot); + writel(val, pcie->base + PCIE_INT_ENABLE); + + /* map to all DDR region. We need to set it before cfg operation. */ + writel(PCIE_BAR_MAP_MAX | PCIE_BAR_ENABLE, + port->base + PCIE_BAR0_SETUP); + + /* configure class code and revision ID */ + writel(PCIE_CLASS_CODE | PCIE_REVISION_ID, port->base + PCIE_CLASS); + + /* configure FC credit */ + writel(PCIE_CONF_ADDR(PCIE_FC_CREDIT, func, slot, 0), + pcie->base + PCIE_CFG_ADDR); + val = readl(pcie->base + PCIE_CFG_DATA); + val &= ~PCIE_FC_CREDIT_MASK; + val |= PCIE_FC_CREDIT_VAL(0x806c); + writel(PCIE_CONF_ADDR(PCIE_FC_CREDIT, func, slot, 0), + pcie->base + PCIE_CFG_ADDR); + writel(val, pcie->base + PCIE_CFG_DATA); + + /* configure RC FTS number to 250 when it leaves L0s */ + writel(PCIE_CONF_ADDR(PCIE_FTS_NUM, func, slot, 0), + pcie->base + PCIE_CFG_ADDR); + val = readl(pcie->base + PCIE_CFG_DATA); + val &= ~PCIE_FTS_NUM_MASK; + val |= PCIE_FTS_NUM_L0(0x50); + writel(PCIE_CONF_ADDR(PCIE_FTS_NUM, func, slot, 0), + pcie->base + PCIE_CFG_ADDR); + writel(val, pcie->base + PCIE_CFG_DATA); + + return 0; +} + +static int mtk_pcie_startup_port_an7583(struct mtk_pcie_port *port) +{ + struct mtk_pcie *pcie = port->pcie; + struct device *dev = pcie->dev; + struct pci_host_bridge *host; + struct resource_entry *entry; + struct regmap *pbus_regmap; + resource_size_t addr; + u32 args[2], size; + + /* + * Configure PBus base address and base address mask to allow + * the hw to detect if a given address is accessible on PCIe + * controller. + */ + pbus_regmap = syscon_regmap_lookup_by_phandle_args(dev->of_node, + "mediatek,pbus-csr", + ARRAY_SIZE(args), + args); + if (IS_ERR(pbus_regmap)) + return PTR_ERR(pbus_regmap); + + host = pci_host_bridge_from_priv(pcie); + entry = resource_list_first_type(&host->windows, IORESOURCE_MEM); + if (!entry) + return -ENODEV; + + addr = entry->res->start - entry->offset; + regmap_write(pbus_regmap, args[0], lower_32_bits(addr)); + size = lower_32_bits(resource_size(entry->res)); + regmap_write(pbus_regmap, args[1], GENMASK(31, __fls(size))); + + return mtk_pcie_startup_port_v2(port); +} + +static int mtk_pcie_enable_port(struct mtk_pcie_port *port) +{ + struct mtk_pcie *pcie = port->pcie; + struct device *dev = pcie->dev; + int err; + + err = clk_prepare_enable(port->sys_ck); + if (err) { + dev_err(dev, "failed to enable sys_ck%d clock\n", port->slot); + return err; + } + + err = clk_prepare_enable(port->ahb_ck); + if (err) { + dev_err(dev, "failed to enable ahb_ck%d\n", port->slot); + goto err_ahb_clk; + } + + err = clk_prepare_enable(port->aux_ck); + if (err) { + dev_err(dev, "failed to enable aux_ck%d\n", port->slot); + goto err_aux_clk; + } + + err = clk_prepare_enable(port->axi_ck); + if (err) { + dev_err(dev, "failed to enable axi_ck%d\n", port->slot); + goto err_axi_clk; + } + + err = clk_prepare_enable(port->obff_ck); + if (err) { + dev_err(dev, "failed to enable obff_ck%d\n", port->slot); + goto err_obff_clk; + } + + err = clk_prepare_enable(port->pipe_ck); + if (err) { + dev_err(dev, "failed to enable pipe_ck%d\n", port->slot); + goto err_pipe_clk; + } + + reset_control_assert(port->reset); + reset_control_deassert(port->reset); + + err = phy_init(port->phy); + if (err) { + dev_err(dev, "failed to initialize port%d phy\n", port->slot); + goto err_phy_init; + } + + err = phy_power_on(port->phy); + if (err) { + dev_err(dev, "failed to power on port%d phy\n", port->slot); + goto err_phy_on; + } + + err = pcie->soc->startup(port); + if (err) { + dev_info(dev, "Port%d link down\n", port->slot); + goto err_soc_startup; + } + + return 0; + +err_soc_startup: + phy_power_off(port->phy); +err_phy_on: + phy_exit(port->phy); +err_phy_init: + clk_disable_unprepare(port->pipe_ck); +err_pipe_clk: + clk_disable_unprepare(port->obff_ck); +err_obff_clk: + clk_disable_unprepare(port->axi_ck); +err_axi_clk: + clk_disable_unprepare(port->aux_ck); +err_aux_clk: + clk_disable_unprepare(port->ahb_ck); +err_ahb_clk: + clk_disable_unprepare(port->sys_ck); + + return err; +} + +static int mtk_pcie_parse_port(struct mtk_pcie *pcie, + struct device_node *node, + int slot) +{ + struct mtk_pcie_port *port; + struct device *dev = pcie->dev; + struct platform_device *pdev = to_platform_device(dev); + struct resource *res; + char name[20]; + int err; + + port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); + if (!port) + return -ENOMEM; + + snprintf(name, sizeof(name), "port%d", slot); + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name); + if (!res) { + dev_err(dev, "failed to get port%d base\n", slot); + return -EINVAL; + } + + port->phys_base = res->start; + port->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(port->base)) { + dev_err(dev, "failed to map port%d base\n", slot); + return PTR_ERR(port->base); + } + + snprintf(name, sizeof(name), "sys_ck%d", slot); + port->sys_ck = devm_clk_get(dev, name); + if (IS_ERR(port->sys_ck)) { + dev_err(dev, "failed to get sys_ck%d clock\n", slot); + return PTR_ERR(port->sys_ck); + } + + /* sys_ck might be divided into the following parts in some chips */ + snprintf(name, sizeof(name), "ahb_ck%d", slot); + port->ahb_ck = devm_clk_get_optional(dev, name); + if (IS_ERR(port->ahb_ck)) + return PTR_ERR(port->ahb_ck); + + snprintf(name, sizeof(name), "axi_ck%d", slot); + port->axi_ck = devm_clk_get_optional(dev, name); + if (IS_ERR(port->axi_ck)) + return PTR_ERR(port->axi_ck); + + snprintf(name, sizeof(name), "aux_ck%d", slot); + port->aux_ck = devm_clk_get_optional(dev, name); + if (IS_ERR(port->aux_ck)) + return PTR_ERR(port->aux_ck); + + snprintf(name, sizeof(name), "obff_ck%d", slot); + port->obff_ck = devm_clk_get_optional(dev, name); + if (IS_ERR(port->obff_ck)) + return PTR_ERR(port->obff_ck); + + snprintf(name, sizeof(name), "pipe_ck%d", slot); + port->pipe_ck = devm_clk_get_optional(dev, name); + if (IS_ERR(port->pipe_ck)) + return PTR_ERR(port->pipe_ck); + + snprintf(name, sizeof(name), "pcie-rst%d", slot); + port->reset = devm_reset_control_get_optional_exclusive(dev, name); + if (PTR_ERR(port->reset) == -EPROBE_DEFER) + return PTR_ERR(port->reset); + + /* some platforms may use default PHY setting */ + snprintf(name, sizeof(name), "pcie-phy%d", slot); + port->phy = devm_phy_optional_get(dev, name); + if (IS_ERR(port->phy)) + return PTR_ERR(port->phy); + + port->slot = slot; + port->pcie = pcie; + + if (pcie->soc->setup_irq) { + err = pcie->soc->setup_irq(port, node); + if (err) + return err; + } + + INIT_LIST_HEAD(&port->list); + list_add_tail(&port->list, &pcie->ports); + + return 0; +} + +static int mtk_pcie_subsys_powerup(struct mtk_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct platform_device *pdev = to_platform_device(dev); + struct resource *regs; + struct device_node *cfg_node; + int err; + + /* get shared registers, which are optional */ + regs = platform_get_resource_byname(pdev, IORESOURCE_MEM, "subsys"); + if (regs) { + pcie->base = devm_ioremap_resource(dev, regs); + if (IS_ERR(pcie->base)) + return PTR_ERR(pcie->base); + } + + cfg_node = of_find_compatible_node(NULL, NULL, + "mediatek,generic-pciecfg"); + if (cfg_node) { + pcie->cfg = syscon_node_to_regmap(cfg_node); + of_node_put(cfg_node); + if (IS_ERR(pcie->cfg)) + return PTR_ERR(pcie->cfg); + } + + pcie->free_ck = devm_clk_get(dev, "free_ck"); + if (IS_ERR(pcie->free_ck)) { + if (PTR_ERR(pcie->free_ck) == -EPROBE_DEFER) + return -EPROBE_DEFER; + + pcie->free_ck = NULL; + } + + pm_runtime_enable(dev); + pm_runtime_get_sync(dev); + + /* enable top level clock */ + err = clk_prepare_enable(pcie->free_ck); + if (err) { + dev_err(dev, "failed to enable free_ck\n"); + goto err_free_ck; + } + + return 0; + +err_free_ck: + pm_runtime_put_sync(dev); + pm_runtime_disable(dev); + + return err; +} + +static int mtk_pcie_setup(struct mtk_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct device_node *node = dev->of_node; + struct mtk_pcie_port *port, *tmp; + int err, slot; + + slot = of_get_pci_domain_nr(dev->of_node); + if (slot < 0) { + for_each_available_child_of_node_scoped(node, child) { + err = of_pci_get_devfn(child); + if (err < 0) + return dev_err_probe(dev, err, "failed to get devfn\n"); + + slot = PCI_SLOT(err); + + err = mtk_pcie_parse_port(pcie, child, slot); + if (err) + return err; + } + } else { + err = mtk_pcie_parse_port(pcie, node, slot); + if (err) + return err; + } + + err = mtk_pcie_subsys_powerup(pcie); + if (err) + return err; + + /* enable each port, and then check link status */ + list_for_each_entry_safe(port, tmp, &pcie->ports, list) { + err = mtk_pcie_enable_port(port); + if (err) { + mtk_pcie_irq_teardown_port(port); + mtk_pcie_port_free(port); + } + } + + /* power down PCIe subsys if slots are all empty (link down) */ + if (list_empty(&pcie->ports)) + mtk_pcie_subsys_powerdown(pcie); + + return 0; +} + +static int mtk_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct mtk_pcie *pcie; + struct pci_host_bridge *host; + int err; + + host = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!host) + return -ENOMEM; + + pcie = pci_host_bridge_priv(host); + + pcie->dev = dev; + pcie->soc = of_device_get_match_data(dev); + platform_set_drvdata(pdev, pcie); + INIT_LIST_HEAD(&pcie->ports); + + err = mtk_pcie_setup(pcie); + if (err) + return err; + + host->ops = pcie->soc->ops; + host->sysdata = pcie; + host->msi_domain = !!(pcie->soc->quirks & MTK_PCIE_NO_MSI); + + err = pci_host_probe(host); + if (err) + goto put_resources; + + /* + * Ignore error because pci_host_probe() was already called, and in any + * case it is possible that the port will still work as Gen1. + */ + if (pcie->soc->quirks & MTK_PCIE_RETRAIN) + mtk_pcie_retrain(dev, host); + + return 0; + +put_resources: + if (!list_empty(&pcie->ports)) + mtk_pcie_put_resources(pcie); + + return err; +} + + +static void mtk_pcie_free_resources(struct mtk_pcie *pcie) +{ + struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); + struct list_head *windows = &host->windows; + + pci_free_resource_list(windows); +} + +static void mtk_pcie_remove(struct platform_device *pdev) +{ + struct mtk_pcie *pcie = platform_get_drvdata(pdev); + struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie); + + pci_lock_rescan_remove(); + pci_stop_root_bus(host->bus); + pci_remove_root_bus(host->bus); + pci_unlock_rescan_remove(); + mtk_pcie_free_resources(pcie); + + mtk_pcie_irq_teardown(pcie); + + mtk_pcie_put_resources(pcie); +} + +static int mtk_pcie_suspend_noirq(struct device *dev) +{ + struct mtk_pcie *pcie = dev_get_drvdata(dev); + struct mtk_pcie_port *port; + + if (list_empty(&pcie->ports)) + return 0; + + list_for_each_entry(port, &pcie->ports, list) { + clk_disable_unprepare(port->pipe_ck); + clk_disable_unprepare(port->obff_ck); + clk_disable_unprepare(port->axi_ck); + clk_disable_unprepare(port->aux_ck); + clk_disable_unprepare(port->ahb_ck); + clk_disable_unprepare(port->sys_ck); + phy_power_off(port->phy); + phy_exit(port->phy); + } + + clk_disable_unprepare(pcie->free_ck); + + return 0; +} + +static int mtk_pcie_resume_noirq(struct device *dev) +{ + struct mtk_pcie *pcie = dev_get_drvdata(dev); + struct mtk_pcie_port *port, *tmp; + int err; + + if (list_empty(&pcie->ports)) + return 0; + + clk_prepare_enable(pcie->free_ck); + + list_for_each_entry_safe(port, tmp, &pcie->ports, list) { + err = mtk_pcie_enable_port(port); + if (err) + mtk_pcie_port_free(port); + } + + /* In case of EP was removed while system suspend. */ + if (list_empty(&pcie->ports)) + clk_disable_unprepare(pcie->free_ck); + + return 0; +} + +static const struct dev_pm_ops mtk_pcie_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_pcie_suspend_noirq, + mtk_pcie_resume_noirq) +}; + +static const struct mtk_pcie_soc mtk_pcie_soc_v1 = { + .ops = &mtk_pcie_ops, + .startup = mtk_pcie_startup_port, + .quirks = MTK_PCIE_NO_MSI, +}; + +static const struct mtk_pcie_soc mtk_pcie_soc_mt2712 = { + .ops = &mtk_pcie_ops_v2, + .startup = mtk_pcie_startup_port_v2, + .setup_irq = mtk_pcie_setup_irq, +}; + +static const struct mtk_pcie_soc mtk_pcie_soc_mt7622 = { + .ops = &mtk_pcie_ops_v2, + .startup = mtk_pcie_startup_port_v2, + .setup_irq = mtk_pcie_setup_irq, + .quirks = MTK_PCIE_FIX_CLASS_ID, +}; + +static const struct mtk_pcie_soc mtk_pcie_soc_an7583 = { + .ops = &mtk_pcie_ops_v2, + .startup = mtk_pcie_startup_port_an7583, + .setup_irq = mtk_pcie_setup_irq, + .quirks = MTK_PCIE_FIX_CLASS_ID | MTK_PCIE_SKIP_RSTB, +}; + +static const struct mtk_pcie_soc mtk_pcie_soc_mt7629 = { + .device_id = PCI_DEVICE_ID_MEDIATEK_7629, + .ops = &mtk_pcie_ops_v2, + .startup = mtk_pcie_startup_port_v2, + .setup_irq = mtk_pcie_setup_irq, + .quirks = MTK_PCIE_FIX_CLASS_ID | MTK_PCIE_FIX_DEVICE_ID, +}; + +static const struct mtk_pcie_soc mtk_pcie_soc_en7528 = { + .ops = &mtk_pcie_ops_v2, + .startup = mtk_pcie_startup_port_en7528, + .setup_irq = mtk_pcie_setup_irq, + .quirks = MTK_PCIE_RETRAIN, +}; + +static const struct of_device_id mtk_pcie_ids[] = { + { .compatible = "airoha,an7583-pcie", .data = &mtk_pcie_soc_an7583 }, + { .compatible = "econet,en7528-pcie", .data = &mtk_pcie_soc_en7528 }, + { .compatible = "mediatek,mt2701-pcie", .data = &mtk_pcie_soc_v1 }, + { .compatible = "mediatek,mt7623-pcie", .data = &mtk_pcie_soc_v1 }, + { .compatible = "mediatek,mt2712-pcie", .data = &mtk_pcie_soc_mt2712 }, + { .compatible = "mediatek,mt7622-pcie", .data = &mtk_pcie_soc_mt7622 }, + { .compatible = "mediatek,mt7629-pcie", .data = &mtk_pcie_soc_mt7629 }, + {}, +}; +MODULE_DEVICE_TABLE(of, mtk_pcie_ids); + +static struct platform_driver mtk_pcie_driver = { + .probe = mtk_pcie_probe, + .remove = mtk_pcie_remove, + .driver = { + .name = "mtk-pcie", + .of_match_table = mtk_pcie_ids, + .suppress_bind_attrs = true, + .pm = &mtk_pcie_pm_ops, + }, +}; +module_platform_driver(mtk_pcie_driver); +MODULE_DESCRIPTION("MediaTek PCIe host controller driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pcie-mt7621.c b/drivers/pci/controller/pcie-mt7621.c new file mode 100644 index 000000000..01ead2f92 --- /dev/null +++ b/drivers/pci/controller/pcie-mt7621.c @@ -0,0 +1,548 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * BRIEF MODULE DESCRIPTION + * PCI init for Ralink RT2880 solution + * + * Copyright 2007 Ralink Inc. (bruce_chang@ralinktech.com.tw) + * + * May 2007 Bruce Chang + * Initial Release + * + * May 2009 Bruce Chang + * support RT2880/RT3883 PCIe + * + * May 2011 Bruce Chang + * support RT6855/MT7620 PCIe + */ + +#include <linux/bitops.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/gpio/consumer.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/pci.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/reset.h> +#include <linux/sys_soc.h> + +#include "../pci.h" + +/* MediaTek-specific configuration registers */ +#define PCIE_FTS_NUM 0x70c +#define PCIE_FTS_NUM_MASK GENMASK(15, 8) +#define PCIE_FTS_NUM_L0(x) (((x) & 0xff) << 8) + +/* Host-PCI bridge registers */ +#define RALINK_PCI_PCICFG_ADDR 0x0000 +#define RALINK_PCI_PCIMSK_ADDR 0x000c +#define RALINK_PCI_CONFIG_ADDR 0x0020 +#define RALINK_PCI_CONFIG_DATA 0x0024 +#define RALINK_PCI_MEMBASE 0x0028 +#define RALINK_PCI_IOBASE 0x002c + +/* PCIe RC control registers */ +#define RALINK_PCI_ID 0x0030 +#define RALINK_PCI_CLASS 0x0034 +#define RALINK_PCI_SUBID 0x0038 +#define RALINK_PCI_STATUS 0x0050 + +/* Some definition values */ +#define PCIE_REVISION_ID BIT(0) +#define PCIE_CLASS_CODE (0x60400 << 8) +#define PCIE_BAR_MAP_MAX GENMASK(30, 16) +#define PCIE_BAR_ENABLE BIT(0) +#define PCIE_PORT_INT_EN(x) BIT(20 + (x)) +#define PCIE_PORT_LINKUP BIT(0) +#define PCIE_PORT_CNT 3 + +#define INIT_PORTS_DELAY_MS 100 +#define PERST_DELAY_MS 100 + +/** + * struct mt7621_pcie_port - PCIe port information + * @base: I/O mapped register base + * @list: port list + * @pcie: pointer to PCIe host info + * @clk: pointer to the port clock gate + * @phy: pointer to PHY control block + * @pcie_rst: pointer to port reset control + * @gpio_rst: gpio reset + * @slot: port slot + * @enabled: indicates if port is enabled + */ +struct mt7621_pcie_port { + void __iomem *base; + struct list_head list; + struct mt7621_pcie *pcie; + struct clk *clk; + struct phy *phy; + struct reset_control *pcie_rst; + struct gpio_desc *gpio_rst; + u32 slot; + bool enabled; +}; + +/** + * struct mt7621_pcie - PCIe host information + * @base: IO Mapped Register Base + * @dev: Pointer to PCIe device + * @ports: pointer to PCIe port information + * @resets_inverted: depends on chip revision + * reset lines are inverted. + */ +struct mt7621_pcie { + struct device *dev; + void __iomem *base; + struct list_head ports; + bool resets_inverted; +}; + +static inline u32 pcie_read(struct mt7621_pcie *pcie, u32 reg) +{ + return readl_relaxed(pcie->base + reg); +} + +static inline void pcie_write(struct mt7621_pcie *pcie, u32 val, u32 reg) +{ + writel_relaxed(val, pcie->base + reg); +} + +static inline u32 pcie_port_read(struct mt7621_pcie_port *port, u32 reg) +{ + return readl_relaxed(port->base + reg); +} + +static inline void pcie_port_write(struct mt7621_pcie_port *port, + u32 val, u32 reg) +{ + writel_relaxed(val, port->base + reg); +} + +static void __iomem *mt7621_pcie_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + struct mt7621_pcie *pcie = bus->sysdata; + u32 address = PCI_CONF1_EXT_ADDRESS(bus->number, PCI_SLOT(devfn), + PCI_FUNC(devfn), where); + + writel_relaxed(address, pcie->base + RALINK_PCI_CONFIG_ADDR); + + return pcie->base + RALINK_PCI_CONFIG_DATA + (where & 3); +} + +static struct pci_ops mt7621_pcie_ops = { + .map_bus = mt7621_pcie_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; + +static u32 read_config(struct mt7621_pcie *pcie, unsigned int dev, u32 reg) +{ + u32 address = PCI_CONF1_EXT_ADDRESS(0, dev, 0, reg); + + pcie_write(pcie, address, RALINK_PCI_CONFIG_ADDR); + return pcie_read(pcie, RALINK_PCI_CONFIG_DATA); +} + +static void write_config(struct mt7621_pcie *pcie, unsigned int dev, + u32 reg, u32 val) +{ + u32 address = PCI_CONF1_EXT_ADDRESS(0, dev, 0, reg); + + pcie_write(pcie, address, RALINK_PCI_CONFIG_ADDR); + pcie_write(pcie, val, RALINK_PCI_CONFIG_DATA); +} + +static inline void mt7621_rst_gpio_pcie_assert(struct mt7621_pcie_port *port) +{ + if (port->gpio_rst) + gpiod_set_value(port->gpio_rst, 1); +} + +static inline void mt7621_rst_gpio_pcie_deassert(struct mt7621_pcie_port *port) +{ + if (port->gpio_rst) + gpiod_set_value(port->gpio_rst, 0); +} + +static inline bool mt7621_pcie_port_is_linkup(struct mt7621_pcie_port *port) +{ + return (pcie_port_read(port, RALINK_PCI_STATUS) & PCIE_PORT_LINKUP) != 0; +} + +static inline void mt7621_control_assert(struct mt7621_pcie_port *port) +{ + struct mt7621_pcie *pcie = port->pcie; + + if (pcie->resets_inverted) + reset_control_assert(port->pcie_rst); + else + reset_control_deassert(port->pcie_rst); +} + +static inline void mt7621_control_deassert(struct mt7621_pcie_port *port) +{ + struct mt7621_pcie *pcie = port->pcie; + + if (pcie->resets_inverted) + reset_control_deassert(port->pcie_rst); + else + reset_control_assert(port->pcie_rst); +} + +static int mt7621_pcie_parse_port(struct mt7621_pcie *pcie, + struct device_node *node, + int slot) +{ + struct mt7621_pcie_port *port; + struct device *dev = pcie->dev; + struct platform_device *pdev = to_platform_device(dev); + char name[11]; + int err; + + port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); + if (!port) + return -ENOMEM; + + port->base = devm_platform_ioremap_resource(pdev, slot + 1); + if (IS_ERR(port->base)) + return PTR_ERR(port->base); + + port->clk = devm_get_clk_from_child(dev, node, NULL); + if (IS_ERR(port->clk)) { + dev_err(dev, "failed to get pcie%d clock\n", slot); + return PTR_ERR(port->clk); + } + + port->pcie_rst = of_reset_control_get_exclusive(node, NULL); + if (PTR_ERR(port->pcie_rst) == -EPROBE_DEFER) { + dev_err(dev, "failed to get pcie%d reset control\n", slot); + return PTR_ERR(port->pcie_rst); + } + + snprintf(name, sizeof(name), "pcie-phy%d", slot); + port->phy = devm_of_phy_get(dev, node, name); + if (IS_ERR(port->phy)) { + dev_err(dev, "failed to get pcie-phy%d\n", slot); + err = PTR_ERR(port->phy); + goto remove_reset; + } + + port->gpio_rst = devm_gpiod_get_index_optional(dev, "reset", slot, + GPIOD_OUT_LOW); + if (IS_ERR(port->gpio_rst)) { + dev_err(dev, "failed to get GPIO for PCIe%d\n", slot); + err = PTR_ERR(port->gpio_rst); + goto remove_reset; + } + + port->slot = slot; + port->pcie = pcie; + + INIT_LIST_HEAD(&port->list); + list_add_tail(&port->list, &pcie->ports); + + return 0; + +remove_reset: + reset_control_put(port->pcie_rst); + return err; +} + +static int mt7621_pcie_parse_dt(struct mt7621_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct platform_device *pdev = to_platform_device(dev); + struct device_node *node = dev->of_node; + int err; + + pcie->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(pcie->base)) + return PTR_ERR(pcie->base); + + for_each_available_child_of_node_scoped(node, child) { + int slot; + + err = of_pci_get_devfn(child); + if (err < 0) + return dev_err_probe(dev, err, "failed to parse devfn\n"); + + slot = PCI_SLOT(err); + + err = mt7621_pcie_parse_port(pcie, child, slot); + if (err) + return err; + } + + return 0; +} + +static int mt7621_pcie_init_port(struct mt7621_pcie_port *port) +{ + struct mt7621_pcie *pcie = port->pcie; + struct device *dev = pcie->dev; + u32 slot = port->slot; + int err; + + err = phy_init(port->phy); + if (err) { + dev_err(dev, "failed to initialize port%d phy\n", slot); + return err; + } + + err = phy_power_on(port->phy); + if (err) { + dev_err(dev, "failed to power on port%d phy\n", slot); + phy_exit(port->phy); + return err; + } + + port->enabled = true; + + return 0; +} + +static void mt7621_pcie_reset_assert(struct mt7621_pcie *pcie) +{ + struct mt7621_pcie_port *port; + + list_for_each_entry(port, &pcie->ports, list) { + /* PCIe RC reset assert */ + mt7621_control_assert(port); + + /* PCIe EP reset assert */ + mt7621_rst_gpio_pcie_assert(port); + } + + msleep(PERST_DELAY_MS); +} + +static void mt7621_pcie_reset_rc_deassert(struct mt7621_pcie *pcie) +{ + struct mt7621_pcie_port *port; + + list_for_each_entry(port, &pcie->ports, list) + mt7621_control_deassert(port); +} + +static void mt7621_pcie_reset_ep_deassert(struct mt7621_pcie *pcie) +{ + struct mt7621_pcie_port *port; + + list_for_each_entry(port, &pcie->ports, list) + mt7621_rst_gpio_pcie_deassert(port); + + msleep(PERST_DELAY_MS); +} + +static int mt7621_pcie_init_ports(struct mt7621_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct mt7621_pcie_port *port, *tmp; + u8 num_disabled = 0; + int err; + + mt7621_pcie_reset_assert(pcie); + mt7621_pcie_reset_rc_deassert(pcie); + + list_for_each_entry_safe(port, tmp, &pcie->ports, list) { + u32 slot = port->slot; + + if (slot == 1) { + port->enabled = true; + continue; + } + + err = mt7621_pcie_init_port(port); + if (err) { + dev_err(dev, "initializing port %d failed\n", slot); + list_del(&port->list); + } + } + + msleep(INIT_PORTS_DELAY_MS); + mt7621_pcie_reset_ep_deassert(pcie); + + tmp = NULL; + list_for_each_entry(port, &pcie->ports, list) { + u32 slot = port->slot; + + if (!mt7621_pcie_port_is_linkup(port)) { + dev_info(dev, "pcie%d no card, disable it (RST & CLK)\n", + slot); + mt7621_control_assert(port); + port->enabled = false; + num_disabled++; + + if (slot == 0) { + tmp = port; + continue; + } + + if (slot == 1 && tmp && !tmp->enabled) + phy_power_off(tmp->phy); + } + } + + return (num_disabled != PCIE_PORT_CNT) ? 0 : -ENODEV; +} + +static void mt7621_pcie_enable_port(struct mt7621_pcie_port *port) +{ + struct mt7621_pcie *pcie = port->pcie; + u32 slot = port->slot; + u32 val; + + /* enable pcie interrupt */ + val = pcie_read(pcie, RALINK_PCI_PCIMSK_ADDR); + val |= PCIE_PORT_INT_EN(slot); + pcie_write(pcie, val, RALINK_PCI_PCIMSK_ADDR); + + /* map 2G DDR region */ + pcie_port_write(port, PCIE_BAR_MAP_MAX | PCIE_BAR_ENABLE, + PCI_BASE_ADDRESS_0); + + /* configure class code and revision ID */ + pcie_port_write(port, PCIE_CLASS_CODE | PCIE_REVISION_ID, + RALINK_PCI_CLASS); + + /* configure RC FTS number to 250 when it leaves L0s */ + val = read_config(pcie, slot, PCIE_FTS_NUM); + val &= ~PCIE_FTS_NUM_MASK; + val |= PCIE_FTS_NUM_L0(0x50); + write_config(pcie, slot, PCIE_FTS_NUM, val); +} + +static int mt7621_pcie_enable_ports(struct pci_host_bridge *host) +{ + struct mt7621_pcie *pcie = pci_host_bridge_priv(host); + struct device *dev = pcie->dev; + struct mt7621_pcie_port *port; + struct resource_entry *entry; + int err; + + entry = resource_list_first_type(&host->windows, IORESOURCE_IO); + if (!entry) { + dev_err(dev, "cannot get io resource\n"); + return -EINVAL; + } + + /* Setup MEMWIN and IOWIN */ + pcie_write(pcie, 0xffffffff, RALINK_PCI_MEMBASE); + pcie_write(pcie, entry->res->start - entry->offset, RALINK_PCI_IOBASE); + + list_for_each_entry(port, &pcie->ports, list) { + if (port->enabled) { + err = clk_prepare_enable(port->clk); + if (err) { + dev_err(dev, "enabling clk pcie%d\n", + port->slot); + return err; + } + + mt7621_pcie_enable_port(port); + dev_info(dev, "PCIE%d enabled\n", port->slot); + } + } + + return 0; +} + +static int mt7621_pcie_register_host(struct pci_host_bridge *host) +{ + struct mt7621_pcie *pcie = pci_host_bridge_priv(host); + + host->ops = &mt7621_pcie_ops; + host->sysdata = pcie; + return pci_host_probe(host); +} + +static const struct soc_device_attribute mt7621_pcie_quirks_match[] = { + { .soc_id = "mt7621", .revision = "E2" }, + { /* sentinel */ } +}; + +static int mt7621_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + const struct soc_device_attribute *attr; + struct mt7621_pcie_port *port; + struct mt7621_pcie *pcie; + struct pci_host_bridge *bridge; + int err; + + if (!dev->of_node) + return -ENODEV; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!bridge) + return -ENOMEM; + + pcie = pci_host_bridge_priv(bridge); + pcie->dev = dev; + platform_set_drvdata(pdev, pcie); + INIT_LIST_HEAD(&pcie->ports); + + attr = soc_device_match(mt7621_pcie_quirks_match); + if (attr) + pcie->resets_inverted = true; + + err = mt7621_pcie_parse_dt(pcie); + if (err) { + dev_err(dev, "parsing DT failed\n"); + return err; + } + + err = mt7621_pcie_init_ports(pcie); + if (err) { + dev_err(dev, "nothing connected in virtual bridges\n"); + return 0; + } + + err = mt7621_pcie_enable_ports(bridge); + if (err) { + dev_err(dev, "error enabling pcie ports\n"); + goto remove_resets; + } + + return mt7621_pcie_register_host(bridge); + +remove_resets: + list_for_each_entry(port, &pcie->ports, list) + reset_control_put(port->pcie_rst); + + return err; +} + +static void mt7621_pcie_remove(struct platform_device *pdev) +{ + struct mt7621_pcie *pcie = platform_get_drvdata(pdev); + struct mt7621_pcie_port *port; + + list_for_each_entry(port, &pcie->ports, list) + reset_control_put(port->pcie_rst); +} + +static const struct of_device_id mt7621_pcie_ids[] = { + { .compatible = "mediatek,mt7621-pci" }, + {}, +}; +MODULE_DEVICE_TABLE(of, mt7621_pcie_ids); + +static struct platform_driver mt7621_pcie_driver = { + .probe = mt7621_pcie_probe, + .remove = mt7621_pcie_remove, + .driver = { + .name = "mt7621-pci", + .of_match_table = mt7621_pcie_ids, + }, +}; +builtin_platform_driver(mt7621_pcie_driver); + +MODULE_DESCRIPTION("MediaTek MT7621 PCIe host controller driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pcie-rcar-ep.c b/drivers/pci/controller/pcie-rcar-ep.c new file mode 100644 index 000000000..c2da8ac1f --- /dev/null +++ b/drivers/pci/controller/pcie-rcar-ep.c @@ -0,0 +1,561 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe endpoint driver for Renesas R-Car SoCs + * Copyright (c) 2020 Renesas Electronics Europe GmbH + * + * Author: Lad Prabhakar <prabhakar.mahadev-lad.rj@bp.renesas.com> + */ + +#include <linux/delay.h> +#include <linux/of_address.h> +#include <linux/of_platform.h> +#include <linux/pci.h> +#include <linux/pci-epc.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> + +#include "pcie-rcar.h" + +#define RCAR_EPC_MAX_FUNCTIONS 1 + +/* Structure representing the PCIe interface */ +struct rcar_pcie_endpoint { + struct rcar_pcie pcie; + phys_addr_t *ob_mapped_addr; + struct pci_epc_mem_window *ob_window; + u8 max_functions; + unsigned int bar_to_atu[MAX_NR_INBOUND_MAPS]; + unsigned long *ib_window_map; + u32 num_ib_windows; + u32 num_ob_windows; +}; + +static void rcar_pcie_ep_hw_init(struct rcar_pcie *pcie) +{ + u32 val; + + rcar_pci_write_reg(pcie, 0, PCIETCTLR); + + /* Set endpoint mode */ + rcar_pci_write_reg(pcie, 0, PCIEMSR); + + /* Initialize default capabilities. */ + rcar_rmw32(pcie, REXPCAP(0), 0xff, PCI_CAP_ID_EXP); + rcar_rmw32(pcie, REXPCAP(PCI_EXP_FLAGS), + PCI_EXP_FLAGS_TYPE, PCI_EXP_TYPE_ENDPOINT << 4); + rcar_rmw32(pcie, RCONF(PCI_HEADER_TYPE), PCI_HEADER_TYPE_MASK, + PCI_HEADER_TYPE_NORMAL); + + /* Write out the physical slot number = 0 */ + rcar_rmw32(pcie, REXPCAP(PCI_EXP_SLTCAP), PCI_EXP_SLTCAP_PSN, 0); + + val = rcar_pci_read_reg(pcie, EXPCAP(1)); + /* device supports fixed 128 bytes MPSS */ + val &= ~GENMASK(2, 0); + rcar_pci_write_reg(pcie, val, EXPCAP(1)); + + val = rcar_pci_read_reg(pcie, EXPCAP(2)); + /* read requests size 128 bytes */ + val &= ~GENMASK(14, 12); + /* payload size 128 bytes */ + val &= ~GENMASK(7, 5); + rcar_pci_write_reg(pcie, val, EXPCAP(2)); + + /* Set target link speed to 5.0 GT/s */ + rcar_rmw32(pcie, EXPCAP(12), PCI_EXP_LNKSTA_CLS, + PCI_EXP_LNKSTA_CLS_5_0GB); + + /* Set the completion timer timeout to the maximum 50ms. */ + rcar_rmw32(pcie, TLCTLR + 1, 0x3f, 50); + + /* Terminate list of capabilities (Next Capability Offset=0) */ + rcar_rmw32(pcie, RVCCAP(0), 0xfff00000, 0); + + /* flush modifications */ + wmb(); +} + +static int rcar_pcie_ep_get_window(struct rcar_pcie_endpoint *ep, + phys_addr_t addr) +{ + int i; + + for (i = 0; i < ep->num_ob_windows; i++) + if (ep->ob_window[i].phys_base == addr) + return i; + + return -EINVAL; +} + +static int rcar_pcie_parse_outbound_ranges(struct rcar_pcie_endpoint *ep, + struct platform_device *pdev) +{ + struct rcar_pcie *pcie = &ep->pcie; + char outbound_name[10]; + struct resource *res; + unsigned int i = 0; + + ep->num_ob_windows = 0; + for (i = 0; i < RCAR_PCI_MAX_RESOURCES; i++) { + sprintf(outbound_name, "memory%u", i); + res = platform_get_resource_byname(pdev, + IORESOURCE_MEM, + outbound_name); + if (!res) { + dev_err(pcie->dev, "missing outbound window %u\n", i); + return -EINVAL; + } + if (!devm_request_mem_region(&pdev->dev, res->start, + resource_size(res), + res->name)) { + dev_err(pcie->dev, "Cannot request memory region %s.\n", + outbound_name); + return -EIO; + } + + ep->ob_window[i].phys_base = res->start; + ep->ob_window[i].size = resource_size(res); + /* controller doesn't support multiple allocation + * from same window, so set page_size to window size + */ + ep->ob_window[i].page_size = resource_size(res); + } + ep->num_ob_windows = i; + + return 0; +} + +static int rcar_pcie_ep_get_pdata(struct rcar_pcie_endpoint *ep, + struct platform_device *pdev) +{ + struct rcar_pcie *pcie = &ep->pcie; + struct pci_epc_mem_window *window; + struct device *dev = pcie->dev; + struct resource res; + int err; + + err = of_address_to_resource(dev->of_node, 0, &res); + if (err) + return err; + pcie->base = devm_ioremap_resource(dev, &res); + if (IS_ERR(pcie->base)) + return PTR_ERR(pcie->base); + + ep->ob_window = devm_kcalloc(dev, RCAR_PCI_MAX_RESOURCES, + sizeof(*window), GFP_KERNEL); + if (!ep->ob_window) + return -ENOMEM; + + rcar_pcie_parse_outbound_ranges(ep, pdev); + + err = of_property_read_u8(dev->of_node, "max-functions", + &ep->max_functions); + if (err < 0 || ep->max_functions > RCAR_EPC_MAX_FUNCTIONS) + ep->max_functions = RCAR_EPC_MAX_FUNCTIONS; + + return 0; +} + +static int rcar_pcie_ep_write_header(struct pci_epc *epc, u8 fn, u8 vfn, + struct pci_epf_header *hdr) +{ + struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc); + struct rcar_pcie *pcie = &ep->pcie; + u32 val; + + if (!fn) + val = hdr->vendorid; + else + val = rcar_pci_read_reg(pcie, IDSETR0); + val |= hdr->deviceid << 16; + rcar_pci_write_reg(pcie, val, IDSETR0); + + val = hdr->revid; + val |= hdr->progif_code << 8; + val |= hdr->subclass_code << 16; + val |= hdr->baseclass_code << 24; + rcar_pci_write_reg(pcie, val, IDSETR1); + + if (!fn) + val = hdr->subsys_vendor_id; + else + val = rcar_pci_read_reg(pcie, SUBIDSETR); + val |= hdr->subsys_id << 16; + rcar_pci_write_reg(pcie, val, SUBIDSETR); + + if (hdr->interrupt_pin > PCI_INTERRUPT_INTA) + return -EINVAL; + val = rcar_pci_read_reg(pcie, PCICONF(15)); + val |= (hdr->interrupt_pin << 8); + rcar_pci_write_reg(pcie, val, PCICONF(15)); + + return 0; +} + +static int rcar_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + struct pci_epf_bar *epf_bar) +{ + int flags = epf_bar->flags | LAR_ENABLE | LAM_64BIT; + struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc); + u64 size = 1ULL << fls64(epf_bar->size - 1); + dma_addr_t cpu_addr = epf_bar->phys_addr; + enum pci_barno bar = epf_bar->barno; + struct rcar_pcie *pcie = &ep->pcie; + u32 mask; + int idx; + int err; + + idx = find_first_zero_bit(ep->ib_window_map, ep->num_ib_windows); + if (idx >= ep->num_ib_windows) { + dev_err(pcie->dev, "no free inbound window\n"); + return -EINVAL; + } + + if ((flags & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_IO) + flags |= IO_SPACE; + + ep->bar_to_atu[bar] = idx; + /* use 64-bit BARs */ + set_bit(idx, ep->ib_window_map); + set_bit(idx + 1, ep->ib_window_map); + + if (cpu_addr > 0) { + unsigned long nr_zeros = __ffs64(cpu_addr); + u64 alignment = 1ULL << nr_zeros; + + size = min(size, alignment); + } + + size = min(size, 1ULL << 32); + + mask = roundup_pow_of_two(size) - 1; + mask &= ~0xf; + + rcar_pcie_set_inbound(pcie, cpu_addr, + 0x0, mask | flags, idx, false); + + err = rcar_pcie_wait_for_phyrdy(pcie); + if (err) { + dev_err(pcie->dev, "phy not ready\n"); + return -EINVAL; + } + + return 0; +} + +static void rcar_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, u8 vfn, + struct pci_epf_bar *epf_bar) +{ + struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc); + enum pci_barno bar = epf_bar->barno; + u32 atu_index = ep->bar_to_atu[bar]; + + rcar_pcie_set_inbound(&ep->pcie, 0x0, 0x0, 0x0, bar, false); + + clear_bit(atu_index, ep->ib_window_map); + clear_bit(atu_index + 1, ep->ib_window_map); +} + +static int rcar_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, u8 vfn, u8 nr_irqs) +{ + struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc); + struct rcar_pcie *pcie = &ep->pcie; + u8 mmc = order_base_2(nr_irqs); + u32 flags; + + flags = rcar_pci_read_reg(pcie, MSICAP(fn)); + flags |= mmc << MSICAP0_MMESCAP_OFFSET; + rcar_pci_write_reg(pcie, flags, MSICAP(fn)); + + return 0; +} + +static int rcar_pcie_ep_get_msi(struct pci_epc *epc, u8 fn, u8 vfn) +{ + struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc); + struct rcar_pcie *pcie = &ep->pcie; + u32 flags; + + flags = rcar_pci_read_reg(pcie, MSICAP(fn)); + if (!(flags & MSICAP0_MSIE)) + return -EINVAL; + + return 1 << ((flags & MSICAP0_MMESE_MASK) >> MSICAP0_MMESE_OFFSET); +} + +static int rcar_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, u8 vfn, + phys_addr_t addr, u64 pci_addr, size_t size) +{ + struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc); + struct rcar_pcie *pcie = &ep->pcie; + struct resource_entry win; + struct resource res; + int window; + int err; + + /* check if we have a link. */ + err = rcar_pcie_wait_for_dl(pcie); + if (err) { + dev_err(pcie->dev, "link not up\n"); + return err; + } + + window = rcar_pcie_ep_get_window(ep, addr); + if (window < 0) { + dev_err(pcie->dev, "failed to get corresponding window\n"); + return -EINVAL; + } + + memset(&win, 0x0, sizeof(win)); + memset(&res, 0x0, sizeof(res)); + res.start = pci_addr; + res.end = pci_addr + size - 1; + res.flags = IORESOURCE_MEM; + win.res = &res; + + rcar_pcie_set_outbound(pcie, window, &win); + + ep->ob_mapped_addr[window] = addr; + + return 0; +} + +static void rcar_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn, u8 vfn, + phys_addr_t addr) +{ + struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc); + struct resource_entry win; + struct resource res; + int idx; + + for (idx = 0; idx < ep->num_ob_windows; idx++) + if (ep->ob_mapped_addr[idx] == addr) + break; + + if (idx >= ep->num_ob_windows) + return; + + memset(&win, 0x0, sizeof(win)); + memset(&res, 0x0, sizeof(res)); + win.res = &res; + rcar_pcie_set_outbound(&ep->pcie, idx, &win); + + ep->ob_mapped_addr[idx] = 0; +} + +static int rcar_pcie_ep_assert_intx(struct rcar_pcie_endpoint *ep, + u8 fn, u8 intx) +{ + struct rcar_pcie *pcie = &ep->pcie; + u32 val; + + val = rcar_pci_read_reg(pcie, PCIEMSITXR); + if ((val & PCI_MSI_FLAGS_ENABLE)) { + dev_err(pcie->dev, "MSI is enabled, cannot assert INTx\n"); + return -EINVAL; + } + + val = rcar_pci_read_reg(pcie, PCICONF(1)); + if ((val & INTDIS)) { + dev_err(pcie->dev, "INTx message transmission is disabled\n"); + return -EINVAL; + } + + val = rcar_pci_read_reg(pcie, PCIEINTXR); + if ((val & ASTINTX)) { + dev_err(pcie->dev, "INTx is already asserted\n"); + return -EINVAL; + } + + val |= ASTINTX; + rcar_pci_write_reg(pcie, val, PCIEINTXR); + usleep_range(1000, 1001); + val = rcar_pci_read_reg(pcie, PCIEINTXR); + val &= ~ASTINTX; + rcar_pci_write_reg(pcie, val, PCIEINTXR); + + return 0; +} + +static int rcar_pcie_ep_assert_msi(struct rcar_pcie *pcie, + u8 fn, u8 interrupt_num) +{ + u16 msi_count; + u32 val; + + /* Check MSI enable bit */ + val = rcar_pci_read_reg(pcie, MSICAP(fn)); + if (!(val & MSICAP0_MSIE)) + return -EINVAL; + + /* Get MSI numbers from MME */ + msi_count = ((val & MSICAP0_MMESE_MASK) >> MSICAP0_MMESE_OFFSET); + msi_count = 1 << msi_count; + + if (!interrupt_num || interrupt_num > msi_count) + return -EINVAL; + + val = rcar_pci_read_reg(pcie, PCIEMSITXR); + rcar_pci_write_reg(pcie, val | (interrupt_num - 1), PCIEMSITXR); + + return 0; +} + +static int rcar_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn, u8 vfn, + unsigned int type, u16 interrupt_num) +{ + struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc); + + switch (type) { + case PCI_IRQ_INTX: + return rcar_pcie_ep_assert_intx(ep, fn, 0); + + case PCI_IRQ_MSI: + return rcar_pcie_ep_assert_msi(&ep->pcie, fn, interrupt_num); + + default: + return -EINVAL; + } +} + +static int rcar_pcie_ep_start(struct pci_epc *epc) +{ + struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc); + + rcar_pci_write_reg(&ep->pcie, MACCTLR_INIT_VAL, MACCTLR); + rcar_pci_write_reg(&ep->pcie, CFINIT, PCIETCTLR); + + return 0; +} + +static void rcar_pcie_ep_stop(struct pci_epc *epc) +{ + struct rcar_pcie_endpoint *ep = epc_get_drvdata(epc); + + rcar_pci_write_reg(&ep->pcie, 0, PCIETCTLR); +} + +static const struct pci_epc_features rcar_pcie_epc_features = { + .msi_capable = true, + /* use 64-bit BARs so mark BAR[1,3,5] as reserved */ + .bar[BAR_0] = { .type = BAR_FIXED, .fixed_size = 128, + .only_64bit = true, }, + .bar[BAR_2] = { .type = BAR_FIXED, .fixed_size = 256, + .only_64bit = true, }, + .bar[BAR_4] = { .type = BAR_FIXED, .fixed_size = 256, + .only_64bit = true, }, +}; + +static const struct pci_epc_features* +rcar_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no) +{ + return &rcar_pcie_epc_features; +} + +static const struct pci_epc_ops rcar_pcie_epc_ops = { + .write_header = rcar_pcie_ep_write_header, + .set_bar = rcar_pcie_ep_set_bar, + .clear_bar = rcar_pcie_ep_clear_bar, + .set_msi = rcar_pcie_ep_set_msi, + .get_msi = rcar_pcie_ep_get_msi, + .map_addr = rcar_pcie_ep_map_addr, + .unmap_addr = rcar_pcie_ep_unmap_addr, + .raise_irq = rcar_pcie_ep_raise_irq, + .start = rcar_pcie_ep_start, + .stop = rcar_pcie_ep_stop, + .get_features = rcar_pcie_ep_get_features, +}; + +static const struct of_device_id rcar_pcie_ep_of_match[] = { + { .compatible = "renesas,r8a774c0-pcie-ep", }, + { .compatible = "renesas,rcar-gen3-pcie-ep" }, + { }, +}; + +static int rcar_pcie_ep_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct rcar_pcie_endpoint *ep; + struct rcar_pcie *pcie; + struct pci_epc *epc; + int err; + + ep = devm_kzalloc(dev, sizeof(*ep), GFP_KERNEL); + if (!ep) + return -ENOMEM; + + pcie = &ep->pcie; + pcie->dev = dev; + + pm_runtime_enable(dev); + err = pm_runtime_resume_and_get(dev); + if (err < 0) { + dev_err(dev, "pm_runtime_resume_and_get failed\n"); + goto err_pm_disable; + } + + err = rcar_pcie_ep_get_pdata(ep, pdev); + if (err < 0) { + dev_err(dev, "failed to request resources: %d\n", err); + goto err_pm_put; + } + + ep->num_ib_windows = MAX_NR_INBOUND_MAPS; + ep->ib_window_map = + devm_kcalloc(dev, BITS_TO_LONGS(ep->num_ib_windows), + sizeof(long), GFP_KERNEL); + if (!ep->ib_window_map) { + err = -ENOMEM; + dev_err(dev, "failed to allocate memory for inbound map\n"); + goto err_pm_put; + } + + ep->ob_mapped_addr = devm_kcalloc(dev, ep->num_ob_windows, + sizeof(*ep->ob_mapped_addr), + GFP_KERNEL); + if (!ep->ob_mapped_addr) { + err = -ENOMEM; + dev_err(dev, "failed to allocate memory for outbound memory pointers\n"); + goto err_pm_put; + } + + epc = devm_pci_epc_create(dev, &rcar_pcie_epc_ops); + if (IS_ERR(epc)) { + dev_err(dev, "failed to create epc device\n"); + err = PTR_ERR(epc); + goto err_pm_put; + } + + epc->max_functions = ep->max_functions; + epc_set_drvdata(epc, ep); + + rcar_pcie_ep_hw_init(pcie); + + err = pci_epc_multi_mem_init(epc, ep->ob_window, ep->num_ob_windows); + if (err < 0) { + dev_err(dev, "failed to initialize the epc memory space\n"); + goto err_pm_put; + } + + pci_epc_init_notify(epc); + + return 0; + +err_pm_put: + pm_runtime_put(dev); + +err_pm_disable: + pm_runtime_disable(dev); + + return err; +} + +static struct platform_driver rcar_pcie_ep_driver = { + .driver = { + .name = "rcar-pcie-ep", + .of_match_table = rcar_pcie_ep_of_match, + .suppress_bind_attrs = true, + }, + .probe = rcar_pcie_ep_probe, +}; +builtin_platform_driver(rcar_pcie_ep_driver); diff --git a/drivers/pci/controller/pcie-rcar-host.c b/drivers/pci/controller/pcie-rcar-host.c new file mode 100644 index 000000000..cd9171eeb --- /dev/null +++ b/drivers/pci/controller/pcie-rcar-host.c @@ -0,0 +1,1140 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe driver for Renesas R-Car SoCs + * Copyright (C) 2014-2020 Renesas Electronics Europe Ltd + * + * Based on: + * arch/sh/drivers/pci/pcie-sh7786.c + * arch/sh/drivers/pci/ops-sh7786.c + * Copyright (C) 2009 - 2011 Paul Mundt + * + * Author: Phil Edworthy <phil.edworthy@renesas.com> + */ + +#include <linux/bitops.h> +#include <linux/cleanup.h> +#include <linux/clk.h> +#include <linux/clk-provider.h> +#include <linux/delay.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/irqdomain.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/iopoll.h> +#include <linux/msi.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/of_platform.h> +#include <linux/pci.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/regulator/consumer.h> + +#include "pcie-rcar.h" + +struct rcar_msi { + DECLARE_BITMAP(used, INT_PCI_MSI_NR); + struct irq_domain *domain; + struct mutex map_lock; + raw_spinlock_t mask_lock; + int irq1; + int irq2; +}; + +/* Structure representing the PCIe interface */ +struct rcar_pcie_host { + struct rcar_pcie pcie; + struct phy *phy; + struct clk *bus_clk; + struct rcar_msi msi; + int (*phy_init_fn)(struct rcar_pcie_host *host); +}; + +static int rcar_pcie_wakeup(struct device *pcie_dev, void __iomem *pcie_base) +{ + u32 pmsr, val; + int ret = 0; + + if (!pcie_base || pm_runtime_suspended(pcie_dev)) + return -EINVAL; + + pmsr = readl(pcie_base + PMSR); + + /* + * Test if the PCIe controller received PM_ENTER_L1 DLLP and + * the PCIe controller is not in L1 link state. If true, apply + * fix, which will put the controller into L1 link state, from + * which it can return to L0s/L0 on its own. + */ + if ((pmsr & PMEL1RX) && ((pmsr & PMSTATE) != PMSTATE_L1)) { + writel(L1IATN, pcie_base + PMCTLR); + ret = readl_poll_timeout_atomic(pcie_base + PMSR, val, + val & L1FAEG, 10, 1000); + if (ret) { + dev_warn_ratelimited(pcie_dev, + "Timeout waiting for L1 link state, ret=%d\n", + ret); + } + writel(L1FAEG | PMEL1RX, pcie_base + PMSR); + } + + return ret; +} + +static struct rcar_pcie_host *msi_to_host(struct rcar_msi *msi) +{ + return container_of(msi, struct rcar_pcie_host, msi); +} + +static u32 rcar_read_conf(struct rcar_pcie *pcie, int where) +{ + unsigned int shift = BITS_PER_BYTE * (where & 3); + u32 val = rcar_pci_read_reg(pcie, where & ~3); + + return val >> shift; +} + +#ifdef CONFIG_ARM +#define __rcar_pci_rw_reg_workaround(instr) \ + " .arch armv7-a\n" \ + "1: " instr " %1, [%2]\n" \ + "2: isb\n" \ + "3: .pushsection .text.fixup,\"ax\"\n" \ + " .align 2\n" \ + "4: mov %0, #" __stringify(PCIBIOS_SET_FAILED) "\n" \ + " b 3b\n" \ + " .popsection\n" \ + " .pushsection __ex_table,\"a\"\n" \ + " .align 3\n" \ + " .long 1b, 4b\n" \ + " .long 2b, 4b\n" \ + " .popsection\n" +#endif + +static int rcar_pci_write_reg_workaround(struct rcar_pcie *pcie, u32 val, + unsigned int reg) +{ + int error = PCIBIOS_SUCCESSFUL; +#ifdef CONFIG_ARM + asm volatile( + __rcar_pci_rw_reg_workaround("str") + : "+r"(error):"r"(val), "r"(pcie->base + reg) : "memory"); +#else + rcar_pci_write_reg(pcie, val, reg); +#endif + return error; +} + +static int rcar_pci_read_reg_workaround(struct rcar_pcie *pcie, u32 *val, + unsigned int reg) +{ + int error = PCIBIOS_SUCCESSFUL; +#ifdef CONFIG_ARM + asm volatile( + __rcar_pci_rw_reg_workaround("ldr") + : "+r"(error), "=r"(*val) : "r"(pcie->base + reg) : "memory"); + + if (error != PCIBIOS_SUCCESSFUL) + PCI_SET_ERROR_RESPONSE(val); +#else + *val = rcar_pci_read_reg(pcie, reg); +#endif + return error; +} + +/* Serialization is provided by 'pci_lock' in drivers/pci/access.c */ +static int rcar_pcie_config_access(struct rcar_pcie_host *host, + unsigned char access_type, struct pci_bus *bus, + unsigned int devfn, int where, u32 *data) +{ + struct rcar_pcie *pcie = &host->pcie; + unsigned int dev, func, reg, index; + int ret; + + /* Wake the bus up in case it is in L1 state. */ + ret = rcar_pcie_wakeup(pcie->dev, pcie->base); + if (ret) { + PCI_SET_ERROR_RESPONSE(data); + return PCIBIOS_SET_FAILED; + } + + dev = PCI_SLOT(devfn); + func = PCI_FUNC(devfn); + reg = where & ~3; + index = reg / 4; + + /* + * While each channel has its own memory-mapped extended config + * space, it's generally only accessible when in endpoint mode. + * When in root complex mode, the controller is unable to target + * itself with either type 0 or type 1 accesses, and indeed, any + * controller-initiated target transfer to its own config space + * results in a completer abort. + * + * Each channel effectively only supports a single device, but as + * the same channel <-> device access works for any PCI_SLOT() + * value, we cheat a bit here and bind the controller's config + * space to devfn 0 in order to enable self-enumeration. In this + * case the regular ECAR/ECDR path is sidelined and the mangled + * config access itself is initiated as an internal bus transaction. + */ + if (pci_is_root_bus(bus)) { + if (dev != 0) + return PCIBIOS_DEVICE_NOT_FOUND; + + if (access_type == RCAR_PCI_ACCESS_READ) + *data = rcar_pci_read_reg(pcie, PCICONF(index)); + else + rcar_pci_write_reg(pcie, *data, PCICONF(index)); + + return PCIBIOS_SUCCESSFUL; + } + + /* Clear errors */ + rcar_pci_write_reg(pcie, rcar_pci_read_reg(pcie, PCIEERRFR), PCIEERRFR); + + /* Set the PIO address */ + rcar_pci_write_reg(pcie, PCIE_CONF_BUS(bus->number) | + PCIE_CONF_DEV(dev) | PCIE_CONF_FUNC(func) | reg, PCIECAR); + + /* Enable the configuration access */ + if (pci_is_root_bus(bus->parent)) + rcar_pci_write_reg(pcie, PCIECCTLR_CCIE | TYPE0, PCIECCTLR); + else + rcar_pci_write_reg(pcie, PCIECCTLR_CCIE | TYPE1, PCIECCTLR); + + /* Check for errors */ + if (rcar_pci_read_reg(pcie, PCIEERRFR) & UNSUPPORTED_REQUEST) + return PCIBIOS_DEVICE_NOT_FOUND; + + /* Check for master and target aborts */ + if (rcar_read_conf(pcie, RCONF(PCI_STATUS)) & + (PCI_STATUS_REC_MASTER_ABORT | PCI_STATUS_REC_TARGET_ABORT)) + return PCIBIOS_DEVICE_NOT_FOUND; + + if (access_type == RCAR_PCI_ACCESS_READ) + ret = rcar_pci_read_reg_workaround(pcie, data, PCIECDR); + else + ret = rcar_pci_write_reg_workaround(pcie, *data, PCIECDR); + + /* Disable the configuration access */ + rcar_pci_write_reg(pcie, 0, PCIECCTLR); + + return ret; +} + +static int rcar_pcie_read_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct rcar_pcie_host *host = bus->sysdata; + int ret; + + ret = rcar_pcie_config_access(host, RCAR_PCI_ACCESS_READ, + bus, devfn, where, val); + if (ret != PCIBIOS_SUCCESSFUL) + return ret; + + if (size == 1) + *val = (*val >> (BITS_PER_BYTE * (where & 3))) & 0xff; + else if (size == 2) + *val = (*val >> (BITS_PER_BYTE * (where & 2))) & 0xffff; + + dev_dbg(&bus->dev, "pcie-config-read: bus=%3d devfn=0x%04x where=0x%04x size=%d val=0x%08x\n", + bus->number, devfn, where, size, *val); + + return ret; +} + +/* Serialization is provided by 'pci_lock' in drivers/pci/access.c */ +static int rcar_pcie_write_conf(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + struct rcar_pcie_host *host = bus->sysdata; + unsigned int shift; + u32 data; + int ret; + + ret = rcar_pcie_config_access(host, RCAR_PCI_ACCESS_READ, + bus, devfn, where, &data); + if (ret != PCIBIOS_SUCCESSFUL) + return ret; + + dev_dbg(&bus->dev, "pcie-config-write: bus=%3d devfn=0x%04x where=0x%04x size=%d val=0x%08x\n", + bus->number, devfn, where, size, val); + + if (size == 1) { + shift = BITS_PER_BYTE * (where & 3); + data &= ~(0xff << shift); + data |= ((val & 0xff) << shift); + } else if (size == 2) { + shift = BITS_PER_BYTE * (where & 2); + data &= ~(0xffff << shift); + data |= ((val & 0xffff) << shift); + } else + data = val; + + ret = rcar_pcie_config_access(host, RCAR_PCI_ACCESS_WRITE, + bus, devfn, where, &data); + + return ret; +} + +static struct pci_ops rcar_pcie_ops = { + .read = rcar_pcie_read_conf, + .write = rcar_pcie_write_conf, +}; + +static void rcar_pcie_force_speedup(struct rcar_pcie *pcie) +{ + struct device *dev = pcie->dev; + unsigned int timeout = 1000; + u32 macsr; + + if ((rcar_pci_read_reg(pcie, MACS2R) & LINK_SPEED) != LINK_SPEED_5_0GTS) + return; + + if (rcar_pci_read_reg(pcie, MACCTLR) & SPEED_CHANGE) { + dev_err(dev, "Speed change already in progress\n"); + return; + } + + macsr = rcar_pci_read_reg(pcie, MACSR); + if ((macsr & LINK_SPEED) == LINK_SPEED_5_0GTS) + goto done; + + /* Set target link speed to 5.0 GT/s */ + rcar_rmw32(pcie, EXPCAP(12), PCI_EXP_LNKSTA_CLS, + PCI_EXP_LNKSTA_CLS_5_0GB); + + /* Set speed change reason as intentional factor */ + rcar_rmw32(pcie, MACCGSPSETR, SPCNGRSN, 0); + + /* Clear SPCHGFIN, SPCHGSUC, and SPCHGFAIL */ + if (macsr & (SPCHGFIN | SPCHGSUC | SPCHGFAIL)) + rcar_pci_write_reg(pcie, macsr, MACSR); + + /* Start link speed change */ + rcar_rmw32(pcie, MACCTLR, SPEED_CHANGE, SPEED_CHANGE); + + while (timeout--) { + macsr = rcar_pci_read_reg(pcie, MACSR); + if (macsr & SPCHGFIN) { + /* Clear the interrupt bits */ + rcar_pci_write_reg(pcie, macsr, MACSR); + + if (macsr & SPCHGFAIL) + dev_err(dev, "Speed change failed\n"); + + goto done; + } + + msleep(1); + } + + dev_err(dev, "Speed change timed out\n"); + +done: + dev_info(dev, "Current link speed is %s GT/s\n", + (macsr & LINK_SPEED) == LINK_SPEED_5_0GTS ? "5" : "2.5"); +} + +static void rcar_pcie_hw_enable(struct rcar_pcie_host *host) +{ + struct rcar_pcie *pcie = &host->pcie; + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(host); + struct resource_entry *win; + int i = 0; + + /* Try setting 5 GT/s link speed */ + rcar_pcie_force_speedup(pcie); + + /* Setup PCI resources */ + resource_list_for_each_entry(win, &bridge->windows) { + struct resource *res = win->res; + + if (!res->flags) + continue; + + switch (resource_type(res)) { + case IORESOURCE_IO: + case IORESOURCE_MEM: + rcar_pcie_set_outbound(pcie, i, win); + i++; + break; + } + } +} + +static int rcar_pcie_enable(struct rcar_pcie_host *host) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(host); + + rcar_pcie_hw_enable(host); + + pci_add_flags(PCI_REASSIGN_ALL_BUS); + + bridge->sysdata = host; + bridge->ops = &rcar_pcie_ops; + + return pci_host_probe(bridge); +} + +static int phy_wait_for_ack(struct rcar_pcie *pcie) +{ + struct device *dev = pcie->dev; + unsigned int timeout = 100; + + while (timeout--) { + if (rcar_pci_read_reg(pcie, H1_PCIEPHYADRR) & PHY_ACK) + return 0; + + udelay(100); + } + + dev_err(dev, "Access to PCIe phy timed out\n"); + + return -ETIMEDOUT; +} + +static void phy_write_reg(struct rcar_pcie *pcie, + unsigned int rate, u32 addr, + unsigned int lane, u32 data) +{ + u32 phyaddr; + + phyaddr = WRITE_CMD | + ((rate & 1) << RATE_POS) | + ((lane & 0xf) << LANE_POS) | + ((addr & 0xff) << ADR_POS); + + /* Set write data */ + rcar_pci_write_reg(pcie, data, H1_PCIEPHYDOUTR); + rcar_pci_write_reg(pcie, phyaddr, H1_PCIEPHYADRR); + + /* Ignore errors as they will be dealt with if the data link is down */ + phy_wait_for_ack(pcie); + + /* Clear command */ + rcar_pci_write_reg(pcie, 0, H1_PCIEPHYDOUTR); + rcar_pci_write_reg(pcie, 0, H1_PCIEPHYADRR); + + /* Ignore errors as they will be dealt with if the data link is down */ + phy_wait_for_ack(pcie); +} + +static int rcar_pcie_hw_init(struct rcar_pcie *pcie) +{ + int err; + + /* Begin initialization */ + rcar_pci_write_reg(pcie, 0, PCIETCTLR); + + /* Set mode */ + rcar_pci_write_reg(pcie, 1, PCIEMSR); + + err = rcar_pcie_wait_for_phyrdy(pcie); + if (err) + return err; + + /* + * Initial header for port config space is type 1, set the device + * class to match. Hardware takes care of propagating the IDSETR + * settings, so there is no need to bother with a quirk. + */ + rcar_pci_write_reg(pcie, PCI_CLASS_BRIDGE_PCI_NORMAL << 8, IDSETR1); + + /* + * Setup Secondary Bus Number & Subordinate Bus Number, even though + * they aren't used, to avoid bridge being detected as broken. + */ + rcar_rmw32(pcie, RCONF(PCI_SECONDARY_BUS), 0xff, 1); + rcar_rmw32(pcie, RCONF(PCI_SUBORDINATE_BUS), 0xff, 1); + + /* Initialize default capabilities. */ + rcar_rmw32(pcie, REXPCAP(0), 0xff, PCI_CAP_ID_EXP); + rcar_rmw32(pcie, REXPCAP(PCI_EXP_FLAGS), + PCI_EXP_FLAGS_TYPE, PCI_EXP_TYPE_ROOT_PORT << 4); + rcar_rmw32(pcie, RCONF(PCI_HEADER_TYPE), PCI_HEADER_TYPE_MASK, + PCI_HEADER_TYPE_BRIDGE); + + /* Enable data link layer active state reporting */ + rcar_rmw32(pcie, REXPCAP(PCI_EXP_LNKCAP), PCI_EXP_LNKCAP_DLLLARC, + PCI_EXP_LNKCAP_DLLLARC); + + /* Write out the physical slot number = 0 */ + rcar_rmw32(pcie, REXPCAP(PCI_EXP_SLTCAP), PCI_EXP_SLTCAP_PSN, 0); + + /* Set the completion timer timeout to the maximum 50ms. */ + rcar_rmw32(pcie, TLCTLR + 1, 0x3f, 50); + + /* Terminate list of capabilities (Next Capability Offset=0) */ + rcar_rmw32(pcie, RVCCAP(0), 0xfff00000, 0); + + /* Enable MSI */ + if (IS_ENABLED(CONFIG_PCI_MSI)) + rcar_pci_write_reg(pcie, 0x801f0000, PCIEMSITXR); + + rcar_pci_write_reg(pcie, MACCTLR_INIT_VAL, MACCTLR); + + /* Finish initialization - establish a PCI Express link */ + rcar_pci_write_reg(pcie, CFINIT, PCIETCTLR); + + /* This will timeout if we don't have a link. */ + err = rcar_pcie_wait_for_dl(pcie); + if (err) + return err; + + /* Enable INTx interrupts */ + rcar_rmw32(pcie, PCIEINTXR, 0, 0xF << 8); + + wmb(); + + return 0; +} + +static int rcar_pcie_phy_init_h1(struct rcar_pcie_host *host) +{ + struct rcar_pcie *pcie = &host->pcie; + + /* Initialize the phy */ + phy_write_reg(pcie, 0, 0x42, 0x1, 0x0EC34191); + phy_write_reg(pcie, 1, 0x42, 0x1, 0x0EC34180); + phy_write_reg(pcie, 0, 0x43, 0x1, 0x00210188); + phy_write_reg(pcie, 1, 0x43, 0x1, 0x00210188); + phy_write_reg(pcie, 0, 0x44, 0x1, 0x015C0014); + phy_write_reg(pcie, 1, 0x44, 0x1, 0x015C0014); + phy_write_reg(pcie, 1, 0x4C, 0x1, 0x786174A0); + phy_write_reg(pcie, 1, 0x4D, 0x1, 0x048000BB); + phy_write_reg(pcie, 0, 0x51, 0x1, 0x079EC062); + phy_write_reg(pcie, 0, 0x52, 0x1, 0x20000000); + phy_write_reg(pcie, 1, 0x52, 0x1, 0x20000000); + phy_write_reg(pcie, 1, 0x56, 0x1, 0x00003806); + + phy_write_reg(pcie, 0, 0x60, 0x1, 0x004B03A5); + phy_write_reg(pcie, 0, 0x64, 0x1, 0x3F0F1F0F); + phy_write_reg(pcie, 0, 0x66, 0x1, 0x00008000); + + return 0; +} + +static int rcar_pcie_phy_init_gen2(struct rcar_pcie_host *host) +{ + struct rcar_pcie *pcie = &host->pcie; + + /* + * These settings come from the R-Car Series, 2nd Generation User's + * Manual, section 50.3.1 (2) Initialization of the physical layer. + */ + rcar_pci_write_reg(pcie, 0x000f0030, GEN2_PCIEPHYADDR); + rcar_pci_write_reg(pcie, 0x00381203, GEN2_PCIEPHYDATA); + rcar_pci_write_reg(pcie, 0x00000001, GEN2_PCIEPHYCTRL); + rcar_pci_write_reg(pcie, 0x00000006, GEN2_PCIEPHYCTRL); + + rcar_pci_write_reg(pcie, 0x000f0054, GEN2_PCIEPHYADDR); + /* The following value is for DC connection, no termination resistor */ + rcar_pci_write_reg(pcie, 0x13802007, GEN2_PCIEPHYDATA); + rcar_pci_write_reg(pcie, 0x00000001, GEN2_PCIEPHYCTRL); + rcar_pci_write_reg(pcie, 0x00000006, GEN2_PCIEPHYCTRL); + + return 0; +} + +static int rcar_pcie_phy_init_gen3(struct rcar_pcie_host *host) +{ + int err; + + err = phy_init(host->phy); + if (err) + return err; + + err = phy_power_on(host->phy); + if (err) + phy_exit(host->phy); + + return err; +} + +static irqreturn_t rcar_pcie_msi_irq(int irq, void *data) +{ + struct rcar_pcie_host *host = data; + struct rcar_pcie *pcie = &host->pcie; + struct rcar_msi *msi = &host->msi; + struct device *dev = pcie->dev; + unsigned long reg; + + reg = rcar_pci_read_reg(pcie, PCIEMSIFR); + + /* MSI & INTx share an interrupt - we only handle MSI here */ + if (!reg) + return IRQ_NONE; + + while (reg) { + unsigned int index = find_first_bit(®, 32); + int ret; + + ret = generic_handle_domain_irq(msi->domain, index); + if (ret) { + /* Unknown MSI, just clear it */ + dev_dbg(dev, "unexpected MSI\n"); + rcar_pci_write_reg(pcie, BIT(index), PCIEMSIFR); + } + + /* see if there's any more pending in this vector */ + reg = rcar_pci_read_reg(pcie, PCIEMSIFR); + } + + return IRQ_HANDLED; +} + +static void rcar_msi_irq_ack(struct irq_data *d) +{ + struct rcar_msi *msi = irq_data_get_irq_chip_data(d); + struct rcar_pcie *pcie = &msi_to_host(msi)->pcie; + + /* clear the interrupt */ + rcar_pci_write_reg(pcie, BIT(d->hwirq), PCIEMSIFR); +} + +static void rcar_msi_irq_mask(struct irq_data *d) +{ + struct rcar_msi *msi = irq_data_get_irq_chip_data(d); + struct rcar_pcie *pcie = &msi_to_host(msi)->pcie; + u32 value; + + scoped_guard(raw_spinlock_irqsave, &msi->mask_lock) { + value = rcar_pci_read_reg(pcie, PCIEMSIIER); + value &= ~BIT(d->hwirq); + rcar_pci_write_reg(pcie, value, PCIEMSIIER); + } +} + +static void rcar_msi_irq_unmask(struct irq_data *d) +{ + struct rcar_msi *msi = irq_data_get_irq_chip_data(d); + struct rcar_pcie *pcie = &msi_to_host(msi)->pcie; + u32 value; + + scoped_guard(raw_spinlock_irqsave, &msi->mask_lock) { + value = rcar_pci_read_reg(pcie, PCIEMSIIER); + value |= BIT(d->hwirq); + rcar_pci_write_reg(pcie, value, PCIEMSIIER); + } +} + +static void rcar_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct rcar_msi *msi = irq_data_get_irq_chip_data(data); + struct rcar_pcie *pcie = &msi_to_host(msi)->pcie; + + msg->address_lo = rcar_pci_read_reg(pcie, PCIEMSIALR) & ~MSIFE; + msg->address_hi = rcar_pci_read_reg(pcie, PCIEMSIAUR); + msg->data = data->hwirq; +} + +static struct irq_chip rcar_msi_bottom_chip = { + .name = "R-Car MSI", + .irq_ack = rcar_msi_irq_ack, + .irq_mask = rcar_msi_irq_mask, + .irq_unmask = rcar_msi_irq_unmask, + .irq_compose_msi_msg = rcar_compose_msi_msg, +}; + +static int rcar_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct rcar_msi *msi = domain->host_data; + unsigned int i; + int hwirq; + + mutex_lock(&msi->map_lock); + + hwirq = bitmap_find_free_region(msi->used, INT_PCI_MSI_NR, order_base_2(nr_irqs)); + + mutex_unlock(&msi->map_lock); + + if (hwirq < 0) + return -ENOSPC; + + for (i = 0; i < nr_irqs; i++) + irq_domain_set_info(domain, virq + i, hwirq + i, + &rcar_msi_bottom_chip, domain->host_data, + handle_edge_irq, NULL, NULL); + + return 0; +} + +static void rcar_msi_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct rcar_msi *msi = domain->host_data; + + mutex_lock(&msi->map_lock); + + bitmap_release_region(msi->used, d->hwirq, order_base_2(nr_irqs)); + + mutex_unlock(&msi->map_lock); +} + +static const struct irq_domain_ops rcar_msi_domain_ops = { + .alloc = rcar_msi_domain_alloc, + .free = rcar_msi_domain_free, +}; + +#define RCAR_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_PCI_MSI_MASK_PARENT | \ + MSI_FLAG_NO_AFFINITY) + +#define RCAR_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_MULTI_PCI_MSI) + +static const struct msi_parent_ops rcar_msi_parent_ops = { + .required_flags = RCAR_MSI_FLAGS_REQUIRED, + .supported_flags = RCAR_MSI_FLAGS_SUPPORTED, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .chip_flags = MSI_CHIP_FLAG_SET_ACK, + .prefix = "RCAR-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +static int rcar_allocate_domains(struct rcar_msi *msi) +{ + struct rcar_pcie *pcie = &msi_to_host(msi)->pcie; + struct irq_domain_info info = { + .fwnode = dev_fwnode(pcie->dev), + .ops = &rcar_msi_domain_ops, + .host_data = msi, + .size = INT_PCI_MSI_NR, + }; + + msi->domain = msi_create_parent_irq_domain(&info, &rcar_msi_parent_ops); + if (!msi->domain) { + dev_err(pcie->dev, "failed to create IRQ domain\n"); + return -ENOMEM; + } + + return 0; +} + +static void rcar_free_domains(struct rcar_msi *msi) +{ + irq_domain_remove(msi->domain); +} + +static int rcar_pcie_enable_msi(struct rcar_pcie_host *host) +{ + struct rcar_pcie *pcie = &host->pcie; + struct device *dev = pcie->dev; + struct rcar_msi *msi = &host->msi; + struct resource res; + int err; + + mutex_init(&msi->map_lock); + raw_spin_lock_init(&msi->mask_lock); + + err = of_address_to_resource(dev->of_node, 0, &res); + if (err) + return err; + + err = rcar_allocate_domains(msi); + if (err) + return err; + + /* Two IRQs are for MSI, but they are also used for non-MSI IRQs */ + err = devm_request_irq(dev, msi->irq1, rcar_pcie_msi_irq, + IRQF_SHARED | IRQF_NO_THREAD, + rcar_msi_bottom_chip.name, host); + if (err < 0) { + dev_err(dev, "failed to request IRQ: %d\n", err); + goto err; + } + + err = devm_request_irq(dev, msi->irq2, rcar_pcie_msi_irq, + IRQF_SHARED | IRQF_NO_THREAD, + rcar_msi_bottom_chip.name, host); + if (err < 0) { + dev_err(dev, "failed to request IRQ: %d\n", err); + goto err; + } + + /* Disable all MSIs */ + rcar_pci_write_reg(pcie, 0, PCIEMSIIER); + + /* + * Setup MSI data target using RC base address, which is guaranteed + * to be in the low 32bit range on any R-Car HW. + */ + rcar_pci_write_reg(pcie, lower_32_bits(res.start) | MSIFE, PCIEMSIALR); + rcar_pci_write_reg(pcie, upper_32_bits(res.start), PCIEMSIAUR); + + return 0; + +err: + rcar_free_domains(msi); + return err; +} + +static void rcar_pcie_teardown_msi(struct rcar_pcie_host *host) +{ + struct rcar_pcie *pcie = &host->pcie; + + /* Disable all MSI interrupts */ + rcar_pci_write_reg(pcie, 0, PCIEMSIIER); + + /* Disable address decoding of the MSI interrupt, MSIFE */ + rcar_pci_write_reg(pcie, 0, PCIEMSIALR); + + rcar_free_domains(&host->msi); +} + +static int rcar_pcie_get_resources(struct rcar_pcie_host *host) +{ + struct rcar_pcie *pcie = &host->pcie; + struct device *dev = pcie->dev; + struct resource res; + int err, i; + + host->phy = devm_phy_optional_get(dev, "pcie"); + if (IS_ERR(host->phy)) + return PTR_ERR(host->phy); + + err = of_address_to_resource(dev->of_node, 0, &res); + if (err) + return err; + + pcie->base = devm_ioremap_resource(dev, &res); + if (IS_ERR(pcie->base)) + return PTR_ERR(pcie->base); + + host->bus_clk = devm_clk_get(dev, "pcie_bus"); + if (IS_ERR(host->bus_clk)) { + dev_err(dev, "cannot get pcie bus clock\n"); + return PTR_ERR(host->bus_clk); + } + + i = irq_of_parse_and_map(dev->of_node, 0); + if (!i) { + dev_err(dev, "cannot get platform resources for msi interrupt\n"); + err = -ENOENT; + goto err_irq1; + } + host->msi.irq1 = i; + + i = irq_of_parse_and_map(dev->of_node, 1); + if (!i) { + dev_err(dev, "cannot get platform resources for msi interrupt\n"); + err = -ENOENT; + goto err_irq2; + } + host->msi.irq2 = i; + + return 0; + +err_irq2: + irq_dispose_mapping(host->msi.irq1); +err_irq1: + return err; +} + +static int rcar_pcie_inbound_ranges(struct rcar_pcie *pcie, + struct resource_entry *entry, + int *index) +{ + u64 restype = entry->res->flags; + u64 cpu_addr = entry->res->start; + u64 cpu_end = entry->res->end; + u64 pci_addr = entry->res->start - entry->offset; + u32 flags = LAM_64BIT | LAR_ENABLE; + u64 mask; + u64 size = resource_size(entry->res); + int idx = *index; + + if (restype & IORESOURCE_PREFETCH) + flags |= LAM_PREFETCH; + + while (cpu_addr < cpu_end) { + if (idx >= MAX_NR_INBOUND_MAPS - 1) { + dev_err(pcie->dev, "Failed to map inbound regions!\n"); + return -EINVAL; + } + + /* + * If the size of the range is larger than the alignment of + * the start address, we have to use multiple entries to + * perform the mapping. + */ + if (cpu_addr > 0) { + unsigned long nr_zeros = __ffs64(cpu_addr); + u64 alignment = 1ULL << nr_zeros; + + size = min(size, alignment); + } + + /* Hardware supports max 4GiB inbound region */ + size = min(size, 1ULL << 32); + + mask = roundup_pow_of_two(size) - 1; + mask &= ~0xf; + + rcar_pcie_set_inbound(pcie, cpu_addr, pci_addr, + lower_32_bits(mask) | flags, idx, true); + + pci_addr += size; + cpu_addr += size; + idx += 2; + } + *index = idx; + + return 0; +} + +static int rcar_pcie_parse_map_dma_ranges(struct rcar_pcie_host *host) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(host); + struct resource_entry *entry; + int index = 0, err = 0; + + resource_list_for_each_entry(entry, &bridge->dma_ranges) { + err = rcar_pcie_inbound_ranges(&host->pcie, entry, &index); + if (err) + break; + } + + return err; +} + +static const struct of_device_id rcar_pcie_of_match[] = { + { .compatible = "renesas,pcie-r8a7779", + .data = rcar_pcie_phy_init_h1 }, + { .compatible = "renesas,pcie-r8a7790", + .data = rcar_pcie_phy_init_gen2 }, + { .compatible = "renesas,pcie-r8a7791", + .data = rcar_pcie_phy_init_gen2 }, + { .compatible = "renesas,pcie-rcar-gen2", + .data = rcar_pcie_phy_init_gen2 }, + { .compatible = "renesas,pcie-r8a7795", + .data = rcar_pcie_phy_init_gen3 }, + { .compatible = "renesas,pcie-rcar-gen3", + .data = rcar_pcie_phy_init_gen3 }, + {}, +}; + +/* Design note 346 from Linear Technology says order is not important. */ +static const char * const rcar_pcie_supplies[] = { + "vpcie1v5", + "vpcie3v3", + "vpcie12v", +}; + +static int rcar_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct pci_host_bridge *bridge; + struct rcar_pcie_host *host; + struct rcar_pcie *pcie; + unsigned int i; + u32 data; + int err; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*host)); + if (!bridge) + return -ENOMEM; + + host = pci_host_bridge_priv(bridge); + pcie = &host->pcie; + pcie->dev = dev; + platform_set_drvdata(pdev, host); + + for (i = 0; i < ARRAY_SIZE(rcar_pcie_supplies); i++) { + err = devm_regulator_get_enable_optional(dev, rcar_pcie_supplies[i]); + if (err < 0 && err != -ENODEV) + return dev_err_probe(dev, err, "failed to enable regulator: %s\n", + rcar_pcie_supplies[i]); + } + + pm_runtime_enable(pcie->dev); + err = pm_runtime_get_sync(pcie->dev); + if (err < 0) { + dev_err(pcie->dev, "pm_runtime_get_sync failed\n"); + goto err_pm_put; + } + + err = rcar_pcie_get_resources(host); + if (err < 0) { + dev_err(dev, "failed to request resources: %d\n", err); + goto err_pm_put; + } + + err = clk_prepare_enable(host->bus_clk); + if (err) { + dev_err(dev, "failed to enable bus clock: %d\n", err); + goto err_unmap_msi_irqs; + } + + err = rcar_pcie_parse_map_dma_ranges(host); + if (err) + goto err_clk_disable; + + host->phy_init_fn = of_device_get_match_data(dev); + err = host->phy_init_fn(host); + if (err) { + dev_err(dev, "failed to init PCIe PHY\n"); + goto err_clk_disable; + } + + /* Failure to get a link might just be that no cards are inserted */ + if (rcar_pcie_hw_init(pcie)) { + dev_info(dev, "PCIe link down\n"); + err = -ENODEV; + goto err_phy_shutdown; + } + + data = rcar_pci_read_reg(pcie, MACSR); + dev_info(dev, "PCIe x%d: link up\n", (data >> 20) & 0x3f); + + if (IS_ENABLED(CONFIG_PCI_MSI)) { + err = rcar_pcie_enable_msi(host); + if (err < 0) { + dev_err(dev, + "failed to enable MSI support: %d\n", + err); + goto err_phy_shutdown; + } + } + + err = rcar_pcie_enable(host); + if (err) + goto err_msi_teardown; + + return 0; + +err_msi_teardown: + if (IS_ENABLED(CONFIG_PCI_MSI)) + rcar_pcie_teardown_msi(host); + +err_phy_shutdown: + if (host->phy) { + phy_power_off(host->phy); + phy_exit(host->phy); + } + +err_clk_disable: + clk_disable_unprepare(host->bus_clk); + +err_unmap_msi_irqs: + irq_dispose_mapping(host->msi.irq2); + irq_dispose_mapping(host->msi.irq1); + +err_pm_put: + pm_runtime_put(dev); + pm_runtime_disable(dev); + + return err; +} + +static int rcar_pcie_resume(struct device *dev) +{ + struct rcar_pcie_host *host = dev_get_drvdata(dev); + struct rcar_pcie *pcie = &host->pcie; + unsigned int data; + int err; + + err = rcar_pcie_parse_map_dma_ranges(host); + if (err) + return 0; + + /* Failure to get a link might just be that no cards are inserted */ + err = host->phy_init_fn(host); + if (err) { + dev_info(dev, "PCIe link down\n"); + return 0; + } + + data = rcar_pci_read_reg(pcie, MACSR); + dev_info(dev, "PCIe x%d: link up\n", (data >> 20) & 0x3f); + + /* Enable MSI */ + if (IS_ENABLED(CONFIG_PCI_MSI)) { + struct resource res; + u32 val; + + of_address_to_resource(dev->of_node, 0, &res); + rcar_pci_write_reg(pcie, upper_32_bits(res.start), PCIEMSIAUR); + rcar_pci_write_reg(pcie, lower_32_bits(res.start) | MSIFE, PCIEMSIALR); + + bitmap_to_arr32(&val, host->msi.used, INT_PCI_MSI_NR); + rcar_pci_write_reg(pcie, val, PCIEMSIIER); + } + + rcar_pcie_hw_enable(host); + + return 0; +} + +static int rcar_pcie_resume_noirq(struct device *dev) +{ + struct rcar_pcie_host *host = dev_get_drvdata(dev); + struct rcar_pcie *pcie = &host->pcie; + + if (rcar_pci_read_reg(pcie, PMSR) && + !(rcar_pci_read_reg(pcie, PCIETCTLR) & DL_DOWN)) + return 0; + + /* Re-establish the PCIe link */ + rcar_pci_write_reg(pcie, MACCTLR_INIT_VAL, MACCTLR); + rcar_pci_write_reg(pcie, CFINIT, PCIETCTLR); + return rcar_pcie_wait_for_dl(pcie); +} + +static const struct dev_pm_ops rcar_pcie_pm_ops = { + SYSTEM_SLEEP_PM_OPS(NULL, rcar_pcie_resume) + .resume_noirq = rcar_pcie_resume_noirq, +}; + +static struct platform_driver rcar_pcie_driver = { + .driver = { + .name = "rcar-pcie", + .of_match_table = rcar_pcie_of_match, + .pm = &rcar_pcie_pm_ops, + .suppress_bind_attrs = true, + }, + .probe = rcar_pcie_probe, +}; + +#ifdef CONFIG_ARM +static int rcar_pcie_aarch32_abort_handler(unsigned long addr, + unsigned int fsr, struct pt_regs *regs) +{ + return !fixup_exception(regs); +} + +static const struct of_device_id rcar_pcie_abort_handler_of_match[] __initconst = { + { .compatible = "renesas,pcie-r8a7779" }, + { .compatible = "renesas,pcie-r8a7790" }, + { .compatible = "renesas,pcie-r8a7791" }, + { .compatible = "renesas,pcie-rcar-gen2" }, + {}, +}; + +static int __init rcar_pcie_init(void) +{ + if (of_find_matching_node(NULL, rcar_pcie_abort_handler_of_match)) { +#ifdef CONFIG_ARM_LPAE + hook_fault_code(17, rcar_pcie_aarch32_abort_handler, SIGBUS, 0, + "asynchronous external abort"); +#else + hook_fault_code(22, rcar_pcie_aarch32_abort_handler, SIGBUS, 0, + "imprecise external abort"); +#endif + } + + return platform_driver_register(&rcar_pcie_driver); +} +device_initcall(rcar_pcie_init); +#else +builtin_platform_driver(rcar_pcie_driver); +#endif diff --git a/drivers/pci/controller/pcie-rcar.c b/drivers/pci/controller/pcie-rcar.c new file mode 100644 index 000000000..7583699ef --- /dev/null +++ b/drivers/pci/controller/pcie-rcar.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe driver for Renesas R-Car SoCs + * Copyright (C) 2014-2020 Renesas Electronics Europe Ltd + * + * Author: Phil Edworthy <phil.edworthy@renesas.com> + */ + +#include <linux/delay.h> +#include <linux/pci.h> + +#include "pcie-rcar.h" + +void rcar_pci_write_reg(struct rcar_pcie *pcie, u32 val, unsigned int reg) +{ + writel(val, pcie->base + reg); +} + +u32 rcar_pci_read_reg(struct rcar_pcie *pcie, unsigned int reg) +{ + return readl(pcie->base + reg); +} + +void rcar_rmw32(struct rcar_pcie *pcie, int where, u32 mask, u32 data) +{ + unsigned int shift = BITS_PER_BYTE * (where & 3); + u32 val = rcar_pci_read_reg(pcie, where & ~3); + + val &= ~(mask << shift); + val |= data << shift; + rcar_pci_write_reg(pcie, val, where & ~3); +} + +int rcar_pcie_wait_for_phyrdy(struct rcar_pcie *pcie) +{ + unsigned int timeout = 10; + + while (timeout--) { + if (rcar_pci_read_reg(pcie, PCIEPHYSR) & PHYRDY) + return 0; + + msleep(5); + } + + return -ETIMEDOUT; +} + +int rcar_pcie_wait_for_dl(struct rcar_pcie *pcie) +{ + unsigned int timeout = 10000; + + while (timeout--) { + if ((rcar_pci_read_reg(pcie, PCIETSTR) & DATA_LINK_ACTIVE)) + return 0; + + udelay(5); + cpu_relax(); + } + + return -ETIMEDOUT; +} + +void rcar_pcie_set_outbound(struct rcar_pcie *pcie, int win, + struct resource_entry *window) +{ + /* Setup PCIe address space mappings for each resource */ + struct resource *res = window->res; + resource_size_t res_start; + resource_size_t size; + u32 mask; + + rcar_pci_write_reg(pcie, 0x00000000, PCIEPTCTLR(win)); + + /* + * The PAMR mask is calculated in units of 128Bytes, which + * keeps things pretty simple. + */ + size = resource_size(res); + if (size > 128) + mask = (roundup_pow_of_two(size) / SZ_128) - 1; + else + mask = 0x0; + rcar_pci_write_reg(pcie, mask << 7, PCIEPAMR(win)); + + if (res->flags & IORESOURCE_IO) + res_start = pci_pio_to_address(res->start) - window->offset; + else + res_start = res->start - window->offset; + + rcar_pci_write_reg(pcie, upper_32_bits(res_start), PCIEPAUR(win)); + rcar_pci_write_reg(pcie, lower_32_bits(res_start) & ~0x7F, + PCIEPALR(win)); + + /* First resource is for IO */ + mask = PAR_ENABLE; + if (res->flags & IORESOURCE_IO) + mask |= IO_SPACE; + + rcar_pci_write_reg(pcie, mask, PCIEPTCTLR(win)); +} + +void rcar_pcie_set_inbound(struct rcar_pcie *pcie, u64 cpu_addr, + u64 pci_addr, u64 flags, int idx, bool host) +{ + /* + * Set up 64-bit inbound regions as the range parser doesn't + * distinguish between 32 and 64-bit types. + */ + if (host) + rcar_pci_write_reg(pcie, lower_32_bits(pci_addr), + PCIEPRAR(idx)); + rcar_pci_write_reg(pcie, lower_32_bits(cpu_addr), PCIELAR(idx)); + rcar_pci_write_reg(pcie, flags, PCIELAMR(idx)); + + if (host) + rcar_pci_write_reg(pcie, upper_32_bits(pci_addr), + PCIEPRAR(idx + 1)); + rcar_pci_write_reg(pcie, upper_32_bits(cpu_addr), PCIELAR(idx + 1)); + rcar_pci_write_reg(pcie, 0, PCIELAMR(idx + 1)); +} diff --git a/drivers/pci/controller/pcie-rcar.h b/drivers/pci/controller/pcie-rcar.h new file mode 100644 index 000000000..6799f8139 --- /dev/null +++ b/drivers/pci/controller/pcie-rcar.h @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * PCIe driver for Renesas R-Car SoCs + * Copyright (C) 2014-2020 Renesas Electronics Europe Ltd + * + * Author: Phil Edworthy <phil.edworthy@renesas.com> + */ + +#ifndef _PCIE_RCAR_H +#define _PCIE_RCAR_H + +#define PCIECAR 0x000010 +#define PCIECCTLR 0x000018 +#define PCIECCTLR_CCIE BIT(31) +#define TYPE0 (0 << 8) +#define TYPE1 BIT(8) +#define PCIECDR 0x000020 +#define PCIEMSR 0x000028 +#define PCIEINTXR 0x000400 +#define ASTINTX BIT(16) +#define PCIEPHYSR 0x0007f0 +#define PHYRDY BIT(0) +#define PCIEMSITXR 0x000840 + +/* Transfer control */ +#define PCIETCTLR 0x02000 +#define DL_DOWN BIT(3) +#define CFINIT BIT(0) +#define PCIETSTR 0x02004 +#define DATA_LINK_ACTIVE BIT(0) +#define PCIEERRFR 0x02020 +#define UNSUPPORTED_REQUEST BIT(4) +#define PCIEMSIFR 0x02044 +#define PCIEMSIALR 0x02048 +#define MSIFE BIT(0) +#define PCIEMSIAUR 0x0204c +#define PCIEMSIIER 0x02050 + +/* root port address */ +#define PCIEPRAR(x) (0x02080 + ((x) * 0x4)) + +/* local address reg & mask */ +#define PCIELAR(x) (0x02200 + ((x) * 0x20)) +#define PCIELAMR(x) (0x02208 + ((x) * 0x20)) +#define LAM_PREFETCH BIT(3) +#define LAM_64BIT BIT(2) +#define LAR_ENABLE BIT(1) + +/* PCIe address reg & mask */ +#define PCIEPALR(x) (0x03400 + ((x) * 0x20)) +#define PCIEPAUR(x) (0x03404 + ((x) * 0x20)) +#define PCIEPAMR(x) (0x03408 + ((x) * 0x20)) +#define PCIEPTCTLR(x) (0x0340c + ((x) * 0x20)) +#define PAR_ENABLE BIT(31) +#define IO_SPACE BIT(8) + +/* Configuration */ +#define PCICONF(x) (0x010000 + ((x) * 0x4)) +#define INTDIS BIT(10) +#define PMCAP(x) (0x010040 + ((x) * 0x4)) +#define MSICAP(x) (0x010050 + ((x) * 0x4)) +#define MSICAP0_MSIE BIT(16) +#define MSICAP0_MMESCAP_OFFSET 17 +#define MSICAP0_MMESE_OFFSET 20 +#define MSICAP0_MMESE_MASK GENMASK(22, 20) +#define EXPCAP(x) (0x010070 + ((x) * 0x4)) +#define VCCAP(x) (0x010100 + ((x) * 0x4)) + +/* link layer */ +#define IDSETR0 0x011000 +#define IDSETR1 0x011004 +#define SUBIDSETR 0x011024 +#define TLCTLR 0x011048 +#define MACSR 0x011054 +#define SPCHGFIN BIT(4) +#define SPCHGFAIL BIT(6) +#define SPCHGSUC BIT(7) +#define LINK_SPEED (0xf << 16) +#define LINK_SPEED_2_5GTS (1 << 16) +#define LINK_SPEED_5_0GTS (2 << 16) +#define MACCTLR 0x011058 +#define MACCTLR_NFTS_MASK GENMASK(23, 16) /* The name is from SH7786 */ +#define SPEED_CHANGE BIT(24) +#define SCRAMBLE_DISABLE BIT(27) +#define LTSMDIS BIT(31) +#define MACCTLR_INIT_VAL (LTSMDIS | MACCTLR_NFTS_MASK) +#define PMSR 0x01105c +#define L1FAEG BIT(31) +#define PMEL1RX BIT(23) +#define PMSTATE GENMASK(18, 16) +#define PMSTATE_L1 (3 << 16) +#define PMCTLR 0x011060 +#define L1IATN BIT(31) + +#define MACS2R 0x011078 +#define MACCGSPSETR 0x011084 +#define SPCNGRSN BIT(31) + +/* R-Car H1 PHY */ +#define H1_PCIEPHYADRR 0x04000c +#define WRITE_CMD BIT(16) +#define PHY_ACK BIT(24) +#define RATE_POS 12 +#define LANE_POS 8 +#define ADR_POS 0 +#define H1_PCIEPHYDOUTR 0x040014 + +/* R-Car Gen2 PHY */ +#define GEN2_PCIEPHYADDR 0x780 +#define GEN2_PCIEPHYDATA 0x784 +#define GEN2_PCIEPHYCTRL 0x78c + +#define INT_PCI_MSI_NR 32 + +#define RCONF(x) (PCICONF(0) + (x)) +#define RPMCAP(x) (PMCAP(0) + (x)) +#define REXPCAP(x) (EXPCAP(0) + (x)) +#define RVCCAP(x) (VCCAP(0) + (x)) + +#define PCIE_CONF_BUS(b) (((b) & 0xff) << 24) +#define PCIE_CONF_DEV(d) (((d) & 0x1f) << 19) +#define PCIE_CONF_FUNC(f) (((f) & 0x7) << 16) + +#define RCAR_PCI_MAX_RESOURCES 4 +#define MAX_NR_INBOUND_MAPS 6 + +struct rcar_pcie { + struct device *dev; + void __iomem *base; +}; + +enum { + RCAR_PCI_ACCESS_READ, + RCAR_PCI_ACCESS_WRITE, +}; + +void rcar_pci_write_reg(struct rcar_pcie *pcie, u32 val, unsigned int reg); +u32 rcar_pci_read_reg(struct rcar_pcie *pcie, unsigned int reg); +void rcar_rmw32(struct rcar_pcie *pcie, int where, u32 mask, u32 data); +int rcar_pcie_wait_for_phyrdy(struct rcar_pcie *pcie); +int rcar_pcie_wait_for_dl(struct rcar_pcie *pcie); +void rcar_pcie_set_outbound(struct rcar_pcie *pcie, int win, + struct resource_entry *window); +void rcar_pcie_set_inbound(struct rcar_pcie *pcie, u64 cpu_addr, + u64 pci_addr, u64 flags, int idx, bool host); + +#endif diff --git a/drivers/pci/controller/pcie-rockchip-ep.c b/drivers/pci/controller/pcie-rockchip-ep.c new file mode 100644 index 000000000..799461335 --- /dev/null +++ b/drivers/pci/controller/pcie-rockchip-ep.c @@ -0,0 +1,910 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Rockchip AXI PCIe endpoint controller driver + * + * Copyright (c) 2018 Rockchip, Inc. + * + * Author: Shawn Lin <shawn.lin@rock-chips.com> + * Simon Xue <xxm@rock-chips.com> + */ + +#include <linux/configfs.h> +#include <linux/delay.h> +#include <linux/gpio/consumer.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/irq.h> +#include <linux/of.h> +#include <linux/pci-epc.h> +#include <linux/platform_device.h> +#include <linux/pci-epf.h> +#include <linux/sizes.h> +#include <linux/workqueue.h> + +#include "pcie-rockchip.h" + +/** + * struct rockchip_pcie_ep - private data for PCIe endpoint controller driver + * @rockchip: Rockchip PCIe controller + * @epc: PCI EPC device + * @max_regions: maximum number of regions supported by hardware + * @ob_region_map: bitmask of mapped outbound regions + * @ob_addr: base addresses in the AXI bus where the outbound regions start + * @irq_phys_addr: base address on the AXI bus where the MSI/INTX IRQ + * dedicated outbound regions is mapped. + * @irq_cpu_addr: base address in the CPU space where a write access triggers + * the sending of a memory write (MSI) / normal message (INTX + * IRQ) TLP through the PCIe bus. + * @irq_pci_addr: used to save the current mapping of the MSI/INTX IRQ + * dedicated outbound region. + * @irq_pci_fn: the latest PCI function that has updated the mapping of + * the MSI/INTX IRQ dedicated outbound region. + * @irq_pending: bitmask of asserted INTX IRQs. + * @perst_irq: IRQ used for the PERST# signal. + * @perst_asserted: True if the PERST# signal was asserted. + * @link_up: True if the PCI link is up. + * @link_training: Work item to execute PCI link training. + */ +struct rockchip_pcie_ep { + struct rockchip_pcie rockchip; + struct pci_epc *epc; + u32 max_regions; + unsigned long ob_region_map; + phys_addr_t *ob_addr; + phys_addr_t irq_phys_addr; + void __iomem *irq_cpu_addr; + u64 irq_pci_addr; + u8 irq_pci_fn; + u8 irq_pending; + int perst_irq; + bool perst_asserted; + bool link_up; + struct delayed_work link_training; +}; + +static void rockchip_pcie_clear_ep_ob_atu(struct rockchip_pcie *rockchip, + u32 region) +{ + rockchip_pcie_write(rockchip, 0, + ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0(region)); + rockchip_pcie_write(rockchip, 0, + ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR1(region)); + rockchip_pcie_write(rockchip, 0, + ROCKCHIP_PCIE_AT_OB_REGION_DESC0(region)); + rockchip_pcie_write(rockchip, 0, + ROCKCHIP_PCIE_AT_OB_REGION_DESC1(region)); +} + +static int rockchip_pcie_ep_ob_atu_num_bits(struct rockchip_pcie *rockchip, + u64 pci_addr, size_t size) +{ + int num_pass_bits = fls64(pci_addr ^ (pci_addr + size - 1)); + + return clamp(num_pass_bits, + ROCKCHIP_PCIE_AT_MIN_NUM_BITS, + ROCKCHIP_PCIE_AT_MAX_NUM_BITS); +} + +static void rockchip_pcie_prog_ep_ob_atu(struct rockchip_pcie *rockchip, u8 fn, + u32 r, u64 cpu_addr, u64 pci_addr, + size_t size) +{ + int num_pass_bits; + u32 addr0, addr1, desc0; + + num_pass_bits = rockchip_pcie_ep_ob_atu_num_bits(rockchip, + pci_addr, size); + + addr0 = ((num_pass_bits - 1) & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS) | + (lower_32_bits(pci_addr) & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR); + addr1 = upper_32_bits(pci_addr); + desc0 = ROCKCHIP_PCIE_AT_OB_REGION_DESC0_DEVFN(fn) | AXI_WRAPPER_MEM_WRITE; + + /* PCI bus address region */ + rockchip_pcie_write(rockchip, addr0, + ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0(r)); + rockchip_pcie_write(rockchip, addr1, + ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR1(r)); + rockchip_pcie_write(rockchip, desc0, + ROCKCHIP_PCIE_AT_OB_REGION_DESC0(r)); + rockchip_pcie_write(rockchip, 0, + ROCKCHIP_PCIE_AT_OB_REGION_DESC1(r)); +} + +static int rockchip_pcie_ep_write_header(struct pci_epc *epc, u8 fn, u8 vfn, + struct pci_epf_header *hdr) +{ + u32 reg; + struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); + struct rockchip_pcie *rockchip = &ep->rockchip; + + /* All functions share the same vendor ID with function 0 */ + if (fn == 0) { + rockchip_pcie_write(rockchip, + hdr->vendorid | hdr->subsys_vendor_id << 16, + PCIE_CORE_CONFIG_VENDOR); + } + + reg = rockchip_pcie_read(rockchip, PCIE_EP_CONFIG_DID_VID); + reg = (reg & 0xFFFF) | (hdr->deviceid << 16); + rockchip_pcie_write(rockchip, reg, PCIE_EP_CONFIG_DID_VID); + + rockchip_pcie_write(rockchip, + hdr->revid | + hdr->progif_code << 8 | + hdr->subclass_code << 16 | + hdr->baseclass_code << 24, + ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + PCI_REVISION_ID); + rockchip_pcie_write(rockchip, hdr->cache_line_size, + ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + + PCI_CACHE_LINE_SIZE); + rockchip_pcie_write(rockchip, hdr->subsys_id << 16, + ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + + PCI_SUBSYSTEM_VENDOR_ID); + rockchip_pcie_write(rockchip, hdr->interrupt_pin << 8, + ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + + PCI_INTERRUPT_LINE); + + return 0; +} + +static int rockchip_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, u8 vfn, + struct pci_epf_bar *epf_bar) +{ + struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); + struct rockchip_pcie *rockchip = &ep->rockchip; + dma_addr_t bar_phys = epf_bar->phys_addr; + enum pci_barno bar = epf_bar->barno; + int flags = epf_bar->flags; + u32 addr0, addr1, reg, cfg, b, aperture, ctrl; + u64 sz; + + /* BAR size is 2^(aperture + 7) */ + sz = max_t(size_t, epf_bar->size, MIN_EP_APERTURE); + + /* + * roundup_pow_of_two() returns an unsigned long, which is not suited + * for 64bit values. + */ + sz = 1ULL << fls64(sz - 1); + aperture = ilog2(sz) - 7; /* 128B -> 0, 256B -> 1, 512B -> 2, ... */ + + if ((flags & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_IO) { + ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_IO_32BITS; + } else { + bool is_prefetch = !!(flags & PCI_BASE_ADDRESS_MEM_PREFETCH); + bool is_64bits = !!(flags & PCI_BASE_ADDRESS_MEM_TYPE_64); + + if (is_64bits && (bar & 1)) + return -EINVAL; + + if (is_64bits && is_prefetch) + ctrl = + ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_PREFETCH_MEM_64BITS; + else if (is_prefetch) + ctrl = + ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_PREFETCH_MEM_32BITS; + else if (is_64bits) + ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_MEM_64BITS; + else + ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_MEM_32BITS; + } + + if (bar < BAR_4) { + reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG0(fn); + b = bar; + } else { + reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG1(fn); + b = bar - BAR_4; + } + + addr0 = lower_32_bits(bar_phys); + addr1 = upper_32_bits(bar_phys); + + cfg = rockchip_pcie_read(rockchip, reg); + cfg &= ~(ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) | + ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b)); + cfg |= (ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE(b, aperture) | + ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl)); + + rockchip_pcie_write(rockchip, cfg, reg); + rockchip_pcie_write(rockchip, addr0, + ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar)); + rockchip_pcie_write(rockchip, addr1, + ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar)); + + return 0; +} + +static void rockchip_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, u8 vfn, + struct pci_epf_bar *epf_bar) +{ + struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); + struct rockchip_pcie *rockchip = &ep->rockchip; + u32 reg, cfg, b, ctrl; + enum pci_barno bar = epf_bar->barno; + + if (bar < BAR_4) { + reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG0(fn); + b = bar; + } else { + reg = ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG1(fn); + b = bar - BAR_4; + } + + ctrl = ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_DISABLED; + cfg = rockchip_pcie_read(rockchip, reg); + cfg &= ~(ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) | + ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b)); + cfg |= ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl); + + rockchip_pcie_write(rockchip, cfg, reg); + rockchip_pcie_write(rockchip, 0x0, + ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar)); + rockchip_pcie_write(rockchip, 0x0, + ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar)); +} + +static inline u32 rockchip_ob_region(phys_addr_t addr) +{ + return (addr >> ilog2(SZ_1M)) & 0x1f; +} + +static u64 rockchip_pcie_ep_align_addr(struct pci_epc *epc, u64 pci_addr, + size_t *pci_size, size_t *addr_offset) +{ + struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); + size_t size = *pci_size; + u64 offset, mask; + int num_bits; + + num_bits = rockchip_pcie_ep_ob_atu_num_bits(&ep->rockchip, + pci_addr, size); + mask = (1ULL << num_bits) - 1; + + offset = pci_addr & mask; + if (size + offset > SZ_1M) + size = SZ_1M - offset; + + *pci_size = ALIGN(offset + size, ROCKCHIP_PCIE_AT_SIZE_ALIGN); + *addr_offset = offset; + + return pci_addr & ~mask; +} + +static int rockchip_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, u8 vfn, + phys_addr_t addr, u64 pci_addr, + size_t size) +{ + struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); + struct rockchip_pcie *pcie = &ep->rockchip; + u32 r = rockchip_ob_region(addr); + + if (test_bit(r, &ep->ob_region_map)) + return -EBUSY; + + rockchip_pcie_prog_ep_ob_atu(pcie, fn, r, addr, pci_addr, size); + + set_bit(r, &ep->ob_region_map); + ep->ob_addr[r] = addr; + + return 0; +} + +static void rockchip_pcie_ep_unmap_addr(struct pci_epc *epc, u8 fn, u8 vfn, + phys_addr_t addr) +{ + struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); + struct rockchip_pcie *rockchip = &ep->rockchip; + u32 r = rockchip_ob_region(addr); + + if (addr != ep->ob_addr[r] || !test_bit(r, &ep->ob_region_map)) + return; + + rockchip_pcie_clear_ep_ob_atu(rockchip, r); + + ep->ob_addr[r] = 0; + clear_bit(r, &ep->ob_region_map); +} + +static int rockchip_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, u8 vfn, + u8 nr_irqs) +{ + struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); + struct rockchip_pcie *rockchip = &ep->rockchip; + u8 mmc = order_base_2(nr_irqs); + u32 flags; + + flags = rockchip_pcie_read(rockchip, + ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + + ROCKCHIP_PCIE_EP_MSI_CTRL_REG); + flags &= ~ROCKCHIP_PCIE_EP_MSI_CTRL_MMC_MASK; + flags |= + (mmc << ROCKCHIP_PCIE_EP_MSI_CTRL_MMC_OFFSET) | + (PCI_MSI_FLAGS_64BIT << ROCKCHIP_PCIE_EP_MSI_FLAGS_OFFSET); + flags &= ~ROCKCHIP_PCIE_EP_MSI_CTRL_MASK_MSI_CAP; + rockchip_pcie_write(rockchip, flags, + ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + + ROCKCHIP_PCIE_EP_MSI_CTRL_REG); + return 0; +} + +static int rockchip_pcie_ep_get_msi(struct pci_epc *epc, u8 fn, u8 vfn) +{ + struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); + struct rockchip_pcie *rockchip = &ep->rockchip; + u32 flags; + + flags = rockchip_pcie_read(rockchip, + ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + + ROCKCHIP_PCIE_EP_MSI_CTRL_REG); + if (!(flags & ROCKCHIP_PCIE_EP_MSI_CTRL_ME)) + return -EINVAL; + + return 1 << ((flags & ROCKCHIP_PCIE_EP_MSI_CTRL_MME_MASK) >> + ROCKCHIP_PCIE_EP_MSI_CTRL_MME_OFFSET); +} + +static void rockchip_pcie_ep_assert_intx(struct rockchip_pcie_ep *ep, u8 fn, + u8 intx, bool do_assert) +{ + struct rockchip_pcie *rockchip = &ep->rockchip; + + intx &= 3; + + if (do_assert) { + ep->irq_pending |= BIT(intx); + rockchip_pcie_write(rockchip, + PCIE_CLIENT_INT_IN_ASSERT | + PCIE_CLIENT_INT_PEND_ST_PEND, + PCIE_CLIENT_LEGACY_INT_CTRL); + } else { + ep->irq_pending &= ~BIT(intx); + rockchip_pcie_write(rockchip, + PCIE_CLIENT_INT_IN_DEASSERT | + PCIE_CLIENT_INT_PEND_ST_NORMAL, + PCIE_CLIENT_LEGACY_INT_CTRL); + } +} + +static int rockchip_pcie_ep_send_intx_irq(struct rockchip_pcie_ep *ep, u8 fn, + u8 intx) +{ + u16 cmd; + + cmd = rockchip_pcie_read(&ep->rockchip, + ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + + ROCKCHIP_PCIE_EP_CMD_STATUS); + + if (cmd & PCI_COMMAND_INTX_DISABLE) + return -EINVAL; + + /* + * Should add some delay between toggling INTx per TRM vaguely saying + * it depends on some cycles of the AHB bus clock to function it. So + * add sufficient 1ms here. + */ + rockchip_pcie_ep_assert_intx(ep, fn, intx, true); + mdelay(1); + rockchip_pcie_ep_assert_intx(ep, fn, intx, false); + return 0; +} + +static int rockchip_pcie_ep_send_msi_irq(struct rockchip_pcie_ep *ep, u8 fn, + u8 interrupt_num) +{ + struct rockchip_pcie *rockchip = &ep->rockchip; + u32 flags, mme, data, data_mask; + size_t irq_pci_size, offset; + u64 irq_pci_addr; + u8 msi_count; + u64 pci_addr; + + /* Check MSI enable bit */ + flags = rockchip_pcie_read(&ep->rockchip, + ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + + ROCKCHIP_PCIE_EP_MSI_CTRL_REG); + if (!(flags & ROCKCHIP_PCIE_EP_MSI_CTRL_ME)) + return -EINVAL; + + /* Get MSI numbers from MME */ + mme = ((flags & ROCKCHIP_PCIE_EP_MSI_CTRL_MME_MASK) >> + ROCKCHIP_PCIE_EP_MSI_CTRL_MME_OFFSET); + msi_count = 1 << mme; + if (!interrupt_num || interrupt_num > msi_count) + return -EINVAL; + + /* Set MSI private data */ + data_mask = msi_count - 1; + data = rockchip_pcie_read(rockchip, + ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + + ROCKCHIP_PCIE_EP_MSI_CTRL_REG + + PCI_MSI_DATA_64); + data = (data & ~data_mask) | ((interrupt_num - 1) & data_mask); + + /* Get MSI PCI address */ + pci_addr = rockchip_pcie_read(rockchip, + ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + + ROCKCHIP_PCIE_EP_MSI_CTRL_REG + + PCI_MSI_ADDRESS_HI); + pci_addr <<= 32; + pci_addr |= rockchip_pcie_read(rockchip, + ROCKCHIP_PCIE_EP_FUNC_BASE(fn) + + ROCKCHIP_PCIE_EP_MSI_CTRL_REG + + PCI_MSI_ADDRESS_LO); + + /* Set the outbound region if needed. */ + irq_pci_size = ~PCIE_ADDR_MASK + 1; + irq_pci_addr = rockchip_pcie_ep_align_addr(ep->epc, + pci_addr & PCIE_ADDR_MASK, + &irq_pci_size, &offset); + if (unlikely(ep->irq_pci_addr != irq_pci_addr || + ep->irq_pci_fn != fn)) { + rockchip_pcie_prog_ep_ob_atu(rockchip, fn, + rockchip_ob_region(ep->irq_phys_addr), + ep->irq_phys_addr, + irq_pci_addr, irq_pci_size); + ep->irq_pci_addr = irq_pci_addr; + ep->irq_pci_fn = fn; + } + + writew(data, ep->irq_cpu_addr + offset + (pci_addr & ~PCIE_ADDR_MASK)); + return 0; +} + +static int rockchip_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn, u8 vfn, + unsigned int type, u16 interrupt_num) +{ + struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); + + switch (type) { + case PCI_IRQ_INTX: + return rockchip_pcie_ep_send_intx_irq(ep, fn, 0); + case PCI_IRQ_MSI: + return rockchip_pcie_ep_send_msi_irq(ep, fn, interrupt_num); + default: + return -EINVAL; + } +} + +static int rockchip_pcie_ep_start(struct pci_epc *epc) +{ + struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); + struct rockchip_pcie *rockchip = &ep->rockchip; + struct pci_epf *epf; + u32 cfg; + + cfg = BIT(0); + list_for_each_entry(epf, &epc->pci_epf, list) + cfg |= BIT(epf->func_no); + + rockchip_pcie_write(rockchip, cfg, PCIE_CORE_PHY_FUNC_CFG); + + if (rockchip->perst_gpio) + enable_irq(ep->perst_irq); + + /* Enable configuration and start link training */ + rockchip_pcie_write(rockchip, + PCIE_CLIENT_LINK_TRAIN_ENABLE | + PCIE_CLIENT_CONF_ENABLE, + PCIE_CLIENT_CONFIG); + + if (!rockchip->perst_gpio) + schedule_delayed_work(&ep->link_training, 0); + + return 0; +} + +static void rockchip_pcie_ep_stop(struct pci_epc *epc) +{ + struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); + struct rockchip_pcie *rockchip = &ep->rockchip; + + if (rockchip->perst_gpio) { + ep->perst_asserted = true; + disable_irq(ep->perst_irq); + } + + cancel_delayed_work_sync(&ep->link_training); + + /* Stop link training and disable configuration */ + rockchip_pcie_write(rockchip, + PCIE_CLIENT_CONF_DISABLE | + PCIE_CLIENT_LINK_TRAIN_DISABLE, + PCIE_CLIENT_CONFIG); +} + +static void rockchip_pcie_ep_retrain_link(struct rockchip_pcie *rockchip) +{ + u32 status; + + status = rockchip_pcie_read(rockchip, PCIE_EP_CONFIG_BASE + PCI_EXP_LNKCTL); + status |= PCI_EXP_LNKCTL_RL; + rockchip_pcie_write(rockchip, status, PCIE_EP_CONFIG_BASE + PCI_EXP_LNKCTL); +} + +static bool rockchip_pcie_ep_link_up(struct rockchip_pcie *rockchip) +{ + u32 val = rockchip_pcie_read(rockchip, PCIE_CLIENT_BASIC_STATUS1); + + return PCIE_LINK_UP(val); +} + +static void rockchip_pcie_ep_link_training(struct work_struct *work) +{ + struct rockchip_pcie_ep *ep = + container_of(work, struct rockchip_pcie_ep, link_training.work); + struct rockchip_pcie *rockchip = &ep->rockchip; + struct device *dev = rockchip->dev; + u32 val; + int ret; + + /* Enable Gen1 training and wait for its completion */ + ret = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL, + val, PCIE_LINK_TRAINING_DONE(val), 50, + LINK_TRAIN_TIMEOUT); + if (ret) + goto again; + + /* Make sure that the link is up */ + ret = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1, + val, PCIE_LINK_UP(val), 50, + LINK_TRAIN_TIMEOUT); + if (ret) + goto again; + + /* + * Check the current speed: if gen2 speed was requested and we are not + * at gen2 speed yet, retrain again for gen2. + */ + val = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL); + if (!PCIE_LINK_IS_GEN2(val) && rockchip->link_gen == 2) { + /* Enable retrain for gen2 */ + rockchip_pcie_ep_retrain_link(rockchip); + readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL, + val, PCIE_LINK_IS_GEN2(val), 50, + LINK_TRAIN_TIMEOUT); + } + + /* Check again that the link is up */ + if (!rockchip_pcie_ep_link_up(rockchip)) + goto again; + + /* + * If PERST# was asserted while polling the link, do not notify + * the function. + */ + if (ep->perst_asserted) + return; + + val = rockchip_pcie_read(rockchip, PCIE_CLIENT_BASIC_STATUS0); + dev_info(dev, + "link up (negotiated speed: %sGT/s, width: x%lu)\n", + (val & PCIE_CLIENT_NEG_LINK_SPEED) ? "5" : "2.5", + ((val & PCIE_CLIENT_NEG_LINK_WIDTH_MASK) >> + PCIE_CLIENT_NEG_LINK_WIDTH_SHIFT) << 1); + + /* Notify the function */ + pci_epc_linkup(ep->epc); + ep->link_up = true; + + return; + +again: + schedule_delayed_work(&ep->link_training, msecs_to_jiffies(5)); +} + +static void rockchip_pcie_ep_perst_assert(struct rockchip_pcie_ep *ep) +{ + struct rockchip_pcie *rockchip = &ep->rockchip; + + dev_dbg(rockchip->dev, "PERST# asserted, link down\n"); + + if (ep->perst_asserted) + return; + + ep->perst_asserted = true; + + cancel_delayed_work_sync(&ep->link_training); + + if (ep->link_up) { + pci_epc_linkdown(ep->epc); + ep->link_up = false; + } +} + +static void rockchip_pcie_ep_perst_deassert(struct rockchip_pcie_ep *ep) +{ + struct rockchip_pcie *rockchip = &ep->rockchip; + + dev_dbg(rockchip->dev, "PERST# de-asserted, starting link training\n"); + + if (!ep->perst_asserted) + return; + + ep->perst_asserted = false; + + /* Enable link re-training */ + rockchip_pcie_ep_retrain_link(rockchip); + + /* Start link training */ + schedule_delayed_work(&ep->link_training, 0); +} + +static irqreturn_t rockchip_pcie_ep_perst_irq_thread(int irq, void *data) +{ + struct pci_epc *epc = data; + struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); + struct rockchip_pcie *rockchip = &ep->rockchip; + u32 perst = gpiod_get_value(rockchip->perst_gpio); + + if (perst) + rockchip_pcie_ep_perst_assert(ep); + else + rockchip_pcie_ep_perst_deassert(ep); + + irq_set_irq_type(ep->perst_irq, + (perst ? IRQF_TRIGGER_HIGH : IRQF_TRIGGER_LOW)); + + return IRQ_HANDLED; +} + +static int rockchip_pcie_ep_setup_irq(struct pci_epc *epc) +{ + struct rockchip_pcie_ep *ep = epc_get_drvdata(epc); + struct rockchip_pcie *rockchip = &ep->rockchip; + struct device *dev = rockchip->dev; + int ret; + + if (!rockchip->perst_gpio) + return 0; + + /* PCIe reset interrupt */ + ep->perst_irq = gpiod_to_irq(rockchip->perst_gpio); + if (ep->perst_irq < 0) { + dev_err(dev, + "failed to get IRQ for PERST# GPIO: %d\n", + ep->perst_irq); + + return ep->perst_irq; + } + + /* + * The perst_gpio is active low, so when it is inactive on start, it + * is high and will trigger the perst_irq handler. So treat this initial + * IRQ as a dummy one by faking the host asserting PERST#. + */ + ep->perst_asserted = true; + irq_set_status_flags(ep->perst_irq, IRQ_NOAUTOEN); + ret = devm_request_threaded_irq(dev, ep->perst_irq, NULL, + rockchip_pcie_ep_perst_irq_thread, + IRQF_TRIGGER_HIGH | IRQF_ONESHOT, + "pcie-ep-perst", epc); + if (ret) { + dev_err(dev, + "failed to request IRQ for PERST# GPIO: %d\n", + ret); + + return ret; + } + + return 0; +} + +static const struct pci_epc_features rockchip_pcie_epc_features = { + .linkup_notifier = true, + .msi_capable = true, + .intx_capable = true, + .align = ROCKCHIP_PCIE_AT_SIZE_ALIGN, +}; + +static const struct pci_epc_features* +rockchip_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no) +{ + return &rockchip_pcie_epc_features; +} + +static const struct pci_epc_ops rockchip_pcie_epc_ops = { + .write_header = rockchip_pcie_ep_write_header, + .set_bar = rockchip_pcie_ep_set_bar, + .clear_bar = rockchip_pcie_ep_clear_bar, + .align_addr = rockchip_pcie_ep_align_addr, + .map_addr = rockchip_pcie_ep_map_addr, + .unmap_addr = rockchip_pcie_ep_unmap_addr, + .set_msi = rockchip_pcie_ep_set_msi, + .get_msi = rockchip_pcie_ep_get_msi, + .raise_irq = rockchip_pcie_ep_raise_irq, + .start = rockchip_pcie_ep_start, + .stop = rockchip_pcie_ep_stop, + .get_features = rockchip_pcie_ep_get_features, +}; + +static int rockchip_pcie_ep_get_resources(struct rockchip_pcie *rockchip, + struct rockchip_pcie_ep *ep) +{ + struct device *dev = rockchip->dev; + int err; + + err = rockchip_pcie_parse_dt(rockchip); + if (err) + return err; + + err = rockchip_pcie_get_phys(rockchip); + if (err) + return err; + + err = of_property_read_u32(dev->of_node, + "rockchip,max-outbound-regions", + &ep->max_regions); + if (err < 0 || ep->max_regions > MAX_REGION_LIMIT) + ep->max_regions = MAX_REGION_LIMIT; + + ep->ob_region_map = 0; + + err = of_property_read_u8(dev->of_node, "max-functions", + &ep->epc->max_functions); + if (err < 0) + ep->epc->max_functions = 1; + + return 0; +} + +static const struct of_device_id rockchip_pcie_ep_of_match[] = { + { .compatible = "rockchip,rk3399-pcie-ep"}, + {}, +}; + +static int rockchip_pcie_ep_init_ob_mem(struct rockchip_pcie_ep *ep) +{ + struct rockchip_pcie *rockchip = &ep->rockchip; + struct device *dev = rockchip->dev; + struct pci_epc_mem_window *windows = NULL; + int err, i; + + ep->ob_addr = devm_kcalloc(dev, ep->max_regions, sizeof(*ep->ob_addr), + GFP_KERNEL); + + if (!ep->ob_addr) + return -ENOMEM; + + windows = devm_kcalloc(dev, ep->max_regions, + sizeof(struct pci_epc_mem_window), GFP_KERNEL); + if (!windows) + return -ENOMEM; + + for (i = 0; i < ep->max_regions; i++) { + windows[i].phys_base = rockchip->mem_res->start + (SZ_1M * i); + windows[i].size = SZ_1M; + windows[i].page_size = SZ_1M; + } + err = pci_epc_multi_mem_init(ep->epc, windows, ep->max_regions); + devm_kfree(dev, windows); + + if (err < 0) { + dev_err(dev, "failed to initialize the memory space\n"); + return err; + } + + ep->irq_cpu_addr = pci_epc_mem_alloc_addr(ep->epc, &ep->irq_phys_addr, + SZ_1M); + if (!ep->irq_cpu_addr) { + dev_err(dev, "failed to reserve memory space for MSI\n"); + err = -ENOMEM; + goto err_epc_mem_exit; + } + + ep->irq_pci_addr = ROCKCHIP_PCIE_EP_DUMMY_IRQ_ADDR; + + return 0; + +err_epc_mem_exit: + pci_epc_mem_exit(ep->epc); + + return err; +} + +static void rockchip_pcie_ep_exit_ob_mem(struct rockchip_pcie_ep *ep) +{ + pci_epc_mem_exit(ep->epc); +} + +static void rockchip_pcie_ep_hide_broken_msix_cap(struct rockchip_pcie *rockchip) +{ + u32 cfg_msi, cfg_msix_cp; + + /* + * MSI-X is not supported but the controller still advertises the MSI-X + * capability by default, which can lead to the Root Complex side + * allocating MSI-X vectors which cannot be used. Avoid this by skipping + * the MSI-X capability entry in the PCIe capabilities linked-list: get + * the next pointer from the MSI-X entry and set that in the MSI + * capability entry (which is the previous entry). This way the MSI-X + * entry is skipped (left out of the linked-list) and not advertised. + */ + cfg_msi = rockchip_pcie_read(rockchip, PCIE_EP_CONFIG_BASE + + ROCKCHIP_PCIE_EP_MSI_CTRL_REG); + + cfg_msi &= ~ROCKCHIP_PCIE_EP_MSI_CP1_MASK; + + cfg_msix_cp = rockchip_pcie_read(rockchip, PCIE_EP_CONFIG_BASE + + ROCKCHIP_PCIE_EP_MSIX_CAP_REG) & + ROCKCHIP_PCIE_EP_MSIX_CAP_CP_MASK; + + cfg_msi |= cfg_msix_cp; + + rockchip_pcie_write(rockchip, cfg_msi, + PCIE_EP_CONFIG_BASE + ROCKCHIP_PCIE_EP_MSI_CTRL_REG); +} + +static int rockchip_pcie_ep_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct rockchip_pcie_ep *ep; + struct rockchip_pcie *rockchip; + struct pci_epc *epc; + int err; + + ep = devm_kzalloc(dev, sizeof(*ep), GFP_KERNEL); + if (!ep) + return -ENOMEM; + + rockchip = &ep->rockchip; + rockchip->is_rc = false; + rockchip->dev = dev; + INIT_DELAYED_WORK(&ep->link_training, rockchip_pcie_ep_link_training); + + epc = devm_pci_epc_create(dev, &rockchip_pcie_epc_ops); + if (IS_ERR(epc)) { + dev_err(dev, "failed to create EPC device\n"); + return PTR_ERR(epc); + } + + ep->epc = epc; + epc_set_drvdata(epc, ep); + + err = rockchip_pcie_ep_get_resources(rockchip, ep); + if (err) + return err; + + err = rockchip_pcie_ep_init_ob_mem(ep); + if (err) + return err; + + err = rockchip_pcie_enable_clocks(rockchip); + if (err) + goto err_exit_ob_mem; + + err = rockchip_pcie_init_port(rockchip); + if (err) + goto err_disable_clocks; + + rockchip_pcie_ep_hide_broken_msix_cap(rockchip); + + /* Only enable function 0 by default */ + rockchip_pcie_write(rockchip, BIT(0), PCIE_CORE_PHY_FUNC_CFG); + + pci_epc_init_notify(epc); + + err = rockchip_pcie_ep_setup_irq(epc); + if (err < 0) + goto err_uninit_port; + + return 0; +err_uninit_port: + rockchip_pcie_deinit_phys(rockchip); +err_disable_clocks: + rockchip_pcie_disable_clocks(rockchip); +err_exit_ob_mem: + rockchip_pcie_ep_exit_ob_mem(ep); + return err; +} + +static struct platform_driver rockchip_pcie_ep_driver = { + .driver = { + .name = "rockchip-pcie-ep", + .of_match_table = rockchip_pcie_ep_of_match, + }, + .probe = rockchip_pcie_ep_probe, +}; + +builtin_platform_driver(rockchip_pcie_ep_driver); diff --git a/drivers/pci/controller/pcie-rockchip-host.c b/drivers/pci/controller/pcie-rockchip-host.c new file mode 100644 index 000000000..d203c4876 --- /dev/null +++ b/drivers/pci/controller/pcie-rockchip-host.c @@ -0,0 +1,1057 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Rockchip AXI PCIe host controller driver + * + * Copyright (c) 2016 Rockchip, Inc. + * + * Author: Shawn Lin <shawn.lin@rock-chips.com> + * Wenrui Li <wenrui.li@rock-chips.com> + * + * Bits taken from Synopsys DesignWare Host controller driver and + * ARM PCI Host generic driver. + */ + +#include <linux/bitfield.h> +#include <linux/bitrev.h> +#include <linux/gpio/consumer.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/irq.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqdomain.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_pci.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> + +#include "../pci.h" +#include "pcie-rockchip.h" + +static void rockchip_pcie_enable_bw_int(struct rockchip_pcie *rockchip) +{ + u32 status; + + status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); + status |= (PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE); + rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); +} + +static void rockchip_pcie_clr_bw_int(struct rockchip_pcie *rockchip) +{ + u32 status; + + status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); + status |= (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS) << 16; + rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); +} + +static void rockchip_pcie_update_txcredit_mui(struct rockchip_pcie *rockchip) +{ + u32 val; + + /* Update Tx credit maximum update interval */ + val = rockchip_pcie_read(rockchip, PCIE_CORE_TXCREDIT_CFG1); + val &= ~PCIE_CORE_TXCREDIT_CFG1_MUI_MASK; + val |= PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(24000); /* ns */ + rockchip_pcie_write(rockchip, val, PCIE_CORE_TXCREDIT_CFG1); +} + +static int rockchip_pcie_valid_device(struct rockchip_pcie *rockchip, + struct pci_bus *bus, int dev) +{ + /* + * Access only one slot on each root port. + * Do not read more than one device on the bus directly attached + * to RC's downstream side. + */ + if (pci_is_root_bus(bus) || pci_is_root_bus(bus->parent)) + return dev == 0; + + return 1; +} + +static u8 rockchip_pcie_lane_map(struct rockchip_pcie *rockchip) +{ + u32 val; + u8 map; + + if (rockchip->legacy_phy) + return GENMASK(MAX_LANE_NUM - 1, 0); + + val = rockchip_pcie_read(rockchip, PCIE_CORE_LANE_MAP); + map = val & PCIE_CORE_LANE_MAP_MASK; + + /* The link may be using a reverse-indexed mapping. */ + if (val & PCIE_CORE_LANE_MAP_REVERSE) + map = bitrev8(map) >> 4; + + return map; +} + +static int rockchip_pcie_rd_own_conf(struct rockchip_pcie *rockchip, + int where, int size, u32 *val) +{ + void __iomem *addr; + + addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + where; + + if (!IS_ALIGNED((uintptr_t)addr, size)) { + *val = 0; + return PCIBIOS_BAD_REGISTER_NUMBER; + } + + if (size == 4) { + *val = readl(addr); + } else if (size == 2) { + *val = readw(addr); + } else if (size == 1) { + *val = readb(addr); + } else { + *val = 0; + return PCIBIOS_BAD_REGISTER_NUMBER; + } + return PCIBIOS_SUCCESSFUL; +} + +static int rockchip_pcie_wr_own_conf(struct rockchip_pcie *rockchip, + int where, int size, u32 val) +{ + u32 mask, tmp, offset; + void __iomem *addr; + + offset = where & ~0x3; + addr = rockchip->apb_base + PCIE_RC_CONFIG_NORMAL_BASE + offset; + + if (size == 4) { + writel(val, addr); + return PCIBIOS_SUCCESSFUL; + } + + mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); + + /* + * N.B. This read/modify/write isn't safe in general because it can + * corrupt RW1C bits in adjacent registers. But the hardware + * doesn't support smaller writes. + */ + tmp = readl(addr) & mask; + tmp |= val << ((where & 0x3) * 8); + writel(tmp, addr); + + return PCIBIOS_SUCCESSFUL; +} + +static int rockchip_pcie_rd_other_conf(struct rockchip_pcie *rockchip, + struct pci_bus *bus, u32 devfn, + int where, int size, u32 *val) +{ + void __iomem *addr; + + addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); + + if (!IS_ALIGNED((uintptr_t)addr, size)) { + *val = 0; + return PCIBIOS_BAD_REGISTER_NUMBER; + } + + if (pci_is_root_bus(bus->parent)) + rockchip_pcie_cfg_configuration_accesses(rockchip, + AXI_WRAPPER_TYPE0_CFG); + else + rockchip_pcie_cfg_configuration_accesses(rockchip, + AXI_WRAPPER_TYPE1_CFG); + + if (size == 4) { + *val = readl(addr); + } else if (size == 2) { + *val = readw(addr); + } else if (size == 1) { + *val = readb(addr); + } else { + *val = 0; + return PCIBIOS_BAD_REGISTER_NUMBER; + } + return PCIBIOS_SUCCESSFUL; +} + +static int rockchip_pcie_wr_other_conf(struct rockchip_pcie *rockchip, + struct pci_bus *bus, u32 devfn, + int where, int size, u32 val) +{ + void __iomem *addr; + + addr = rockchip->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); + + if (!IS_ALIGNED((uintptr_t)addr, size)) + return PCIBIOS_BAD_REGISTER_NUMBER; + + if (pci_is_root_bus(bus->parent)) + rockchip_pcie_cfg_configuration_accesses(rockchip, + AXI_WRAPPER_TYPE0_CFG); + else + rockchip_pcie_cfg_configuration_accesses(rockchip, + AXI_WRAPPER_TYPE1_CFG); + + if (size == 4) + writel(val, addr); + else if (size == 2) + writew(val, addr); + else if (size == 1) + writeb(val, addr); + else + return PCIBIOS_BAD_REGISTER_NUMBER; + + return PCIBIOS_SUCCESSFUL; +} + +static int rockchip_pcie_rd_conf(struct pci_bus *bus, u32 devfn, int where, + int size, u32 *val) +{ + struct rockchip_pcie *rockchip = bus->sysdata; + + if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) + return PCIBIOS_DEVICE_NOT_FOUND; + + if (pci_is_root_bus(bus)) + return rockchip_pcie_rd_own_conf(rockchip, where, size, val); + + return rockchip_pcie_rd_other_conf(rockchip, bus, devfn, where, size, + val); +} + +static int rockchip_pcie_wr_conf(struct pci_bus *bus, u32 devfn, + int where, int size, u32 val) +{ + struct rockchip_pcie *rockchip = bus->sysdata; + + if (!rockchip_pcie_valid_device(rockchip, bus, PCI_SLOT(devfn))) + return PCIBIOS_DEVICE_NOT_FOUND; + + if (pci_is_root_bus(bus)) + return rockchip_pcie_wr_own_conf(rockchip, where, size, val); + + return rockchip_pcie_wr_other_conf(rockchip, bus, devfn, where, size, + val); +} + +static struct pci_ops rockchip_pcie_ops = { + .read = rockchip_pcie_rd_conf, + .write = rockchip_pcie_wr_conf, +}; + +static void rockchip_pcie_set_power_limit(struct rockchip_pcie *rockchip) +{ + int curr; + u32 status, scale, power; + + if (IS_ERR(rockchip->vpcie3v3)) + return; + + /* + * Set RC's captured slot power limit and scale if + * vpcie3v3 available. The default values are both zero + * which means the software should set these two according + * to the actual power supply. + */ + curr = regulator_get_current_limit(rockchip->vpcie3v3); + if (curr <= 0) + return; + + scale = 3; /* 0.001x */ + curr = curr / 1000; /* convert to mA */ + power = (curr * 3300) / 1000; /* milliwatt */ + while (power > FIELD_MAX(PCI_EXP_DEVCAP_PWR_VAL)) { + if (!scale) { + dev_warn(rockchip->dev, "invalid power supply\n"); + return; + } + scale--; + power = power / 10; + } + + status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCAP); + status |= FIELD_PREP(PCI_EXP_DEVCAP_PWR_VAL, power); + status |= FIELD_PREP(PCI_EXP_DEVCAP_PWR_SCL, scale); + rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCAP); +} + +/** + * rockchip_pcie_host_init_port - Initialize hardware + * @rockchip: PCIe port information + */ +static int rockchip_pcie_host_init_port(struct rockchip_pcie *rockchip) +{ + struct device *dev = rockchip->dev; + int err, i = MAX_LANE_NUM; + u32 status; + + gpiod_set_value_cansleep(rockchip->perst_gpio, 0); + + err = rockchip_pcie_init_port(rockchip); + if (err) + return err; + + /* Fix the transmitted FTS count desired to exit from L0s. */ + status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL_PLC1); + status = (status & ~PCIE_CORE_CTRL_PLC1_FTS_MASK) | + (PCIE_CORE_CTRL_PLC1_FTS_CNT << PCIE_CORE_CTRL_PLC1_FTS_SHIFT); + rockchip_pcie_write(rockchip, status, PCIE_CORE_CTRL_PLC1); + + rockchip_pcie_set_power_limit(rockchip); + + /* Set RC's clock architecture as common clock */ + status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); + status |= PCI_EXP_LNKSTA_SLC << 16; + rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); + + /* Set RC's RCB to 128 */ + status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); + status |= PCI_EXP_LNKCTL_RCB; + rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); + + /* Enable Gen1 training */ + rockchip_pcie_write(rockchip, PCIE_CLIENT_LINK_TRAIN_ENABLE, + PCIE_CLIENT_CONFIG); + + msleep(PCIE_T_PVPERL_MS); + gpiod_set_value_cansleep(rockchip->perst_gpio, 1); + + msleep(PCIE_RESET_CONFIG_WAIT_MS); + + /* 500ms timeout value should be enough for Gen1/2 training */ + err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_BASIC_STATUS1, + status, PCIE_LINK_UP(status), 20, + 500 * USEC_PER_MSEC); + if (err) { + dev_err(dev, "PCIe link training gen1 timeout!\n"); + goto err_power_off_phy; + } + + if (rockchip->link_gen == 2) { + /* + * Enable retrain for gen2. This should be configured only after + * gen1 finished. + */ + status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL2); + status &= ~PCI_EXP_LNKCTL2_TLS; + status |= PCI_EXP_LNKCTL2_TLS_5_0GT; + rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL2); + status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); + status |= PCI_EXP_LNKCTL_RL; + rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCTL); + + err = readl_poll_timeout(rockchip->apb_base + PCIE_CORE_CTRL, + status, PCIE_LINK_IS_GEN2(status), 20, + 500 * USEC_PER_MSEC); + if (err) + dev_dbg(dev, "PCIe link training gen2 timeout, fall back to gen1!\n"); + } + + /* Check the final link width from negotiated lane counter from MGMT */ + status = rockchip_pcie_read(rockchip, PCIE_CORE_CTRL); + status = 0x1 << ((status & PCIE_CORE_PL_CONF_LANE_MASK) >> + PCIE_CORE_PL_CONF_LANE_SHIFT); + dev_dbg(dev, "current link width is x%d\n", status); + + /* Power off unused lane(s) */ + rockchip->lanes_map = rockchip_pcie_lane_map(rockchip); + for (i = 0; i < MAX_LANE_NUM; i++) { + if (!(rockchip->lanes_map & BIT(i))) { + dev_dbg(dev, "idling lane %d\n", i); + phy_power_off(rockchip->phys[i]); + } + } + + rockchip_pcie_write(rockchip, PCI_VENDOR_ID_ROCKCHIP, + PCIE_CORE_CONFIG_VENDOR); + rockchip_pcie_write(rockchip, + PCI_CLASS_BRIDGE_PCI_NORMAL << 8, + PCIE_RC_CONFIG_RID_CCR); + + /* Clear THP cap's next cap pointer to remove L1 substate cap */ + status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_THP_CAP); + status &= ~PCIE_RC_CONFIG_THP_CAP_NEXT_MASK; + rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_THP_CAP); + + /* Clear L0s from RC's link cap */ + if (of_property_read_bool(dev->of_node, "aspm-no-l0s")) { + status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCAP); + status &= ~PCI_EXP_LNKCAP_ASPM_L0S; + rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_LNKCAP); + } + + status = rockchip_pcie_read(rockchip, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCTL); + status &= ~PCI_EXP_DEVCTL_PAYLOAD; + status |= PCI_EXP_DEVCTL_PAYLOAD_256B; + rockchip_pcie_write(rockchip, status, PCIE_RC_CONFIG_CR + PCI_EXP_DEVCTL); + + return 0; +err_power_off_phy: + while (i--) + phy_power_off(rockchip->phys[i]); + i = MAX_LANE_NUM; + while (i--) + phy_exit(rockchip->phys[i]); + return err; +} + +static irqreturn_t rockchip_pcie_subsys_irq_handler(int irq, void *arg) +{ + struct rockchip_pcie *rockchip = arg; + struct device *dev = rockchip->dev; + u32 reg; + u32 sub_reg; + + reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); + if (reg & PCIE_CLIENT_INT_LOCAL) { + dev_dbg(dev, "local interrupt received\n"); + sub_reg = rockchip_pcie_read(rockchip, PCIE_CORE_INT_STATUS); + if (sub_reg & PCIE_CORE_INT_PRFPE) + dev_dbg(dev, "parity error detected while reading from the PNP receive FIFO RAM\n"); + + if (sub_reg & PCIE_CORE_INT_CRFPE) + dev_dbg(dev, "parity error detected while reading from the Completion Receive FIFO RAM\n"); + + if (sub_reg & PCIE_CORE_INT_RRPE) + dev_dbg(dev, "parity error detected while reading from replay buffer RAM\n"); + + if (sub_reg & PCIE_CORE_INT_PRFO) + dev_dbg(dev, "overflow occurred in the PNP receive FIFO\n"); + + if (sub_reg & PCIE_CORE_INT_CRFO) + dev_dbg(dev, "overflow occurred in the completion receive FIFO\n"); + + if (sub_reg & PCIE_CORE_INT_RT) + dev_dbg(dev, "replay timer timed out\n"); + + if (sub_reg & PCIE_CORE_INT_RTR) + dev_dbg(dev, "replay timer rolled over after 4 transmissions of the same TLP\n"); + + if (sub_reg & PCIE_CORE_INT_PE) + dev_dbg(dev, "phy error detected on receive side\n"); + + if (sub_reg & PCIE_CORE_INT_MTR) + dev_dbg(dev, "malformed TLP received from the link\n"); + + if (sub_reg & PCIE_CORE_INT_UCR) + dev_dbg(dev, "Unexpected Completion received from the link\n"); + + if (sub_reg & PCIE_CORE_INT_FCE) + dev_dbg(dev, "an error was observed in the flow control advertisements from the other side\n"); + + if (sub_reg & PCIE_CORE_INT_CT) + dev_dbg(dev, "a request timed out waiting for completion\n"); + + if (sub_reg & PCIE_CORE_INT_UTC) + dev_dbg(dev, "unmapped TC error\n"); + + if (sub_reg & PCIE_CORE_INT_MMVC) + dev_dbg(dev, "MSI mask register changes\n"); + + rockchip_pcie_write(rockchip, sub_reg, PCIE_CORE_INT_STATUS); + } else if (reg & PCIE_CLIENT_INT_PHY) { + dev_dbg(dev, "phy link changes\n"); + rockchip_pcie_update_txcredit_mui(rockchip); + rockchip_pcie_clr_bw_int(rockchip); + } + + rockchip_pcie_write(rockchip, reg & PCIE_CLIENT_INT_LOCAL, + PCIE_CLIENT_INT_STATUS); + + return IRQ_HANDLED; +} + +static irqreturn_t rockchip_pcie_client_irq_handler(int irq, void *arg) +{ + struct rockchip_pcie *rockchip = arg; + struct device *dev = rockchip->dev; + u32 reg; + + reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); + if (reg & PCIE_CLIENT_INT_LEGACY_DONE) + dev_dbg(dev, "legacy done interrupt received\n"); + + if (reg & PCIE_CLIENT_INT_MSG) + dev_dbg(dev, "message done interrupt received\n"); + + if (reg & PCIE_CLIENT_INT_HOT_RST) + dev_dbg(dev, "hot reset interrupt received\n"); + + if (reg & PCIE_CLIENT_INT_DPA) + dev_dbg(dev, "dpa interrupt received\n"); + + if (reg & PCIE_CLIENT_INT_FATAL_ERR) + dev_dbg(dev, "fatal error interrupt received\n"); + + if (reg & PCIE_CLIENT_INT_NFATAL_ERR) + dev_dbg(dev, "non fatal error interrupt received\n"); + + if (reg & PCIE_CLIENT_INT_CORR_ERR) + dev_dbg(dev, "correctable error interrupt received\n"); + + if (reg & PCIE_CLIENT_INT_PHY) + dev_dbg(dev, "phy interrupt received\n"); + + rockchip_pcie_write(rockchip, reg & (PCIE_CLIENT_INT_LEGACY_DONE | + PCIE_CLIENT_INT_MSG | PCIE_CLIENT_INT_HOT_RST | + PCIE_CLIENT_INT_DPA | PCIE_CLIENT_INT_FATAL_ERR | + PCIE_CLIENT_INT_NFATAL_ERR | + PCIE_CLIENT_INT_CORR_ERR | + PCIE_CLIENT_INT_PHY), + PCIE_CLIENT_INT_STATUS); + + return IRQ_HANDLED; +} + +static void rockchip_pcie_intx_handler(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + struct rockchip_pcie *rockchip = irq_desc_get_handler_data(desc); + struct device *dev = rockchip->dev; + u32 reg; + u32 hwirq; + int ret; + + chained_irq_enter(chip, desc); + + reg = rockchip_pcie_read(rockchip, PCIE_CLIENT_INT_STATUS); + reg = (reg & PCIE_CLIENT_INTR_MASK) >> PCIE_CLIENT_INTR_SHIFT; + + while (reg) { + hwirq = ffs(reg) - 1; + reg &= ~BIT(hwirq); + + ret = generic_handle_domain_irq(rockchip->irq_domain, hwirq); + if (ret) + dev_err(dev, "unexpected IRQ, INT%d\n", hwirq); + } + + chained_irq_exit(chip, desc); +} + +static int rockchip_pcie_setup_irq(struct rockchip_pcie *rockchip) +{ + int irq, err; + struct device *dev = rockchip->dev; + struct platform_device *pdev = to_platform_device(dev); + + irq = platform_get_irq_byname(pdev, "sys"); + if (irq < 0) + return irq; + + err = devm_request_irq(dev, irq, rockchip_pcie_subsys_irq_handler, + IRQF_SHARED, "pcie-sys", rockchip); + if (err) { + dev_err(dev, "failed to request PCIe subsystem IRQ\n"); + return err; + } + + irq = platform_get_irq_byname(pdev, "legacy"); + if (irq < 0) + return irq; + + irq_set_chained_handler_and_data(irq, + rockchip_pcie_intx_handler, + rockchip); + + irq = platform_get_irq_byname(pdev, "client"); + if (irq < 0) + return irq; + + err = devm_request_irq(dev, irq, rockchip_pcie_client_irq_handler, + IRQF_SHARED, "pcie-client", rockchip); + if (err) { + dev_err(dev, "failed to request PCIe client IRQ\n"); + return err; + } + + return 0; +} + +/** + * rockchip_pcie_parse_host_dt - Parse Device Tree + * @rockchip: PCIe port information + * + * Return: '0' on success and error value on failure + */ +static int rockchip_pcie_parse_host_dt(struct rockchip_pcie *rockchip) +{ + struct device *dev = rockchip->dev; + int err; + + err = rockchip_pcie_parse_dt(rockchip); + if (err) + return err; + + rockchip->vpcie12v = devm_regulator_get_optional(dev, "vpcie12v"); + if (IS_ERR(rockchip->vpcie12v)) { + if (PTR_ERR(rockchip->vpcie12v) != -ENODEV) + return PTR_ERR(rockchip->vpcie12v); + dev_info(dev, "no vpcie12v regulator found\n"); + } + + rockchip->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3"); + if (IS_ERR(rockchip->vpcie3v3)) { + if (PTR_ERR(rockchip->vpcie3v3) != -ENODEV) + return PTR_ERR(rockchip->vpcie3v3); + dev_info(dev, "no vpcie3v3 regulator found\n"); + } + + rockchip->vpcie1v8 = devm_regulator_get(dev, "vpcie1v8"); + if (IS_ERR(rockchip->vpcie1v8)) + return PTR_ERR(rockchip->vpcie1v8); + + rockchip->vpcie0v9 = devm_regulator_get(dev, "vpcie0v9"); + if (IS_ERR(rockchip->vpcie0v9)) + return PTR_ERR(rockchip->vpcie0v9); + + return 0; +} + +static int rockchip_pcie_set_vpcie(struct rockchip_pcie *rockchip) +{ + struct device *dev = rockchip->dev; + int err; + + if (!IS_ERR(rockchip->vpcie12v)) { + err = regulator_enable(rockchip->vpcie12v); + if (err) { + dev_err(dev, "fail to enable vpcie12v regulator\n"); + goto err_out; + } + } + + if (!IS_ERR(rockchip->vpcie3v3)) { + err = regulator_enable(rockchip->vpcie3v3); + if (err) { + dev_err(dev, "fail to enable vpcie3v3 regulator\n"); + goto err_disable_12v; + } + } + + err = regulator_enable(rockchip->vpcie1v8); + if (err) { + dev_err(dev, "fail to enable vpcie1v8 regulator\n"); + goto err_disable_3v3; + } + + err = regulator_enable(rockchip->vpcie0v9); + if (err) { + dev_err(dev, "fail to enable vpcie0v9 regulator\n"); + goto err_disable_1v8; + } + + return 0; + +err_disable_1v8: + regulator_disable(rockchip->vpcie1v8); +err_disable_3v3: + if (!IS_ERR(rockchip->vpcie3v3)) + regulator_disable(rockchip->vpcie3v3); +err_disable_12v: + if (!IS_ERR(rockchip->vpcie12v)) + regulator_disable(rockchip->vpcie12v); +err_out: + return err; +} + +static void rockchip_pcie_enable_interrupts(struct rockchip_pcie *rockchip) +{ + rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) & + (~PCIE_CLIENT_INT_CLI), PCIE_CLIENT_INT_MASK); + rockchip_pcie_write(rockchip, (u32)(~PCIE_CORE_INT), + PCIE_CORE_INT_MASK); + + rockchip_pcie_enable_bw_int(rockchip); +} + +static int rockchip_pcie_intx_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq); + irq_set_chip_data(irq, domain->host_data); + + return 0; +} + +static const struct irq_domain_ops intx_domain_ops = { + .map = rockchip_pcie_intx_map, +}; + +static int rockchip_pcie_init_irq_domain(struct rockchip_pcie *rockchip) +{ + struct device *dev = rockchip->dev; + struct device_node *intc = of_get_next_child(dev->of_node, NULL); + + if (!intc) { + dev_err(dev, "missing child interrupt-controller node\n"); + return -EINVAL; + } + + rockchip->irq_domain = irq_domain_create_linear(of_fwnode_handle(intc), PCI_NUM_INTX, + &intx_domain_ops, rockchip); + of_node_put(intc); + if (!rockchip->irq_domain) { + dev_err(dev, "failed to get a INTx IRQ domain\n"); + return -EINVAL; + } + + return 0; +} + +static int rockchip_pcie_prog_ob_atu(struct rockchip_pcie *rockchip, + int region_no, int type, u8 num_pass_bits, + u32 lower_addr, u32 upper_addr) +{ + u32 ob_addr_0; + u32 ob_addr_1; + u32 ob_desc_0; + u32 aw_offset; + + if (region_no >= MAX_AXI_WRAPPER_REGION_NUM) + return -EINVAL; + if (num_pass_bits + 1 < 8) + return -EINVAL; + if (num_pass_bits > 63) + return -EINVAL; + if (region_no == 0) { + if (AXI_REGION_0_SIZE < (2ULL << num_pass_bits)) + return -EINVAL; + } + if (region_no != 0) { + if (AXI_REGION_SIZE < (2ULL << num_pass_bits)) + return -EINVAL; + } + + aw_offset = (region_no << OB_REG_SIZE_SHIFT); + + ob_addr_0 = num_pass_bits & PCIE_CORE_OB_REGION_ADDR0_NUM_BITS; + ob_addr_0 |= lower_addr & PCIE_CORE_OB_REGION_ADDR0_LO_ADDR; + ob_addr_1 = upper_addr; + ob_desc_0 = (1 << 23 | type); + + rockchip_pcie_write(rockchip, ob_addr_0, + PCIE_CORE_OB_REGION_ADDR0 + aw_offset); + rockchip_pcie_write(rockchip, ob_addr_1, + PCIE_CORE_OB_REGION_ADDR1 + aw_offset); + rockchip_pcie_write(rockchip, ob_desc_0, + PCIE_CORE_OB_REGION_DESC0 + aw_offset); + rockchip_pcie_write(rockchip, 0, + PCIE_CORE_OB_REGION_DESC1 + aw_offset); + + return 0; +} + +static int rockchip_pcie_prog_ib_atu(struct rockchip_pcie *rockchip, + int region_no, u8 num_pass_bits, + u32 lower_addr, u32 upper_addr) +{ + u32 ib_addr_0; + u32 ib_addr_1; + u32 aw_offset; + + if (region_no > MAX_AXI_IB_ROOTPORT_REGION_NUM) + return -EINVAL; + if (num_pass_bits + 1 < MIN_AXI_ADDR_BITS_PASSED) + return -EINVAL; + if (num_pass_bits > 63) + return -EINVAL; + + aw_offset = (region_no << IB_ROOT_PORT_REG_SIZE_SHIFT); + + ib_addr_0 = num_pass_bits & PCIE_CORE_IB_REGION_ADDR0_NUM_BITS; + ib_addr_0 |= (lower_addr << 8) & PCIE_CORE_IB_REGION_ADDR0_LO_ADDR; + ib_addr_1 = upper_addr; + + rockchip_pcie_write(rockchip, ib_addr_0, PCIE_RP_IB_ADDR0 + aw_offset); + rockchip_pcie_write(rockchip, ib_addr_1, PCIE_RP_IB_ADDR1 + aw_offset); + + return 0; +} + +static int rockchip_pcie_cfg_atu(struct rockchip_pcie *rockchip) +{ + struct device *dev = rockchip->dev; + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip); + struct resource_entry *entry; + u64 pci_addr, size; + int offset; + int err; + int reg_no; + + rockchip_pcie_cfg_configuration_accesses(rockchip, + AXI_WRAPPER_TYPE0_CFG); + entry = resource_list_first_type(&bridge->windows, IORESOURCE_MEM); + if (!entry) + return -ENODEV; + + size = resource_size(entry->res); + pci_addr = entry->res->start - entry->offset; + rockchip->msg_bus_addr = pci_addr; + + for (reg_no = 0; reg_no < (size >> 20); reg_no++) { + err = rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1, + AXI_WRAPPER_MEM_WRITE, + 20 - 1, + pci_addr + (reg_no << 20), + 0); + if (err) { + dev_err(dev, "program RC mem outbound ATU failed\n"); + return err; + } + } + + err = rockchip_pcie_prog_ib_atu(rockchip, 2, 32 - 1, 0x0, 0); + if (err) { + dev_err(dev, "program RC mem inbound ATU failed\n"); + return err; + } + + entry = resource_list_first_type(&bridge->windows, IORESOURCE_IO); + if (!entry) + return -ENODEV; + + /* store the register number offset to program RC io outbound ATU */ + offset = size >> 20; + + size = resource_size(entry->res); + pci_addr = entry->res->start - entry->offset; + + for (reg_no = 0; reg_no < (size >> 20); reg_no++) { + err = rockchip_pcie_prog_ob_atu(rockchip, + reg_no + 1 + offset, + AXI_WRAPPER_IO_WRITE, + 20 - 1, + pci_addr + (reg_no << 20), + 0); + if (err) { + dev_err(dev, "program RC io outbound ATU failed\n"); + return err; + } + } + + /* assign message regions */ + rockchip_pcie_prog_ob_atu(rockchip, reg_no + 1 + offset, + AXI_WRAPPER_NOR_MSG, + 20 - 1, 0, 0); + + rockchip->msg_bus_addr += ((reg_no + offset) << 20); + return err; +} + +static int rockchip_pcie_wait_l2(struct rockchip_pcie *rockchip) +{ + u32 value; + int err; + + /* send PME_TURN_OFF message */ + writel(0x0, rockchip->msg_region + PCIE_RC_SEND_PME_OFF); + + /* read LTSSM and wait for falling into L2 link state */ + err = readl_poll_timeout(rockchip->apb_base + PCIE_CLIENT_DEBUG_OUT_0, + value, PCIE_LINK_IS_L2(value), 20, + jiffies_to_usecs(5 * HZ)); + if (err) { + dev_err(rockchip->dev, "PCIe link enter L2 timeout!\n"); + return err; + } + + return 0; +} + +static int rockchip_pcie_suspend_noirq(struct device *dev) +{ + struct rockchip_pcie *rockchip = dev_get_drvdata(dev); + int ret; + + /* disable core and cli int since we don't need to ack PME_ACK */ + rockchip_pcie_write(rockchip, (PCIE_CLIENT_INT_CLI << 16) | + PCIE_CLIENT_INT_CLI, PCIE_CLIENT_INT_MASK); + rockchip_pcie_write(rockchip, (u32)PCIE_CORE_INT, PCIE_CORE_INT_MASK); + + ret = rockchip_pcie_wait_l2(rockchip); + if (ret) { + rockchip_pcie_enable_interrupts(rockchip); + return ret; + } + + rockchip_pcie_deinit_phys(rockchip); + + rockchip_pcie_disable_clocks(rockchip); + + regulator_disable(rockchip->vpcie0v9); + + return ret; +} + +static int rockchip_pcie_resume_noirq(struct device *dev) +{ + struct rockchip_pcie *rockchip = dev_get_drvdata(dev); + int err; + + err = regulator_enable(rockchip->vpcie0v9); + if (err) { + dev_err(dev, "fail to enable vpcie0v9 regulator\n"); + return err; + } + + err = rockchip_pcie_enable_clocks(rockchip); + if (err) + goto err_disable_0v9; + + err = rockchip_pcie_host_init_port(rockchip); + if (err) + goto err_pcie_resume; + + err = rockchip_pcie_cfg_atu(rockchip); + if (err) + goto err_err_deinit_port; + + /* Need this to enter L1 again */ + rockchip_pcie_update_txcredit_mui(rockchip); + rockchip_pcie_enable_interrupts(rockchip); + + return 0; + +err_err_deinit_port: + rockchip_pcie_deinit_phys(rockchip); +err_pcie_resume: + rockchip_pcie_disable_clocks(rockchip); +err_disable_0v9: + regulator_disable(rockchip->vpcie0v9); + return err; +} + +static int rockchip_pcie_probe(struct platform_device *pdev) +{ + struct rockchip_pcie *rockchip; + struct device *dev = &pdev->dev; + struct pci_host_bridge *bridge; + int err; + + if (!dev->of_node) + return -ENODEV; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*rockchip)); + if (!bridge) + return -ENOMEM; + + rockchip = pci_host_bridge_priv(bridge); + + platform_set_drvdata(pdev, rockchip); + rockchip->dev = dev; + rockchip->is_rc = true; + + err = rockchip_pcie_parse_host_dt(rockchip); + if (err) + return err; + + err = rockchip_pcie_enable_clocks(rockchip); + if (err) + return err; + + err = rockchip_pcie_set_vpcie(rockchip); + if (err) { + dev_err(dev, "failed to set vpcie regulator\n"); + goto err_set_vpcie; + } + + err = rockchip_pcie_host_init_port(rockchip); + if (err) + goto err_vpcie; + + err = rockchip_pcie_init_irq_domain(rockchip); + if (err < 0) + goto err_deinit_port; + + err = rockchip_pcie_cfg_atu(rockchip); + if (err) + goto err_remove_irq_domain; + + rockchip->msg_region = devm_ioremap(dev, rockchip->msg_bus_addr, SZ_1M); + if (!rockchip->msg_region) { + err = -ENOMEM; + goto err_remove_irq_domain; + } + + bridge->sysdata = rockchip; + bridge->ops = &rockchip_pcie_ops; + + err = rockchip_pcie_setup_irq(rockchip); + if (err) + goto err_remove_irq_domain; + + rockchip_pcie_enable_interrupts(rockchip); + + err = pci_host_probe(bridge); + if (err < 0) + goto err_remove_irq_domain; + + return 0; + +err_remove_irq_domain: + irq_domain_remove(rockchip->irq_domain); +err_deinit_port: + rockchip_pcie_deinit_phys(rockchip); +err_vpcie: + if (!IS_ERR(rockchip->vpcie12v)) + regulator_disable(rockchip->vpcie12v); + if (!IS_ERR(rockchip->vpcie3v3)) + regulator_disable(rockchip->vpcie3v3); + regulator_disable(rockchip->vpcie1v8); + regulator_disable(rockchip->vpcie0v9); +err_set_vpcie: + rockchip_pcie_disable_clocks(rockchip); + return err; +} + +static void rockchip_pcie_remove(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct rockchip_pcie *rockchip = dev_get_drvdata(dev); + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(rockchip); + + pci_lock_rescan_remove(); + pci_stop_root_bus(bridge->bus); + pci_remove_root_bus(bridge->bus); + pci_unlock_rescan_remove(); + irq_domain_remove(rockchip->irq_domain); + + rockchip_pcie_deinit_phys(rockchip); + + rockchip_pcie_disable_clocks(rockchip); + + if (!IS_ERR(rockchip->vpcie12v)) + regulator_disable(rockchip->vpcie12v); + if (!IS_ERR(rockchip->vpcie3v3)) + regulator_disable(rockchip->vpcie3v3); + regulator_disable(rockchip->vpcie1v8); + regulator_disable(rockchip->vpcie0v9); +} + +static const struct dev_pm_ops rockchip_pcie_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(rockchip_pcie_suspend_noirq, + rockchip_pcie_resume_noirq) +}; + +static const struct of_device_id rockchip_pcie_of_match[] = { + { .compatible = "rockchip,rk3399-pcie", }, + {} +}; +MODULE_DEVICE_TABLE(of, rockchip_pcie_of_match); + +static struct platform_driver rockchip_pcie_driver = { + .driver = { + .name = "rockchip-pcie", + .of_match_table = rockchip_pcie_of_match, + .pm = &rockchip_pcie_pm_ops, + }, + .probe = rockchip_pcie_probe, + .remove = rockchip_pcie_remove, +}; +module_platform_driver(rockchip_pcie_driver); + +MODULE_AUTHOR("Rockchip Inc"); +MODULE_DESCRIPTION("Rockchip AXI PCIe driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/pcie-rockchip.c b/drivers/pci/controller/pcie-rockchip.c new file mode 100644 index 000000000..8d78ab1b3 --- /dev/null +++ b/drivers/pci/controller/pcie-rockchip.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Rockchip AXI PCIe host controller driver + * + * Copyright (c) 2016 Rockchip, Inc. + * + * Author: Shawn Lin <shawn.lin@rock-chips.com> + * Wenrui Li <wenrui.li@rock-chips.com> + * + * Bits taken from Synopsys DesignWare Host controller driver and + * ARM PCI Host generic driver. + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/gpio/consumer.h> +#include <linux/iopoll.h> +#include <linux/of.h> +#include <linux/of_pci.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/reset.h> + +#include "../pci.h" +#include "pcie-rockchip.h" + +int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip) +{ + struct device *dev = rockchip->dev; + struct platform_device *pdev = to_platform_device(dev); + struct device_node *node = dev->of_node; + struct resource *regs; + int err, i; + + if (rockchip->is_rc) { + regs = platform_get_resource_byname(pdev, + IORESOURCE_MEM, + "axi-base"); + rockchip->reg_base = devm_pci_remap_cfg_resource(dev, regs); + if (IS_ERR(rockchip->reg_base)) + return PTR_ERR(rockchip->reg_base); + } else { + rockchip->mem_res = + platform_get_resource_byname(pdev, IORESOURCE_MEM, + "mem-base"); + if (!rockchip->mem_res) + return -EINVAL; + } + + rockchip->apb_base = + devm_platform_ioremap_resource_byname(pdev, "apb-base"); + if (IS_ERR(rockchip->apb_base)) + return PTR_ERR(rockchip->apb_base); + + err = rockchip_pcie_get_phys(rockchip); + if (err) + return err; + + rockchip->lanes = 1; + err = of_property_read_u32(node, "num-lanes", &rockchip->lanes); + if (!err && (rockchip->lanes == 0 || + rockchip->lanes == 3 || + rockchip->lanes > 4)) { + dev_warn(dev, "invalid num-lanes, default to use one lane\n"); + rockchip->lanes = 1; + } + + rockchip->link_gen = of_pci_get_max_link_speed(node); + if (rockchip->link_gen < 0 || rockchip->link_gen > 2) + rockchip->link_gen = 2; + + for (i = 0; i < ROCKCHIP_NUM_PM_RSTS; i++) + rockchip->pm_rsts[i].id = rockchip_pci_pm_rsts[i]; + + err = devm_reset_control_bulk_get_exclusive(dev, + ROCKCHIP_NUM_PM_RSTS, + rockchip->pm_rsts); + if (err) + return dev_err_probe(dev, err, "Cannot get the PM reset\n"); + + for (i = 0; i < ROCKCHIP_NUM_CORE_RSTS; i++) + rockchip->core_rsts[i].id = rockchip_pci_core_rsts[i]; + + err = devm_reset_control_bulk_get_exclusive(dev, + ROCKCHIP_NUM_CORE_RSTS, + rockchip->core_rsts); + if (err) + return dev_err_probe(dev, err, "Cannot get the Core resets\n"); + + if (rockchip->is_rc) + rockchip->perst_gpio = devm_gpiod_get_optional(dev, "ep", + GPIOD_OUT_LOW); + else + rockchip->perst_gpio = devm_gpiod_get_optional(dev, "reset", + GPIOD_IN); + if (IS_ERR(rockchip->perst_gpio)) + return dev_err_probe(dev, PTR_ERR(rockchip->perst_gpio), + "failed to get PERST# GPIO\n"); + + rockchip->num_clks = devm_clk_bulk_get_all(dev, &rockchip->clks); + if (rockchip->num_clks < 0) + return dev_err_probe(dev, rockchip->num_clks, + "failed to get clocks\n"); + + return 0; +} +EXPORT_SYMBOL_GPL(rockchip_pcie_parse_dt); + +#define rockchip_pcie_read_addr(addr) rockchip_pcie_read(rockchip, addr) +/* 100 ms max wait time for PHY PLLs to lock */ +#define RK_PHY_PLL_LOCK_TIMEOUT_US 100000 +/* Sleep should be less than 20ms */ +#define RK_PHY_PLL_LOCK_SLEEP_US 1000 + +int rockchip_pcie_init_port(struct rockchip_pcie *rockchip) +{ + struct device *dev = rockchip->dev; + int err, i; + u32 regs; + + err = reset_control_bulk_assert(ROCKCHIP_NUM_PM_RSTS, + rockchip->pm_rsts); + if (err) + return dev_err_probe(dev, err, "Couldn't assert PM resets\n"); + + for (i = 0; i < MAX_LANE_NUM; i++) { + err = phy_init(rockchip->phys[i]); + if (err) { + dev_err(dev, "init phy%d err %d\n", i, err); + goto err_exit_phy; + } + } + + err = reset_control_bulk_assert(ROCKCHIP_NUM_CORE_RSTS, + rockchip->core_rsts); + if (err) { + dev_err_probe(dev, err, "Couldn't assert Core resets\n"); + goto err_exit_phy; + } + + udelay(10); + + err = reset_control_bulk_deassert(ROCKCHIP_NUM_PM_RSTS, + rockchip->pm_rsts); + if (err) { + dev_err(dev, "Couldn't deassert PM resets %d\n", err); + goto err_exit_phy; + } + + if (rockchip->link_gen == 2) + rockchip_pcie_write(rockchip, PCIE_CLIENT_GEN_SEL_2, + PCIE_CLIENT_CONFIG); + else + rockchip_pcie_write(rockchip, PCIE_CLIENT_GEN_SEL_1, + PCIE_CLIENT_CONFIG); + + regs = PCIE_CLIENT_ARI_ENABLE | + PCIE_CLIENT_CONF_LANE_NUM(rockchip->lanes); + + if (rockchip->is_rc) + regs |= PCIE_CLIENT_LINK_TRAIN_ENABLE | + PCIE_CLIENT_CONF_ENABLE | PCIE_CLIENT_MODE_RC; + else + regs |= PCIE_CLIENT_CONF_DISABLE | PCIE_CLIENT_MODE_EP; + + rockchip_pcie_write(rockchip, regs, PCIE_CLIENT_CONFIG); + + for (i = 0; i < MAX_LANE_NUM; i++) { + err = phy_power_on(rockchip->phys[i]); + if (err) { + dev_err(dev, "power on phy%d err %d\n", i, err); + goto err_power_off_phy; + } + } + + err = readx_poll_timeout(rockchip_pcie_read_addr, + PCIE_CLIENT_SIDE_BAND_STATUS, + regs, !(regs & PCIE_CLIENT_PHY_ST), + RK_PHY_PLL_LOCK_SLEEP_US, + RK_PHY_PLL_LOCK_TIMEOUT_US); + if (err) { + dev_err(dev, "PHY PLLs could not lock, %d\n", err); + goto err_power_off_phy; + } + + err = reset_control_bulk_deassert(ROCKCHIP_NUM_CORE_RSTS, + rockchip->core_rsts); + if (err) { + dev_err(dev, "Couldn't deassert Core reset %d\n", err); + goto err_power_off_phy; + } + + return 0; +err_power_off_phy: + while (i--) + phy_power_off(rockchip->phys[i]); + i = MAX_LANE_NUM; +err_exit_phy: + while (i--) + phy_exit(rockchip->phys[i]); + return err; +} +EXPORT_SYMBOL_GPL(rockchip_pcie_init_port); + +int rockchip_pcie_get_phys(struct rockchip_pcie *rockchip) +{ + struct device *dev = rockchip->dev; + struct phy *phy; + char *name; + u32 i; + + phy = devm_phy_get(dev, "pcie-phy"); + if (!IS_ERR(phy)) { + rockchip->legacy_phy = true; + rockchip->phys[0] = phy; + dev_warn(dev, "legacy phy model is deprecated!\n"); + return 0; + } + + if (PTR_ERR(phy) == -EPROBE_DEFER) + return PTR_ERR(phy); + + dev_dbg(dev, "missing legacy phy; search for per-lane PHY\n"); + + for (i = 0; i < MAX_LANE_NUM; i++) { + name = kasprintf(GFP_KERNEL, "pcie-phy-%u", i); + if (!name) + return -ENOMEM; + + phy = devm_of_phy_get(dev, dev->of_node, name); + kfree(name); + + if (IS_ERR(phy)) { + if (PTR_ERR(phy) != -EPROBE_DEFER) + dev_err(dev, "missing phy for lane %d: %pe\n", + i, phy); + return PTR_ERR(phy); + } + + rockchip->phys[i] = phy; + } + + return 0; +} +EXPORT_SYMBOL_GPL(rockchip_pcie_get_phys); + +void rockchip_pcie_deinit_phys(struct rockchip_pcie *rockchip) +{ + int i; + + for (i = 0; i < MAX_LANE_NUM; i++) { + /* inactive lanes are already powered off */ + if (rockchip->lanes_map & BIT(i)) + phy_power_off(rockchip->phys[i]); + phy_exit(rockchip->phys[i]); + } +} +EXPORT_SYMBOL_GPL(rockchip_pcie_deinit_phys); + +int rockchip_pcie_enable_clocks(struct rockchip_pcie *rockchip) +{ + struct device *dev = rockchip->dev; + int err; + + err = clk_bulk_prepare_enable(rockchip->num_clks, rockchip->clks); + if (err) + return dev_err_probe(dev, err, "failed to enable clocks\n"); + + return 0; +} +EXPORT_SYMBOL_GPL(rockchip_pcie_enable_clocks); + +void rockchip_pcie_disable_clocks(struct rockchip_pcie *rockchip) +{ + + clk_bulk_disable_unprepare(rockchip->num_clks, rockchip->clks); +} +EXPORT_SYMBOL_GPL(rockchip_pcie_disable_clocks); + +void rockchip_pcie_cfg_configuration_accesses( + struct rockchip_pcie *rockchip, u32 type) +{ + u32 ob_desc_0; + + /* Configuration Accesses for region 0 */ + rockchip_pcie_write(rockchip, 0x0, PCIE_RC_BAR_CONF); + + rockchip_pcie_write(rockchip, + (RC_REGION_0_ADDR_TRANS_L + RC_REGION_0_PASS_BITS), + PCIE_CORE_OB_REGION_ADDR0); + rockchip_pcie_write(rockchip, RC_REGION_0_ADDR_TRANS_H, + PCIE_CORE_OB_REGION_ADDR1); + ob_desc_0 = rockchip_pcie_read(rockchip, PCIE_CORE_OB_REGION_DESC0); + ob_desc_0 &= ~(RC_REGION_0_TYPE_MASK); + ob_desc_0 |= (type | (0x1 << 23)); + rockchip_pcie_write(rockchip, ob_desc_0, PCIE_CORE_OB_REGION_DESC0); + rockchip_pcie_write(rockchip, 0x0, PCIE_CORE_OB_REGION_DESC1); +} +EXPORT_SYMBOL_GPL(rockchip_pcie_cfg_configuration_accesses); diff --git a/drivers/pci/controller/pcie-rockchip.h b/drivers/pci/controller/pcie-rockchip.h new file mode 100644 index 000000000..3e82a69b9 --- /dev/null +++ b/drivers/pci/controller/pcie-rockchip.h @@ -0,0 +1,353 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Rockchip AXI PCIe controller driver + * + * Copyright (c) 2018 Rockchip, Inc. + * + * Author: Shawn Lin <shawn.lin@rock-chips.com> + * + */ + +#ifndef _PCIE_ROCKCHIP_H +#define _PCIE_ROCKCHIP_H + +#include <linux/clk.h> +#include <linux/hw_bitfield.h> +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/pci-ecam.h> +#include <linux/reset.h> + +/* + * The upper 16 bits of PCIE_CLIENT_CONFIG are a write mask for the lower 16 + * bits. This allows atomic updates of the register without locking. + */ +#define HWORD_SET_BIT(val) (FIELD_PREP_WM16_CONST((val), 1)) +#define HWORD_CLR_BIT(val) (FIELD_PREP_WM16_CONST((val), 0)) + +#define ENCODE_LANES(x) ((((x) >> 1) & 3)) +#define MAX_LANE_NUM 4 +#define MAX_REGION_LIMIT 32 +#define MIN_EP_APERTURE 28 +#define LINK_TRAIN_TIMEOUT (500 * USEC_PER_MSEC) + +#define PCIE_CLIENT_BASE 0x0 +#define PCIE_CLIENT_CONFIG (PCIE_CLIENT_BASE + 0x00) +#define PCIE_CLIENT_CONF_ENABLE HWORD_SET_BIT(0x0001) +#define PCIE_CLIENT_CONF_DISABLE HWORD_CLR_BIT(0x0001) +#define PCIE_CLIENT_LINK_TRAIN_ENABLE HWORD_SET_BIT(0x0002) +#define PCIE_CLIENT_LINK_TRAIN_DISABLE HWORD_CLR_BIT(0x0002) +#define PCIE_CLIENT_ARI_ENABLE HWORD_SET_BIT(0x0008) +#define PCIE_CLIENT_CONF_LANE_NUM(x) FIELD_PREP_WM16(0x0030, ENCODE_LANES(x)) +#define PCIE_CLIENT_MODE_RC HWORD_SET_BIT(0x0040) +#define PCIE_CLIENT_MODE_EP HWORD_CLR_BIT(0x0040) +#define PCIE_CLIENT_GEN_SEL_1 HWORD_CLR_BIT(0x0080) +#define PCIE_CLIENT_GEN_SEL_2 HWORD_SET_BIT(0x0080) +#define PCIE_CLIENT_LEGACY_INT_CTRL (PCIE_CLIENT_BASE + 0x0c) +#define PCIE_CLIENT_INT_IN_ASSERT HWORD_SET_BIT(0x0002) +#define PCIE_CLIENT_INT_IN_DEASSERT HWORD_CLR_BIT(0x0002) +#define PCIE_CLIENT_INT_PEND_ST_PEND HWORD_SET_BIT(0x0001) +#define PCIE_CLIENT_INT_PEND_ST_NORMAL HWORD_CLR_BIT(0x0001) +#define PCIE_CLIENT_SIDE_BAND_STATUS (PCIE_CLIENT_BASE + 0x20) +#define PCIE_CLIENT_PHY_ST BIT(12) +#define PCIE_CLIENT_DEBUG_OUT_0 (PCIE_CLIENT_BASE + 0x3c) +#define PCIE_CLIENT_DEBUG_LTSSM_MASK GENMASK(5, 0) +#define PCIE_CLIENT_DEBUG_LTSSM_L1 0x18 +#define PCIE_CLIENT_DEBUG_LTSSM_L2 0x19 +#define PCIE_CLIENT_BASIC_STATUS0 (PCIE_CLIENT_BASE + 0x44) +#define PCIE_CLIENT_NEG_LINK_WIDTH_MASK GENMASK(7, 6) +#define PCIE_CLIENT_NEG_LINK_WIDTH_SHIFT 6 +#define PCIE_CLIENT_NEG_LINK_SPEED BIT(5) +#define PCIE_CLIENT_BASIC_STATUS1 (PCIE_CLIENT_BASE + 0x48) +#define PCIE_CLIENT_LINK_STATUS_UP 0x00300000 +#define PCIE_CLIENT_LINK_STATUS_MASK 0x00300000 +#define PCIE_CLIENT_INT_MASK (PCIE_CLIENT_BASE + 0x4c) +#define PCIE_CLIENT_INT_STATUS (PCIE_CLIENT_BASE + 0x50) +#define PCIE_CLIENT_INTR_MASK GENMASK(8, 5) +#define PCIE_CLIENT_INTR_SHIFT 5 +#define PCIE_CLIENT_INT_LEGACY_DONE BIT(15) +#define PCIE_CLIENT_INT_MSG BIT(14) +#define PCIE_CLIENT_INT_HOT_RST BIT(13) +#define PCIE_CLIENT_INT_DPA BIT(12) +#define PCIE_CLIENT_INT_FATAL_ERR BIT(11) +#define PCIE_CLIENT_INT_NFATAL_ERR BIT(10) +#define PCIE_CLIENT_INT_CORR_ERR BIT(9) +#define PCIE_CLIENT_INT_INTD BIT(8) +#define PCIE_CLIENT_INT_INTC BIT(7) +#define PCIE_CLIENT_INT_INTB BIT(6) +#define PCIE_CLIENT_INT_INTA BIT(5) +#define PCIE_CLIENT_INT_LOCAL BIT(4) +#define PCIE_CLIENT_INT_UDMA BIT(3) +#define PCIE_CLIENT_INT_PHY BIT(2) +#define PCIE_CLIENT_INT_HOT_PLUG BIT(1) +#define PCIE_CLIENT_INT_PWR_STCG BIT(0) + +#define PCIE_CLIENT_INT_LEGACY \ + (PCIE_CLIENT_INT_INTA | PCIE_CLIENT_INT_INTB | \ + PCIE_CLIENT_INT_INTC | PCIE_CLIENT_INT_INTD) + +#define PCIE_CLIENT_INT_CLI \ + (PCIE_CLIENT_INT_CORR_ERR | PCIE_CLIENT_INT_NFATAL_ERR | \ + PCIE_CLIENT_INT_FATAL_ERR | PCIE_CLIENT_INT_DPA | \ + PCIE_CLIENT_INT_HOT_RST | PCIE_CLIENT_INT_MSG | \ + PCIE_CLIENT_INT_LEGACY_DONE | PCIE_CLIENT_INT_LEGACY | \ + PCIE_CLIENT_INT_PHY) + +#define PCIE_CORE_CTRL_MGMT_BASE 0x900000 +#define PCIE_CORE_CTRL (PCIE_CORE_CTRL_MGMT_BASE + 0x000) +#define PCIE_CORE_PL_CONF_LS_MASK 0x00000001 +#define PCIE_CORE_PL_CONF_LS_READY 0x00000001 +#define PCIE_CORE_PL_CONF_SPEED_5G 0x00000008 +#define PCIE_CORE_PL_CONF_SPEED_MASK 0x00000018 +#define PCIE_CORE_PL_CONF_LANE_MASK 0x00000006 +#define PCIE_CORE_PL_CONF_LANE_SHIFT 1 +#define PCIE_CORE_CTRL_PLC1 (PCIE_CORE_CTRL_MGMT_BASE + 0x004) +#define PCIE_CORE_CTRL_PLC1_FTS_MASK GENMASK(23, 8) +#define PCIE_CORE_CTRL_PLC1_FTS_SHIFT 8 +#define PCIE_CORE_CTRL_PLC1_FTS_CNT 0xffff +#define PCIE_CORE_TXCREDIT_CFG1 (PCIE_CORE_CTRL_MGMT_BASE + 0x020) +#define PCIE_CORE_TXCREDIT_CFG1_MUI_MASK 0xFFFF0000 +#define PCIE_CORE_TXCREDIT_CFG1_MUI_SHIFT 16 +#define PCIE_CORE_TXCREDIT_CFG1_MUI_ENCODE(x) \ + (((x) >> 3) << PCIE_CORE_TXCREDIT_CFG1_MUI_SHIFT) +#define PCIE_CORE_LANE_MAP (PCIE_CORE_CTRL_MGMT_BASE + 0x200) +#define PCIE_CORE_LANE_MAP_MASK 0x0000000f +#define PCIE_CORE_LANE_MAP_REVERSE BIT(16) +#define PCIE_CORE_INT_STATUS (PCIE_CORE_CTRL_MGMT_BASE + 0x20c) +#define PCIE_CORE_INT_PRFPE BIT(0) +#define PCIE_CORE_INT_CRFPE BIT(1) +#define PCIE_CORE_INT_RRPE BIT(2) +#define PCIE_CORE_INT_PRFO BIT(3) +#define PCIE_CORE_INT_CRFO BIT(4) +#define PCIE_CORE_INT_RT BIT(5) +#define PCIE_CORE_INT_RTR BIT(6) +#define PCIE_CORE_INT_PE BIT(7) +#define PCIE_CORE_INT_MTR BIT(8) +#define PCIE_CORE_INT_UCR BIT(9) +#define PCIE_CORE_INT_FCE BIT(10) +#define PCIE_CORE_INT_CT BIT(11) +#define PCIE_CORE_INT_UTC BIT(18) +#define PCIE_CORE_INT_MMVC BIT(19) +#define PCIE_CORE_CONFIG_VENDOR (PCIE_CORE_CTRL_MGMT_BASE + 0x44) +#define PCIE_CORE_INT_MASK (PCIE_CORE_CTRL_MGMT_BASE + 0x210) +#define PCIE_CORE_PHY_FUNC_CFG (PCIE_CORE_CTRL_MGMT_BASE + 0x2c0) +#define PCIE_RC_BAR_CONF (PCIE_CORE_CTRL_MGMT_BASE + 0x300) +#define ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_DISABLED 0x0 +#define ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_IO_32BITS 0x1 +#define ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_MEM_32BITS 0x4 +#define ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_PREFETCH_MEM_32BITS 0x5 +#define ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_MEM_64BITS 0x6 +#define ROCKCHIP_PCIE_CORE_BAR_CFG_CTRL_PREFETCH_MEM_64BITS 0x7 + +#define PCIE_CORE_INT \ + (PCIE_CORE_INT_PRFPE | PCIE_CORE_INT_CRFPE | \ + PCIE_CORE_INT_RRPE | PCIE_CORE_INT_CRFO | \ + PCIE_CORE_INT_RT | PCIE_CORE_INT_RTR | \ + PCIE_CORE_INT_PE | PCIE_CORE_INT_MTR | \ + PCIE_CORE_INT_UCR | PCIE_CORE_INT_FCE | \ + PCIE_CORE_INT_CT | PCIE_CORE_INT_UTC | \ + PCIE_CORE_INT_MMVC) + +#define PCIE_RC_RP_ATS_BASE 0x400000 +#define PCIE_RC_CONFIG_NORMAL_BASE 0x800000 +#define PCIE_EP_PF_CONFIG_REGS_BASE 0x800000 +#define PCIE_RC_CONFIG_BASE 0xa00000 +#define PCIE_EP_CONFIG_BASE 0xa00000 +#define PCIE_EP_CONFIG_DID_VID (PCIE_EP_CONFIG_BASE + 0x00) +#define PCIE_EP_CONFIG_LCS (PCIE_EP_CONFIG_BASE + 0xd0) +#define PCIE_RC_CONFIG_RID_CCR (PCIE_RC_CONFIG_BASE + 0x08) +#define PCIE_RC_CONFIG_CR (PCIE_RC_CONFIG_BASE + 0xc0) +#define PCIE_RC_CONFIG_L1_SUBSTATE_CTRL2 (PCIE_RC_CONFIG_BASE + 0x90c) +#define PCIE_RC_CONFIG_THP_CAP (PCIE_RC_CONFIG_BASE + 0x274) +#define PCIE_RC_CONFIG_THP_CAP_NEXT_MASK GENMASK(31, 20) + +#define MAX_AXI_IB_ROOTPORT_REGION_NUM 3 +#define MIN_AXI_ADDR_BITS_PASSED 8 +#define PCIE_ADDR_MASK GENMASK_ULL(63, MIN_AXI_ADDR_BITS_PASSED) +#define PCIE_CORE_AXI_CONF_BASE 0xc00000 +#define PCIE_CORE_OB_REGION_ADDR0 (PCIE_CORE_AXI_CONF_BASE + 0x0) +#define PCIE_CORE_OB_REGION_ADDR0_NUM_BITS 0x3f +#define PCIE_CORE_OB_REGION_ADDR0_LO_ADDR PCIE_ADDR_MASK +#define PCIE_CORE_OB_REGION_ADDR1 (PCIE_CORE_AXI_CONF_BASE + 0x4) +#define PCIE_CORE_OB_REGION_DESC0 (PCIE_CORE_AXI_CONF_BASE + 0x8) +#define PCIE_CORE_OB_REGION_DESC1 (PCIE_CORE_AXI_CONF_BASE + 0xc) + +#define PCIE_CORE_AXI_INBOUND_BASE 0xc00800 +#define PCIE_RP_IB_ADDR0 (PCIE_CORE_AXI_INBOUND_BASE + 0x0) +#define PCIE_CORE_IB_REGION_ADDR0_NUM_BITS 0x3f +#define PCIE_CORE_IB_REGION_ADDR0_LO_ADDR PCIE_ADDR_MASK +#define PCIE_RP_IB_ADDR1 (PCIE_CORE_AXI_INBOUND_BASE + 0x4) + +/* Size of one AXI Region (not Region 0) */ +#define AXI_REGION_SIZE BIT(20) +/* Size of Region 0, equal to sum of sizes of other regions */ +#define AXI_REGION_0_SIZE (32 * (0x1 << 20)) +#define OB_REG_SIZE_SHIFT 5 +#define IB_ROOT_PORT_REG_SIZE_SHIFT 3 +#define AXI_WRAPPER_IO_WRITE 0x6 +#define AXI_WRAPPER_MEM_WRITE 0x2 +#define AXI_WRAPPER_TYPE0_CFG 0xa +#define AXI_WRAPPER_TYPE1_CFG 0xb +#define AXI_WRAPPER_NOR_MSG 0xc + +#define PCIE_RC_SEND_PME_OFF 0x11960 +#define PCIE_LINK_IS_L2(x) \ + (((x) & PCIE_CLIENT_DEBUG_LTSSM_MASK) == PCIE_CLIENT_DEBUG_LTSSM_L2) +#define PCIE_LINK_TRAINING_DONE(x) \ + (((x) & PCIE_CORE_PL_CONF_LS_MASK) == PCIE_CORE_PL_CONF_LS_READY) +#define PCIE_LINK_UP(x) \ + (((x) & PCIE_CLIENT_LINK_STATUS_MASK) == PCIE_CLIENT_LINK_STATUS_UP) +#define PCIE_LINK_IS_GEN2(x) \ + (((x) & PCIE_CORE_PL_CONF_SPEED_MASK) == PCIE_CORE_PL_CONF_SPEED_5G) + +#define RC_REGION_0_ADDR_TRANS_H 0x00000000 +#define RC_REGION_0_ADDR_TRANS_L 0x00000000 +#define RC_REGION_0_PASS_BITS (25 - 1) +#define RC_REGION_0_TYPE_MASK GENMASK(3, 0) +#define MAX_AXI_WRAPPER_REGION_NUM 33 + +#define ROCKCHIP_PCIE_MSG_ROUTING_MASK GENMASK(7, 5) +#define ROCKCHIP_PCIE_MSG_ROUTING(route) \ + (((route) << 5) & ROCKCHIP_PCIE_MSG_ROUTING_MASK) +#define ROCKCHIP_PCIE_MSG_CODE_MASK GENMASK(15, 8) +#define ROCKCHIP_PCIE_MSG_CODE(code) \ + (((code) << 8) & ROCKCHIP_PCIE_MSG_CODE_MASK) +#define ROCKCHIP_PCIE_MSG_NO_DATA BIT(16) + +#define ROCKCHIP_PCIE_EP_CMD_STATUS 0x4 +#define ROCKCHIP_PCIE_EP_CMD_STATUS_IS BIT(19) +#define ROCKCHIP_PCIE_EP_MSI_CTRL_REG 0x90 +#define ROCKCHIP_PCIE_EP_MSI_CP1_OFFSET 8 +#define ROCKCHIP_PCIE_EP_MSI_CP1_MASK GENMASK(15, 8) +#define ROCKCHIP_PCIE_EP_MSI_FLAGS_OFFSET 16 +#define ROCKCHIP_PCIE_EP_MSI_CTRL_MMC_OFFSET 17 +#define ROCKCHIP_PCIE_EP_MSI_CTRL_MMC_MASK GENMASK(19, 17) +#define ROCKCHIP_PCIE_EP_MSI_CTRL_MME_OFFSET 20 +#define ROCKCHIP_PCIE_EP_MSI_CTRL_MME_MASK GENMASK(22, 20) +#define ROCKCHIP_PCIE_EP_MSI_CTRL_ME BIT(16) +#define ROCKCHIP_PCIE_EP_MSI_CTRL_MASK_MSI_CAP BIT(24) +#define ROCKCHIP_PCIE_EP_MSIX_CAP_REG 0xb0 +#define ROCKCHIP_PCIE_EP_MSIX_CAP_CP_OFFSET 8 +#define ROCKCHIP_PCIE_EP_MSIX_CAP_CP_MASK GENMASK(15, 8) +#define ROCKCHIP_PCIE_EP_DUMMY_IRQ_ADDR 0x1 +#define ROCKCHIP_PCIE_EP_PCI_LEGACY_IRQ_ADDR 0x3 + +#define ROCKCHIP_PCIE_AT_MIN_NUM_BITS 8 +#define ROCKCHIP_PCIE_AT_MAX_NUM_BITS 20 +#define ROCKCHIP_PCIE_AT_SIZE_ALIGN (1UL << ROCKCHIP_PCIE_AT_MIN_NUM_BITS) + +#define ROCKCHIP_PCIE_EP_FUNC_BASE(fn) \ + (PCIE_EP_PF_CONFIG_REGS_BASE + (((fn) << 12) & GENMASK(19, 12))) +#define ROCKCHIP_PCIE_EP_VIRT_FUNC_BASE(fn) \ + (PCIE_EP_PF_CONFIG_REGS_BASE + 0x10000 + (((fn) << 12) & GENMASK(19, 12))) + +#define ROCKCHIP_PCIE_AT_MIN_NUM_BITS 8 +#define ROCKCHIP_PCIE_AT_MAX_NUM_BITS 20 +#define ROCKCHIP_PCIE_AT_SIZE_ALIGN (1UL << ROCKCHIP_PCIE_AT_MIN_NUM_BITS) + +#define ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar) \ + (PCIE_CORE_AXI_CONF_BASE + 0x0828 + (fn) * 0x0040 + (bar) * 0x0008) +#define ROCKCHIP_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar) \ + (PCIE_CORE_AXI_CONF_BASE + 0x082c + (fn) * 0x0040 + (bar) * 0x0008) +#define ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN_MASK GENMASK(19, 12) +#define ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN(devfn) \ + (((devfn) << 12) & \ + ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0_DEVFN_MASK) +#define ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0_BUS_MASK GENMASK(27, 20) +#define ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0_BUS(bus) \ + (((bus) << 20) & ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0_BUS_MASK) +#define PCIE_RC_EP_ATR_OB_REGIONS_1_32 (PCIE_CORE_AXI_CONF_BASE + 0x0020) +#define ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR0(r) \ + (PCIE_RC_EP_ATR_OB_REGIONS_1_32 + 0x0000 + ((r) & 0x1f) * 0x0020) +#define ROCKCHIP_PCIE_AT_OB_REGION_PCI_ADDR1(r) \ + (PCIE_RC_EP_ATR_OB_REGIONS_1_32 + 0x0004 + ((r) & 0x1f) * 0x0020) +#define ROCKCHIP_PCIE_AT_OB_REGION_DESC0_HARDCODED_RID BIT(23) +#define ROCKCHIP_PCIE_AT_OB_REGION_DESC0_DEVFN_MASK GENMASK(31, 24) +#define ROCKCHIP_PCIE_AT_OB_REGION_DESC0_DEVFN(devfn) \ + (((devfn) << 24) & ROCKCHIP_PCIE_AT_OB_REGION_DESC0_DEVFN_MASK) +#define ROCKCHIP_PCIE_AT_OB_REGION_DESC0(r) \ + (PCIE_RC_EP_ATR_OB_REGIONS_1_32 + 0x0008 + ((r) & 0x1f) * 0x0020) +#define ROCKCHIP_PCIE_AT_OB_REGION_DESC1(r) \ + (PCIE_RC_EP_ATR_OB_REGIONS_1_32 + 0x000c + ((r) & 0x1f) * 0x0020) +#define ROCKCHIP_PCIE_AT_OB_REGION_DESC2(r) \ + (PCIE_RC_EP_ATR_OB_REGIONS_1_32 + 0x0010 + ((r) & 0x1f) * 0x0020) + +#define ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG0(fn) \ + (PCIE_CORE_CTRL_MGMT_BASE + 0x0240 + (fn) * 0x0008) +#define ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG1(fn) \ + (PCIE_CORE_CTRL_MGMT_BASE + 0x0244 + (fn) * 0x0008) +#define ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) \ + (GENMASK(4, 0) << ((b) * 8)) +#define ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE(b, a) \ + (((a) << ((b) * 8)) & \ + ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b)) +#define ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b) \ + (GENMASK(7, 5) << ((b) * 8)) +#define ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL(b, c) \ + (((c) << ((b) * 8 + 5)) & \ + ROCKCHIP_PCIE_CORE_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b)) + +#define ROCKCHIP_NUM_PM_RSTS ARRAY_SIZE(rockchip_pci_pm_rsts) +#define ROCKCHIP_NUM_CORE_RSTS ARRAY_SIZE(rockchip_pci_core_rsts) + +static const char * const rockchip_pci_pm_rsts[] = { + "pm", + "pclk", + "aclk", +}; + +/* NOTE: Do not reorder the deassert sequence of the following reset pins */ +static const char * const rockchip_pci_core_rsts[] = { + "pipe", + "mgmt", + "core", + "mgmt-sticky", +}; + +struct rockchip_pcie { + void __iomem *reg_base; /* DT axi-base */ + void __iomem *apb_base; /* DT apb-base */ + bool legacy_phy; + struct phy *phys[MAX_LANE_NUM]; + struct reset_control_bulk_data pm_rsts[ROCKCHIP_NUM_PM_RSTS]; + struct reset_control_bulk_data core_rsts[ROCKCHIP_NUM_CORE_RSTS]; + struct clk_bulk_data *clks; + int num_clks; + struct regulator *vpcie12v; /* 12V power supply */ + struct regulator *vpcie3v3; /* 3.3V power supply */ + struct regulator *vpcie1v8; /* 1.8V power supply */ + struct regulator *vpcie0v9; /* 0.9V power supply */ + struct gpio_desc *perst_gpio; + u32 lanes; + u8 lanes_map; + int link_gen; + struct device *dev; + struct irq_domain *irq_domain; + int offset; + void __iomem *msg_region; + phys_addr_t msg_bus_addr; + bool is_rc; + struct resource *mem_res; +}; + +static u32 rockchip_pcie_read(struct rockchip_pcie *rockchip, u32 reg) +{ + return readl(rockchip->apb_base + reg); +} + +static void rockchip_pcie_write(struct rockchip_pcie *rockchip, u32 val, + u32 reg) +{ + writel(val, rockchip->apb_base + reg); +} + +int rockchip_pcie_parse_dt(struct rockchip_pcie *rockchip); +int rockchip_pcie_init_port(struct rockchip_pcie *rockchip); +int rockchip_pcie_get_phys(struct rockchip_pcie *rockchip); +void rockchip_pcie_deinit_phys(struct rockchip_pcie *rockchip); +int rockchip_pcie_enable_clocks(struct rockchip_pcie *rockchip); +void rockchip_pcie_disable_clocks(struct rockchip_pcie *rockchip); +void rockchip_pcie_cfg_configuration_accesses( + struct rockchip_pcie *rockchip, u32 type); + +#endif /* _PCIE_ROCKCHIP_H */ diff --git a/drivers/pci/controller/pcie-rzg3s-host.c b/drivers/pci/controller/pcie-rzg3s-host.c new file mode 100644 index 000000000..a1a78bdb7 --- /dev/null +++ b/drivers/pci/controller/pcie-rzg3s-host.c @@ -0,0 +1,2147 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCIe driver for Renesas RZ/G3S SoCs + * + * Copyright (C) 2025 Renesas Electronics Corp. + * + * Based on: + * drivers/pci/controller/pcie-rcar-host.c + * Copyright (C) 2009 - 2011 Paul Mundt + */ + +#include <linux/bitfield.h> +#include <linux/bitmap.h> +#include <linux/bitops.h> +#include <linux/cleanup.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/iopoll.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/irqdomain.h> +#include <linux/kernel.h> +#include <linux/mfd/syscon.h> +#include <linux/mutex.h> +#include <linux/msi.h> +#include <linux/of_irq.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/regmap.h> +#include <linux/reset.h> +#include <linux/sizes.h> +#include <linux/slab.h> +#include <linux/units.h> + +#include "pci-host-common.h" +#include "../pci.h" + +/* AXI registers */ +#define RZG3S_PCI_REQDATA(id) (0x80 + (id) * 0x4) +#define RZG3S_PCI_REQRCVDAT 0x8c + +#define RZG3S_PCI_REQADR1 0x90 +#define RZG3S_PCI_REQADR1_BUS GENMASK(31, 24) +#define RZG3S_PCI_REQADR1_DEV GENMASK(23, 19) +#define RZG3S_PCI_REQADR1_FUNC GENMASK(18, 16) +#define RZG3S_PCI_REQADR1_REG GENMASK(11, 0) + +#define RZG3S_PCI_REQBE 0x98 +#define RZG3S_PCI_REQBE_BYTE_EN GENMASK(3, 0) + +#define RZG3S_PCI_REQISS 0x9c +#define RZG3S_PCI_REQISS_MOR_STATUS GENMASK(18, 16) +#define RZG3S_PCI_REQISS_TR_TYPE GENMASK(11, 8) +#define RZG3S_PCI_REQISS_TR_TP0_RD FIELD_PREP(RZG3S_PCI_REQISS_TR_TYPE, 0x4) +#define RZG3S_PCI_REQISS_TR_TP0_WR FIELD_PREP(RZG3S_PCI_REQISS_TR_TYPE, 0x5) +#define RZG3S_PCI_REQISS_TR_TP1_RD FIELD_PREP(RZG3S_PCI_REQISS_TR_TYPE, 0x6) +#define RZG3S_PCI_REQISS_TR_TP1_WR FIELD_PREP(RZG3S_PCI_REQISS_TR_TYPE, 0x7) +#define RZG3S_PCI_REQISS_REQ_ISSUE BIT(0) + +#define RZG3S_PCI_MSIRCVWADRL 0x100 +#define RZG3S_PCI_MSIRCVWADRL_MASK GENMASK(31, 3) +#define RZG3S_PCI_MSIRCVWADRL_MSG_DATA_ENA BIT(1) +#define RZG3S_PCI_MSIRCVWADRL_ENA BIT(0) + +#define RZG3S_PCI_MSIRCVWADRU 0x104 + +#define RZG3S_PCI_MSIRCVWMSKL 0x108 +#define RZG3S_PCI_MSIRCVWMSKL_MASK GENMASK(31, 2) + +#define RZG3S_PCI_PINTRCVIE 0x110 +#define RZG3S_PCI_PINTRCVIE_INTX(i) BIT(i) +#define RZG3S_PCI_PINTRCVIE_MSI BIT(4) + +/* Register is R/W1C, it doesn't require locking. */ +#define RZG3S_PCI_PINTRCVIS 0x114 +#define RZG3S_PCI_PINTRCVIS_INTX(i) BIT(i) +#define RZG3S_PCI_PINTRCVIS_MSI BIT(4) + +#define RZG3S_PCI_MSGRCVIE 0x120 +#define RZG3S_PCI_MSGRCVIE_MSG_RCV BIT(24) + +#define RZG3S_PCI_MSGRCVIS 0x124 +#define RZG3S_PCI_MSGRCVIS_MRI BIT(24) + +#define RZG3S_PCI_PEIE0 0x200 + +#define RZG3S_PCI_PEIS0 0x204 +#define RZG3S_PCI_PEIS0_RX_DLLP_PM_ENTER BIT(12) +#define RZG3S_PCI_PEIS0_DL_UPDOWN BIT(9) + +#define RZG3S_PCI_PEIE1 0x208 +#define RZG3S_PCI_PEIS1 0x20c +#define RZG3S_PCI_AMEIS 0x214 +#define RZG3S_PCI_ASEIS1 0x224 + +#define RZG3S_PCI_PCSTAT1 0x408 +#define RZG3S_PCI_PCSTAT1_LTSSM_STATE GENMASK(14, 10) +#define RZG3S_PCI_PCSTAT1_DL_DOWN_STS BIT(0) + +#define RZG3S_PCI_PCCTRL2 0x410 +#define RZG3S_PCI_PCCTRL2_LS_CHG GENMASK(9, 8) +#define RZG3S_PCI_PCCTRL2_LS_CHG_REQ BIT(0) + +#define RZG3S_PCI_PCSTAT2 0x414 +#define RZG3S_PCI_PCSTAT2_LS_CHG_DONE BIT(28) +#define RZG3S_PCI_PCSTAT2_SDRIRE GENMASK(7, 1) + +#define RZG3S_PCI_PERM 0x300 +#define RZG3S_PCI_PERM_CFG_HWINIT_EN BIT(2) +#define RZG3S_PCI_PERM_PIPE_PHY_REG_EN BIT(1) + +#define RZG3S_PCI_RESET 0x310 +#define RZG3S_PCI_RESET_RST_OUT_B BIT(6) +#define RZG3S_PCI_RESET_RST_PS_B BIT(5) +#define RZG3S_PCI_RESET_RST_LOAD_B BIT(4) +#define RZG3S_PCI_RESET_RST_CFG_B BIT(3) +#define RZG3S_PCI_RESET_RST_RSM_B BIT(2) +#define RZG3S_PCI_RESET_RST_GP_B BIT(1) +#define RZG3S_PCI_RESET_RST_B BIT(0) + +#define RZG3S_PCI_MSIRE(id) (0x600 + (id) * 0x10) +#define RZG3S_PCI_MSIRE_ENA BIT(0) + +#define RZG3S_PCI_MSIRM(id) (0x608 + (id) * 0x10) + +/* Register is R/W1C, it doesn't require locking. */ +#define RZG3S_PCI_MSIRS(id) (0x60c + (id) * 0x10) + +#define RZG3S_PCI_AWBASEL(id) (0x1000 + (id) * 0x20) +#define RZG3S_PCI_AWBASEL_WIN_ENA BIT(0) + +#define RZG3S_PCI_AWBASEU(id) (0x1004 + (id) * 0x20) +#define RZG3S_PCI_AWMASKL(id) (0x1008 + (id) * 0x20) +#define RZG3S_PCI_AWMASKU(id) (0x100c + (id) * 0x20) +#define RZG3S_PCI_ADESTL(id) (0x1010 + (id) * 0x20) +#define RZG3S_PCI_ADESTU(id) (0x1014 + (id) * 0x20) + +#define RZG3S_PCI_PWBASEL(id) (0x1100 + (id) * 0x20) +#define RZG3S_PCI_PWBASEL_ENA BIT(0) + +#define RZG3S_PCI_PWBASEU(id) (0x1104 + (id) * 0x20) +#define RZG3S_PCI_PDESTL(id) (0x1110 + (id) * 0x20) +#define RZG3S_PCI_PDESTU(id) (0x1114 + (id) * 0x20) +#define RZG3S_PCI_PWMASKL(id) (0x1108 + (id) * 0x20) +#define RZG3S_PCI_PWMASKU(id) (0x110c + (id) * 0x20) + +/* PHY control registers */ +#define RZG3S_PCI_PHY_XCFGD(id) (0x2000 + (id) * 0x10) +#define RZG3S_PCI_PHY_XCFGD_NUM 39 + +#define RZG3S_PCI_PHY_XCFGA_CMN(id) (0x2400 + (id) * 0x10) +#define RZG3S_PCI_PHY_XCFGA_CMN_NUM 16 + +#define RZG3S_PCI_PHY_XCFGA_RX(id) (0x2500 + (id) * 0x10) +#define RZG3S_PCI_PHY_XCFGA_RX_NUM 13 + +#define RZG3S_PCI_PHY_XCFGA_TX 0x25d0 + +#define RZG3S_PCI_PHY_XCFG_CTRL 0x2a20 +#define RZG3S_PCI_PHY_XCFG_CTRL_PHYREG_SEL BIT(0) + +/* PCIe registers */ +#define RZG3S_PCI_CFG_BASE 0x6000 +#define RZG3S_PCI_CFG_BARMSK00L 0xa0 +#define RZG3S_PCI_CFG_BARMSK00U 0xa4 + +#define RZG3S_PCI_CFG_PCIEC 0x60 + +/* Maximum number of windows */ +#define RZG3S_MAX_WINDOWS 8 + +/* Number of MSI interrupts per register */ +#define RZG3S_PCI_MSI_INT_PER_REG 32 +/* The number of MSI interrupts */ +#define RZG3S_PCI_MSI_INT_NR RZG3S_PCI_MSI_INT_PER_REG + +/* Timeouts experimentally determined */ +#define RZG3S_REQ_ISSUE_TIMEOUT_US 2500 + +/** + * enum rzg3s_sysc_link_mode - PCIe link configuration modes + * @RZG3S_SYSC_LINK_MODE_SINGLE_X4: Single port with x4 lanes + * @RZG3S_SYSC_LINK_MODE_DUAL_X2: Dual ports with x2 lanes each + */ +enum rzg3s_sysc_link_mode { + RZG3S_SYSC_LINK_MODE_SINGLE_X4 = 1, + RZG3S_SYSC_LINK_MODE_DUAL_X2 = 3, +}; + +/** + * struct rzg3s_sysc_function - System Controller function descriptor + * @offset: Register offset from the System Controller base address + * @mask: Bit mask for the function within the register + */ +struct rzg3s_sysc_function { + u32 offset; + u32 mask; +}; + +/** + * enum rzg3s_sysc_func_id - System controller function IDs + * @RZG3S_SYSC_FUNC_ID_RST_RSM_B: RST_RSM_B SYSC function ID + * @RZG3S_SYSC_FUNC_ID_L1_ALLOW: L1 allow SYSC function ID + * @RZG3S_SYSC_FUNC_ID_MODE: Mode SYSC function ID + * @RZG3S_SYSC_FUNC_ID_LINK_MASTER: Link master SYSC function ID + * @RZG3S_SYSC_FUNC_ID_MAX: Max SYSC function ID + */ +enum rzg3s_sysc_func_id { + RZG3S_SYSC_FUNC_ID_RST_RSM_B, + RZG3S_SYSC_FUNC_ID_L1_ALLOW, + RZG3S_SYSC_FUNC_ID_MODE, + RZG3S_SYSC_FUNC_ID_LINK_MASTER, + RZG3S_SYSC_FUNC_ID_MAX, +}; + +/** + * struct rzg3s_sysc_info - RZ/G3S System Controller info + * @functions: SYSC function descriptors array + */ +struct rzg3s_sysc_info { + const struct rzg3s_sysc_function functions[RZG3S_SYSC_FUNC_ID_MAX]; +}; + +/** + * struct rzg3s_sysc - RZ/G3S System Controller descriptor + * @regmap: System controller regmap + * @info: System controller info + */ +struct rzg3s_sysc { + struct regmap *regmap; + const struct rzg3s_sysc_info *info; +}; + +/** + * struct rzg3s_pcie_msi - RZ/G3S PCIe MSI data structure + * @domain: IRQ domain + * @map: bitmap with the allocated MSIs + * @dma_addr: address of the allocated MSI window + * @window_base: base address of the MSI window + * @pages: allocated pages for MSI window mapping + * @map_lock: lock for bitmap with the allocated MSIs + * @irq: MSI interrupt + */ +struct rzg3s_pcie_msi { + struct irq_domain *domain; + DECLARE_BITMAP(map, RZG3S_PCI_MSI_INT_NR); + dma_addr_t dma_addr; + dma_addr_t window_base; + unsigned long pages; + struct mutex map_lock; + int irq; +}; + +/** + * enum rzg3s_pcie_controller_id - RZ/G3S PCIe controller IDs + * @RZG3S_PCIE_CONTROLLER_ID_0: PCIe controller 0 + * @RZG3S_PCIE_CONTROLLER_ID_1: PCIe controller 1 + * @RZG3S_PCIE_CONTROLLER_ID_MAX: Max PCIe controllers + */ +enum rzg3s_pcie_controller_id { + RZG3S_PCIE_CONTROLLER_ID_0, + RZG3S_PCIE_CONTROLLER_ID_1, + RZG3S_PCIE_CONTROLLER_ID_MAX, +}; + +struct rzg3s_pcie_host; + +/** + * struct rzg3s_pcie_soc_data - SoC specific data + * @init_phy: PHY initialization function + * @config_pre_init: Optional callback for SoC-specific pre-configuration + * @config_post_init: Callback for SoC-specific post-configuration + * @config_deinit: Callback for SoC-specific de-initialization + * @setup_lanes: Callback for setting up the number of lanes + * @power_resets: array with the resets that need to be de-asserted after + * power-on + * @cfg_resets: array with the resets that need to be de-asserted after + * configuration + * @sysc_info: System Controller info for each controller + * @num_power_resets: number of power resets + * @num_cfg_resets: number of configuration resets + * @num_pcie_controllers: number of PCIe controllers + */ +struct rzg3s_pcie_soc_data { + int (*init_phy)(struct rzg3s_pcie_host *host); + void (*config_pre_init)(struct rzg3s_pcie_host *host); + int (*config_post_init)(struct rzg3s_pcie_host *host); + int (*config_deinit)(struct rzg3s_pcie_host *host); + int (*setup_lanes)(struct rzg3s_pcie_host *host); + const char * const *power_resets; + const char * const *cfg_resets; + struct rzg3s_sysc_info sysc_info[RZG3S_PCIE_CONTROLLER_ID_MAX]; + u8 num_power_resets; + u8 num_cfg_resets; + u8 num_pcie_controllers; +}; + +/** + * struct rzg3s_pcie_port - RZ/G3S PCIe Root Port data structure + * @refclk: PCIe reference clock + * @vendor_id: Vendor ID + * @device_id: Device ID + */ +struct rzg3s_pcie_port { + struct clk *refclk; + u32 vendor_id; + u32 device_id; +}; + +/** + * struct rzg3s_pcie_host - RZ/G3S PCIe data structure + * @axi: base address for AXI registers + * @pcie: base address for PCIe registers + * @dev: struct device + * @power_resets: reset control signals that should be set after power up + * @cfg_resets: reset control signals that should be set after configuration + * @sysc: SYSC descriptor + * @intx_domain: INTx IRQ domain + * @data: SoC specific data + * @msi: MSI data structure + * @port: PCIe Root Port + * @hw_lock: lock for access to the HW resources + * @intx_irqs: INTx interrupts + * @max_link_speed: maximum supported link speed + * @controller_id: PCIe controller identifier, used for System Controller access + * @num_lanes: The number of lanes + */ +struct rzg3s_pcie_host { + void __iomem *axi; + void __iomem *pcie; + struct device *dev; + struct reset_control_bulk_data *power_resets; + struct reset_control_bulk_data *cfg_resets; + struct rzg3s_sysc *sysc; + struct irq_domain *intx_domain; + const struct rzg3s_pcie_soc_data *data; + struct rzg3s_pcie_msi msi; + struct rzg3s_pcie_port port; + raw_spinlock_t hw_lock; + int intx_irqs[PCI_NUM_INTX]; + int max_link_speed; + enum rzg3s_pcie_controller_id controller_id; + u8 num_lanes; +}; + +#define rzg3s_msi_to_host(_msi) container_of(_msi, struct rzg3s_pcie_host, msi) + +/* + * RZ/V2H(P) supports a total of 4 lanes shared across two controllers. + * rzv2h_lane_lock serialises both the counter update and the SYSC + * register write so that concurrent async probes cannot race on the + * shared LINK_MASTER register (offset 0x1060). + * rzv2h_num_total_lanes tracks global lane usage to prevent + * over-allocation or invalid bifurcation modes. + */ +#define RZV2H_PCIE_MAX_LANES 4 +static DEFINE_SPINLOCK(rzv2h_lane_lock); +static u8 rzv2h_num_total_lanes; + +static int rzg3s_sysc_config_func(struct rzg3s_sysc *sysc, + enum rzg3s_sysc_func_id fid, u32 val) +{ + const struct rzg3s_sysc_info *info = sysc->info; + const struct rzg3s_sysc_function *functions = info->functions; + + if (fid >= RZG3S_SYSC_FUNC_ID_MAX) + return -EINVAL; + + if (!functions[fid].mask) + return 0; + + return regmap_update_bits(sysc->regmap, functions[fid].offset, + functions[fid].mask, + field_prep(functions[fid].mask, val)); +} + +static void rzg3s_pcie_update_bits(void __iomem *base, u32 offset, u32 mask, + u32 val) +{ + u32 tmp; + + tmp = readl_relaxed(base + offset); + tmp &= ~mask; + tmp |= val & mask; + writel_relaxed(tmp, base + offset); +} + +static int rzg3s_pcie_child_issue_request(struct rzg3s_pcie_host *host) +{ + u32 val; + int ret; + + rzg3s_pcie_update_bits(host->axi, RZG3S_PCI_REQISS, + RZG3S_PCI_REQISS_REQ_ISSUE, + RZG3S_PCI_REQISS_REQ_ISSUE); + ret = readl_poll_timeout_atomic(host->axi + RZG3S_PCI_REQISS, val, + !(val & RZG3S_PCI_REQISS_REQ_ISSUE), + 5, RZG3S_REQ_ISSUE_TIMEOUT_US); + + if (val & RZG3S_PCI_REQISS_MOR_STATUS) + return -EIO; + + return ret; +} + +static void rzg3s_pcie_child_prepare_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + struct rzg3s_pcie_host *host = bus->sysdata; + unsigned int dev, func, reg; + + dev = PCI_SLOT(devfn); + func = PCI_FUNC(devfn); + reg = where & ~0x3; + + /* Set the destination */ + writel_relaxed(FIELD_PREP(RZG3S_PCI_REQADR1_BUS, bus->number) | + FIELD_PREP(RZG3S_PCI_REQADR1_DEV, dev) | + FIELD_PREP(RZG3S_PCI_REQADR1_FUNC, func) | + FIELD_PREP(RZG3S_PCI_REQADR1_REG, reg), + host->axi + RZG3S_PCI_REQADR1); + + /* Set byte enable */ + writel_relaxed(RZG3S_PCI_REQBE_BYTE_EN, host->axi + RZG3S_PCI_REQBE); +} + +static int rzg3s_pcie_child_read_conf(struct rzg3s_pcie_host *host, + struct pci_bus *bus, unsigned int devfn, + int where, u32 *data) +{ + bool type0 = pci_is_root_bus(bus->parent) ? true : false; + int ret; + + rzg3s_pcie_child_prepare_bus(bus, devfn, where); + + /* Set the type of request */ + writel_relaxed(type0 ? RZG3S_PCI_REQISS_TR_TP0_RD : + RZG3S_PCI_REQISS_TR_TP1_RD, + host->axi + RZG3S_PCI_REQISS); + + /* Issue the request and wait to finish */ + ret = rzg3s_pcie_child_issue_request(host); + if (ret) + return PCIBIOS_SET_FAILED; + + /* Read the data */ + *data = readl_relaxed(host->axi + RZG3S_PCI_REQRCVDAT); + + return PCIBIOS_SUCCESSFUL; +} + +/* Serialization is provided by 'pci_lock' in drivers/pci/access.c */ +static int rzg3s_pcie_child_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct rzg3s_pcie_host *host = bus->sysdata; + int ret; + + ret = rzg3s_pcie_child_read_conf(host, bus, devfn, where, val); + if (ret != PCIBIOS_SUCCESSFUL) + return ret; + + if (size <= 2) + *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); + + return PCIBIOS_SUCCESSFUL; +} + +static int rzg3s_pcie_child_write_conf(struct rzg3s_pcie_host *host, + struct pci_bus *bus, unsigned int devfn, + int where, u32 data) +{ + bool type0 = pci_is_root_bus(bus->parent) ? true : false; + int ret; + + rzg3s_pcie_child_prepare_bus(bus, devfn, where); + + /* Set the write data */ + writel_relaxed(0, host->axi + RZG3S_PCI_REQDATA(0)); + writel_relaxed(0, host->axi + RZG3S_PCI_REQDATA(1)); + writel_relaxed(data, host->axi + RZG3S_PCI_REQDATA(2)); + + /* Set the type of request */ + writel_relaxed(type0 ? RZG3S_PCI_REQISS_TR_TP0_WR : + RZG3S_PCI_REQISS_TR_TP1_WR, + host->axi + RZG3S_PCI_REQISS); + + /* Issue the request and wait to finish */ + ret = rzg3s_pcie_child_issue_request(host); + if (ret) + return PCIBIOS_SET_FAILED; + + return PCIBIOS_SUCCESSFUL; +} + +/* Serialization is provided by 'pci_lock' in drivers/pci/access.c */ +static int rzg3s_pcie_child_write(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + struct rzg3s_pcie_host *host = bus->sysdata; + u32 data, shift; + int ret; + + if (size == 4) + return rzg3s_pcie_child_write_conf(host, bus, devfn, where, val); + + /* + * Controller does 32 bit accesses. To do byte accesses software need + * to do read/modify/write. This may have potential side effects. For + * example, software may perform a 16-bit write. If the hardware only + * supports 32-bit accesses, we must do a 32-bit read, merge in the 16 + * bits we intend to write, followed by a 32-bit write. If the 16 bits + * we *don't* intend to write happen to have any RW1C + * (write-one-to-clear) bits set, we just inadvertently cleared + * something we shouldn't have. + */ + if (!bus->unsafe_warn) { + dev_warn(&bus->dev, "%d-byte config write to %04x:%02x:%02x.%d offset %#x may corrupt adjacent RW1C bits\n", + size, pci_domain_nr(bus), bus->number, + PCI_SLOT(devfn), PCI_FUNC(devfn), where); + bus->unsafe_warn = 1; + } + + ret = rzg3s_pcie_child_read_conf(host, bus, devfn, where, &data); + if (ret != PCIBIOS_SUCCESSFUL) + return ret; + + if (size == 1) { + shift = BITS_PER_BYTE * (where & 3); + data &= ~(0xff << shift); + data |= ((val & 0xff) << shift); + } else if (size == 2) { + shift = BITS_PER_BYTE * (where & 2); + data &= ~(0xffff << shift); + data |= ((val & 0xffff) << shift); + } else { + data = val; + } + + return rzg3s_pcie_child_write_conf(host, bus, devfn, where, data); +} + +static struct pci_ops rzg3s_pcie_child_ops = { + .read = rzg3s_pcie_child_read, + .write = rzg3s_pcie_child_write, +}; + +static void __iomem *rzg3s_pcie_root_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + struct rzg3s_pcie_host *host = bus->sysdata; + + if (devfn) + return NULL; + + return host->pcie + where; +} + +static struct pci_ops rzg3s_pcie_root_ops = { + .read = pci_generic_config_read, + .write = pci_generic_config_write, + .map_bus = rzg3s_pcie_root_map_bus, +}; + +static void rzg3s_pcie_intx_irq_handler(struct irq_desc *desc) +{ + struct rzg3s_pcie_host *host = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + unsigned int irq = irq_desc_get_irq(desc); + u32 intx = irq - host->intx_irqs[0]; + + chained_irq_enter(chip, desc); + generic_handle_domain_irq(host->intx_domain, intx); + chained_irq_exit(chip, desc); +} + +static irqreturn_t rzg3s_pcie_msi_irq(int irq, void *data) +{ + u8 regs = RZG3S_PCI_MSI_INT_NR / RZG3S_PCI_MSI_INT_PER_REG; + DECLARE_BITMAP(bitmap, RZG3S_PCI_MSI_INT_NR) = {0}; + struct rzg3s_pcie_host *host = data; + struct rzg3s_pcie_msi *msi = &host->msi; + unsigned long bit; + u32 status; + + status = readl_relaxed(host->axi + RZG3S_PCI_PINTRCVIS); + if (!(status & RZG3S_PCI_PINTRCVIS_MSI)) + return IRQ_NONE; + + /* Clear the MSI */ + rzg3s_pcie_update_bits(host->axi, RZG3S_PCI_PINTRCVIS, + RZG3S_PCI_PINTRCVIS_MSI, + RZG3S_PCI_PINTRCVIS_MSI); + rzg3s_pcie_update_bits(host->axi, RZG3S_PCI_MSGRCVIS, + RZG3S_PCI_MSGRCVIS_MRI, RZG3S_PCI_MSGRCVIS_MRI); + + for (u8 reg_id = 0; reg_id < regs; reg_id++) { + status = readl_relaxed(host->axi + RZG3S_PCI_MSIRS(reg_id)); + bitmap_write(bitmap, status, reg_id * RZG3S_PCI_MSI_INT_PER_REG, + RZG3S_PCI_MSI_INT_PER_REG); + } + + for_each_set_bit(bit, bitmap, RZG3S_PCI_MSI_INT_NR) { + int ret; + + ret = generic_handle_domain_irq(msi->domain, bit); + if (ret) { + u8 reg_bit = bit % RZG3S_PCI_MSI_INT_PER_REG; + u8 reg_id = bit / RZG3S_PCI_MSI_INT_PER_REG; + + /* Unknown MSI, just clear it */ + writel_relaxed(BIT(reg_bit), + host->axi + RZG3S_PCI_MSIRS(reg_id)); + } + } + + return IRQ_HANDLED; +} + +static void rzg3s_pcie_msi_irq_ack(struct irq_data *d) +{ + struct rzg3s_pcie_msi *msi = irq_data_get_irq_chip_data(d); + struct rzg3s_pcie_host *host = rzg3s_msi_to_host(msi); + u8 reg_bit = d->hwirq % RZG3S_PCI_MSI_INT_PER_REG; + u8 reg_id = d->hwirq / RZG3S_PCI_MSI_INT_PER_REG; + + writel_relaxed(BIT(reg_bit), host->axi + RZG3S_PCI_MSIRS(reg_id)); +} + +static void rzg3s_pcie_msi_irq_mask(struct irq_data *d) +{ + struct rzg3s_pcie_msi *msi = irq_data_get_irq_chip_data(d); + struct rzg3s_pcie_host *host = rzg3s_msi_to_host(msi); + u8 reg_bit = d->hwirq % RZG3S_PCI_MSI_INT_PER_REG; + u8 reg_id = d->hwirq / RZG3S_PCI_MSI_INT_PER_REG; + + guard(raw_spinlock_irqsave)(&host->hw_lock); + + rzg3s_pcie_update_bits(host->axi, RZG3S_PCI_MSIRM(reg_id), BIT(reg_bit), + BIT(reg_bit)); +} + +static void rzg3s_pcie_msi_irq_unmask(struct irq_data *d) +{ + struct rzg3s_pcie_msi *msi = irq_data_get_irq_chip_data(d); + struct rzg3s_pcie_host *host = rzg3s_msi_to_host(msi); + u8 reg_bit = d->hwirq % RZG3S_PCI_MSI_INT_PER_REG; + u8 reg_id = d->hwirq / RZG3S_PCI_MSI_INT_PER_REG; + + guard(raw_spinlock_irqsave)(&host->hw_lock); + + rzg3s_pcie_update_bits(host->axi, RZG3S_PCI_MSIRM(reg_id), BIT(reg_bit), + 0); +} + +static void rzg3s_pcie_irq_compose_msi_msg(struct irq_data *data, + struct msi_msg *msg) +{ + struct rzg3s_pcie_msi *msi = irq_data_get_irq_chip_data(data); + struct rzg3s_pcie_host *host = rzg3s_msi_to_host(msi); + u32 lo, hi; + + /* + * Enable and msg data enable bits are part of the address lo. Drop + * them along with the unused bit. + */ + lo = readl_relaxed(host->axi + RZG3S_PCI_MSIRCVWADRL) & + RZG3S_PCI_MSIRCVWADRL_MASK; + hi = readl_relaxed(host->axi + RZG3S_PCI_MSIRCVWADRU); + + msg->address_lo = lo; + msg->address_hi = hi; + msg->data = data->hwirq; +} + +static struct irq_chip rzg3s_pcie_msi_bottom_chip = { + .name = "rzg3s-pcie-msi", + .irq_ack = rzg3s_pcie_msi_irq_ack, + .irq_mask = rzg3s_pcie_msi_irq_mask, + .irq_unmask = rzg3s_pcie_msi_irq_unmask, + .irq_compose_msi_msg = rzg3s_pcie_irq_compose_msi_msg, +}; + +static int rzg3s_pcie_msi_domain_alloc(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs, + void *args) +{ + struct rzg3s_pcie_msi *msi = domain->host_data; + int hwirq; + + scoped_guard(mutex, &msi->map_lock) { + hwirq = bitmap_find_free_region(msi->map, RZG3S_PCI_MSI_INT_NR, + order_base_2(nr_irqs)); + } + + if (hwirq < 0) + return -ENOSPC; + + for (unsigned int i = 0; i < nr_irqs; i++) { + irq_domain_set_info(domain, virq + i, hwirq + i, + &rzg3s_pcie_msi_bottom_chip, + domain->host_data, handle_edge_irq, NULL, + NULL); + } + + return 0; +} + +static void rzg3s_pcie_msi_domain_free(struct irq_domain *domain, + unsigned int virq, unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct rzg3s_pcie_msi *msi = domain->host_data; + + guard(mutex)(&msi->map_lock); + + bitmap_release_region(msi->map, d->hwirq, order_base_2(nr_irqs)); +} + +static const struct irq_domain_ops rzg3s_pcie_msi_domain_ops = { + .alloc = rzg3s_pcie_msi_domain_alloc, + .free = rzg3s_pcie_msi_domain_free, +}; + +#define RZG3S_PCIE_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_NO_AFFINITY | \ + MSI_FLAG_PCI_MSI_MASK_PARENT) + +#define RZG3S_PCIE_MSI_FLAGS_SUPPORTED (MSI_FLAG_MULTI_PCI_MSI | \ + MSI_GENERIC_FLAGS_MASK) + +static const struct msi_parent_ops rzg3s_pcie_msi_parent_ops = { + .required_flags = RZG3S_PCIE_MSI_FLAGS_REQUIRED, + .supported_flags = RZG3S_PCIE_MSI_FLAGS_SUPPORTED, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .chip_flags = MSI_CHIP_FLAG_SET_ACK, + .prefix = "RZG3S-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +static int rzg3s_pcie_msi_allocate_domains(struct rzg3s_pcie_msi *msi) +{ + struct rzg3s_pcie_host *host = rzg3s_msi_to_host(msi); + struct device *dev = host->dev; + struct irq_domain_info info = { + .fwnode = dev_fwnode(dev), + .ops = &rzg3s_pcie_msi_domain_ops, + .size = RZG3S_PCI_MSI_INT_NR, + .host_data = msi, + }; + + msi->domain = msi_create_parent_irq_domain(&info, + &rzg3s_pcie_msi_parent_ops); + if (!msi->domain) + return dev_err_probe(dev, -ENOMEM, + "failed to create IRQ domain\n"); + + return 0; +} + +static int rzg3s_pcie_msi_hw_setup(struct rzg3s_pcie_host *host) +{ + u8 regs = RZG3S_PCI_MSI_INT_NR / RZG3S_PCI_MSI_INT_PER_REG; + struct rzg3s_pcie_msi *msi = &host->msi; + + /* + * Set MSI window size. HW will set the window to + * RZG3S_PCI_MSI_INT_NR * 4 bytes. + */ + writel_relaxed(FIELD_PREP(RZG3S_PCI_MSIRCVWMSKL_MASK, + RZG3S_PCI_MSI_INT_NR - 1), + host->axi + RZG3S_PCI_MSIRCVWMSKL); + + /* Set MSI window address and enable MSI window */ + writel_relaxed(upper_32_bits(msi->window_base), + host->axi + RZG3S_PCI_MSIRCVWADRU); + writel_relaxed(lower_32_bits(msi->window_base) | + RZG3S_PCI_MSIRCVWADRL_ENA | + RZG3S_PCI_MSIRCVWADRL_MSG_DATA_ENA, + host->axi + RZG3S_PCI_MSIRCVWADRL); + + /* Set MSI receive enable */ + for (u8 reg_id = 0; reg_id < regs; reg_id++) { + writel_relaxed(RZG3S_PCI_MSIRE_ENA, + host->axi + RZG3S_PCI_MSIRE(reg_id)); + } + + /* Enable message receive interrupts */ + writel_relaxed(RZG3S_PCI_MSGRCVIE_MSG_RCV, + host->axi + RZG3S_PCI_MSGRCVIE); + + /* Enable MSI */ + rzg3s_pcie_update_bits(host->axi, RZG3S_PCI_PINTRCVIE, + RZG3S_PCI_PINTRCVIE_MSI, + RZG3S_PCI_PINTRCVIE_MSI); + + return 0; +} + +static int rzg3s_pcie_msi_setup(struct rzg3s_pcie_host *host) +{ + size_t size = RZG3S_PCI_MSI_INT_NR * sizeof(u32); + struct rzg3s_pcie_msi *msi = &host->msi; + struct device *dev = host->dev; + int id, ret; + + msi->pages = __get_free_pages(GFP_KERNEL | GFP_DMA, 0); + if (!msi->pages) + return -ENOMEM; + + msi->dma_addr = dma_map_single(dev, (void *)msi->pages, size * 2, + DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, msi->dma_addr)) { + ret = -ENOMEM; + goto free_pages; + } + + /* + * According to the RZ/G3S HW manual (Rev.1.10, section 34.4.5.2 Setting + * the MSI Window) the MSI window needs to fall within one of the + * enabled AXI windows. Find an enabled AXI window to setup the MSI + * window. + */ + for (id = 0; id < RZG3S_MAX_WINDOWS; id++) { + u64 base, basel, baseu; + u64 mask, maskl, masku; + + basel = readl_relaxed(host->axi + RZG3S_PCI_AWBASEL(id)); + /* Skip checking this AXI window if it's not enabled */ + if (!(basel & RZG3S_PCI_AWBASEL_WIN_ENA)) + continue; + + baseu = readl_relaxed(host->axi + RZG3S_PCI_AWBASEU(id)); + base = baseu << 32 | basel; + + maskl = readl_relaxed(host->axi + RZG3S_PCI_AWMASKL(id)); + masku = readl_relaxed(host->axi + RZG3S_PCI_AWMASKU(id)); + mask = masku << 32 | maskl; + + if (msi->dma_addr < base || msi->dma_addr > base + mask) + continue; + + break; + } + + if (id == RZG3S_MAX_WINDOWS) { + ret = -EINVAL; + goto dma_unmap; + } + + /* The MSI base address must be aligned to the MSI size */ + msi->window_base = ALIGN(msi->dma_addr, size); + if (msi->window_base < msi->dma_addr) { + ret = -EINVAL; + goto dma_unmap; + } + + rzg3s_pcie_msi_hw_setup(host); + + return 0; + +dma_unmap: + dma_unmap_single(dev, msi->dma_addr, size * 2, DMA_BIDIRECTIONAL); +free_pages: + free_pages(msi->pages, 0); + return ret; +} + +static void rzg3s_pcie_msi_hw_teardown(struct rzg3s_pcie_host *host) +{ + u8 regs = RZG3S_PCI_MSI_INT_NR / RZG3S_PCI_MSI_INT_PER_REG; + + /* Disable MSI */ + rzg3s_pcie_update_bits(host->axi, RZG3S_PCI_PINTRCVIE, + RZG3S_PCI_PINTRCVIE_MSI, 0); + + /* Disable message receive interrupts */ + rzg3s_pcie_update_bits(host->axi, RZG3S_PCI_MSGRCVIE, + RZG3S_PCI_MSGRCVIE_MSG_RCV, 0); + + /* Disable MSI receive enable */ + for (u8 reg_id = 0; reg_id < regs; reg_id++) + writel_relaxed(0, host->axi + RZG3S_PCI_MSIRE(reg_id)); + + /* Disable MSI window */ + writel_relaxed(0, host->axi + RZG3S_PCI_MSIRCVWADRL); +} + +static void rzg3s_pcie_teardown_msi(struct rzg3s_pcie_host *host) +{ + size_t size = RZG3S_PCI_MSI_INT_NR * sizeof(u32); + struct rzg3s_pcie_msi *msi = &host->msi; + + rzg3s_pcie_msi_hw_teardown(host); + + free_irq(msi->irq, host); + irq_domain_remove(msi->domain); + + /* Free unused memory */ + dma_unmap_single(host->dev, msi->dma_addr, size * 2, DMA_BIDIRECTIONAL); + free_pages(msi->pages, 0); +} + +static int rzg3s_pcie_init_msi(struct rzg3s_pcie_host *host) +{ + struct platform_device *pdev = to_platform_device(host->dev); + struct rzg3s_pcie_msi *msi = &host->msi; + struct device *dev = host->dev; + const char *devname; + int ret; + + ret = devm_mutex_init(dev, &msi->map_lock); + if (ret) + return ret; + + msi->irq = platform_get_irq_byname(pdev, "msi"); + if (msi->irq < 0) + return dev_err_probe(dev, msi->irq, "Failed to get MSI IRQ!\n"); + + devname = devm_kasprintf(dev, GFP_KERNEL, "%s-msi", dev_name(dev)); + if (!devname) + return -ENOMEM; + + ret = rzg3s_pcie_msi_allocate_domains(msi); + if (ret) + return ret; + + /* + * Don't use devm_request_irq() as the driver uses non-devm helpers + * to control clocks. Mixing them may lead to subtle bugs. + */ + ret = request_irq(msi->irq, rzg3s_pcie_msi_irq, 0, devname, host); + if (ret) { + dev_err_probe(dev, ret, "Failed to request IRQ: %d\n", ret); + goto free_domains; + } + + ret = rzg3s_pcie_msi_setup(host); + if (ret) { + dev_err_probe(dev, ret, "Failed to setup MSI!\n"); + goto free_irq; + } + + return 0; + +free_irq: + free_irq(msi->irq, host); +free_domains: + irq_domain_remove(msi->domain); + return ret; +} + +static void rzg3s_pcie_intx_irq_ack(struct irq_data *d) +{ + struct rzg3s_pcie_host *host = irq_data_get_irq_chip_data(d); + + rzg3s_pcie_update_bits(host->axi, RZG3S_PCI_PINTRCVIS, + RZG3S_PCI_PINTRCVIS_INTX(d->hwirq), + RZG3S_PCI_PINTRCVIS_INTX(d->hwirq)); +} + +static void rzg3s_pcie_intx_irq_mask(struct irq_data *d) +{ + struct rzg3s_pcie_host *host = irq_data_get_irq_chip_data(d); + + guard(raw_spinlock_irqsave)(&host->hw_lock); + + rzg3s_pcie_update_bits(host->axi, RZG3S_PCI_PINTRCVIE, + RZG3S_PCI_PINTRCVIE_INTX(d->hwirq), 0); +} + +static void rzg3s_pcie_intx_irq_unmask(struct irq_data *d) +{ + struct rzg3s_pcie_host *host = irq_data_get_irq_chip_data(d); + + guard(raw_spinlock_irqsave)(&host->hw_lock); + + rzg3s_pcie_update_bits(host->axi, RZG3S_PCI_PINTRCVIE, + RZG3S_PCI_PINTRCVIE_INTX(d->hwirq), + RZG3S_PCI_PINTRCVIE_INTX(d->hwirq)); +} + +static struct irq_chip rzg3s_pcie_intx_irq_chip = { + .name = "PCIe INTx", + .irq_ack = rzg3s_pcie_intx_irq_ack, + .irq_mask = rzg3s_pcie_intx_irq_mask, + .irq_unmask = rzg3s_pcie_intx_irq_unmask, +}; + +static int rzg3s_pcie_intx_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &rzg3s_pcie_intx_irq_chip, + handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + + return 0; +} + +static const struct irq_domain_ops rzg3s_pcie_intx_domain_ops = { + .map = rzg3s_pcie_intx_map, + .xlate = irq_domain_xlate_onetwocell, +}; + +static int rzg3s_pcie_init_irqdomain(struct rzg3s_pcie_host *host) +{ + struct device *dev = host->dev; + struct platform_device *pdev = to_platform_device(dev); + + for (int i = 0; i < PCI_NUM_INTX; i++) { + char irq_name[5] = {0}; + int irq; + + scnprintf(irq_name, ARRAY_SIZE(irq_name), "int%c", 'a' + i); + + irq = platform_get_irq_byname(pdev, irq_name); + if (irq < 0) + return dev_err_probe(dev, -EINVAL, + "Failed to parse and map INT%c IRQ\n", + 'A' + i); + + host->intx_irqs[i] = irq; + irq_set_chained_handler_and_data(irq, + rzg3s_pcie_intx_irq_handler, + host); + } + + host->intx_domain = irq_domain_create_linear(dev_fwnode(dev), + PCI_NUM_INTX, + &rzg3s_pcie_intx_domain_ops, + host); + if (!host->intx_domain) + return dev_err_probe(dev, -EINVAL, + "Failed to add irq domain for INTx IRQs\n"); + irq_domain_update_bus_token(host->intx_domain, DOMAIN_BUS_WIRED); + + if (IS_ENABLED(CONFIG_PCI_MSI)) { + int ret = rzg3s_pcie_init_msi(host); + + if (ret) { + irq_domain_remove(host->intx_domain); + return ret; + } + } + + return 0; +} + +static void rzg3s_pcie_teardown_irqdomain(struct rzg3s_pcie_host *host) +{ + if (IS_ENABLED(CONFIG_PCI_MSI)) + rzg3s_pcie_teardown_msi(host); + + irq_domain_remove(host->intx_domain); +} + +static int rzg3s_pcie_set_max_link_speed(struct rzg3s_pcie_host *host) +{ + u32 remote_supported_link_speeds, max_supported_link_speeds; + u32 cs2, tmp, pcie_cap = RZG3S_PCI_CFG_PCIEC; + u32 cur_link_speed, link_speed, hw_max_speed; + u8 ltssm_state_l0 = 0xc; + u32 lnkcap; + int ret; + u16 ls; + + /* + * According to the RZ/G3S HW manual (Rev.1.10, section 34.6.3 Caution + * when Changing the Speed Spontaneously) link speed change can be done + * only when the LTSSM is in L0. + */ + ret = readl_poll_timeout(host->axi + RZG3S_PCI_PCSTAT1, tmp, + FIELD_GET(RZG3S_PCI_PCSTAT1_LTSSM_STATE, tmp) == ltssm_state_l0, + PCIE_LINK_WAIT_SLEEP_MS * MILLI, + PCIE_LINK_WAIT_SLEEP_MS * MILLI * + PCIE_LINK_WAIT_MAX_RETRIES); + if (ret) + return ret; + + ls = readw_relaxed(host->pcie + pcie_cap + PCI_EXP_LNKSTA); + cs2 = readl_relaxed(host->axi + RZG3S_PCI_PCSTAT2); + + /* Read hardware supported link speed from Link Capabilities Register */ + lnkcap = readl_relaxed(host->pcie + pcie_cap + PCI_EXP_LNKCAP); + hw_max_speed = FIELD_GET(PCI_EXP_LNKCAP_SLS, lnkcap); + + /* + * Use DT max-link-speed only as a limit. If specified and lower + * than hardware capability, cap to that value. + */ + if (host->max_link_speed > 0 && host->max_link_speed < hw_max_speed) + hw_max_speed = host->max_link_speed; + + switch (pcie_get_link_speed(hw_max_speed)) { + case PCIE_SPEED_8_0GT: + max_supported_link_speeds = GENMASK(PCI_EXP_LNKSTA_CLS_8_0GB - 1, 0); + link_speed = PCI_EXP_LNKCTL2_TLS_8_0GT; + break; + case PCIE_SPEED_5_0GT: + max_supported_link_speeds = GENMASK(PCI_EXP_LNKSTA_CLS_5_0GB - 1, 0); + link_speed = PCI_EXP_LNKCTL2_TLS_5_0GT; + break; + default: + /* Should not happen */ + return -EINVAL; + } + + cur_link_speed = FIELD_GET(PCI_EXP_LNKSTA_CLS, ls); + remote_supported_link_speeds = FIELD_GET(RZG3S_PCI_PCSTAT2_SDRIRE, cs2); + /* Drop reserved bits */ + remote_supported_link_speeds &= max_supported_link_speeds; + + /* + * Return if target link speed is already set or the connected device + * doesn't support it. + */ + if (cur_link_speed == hw_max_speed || + remote_supported_link_speeds != max_supported_link_speeds) + return 0; + + /* Set target Link speed */ + rzg3s_pcie_update_bits(host->pcie, pcie_cap + PCI_EXP_LNKCTL2, + PCI_EXP_LNKCTL2_TLS, + FIELD_PREP(PCI_EXP_LNKCTL2_TLS, link_speed)); + + /* Request link speed change */ + rzg3s_pcie_update_bits(host->axi, RZG3S_PCI_PCCTRL2, + RZG3S_PCI_PCCTRL2_LS_CHG_REQ | + RZG3S_PCI_PCCTRL2_LS_CHG, + RZG3S_PCI_PCCTRL2_LS_CHG_REQ | + FIELD_PREP(RZG3S_PCI_PCCTRL2_LS_CHG, + link_speed - 1)); + + ret = readl_poll_timeout(host->axi + RZG3S_PCI_PCSTAT2, cs2, + (cs2 & RZG3S_PCI_PCSTAT2_LS_CHG_DONE), + PCIE_LINK_WAIT_SLEEP_MS * MILLI, + PCIE_LINK_WAIT_SLEEP_MS * MILLI * + PCIE_LINK_WAIT_MAX_RETRIES); + + /* + * According to the RZ/G3S HW manual (Rev.1.10, section 34.6.3 Caution + * when Changing the Speed Spontaneously) the PCI_PCCTRL2_LS_CHG_REQ + * should be de-asserted after checking for PCI_PCSTAT2_LS_CHG_DONE. + */ + rzg3s_pcie_update_bits(host->axi, RZG3S_PCI_PCCTRL2, + RZG3S_PCI_PCCTRL2_LS_CHG_REQ, 0); + + return ret; +} + +static int rzg3s_pcie_config_init(struct rzg3s_pcie_host *host) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(host); + u32 mask = GENMASK(31, 8); + struct resource_entry *ft; + struct resource *bus; + u8 subordinate_bus; + u8 secondary_bus; + u8 primary_bus; + + ft = resource_list_first_type(&bridge->windows, IORESOURCE_BUS); + if (!ft) + return -ENODEV; + + bus = ft->res; + primary_bus = bus->start; + secondary_bus = bus->start + 1; + subordinate_bus = bus->end; + + /* Enable access control to the CFGU */ + writel_relaxed(RZG3S_PCI_PERM_CFG_HWINIT_EN, + host->axi + RZG3S_PCI_PERM); + + /* HW manual recommends to write 0xffffffff on initialization */ + writel_relaxed(0xffffffff, host->pcie + RZG3S_PCI_CFG_BARMSK00L); + writel_relaxed(0xffffffff, host->pcie + RZG3S_PCI_CFG_BARMSK00U); + + /* + * Explicitly program class code. RZ/G3E requires this configuration. + * Harmless for RZ/G3S where this matches the hardware default. + */ + rzg3s_pcie_update_bits(host->pcie, PCI_CLASS_REVISION, mask, + field_prep(mask, PCI_CLASS_BRIDGE_PCI_NORMAL)); + + if (host->num_lanes) { + rzg3s_pcie_update_bits(host->pcie + RZG3S_PCI_CFG_PCIEC, + PCI_EXP_LNKCAP, PCI_EXP_LNKCAP_MLW, + FIELD_PREP(PCI_EXP_LNKCAP_MLW, + host->num_lanes)); + } + + /* Disable access control to the CFGU */ + writel_relaxed(0, host->axi + RZG3S_PCI_PERM); + + /* Update bus info */ + writeb_relaxed(primary_bus, host->pcie + PCI_PRIMARY_BUS); + writeb_relaxed(secondary_bus, host->pcie + PCI_SECONDARY_BUS); + writeb_relaxed(subordinate_bus, host->pcie + PCI_SUBORDINATE_BUS); + + return 0; +} + +static int rzg3s_pcie_config_post_init(struct rzg3s_pcie_host *host) +{ + return reset_control_bulk_deassert(host->data->num_cfg_resets, + host->cfg_resets); +} + +static int rzg3s_pcie_config_deinit(struct rzg3s_pcie_host *host) +{ + return reset_control_bulk_assert(host->data->num_cfg_resets, + host->cfg_resets); +} + +static void rzg3e_pcie_config_pre_init(struct rzg3s_pcie_host *host) +{ + u32 mask = RZG3S_PCI_RESET_RST_LOAD_B | RZG3S_PCI_RESET_RST_CFG_B; + + /* De-assert LOAD_B and CFG_B */ + rzg3s_pcie_update_bits(host->axi, RZG3S_PCI_RESET, mask, mask); +} + +static int rzg3e_pcie_config_deinit(struct rzg3s_pcie_host *host) +{ + writel_relaxed(0, host->axi + RZG3S_PCI_RESET); + return 0; +} + +static int rzg3e_pcie_config_post_init(struct rzg3s_pcie_host *host) +{ + u32 mask = RZG3S_PCI_RESET_RST_PS_B | RZG3S_PCI_RESET_RST_GP_B | + RZG3S_PCI_RESET_RST_B; + + /* De-assert PS_B, GP_B, RST_B */ + rzg3s_pcie_update_bits(host->axi, RZG3S_PCI_RESET, mask, mask); + + /* Flush deassert */ + readl_relaxed(host->axi + RZG3S_PCI_RESET); + + /* + * According to the RZ/G3E HW manual (Rev.1.15, Table 6.6-130 + * Initialization Procedure (RC)), hardware requires >= 500us delay + * before final reset deassert. + */ + fsleep(500); + + /* De-assert OUT_B and RSM_B */ + mask = RZG3S_PCI_RESET_RST_OUT_B | RZG3S_PCI_RESET_RST_RSM_B; + rzg3s_pcie_update_bits(host->axi, RZG3S_PCI_RESET, mask, mask); + + return 0; +} + +static void rzg3s_pcie_irq_init(struct rzg3s_pcie_host *host) +{ + /* + * According to the HW manual of the RZ/G3S (Rev.1.10, sections + * corresponding to all registers written with ~0U), the hardware + * ignores value written to unused bits. Writing ~0U to these registers + * should be safe. + */ + + /* Clear the link state and PM transitions */ + writel_relaxed(RZG3S_PCI_PEIS0_DL_UPDOWN | + RZG3S_PCI_PEIS0_RX_DLLP_PM_ENTER, + host->axi + RZG3S_PCI_PEIS0); + + /* Disable all interrupts */ + writel_relaxed(0, host->axi + RZG3S_PCI_PEIE0); + + /* Clear all parity and ecc error interrupts */ + writel_relaxed(~0U, host->axi + RZG3S_PCI_PEIS1); + + /* Disable all parity and ecc error interrupts */ + writel_relaxed(0, host->axi + RZG3S_PCI_PEIE1); + + /* Clear all AXI master error interrupts */ + writel_relaxed(~0U, host->axi + RZG3S_PCI_AMEIS); + + /* Clear all AXI slave error interrupts */ + writel_relaxed(~0U, host->axi + RZG3S_PCI_ASEIS1); + + /* Clear all message receive interrupts */ + writel_relaxed(~0U, host->axi + RZG3S_PCI_MSGRCVIS); +} + +static int rzg3s_pcie_power_resets_deassert(struct rzg3s_pcie_host *host) +{ + const struct rzg3s_pcie_soc_data *data = host->data; + + /* + * According to the RZ/G3S HW manual (Rev.1.10, section + * 34.5.1.2 De-asserting the Reset) the PCIe IP needs to wait 5ms from + * power on to the de-assertion of reset. + */ + fsleep(5000); + return reset_control_bulk_deassert(data->num_power_resets, + host->power_resets); +} + +static int rzg3s_pcie_resets_prepare_and_get(struct rzg3s_pcie_host *host) +{ + const struct rzg3s_pcie_soc_data *data = host->data; + unsigned int i; + int ret; + + host->power_resets = devm_kmalloc_array(host->dev, + data->num_power_resets, + sizeof(*host->power_resets), + GFP_KERNEL); + if (!host->power_resets) + return -ENOMEM; + + for (i = 0; i < data->num_power_resets; i++) + host->power_resets[i].id = data->power_resets[i]; + + host->cfg_resets = devm_kmalloc_array(host->dev, + data->num_cfg_resets, + sizeof(*host->cfg_resets), + GFP_KERNEL); + if (!host->cfg_resets) + return -ENOMEM; + + for (i = 0; i < data->num_cfg_resets; i++) + host->cfg_resets[i].id = data->cfg_resets[i]; + + ret = devm_reset_control_bulk_get_shared(host->dev, + data->num_power_resets, + host->power_resets); + if (ret) + return ret; + + return devm_reset_control_bulk_get_optional_exclusive(host->dev, + data->num_cfg_resets, + host->cfg_resets); +} + +static int rzg3s_pcie_host_parse_port(struct rzg3s_pcie_host *host) +{ + struct device_node *of_port __free(device_node) = + of_get_next_child(host->dev->of_node, NULL); + struct rzg3s_pcie_port *port = &host->port; + int ret; + + ret = of_property_read_u32(of_port, "vendor-id", &port->vendor_id); + if (ret) + return ret; + + ret = of_property_read_u32(of_port, "device-id", &port->device_id); + if (ret) + return ret; + + port->refclk = of_clk_get_by_name(of_port, "ref"); + if (IS_ERR(port->refclk)) + return PTR_ERR(port->refclk); + + return 0; +} + +static int rzg3s_pcie_host_init_port(struct rzg3s_pcie_host *host) +{ + struct rzg3s_pcie_port *port = &host->port; + struct device *dev = host->dev; + int ret; + + /* Enable access control to the CFGU */ + writel_relaxed(RZG3S_PCI_PERM_CFG_HWINIT_EN, + host->axi + RZG3S_PCI_PERM); + + /* Update vendor ID and device ID */ + writew_relaxed(port->vendor_id, host->pcie + PCI_VENDOR_ID); + writew_relaxed(port->device_id, host->pcie + PCI_DEVICE_ID); + + /* Disable access control to the CFGU */ + writel_relaxed(0, host->axi + RZG3S_PCI_PERM); + + ret = clk_prepare_enable(port->refclk); + if (ret) + return dev_err_probe(dev, ret, "Failed to enable refclk!\n"); + + /* Set the PHY, if any */ + if (host->data->init_phy) { + ret = host->data->init_phy(host); + if (ret) { + dev_err_probe(dev, ret, "Failed to set the PHY!\n"); + goto refclk_disable; + } + } + + return 0; + +refclk_disable: + clk_disable_unprepare(port->refclk); + return ret; +} + +static int rzg3s_pcie_host_init(struct rzg3s_pcie_host *host) +{ + u32 val; + int ret; + + /* SoC-specific pre-configuration */ + if (host->data->config_pre_init) + host->data->config_pre_init(host); + + /* Initialize the PCIe related registers */ + ret = rzg3s_pcie_config_init(host); + if (ret) + goto config_deinit; + + ret = rzg3s_pcie_host_init_port(host); + if (ret) + goto config_deinit; + + /* Enable ASPM L1 transition for SoCs that use it */ + ret = rzg3s_sysc_config_func(host->sysc, + RZG3S_SYSC_FUNC_ID_L1_ALLOW, 1); + if (ret) + goto config_deinit_and_refclk; + + /* Initialize the interrupts */ + rzg3s_pcie_irq_init(host); + + /* SoC-specific post-configuration */ + ret = host->data->config_post_init(host); + if (ret) + goto config_deinit_and_refclk; + + /* Wait for link up */ + ret = readl_poll_timeout(host->axi + RZG3S_PCI_PCSTAT1, val, + !(val & RZG3S_PCI_PCSTAT1_DL_DOWN_STS), + PCIE_LINK_WAIT_SLEEP_MS * MILLI, + PCIE_LINK_WAIT_SLEEP_MS * MILLI * + PCIE_LINK_WAIT_MAX_RETRIES); + if (ret) + goto config_deinit_post; + + val = readl_relaxed(host->axi + RZG3S_PCI_PCSTAT2); + dev_info(host->dev, "PCIe link status [0x%x]\n", val); + + return 0; + +config_deinit_post: + host->data->config_deinit(host); +config_deinit_and_refclk: + clk_disable_unprepare(host->port.refclk); +config_deinit: + if (host->data->config_pre_init) + host->data->config_deinit(host); + return ret; +} + +static void rzg3s_pcie_set_inbound_window(struct rzg3s_pcie_host *host, + u64 cpu_addr, u64 pci_addr, u64 size, + int id) +{ + /* Set CPU window base address */ + writel_relaxed(upper_32_bits(cpu_addr), + host->axi + RZG3S_PCI_ADESTU(id)); + writel_relaxed(lower_32_bits(cpu_addr), + host->axi + RZG3S_PCI_ADESTL(id)); + + /* Set window size */ + writel_relaxed(upper_32_bits(size), host->axi + RZG3S_PCI_AWMASKU(id)); + writel_relaxed(lower_32_bits(size), host->axi + RZG3S_PCI_AWMASKL(id)); + + /* Set PCIe window base address and enable the window */ + writel_relaxed(upper_32_bits(pci_addr), + host->axi + RZG3S_PCI_AWBASEU(id)); + writel_relaxed(lower_32_bits(pci_addr) | RZG3S_PCI_AWBASEL_WIN_ENA, + host->axi + RZG3S_PCI_AWBASEL(id)); +} + +static int rzg3s_pcie_set_inbound_windows(struct rzg3s_pcie_host *host, + struct resource_entry *entry, + int *index) +{ + u64 pci_addr = entry->res->start - entry->offset; + u64 cpu_addr = entry->res->start; + u64 cpu_end = entry->res->end; + int id = *index; + u64 size; + + /* + * According to the RZ/G3S HW manual (Rev.1.10, section 34.6.6.7) and + * RZ/G3E HW manual (Rev.1.15, section 6.6.7.6): + * - Each window must be a single memory size of power of two + * - Mask registers must be set to (2^N - 1) + * - Bit carry must not occur when adding base and mask registers, + * meaning the base address must be aligned to the window size + * + * Split non-power-of-2 regions into multiple windows to satisfy + * these constraints without over-mapping. + */ + while (cpu_addr <= cpu_end) { + u64 remaining_size = cpu_end - cpu_addr + 1; + u64 align_limit; + + if (id >= RZG3S_MAX_WINDOWS) + return dev_err_probe(host->dev, -ENOSPC, + "Failed to map inbound window for resource (%s)\n", + entry->res->name); + + /* Start with largest power-of-two that fits in remaining size */ + size = 1ULL << __fls(remaining_size); + + /* + * The "no bit carry" rule requires base addresses to be + * aligned to the window size. Find the maximum window size + * that both addresses can support based on their natural + * alignment (lowest set bit). + */ + align_limit = min(cpu_addr ? (1ULL << __ffs(cpu_addr)) : ~0ULL, + pci_addr ? (1ULL << __ffs(pci_addr)) : ~0ULL); + + size = min(size, align_limit); + + /* + * Minimum window size is 4KB. + * See RZ/G3S HW manual (Rev.1.10, section 34.3.1.71) and + * RZ/G3E HW manual (Rev.1.15, section 6.6.4.1.3.(74)). + */ + size = max(size, SZ_4K); + + rzg3s_pcie_set_inbound_window(host, cpu_addr, pci_addr, + size - 1, id); + + pci_addr += size; + cpu_addr += size; + id++; + } + *index = id; + + return 0; +} + +static int rzg3s_pcie_parse_map_dma_ranges(struct rzg3s_pcie_host *host) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(host); + struct resource_entry *entry; + int i = 0, ret; + + resource_list_for_each_entry(entry, &bridge->dma_ranges) { + ret = rzg3s_pcie_set_inbound_windows(host, entry, &i); + if (ret) + return ret; + } + + return 0; +} + +static void rzg3s_pcie_set_outbound_window(struct rzg3s_pcie_host *host, + struct resource_entry *win, int id) +{ + struct resource *res = win->res; + resource_size_t size = resource_size(res); + resource_size_t res_start; + + if (res->flags & IORESOURCE_IO) + res_start = pci_pio_to_address(res->start) - win->offset; + else + res_start = res->start - win->offset; + + /* + * According to the RZ/G3S HW manual (Rev.1.10, section 34.3.1.75 PCIe + * Window Base (Lower) Registers) the window base address need to be 4K + * aligned. + */ + res_start = ALIGN(res_start, SZ_4K); + + size = ALIGN(size, SZ_4K); + size = roundup_pow_of_two(size) - 1; + + /* Set PCIe destination */ + writel_relaxed(upper_32_bits(res_start), + host->axi + RZG3S_PCI_PDESTU(id)); + writel_relaxed(lower_32_bits(res_start), + host->axi + RZG3S_PCI_PDESTL(id)); + + /* Set PCIe window mask */ + writel_relaxed(upper_32_bits(size), host->axi + RZG3S_PCI_PWMASKU(id)); + writel_relaxed(lower_32_bits(size), host->axi + RZG3S_PCI_PWMASKL(id)); + + /* Set PCIe window base and enable the window */ + writel_relaxed(upper_32_bits(res_start), + host->axi + RZG3S_PCI_PWBASEU(id)); + writel_relaxed(lower_32_bits(res_start) | RZG3S_PCI_PWBASEL_ENA, + host->axi + RZG3S_PCI_PWBASEL(id)); +} + +static int rzg3s_pcie_parse_map_ranges(struct rzg3s_pcie_host *host) +{ + struct pci_host_bridge *bridge = pci_host_bridge_from_priv(host); + struct resource_entry *win; + int i = 0; + + resource_list_for_each_entry(win, &bridge->windows) { + struct resource *res = win->res; + + if (i >= RZG3S_MAX_WINDOWS) + return dev_err_probe(host->dev, -ENOSPC, + "Failed to map outbound window for resource (%s)\n", + res->name); + + if (!res->flags) + continue; + + switch (resource_type(res)) { + case IORESOURCE_IO: + case IORESOURCE_MEM: + rzg3s_pcie_set_outbound_window(host, win, i); + i++; + break; + } + } + + return 0; +} + +static int rzg3s_soc_pcie_init_phy(struct rzg3s_pcie_host *host) +{ + static const u32 xcfgd_settings[RZG3S_PCI_PHY_XCFGD_NUM] = { + [8] = 0xe0006801, 0x007f7e30, 0x183e0000, 0x978ff500, + 0xec000000, 0x009f1400, 0x0000d009, + [17] = 0x78000000, + [19] = 0x00880000, 0x000005c0, 0x07000000, 0x00780920, + 0xc9400ce2, 0x90000c0c, 0x000c1414, 0x00005034, + 0x00006000, 0x00000001, + }; + static const u32 xcfga_cmn_settings[RZG3S_PCI_PHY_XCFGA_CMN_NUM] = { + 0x00000d10, 0x08310100, 0x00c21404, 0x013c0010, 0x01874440, + 0x1a216082, 0x00103440, 0x00000080, 0x00000010, 0x0c1000c1, + 0x1000c100, 0x0222000c, 0x00640019, 0x00a00028, 0x01d11228, + 0x0201001d, + }; + static const u32 xcfga_rx_settings[RZG3S_PCI_PHY_XCFGA_RX_NUM] = { + 0x07d55000, 0x030e3f00, 0x00000288, 0x102c5880, 0x0000000b, + 0x04141441, 0x00641641, 0x00d63d63, 0x00641641, 0x01970377, + 0x00190287, 0x00190028, 0x00000028, + }; + unsigned int i; + + /* + * Enable access permission for physical layer control and status + * registers. + */ + writel_relaxed(RZG3S_PCI_PERM_PIPE_PHY_REG_EN, + host->axi + RZG3S_PCI_PERM); + + for (i = 0; i < RZG3S_PCI_PHY_XCFGD_NUM; i++) { + writel_relaxed(xcfgd_settings[i], + host->axi + RZG3S_PCI_PHY_XCFGD(i)); + } + + for (i = 0; i < RZG3S_PCI_PHY_XCFGA_CMN_NUM; i++) { + writel_relaxed(xcfga_cmn_settings[i], + host->axi + RZG3S_PCI_PHY_XCFGA_CMN(i)); + } + + for (i = 0; i < RZG3S_PCI_PHY_XCFGA_RX_NUM; i++) { + writel_relaxed(xcfga_rx_settings[i], + host->axi + RZG3S_PCI_PHY_XCFGA_RX(i)); + } + + writel_relaxed(0x107, host->axi + RZG3S_PCI_PHY_XCFGA_TX); + + /* Select PHY settings values */ + writel_relaxed(RZG3S_PCI_PHY_XCFG_CTRL_PHYREG_SEL, + host->axi + RZG3S_PCI_PHY_XCFG_CTRL); + + /* + * Disable access permission for physical layer control and status + * registers. + */ + writel_relaxed(0, host->axi + RZG3S_PCI_PERM); + + return 0; +} + +static int +rzg3s_pcie_host_setup(struct rzg3s_pcie_host *host, + int (*init_irqdomain)(struct rzg3s_pcie_host *host), + void (*teardown_irqdomain)(struct rzg3s_pcie_host *host)) +{ + struct device *dev = host->dev; + int ret; + + /* Set inbound windows */ + ret = rzg3s_pcie_parse_map_dma_ranges(host); + if (ret) + return dev_err_probe(dev, ret, + "Failed to set inbound windows!\n"); + + /* Set outbound windows */ + ret = rzg3s_pcie_parse_map_ranges(host); + if (ret) + return dev_err_probe(dev, ret, + "Failed to set outbound windows!\n"); + + ret = init_irqdomain(host); + if (ret) + return dev_err_probe(dev, ret, "Failed to init IRQ domain\n"); + + ret = rzg3s_pcie_host_init(host); + if (ret) { + dev_err_probe(dev, ret, "Failed to initialize the HW!\n"); + goto teardown_irqdomain; + } + + ret = rzg3s_pcie_set_max_link_speed(host); + if (ret) + dev_info(dev, "Failed to set max link speed\n"); + + pci_host_common_link_train_delay(host->max_link_speed); + + return 0; + +teardown_irqdomain: + teardown_irqdomain(host); + + return ret; +} + +static int rzg3s_pcie_get_controller_id(struct rzg3s_pcie_host *host) +{ + struct device_node *np = host->dev->of_node; + struct of_phandle_args sysc_args; + int ret; + + if (host->data->num_pcie_controllers == 1) + return 0; + + ret = of_parse_phandle_with_fixed_args(np, "renesas,sysc", 1, 0, &sysc_args); + if (ret) + return ret; + + of_node_put(sysc_args.np); + + if (sysc_args.args[0] >= host->data->num_pcie_controllers || + sysc_args.args[0] >= RZG3S_PCIE_CONTROLLER_ID_MAX) + return -EINVAL; + + host->controller_id = sysc_args.args[0]; + + return 0; +} + +static int rzv2h_pcie_setup_lanes(struct rzg3s_pcie_host *host) +{ + struct device_node *np = host->dev->of_node; + u32 num_lanes; + int ret; + + ret = of_property_read_u32(np, "num-lanes", &num_lanes); + if (ret) + return ret; + + /* + * RZ/V2H(P) supports up to 4 lanes, but only in single x4 mode + * for the first controller. Dual x2 mode is supported with 2 + * lanes for both controllers. + */ + if (num_lanes != 4 && num_lanes != 2) + return -EINVAL; + + if (host->controller_id == RZG3S_PCIE_CONTROLLER_ID_1 && num_lanes > 2) + return -EINVAL; + + guard(spinlock)(&rzv2h_lane_lock); + if (rzv2h_num_total_lanes + num_lanes > RZV2H_PCIE_MAX_LANES) + return -EINVAL; + + ret = rzg3s_sysc_config_func(host->sysc, RZG3S_SYSC_FUNC_ID_LINK_MASTER, + num_lanes == 2 ? + RZG3S_SYSC_LINK_MODE_DUAL_X2 : + RZG3S_SYSC_LINK_MODE_SINGLE_X4); + if (ret) + return ret; + + rzv2h_num_total_lanes += num_lanes; + host->num_lanes = num_lanes; + + return 0; +} + +static void rzv2h_pcie_release_lanes(void *data) +{ + struct rzg3s_pcie_host *host = data; + + guard(spinlock)(&rzv2h_lane_lock); + rzv2h_num_total_lanes -= host->num_lanes; +} + +static int rzg3s_pcie_probe(struct platform_device *pdev) +{ + struct pci_host_bridge *bridge; + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + struct device_node *sysc_np __free(device_node) = + of_parse_phandle(np, "renesas,sysc", 0); + struct rzg3s_pcie_host *host; + struct rzg3s_sysc *sysc; + int ret; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*host)); + if (!bridge) + return -ENOMEM; + + host = pci_host_bridge_priv(bridge); + host->dev = dev; + host->data = device_get_match_data(dev); + platform_set_drvdata(pdev, host); + + host->sysc = devm_kzalloc(dev, sizeof(*host->sysc), GFP_KERNEL); + if (!host->sysc) + return -ENOMEM; + + ret = rzg3s_pcie_get_controller_id(host); + if (ret) + return ret; + + sysc = host->sysc; + sysc->info = &host->data->sysc_info[host->controller_id]; + + host->axi = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(host->axi)) + return PTR_ERR(host->axi); + host->pcie = host->axi + RZG3S_PCI_CFG_BASE; + + host->max_link_speed = of_pci_get_max_link_speed(np); + + ret = rzg3s_pcie_host_parse_port(host); + if (ret) + return ret; + + sysc->regmap = syscon_node_to_regmap(sysc_np); + if (IS_ERR(sysc->regmap)) { + ret = PTR_ERR(sysc->regmap); + goto port_refclk_put; + } + + /* Put controller in RC mode */ + ret = rzg3s_sysc_config_func(sysc, RZG3S_SYSC_FUNC_ID_MODE, 1); + if (ret) + goto port_refclk_put; + + ret = rzg3s_sysc_config_func(sysc, RZG3S_SYSC_FUNC_ID_RST_RSM_B, 1); + if (ret) + goto port_refclk_put; + + if (host->data->setup_lanes) { + ret = host->data->setup_lanes(host); + if (ret) + goto sysc_signal_restore; + + ret = devm_add_action_or_reset(dev, rzv2h_pcie_release_lanes, host); + if (ret) + goto sysc_signal_restore; + } + + ret = rzg3s_pcie_resets_prepare_and_get(host); + if (ret) + goto sysc_signal_restore; + + ret = rzg3s_pcie_power_resets_deassert(host); + if (ret) + goto sysc_signal_restore; + + pm_runtime_enable(dev); + + /* + * Controller clocks are part of a clock power domain. Enable them + * through runtime PM. + */ + ret = pm_runtime_resume_and_get(dev); + if (ret) + goto rpm_disable; + + raw_spin_lock_init(&host->hw_lock); + + ret = rzg3s_pcie_host_setup(host, rzg3s_pcie_init_irqdomain, + rzg3s_pcie_teardown_irqdomain); + if (ret) + goto rpm_put; + + bridge->sysdata = host; + bridge->ops = &rzg3s_pcie_root_ops; + bridge->child_ops = &rzg3s_pcie_child_ops; + ret = pci_host_probe(bridge); + if (ret) + goto host_probe_teardown; + + return 0; + +host_probe_teardown: + rzg3s_pcie_teardown_irqdomain(host); + host->data->config_deinit(host); +rpm_put: + pm_runtime_put_sync(dev); +rpm_disable: + pm_runtime_disable(dev); + reset_control_bulk_assert(host->data->num_power_resets, + host->power_resets); +sysc_signal_restore: + /* + * SYSC RST_RSM_B signal need to be asserted before turning off the + * power to the PHY. + */ + rzg3s_sysc_config_func(sysc, RZG3S_SYSC_FUNC_ID_RST_RSM_B, 0); +port_refclk_put: + clk_put(host->port.refclk); + + return ret; +} + +static int rzg3s_pcie_suspend_noirq(struct device *dev) +{ + struct rzg3s_pcie_host *host = dev_get_drvdata(dev); + const struct rzg3s_pcie_soc_data *data = host->data; + struct rzg3s_pcie_port *port = &host->port; + struct rzg3s_sysc *sysc = host->sysc; + int ret; + + ret = pm_runtime_put_sync(dev); + if (ret) + return ret; + + clk_disable_unprepare(port->refclk); + + /* SoC-specific de-initialization */ + ret = data->config_deinit(host); + if (ret) + goto refclk_restore; + + ret = reset_control_bulk_assert(data->num_power_resets, + host->power_resets); + if (ret) + goto config_reinit; + + ret = rzg3s_sysc_config_func(sysc, RZG3S_SYSC_FUNC_ID_RST_RSM_B, 0); + if (ret) + goto power_resets_restore; + + return 0; + + /* Restore the previous state if any error happens */ +power_resets_restore: + reset_control_bulk_deassert(data->num_power_resets, + host->power_resets); +config_reinit: + if (data->config_pre_init) + data->config_pre_init(host); + data->config_post_init(host); +refclk_restore: + clk_prepare_enable(port->refclk); + pm_runtime_resume_and_get(dev); + return ret; +} + +static int rzg3s_pcie_resume_noirq(struct device *dev) +{ + struct rzg3s_pcie_host *host = dev_get_drvdata(dev); + const struct rzg3s_pcie_soc_data *data = host->data; + struct rzg3s_sysc *sysc = host->sysc; + int ret; + + ret = rzg3s_sysc_config_func(sysc, RZG3S_SYSC_FUNC_ID_MODE, 1); + if (ret) + return ret; + + ret = rzg3s_sysc_config_func(sysc, RZG3S_SYSC_FUNC_ID_RST_RSM_B, 1); + if (ret) + return ret; + + if (host->num_lanes) { + ret = rzg3s_sysc_config_func(host->sysc, + RZG3S_SYSC_FUNC_ID_LINK_MASTER, + host->num_lanes == 2 ? + RZG3S_SYSC_LINK_MODE_DUAL_X2 : + RZG3S_SYSC_LINK_MODE_SINGLE_X4); + if (ret) + goto assert_rst_rsm_b; + } + + ret = rzg3s_pcie_power_resets_deassert(host); + if (ret) + goto assert_rst_rsm_b; + + ret = pm_runtime_resume_and_get(dev); + if (ret) + goto assert_power_resets; + + ret = rzg3s_pcie_host_setup(host, rzg3s_pcie_msi_hw_setup, + rzg3s_pcie_msi_hw_teardown); + if (ret) + goto rpm_put; + + return 0; + + /* + * If any error happens there is no way to recover the IP. Put it in the + * lowest possible power state. + */ +rpm_put: + pm_runtime_put_sync(dev); +assert_power_resets: + reset_control_bulk_assert(data->num_power_resets, + host->power_resets); +assert_rst_rsm_b: + rzg3s_sysc_config_func(sysc, RZG3S_SYSC_FUNC_ID_RST_RSM_B, 0); + return ret; +} + +static const struct dev_pm_ops rzg3s_pcie_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(rzg3s_pcie_suspend_noirq, + rzg3s_pcie_resume_noirq) +}; + +static const char * const rzg3s_soc_power_resets[] = { + "aresetn", "rst_cfg_b", "rst_load_b", +}; + +static const char * const rzg3s_soc_cfg_resets[] = { + "rst_b", "rst_ps_b", "rst_gp_b", "rst_rsm_b", +}; + +static const struct rzg3s_pcie_soc_data rzg3s_soc_data = { + .power_resets = rzg3s_soc_power_resets, + .num_power_resets = ARRAY_SIZE(rzg3s_soc_power_resets), + .cfg_resets = rzg3s_soc_cfg_resets, + .num_cfg_resets = ARRAY_SIZE(rzg3s_soc_cfg_resets), + .num_pcie_controllers = 1, + .config_post_init = rzg3s_pcie_config_post_init, + .config_deinit = rzg3s_pcie_config_deinit, + .init_phy = rzg3s_soc_pcie_init_phy, + .sysc_info = { + [RZG3S_PCIE_CONTROLLER_ID_0] = { + .functions = { + [RZG3S_SYSC_FUNC_ID_RST_RSM_B] = { + .offset = 0xd74, + .mask = BIT(0), + }, + }, + }, + }, +}; + +static const char * const rzg3e_soc_power_resets[] = { "aresetn" }; + +static const struct rzg3s_pcie_soc_data rzg3e_soc_data = { + .power_resets = rzg3e_soc_power_resets, + .num_power_resets = ARRAY_SIZE(rzg3e_soc_power_resets), + .num_pcie_controllers = 1, + .config_pre_init = rzg3e_pcie_config_pre_init, + .config_post_init = rzg3e_pcie_config_post_init, + .config_deinit = rzg3e_pcie_config_deinit, + .sysc_info = { + [RZG3S_PCIE_CONTROLLER_ID_0] = { + .functions = { + [RZG3S_SYSC_FUNC_ID_L1_ALLOW] = { + .offset = 0x1020, + .mask = BIT(0), + }, + [RZG3S_SYSC_FUNC_ID_MODE] = { + .offset = 0x1024, + .mask = BIT(0), + }, + }, + }, + }, +}; + +static const struct rzg3s_pcie_soc_data rzv2h_soc_data = { + .power_resets = rzg3e_soc_power_resets, + .num_power_resets = ARRAY_SIZE(rzg3e_soc_power_resets), + .num_pcie_controllers = 2, + .config_pre_init = rzg3e_pcie_config_pre_init, + .config_post_init = rzg3e_pcie_config_post_init, + .config_deinit = rzg3e_pcie_config_deinit, + .setup_lanes = rzv2h_pcie_setup_lanes, + .sysc_info = { + [RZG3S_PCIE_CONTROLLER_ID_0] = { + .functions = { + [RZG3S_SYSC_FUNC_ID_L1_ALLOW] = { + .offset = 0x1020, + .mask = BIT(0), + }, + [RZG3S_SYSC_FUNC_ID_MODE] = { + .offset = 0x1024, + .mask = BIT(0), + }, + [RZG3S_SYSC_FUNC_ID_LINK_MASTER] = { + .offset = 0x1060, + .mask = GENMASK(9, 8), + }, + }, + }, + [RZG3S_PCIE_CONTROLLER_ID_1] = { + .functions = { + [RZG3S_SYSC_FUNC_ID_L1_ALLOW] = { + .offset = 0x1050, + .mask = BIT(0), + }, + [RZG3S_SYSC_FUNC_ID_MODE] = { + .offset = 0x1054, + .mask = BIT(0), + }, + [RZG3S_SYSC_FUNC_ID_LINK_MASTER] = { + .offset = 0x1060, + .mask = GENMASK(9, 8), + }, + }, + }, + }, +}; + +static const struct of_device_id rzg3s_pcie_of_match[] = { + { + .compatible = "renesas,r9a08g045-pcie", + .data = &rzg3s_soc_data, + }, + { + .compatible = "renesas,r9a09g047-pcie", + .data = &rzg3e_soc_data, + }, + { + .compatible = "renesas,r9a09g057-pcie", + .data = &rzv2h_soc_data, + }, + {} +}; + +static struct platform_driver rzg3s_pcie_driver = { + .driver = { + .name = "rzg3s-pcie-host", + .of_match_table = rzg3s_pcie_of_match, + .pm = pm_ptr(&rzg3s_pcie_pm_ops), + .suppress_bind_attrs = true, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, + .probe = rzg3s_pcie_probe, +}; +builtin_platform_driver(rzg3s_pcie_driver); diff --git a/drivers/pci/controller/pcie-tegra264.c b/drivers/pci/controller/pcie-tegra264.c new file mode 100644 index 000000000..653136db4 --- /dev/null +++ b/drivers/pci/controller/pcie-tegra264.c @@ -0,0 +1,467 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PCIe host controller driver for Tegra264 SoC + * + * Copyright (c) 2022-2026, NVIDIA CORPORATION. All rights reserved. + */ + +#include <linux/delay.h> +#include <linux/init.h> +#include <linux/interconnect.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/of_device.h> +#include <linux/of.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/pci-ecam.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> + +#include <soc/tegra/bpmp.h> +#include <soc/tegra/bpmp-abi.h> +#include <soc/tegra/fuse.h> + +#include "../pci.h" + +/* XAL registers */ +#define XAL_RC_ECAM_BASE_HI 0x00 +#define XAL_RC_ECAM_BASE_LO 0x04 +#define XAL_RC_ECAM_BUSMASK 0x08 +#define XAL_RC_IO_BASE_HI 0x0c +#define XAL_RC_IO_BASE_LO 0x10 +#define XAL_RC_IO_LIMIT_HI 0x14 +#define XAL_RC_IO_LIMIT_LO 0x18 +#define XAL_RC_MEM_32BIT_BASE_HI 0x1c +#define XAL_RC_MEM_32BIT_BASE_LO 0x20 +#define XAL_RC_MEM_32BIT_LIMIT_HI 0x24 +#define XAL_RC_MEM_32BIT_LIMIT_LO 0x28 +#define XAL_RC_MEM_64BIT_BASE_HI 0x2c +#define XAL_RC_MEM_64BIT_BASE_LO 0x30 +#define XAL_RC_MEM_64BIT_LIMIT_HI 0x34 +#define XAL_RC_MEM_64BIT_LIMIT_LO 0x38 +#define XAL_RC_BAR_CNTL_STANDARD 0x40 +#define XAL_RC_BAR_CNTL_STANDARD_IOBAR_EN BIT(0) +#define XAL_RC_BAR_CNTL_STANDARD_32B_BAR_EN BIT(1) +#define XAL_RC_BAR_CNTL_STANDARD_64B_BAR_EN BIT(2) + +/* XTL registers */ +#define XTL_RC_PCIE_CFG_LINK_CAPS 0x56 +#define XTL_RC_PCIE_CFG_LINK_STATUS 0x5a + +#define XTL_RC_MGMT_PERST_CONTROL 0x218 +#define XTL_RC_MGMT_PERST_CONTROL_PERST_O_N BIT(0) + +#define XTL_RC_MGMT_CLOCK_CONTROL 0x47c +#define XTL_RC_MGMT_CLOCK_CONTROL_PEX_CLKREQ_I_N_PIN_USE_CONV_TO_PRSNT BIT(9) + +struct tegra264_pcie { + struct device *dev; + + /* I/O memory */ + void __iomem *xal; + void __iomem *xtl; + void __iomem *ecam; + + /* bridge configuration */ + struct pci_config_window *cfg; + struct pci_host_bridge *bridge; + + /* BPMP and bandwidth management */ + struct icc_path *icc_path; + struct tegra_bpmp *bpmp; + u32 ctl_id; + + bool supports_hotplug; + bool link_up; +}; + +static void tegra264_pcie_power_off(struct tegra264_pcie *pcie) +{ + struct tegra_bpmp_message msg = {}; + struct mrq_pcie_request req = {}; + int err; + + req.cmd = CMD_PCIE_RP_CONTROLLER_OFF; + req.rp_ctrlr_off.rp_controller = pcie->ctl_id; + + msg.mrq = MRQ_PCIE; + msg.tx.data = &req; + msg.tx.size = sizeof(req); + + err = tegra_bpmp_transfer(pcie->bpmp, &msg); + if (err) + dev_err(pcie->dev, "failed to turn off PCIe #%u: %pe\n", + pcie->ctl_id, ERR_PTR(err)); + + if (msg.rx.ret) + dev_err(pcie->dev, "failed to turn off PCIe #%u: %d\n", + pcie->ctl_id, msg.rx.ret); +} + +static void tegra264_pcie_icc_set(struct tegra264_pcie *pcie) +{ + u32 value, speed, width; + int err; + + /* + * If the link is up, read the negotiated speed and width fields from + * the link status register. Otherwise, read the corresponding values + * from the link capabilities register to ensure the link works after + * hotplug. + * + * Ideally we'll want to update this dynamically, either on hotplug + * or bandwidth change notifications. Neither of those are currently + * possible, so this is as good as it gets for now. + */ + if (pcie->link_up) { + value = readw(pcie->ecam + XTL_RC_PCIE_CFG_LINK_STATUS); + speed = FIELD_GET(PCI_EXP_LNKSTA_CLS, value); + width = FIELD_GET(PCI_EXP_LNKSTA_NLW, value); + } else { + value = readw(pcie->ecam + XTL_RC_PCIE_CFG_LINK_CAPS); + speed = FIELD_GET(PCI_EXP_LNKCAP_SLS, value); + width = FIELD_GET(PCI_EXP_LNKCAP_MLW, value); + } + + value = Mbps_to_icc(width * PCIE_SPEED2MBS_ENC(pcie_link_speed[speed])); + + /* + * We don't want to error out here because a boot-critical device + * could be connected to this root port. Failure to set the bandwidth + * request may have an adverse impact on performance, but it is not + * generally fatal, so we opt to continue regardless so that users + * get a chance to fix things. + */ + err = icc_set_bw(pcie->icc_path, value, value); + if (err < 0) + dev_err(pcie->dev, + "failed to request bandwidth (%u kBps): %pe\n", + value, ERR_PTR(err)); +} + +/* + * The various memory regions used by the controller (I/O, memory, ECAM) are + * set up during early boot and have hardware-level protections in place. If + * the DT ranges don't match what's been setup, the controller won't be able + * to write the address endpoints properly, so make sure to validate that DT + * and firmware programming agree on these ranges. + */ +static bool tegra264_pcie_valid_ranges(struct platform_device *pdev) +{ + struct tegra264_pcie *pcie = platform_get_drvdata(pdev); + struct device_node *np = pcie->dev->of_node; + struct of_pci_range_parser parser; + phys_addr_t phys, limit, hi, lo; + struct of_pci_range range; + struct resource *res; + bool status = true; + u32 value; + int err; + + err = of_pci_range_parser_init(&parser, np); + if (err < 0) + return false; + + for_each_of_pci_range(&parser, &range) { + unsigned int addr_hi, addr_lo, limit_hi, limit_lo, enable; + unsigned long type = range.flags & IORESOURCE_TYPE_BITS; + phys_addr_t start, end, mask; + const char *region = NULL; + + end = range.cpu_addr + range.size - 1; + start = range.cpu_addr; + + switch (type) { + case IORESOURCE_IO: + addr_hi = XAL_RC_IO_BASE_HI; + addr_lo = XAL_RC_IO_BASE_LO; + limit_hi = XAL_RC_IO_LIMIT_HI; + limit_lo = XAL_RC_IO_LIMIT_LO; + enable = XAL_RC_BAR_CNTL_STANDARD_IOBAR_EN; + mask = SZ_64K - 1; + region = "I/O"; + break; + + case IORESOURCE_MEM: + if (range.flags & IORESOURCE_PREFETCH) { + addr_hi = XAL_RC_MEM_64BIT_BASE_HI; + addr_lo = XAL_RC_MEM_64BIT_BASE_LO; + limit_hi = XAL_RC_MEM_64BIT_LIMIT_HI; + limit_lo = XAL_RC_MEM_64BIT_LIMIT_LO; + enable = XAL_RC_BAR_CNTL_STANDARD_64B_BAR_EN; + region = "prefetchable memory"; + } else { + addr_hi = XAL_RC_MEM_32BIT_BASE_HI; + addr_lo = XAL_RC_MEM_32BIT_BASE_LO; + limit_hi = XAL_RC_MEM_32BIT_LIMIT_HI; + limit_lo = XAL_RC_MEM_32BIT_LIMIT_LO; + enable = XAL_RC_BAR_CNTL_STANDARD_32B_BAR_EN; + region = "memory"; + } + + mask = SZ_1M - 1; + break; + } + + /* not interested in anything that's not I/O or memory */ + if (!region) + continue; + + /* don't check regions that haven't been enabled */ + value = readl(pcie->xal + XAL_RC_BAR_CNTL_STANDARD); + if ((value & enable) == 0) + continue; + + hi = readl(pcie->xal + addr_hi); + lo = readl(pcie->xal + addr_lo); + phys = ((hi << 16) << 16) | lo; + + hi = readl(pcie->xal + limit_hi); + lo = readl(pcie->xal + limit_lo); + limit = ((hi << 16) << 16) | lo | mask; + + if (phys != start || limit != end) { + dev_err(pcie->dev, + "%s region mismatch: %pap-%pap -> %pap-%pap\n", + region, &phys, &limit, &start, &end); + status = false; + } + } + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ecam"); + if (!res) + return false; + + hi = readl(pcie->xal + XAL_RC_ECAM_BASE_HI); + lo = readl(pcie->xal + XAL_RC_ECAM_BASE_LO); + phys = ((hi << 16) << 16) | lo; + + value = readl(pcie->xal + XAL_RC_ECAM_BUSMASK); + limit = phys + ((value + 1) << 20) - 1; + + if (phys != res->start || limit != res->end) { + dev_err(pcie->dev, + "ECAM region mismatch: %pap-%pap -> %pap-%pap\n", + &phys, &limit, &res->start, &res->end); + status = false; + } + + return status; +} + +static bool tegra264_pcie_supports_hotplug(struct tegra264_pcie *pcie) +{ + u32 value = readl(pcie->xtl + XTL_RC_MGMT_CLOCK_CONTROL); + + return (value & XTL_RC_MGMT_CLOCK_CONTROL_PEX_CLKREQ_I_N_PIN_USE_CONV_TO_PRSNT) != 0; +} + +static bool tegra264_pcie_link_up(struct tegra264_pcie *pcie, + enum pci_bus_speed *speed) +{ + u16 value = readw(pcie->ecam + XTL_RC_PCIE_CFG_LINK_STATUS); + + if (value & PCI_EXP_LNKSTA_DLLLA) { + if (speed) + *speed = pcie_link_speed[FIELD_GET(PCI_EXP_LNKSTA_CLS, + value)]; + + return true; + } + + return false; +} + +static void tegra264_pcie_init(struct tegra264_pcie *pcie) +{ + enum pci_bus_speed speed; + unsigned int i; + u32 value; + + /* bring the endpoint out of reset */ + value = readl(pcie->xtl + XTL_RC_MGMT_PERST_CONTROL); + value |= XTL_RC_MGMT_PERST_CONTROL_PERST_O_N; + writel(value, pcie->xtl + XTL_RC_MGMT_PERST_CONTROL); + + for (i = 0; i < PCIE_LINK_WAIT_MAX_RETRIES; i++) { + if (tegra264_pcie_link_up(pcie, NULL)) + break; + + msleep(PCIE_LINK_WAIT_SLEEP_MS); + } + + pcie->supports_hotplug = tegra264_pcie_supports_hotplug(pcie); + pcie->link_up = tegra264_pcie_link_up(pcie, &speed); + + if (pcie->link_up) { + msleep(PCIE_RESET_CONFIG_WAIT_MS); + dev_info(pcie->dev, "PCIe #%u link is up (speed: %s)\n", + pcie->ctl_id, pci_speed_string(speed)); + tegra264_pcie_icc_set(pcie); + } else { + dev_info(pcie->dev, "PCIe #%u link is down\n", pcie->ctl_id); + + /* + * Make sure to reset the bandwidth requirements if the link + * is down but hotplug-capable. + */ + if (pcie->supports_hotplug) + tegra264_pcie_icc_set(pcie); + } +} + +static int tegra264_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct pci_host_bridge *bridge; + struct tegra264_pcie *pcie; + struct resource_entry *bus; + struct resource *res; + int err; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(struct tegra264_pcie)); + if (!bridge) + return dev_err_probe(dev, -ENOMEM, + "failed to allocate host bridge\n"); + + pcie = pci_host_bridge_priv(bridge); + platform_set_drvdata(pdev, pcie); + pcie->bridge = bridge; + pcie->dev = dev; + + pcie->xal = devm_platform_ioremap_resource_byname(pdev, "xal"); + if (IS_ERR(pcie->xal)) + return dev_err_probe(dev, PTR_ERR(pcie->xal), + "failed to map XAL memory\n"); + + pcie->xtl = devm_platform_ioremap_resource_byname(pdev, "xtl-pri"); + if (IS_ERR(pcie->xtl)) + return dev_err_probe(dev, PTR_ERR(pcie->xtl), + "failed to map XTL-PRI memory\n"); + + bus = resource_list_first_type(&bridge->windows, IORESOURCE_BUS); + if (!bus) + return dev_err_probe(dev, -ENODEV, + "failed to get bus resources\n"); + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ecam"); + if (!res) + return dev_err_probe(dev, -ENXIO, + "failed to get ECAM resource\n"); + + pcie->icc_path = devm_of_icc_get(dev, "write"); + if (IS_ERR(pcie->icc_path)) + return dev_err_probe(dev, PTR_ERR(pcie->icc_path), + "failed to get ICC\n"); + + pcie->bpmp = tegra_bpmp_get_with_id(dev, &pcie->ctl_id); + if (IS_ERR(pcie->bpmp)) + return dev_err_probe(dev, PTR_ERR(pcie->bpmp), + "failed to get BPMP\n"); + + err = devm_pm_runtime_set_active_enabled(dev); + if (err < 0) { + dev_err_probe(dev, err, "failed to enable runtime PM\n"); + goto err_put_bpmp; + } + + err = pm_runtime_resume_and_get(dev); + if (err < 0) { + dev_err_probe(dev, err, "failed to power on device\n"); + goto err_put_bpmp; + } + + /* sanity check that programmed ranges match what's in DT */ + if (!tegra264_pcie_valid_ranges(pdev)) { + err = -EINVAL; + goto err_put_pm; + } + + pcie->cfg = pci_ecam_create(dev, res, bus->res, &pci_generic_ecam_ops); + if (IS_ERR(pcie->cfg)) { + err = dev_err_probe(dev, PTR_ERR(pcie->cfg), + "failed to create ECAM\n"); + goto err_put_pm; + } + + bridge->ops = (struct pci_ops *)&pci_generic_ecam_ops.pci_ops; + bridge->sysdata = pcie->cfg; + pcie->ecam = pcie->cfg->win; + + tegra264_pcie_init(pcie); + + /* + * Fail if the link isn't up and doesn't support hotplug, no device + * will ever be able to be added on this bus. + */ + if (!pcie->link_up && !pcie->supports_hotplug) { + err = dev_err_probe(pcie->dev, -ENODEV, + "PCIe #%u link is down and not hotplug-capable, turning off\n", + pcie->ctl_id); + tegra264_pcie_power_off(pcie); + goto err_free_ecam; + } + + err = pci_host_probe(bridge); + if (err < 0) { + dev_err_probe(dev, err, "failed to register Host Bridge\n"); + goto err_free_ecam; + } + + return 0; + +err_free_ecam: + pci_ecam_free(pcie->cfg); +err_put_pm: + pm_runtime_put_sync(dev); +err_put_bpmp: + tegra_bpmp_put(pcie->bpmp); + + return err; +} + +static void tegra264_pcie_remove(struct platform_device *pdev) +{ + struct tegra264_pcie *pcie = platform_get_drvdata(pdev); + + /* + * If we undo tegra264_pcie_init() then link goes down and need + * controller reset to bring up the link again. Remove intention is + * to clean up the root bridge and re-enumerate during bind. + */ + pci_lock_rescan_remove(); + pci_stop_root_bus(pcie->bridge->bus); + pci_remove_root_bus(pcie->bridge->bus); + pci_unlock_rescan_remove(); + + pm_runtime_put_sync(&pdev->dev); + tegra_bpmp_put(pcie->bpmp); + pci_ecam_free(pcie->cfg); +} + +static const struct of_device_id tegra264_pcie_of_match[] = { + { + .compatible = "nvidia,tegra264-pcie", + }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, tegra264_pcie_of_match); + +static struct platform_driver tegra264_pcie_driver = { + .probe = tegra264_pcie_probe, + .remove = tegra264_pcie_remove, + .driver = { + .name = "tegra264-pcie", + .of_match_table = tegra264_pcie_of_match, + }, +}; +module_platform_driver(tegra264_pcie_driver); + +MODULE_AUTHOR("Manikanta Maddireddy <mmaddireddy@nvidia.com>"); +MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>"); +MODULE_DESCRIPTION("NVIDIA Tegra264 PCIe host controller driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pci/controller/pcie-xilinx-common.h b/drivers/pci/controller/pcie-xilinx-common.h new file mode 100644 index 000000000..1832770f3 --- /dev/null +++ b/drivers/pci/controller/pcie-xilinx-common.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * (C) Copyright 2023, Xilinx, Inc. + */ + +#include <linux/pci.h> +#include <linux/pci-ecam.h> +#include <linux/platform_device.h> + +/* Interrupt registers definitions */ +#define XILINX_PCIE_INTR_LINK_DOWN 0 +#define XILINX_PCIE_INTR_HOT_RESET 3 +#define XILINX_PCIE_INTR_CFG_PCIE_TIMEOUT 4 +#define XILINX_PCIE_INTR_CFG_TIMEOUT 8 +#define XILINX_PCIE_INTR_CORRECTABLE 9 +#define XILINX_PCIE_INTR_NONFATAL 10 +#define XILINX_PCIE_INTR_FATAL 11 +#define XILINX_PCIE_INTR_CFG_ERR_POISON 12 +#define XILINX_PCIE_INTR_PME_TO_ACK_RCVD 15 +#define XILINX_PCIE_INTR_INTX 16 +#define XILINX_PCIE_INTR_PM_PME_RCVD 17 +#define XILINX_PCIE_INTR_MSI 17 +#define XILINX_PCIE_INTR_SLV_UNSUPP 20 +#define XILINX_PCIE_INTR_SLV_UNEXP 21 +#define XILINX_PCIE_INTR_SLV_COMPL 22 +#define XILINX_PCIE_INTR_SLV_ERRP 23 +#define XILINX_PCIE_INTR_SLV_CMPABT 24 +#define XILINX_PCIE_INTR_SLV_ILLBUR 25 +#define XILINX_PCIE_INTR_MST_DECERR 26 +#define XILINX_PCIE_INTR_MST_SLVERR 27 +#define XILINX_PCIE_INTR_SLV_PCIE_TIMEOUT 28 diff --git a/drivers/pci/controller/pcie-xilinx-cpm.c b/drivers/pci/controller/pcie-xilinx-cpm.c new file mode 100644 index 000000000..d38f27e20 --- /dev/null +++ b/drivers/pci/controller/pcie-xilinx-cpm.c @@ -0,0 +1,686 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * PCIe host controller driver for Xilinx Versal CPM DMA Bridge + * + * (C) Copyright 2019 - 2020, Xilinx, Inc. + */ + +#include <linux/bitfield.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/irqchip.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqdomain.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> + +#include "../pci.h" +#include "pcie-xilinx-common.h" + +/* Register definitions */ +#define XILINX_CPM_PCIE_REG_IDR 0x00000E10 +#define XILINX_CPM_PCIE_REG_IMR 0x00000E14 +#define XILINX_CPM_PCIE_REG_PSCR 0x00000E1C +#define XILINX_CPM_PCIE_REG_RPSC 0x00000E20 +#define XILINX_CPM_PCIE_REG_RPEFR 0x00000E2C +#define XILINX_CPM_PCIE_REG_IDRN 0x00000E38 +#define XILINX_CPM_PCIE_REG_IDRN_MASK 0x00000E3C +#define XILINX_CPM_PCIE_MISC_IR_STATUS 0x00000340 +#define XILINX_CPM_PCIE_MISC_IR_ENABLE 0x00000348 +#define XILINX_CPM_PCIE0_MISC_IR_LOCAL BIT(1) +#define XILINX_CPM_PCIE1_MISC_IR_LOCAL BIT(2) + +#define XILINX_CPM_PCIE0_IR_STATUS 0x000002A0 +#define XILINX_CPM_PCIE1_IR_STATUS 0x000002B4 +#define XILINX_CPM_PCIE0_IR_ENABLE 0x000002A8 +#define XILINX_CPM_PCIE1_IR_ENABLE 0x000002BC +#define XILINX_CPM_PCIE_IR_LOCAL BIT(0) + +#define IMR(x) BIT(XILINX_PCIE_INTR_ ##x) + +#define XILINX_CPM_PCIE_IMR_ALL_MASK \ + ( \ + IMR(LINK_DOWN) | \ + IMR(HOT_RESET) | \ + IMR(CFG_PCIE_TIMEOUT) | \ + IMR(CFG_TIMEOUT) | \ + IMR(CORRECTABLE) | \ + IMR(NONFATAL) | \ + IMR(FATAL) | \ + IMR(CFG_ERR_POISON) | \ + IMR(PME_TO_ACK_RCVD) | \ + IMR(INTX) | \ + IMR(PM_PME_RCVD) | \ + IMR(SLV_UNSUPP) | \ + IMR(SLV_UNEXP) | \ + IMR(SLV_COMPL) | \ + IMR(SLV_ERRP) | \ + IMR(SLV_CMPABT) | \ + IMR(SLV_ILLBUR) | \ + IMR(MST_DECERR) | \ + IMR(MST_SLVERR) | \ + IMR(SLV_PCIE_TIMEOUT) \ + ) + +#define XILINX_CPM_PCIE_IDR_ALL_MASK 0xFFFFFFFF +#define XILINX_CPM_PCIE_IDRN_MASK GENMASK(19, 16) +#define XILINX_CPM_PCIE_IDRN_SHIFT 16 + +/* Root Port Error FIFO Read Register definitions */ +#define XILINX_CPM_PCIE_RPEFR_ERR_VALID BIT(18) +#define XILINX_CPM_PCIE_RPEFR_REQ_ID GENMASK(15, 0) +#define XILINX_CPM_PCIE_RPEFR_ALL_MASK 0xFFFFFFFF + +/* Root Port Status/control Register definitions */ +#define XILINX_CPM_PCIE_REG_RPSC_BEN BIT(0) + +/* Phy Status/Control Register definitions */ +#define XILINX_CPM_PCIE_REG_PSCR_LNKUP BIT(11) + +enum xilinx_cpm_version { + CPM, + CPM5, + CPM5_HOST1, + CPM5NC_HOST, +}; + +/** + * struct xilinx_cpm_variant - CPM variant information + * @version: CPM version + * @ir_status: Offset for the error interrupt status register + * @ir_enable: Offset for the CPM5 local error interrupt enable register + * @ir_misc_value: A bitmask for the miscellaneous interrupt status + */ +struct xilinx_cpm_variant { + enum xilinx_cpm_version version; + u32 ir_status; + u32 ir_enable; + u32 ir_misc_value; +}; + +/** + * struct xilinx_cpm_pcie - PCIe port information + * @dev: Device pointer + * @reg_base: Bridge Register Base + * @cpm_base: CPM System Level Control and Status Register(SLCR) Base + * @intx_domain: Legacy IRQ domain pointer + * @cpm_domain: CPM IRQ domain pointer + * @cfg: Holds mappings of config space window + * @intx_irq: legacy interrupt number + * @irq: Error interrupt number + * @lock: lock protecting shared register access + * @variant: CPM version check pointer + */ +struct xilinx_cpm_pcie { + struct device *dev; + void __iomem *reg_base; + void __iomem *cpm_base; + struct irq_domain *intx_domain; + struct irq_domain *cpm_domain; + struct pci_config_window *cfg; + int intx_irq; + int irq; + raw_spinlock_t lock; + const struct xilinx_cpm_variant *variant; +}; + +static u32 pcie_read(struct xilinx_cpm_pcie *port, u32 reg) +{ + return readl_relaxed(port->reg_base + reg); +} + +static void pcie_write(struct xilinx_cpm_pcie *port, + u32 val, u32 reg) +{ + writel_relaxed(val, port->reg_base + reg); +} + +static bool cpm_pcie_link_up(struct xilinx_cpm_pcie *port) +{ + return (pcie_read(port, XILINX_CPM_PCIE_REG_PSCR) & + XILINX_CPM_PCIE_REG_PSCR_LNKUP); +} + +static void cpm_pcie_clear_err_interrupts(struct xilinx_cpm_pcie *port) +{ + unsigned long val = pcie_read(port, XILINX_CPM_PCIE_REG_RPEFR); + + if (val & XILINX_CPM_PCIE_RPEFR_ERR_VALID) { + dev_dbg(port->dev, "Requester ID %lu\n", + val & XILINX_CPM_PCIE_RPEFR_REQ_ID); + pcie_write(port, XILINX_CPM_PCIE_RPEFR_ALL_MASK, + XILINX_CPM_PCIE_REG_RPEFR); + } +} + +static void xilinx_cpm_mask_leg_irq(struct irq_data *data) +{ + struct xilinx_cpm_pcie *port = irq_data_get_irq_chip_data(data); + unsigned long flags; + u32 mask; + u32 val; + + mask = BIT(data->hwirq + XILINX_CPM_PCIE_IDRN_SHIFT); + raw_spin_lock_irqsave(&port->lock, flags); + val = pcie_read(port, XILINX_CPM_PCIE_REG_IDRN_MASK); + pcie_write(port, (val & (~mask)), XILINX_CPM_PCIE_REG_IDRN_MASK); + raw_spin_unlock_irqrestore(&port->lock, flags); +} + +static void xilinx_cpm_unmask_leg_irq(struct irq_data *data) +{ + struct xilinx_cpm_pcie *port = irq_data_get_irq_chip_data(data); + unsigned long flags; + u32 mask; + u32 val; + + mask = BIT(data->hwirq + XILINX_CPM_PCIE_IDRN_SHIFT); + raw_spin_lock_irqsave(&port->lock, flags); + val = pcie_read(port, XILINX_CPM_PCIE_REG_IDRN_MASK); + pcie_write(port, (val | mask), XILINX_CPM_PCIE_REG_IDRN_MASK); + raw_spin_unlock_irqrestore(&port->lock, flags); +} + +static struct irq_chip xilinx_cpm_leg_irq_chip = { + .name = "INTx", + .irq_mask = xilinx_cpm_mask_leg_irq, + .irq_unmask = xilinx_cpm_unmask_leg_irq, +}; + +/** + * xilinx_cpm_pcie_intx_map - Set the handler for the INTx and mark IRQ as valid + * @domain: IRQ domain + * @irq: Virtual IRQ number + * @hwirq: HW interrupt number + * + * Return: Always returns 0. + */ +static int xilinx_cpm_pcie_intx_map(struct irq_domain *domain, + unsigned int irq, irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &xilinx_cpm_leg_irq_chip, + handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + irq_set_status_flags(irq, IRQ_LEVEL); + + return 0; +} + +/* INTx IRQ Domain operations */ +static const struct irq_domain_ops intx_domain_ops = { + .map = xilinx_cpm_pcie_intx_map, +}; + +static void xilinx_cpm_pcie_intx_flow(struct irq_desc *desc) +{ + struct xilinx_cpm_pcie *port = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + unsigned long val; + int i; + + chained_irq_enter(chip, desc); + + val = FIELD_GET(XILINX_CPM_PCIE_IDRN_MASK, + pcie_read(port, XILINX_CPM_PCIE_REG_IDRN)); + + for_each_set_bit(i, &val, PCI_NUM_INTX) + generic_handle_domain_irq(port->intx_domain, i); + + chained_irq_exit(chip, desc); +} + +static void xilinx_cpm_mask_event_irq(struct irq_data *d) +{ + struct xilinx_cpm_pcie *port = irq_data_get_irq_chip_data(d); + u32 val; + + raw_spin_lock(&port->lock); + val = pcie_read(port, XILINX_CPM_PCIE_REG_IMR); + val &= ~BIT(d->hwirq); + pcie_write(port, val, XILINX_CPM_PCIE_REG_IMR); + raw_spin_unlock(&port->lock); +} + +static void xilinx_cpm_unmask_event_irq(struct irq_data *d) +{ + struct xilinx_cpm_pcie *port = irq_data_get_irq_chip_data(d); + u32 val; + + raw_spin_lock(&port->lock); + val = pcie_read(port, XILINX_CPM_PCIE_REG_IMR); + val |= BIT(d->hwirq); + pcie_write(port, val, XILINX_CPM_PCIE_REG_IMR); + raw_spin_unlock(&port->lock); +} + +static struct irq_chip xilinx_cpm_event_irq_chip = { + .name = "RC-Event", + .irq_mask = xilinx_cpm_mask_event_irq, + .irq_unmask = xilinx_cpm_unmask_event_irq, +}; + +static int xilinx_cpm_pcie_event_map(struct irq_domain *domain, + unsigned int irq, irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &xilinx_cpm_event_irq_chip, + handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + irq_set_status_flags(irq, IRQ_LEVEL); + return 0; +} + +static const struct irq_domain_ops event_domain_ops = { + .map = xilinx_cpm_pcie_event_map, +}; + +static void xilinx_cpm_pcie_event_flow(struct irq_desc *desc) +{ + struct xilinx_cpm_pcie *port = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + const struct xilinx_cpm_variant *variant = port->variant; + unsigned long val; + int i; + + chained_irq_enter(chip, desc); + val = pcie_read(port, XILINX_CPM_PCIE_REG_IDR); + val &= pcie_read(port, XILINX_CPM_PCIE_REG_IMR); + for_each_set_bit(i, &val, 32) + generic_handle_domain_irq(port->cpm_domain, i); + pcie_write(port, val, XILINX_CPM_PCIE_REG_IDR); + + if (variant->ir_status) { + val = readl_relaxed(port->cpm_base + variant->ir_status); + if (val) + writel_relaxed(val, port->cpm_base + + variant->ir_status); + } + + /* + * XILINX_CPM_PCIE_MISC_IR_STATUS register is mapped to + * CPM SLCR block. + */ + val = readl_relaxed(port->cpm_base + XILINX_CPM_PCIE_MISC_IR_STATUS); + if (val) + writel_relaxed(val, + port->cpm_base + XILINX_CPM_PCIE_MISC_IR_STATUS); + + chained_irq_exit(chip, desc); +} + +#define _IC(x, s) \ + [XILINX_PCIE_INTR_ ## x] = { __stringify(x), s } + +static const struct { + const char *sym; + const char *str; +} intr_cause[32] = { + _IC(LINK_DOWN, "Link Down"), + _IC(HOT_RESET, "Hot reset"), + _IC(CFG_TIMEOUT, "ECAM access timeout"), + _IC(CORRECTABLE, "Correctable error message"), + _IC(NONFATAL, "Non fatal error message"), + _IC(FATAL, "Fatal error message"), + _IC(SLV_UNSUPP, "Slave unsupported request"), + _IC(SLV_UNEXP, "Slave unexpected completion"), + _IC(SLV_COMPL, "Slave completion timeout"), + _IC(SLV_ERRP, "Slave Error Poison"), + _IC(SLV_CMPABT, "Slave Completer Abort"), + _IC(SLV_ILLBUR, "Slave Illegal Burst"), + _IC(MST_DECERR, "Master decode error"), + _IC(MST_SLVERR, "Master slave error"), + _IC(CFG_PCIE_TIMEOUT, "PCIe ECAM access timeout"), + _IC(CFG_ERR_POISON, "ECAM poisoned completion received"), + _IC(PME_TO_ACK_RCVD, "PME_TO_ACK message received"), + _IC(PM_PME_RCVD, "PM_PME message received"), + _IC(SLV_PCIE_TIMEOUT, "PCIe completion timeout received"), +}; + +static irqreturn_t xilinx_cpm_pcie_intr_handler(int irq, void *dev_id) +{ + struct xilinx_cpm_pcie *port = dev_id; + struct device *dev = port->dev; + struct irq_data *d; + + d = irq_domain_get_irq_data(port->cpm_domain, irq); + + switch (d->hwirq) { + case XILINX_PCIE_INTR_CORRECTABLE: + case XILINX_PCIE_INTR_NONFATAL: + case XILINX_PCIE_INTR_FATAL: + cpm_pcie_clear_err_interrupts(port); + fallthrough; + + default: + if (intr_cause[d->hwirq].str) + dev_warn(dev, "%s\n", intr_cause[d->hwirq].str); + else + dev_warn(dev, "Unknown IRQ %ld\n", d->hwirq); + } + + return IRQ_HANDLED; +} + +static void xilinx_cpm_free_irq_domains(struct xilinx_cpm_pcie *port) +{ + if (port->intx_domain) { + irq_domain_remove(port->intx_domain); + port->intx_domain = NULL; + } + + if (port->cpm_domain) { + irq_domain_remove(port->cpm_domain); + port->cpm_domain = NULL; + } +} + +/** + * xilinx_cpm_pcie_init_irq_domain - Initialize IRQ domain + * @port: PCIe port information + * + * Return: '0' on success and error value on failure + */ +static int xilinx_cpm_pcie_init_irq_domain(struct xilinx_cpm_pcie *port) +{ + struct device *dev = port->dev; + struct device_node *node = dev->of_node; + struct device_node *pcie_intc_node; + + /* Setup INTx */ + pcie_intc_node = of_get_next_child(node, NULL); + if (!pcie_intc_node) { + dev_err(dev, "No PCIe Intc node found\n"); + return -EINVAL; + } + + port->cpm_domain = irq_domain_create_linear(of_fwnode_handle(pcie_intc_node), 32, + &event_domain_ops, port); + if (!port->cpm_domain) + goto out; + + irq_domain_update_bus_token(port->cpm_domain, DOMAIN_BUS_NEXUS); + + port->intx_domain = irq_domain_create_linear(of_fwnode_handle(pcie_intc_node), PCI_NUM_INTX, + &intx_domain_ops, port); + if (!port->intx_domain) + goto out; + + irq_domain_update_bus_token(port->intx_domain, DOMAIN_BUS_WIRED); + + of_node_put(pcie_intc_node); + raw_spin_lock_init(&port->lock); + + return 0; +out: + xilinx_cpm_free_irq_domains(port); + of_node_put(pcie_intc_node); + dev_err(dev, "Failed to allocate IRQ domains\n"); + + return -ENOMEM; +} + +static int xilinx_cpm_setup_irq(struct xilinx_cpm_pcie *port) +{ + struct device *dev = port->dev; + struct platform_device *pdev = to_platform_device(dev); + int i, irq; + + port->irq = platform_get_irq(pdev, 0); + if (port->irq < 0) + return port->irq; + + for (i = 0; i < ARRAY_SIZE(intr_cause); i++) { + int err; + + if (!intr_cause[i].str) + continue; + + irq = irq_create_mapping(port->cpm_domain, i); + if (!irq) { + dev_err(dev, "Failed to map interrupt\n"); + return -ENXIO; + } + + err = devm_request_irq(dev, irq, xilinx_cpm_pcie_intr_handler, + 0, intr_cause[i].sym, port); + if (err) { + dev_err(dev, "Failed to request IRQ %d\n", irq); + return err; + } + } + + port->intx_irq = irq_create_mapping(port->cpm_domain, + XILINX_PCIE_INTR_INTX); + if (!port->intx_irq) { + dev_err(dev, "Failed to map INTx interrupt\n"); + return -ENXIO; + } + + /* Plug the INTx chained handler */ + irq_set_chained_handler_and_data(port->intx_irq, + xilinx_cpm_pcie_intx_flow, port); + + /* Plug the main event chained handler */ + irq_set_chained_handler_and_data(port->irq, + xilinx_cpm_pcie_event_flow, port); + + return 0; +} + +/** + * xilinx_cpm_pcie_init_port - Initialize hardware + * @port: PCIe port information + */ +static void xilinx_cpm_pcie_init_port(struct xilinx_cpm_pcie *port) +{ + const struct xilinx_cpm_variant *variant = port->variant; + + if (variant->version == CPM5NC_HOST) + return; + + if (cpm_pcie_link_up(port)) + dev_info(port->dev, "PCIe Link is UP\n"); + else + dev_info(port->dev, "PCIe Link is DOWN\n"); + + /* Disable all interrupts */ + pcie_write(port, ~XILINX_CPM_PCIE_IDR_ALL_MASK, + XILINX_CPM_PCIE_REG_IMR); + + /* Clear pending interrupts */ + pcie_write(port, pcie_read(port, XILINX_CPM_PCIE_REG_IDR) & + XILINX_CPM_PCIE_IMR_ALL_MASK, + XILINX_CPM_PCIE_REG_IDR); + + /* + * XILINX_CPM_PCIE_MISC_IR_ENABLE register is mapped to + * CPM SLCR block. + */ + writel(variant->ir_misc_value, + port->cpm_base + XILINX_CPM_PCIE_MISC_IR_ENABLE); + + if (variant->ir_enable) { + writel(XILINX_CPM_PCIE_IR_LOCAL, + port->cpm_base + variant->ir_enable); + } + + /* Set Bridge enable bit */ + pcie_write(port, pcie_read(port, XILINX_CPM_PCIE_REG_RPSC) | + XILINX_CPM_PCIE_REG_RPSC_BEN, + XILINX_CPM_PCIE_REG_RPSC); +} + +/** + * xilinx_cpm_pcie_parse_dt - Parse Device tree + * @port: PCIe port information + * @bus_range: Bus resource + * + * Return: '0' on success and error value on failure + */ +static int xilinx_cpm_pcie_parse_dt(struct xilinx_cpm_pcie *port, + struct resource *bus_range) +{ + struct device *dev = port->dev; + struct platform_device *pdev = to_platform_device(dev); + struct resource *res; + + port->cpm_base = devm_platform_ioremap_resource_byname(pdev, + "cpm_slcr"); + if (IS_ERR(port->cpm_base)) + return PTR_ERR(port->cpm_base); + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg"); + if (!res) + return -ENXIO; + + port->cfg = pci_ecam_create(dev, res, bus_range, + &pci_generic_ecam_ops); + if (IS_ERR(port->cfg)) + return PTR_ERR(port->cfg); + + if (port->variant->version == CPM5 || + port->variant->version == CPM5_HOST1) { + port->reg_base = devm_platform_ioremap_resource_byname(pdev, + "cpm_csr"); + if (IS_ERR(port->reg_base)) + return PTR_ERR(port->reg_base); + } else { + port->reg_base = port->cfg->win; + } + + return 0; +} + +static void xilinx_cpm_free_interrupts(struct xilinx_cpm_pcie *port) +{ + irq_set_chained_handler_and_data(port->intx_irq, NULL, NULL); + irq_set_chained_handler_and_data(port->irq, NULL, NULL); +} + +/** + * xilinx_cpm_pcie_probe - Probe function + * @pdev: Platform device pointer + * + * Return: '0' on success and error value on failure + */ +static int xilinx_cpm_pcie_probe(struct platform_device *pdev) +{ + struct xilinx_cpm_pcie *port; + struct device *dev = &pdev->dev; + struct pci_host_bridge *bridge; + struct resource_entry *bus; + int err; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*port)); + if (!bridge) + return -ENODEV; + + port = pci_host_bridge_priv(bridge); + + port->dev = dev; + + port->variant = of_device_get_match_data(dev); + + if (port->variant->version != CPM5NC_HOST) { + err = xilinx_cpm_pcie_init_irq_domain(port); + if (err) + return err; + } + + bus = resource_list_first_type(&bridge->windows, IORESOURCE_BUS); + if (!bus) { + err = -ENODEV; + goto err_free_irq_domains; + } + + err = xilinx_cpm_pcie_parse_dt(port, bus->res); + if (err) { + dev_err(dev, "Parsing DT failed\n"); + goto err_free_irq_domains; + } + + xilinx_cpm_pcie_init_port(port); + + if (port->variant->version != CPM5NC_HOST) { + err = xilinx_cpm_setup_irq(port); + if (err) { + dev_err(dev, "Failed to set up interrupts\n"); + goto err_setup_irq; + } + } + + bridge->sysdata = port->cfg; + bridge->ops = (struct pci_ops *)&pci_generic_ecam_ops.pci_ops; + + err = pci_host_probe(bridge); + if (err < 0) + goto err_host_bridge; + + return 0; + +err_host_bridge: + if (port->variant->version != CPM5NC_HOST) + xilinx_cpm_free_interrupts(port); +err_setup_irq: + pci_ecam_free(port->cfg); +err_free_irq_domains: + if (port->variant->version != CPM5NC_HOST) + xilinx_cpm_free_irq_domains(port); + return err; +} + +static const struct xilinx_cpm_variant cpm_host = { + .version = CPM, + .ir_misc_value = XILINX_CPM_PCIE0_MISC_IR_LOCAL, +}; + +static const struct xilinx_cpm_variant cpm5_host = { + .version = CPM5, + .ir_misc_value = XILINX_CPM_PCIE0_MISC_IR_LOCAL, + .ir_status = XILINX_CPM_PCIE0_IR_STATUS, + .ir_enable = XILINX_CPM_PCIE0_IR_ENABLE, +}; + +static const struct xilinx_cpm_variant cpm5_host1 = { + .version = CPM5_HOST1, + .ir_misc_value = XILINX_CPM_PCIE1_MISC_IR_LOCAL, + .ir_status = XILINX_CPM_PCIE1_IR_STATUS, + .ir_enable = XILINX_CPM_PCIE1_IR_ENABLE, +}; + +static const struct xilinx_cpm_variant cpm5n_host = { + .version = CPM5NC_HOST, +}; + +static const struct of_device_id xilinx_cpm_pcie_of_match[] = { + { + .compatible = "xlnx,versal-cpm-host-1.00", + .data = &cpm_host, + }, + { + .compatible = "xlnx,versal-cpm5-host", + .data = &cpm5_host, + }, + { + .compatible = "xlnx,versal-cpm5-host1", + .data = &cpm5_host1, + }, + { + .compatible = "xlnx,versal-cpm5nc-host", + .data = &cpm5n_host, + }, + {} +}; + +static struct platform_driver xilinx_cpm_pcie_driver = { + .driver = { + .name = "xilinx-cpm-pcie", + .of_match_table = xilinx_cpm_pcie_of_match, + .suppress_bind_attrs = true, + }, + .probe = xilinx_cpm_pcie_probe, +}; + +builtin_platform_driver(xilinx_cpm_pcie_driver); diff --git a/drivers/pci/controller/pcie-xilinx-dma-pl.c b/drivers/pci/controller/pcie-xilinx-dma-pl.c new file mode 100644 index 000000000..b037c8f31 --- /dev/null +++ b/drivers/pci/controller/pcie-xilinx-dma-pl.c @@ -0,0 +1,847 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * PCIe host controller driver for Xilinx XDMA PCIe Bridge + * + * Copyright (C) 2023 Xilinx, Inc. All rights reserved. + */ +#include <linux/bitfield.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/irqdomain.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> + +#include "../pci.h" +#include "pcie-xilinx-common.h" + +/* Register definitions */ +#define XILINX_PCIE_DMA_REG_IDR 0x00000138 +#define XILINX_PCIE_DMA_REG_IMR 0x0000013c +#define XILINX_PCIE_DMA_REG_PSCR 0x00000144 +#define XILINX_PCIE_DMA_REG_RPSC 0x00000148 +#define XILINX_PCIE_DMA_REG_MSIBASE1 0x0000014c +#define XILINX_PCIE_DMA_REG_MSIBASE2 0x00000150 +#define XILINX_PCIE_DMA_REG_RPEFR 0x00000154 +#define XILINX_PCIE_DMA_REG_IDRN 0x00000160 +#define XILINX_PCIE_DMA_REG_IDRN_MASK 0x00000164 +#define XILINX_PCIE_DMA_REG_MSI_LOW 0x00000170 +#define XILINX_PCIE_DMA_REG_MSI_HI 0x00000174 +#define XILINX_PCIE_DMA_REG_MSI_LOW_MASK 0x00000178 +#define XILINX_PCIE_DMA_REG_MSI_HI_MASK 0x0000017c + +#define IMR(x) BIT(XILINX_PCIE_INTR_ ##x) + +#define XILINX_PCIE_INTR_IMR_ALL_MASK \ + ( \ + IMR(LINK_DOWN) | \ + IMR(HOT_RESET) | \ + IMR(CFG_TIMEOUT) | \ + IMR(CORRECTABLE) | \ + IMR(NONFATAL) | \ + IMR(FATAL) | \ + IMR(INTX) | \ + IMR(MSI) | \ + IMR(SLV_UNSUPP) | \ + IMR(SLV_UNEXP) | \ + IMR(SLV_COMPL) | \ + IMR(SLV_ERRP) | \ + IMR(SLV_CMPABT) | \ + IMR(SLV_ILLBUR) | \ + IMR(MST_DECERR) | \ + IMR(MST_SLVERR) | \ + ) + +#define XILINX_PCIE_DMA_IMR_ALL_MASK 0x0ff30fe9 +#define XILINX_PCIE_DMA_IDR_ALL_MASK 0xffffffff +#define XILINX_PCIE_DMA_IDRN_MASK GENMASK(19, 16) + +/* Root Port Error Register definitions */ +#define XILINX_PCIE_DMA_RPEFR_ERR_VALID BIT(18) +#define XILINX_PCIE_DMA_RPEFR_REQ_ID GENMASK(15, 0) +#define XILINX_PCIE_DMA_RPEFR_ALL_MASK 0xffffffff + +/* Root Port Interrupt Register definitions */ +#define XILINX_PCIE_DMA_IDRN_SHIFT 16 + +/* Root Port Status/control Register definitions */ +#define XILINX_PCIE_DMA_REG_RPSC_BEN BIT(0) + +/* Phy Status/Control Register definitions */ +#define XILINX_PCIE_DMA_REG_PSCR_LNKUP BIT(11) +#define QDMA_BRIDGE_BASE_OFF 0xcd8 + +/* Number of MSI IRQs */ +#define XILINX_NUM_MSI_IRQS 64 + +enum xilinx_pl_dma_version { + XDMA, + QDMA, +}; + +/** + * struct xilinx_pl_dma_variant - PL DMA PCIe variant information + * @version: DMA version + */ +struct xilinx_pl_dma_variant { + enum xilinx_pl_dma_version version; +}; + +struct xilinx_msi { + unsigned long *bitmap; + struct irq_domain *dev_domain; + struct mutex lock; /* Protect bitmap variable */ + int irq_msi0; + int irq_msi1; +}; + +/** + * struct pl_dma_pcie - PCIe port information + * @dev: Device pointer + * @reg_base: IO Mapped Register Base + * @cfg_base: IO Mapped Configuration Base + * @irq: Interrupt number + * @cfg: Holds mappings of config space window + * @phys_reg_base: Physical address of reg base + * @intx_domain: Legacy IRQ domain pointer + * @pldma_domain: PL DMA IRQ domain pointer + * @resources: Bus Resources + * @msi: MSI information + * @intx_irq: INTx error interrupt number + * @lock: Lock protecting shared register access + * @variant: PL DMA PCIe version check pointer + */ +struct pl_dma_pcie { + struct device *dev; + void __iomem *reg_base; + void __iomem *cfg_base; + int irq; + struct pci_config_window *cfg; + phys_addr_t phys_reg_base; + struct irq_domain *intx_domain; + struct irq_domain *pldma_domain; + struct list_head resources; + struct xilinx_msi msi; + int intx_irq; + raw_spinlock_t lock; + const struct xilinx_pl_dma_variant *variant; +}; + +static inline u32 pcie_read(struct pl_dma_pcie *port, u32 reg) +{ + if (port->variant->version == QDMA) + return readl(port->reg_base + reg + QDMA_BRIDGE_BASE_OFF); + + return readl(port->reg_base + reg); +} + +static inline void pcie_write(struct pl_dma_pcie *port, u32 val, u32 reg) +{ + if (port->variant->version == QDMA) + writel(val, port->reg_base + reg + QDMA_BRIDGE_BASE_OFF); + else + writel(val, port->reg_base + reg); +} + +static inline bool xilinx_pl_dma_pcie_link_up(struct pl_dma_pcie *port) +{ + return (pcie_read(port, XILINX_PCIE_DMA_REG_PSCR) & + XILINX_PCIE_DMA_REG_PSCR_LNKUP) ? true : false; +} + +static void xilinx_pl_dma_pcie_clear_err_interrupts(struct pl_dma_pcie *port) +{ + unsigned long val = pcie_read(port, XILINX_PCIE_DMA_REG_RPEFR); + + if (val & XILINX_PCIE_DMA_RPEFR_ERR_VALID) { + dev_dbg(port->dev, "Requester ID %lu\n", + val & XILINX_PCIE_DMA_RPEFR_REQ_ID); + pcie_write(port, XILINX_PCIE_DMA_RPEFR_ALL_MASK, + XILINX_PCIE_DMA_REG_RPEFR); + } +} + +static bool xilinx_pl_dma_pcie_valid_device(struct pci_bus *bus, + unsigned int devfn) +{ + struct pl_dma_pcie *port = bus->sysdata; + + if (!pci_is_root_bus(bus)) { + /* + * Checking whether the link is up is the last line of + * defense, and this check is inherently racy by definition. + * Sending a PIO request to a downstream device when the link is + * down causes an unrecoverable error, and a reset of the entire + * PCIe controller will be needed. We can reduce the likelihood + * of that unrecoverable error by checking whether the link is + * up, but we can't completely prevent it because the link may + * go down between the link-up check and the PIO request. + */ + if (!xilinx_pl_dma_pcie_link_up(port)) + return false; + } else if (devfn > 0) + /* Only one device down on each root port */ + return false; + + return true; +} + +static void __iomem *xilinx_pl_dma_pcie_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + struct pl_dma_pcie *port = bus->sysdata; + + if (!xilinx_pl_dma_pcie_valid_device(bus, devfn)) + return NULL; + + if (port->variant->version == QDMA) + return port->cfg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); + + return port->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); +} + +/* PCIe operations */ +static struct pci_ecam_ops xilinx_pl_dma_pcie_ops = { + .pci_ops = { + .map_bus = xilinx_pl_dma_pcie_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, + } +}; + +static void xilinx_pl_dma_pcie_enable_msi(struct pl_dma_pcie *port) +{ + phys_addr_t msi_addr = port->phys_reg_base; + + pcie_write(port, upper_32_bits(msi_addr), XILINX_PCIE_DMA_REG_MSIBASE1); + pcie_write(port, lower_32_bits(msi_addr), XILINX_PCIE_DMA_REG_MSIBASE2); +} + +static void xilinx_mask_intx_irq(struct irq_data *data) +{ + struct pl_dma_pcie *port = irq_data_get_irq_chip_data(data); + unsigned long flags; + u32 mask, val; + + mask = BIT(data->hwirq + XILINX_PCIE_DMA_IDRN_SHIFT); + raw_spin_lock_irqsave(&port->lock, flags); + val = pcie_read(port, XILINX_PCIE_DMA_REG_IDRN_MASK); + pcie_write(port, (val & (~mask)), XILINX_PCIE_DMA_REG_IDRN_MASK); + raw_spin_unlock_irqrestore(&port->lock, flags); +} + +static void xilinx_unmask_intx_irq(struct irq_data *data) +{ + struct pl_dma_pcie *port = irq_data_get_irq_chip_data(data); + unsigned long flags; + u32 mask, val; + + mask = BIT(data->hwirq + XILINX_PCIE_DMA_IDRN_SHIFT); + raw_spin_lock_irqsave(&port->lock, flags); + val = pcie_read(port, XILINX_PCIE_DMA_REG_IDRN_MASK); + pcie_write(port, (val | mask), XILINX_PCIE_DMA_REG_IDRN_MASK); + raw_spin_unlock_irqrestore(&port->lock, flags); +} + +static struct irq_chip xilinx_leg_irq_chip = { + .name = "pl_dma:INTx", + .irq_mask = xilinx_mask_intx_irq, + .irq_unmask = xilinx_unmask_intx_irq, +}; + +static int xilinx_pl_dma_pcie_intx_map(struct irq_domain *domain, + unsigned int irq, irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &xilinx_leg_irq_chip, handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + irq_set_status_flags(irq, IRQ_LEVEL); + + return 0; +} + +/* INTx IRQ Domain operations */ +static const struct irq_domain_ops intx_domain_ops = { + .map = xilinx_pl_dma_pcie_intx_map, +}; + +static irqreturn_t xilinx_pl_dma_pcie_msi_handler_high(int irq, void *args) +{ + struct xilinx_msi *msi; + unsigned long status; + u32 bit, virq; + struct pl_dma_pcie *port = args; + + msi = &port->msi; + + while ((status = pcie_read(port, XILINX_PCIE_DMA_REG_MSI_HI)) != 0) { + for_each_set_bit(bit, &status, 32) { + pcie_write(port, 1 << bit, XILINX_PCIE_DMA_REG_MSI_HI); + bit = bit + 32; + virq = irq_find_mapping(msi->dev_domain, bit); + if (virq) + generic_handle_irq(virq); + } + } + + return IRQ_HANDLED; +} + +static irqreturn_t xilinx_pl_dma_pcie_msi_handler_low(int irq, void *args) +{ + struct pl_dma_pcie *port = args; + struct xilinx_msi *msi; + unsigned long status; + u32 bit, virq; + + msi = &port->msi; + + while ((status = pcie_read(port, XILINX_PCIE_DMA_REG_MSI_LOW)) != 0) { + for_each_set_bit(bit, &status, 32) { + pcie_write(port, 1 << bit, XILINX_PCIE_DMA_REG_MSI_LOW); + virq = irq_find_mapping(msi->dev_domain, bit); + if (virq) + generic_handle_irq(virq); + } + } + + return IRQ_HANDLED; +} + +static irqreturn_t xilinx_pl_dma_pcie_event_flow(int irq, void *args) +{ + struct pl_dma_pcie *port = args; + unsigned long val; + int i; + + val = pcie_read(port, XILINX_PCIE_DMA_REG_IDR); + val &= pcie_read(port, XILINX_PCIE_DMA_REG_IMR); + for_each_set_bit(i, &val, 32) + generic_handle_domain_irq(port->pldma_domain, i); + + pcie_write(port, val, XILINX_PCIE_DMA_REG_IDR); + + return IRQ_HANDLED; +} + +#define _IC(x, s) \ + [XILINX_PCIE_INTR_ ## x] = { __stringify(x), s } + +static const struct { + const char *sym; + const char *str; +} intr_cause[32] = { + _IC(LINK_DOWN, "Link Down"), + _IC(HOT_RESET, "Hot reset"), + _IC(CFG_TIMEOUT, "ECAM access timeout"), + _IC(CORRECTABLE, "Correctable error message"), + _IC(NONFATAL, "Non fatal error message"), + _IC(FATAL, "Fatal error message"), + _IC(SLV_UNSUPP, "Slave unsupported request"), + _IC(SLV_UNEXP, "Slave unexpected completion"), + _IC(SLV_COMPL, "Slave completion timeout"), + _IC(SLV_ERRP, "Slave Error Poison"), + _IC(SLV_CMPABT, "Slave Completer Abort"), + _IC(SLV_ILLBUR, "Slave Illegal Burst"), + _IC(MST_DECERR, "Master decode error"), + _IC(MST_SLVERR, "Master slave error"), +}; + +static irqreturn_t xilinx_pl_dma_pcie_intr_handler(int irq, void *dev_id) +{ + struct pl_dma_pcie *port = (struct pl_dma_pcie *)dev_id; + struct device *dev = port->dev; + struct irq_data *d; + + d = irq_domain_get_irq_data(port->pldma_domain, irq); + switch (d->hwirq) { + case XILINX_PCIE_INTR_CORRECTABLE: + case XILINX_PCIE_INTR_NONFATAL: + case XILINX_PCIE_INTR_FATAL: + xilinx_pl_dma_pcie_clear_err_interrupts(port); + fallthrough; + + default: + if (intr_cause[d->hwirq].str) + dev_warn(dev, "%s\n", intr_cause[d->hwirq].str); + else + dev_warn(dev, "Unknown IRQ %ld\n", d->hwirq); + } + + return IRQ_HANDLED; +} + +#define XILINX_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_NO_AFFINITY) + +#define XILINX_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_MULTI_PCI_MSI) + +static const struct msi_parent_ops xilinx_msi_parent_ops = { + .required_flags = XILINX_MSI_FLAGS_REQUIRED, + .supported_flags = XILINX_MSI_FLAGS_SUPPORTED, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .prefix = "pl_dma-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; +static void xilinx_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct pl_dma_pcie *pcie = irq_data_get_irq_chip_data(data); + phys_addr_t msi_addr = pcie->phys_reg_base; + + msg->address_lo = lower_32_bits(msi_addr); + msg->address_hi = upper_32_bits(msi_addr); + msg->data = data->hwirq; +} + +static struct irq_chip xilinx_irq_chip = { + .name = "pl_dma:MSI", + .irq_compose_msi_msg = xilinx_compose_msi_msg, +}; + +static int xilinx_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct pl_dma_pcie *pcie = domain->host_data; + struct xilinx_msi *msi = &pcie->msi; + int bit, i; + + mutex_lock(&msi->lock); + bit = bitmap_find_free_region(msi->bitmap, XILINX_NUM_MSI_IRQS, + get_count_order(nr_irqs)); + if (bit < 0) { + mutex_unlock(&msi->lock); + return -ENOSPC; + } + + for (i = 0; i < nr_irqs; i++) { + irq_domain_set_info(domain, virq + i, bit + i, &xilinx_irq_chip, + domain->host_data, handle_simple_irq, + NULL, NULL); + } + mutex_unlock(&msi->lock); + + return 0; +} + +static void xilinx_irq_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *data = irq_domain_get_irq_data(domain, virq); + struct pl_dma_pcie *pcie = irq_data_get_irq_chip_data(data); + struct xilinx_msi *msi = &pcie->msi; + + mutex_lock(&msi->lock); + bitmap_release_region(msi->bitmap, data->hwirq, + get_count_order(nr_irqs)); + mutex_unlock(&msi->lock); +} + +static const struct irq_domain_ops dev_msi_domain_ops = { + .alloc = xilinx_irq_domain_alloc, + .free = xilinx_irq_domain_free, +}; + +static void xilinx_pl_dma_pcie_free_irq_domains(struct pl_dma_pcie *port) +{ + struct xilinx_msi *msi = &port->msi; + + if (port->intx_domain) { + irq_domain_remove(port->intx_domain); + port->intx_domain = NULL; + } + + if (msi->dev_domain) { + irq_domain_remove(msi->dev_domain); + msi->dev_domain = NULL; + } +} + +static int xilinx_pl_dma_pcie_init_msi_irq_domain(struct pl_dma_pcie *port) +{ + struct device *dev = port->dev; + struct xilinx_msi *msi = &port->msi; + int size = BITS_TO_LONGS(XILINX_NUM_MSI_IRQS) * sizeof(long); + struct irq_domain_info info = { + .fwnode = dev_fwnode(port->dev), + .ops = &dev_msi_domain_ops, + .host_data = port, + .size = XILINX_NUM_MSI_IRQS, + }; + + msi->dev_domain = msi_create_parent_irq_domain(&info, &xilinx_msi_parent_ops); + if (!msi->dev_domain) + goto out; + + mutex_init(&msi->lock); + msi->bitmap = kzalloc(size, GFP_KERNEL); + if (!msi->bitmap) + goto out; + + raw_spin_lock_init(&port->lock); + xilinx_pl_dma_pcie_enable_msi(port); + + return 0; + +out: + xilinx_pl_dma_pcie_free_irq_domains(port); + dev_err(dev, "Failed to allocate MSI IRQ domains\n"); + + return -ENOMEM; +} + +/* + * INTx error interrupts are Xilinx controller specific interrupt, used to + * notify user about errors such as cfg timeout, slave unsupported requests, + * fatal and non fatal error etc. + */ + +static irqreturn_t xilinx_pl_dma_pcie_intx_flow(int irq, void *args) +{ + unsigned long val; + int i; + struct pl_dma_pcie *port = args; + + val = FIELD_GET(XILINX_PCIE_DMA_IDRN_MASK, + pcie_read(port, XILINX_PCIE_DMA_REG_IDRN)); + + for_each_set_bit(i, &val, PCI_NUM_INTX) + generic_handle_domain_irq(port->intx_domain, i); + return IRQ_HANDLED; +} + +static void xilinx_pl_dma_pcie_mask_event_irq(struct irq_data *d) +{ + struct pl_dma_pcie *port = irq_data_get_irq_chip_data(d); + u32 val; + + raw_spin_lock(&port->lock); + val = pcie_read(port, XILINX_PCIE_DMA_REG_IMR); + val &= ~BIT(d->hwirq); + pcie_write(port, val, XILINX_PCIE_DMA_REG_IMR); + raw_spin_unlock(&port->lock); +} + +static void xilinx_pl_dma_pcie_unmask_event_irq(struct irq_data *d) +{ + struct pl_dma_pcie *port = irq_data_get_irq_chip_data(d); + u32 val; + + raw_spin_lock(&port->lock); + val = pcie_read(port, XILINX_PCIE_DMA_REG_IMR); + val |= BIT(d->hwirq); + pcie_write(port, val, XILINX_PCIE_DMA_REG_IMR); + raw_spin_unlock(&port->lock); +} + +static struct irq_chip xilinx_pl_dma_pcie_event_irq_chip = { + .name = "pl_dma:RC-Event", + .irq_mask = xilinx_pl_dma_pcie_mask_event_irq, + .irq_unmask = xilinx_pl_dma_pcie_unmask_event_irq, +}; + +static int xilinx_pl_dma_pcie_event_map(struct irq_domain *domain, + unsigned int irq, irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &xilinx_pl_dma_pcie_event_irq_chip, + handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + irq_set_status_flags(irq, IRQ_LEVEL); + + return 0; +} + +static const struct irq_domain_ops event_domain_ops = { + .map = xilinx_pl_dma_pcie_event_map, +}; + +/** + * xilinx_pl_dma_pcie_init_irq_domain - Initialize IRQ domain + * @port: PCIe port information + * + * Return: '0' on success and error value on failure. + */ +static int xilinx_pl_dma_pcie_init_irq_domain(struct pl_dma_pcie *port) +{ + struct device *dev = port->dev; + struct device_node *node = dev->of_node; + struct device_node *pcie_intc_node; + int ret; + + /* Setup INTx */ + pcie_intc_node = of_get_child_by_name(node, "interrupt-controller"); + if (!pcie_intc_node) { + dev_err(dev, "No PCIe Intc node found\n"); + return -EINVAL; + } + + port->pldma_domain = irq_domain_create_linear(of_fwnode_handle(pcie_intc_node), 32, + &event_domain_ops, port); + if (!port->pldma_domain) + return -ENOMEM; + + irq_domain_update_bus_token(port->pldma_domain, DOMAIN_BUS_NEXUS); + + port->intx_domain = irq_domain_create_linear(of_fwnode_handle(pcie_intc_node), PCI_NUM_INTX, + &intx_domain_ops, port); + if (!port->intx_domain) { + dev_err(dev, "Failed to get a INTx IRQ domain\n"); + return -ENOMEM; + } + + irq_domain_update_bus_token(port->intx_domain, DOMAIN_BUS_WIRED); + + ret = xilinx_pl_dma_pcie_init_msi_irq_domain(port); + if (ret != 0) { + irq_domain_remove(port->intx_domain); + return -ENOMEM; + } + + of_node_put(pcie_intc_node); + raw_spin_lock_init(&port->lock); + + return 0; +} + +static int xilinx_pl_dma_pcie_setup_irq(struct pl_dma_pcie *port) +{ + struct device *dev = port->dev; + struct platform_device *pdev = to_platform_device(dev); + int i, irq, err; + + port->irq = platform_get_irq(pdev, 0); + if (port->irq < 0) + return port->irq; + + for (i = 0; i < ARRAY_SIZE(intr_cause); i++) { + int err; + + if (!intr_cause[i].str) + continue; + + irq = irq_create_mapping(port->pldma_domain, i); + if (!irq) { + dev_err(dev, "Failed to map interrupt\n"); + return -ENXIO; + } + + err = devm_request_irq(dev, irq, + xilinx_pl_dma_pcie_intr_handler, + IRQF_SHARED | IRQF_NO_THREAD, + intr_cause[i].sym, port); + if (err) { + dev_err(dev, "Failed to request IRQ %d\n", irq); + return err; + } + } + + port->intx_irq = irq_create_mapping(port->pldma_domain, + XILINX_PCIE_INTR_INTX); + if (!port->intx_irq) { + dev_err(dev, "Failed to map INTx interrupt\n"); + return -ENXIO; + } + + err = devm_request_irq(dev, port->intx_irq, xilinx_pl_dma_pcie_intx_flow, + IRQF_SHARED | IRQF_NO_THREAD, NULL, port); + if (err) { + dev_err(dev, "Failed to request INTx IRQ %d\n", port->intx_irq); + return err; + } + + err = devm_request_irq(dev, port->irq, xilinx_pl_dma_pcie_event_flow, + IRQF_SHARED | IRQF_NO_THREAD, NULL, port); + if (err) { + dev_err(dev, "Failed to request event IRQ %d\n", port->irq); + return err; + } + + return 0; +} + +static void xilinx_pl_dma_pcie_init_port(struct pl_dma_pcie *port) +{ + if (xilinx_pl_dma_pcie_link_up(port)) + dev_info(port->dev, "PCIe Link is UP\n"); + else + dev_info(port->dev, "PCIe Link is DOWN\n"); + + /* Disable all interrupts */ + pcie_write(port, ~XILINX_PCIE_DMA_IDR_ALL_MASK, + XILINX_PCIE_DMA_REG_IMR); + + /* Clear pending interrupts */ + pcie_write(port, pcie_read(port, XILINX_PCIE_DMA_REG_IDR) & + XILINX_PCIE_DMA_IMR_ALL_MASK, + XILINX_PCIE_DMA_REG_IDR); + + /* Needed for MSI DECODE MODE */ + pcie_write(port, XILINX_PCIE_DMA_IDR_ALL_MASK, + XILINX_PCIE_DMA_REG_MSI_LOW_MASK); + pcie_write(port, XILINX_PCIE_DMA_IDR_ALL_MASK, + XILINX_PCIE_DMA_REG_MSI_HI_MASK); + + /* Set the Bridge enable bit */ + pcie_write(port, pcie_read(port, XILINX_PCIE_DMA_REG_RPSC) | + XILINX_PCIE_DMA_REG_RPSC_BEN, + XILINX_PCIE_DMA_REG_RPSC); +} + +static int xilinx_request_msi_irq(struct pl_dma_pcie *port) +{ + struct device *dev = port->dev; + struct platform_device *pdev = to_platform_device(dev); + int ret; + + port->msi.irq_msi0 = platform_get_irq_byname(pdev, "msi0"); + if (port->msi.irq_msi0 <= 0) + return port->msi.irq_msi0; + + ret = devm_request_irq(dev, port->msi.irq_msi0, xilinx_pl_dma_pcie_msi_handler_low, + IRQF_SHARED | IRQF_NO_THREAD, "xlnx-pcie-dma-pl", + port); + if (ret) { + dev_err(dev, "Failed to register interrupt\n"); + return ret; + } + + port->msi.irq_msi1 = platform_get_irq_byname(pdev, "msi1"); + if (port->msi.irq_msi1 <= 0) + return port->msi.irq_msi1; + + ret = devm_request_irq(dev, port->msi.irq_msi1, xilinx_pl_dma_pcie_msi_handler_high, + IRQF_SHARED | IRQF_NO_THREAD, "xlnx-pcie-dma-pl", + port); + if (ret) { + dev_err(dev, "Failed to register interrupt\n"); + return ret; + } + + return 0; +} + +static int xilinx_pl_dma_pcie_parse_dt(struct pl_dma_pcie *port, + struct resource *bus_range) +{ + struct device *dev = port->dev; + struct platform_device *pdev = to_platform_device(dev); + struct resource *res; + int err; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + dev_err(dev, "Missing \"reg\" property\n"); + return -ENXIO; + } + port->phys_reg_base = res->start; + + port->cfg = pci_ecam_create(dev, res, bus_range, &xilinx_pl_dma_pcie_ops); + if (IS_ERR(port->cfg)) + return PTR_ERR(port->cfg); + + port->reg_base = port->cfg->win; + + if (port->variant->version == QDMA) { + port->cfg_base = port->cfg->win; + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "breg"); + port->reg_base = devm_ioremap_resource(dev, res); + if (IS_ERR(port->reg_base)) + return PTR_ERR(port->reg_base); + port->phys_reg_base = res->start; + } + + err = xilinx_request_msi_irq(port); + if (err) { + pci_ecam_free(port->cfg); + return err; + } + + return 0; +} + +static int xilinx_pl_dma_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct pl_dma_pcie *port; + struct pci_host_bridge *bridge; + struct resource_entry *bus; + int err; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*port)); + if (!bridge) + return -ENODEV; + + port = pci_host_bridge_priv(bridge); + + port->dev = dev; + + bus = resource_list_first_type(&bridge->windows, IORESOURCE_BUS); + if (!bus) + return -ENODEV; + + port->variant = of_device_get_match_data(dev); + + err = xilinx_pl_dma_pcie_parse_dt(port, bus->res); + if (err) { + dev_err(dev, "Parsing DT failed\n"); + return err; + } + + xilinx_pl_dma_pcie_init_port(port); + + err = xilinx_pl_dma_pcie_init_irq_domain(port); + if (err) + goto err_irq_domain; + + err = xilinx_pl_dma_pcie_setup_irq(port); + + bridge->sysdata = port; + bridge->ops = &xilinx_pl_dma_pcie_ops.pci_ops; + + err = pci_host_probe(bridge); + if (err < 0) + goto err_host_bridge; + + return 0; + +err_host_bridge: + xilinx_pl_dma_pcie_free_irq_domains(port); + +err_irq_domain: + pci_ecam_free(port->cfg); + return err; +} + +static const struct xilinx_pl_dma_variant xdma_host = { + .version = XDMA, +}; + +static const struct xilinx_pl_dma_variant qdma_host = { + .version = QDMA, +}; + +static const struct of_device_id xilinx_pl_dma_pcie_of_match[] = { + { + .compatible = "xlnx,xdma-host-3.00", + .data = &xdma_host, + }, + { + .compatible = "xlnx,qdma-host-3.00", + .data = &qdma_host, + }, + {} +}; + +static struct platform_driver xilinx_pl_dma_pcie_driver = { + .driver = { + .name = "xilinx-xdma-pcie", + .of_match_table = xilinx_pl_dma_pcie_of_match, + .suppress_bind_attrs = true, + }, + .probe = xilinx_pl_dma_pcie_probe, +}; + +builtin_platform_driver(xilinx_pl_dma_pcie_driver); diff --git a/drivers/pci/controller/pcie-xilinx-nwl.c b/drivers/pci/controller/pcie-xilinx-nwl.c new file mode 100644 index 000000000..0dee19fa2 --- /dev/null +++ b/drivers/pci/controller/pcie-xilinx-nwl.c @@ -0,0 +1,914 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * PCIe host controller driver for NWL PCIe Bridge + * Based on pcie-xilinx.c, pci-tegra.c + * + * (C) Copyright 2014 - 2015, Xilinx, Inc. + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/irqdomain.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/msi.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/pci.h> +#include <linux/pci-ecam.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/irqchip/chained_irq.h> + +#include "../pci.h" + +/* Bridge core config registers */ +#define BRCFG_PCIE_RX0 0x00000000 +#define BRCFG_PCIE_RX1 0x00000004 +#define BRCFG_INTERRUPT 0x00000010 +#define BRCFG_PCIE_RX_MSG_FILTER 0x00000020 + +/* Egress - Bridge translation registers */ +#define E_BREG_CAPABILITIES 0x00000200 +#define E_BREG_CONTROL 0x00000208 +#define E_BREG_BASE_LO 0x00000210 +#define E_BREG_BASE_HI 0x00000214 +#define E_ECAM_CAPABILITIES 0x00000220 +#define E_ECAM_CONTROL 0x00000228 +#define E_ECAM_BASE_LO 0x00000230 +#define E_ECAM_BASE_HI 0x00000234 + +/* Ingress - address translations */ +#define I_MSII_CAPABILITIES 0x00000300 +#define I_MSII_CONTROL 0x00000308 +#define I_MSII_BASE_LO 0x00000310 +#define I_MSII_BASE_HI 0x00000314 + +#define I_ISUB_CONTROL 0x000003E8 +#define SET_ISUB_CONTROL BIT(0) +/* Rxed msg fifo - Interrupt status registers */ +#define MSGF_MISC_STATUS 0x00000400 +#define MSGF_MISC_MASK 0x00000404 +#define MSGF_LEG_STATUS 0x00000420 +#define MSGF_LEG_MASK 0x00000424 +#define MSGF_MSI_STATUS_LO 0x00000440 +#define MSGF_MSI_STATUS_HI 0x00000444 +#define MSGF_MSI_MASK_LO 0x00000448 +#define MSGF_MSI_MASK_HI 0x0000044C + +/* Msg filter mask bits */ +#define CFG_ENABLE_PM_MSG_FWD BIT(1) +#define CFG_ENABLE_INT_MSG_FWD BIT(2) +#define CFG_ENABLE_ERR_MSG_FWD BIT(3) +#define CFG_ENABLE_MSG_FILTER_MASK (CFG_ENABLE_PM_MSG_FWD | \ + CFG_ENABLE_INT_MSG_FWD | \ + CFG_ENABLE_ERR_MSG_FWD) + +/* Misc interrupt status mask bits */ +#define MSGF_MISC_SR_RXMSG_AVAIL BIT(0) +#define MSGF_MISC_SR_RXMSG_OVER BIT(1) +#define MSGF_MISC_SR_SLAVE_ERR BIT(4) +#define MSGF_MISC_SR_MASTER_ERR BIT(5) +#define MSGF_MISC_SR_I_ADDR_ERR BIT(6) +#define MSGF_MISC_SR_E_ADDR_ERR BIT(7) +#define MSGF_MISC_SR_FATAL_AER BIT(16) +#define MSGF_MISC_SR_NON_FATAL_AER BIT(17) +#define MSGF_MISC_SR_CORR_AER BIT(18) +#define MSGF_MISC_SR_UR_DETECT BIT(20) +#define MSGF_MISC_SR_NON_FATAL_DEV BIT(22) +#define MSGF_MISC_SR_FATAL_DEV BIT(23) +#define MSGF_MISC_SR_LINK_DOWN BIT(24) +#define MSGF_MISC_SR_LINK_AUTO_BWIDTH BIT(25) +#define MSGF_MISC_SR_LINK_BWIDTH BIT(26) + +#define MSGF_MISC_SR_MASKALL (MSGF_MISC_SR_RXMSG_AVAIL | \ + MSGF_MISC_SR_RXMSG_OVER | \ + MSGF_MISC_SR_SLAVE_ERR | \ + MSGF_MISC_SR_MASTER_ERR | \ + MSGF_MISC_SR_I_ADDR_ERR | \ + MSGF_MISC_SR_E_ADDR_ERR | \ + MSGF_MISC_SR_FATAL_AER | \ + MSGF_MISC_SR_NON_FATAL_AER | \ + MSGF_MISC_SR_CORR_AER | \ + MSGF_MISC_SR_UR_DETECT | \ + MSGF_MISC_SR_NON_FATAL_DEV | \ + MSGF_MISC_SR_FATAL_DEV | \ + MSGF_MISC_SR_LINK_DOWN | \ + MSGF_MISC_SR_LINK_AUTO_BWIDTH | \ + MSGF_MISC_SR_LINK_BWIDTH) + +/* Legacy interrupt status mask bits */ +#define MSGF_LEG_SR_INTA BIT(0) +#define MSGF_LEG_SR_INTB BIT(1) +#define MSGF_LEG_SR_INTC BIT(2) +#define MSGF_LEG_SR_INTD BIT(3) +#define MSGF_LEG_SR_MASKALL (MSGF_LEG_SR_INTA | MSGF_LEG_SR_INTB | \ + MSGF_LEG_SR_INTC | MSGF_LEG_SR_INTD) + +/* MSI interrupt status mask bits */ +#define MSGF_MSI_SR_LO_MASK GENMASK(31, 0) +#define MSGF_MSI_SR_HI_MASK GENMASK(31, 0) + +#define MSII_PRESENT BIT(0) +#define MSII_ENABLE BIT(0) +#define MSII_STATUS_ENABLE BIT(15) + +/* Bridge config interrupt mask */ +#define BRCFG_INTERRUPT_MASK BIT(0) +#define BREG_PRESENT BIT(0) +#define BREG_ENABLE BIT(0) +#define BREG_ENABLE_FORCE BIT(1) + +/* E_ECAM status mask bits */ +#define E_ECAM_PRESENT BIT(0) +#define E_ECAM_CR_ENABLE BIT(0) +#define E_ECAM_SIZE_LOC GENMASK(20, 16) +#define E_ECAM_SIZE_SHIFT 16 +#define NWL_ECAM_MAX_SIZE 16 + +#define CFG_DMA_REG_BAR GENMASK(2, 0) +#define CFG_PCIE_CACHE GENMASK(7, 0) + +#define INT_PCI_MSI_NR (2 * 32) + +/* Readin the PS_LINKUP */ +#define PS_LINKUP_OFFSET 0x00000238 +#define PCIE_PHY_LINKUP_BIT BIT(0) +#define PHY_RDY_LINKUP_BIT BIT(1) + +struct nwl_msi { /* MSI information */ + DECLARE_BITMAP(bitmap, INT_PCI_MSI_NR); + struct irq_domain *dev_domain; + struct mutex lock; /* protect bitmap variable */ + int irq_msi0; + int irq_msi1; +}; + +struct nwl_pcie { + struct device *dev; + void __iomem *breg_base; + void __iomem *pcireg_base; + void __iomem *ecam_base; + struct phy *phy[4]; + phys_addr_t phys_breg_base; /* Physical Bridge Register Base */ + phys_addr_t phys_pcie_reg_base; /* Physical PCIe Controller Base */ + phys_addr_t phys_ecam_base; /* Physical Configuration Base */ + u32 breg_size; + u32 pcie_reg_size; + u32 ecam_size; + int irq_intx; + int irq_misc; + struct nwl_msi msi; + struct irq_domain *intx_irq_domain; + struct clk *clk; + raw_spinlock_t leg_mask_lock; +}; + +static inline u32 nwl_bridge_readl(struct nwl_pcie *pcie, u32 off) +{ + return readl(pcie->breg_base + off); +} + +static inline void nwl_bridge_writel(struct nwl_pcie *pcie, u32 val, u32 off) +{ + writel(val, pcie->breg_base + off); +} + +static bool nwl_pcie_link_up(struct nwl_pcie *pcie) +{ + if (readl(pcie->pcireg_base + PS_LINKUP_OFFSET) & PCIE_PHY_LINKUP_BIT) + return true; + return false; +} + +static bool nwl_phy_link_up(struct nwl_pcie *pcie) +{ + if (readl(pcie->pcireg_base + PS_LINKUP_OFFSET) & PHY_RDY_LINKUP_BIT) + return true; + return false; +} + +static int nwl_wait_for_link(struct nwl_pcie *pcie) +{ + struct device *dev = pcie->dev; + int retries; + + /* check if the link is up or not */ + for (retries = 0; retries < PCIE_LINK_WAIT_MAX_RETRIES; retries++) { + if (nwl_phy_link_up(pcie)) + return 0; + msleep(PCIE_LINK_WAIT_SLEEP_MS); + } + + dev_err(dev, "PHY link never came up\n"); + return -ETIMEDOUT; +} + +static bool nwl_pcie_valid_device(struct pci_bus *bus, unsigned int devfn) +{ + struct nwl_pcie *pcie = bus->sysdata; + + /* Check link before accessing downstream ports */ + if (!pci_is_root_bus(bus)) { + if (!nwl_pcie_link_up(pcie)) + return false; + } else if (devfn > 0) + /* Only one device down on each root port */ + return false; + + return true; +} + +/** + * nwl_pcie_map_bus - Get configuration base + * + * @bus: Bus structure of current bus + * @devfn: Device/function + * @where: Offset from base + * + * Return: Base address of the configuration space needed to be + * accessed. + */ +static void __iomem *nwl_pcie_map_bus(struct pci_bus *bus, unsigned int devfn, + int where) +{ + struct nwl_pcie *pcie = bus->sysdata; + + if (!nwl_pcie_valid_device(bus, devfn)) + return NULL; + + return pcie->ecam_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); +} + +/* PCIe operations */ +static struct pci_ops nwl_pcie_ops = { + .map_bus = nwl_pcie_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; + +static irqreturn_t nwl_pcie_misc_handler(int irq, void *data) +{ + struct nwl_pcie *pcie = data; + struct device *dev = pcie->dev; + u32 misc_stat; + + /* Checking for misc interrupts */ + misc_stat = nwl_bridge_readl(pcie, MSGF_MISC_STATUS) & + MSGF_MISC_SR_MASKALL; + if (!misc_stat) + return IRQ_NONE; + + if (misc_stat & MSGF_MISC_SR_RXMSG_OVER) + dev_err_ratelimited(dev, "Received Message FIFO Overflow\n"); + + if (misc_stat & MSGF_MISC_SR_SLAVE_ERR) + dev_err_ratelimited(dev, "Slave error\n"); + + if (misc_stat & MSGF_MISC_SR_MASTER_ERR) + dev_err_ratelimited(dev, "Master error\n"); + + if (misc_stat & MSGF_MISC_SR_I_ADDR_ERR) + dev_err_ratelimited(dev, "In Misc Ingress address translation error\n"); + + if (misc_stat & MSGF_MISC_SR_E_ADDR_ERR) + dev_err_ratelimited(dev, "In Misc Egress address translation error\n"); + + if (misc_stat & MSGF_MISC_SR_FATAL_AER) + dev_err_ratelimited(dev, "Fatal Error in AER Capability\n"); + + if (misc_stat & MSGF_MISC_SR_NON_FATAL_AER) + dev_err_ratelimited(dev, "Non-Fatal Error in AER Capability\n"); + + if (misc_stat & MSGF_MISC_SR_CORR_AER) + dev_err_ratelimited(dev, "Correctable Error in AER Capability\n"); + + if (misc_stat & MSGF_MISC_SR_UR_DETECT) + dev_err_ratelimited(dev, "Unsupported request Detected\n"); + + if (misc_stat & MSGF_MISC_SR_NON_FATAL_DEV) + dev_err_ratelimited(dev, "Non-Fatal Error Detected\n"); + + if (misc_stat & MSGF_MISC_SR_FATAL_DEV) + dev_err_ratelimited(dev, "Fatal Error Detected\n"); + + if (misc_stat & MSGF_MISC_SR_LINK_AUTO_BWIDTH) + dev_info(dev, "Link Autonomous Bandwidth Management Status bit set\n"); + + if (misc_stat & MSGF_MISC_SR_LINK_BWIDTH) + dev_info(dev, "Link Bandwidth Management Status bit set\n"); + + /* Clear misc interrupt status */ + nwl_bridge_writel(pcie, misc_stat, MSGF_MISC_STATUS); + + return IRQ_HANDLED; +} + +static void nwl_pcie_leg_handler(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + struct nwl_pcie *pcie; + unsigned long status; + u32 bit; + + chained_irq_enter(chip, desc); + pcie = irq_desc_get_handler_data(desc); + + while ((status = nwl_bridge_readl(pcie, MSGF_LEG_STATUS) & + MSGF_LEG_SR_MASKALL) != 0) { + for_each_set_bit(bit, &status, PCI_NUM_INTX) + generic_handle_domain_irq(pcie->intx_irq_domain, bit); + } + + chained_irq_exit(chip, desc); +} + +static void nwl_pcie_handle_msi_irq(struct nwl_pcie *pcie, u32 status_reg) +{ + struct nwl_msi *msi = &pcie->msi; + unsigned long status; + u32 bit; + + while ((status = nwl_bridge_readl(pcie, status_reg)) != 0) { + for_each_set_bit(bit, &status, 32) { + nwl_bridge_writel(pcie, 1 << bit, status_reg); + generic_handle_domain_irq(msi->dev_domain, bit); + } + } +} + +static void nwl_pcie_msi_handler_high(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + struct nwl_pcie *pcie = irq_desc_get_handler_data(desc); + + chained_irq_enter(chip, desc); + nwl_pcie_handle_msi_irq(pcie, MSGF_MSI_STATUS_HI); + chained_irq_exit(chip, desc); +} + +static void nwl_pcie_msi_handler_low(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + struct nwl_pcie *pcie = irq_desc_get_handler_data(desc); + + chained_irq_enter(chip, desc); + nwl_pcie_handle_msi_irq(pcie, MSGF_MSI_STATUS_LO); + chained_irq_exit(chip, desc); +} + +static void nwl_mask_intx_irq(struct irq_data *data) +{ + struct nwl_pcie *pcie = irq_data_get_irq_chip_data(data); + unsigned long flags; + u32 mask; + u32 val; + + mask = 1 << data->hwirq; + raw_spin_lock_irqsave(&pcie->leg_mask_lock, flags); + val = nwl_bridge_readl(pcie, MSGF_LEG_MASK); + nwl_bridge_writel(pcie, (val & (~mask)), MSGF_LEG_MASK); + raw_spin_unlock_irqrestore(&pcie->leg_mask_lock, flags); +} + +static void nwl_unmask_intx_irq(struct irq_data *data) +{ + struct nwl_pcie *pcie = irq_data_get_irq_chip_data(data); + unsigned long flags; + u32 mask; + u32 val; + + mask = 1 << data->hwirq; + raw_spin_lock_irqsave(&pcie->leg_mask_lock, flags); + val = nwl_bridge_readl(pcie, MSGF_LEG_MASK); + nwl_bridge_writel(pcie, (val | mask), MSGF_LEG_MASK); + raw_spin_unlock_irqrestore(&pcie->leg_mask_lock, flags); +} + +static struct irq_chip nwl_intx_irq_chip = { + .name = "nwl_pcie:legacy", + .irq_enable = nwl_unmask_intx_irq, + .irq_disable = nwl_mask_intx_irq, + .irq_mask = nwl_mask_intx_irq, + .irq_unmask = nwl_unmask_intx_irq, +}; + +static int nwl_intx_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &nwl_intx_irq_chip, handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + irq_set_status_flags(irq, IRQ_LEVEL); + + return 0; +} + +static const struct irq_domain_ops intx_domain_ops = { + .map = nwl_intx_map, + .xlate = pci_irqd_intx_xlate, +}; + +#ifdef CONFIG_PCI_MSI + +#define NWL_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_NO_AFFINITY) + +#define NWL_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_MULTI_PCI_MSI) + +static const struct msi_parent_ops nwl_msi_parent_ops = { + .required_flags = NWL_MSI_FLAGS_REQUIRED, + .supported_flags = NWL_MSI_FLAGS_SUPPORTED, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .prefix = "nwl-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +#endif + +static void nwl_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct nwl_pcie *pcie = irq_data_get_irq_chip_data(data); + phys_addr_t msi_addr = pcie->phys_pcie_reg_base; + + msg->address_lo = lower_32_bits(msi_addr); + msg->address_hi = upper_32_bits(msi_addr); + msg->data = data->hwirq; +} + +static struct irq_chip nwl_irq_chip = { + .name = "Xilinx MSI", + .irq_compose_msi_msg = nwl_compose_msi_msg, +}; + +static int nwl_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct nwl_pcie *pcie = domain->host_data; + struct nwl_msi *msi = &pcie->msi; + int bit; + int i; + + mutex_lock(&msi->lock); + bit = bitmap_find_free_region(msi->bitmap, INT_PCI_MSI_NR, + get_count_order(nr_irqs)); + if (bit < 0) { + mutex_unlock(&msi->lock); + return -ENOSPC; + } + + for (i = 0; i < nr_irqs; i++) { + irq_domain_set_info(domain, virq + i, bit + i, &nwl_irq_chip, + domain->host_data, handle_simple_irq, + NULL, NULL); + } + mutex_unlock(&msi->lock); + return 0; +} + +static void nwl_irq_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *data = irq_domain_get_irq_data(domain, virq); + struct nwl_pcie *pcie = irq_data_get_irq_chip_data(data); + struct nwl_msi *msi = &pcie->msi; + + mutex_lock(&msi->lock); + bitmap_release_region(msi->bitmap, data->hwirq, + get_count_order(nr_irqs)); + mutex_unlock(&msi->lock); +} + +static const struct irq_domain_ops dev_msi_domain_ops = { + .alloc = nwl_irq_domain_alloc, + .free = nwl_irq_domain_free, +}; + +static int nwl_pcie_init_msi_irq_domain(struct nwl_pcie *pcie) +{ +#ifdef CONFIG_PCI_MSI + struct device *dev = pcie->dev; + struct nwl_msi *msi = &pcie->msi; + struct irq_domain_info info = { + .fwnode = dev_fwnode(dev), + .ops = &dev_msi_domain_ops, + .host_data = pcie, + .size = INT_PCI_MSI_NR, + }; + + msi->dev_domain = msi_create_parent_irq_domain(&info, &nwl_msi_parent_ops); + if (!msi->dev_domain) { + dev_err(dev, "failed to create dev IRQ domain\n"); + return -ENOMEM; + } +#endif + return 0; +} + +static void nwl_pcie_phy_power_off(struct nwl_pcie *pcie, int i) +{ + int err = phy_power_off(pcie->phy[i]); + + if (err) + dev_err(pcie->dev, "could not power off phy %d (err=%d)\n", i, + err); +} + +static void nwl_pcie_phy_exit(struct nwl_pcie *pcie, int i) +{ + int err = phy_exit(pcie->phy[i]); + + if (err) + dev_err(pcie->dev, "could not exit phy %d (err=%d)\n", i, err); +} + +static int nwl_pcie_phy_enable(struct nwl_pcie *pcie) +{ + int i, ret; + + for (i = 0; i < ARRAY_SIZE(pcie->phy); i++) { + ret = phy_init(pcie->phy[i]); + if (ret) + goto err; + + ret = phy_power_on(pcie->phy[i]); + if (ret) { + nwl_pcie_phy_exit(pcie, i); + goto err; + } + } + + return 0; + +err: + while (i--) { + nwl_pcie_phy_power_off(pcie, i); + nwl_pcie_phy_exit(pcie, i); + } + + return ret; +} + +static void nwl_pcie_phy_disable(struct nwl_pcie *pcie) +{ + int i; + + for (i = ARRAY_SIZE(pcie->phy); i--;) { + nwl_pcie_phy_power_off(pcie, i); + nwl_pcie_phy_exit(pcie, i); + } +} + +static int nwl_pcie_init_irq_domain(struct nwl_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct device_node *node = dev->of_node; + struct device_node *intc_node; + + intc_node = of_get_next_child(node, NULL); + if (!intc_node) { + dev_err(dev, "No legacy intc node found\n"); + return -EINVAL; + } + + pcie->intx_irq_domain = irq_domain_create_linear(of_fwnode_handle(intc_node), PCI_NUM_INTX, + &intx_domain_ops, pcie); + of_node_put(intc_node); + if (!pcie->intx_irq_domain) { + dev_err(dev, "failed to create IRQ domain\n"); + return -ENOMEM; + } + + raw_spin_lock_init(&pcie->leg_mask_lock); + nwl_pcie_init_msi_irq_domain(pcie); + return 0; +} + +static int nwl_pcie_enable_msi(struct nwl_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct platform_device *pdev = to_platform_device(dev); + struct nwl_msi *msi = &pcie->msi; + unsigned long base; + int ret; + + mutex_init(&msi->lock); + + /* Get msi_1 IRQ number */ + msi->irq_msi1 = platform_get_irq_byname(pdev, "msi1"); + if (msi->irq_msi1 < 0) + return -EINVAL; + + irq_set_chained_handler_and_data(msi->irq_msi1, + nwl_pcie_msi_handler_high, pcie); + + /* Get msi_0 IRQ number */ + msi->irq_msi0 = platform_get_irq_byname(pdev, "msi0"); + if (msi->irq_msi0 < 0) + return -EINVAL; + + irq_set_chained_handler_and_data(msi->irq_msi0, + nwl_pcie_msi_handler_low, pcie); + + /* Check for msii_present bit */ + ret = nwl_bridge_readl(pcie, I_MSII_CAPABILITIES) & MSII_PRESENT; + if (!ret) { + dev_err(dev, "MSI not present\n"); + return -EIO; + } + + /* Enable MSII */ + nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, I_MSII_CONTROL) | + MSII_ENABLE, I_MSII_CONTROL); + + /* Enable MSII status */ + nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, I_MSII_CONTROL) | + MSII_STATUS_ENABLE, I_MSII_CONTROL); + + /* setup AFI/FPCI range */ + base = pcie->phys_pcie_reg_base; + nwl_bridge_writel(pcie, lower_32_bits(base), I_MSII_BASE_LO); + nwl_bridge_writel(pcie, upper_32_bits(base), I_MSII_BASE_HI); + + /* + * For high range MSI interrupts: disable, clear any pending, + * and enable + */ + nwl_bridge_writel(pcie, 0, MSGF_MSI_MASK_HI); + + nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, MSGF_MSI_STATUS_HI) & + MSGF_MSI_SR_HI_MASK, MSGF_MSI_STATUS_HI); + + nwl_bridge_writel(pcie, MSGF_MSI_SR_HI_MASK, MSGF_MSI_MASK_HI); + + /* + * For low range MSI interrupts: disable, clear any pending, + * and enable + */ + nwl_bridge_writel(pcie, 0, MSGF_MSI_MASK_LO); + + nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, MSGF_MSI_STATUS_LO) & + MSGF_MSI_SR_LO_MASK, MSGF_MSI_STATUS_LO); + + nwl_bridge_writel(pcie, MSGF_MSI_SR_LO_MASK, MSGF_MSI_MASK_LO); + + return 0; +} + +static int nwl_pcie_bridge_init(struct nwl_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct platform_device *pdev = to_platform_device(dev); + u32 breg_val, ecam_val; + int err; + + breg_val = nwl_bridge_readl(pcie, E_BREG_CAPABILITIES) & BREG_PRESENT; + if (!breg_val) { + dev_err(dev, "BREG is not present\n"); + return breg_val; + } + + /* Write bridge_off to breg base */ + nwl_bridge_writel(pcie, lower_32_bits(pcie->phys_breg_base), + E_BREG_BASE_LO); + nwl_bridge_writel(pcie, upper_32_bits(pcie->phys_breg_base), + E_BREG_BASE_HI); + + /* Enable BREG */ + nwl_bridge_writel(pcie, ~BREG_ENABLE_FORCE & BREG_ENABLE, + E_BREG_CONTROL); + + /* Disable DMA channel registers */ + nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, BRCFG_PCIE_RX0) | + CFG_DMA_REG_BAR, BRCFG_PCIE_RX0); + + /* Enable Ingress subtractive decode translation */ + nwl_bridge_writel(pcie, SET_ISUB_CONTROL, I_ISUB_CONTROL); + + /* Enable msg filtering details */ + nwl_bridge_writel(pcie, CFG_ENABLE_MSG_FILTER_MASK, + BRCFG_PCIE_RX_MSG_FILTER); + + /* This routes the PCIe DMA traffic to go through CCI path */ + if (of_dma_is_coherent(dev->of_node)) + nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, BRCFG_PCIE_RX1) | + CFG_PCIE_CACHE, BRCFG_PCIE_RX1); + + err = nwl_wait_for_link(pcie); + if (err) + return err; + + ecam_val = nwl_bridge_readl(pcie, E_ECAM_CAPABILITIES) & E_ECAM_PRESENT; + if (!ecam_val) { + dev_err(dev, "ECAM is not present\n"); + return ecam_val; + } + + /* Enable ECAM */ + nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, E_ECAM_CONTROL) | + E_ECAM_CR_ENABLE, E_ECAM_CONTROL); + + ecam_val = nwl_bridge_readl(pcie, E_ECAM_CONTROL); + ecam_val &= ~E_ECAM_SIZE_LOC; + ecam_val |= NWL_ECAM_MAX_SIZE << E_ECAM_SIZE_SHIFT; + nwl_bridge_writel(pcie, ecam_val, E_ECAM_CONTROL); + + nwl_bridge_writel(pcie, lower_32_bits(pcie->phys_ecam_base), + E_ECAM_BASE_LO); + nwl_bridge_writel(pcie, upper_32_bits(pcie->phys_ecam_base), + E_ECAM_BASE_HI); + + if (nwl_pcie_link_up(pcie)) + dev_info(dev, "Link is UP\n"); + else + dev_info(dev, "Link is DOWN\n"); + + /* Get misc IRQ number */ + pcie->irq_misc = platform_get_irq_byname(pdev, "misc"); + if (pcie->irq_misc < 0) + return -EINVAL; + + err = devm_request_irq(dev, pcie->irq_misc, + nwl_pcie_misc_handler, IRQF_SHARED, + "nwl_pcie:misc", pcie); + if (err) { + dev_err(dev, "fail to register misc IRQ#%d\n", + pcie->irq_misc); + return err; + } + + /* Disable all misc interrupts */ + nwl_bridge_writel(pcie, (u32)~MSGF_MISC_SR_MASKALL, MSGF_MISC_MASK); + + /* Clear pending misc interrupts */ + nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, MSGF_MISC_STATUS) & + MSGF_MISC_SR_MASKALL, MSGF_MISC_STATUS); + + /* Enable all misc interrupts */ + nwl_bridge_writel(pcie, MSGF_MISC_SR_MASKALL, MSGF_MISC_MASK); + + /* Disable all INTX interrupts */ + nwl_bridge_writel(pcie, (u32)~MSGF_LEG_SR_MASKALL, MSGF_LEG_MASK); + + /* Clear pending INTX interrupts */ + nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, MSGF_LEG_STATUS) & + MSGF_LEG_SR_MASKALL, MSGF_LEG_STATUS); + + /* Enable all INTX interrupts */ + nwl_bridge_writel(pcie, MSGF_LEG_SR_MASKALL, MSGF_LEG_MASK); + + /* Enable the bridge config interrupt */ + nwl_bridge_writel(pcie, nwl_bridge_readl(pcie, BRCFG_INTERRUPT) | + BRCFG_INTERRUPT_MASK, BRCFG_INTERRUPT); + + return 0; +} + +static int nwl_pcie_parse_dt(struct nwl_pcie *pcie, + struct platform_device *pdev) +{ + struct device *dev = pcie->dev; + struct resource *res; + int i; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "breg"); + pcie->breg_base = devm_ioremap_resource(dev, res); + if (IS_ERR(pcie->breg_base)) + return PTR_ERR(pcie->breg_base); + pcie->phys_breg_base = res->start; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pcireg"); + pcie->pcireg_base = devm_ioremap_resource(dev, res); + if (IS_ERR(pcie->pcireg_base)) + return PTR_ERR(pcie->pcireg_base); + pcie->phys_pcie_reg_base = res->start; + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg"); + pcie->ecam_base = devm_pci_remap_cfg_resource(dev, res); + if (IS_ERR(pcie->ecam_base)) + return PTR_ERR(pcie->ecam_base); + pcie->phys_ecam_base = res->start; + + /* Get intx IRQ number */ + pcie->irq_intx = platform_get_irq_byname(pdev, "intx"); + if (pcie->irq_intx < 0) + return pcie->irq_intx; + + irq_set_chained_handler_and_data(pcie->irq_intx, + nwl_pcie_leg_handler, pcie); + + + for (i = 0; i < ARRAY_SIZE(pcie->phy); i++) { + pcie->phy[i] = devm_of_phy_get_by_index(dev, dev->of_node, i); + if (PTR_ERR(pcie->phy[i]) == -ENODEV) { + pcie->phy[i] = NULL; + break; + } + + if (IS_ERR(pcie->phy[i])) + return PTR_ERR(pcie->phy[i]); + } + + return 0; +} + +static const struct of_device_id nwl_pcie_of_match[] = { + { .compatible = "xlnx,nwl-pcie-2.11", }, + {} +}; + +static int nwl_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct nwl_pcie *pcie; + struct pci_host_bridge *bridge; + int err; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!bridge) + return -ENODEV; + + pcie = pci_host_bridge_priv(bridge); + platform_set_drvdata(pdev, pcie); + + pcie->dev = dev; + + err = nwl_pcie_parse_dt(pcie, pdev); + if (err) { + dev_err(dev, "Parsing DT failed\n"); + return err; + } + + pcie->clk = devm_clk_get(dev, NULL); + if (IS_ERR(pcie->clk)) + return PTR_ERR(pcie->clk); + + err = clk_prepare_enable(pcie->clk); + if (err) { + dev_err(dev, "can't enable PCIe ref clock\n"); + return err; + } + + err = nwl_pcie_phy_enable(pcie); + if (err) { + dev_err(dev, "could not enable PHYs\n"); + goto err_clk; + } + + err = nwl_pcie_bridge_init(pcie); + if (err) { + dev_err(dev, "HW Initialization failed\n"); + goto err_phy; + } + + err = nwl_pcie_init_irq_domain(pcie); + if (err) { + dev_err(dev, "Failed creating IRQ Domain\n"); + goto err_phy; + } + + bridge->sysdata = pcie; + bridge->ops = &nwl_pcie_ops; + + if (IS_ENABLED(CONFIG_PCI_MSI)) { + err = nwl_pcie_enable_msi(pcie); + if (err < 0) { + dev_err(dev, "failed to enable MSI support: %d\n", err); + goto err_phy; + } + } + + err = pci_host_probe(bridge); + if (!err) + return 0; + +err_phy: + nwl_pcie_phy_disable(pcie); +err_clk: + clk_disable_unprepare(pcie->clk); + return err; +} + +static void nwl_pcie_remove(struct platform_device *pdev) +{ + struct nwl_pcie *pcie = platform_get_drvdata(pdev); + + nwl_pcie_phy_disable(pcie); + clk_disable_unprepare(pcie->clk); +} + +static struct platform_driver nwl_pcie_driver = { + .driver = { + .name = "nwl-pcie", + .suppress_bind_attrs = true, + .of_match_table = nwl_pcie_of_match, + }, + .probe = nwl_pcie_probe, + .remove = nwl_pcie_remove, +}; +builtin_platform_driver(nwl_pcie_driver); diff --git a/drivers/pci/controller/pcie-xilinx.c b/drivers/pci/controller/pcie-xilinx.c new file mode 100644 index 000000000..4aa139aba --- /dev/null +++ b/drivers/pci/controller/pcie-xilinx.c @@ -0,0 +1,624 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * PCIe host controller driver for Xilinx AXI PCIe Bridge + * + * Copyright (c) 2012 - 2014 Xilinx, Inc. + * + * Based on the Tegra PCIe driver + * + * Bits taken from Synopsys DesignWare Host controller driver and + * ARM PCI Host generic driver. + */ + +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/irqdomain.h> +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/msi.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/of_platform.h> +#include <linux/of_irq.h> +#include <linux/pci.h> +#include <linux/pci-ecam.h> +#include <linux/platform_device.h> + +#include "../pci.h" + +/* Register definitions */ +#define XILINX_PCIE_REG_BIR 0x00000130 +#define XILINX_PCIE_REG_IDR 0x00000138 +#define XILINX_PCIE_REG_IMR 0x0000013c +#define XILINX_PCIE_REG_PSCR 0x00000144 +#define XILINX_PCIE_REG_RPSC 0x00000148 +#define XILINX_PCIE_REG_MSIBASE1 0x0000014c +#define XILINX_PCIE_REG_MSIBASE2 0x00000150 +#define XILINX_PCIE_REG_RPEFR 0x00000154 +#define XILINX_PCIE_REG_RPIFR1 0x00000158 +#define XILINX_PCIE_REG_RPIFR2 0x0000015c + +/* Interrupt registers definitions */ +#define XILINX_PCIE_INTR_LINK_DOWN BIT(0) +#define XILINX_PCIE_INTR_ECRC_ERR BIT(1) +#define XILINX_PCIE_INTR_STR_ERR BIT(2) +#define XILINX_PCIE_INTR_HOT_RESET BIT(3) +#define XILINX_PCIE_INTR_CFG_TIMEOUT BIT(8) +#define XILINX_PCIE_INTR_CORRECTABLE BIT(9) +#define XILINX_PCIE_INTR_NONFATAL BIT(10) +#define XILINX_PCIE_INTR_FATAL BIT(11) +#define XILINX_PCIE_INTR_INTX BIT(16) +#define XILINX_PCIE_INTR_MSI BIT(17) +#define XILINX_PCIE_INTR_SLV_UNSUPP BIT(20) +#define XILINX_PCIE_INTR_SLV_UNEXP BIT(21) +#define XILINX_PCIE_INTR_SLV_COMPL BIT(22) +#define XILINX_PCIE_INTR_SLV_ERRP BIT(23) +#define XILINX_PCIE_INTR_SLV_CMPABT BIT(24) +#define XILINX_PCIE_INTR_SLV_ILLBUR BIT(25) +#define XILINX_PCIE_INTR_MST_DECERR BIT(26) +#define XILINX_PCIE_INTR_MST_SLVERR BIT(27) +#define XILINX_PCIE_INTR_MST_ERRP BIT(28) +#define XILINX_PCIE_IMR_ALL_MASK 0x1FF30FED +#define XILINX_PCIE_IMR_ENABLE_MASK 0x1FF30F0D +#define XILINX_PCIE_IDR_ALL_MASK 0xFFFFFFFF + +/* Root Port Error FIFO Read Register definitions */ +#define XILINX_PCIE_RPEFR_ERR_VALID BIT(18) +#define XILINX_PCIE_RPEFR_REQ_ID GENMASK(15, 0) +#define XILINX_PCIE_RPEFR_ALL_MASK 0xFFFFFFFF + +/* Root Port Interrupt FIFO Read Register 1 definitions */ +#define XILINX_PCIE_RPIFR1_INTR_VALID BIT(31) +#define XILINX_PCIE_RPIFR1_MSI_INTR BIT(30) +#define XILINX_PCIE_RPIFR1_INTR_MASK GENMASK(28, 27) +#define XILINX_PCIE_RPIFR1_ALL_MASK 0xFFFFFFFF +#define XILINX_PCIE_RPIFR1_INTR_SHIFT 27 + +/* Bridge Info Register definitions */ +#define XILINX_PCIE_BIR_ECAM_SZ_MASK GENMASK(18, 16) +#define XILINX_PCIE_BIR_ECAM_SZ_SHIFT 16 + +/* Root Port Interrupt FIFO Read Register 2 definitions */ +#define XILINX_PCIE_RPIFR2_MSG_DATA GENMASK(15, 0) + +/* Root Port Status/control Register definitions */ +#define XILINX_PCIE_REG_RPSC_BEN BIT(0) + +/* Phy Status/Control Register definitions */ +#define XILINX_PCIE_REG_PSCR_LNKUP BIT(11) + +/* Number of MSI IRQs */ +#define XILINX_NUM_MSI_IRQS 128 + +/** + * struct xilinx_pcie - PCIe port information + * @dev: Device pointer + * @reg_base: IO Mapped Register Base + * @msi_map: Bitmap of allocated MSIs + * @map_lock: Mutex protecting the MSI allocation + * @msi_domain: MSI IRQ domain pointer + * @leg_domain: Legacy IRQ domain pointer + * @resources: Bus Resources + */ +struct xilinx_pcie { + struct device *dev; + void __iomem *reg_base; + unsigned long msi_map[BITS_TO_LONGS(XILINX_NUM_MSI_IRQS)]; + struct mutex map_lock; + struct irq_domain *msi_domain; + struct irq_domain *leg_domain; + struct list_head resources; +}; + +static inline u32 pcie_read(struct xilinx_pcie *pcie, u32 reg) +{ + return readl(pcie->reg_base + reg); +} + +static inline void pcie_write(struct xilinx_pcie *pcie, u32 val, u32 reg) +{ + writel(val, pcie->reg_base + reg); +} + +static inline bool xilinx_pcie_link_up(struct xilinx_pcie *pcie) +{ + return (pcie_read(pcie, XILINX_PCIE_REG_PSCR) & + XILINX_PCIE_REG_PSCR_LNKUP) ? 1 : 0; +} + +/** + * xilinx_pcie_clear_err_interrupts - Clear Error Interrupts + * @pcie: PCIe port information + */ +static void xilinx_pcie_clear_err_interrupts(struct xilinx_pcie *pcie) +{ + struct device *dev = pcie->dev; + unsigned long val = pcie_read(pcie, XILINX_PCIE_REG_RPEFR); + + if (val & XILINX_PCIE_RPEFR_ERR_VALID) { + dev_dbg(dev, "Requester ID %lu\n", + val & XILINX_PCIE_RPEFR_REQ_ID); + pcie_write(pcie, XILINX_PCIE_RPEFR_ALL_MASK, + XILINX_PCIE_REG_RPEFR); + } +} + +/** + * xilinx_pcie_valid_device - Check if a valid device is present on bus + * @bus: PCI Bus structure + * @devfn: device/function + * + * Return: 'true' on success and 'false' if invalid device is found + */ +static bool xilinx_pcie_valid_device(struct pci_bus *bus, unsigned int devfn) +{ + struct xilinx_pcie *pcie = bus->sysdata; + + /* Check if link is up when trying to access downstream pcie ports */ + if (!pci_is_root_bus(bus)) { + if (!xilinx_pcie_link_up(pcie)) + return false; + } else if (devfn > 0) { + /* Only one device down on each root port */ + return false; + } + return true; +} + +/** + * xilinx_pcie_map_bus - Get configuration base + * @bus: PCI Bus structure + * @devfn: Device/function + * @where: Offset from base + * + * Return: Base address of the configuration space needed to be + * accessed. + */ +static void __iomem *xilinx_pcie_map_bus(struct pci_bus *bus, + unsigned int devfn, int where) +{ + struct xilinx_pcie *pcie = bus->sysdata; + + if (!xilinx_pcie_valid_device(bus, devfn)) + return NULL; + + return pcie->reg_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); +} + +/* PCIe operations */ +static struct pci_ops xilinx_pcie_ops = { + .map_bus = xilinx_pcie_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, +}; + +/* MSI functions */ + +static void xilinx_msi_top_irq_ack(struct irq_data *d) +{ + /* + * xilinx_pcie_intr_handler() will have performed the Ack. + * Eventually, this should be fixed and the Ack be moved in + * the respective callbacks for INTx and MSI. + */ +} + +static void xilinx_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct xilinx_pcie *pcie = irq_data_get_irq_chip_data(data); + phys_addr_t pa = ALIGN_DOWN(virt_to_phys(pcie), SZ_4K); + + msg->address_lo = lower_32_bits(pa); + msg->address_hi = upper_32_bits(pa); + msg->data = data->hwirq; +} + +static struct irq_chip xilinx_msi_bottom_chip = { + .name = "Xilinx MSI", + .irq_compose_msi_msg = xilinx_compose_msi_msg, +}; + +static int xilinx_msi_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct xilinx_pcie *pcie = domain->host_data; + int hwirq, i; + + mutex_lock(&pcie->map_lock); + + hwirq = bitmap_find_free_region(pcie->msi_map, XILINX_NUM_MSI_IRQS, order_base_2(nr_irqs)); + + mutex_unlock(&pcie->map_lock); + + if (hwirq < 0) + return -ENOSPC; + + for (i = 0; i < nr_irqs; i++) + irq_domain_set_info(domain, virq + i, hwirq + i, + &xilinx_msi_bottom_chip, domain->host_data, + handle_edge_irq, NULL, NULL); + + return 0; +} + +static void xilinx_msi_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct xilinx_pcie *pcie = domain->host_data; + + mutex_lock(&pcie->map_lock); + + bitmap_release_region(pcie->msi_map, d->hwirq, order_base_2(nr_irqs)); + + mutex_unlock(&pcie->map_lock); +} + +static const struct irq_domain_ops xilinx_msi_domain_ops = { + .alloc = xilinx_msi_domain_alloc, + .free = xilinx_msi_domain_free, +}; + +static bool xilinx_init_dev_msi_info(struct device *dev, struct irq_domain *domain, + struct irq_domain *real_parent, struct msi_domain_info *info) +{ + struct irq_chip *chip = info->chip; + + if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info)) + return false; + + chip->irq_ack = xilinx_msi_top_irq_ack; + return true; +} + +#define XILINX_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_NO_AFFINITY) + +static const struct msi_parent_ops xilinx_msi_parent_ops = { + .required_flags = XILINX_MSI_FLAGS_REQUIRED, + .supported_flags = MSI_GENERIC_FLAGS_MASK, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .prefix = "xilinx-", + .init_dev_msi_info = xilinx_init_dev_msi_info, +}; + +static int xilinx_allocate_msi_domains(struct xilinx_pcie *pcie) +{ + struct irq_domain_info info = { + .fwnode = dev_fwnode(pcie->dev), + .ops = &xilinx_msi_domain_ops, + .host_data = pcie, + .size = XILINX_NUM_MSI_IRQS, + }; + + pcie->msi_domain = msi_create_parent_irq_domain(&info, &xilinx_msi_parent_ops); + if (!pcie->msi_domain) { + dev_err(pcie->dev, "failed to create MSI domain\n"); + return -ENOMEM; + } + + return 0; +} + +static void xilinx_free_irq_domains(struct xilinx_pcie *pcie) +{ + irq_domain_remove(pcie->msi_domain); + irq_domain_remove(pcie->leg_domain); +} + +/* INTx Functions */ + +/** + * xilinx_pcie_intx_map - Set the handler for the INTx and mark IRQ as valid + * @domain: IRQ domain + * @irq: Virtual IRQ number + * @hwirq: HW interrupt number + * + * Return: Always returns 0. + */ +static int xilinx_pcie_intx_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq); + irq_set_chip_data(irq, domain->host_data); + + return 0; +} + +/* INTx IRQ Domain operations */ +static const struct irq_domain_ops intx_domain_ops = { + .map = xilinx_pcie_intx_map, + .xlate = pci_irqd_intx_xlate, +}; + +/* PCIe HW Functions */ + +/** + * xilinx_pcie_intr_handler - Interrupt Service Handler + * @irq: IRQ number + * @data: PCIe port information + * + * Return: IRQ_HANDLED on success and IRQ_NONE on failure + */ +static irqreturn_t xilinx_pcie_intr_handler(int irq, void *data) +{ + struct xilinx_pcie *pcie = (struct xilinx_pcie *)data; + struct device *dev = pcie->dev; + u32 val, mask, status; + + /* Read interrupt decode and mask registers */ + val = pcie_read(pcie, XILINX_PCIE_REG_IDR); + mask = pcie_read(pcie, XILINX_PCIE_REG_IMR); + + status = val & mask; + if (!status) + return IRQ_NONE; + + if (status & XILINX_PCIE_INTR_LINK_DOWN) + dev_warn(dev, "Link Down\n"); + + if (status & XILINX_PCIE_INTR_ECRC_ERR) + dev_warn(dev, "ECRC failed\n"); + + if (status & XILINX_PCIE_INTR_STR_ERR) + dev_warn(dev, "Streaming error\n"); + + if (status & XILINX_PCIE_INTR_HOT_RESET) + dev_info(dev, "Hot reset\n"); + + if (status & XILINX_PCIE_INTR_CFG_TIMEOUT) + dev_warn(dev, "ECAM access timeout\n"); + + if (status & XILINX_PCIE_INTR_CORRECTABLE) { + dev_warn(dev, "Correctable error message\n"); + xilinx_pcie_clear_err_interrupts(pcie); + } + + if (status & XILINX_PCIE_INTR_NONFATAL) { + dev_warn(dev, "Non fatal error message\n"); + xilinx_pcie_clear_err_interrupts(pcie); + } + + if (status & XILINX_PCIE_INTR_FATAL) { + dev_warn(dev, "Fatal error message\n"); + xilinx_pcie_clear_err_interrupts(pcie); + } + + if (status & (XILINX_PCIE_INTR_INTX | XILINX_PCIE_INTR_MSI)) { + struct irq_domain *domain; + + val = pcie_read(pcie, XILINX_PCIE_REG_RPIFR1); + + /* Check whether interrupt valid */ + if (!(val & XILINX_PCIE_RPIFR1_INTR_VALID)) { + dev_warn(dev, "RP Intr FIFO1 read error\n"); + goto error; + } + + /* Decode the IRQ number */ + if (val & XILINX_PCIE_RPIFR1_MSI_INTR) { + val = pcie_read(pcie, XILINX_PCIE_REG_RPIFR2) & + XILINX_PCIE_RPIFR2_MSG_DATA; + domain = pcie->msi_domain; + } else { + val = (val & XILINX_PCIE_RPIFR1_INTR_MASK) >> + XILINX_PCIE_RPIFR1_INTR_SHIFT; + domain = pcie->leg_domain; + } + + /* Clear interrupt FIFO register 1 */ + pcie_write(pcie, XILINX_PCIE_RPIFR1_ALL_MASK, + XILINX_PCIE_REG_RPIFR1); + + generic_handle_domain_irq(domain, val); + } + + if (status & XILINX_PCIE_INTR_SLV_UNSUPP) + dev_warn(dev, "Slave unsupported request\n"); + + if (status & XILINX_PCIE_INTR_SLV_UNEXP) + dev_warn(dev, "Slave unexpected completion\n"); + + if (status & XILINX_PCIE_INTR_SLV_COMPL) + dev_warn(dev, "Slave completion timeout\n"); + + if (status & XILINX_PCIE_INTR_SLV_ERRP) + dev_warn(dev, "Slave Error Poison\n"); + + if (status & XILINX_PCIE_INTR_SLV_CMPABT) + dev_warn(dev, "Slave Completer Abort\n"); + + if (status & XILINX_PCIE_INTR_SLV_ILLBUR) + dev_warn(dev, "Slave Illegal Burst\n"); + + if (status & XILINX_PCIE_INTR_MST_DECERR) + dev_warn(dev, "Master decode error\n"); + + if (status & XILINX_PCIE_INTR_MST_SLVERR) + dev_warn(dev, "Master slave error\n"); + + if (status & XILINX_PCIE_INTR_MST_ERRP) + dev_warn(dev, "Master error poison\n"); + +error: + /* Clear the Interrupt Decode register */ + pcie_write(pcie, status, XILINX_PCIE_REG_IDR); + + return IRQ_HANDLED; +} + +/** + * xilinx_pcie_init_irq_domain - Initialize IRQ domain + * @pcie: PCIe port information + * + * Return: '0' on success and error value on failure + */ +static int xilinx_pcie_init_irq_domain(struct xilinx_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct device_node *pcie_intc_node; + int ret; + + /* Setup INTx */ + pcie_intc_node = of_get_next_child(dev->of_node, NULL); + if (!pcie_intc_node) { + dev_err(dev, "No PCIe Intc node found\n"); + return -ENODEV; + } + + pcie->leg_domain = irq_domain_create_linear(of_fwnode_handle(pcie_intc_node), PCI_NUM_INTX, + &intx_domain_ops, pcie); + of_node_put(pcie_intc_node); + if (!pcie->leg_domain) { + dev_err(dev, "Failed to get a INTx IRQ domain\n"); + return -ENODEV; + } + + /* Setup MSI */ + if (IS_ENABLED(CONFIG_PCI_MSI)) { + phys_addr_t pa = ALIGN_DOWN(virt_to_phys(pcie), SZ_4K); + + ret = xilinx_allocate_msi_domains(pcie); + if (ret) { + irq_domain_remove(pcie->leg_domain); + return ret; + } + + pcie_write(pcie, upper_32_bits(pa), XILINX_PCIE_REG_MSIBASE1); + pcie_write(pcie, lower_32_bits(pa), XILINX_PCIE_REG_MSIBASE2); + } + + return 0; +} + +/** + * xilinx_pcie_init_port - Initialize hardware + * @pcie: PCIe port information + */ +static void xilinx_pcie_init_port(struct xilinx_pcie *pcie) +{ + struct device *dev = pcie->dev; + + if (xilinx_pcie_link_up(pcie)) + dev_info(dev, "PCIe Link is UP\n"); + else + dev_info(dev, "PCIe Link is DOWN\n"); + + /* Disable all interrupts */ + pcie_write(pcie, ~XILINX_PCIE_IDR_ALL_MASK, + XILINX_PCIE_REG_IMR); + + /* Clear pending interrupts */ + pcie_write(pcie, pcie_read(pcie, XILINX_PCIE_REG_IDR) & + XILINX_PCIE_IMR_ALL_MASK, + XILINX_PCIE_REG_IDR); + + /* Enable all interrupts we handle */ + pcie_write(pcie, XILINX_PCIE_IMR_ENABLE_MASK, XILINX_PCIE_REG_IMR); + + /* Enable the Bridge enable bit */ + pcie_write(pcie, pcie_read(pcie, XILINX_PCIE_REG_RPSC) | + XILINX_PCIE_REG_RPSC_BEN, + XILINX_PCIE_REG_RPSC); +} + +/** + * xilinx_pcie_parse_dt - Parse Device tree + * @pcie: PCIe port information + * + * Return: '0' on success and error value on failure + */ +static int xilinx_pcie_parse_dt(struct xilinx_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct device_node *node = dev->of_node; + struct resource regs; + unsigned int irq; + int err; + + err = of_address_to_resource(node, 0, ®s); + if (err) { + dev_err(dev, "missing \"reg\" property\n"); + return err; + } + + pcie->reg_base = devm_pci_remap_cfg_resource(dev, ®s); + if (IS_ERR(pcie->reg_base)) + return PTR_ERR(pcie->reg_base); + + irq = irq_of_parse_and_map(node, 0); + err = devm_request_irq(dev, irq, xilinx_pcie_intr_handler, + IRQF_SHARED | IRQF_NO_THREAD, + "xilinx-pcie", pcie); + if (err) { + dev_err(dev, "unable to request irq %d\n", irq); + return err; + } + + return 0; +} + +/** + * xilinx_pcie_probe - Probe function + * @pdev: Platform device pointer + * + * Return: '0' on success and error value on failure + */ +static int xilinx_pcie_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct xilinx_pcie *pcie; + struct pci_host_bridge *bridge; + int err; + + if (!dev->of_node) + return -ENODEV; + + bridge = devm_pci_alloc_host_bridge(dev, sizeof(*pcie)); + if (!bridge) + return -ENODEV; + + pcie = pci_host_bridge_priv(bridge); + mutex_init(&pcie->map_lock); + pcie->dev = dev; + + err = xilinx_pcie_parse_dt(pcie); + if (err) { + dev_err(dev, "Parsing DT failed\n"); + return err; + } + + xilinx_pcie_init_port(pcie); + + err = xilinx_pcie_init_irq_domain(pcie); + if (err) { + dev_err(dev, "Failed creating IRQ Domain\n"); + return err; + } + + bridge->sysdata = pcie; + bridge->ops = &xilinx_pcie_ops; + + err = pci_host_probe(bridge); + if (err) + xilinx_free_irq_domains(pcie); + + return err; +} + +static const struct of_device_id xilinx_pcie_of_match[] = { + { .compatible = "xlnx,axi-pcie-host-1.00.a", }, + {} +}; + +static struct platform_driver xilinx_pcie_driver = { + .driver = { + .name = "xilinx-pcie", + .of_match_table = xilinx_pcie_of_match, + .suppress_bind_attrs = true, + }, + .probe = xilinx_pcie_probe, +}; +builtin_platform_driver(xilinx_pcie_driver); diff --git a/drivers/pci/controller/plda/Kconfig b/drivers/pci/controller/plda/Kconfig new file mode 100644 index 000000000..621201011 --- /dev/null +++ b/drivers/pci/controller/plda/Kconfig @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "PLDA-based PCIe controllers" + depends on PCI + +config PCIE_PLDA_HOST + bool + select IRQ_MSI_LIB + +config PCIE_MICROCHIP_HOST + tristate "Microchip AXI PCIe controller" + depends on PCI_MSI && OF + select PCI_HOST_COMMON + select PCIE_PLDA_HOST + help + Say Y here if you want kernel to support the Microchip AXI PCIe + Host Bridge driver. + +config PCIE_STARFIVE_HOST + tristate "StarFive PCIe host controller" + depends on PCI_MSI && OF + depends on ARCH_STARFIVE || COMPILE_TEST + select PCIE_PLDA_HOST + help + Say Y here if you want to support the StarFive PCIe controller in + host mode. StarFive PCIe controller uses PLDA PCIe core. + + If you choose to build this driver as module it will be dynamically + linked and module will be called pcie-starfive.ko. + +endmenu diff --git a/drivers/pci/controller/plda/Makefile b/drivers/pci/controller/plda/Makefile new file mode 100644 index 000000000..0ac6851be --- /dev/null +++ b/drivers/pci/controller/plda/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_PCIE_PLDA_HOST) += pcie-plda-host.o +obj-$(CONFIG_PCIE_MICROCHIP_HOST) += pcie-microchip-host.o +obj-$(CONFIG_PCIE_STARFIVE_HOST) += pcie-starfive.o diff --git a/drivers/pci/controller/plda/pcie-microchip-host.c b/drivers/pci/controller/plda/pcie-microchip-host.c new file mode 100644 index 000000000..24bbf93b8 --- /dev/null +++ b/drivers/pci/controller/plda/pcie-microchip-host.c @@ -0,0 +1,834 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Microchip AXI PCIe Bridge host controller driver + * + * Copyright (c) 2018 - 2020 Microchip Corporation. All rights reserved. + * + * Author: Daire McNamara <daire.mcnamara@microchip.com> + */ + +#include <linux/align.h> +#include <linux/bits.h> +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqdomain.h> +#include <linux/log2.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/of_address.h> +#include <linux/of_pci.h> +#include <linux/pci-ecam.h> +#include <linux/platform_device.h> +#include <linux/wordpart.h> + +#include "../../pci.h" +#include "../pci-host-common.h" +#include "pcie-plda.h" + +#define MC_MAX_NUM_INBOUND_WINDOWS 8 +#define MPFS_NC_BOUNCE_ADDR 0x80000000 + +/* PCIe Bridge Phy and Controller Phy offsets */ +#define MC_PCIE1_BRIDGE_ADDR 0x00008000u +#define MC_PCIE1_CTRL_ADDR 0x0000a000u + +/* PCIe Controller Phy Regs */ +#define SEC_ERROR_EVENT_CNT 0x20 +#define DED_ERROR_EVENT_CNT 0x24 +#define SEC_ERROR_INT 0x28 +#define SEC_ERROR_INT_TX_RAM_SEC_ERR_INT GENMASK(3, 0) +#define SEC_ERROR_INT_RX_RAM_SEC_ERR_INT GENMASK(7, 4) +#define SEC_ERROR_INT_PCIE2AXI_RAM_SEC_ERR_INT GENMASK(11, 8) +#define SEC_ERROR_INT_AXI2PCIE_RAM_SEC_ERR_INT GENMASK(15, 12) +#define SEC_ERROR_INT_ALL_RAM_SEC_ERR_INT GENMASK(15, 0) +#define NUM_SEC_ERROR_INTS (4) +#define SEC_ERROR_INT_MASK 0x2c +#define DED_ERROR_INT 0x30 +#define DED_ERROR_INT_TX_RAM_DED_ERR_INT GENMASK(3, 0) +#define DED_ERROR_INT_RX_RAM_DED_ERR_INT GENMASK(7, 4) +#define DED_ERROR_INT_PCIE2AXI_RAM_DED_ERR_INT GENMASK(11, 8) +#define DED_ERROR_INT_AXI2PCIE_RAM_DED_ERR_INT GENMASK(15, 12) +#define DED_ERROR_INT_ALL_RAM_DED_ERR_INT GENMASK(15, 0) +#define NUM_DED_ERROR_INTS (4) +#define DED_ERROR_INT_MASK 0x34 +#define ECC_CONTROL 0x38 +#define ECC_CONTROL_TX_RAM_INJ_ERROR_0 BIT(0) +#define ECC_CONTROL_TX_RAM_INJ_ERROR_1 BIT(1) +#define ECC_CONTROL_TX_RAM_INJ_ERROR_2 BIT(2) +#define ECC_CONTROL_TX_RAM_INJ_ERROR_3 BIT(3) +#define ECC_CONTROL_RX_RAM_INJ_ERROR_0 BIT(4) +#define ECC_CONTROL_RX_RAM_INJ_ERROR_1 BIT(5) +#define ECC_CONTROL_RX_RAM_INJ_ERROR_2 BIT(6) +#define ECC_CONTROL_RX_RAM_INJ_ERROR_3 BIT(7) +#define ECC_CONTROL_PCIE2AXI_RAM_INJ_ERROR_0 BIT(8) +#define ECC_CONTROL_PCIE2AXI_RAM_INJ_ERROR_1 BIT(9) +#define ECC_CONTROL_PCIE2AXI_RAM_INJ_ERROR_2 BIT(10) +#define ECC_CONTROL_PCIE2AXI_RAM_INJ_ERROR_3 BIT(11) +#define ECC_CONTROL_AXI2PCIE_RAM_INJ_ERROR_0 BIT(12) +#define ECC_CONTROL_AXI2PCIE_RAM_INJ_ERROR_1 BIT(13) +#define ECC_CONTROL_AXI2PCIE_RAM_INJ_ERROR_2 BIT(14) +#define ECC_CONTROL_AXI2PCIE_RAM_INJ_ERROR_3 BIT(15) +#define ECC_CONTROL_TX_RAM_ECC_BYPASS BIT(24) +#define ECC_CONTROL_RX_RAM_ECC_BYPASS BIT(25) +#define ECC_CONTROL_PCIE2AXI_RAM_ECC_BYPASS BIT(26) +#define ECC_CONTROL_AXI2PCIE_RAM_ECC_BYPASS BIT(27) +#define PCIE_EVENT_INT 0x14c +#define PCIE_EVENT_INT_L2_EXIT_INT BIT(0) +#define PCIE_EVENT_INT_HOTRST_EXIT_INT BIT(1) +#define PCIE_EVENT_INT_DLUP_EXIT_INT BIT(2) +#define PCIE_EVENT_INT_MASK GENMASK(2, 0) +#define PCIE_EVENT_INT_L2_EXIT_INT_MASK BIT(16) +#define PCIE_EVENT_INT_HOTRST_EXIT_INT_MASK BIT(17) +#define PCIE_EVENT_INT_DLUP_EXIT_INT_MASK BIT(18) +#define PCIE_EVENT_INT_ENB_MASK GENMASK(18, 16) +#define PCIE_EVENT_INT_ENB_SHIFT 16 +#define NUM_PCIE_EVENTS (3) + +/* PCIe Config space MSI capability structure */ +#define MC_MSI_CAP_CTRL_OFFSET 0xe0u + +/* Events */ +#define EVENT_PCIE_L2_EXIT 0 +#define EVENT_PCIE_HOTRST_EXIT 1 +#define EVENT_PCIE_DLUP_EXIT 2 +#define EVENT_SEC_TX_RAM_SEC_ERR 3 +#define EVENT_SEC_RX_RAM_SEC_ERR 4 +#define EVENT_SEC_PCIE2AXI_RAM_SEC_ERR 5 +#define EVENT_SEC_AXI2PCIE_RAM_SEC_ERR 6 +#define EVENT_DED_TX_RAM_DED_ERR 7 +#define EVENT_DED_RX_RAM_DED_ERR 8 +#define EVENT_DED_PCIE2AXI_RAM_DED_ERR 9 +#define EVENT_DED_AXI2PCIE_RAM_DED_ERR 10 +#define EVENT_LOCAL_DMA_END_ENGINE_0 11 +#define EVENT_LOCAL_DMA_END_ENGINE_1 12 +#define EVENT_LOCAL_DMA_ERROR_ENGINE_0 13 +#define EVENT_LOCAL_DMA_ERROR_ENGINE_1 14 +#define NUM_MC_EVENTS 15 +#define EVENT_LOCAL_A_ATR_EVT_POST_ERR (NUM_MC_EVENTS + PLDA_AXI_POST_ERR) +#define EVENT_LOCAL_A_ATR_EVT_FETCH_ERR (NUM_MC_EVENTS + PLDA_AXI_FETCH_ERR) +#define EVENT_LOCAL_A_ATR_EVT_DISCARD_ERR (NUM_MC_EVENTS + PLDA_AXI_DISCARD_ERR) +#define EVENT_LOCAL_A_ATR_EVT_DOORBELL (NUM_MC_EVENTS + PLDA_AXI_DOORBELL) +#define EVENT_LOCAL_P_ATR_EVT_POST_ERR (NUM_MC_EVENTS + PLDA_PCIE_POST_ERR) +#define EVENT_LOCAL_P_ATR_EVT_FETCH_ERR (NUM_MC_EVENTS + PLDA_PCIE_FETCH_ERR) +#define EVENT_LOCAL_P_ATR_EVT_DISCARD_ERR (NUM_MC_EVENTS + PLDA_PCIE_DISCARD_ERR) +#define EVENT_LOCAL_P_ATR_EVT_DOORBELL (NUM_MC_EVENTS + PLDA_PCIE_DOORBELL) +#define EVENT_LOCAL_PM_MSI_INT_INTX (NUM_MC_EVENTS + PLDA_INTX) +#define EVENT_LOCAL_PM_MSI_INT_MSI (NUM_MC_EVENTS + PLDA_MSI) +#define EVENT_LOCAL_PM_MSI_INT_AER_EVT (NUM_MC_EVENTS + PLDA_AER_EVENT) +#define EVENT_LOCAL_PM_MSI_INT_EVENTS (NUM_MC_EVENTS + PLDA_MISC_EVENTS) +#define EVENT_LOCAL_PM_MSI_INT_SYS_ERR (NUM_MC_EVENTS + PLDA_SYS_ERR) +#define NUM_EVENTS (NUM_MC_EVENTS + PLDA_INT_EVENT_NUM) + +#define PCIE_EVENT_CAUSE(x, s) \ + [EVENT_PCIE_ ## x] = { __stringify(x), s } + +#define SEC_ERROR_CAUSE(x, s) \ + [EVENT_SEC_ ## x] = { __stringify(x), s } + +#define DED_ERROR_CAUSE(x, s) \ + [EVENT_DED_ ## x] = { __stringify(x), s } + +#define LOCAL_EVENT_CAUSE(x, s) \ + [EVENT_LOCAL_ ## x] = { __stringify(x), s } + +#define PCIE_EVENT(x) \ + .offset = PCIE_EVENT_INT, \ + .mask_offset = PCIE_EVENT_INT, \ + .mask_high = 1, \ + .mask = PCIE_EVENT_INT_ ## x ## _INT, \ + .enb_mask = PCIE_EVENT_INT_ENB_MASK + +#define SEC_EVENT(x) \ + .offset = SEC_ERROR_INT, \ + .mask_offset = SEC_ERROR_INT_MASK, \ + .mask = SEC_ERROR_INT_ ## x ## _INT, \ + .mask_high = 1, \ + .enb_mask = 0 + +#define DED_EVENT(x) \ + .offset = DED_ERROR_INT, \ + .mask_offset = DED_ERROR_INT_MASK, \ + .mask_high = 1, \ + .mask = DED_ERROR_INT_ ## x ## _INT, \ + .enb_mask = 0 + +#define LOCAL_EVENT(x) \ + .offset = ISTATUS_LOCAL, \ + .mask_offset = IMASK_LOCAL, \ + .mask_high = 0, \ + .mask = x ## _MASK, \ + .enb_mask = 0 + +#define PCIE_EVENT_TO_EVENT_MAP(x) \ + { PCIE_EVENT_INT_ ## x ## _INT, EVENT_PCIE_ ## x } + +#define SEC_ERROR_TO_EVENT_MAP(x) \ + { SEC_ERROR_INT_ ## x ## _INT, EVENT_SEC_ ## x } + +#define DED_ERROR_TO_EVENT_MAP(x) \ + { DED_ERROR_INT_ ## x ## _INT, EVENT_DED_ ## x } + +#define LOCAL_STATUS_TO_EVENT_MAP(x) \ + { x ## _MASK, EVENT_LOCAL_ ## x } + +struct event_map { + u32 reg_mask; + u32 event_bit; +}; + + +struct mc_pcie { + struct plda_pcie_rp plda; + void __iomem *bridge_base_addr; + void __iomem *ctrl_base_addr; +}; + +struct cause { + const char *sym; + const char *str; +}; + +static const struct cause event_cause[NUM_EVENTS] = { + PCIE_EVENT_CAUSE(L2_EXIT, "L2 exit event"), + PCIE_EVENT_CAUSE(HOTRST_EXIT, "Hot reset exit event"), + PCIE_EVENT_CAUSE(DLUP_EXIT, "DLUP exit event"), + SEC_ERROR_CAUSE(TX_RAM_SEC_ERR, "sec error in tx buffer"), + SEC_ERROR_CAUSE(RX_RAM_SEC_ERR, "sec error in rx buffer"), + SEC_ERROR_CAUSE(PCIE2AXI_RAM_SEC_ERR, "sec error in pcie2axi buffer"), + SEC_ERROR_CAUSE(AXI2PCIE_RAM_SEC_ERR, "sec error in axi2pcie buffer"), + DED_ERROR_CAUSE(TX_RAM_DED_ERR, "ded error in tx buffer"), + DED_ERROR_CAUSE(RX_RAM_DED_ERR, "ded error in rx buffer"), + DED_ERROR_CAUSE(PCIE2AXI_RAM_DED_ERR, "ded error in pcie2axi buffer"), + DED_ERROR_CAUSE(AXI2PCIE_RAM_DED_ERR, "ded error in axi2pcie buffer"), + LOCAL_EVENT_CAUSE(DMA_ERROR_ENGINE_0, "dma engine 0 error"), + LOCAL_EVENT_CAUSE(DMA_ERROR_ENGINE_1, "dma engine 1 error"), + LOCAL_EVENT_CAUSE(A_ATR_EVT_POST_ERR, "axi write request error"), + LOCAL_EVENT_CAUSE(A_ATR_EVT_FETCH_ERR, "axi read request error"), + LOCAL_EVENT_CAUSE(A_ATR_EVT_DISCARD_ERR, "axi read timeout"), + LOCAL_EVENT_CAUSE(P_ATR_EVT_POST_ERR, "pcie write request error"), + LOCAL_EVENT_CAUSE(P_ATR_EVT_FETCH_ERR, "pcie read request error"), + LOCAL_EVENT_CAUSE(P_ATR_EVT_DISCARD_ERR, "pcie read timeout"), + LOCAL_EVENT_CAUSE(PM_MSI_INT_AER_EVT, "aer event"), + LOCAL_EVENT_CAUSE(PM_MSI_INT_EVENTS, "pm/ltr/hotplug event"), + LOCAL_EVENT_CAUSE(PM_MSI_INT_SYS_ERR, "system error"), +}; + +static struct event_map pcie_event_to_event[] = { + PCIE_EVENT_TO_EVENT_MAP(L2_EXIT), + PCIE_EVENT_TO_EVENT_MAP(HOTRST_EXIT), + PCIE_EVENT_TO_EVENT_MAP(DLUP_EXIT), +}; + +static struct event_map sec_error_to_event[] = { + SEC_ERROR_TO_EVENT_MAP(TX_RAM_SEC_ERR), + SEC_ERROR_TO_EVENT_MAP(RX_RAM_SEC_ERR), + SEC_ERROR_TO_EVENT_MAP(PCIE2AXI_RAM_SEC_ERR), + SEC_ERROR_TO_EVENT_MAP(AXI2PCIE_RAM_SEC_ERR), +}; + +static struct event_map ded_error_to_event[] = { + DED_ERROR_TO_EVENT_MAP(TX_RAM_DED_ERR), + DED_ERROR_TO_EVENT_MAP(RX_RAM_DED_ERR), + DED_ERROR_TO_EVENT_MAP(PCIE2AXI_RAM_DED_ERR), + DED_ERROR_TO_EVENT_MAP(AXI2PCIE_RAM_DED_ERR), +}; + +static struct event_map local_status_to_event[] = { + LOCAL_STATUS_TO_EVENT_MAP(DMA_END_ENGINE_0), + LOCAL_STATUS_TO_EVENT_MAP(DMA_END_ENGINE_1), + LOCAL_STATUS_TO_EVENT_MAP(DMA_ERROR_ENGINE_0), + LOCAL_STATUS_TO_EVENT_MAP(DMA_ERROR_ENGINE_1), + LOCAL_STATUS_TO_EVENT_MAP(A_ATR_EVT_POST_ERR), + LOCAL_STATUS_TO_EVENT_MAP(A_ATR_EVT_FETCH_ERR), + LOCAL_STATUS_TO_EVENT_MAP(A_ATR_EVT_DISCARD_ERR), + LOCAL_STATUS_TO_EVENT_MAP(A_ATR_EVT_DOORBELL), + LOCAL_STATUS_TO_EVENT_MAP(P_ATR_EVT_POST_ERR), + LOCAL_STATUS_TO_EVENT_MAP(P_ATR_EVT_FETCH_ERR), + LOCAL_STATUS_TO_EVENT_MAP(P_ATR_EVT_DISCARD_ERR), + LOCAL_STATUS_TO_EVENT_MAP(P_ATR_EVT_DOORBELL), + LOCAL_STATUS_TO_EVENT_MAP(PM_MSI_INT_INTX), + LOCAL_STATUS_TO_EVENT_MAP(PM_MSI_INT_MSI), + LOCAL_STATUS_TO_EVENT_MAP(PM_MSI_INT_AER_EVT), + LOCAL_STATUS_TO_EVENT_MAP(PM_MSI_INT_EVENTS), + LOCAL_STATUS_TO_EVENT_MAP(PM_MSI_INT_SYS_ERR), +}; + +static struct { + u32 offset; + u32 mask; + u32 shift; + u32 enb_mask; + u32 mask_high; + u32 mask_offset; +} event_descs[] = { + { PCIE_EVENT(L2_EXIT) }, + { PCIE_EVENT(HOTRST_EXIT) }, + { PCIE_EVENT(DLUP_EXIT) }, + { SEC_EVENT(TX_RAM_SEC_ERR) }, + { SEC_EVENT(RX_RAM_SEC_ERR) }, + { SEC_EVENT(PCIE2AXI_RAM_SEC_ERR) }, + { SEC_EVENT(AXI2PCIE_RAM_SEC_ERR) }, + { DED_EVENT(TX_RAM_DED_ERR) }, + { DED_EVENT(RX_RAM_DED_ERR) }, + { DED_EVENT(PCIE2AXI_RAM_DED_ERR) }, + { DED_EVENT(AXI2PCIE_RAM_DED_ERR) }, + { LOCAL_EVENT(DMA_END_ENGINE_0) }, + { LOCAL_EVENT(DMA_END_ENGINE_1) }, + { LOCAL_EVENT(DMA_ERROR_ENGINE_0) }, + { LOCAL_EVENT(DMA_ERROR_ENGINE_1) }, + { LOCAL_EVENT(A_ATR_EVT_POST_ERR) }, + { LOCAL_EVENT(A_ATR_EVT_FETCH_ERR) }, + { LOCAL_EVENT(A_ATR_EVT_DISCARD_ERR) }, + { LOCAL_EVENT(A_ATR_EVT_DOORBELL) }, + { LOCAL_EVENT(P_ATR_EVT_POST_ERR) }, + { LOCAL_EVENT(P_ATR_EVT_FETCH_ERR) }, + { LOCAL_EVENT(P_ATR_EVT_DISCARD_ERR) }, + { LOCAL_EVENT(P_ATR_EVT_DOORBELL) }, + { LOCAL_EVENT(PM_MSI_INT_INTX) }, + { LOCAL_EVENT(PM_MSI_INT_MSI) }, + { LOCAL_EVENT(PM_MSI_INT_AER_EVT) }, + { LOCAL_EVENT(PM_MSI_INT_EVENTS) }, + { LOCAL_EVENT(PM_MSI_INT_SYS_ERR) }, +}; + +static char poss_clks[][5] = { "fic0", "fic1", "fic2", "fic3" }; + +static struct mc_pcie *port; + +static void mc_pcie_enable_msi(struct mc_pcie *port, void __iomem *ecam) +{ + struct plda_msi *msi = &port->plda.msi; + u16 reg; + u8 queue_size; + + /* Fixup MSI enable flag */ + reg = readw_relaxed(ecam + MC_MSI_CAP_CTRL_OFFSET + PCI_MSI_FLAGS); + reg |= PCI_MSI_FLAGS_ENABLE; + writew_relaxed(reg, ecam + MC_MSI_CAP_CTRL_OFFSET + PCI_MSI_FLAGS); + + /* Fixup PCI MSI queue flags */ + queue_size = FIELD_GET(PCI_MSI_FLAGS_QMASK, reg); + reg |= FIELD_PREP(PCI_MSI_FLAGS_QSIZE, queue_size); + writew_relaxed(reg, ecam + MC_MSI_CAP_CTRL_OFFSET + PCI_MSI_FLAGS); + + /* Fixup MSI addr fields */ + writel_relaxed(lower_32_bits(msi->vector_phy), + ecam + MC_MSI_CAP_CTRL_OFFSET + PCI_MSI_ADDRESS_LO); + writel_relaxed(upper_32_bits(msi->vector_phy), + ecam + MC_MSI_CAP_CTRL_OFFSET + PCI_MSI_ADDRESS_HI); +} + +static inline u32 reg_to_event(u32 reg, struct event_map field) +{ + return (reg & field.reg_mask) ? BIT(field.event_bit) : 0; +} + +static u32 pcie_events(struct mc_pcie *port) +{ + u32 reg = readl_relaxed(port->ctrl_base_addr + PCIE_EVENT_INT); + u32 val = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(pcie_event_to_event); i++) + val |= reg_to_event(reg, pcie_event_to_event[i]); + + return val; +} + +static u32 sec_errors(struct mc_pcie *port) +{ + u32 reg = readl_relaxed(port->ctrl_base_addr + SEC_ERROR_INT); + u32 val = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(sec_error_to_event); i++) + val |= reg_to_event(reg, sec_error_to_event[i]); + + return val; +} + +static u32 ded_errors(struct mc_pcie *port) +{ + u32 reg = readl_relaxed(port->ctrl_base_addr + DED_ERROR_INT); + u32 val = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(ded_error_to_event); i++) + val |= reg_to_event(reg, ded_error_to_event[i]); + + return val; +} + +static u32 local_events(struct mc_pcie *port) +{ + u32 reg = readl_relaxed(port->bridge_base_addr + ISTATUS_LOCAL); + u32 val = 0; + int i; + + for (i = 0; i < ARRAY_SIZE(local_status_to_event); i++) + val |= reg_to_event(reg, local_status_to_event[i]); + + return val; +} + +static u32 mc_get_events(struct plda_pcie_rp *port) +{ + struct mc_pcie *mc_port = container_of(port, struct mc_pcie, plda); + u32 events = 0; + + events |= pcie_events(mc_port); + events |= sec_errors(mc_port); + events |= ded_errors(mc_port); + events |= local_events(mc_port); + + return events; +} + +static irqreturn_t mc_event_handler(int irq, void *dev_id) +{ + struct plda_pcie_rp *port = dev_id; + struct device *dev = port->dev; + struct irq_data *data; + + data = irq_domain_get_irq_data(port->event_domain, irq); + + if (event_cause[data->hwirq].str) + dev_err_ratelimited(dev, "%s\n", event_cause[data->hwirq].str); + else + dev_err_ratelimited(dev, "bad event IRQ %ld\n", data->hwirq); + + return IRQ_HANDLED; +} + +static void mc_ack_event_irq(struct irq_data *data) +{ + struct plda_pcie_rp *port = irq_data_get_irq_chip_data(data); + struct mc_pcie *mc_port = container_of(port, struct mc_pcie, plda); + u32 event = data->hwirq; + void __iomem *addr; + u32 mask; + + if (event_descs[event].offset == ISTATUS_LOCAL) + addr = mc_port->bridge_base_addr; + else + addr = mc_port->ctrl_base_addr; + + addr += event_descs[event].offset; + mask = event_descs[event].mask; + mask |= event_descs[event].enb_mask; + + writel_relaxed(mask, addr); +} + +static void mc_mask_event_irq(struct irq_data *data) +{ + struct plda_pcie_rp *port = irq_data_get_irq_chip_data(data); + struct mc_pcie *mc_port = container_of(port, struct mc_pcie, plda); + u32 event = data->hwirq; + void __iomem *addr; + u32 mask; + u32 val; + + if (event_descs[event].offset == ISTATUS_LOCAL) + addr = mc_port->bridge_base_addr; + else + addr = mc_port->ctrl_base_addr; + + addr += event_descs[event].mask_offset; + mask = event_descs[event].mask; + if (event_descs[event].enb_mask) { + mask <<= PCIE_EVENT_INT_ENB_SHIFT; + mask &= PCIE_EVENT_INT_ENB_MASK; + } + + if (!event_descs[event].mask_high) + mask = ~mask; + + raw_spin_lock(&port->lock); + val = readl_relaxed(addr); + if (event_descs[event].mask_high) + val |= mask; + else + val &= mask; + + writel_relaxed(val, addr); + raw_spin_unlock(&port->lock); +} + +static void mc_unmask_event_irq(struct irq_data *data) +{ + struct plda_pcie_rp *port = irq_data_get_irq_chip_data(data); + struct mc_pcie *mc_port = container_of(port, struct mc_pcie, plda); + u32 event = data->hwirq; + void __iomem *addr; + u32 mask; + u32 val; + + if (event_descs[event].offset == ISTATUS_LOCAL) + addr = mc_port->bridge_base_addr; + else + addr = mc_port->ctrl_base_addr; + + addr += event_descs[event].mask_offset; + mask = event_descs[event].mask; + + if (event_descs[event].enb_mask) + mask <<= PCIE_EVENT_INT_ENB_SHIFT; + + if (event_descs[event].mask_high) + mask = ~mask; + + if (event_descs[event].enb_mask) + mask &= PCIE_EVENT_INT_ENB_MASK; + + raw_spin_lock(&port->lock); + val = readl_relaxed(addr); + if (event_descs[event].mask_high) + val &= mask; + else + val |= mask; + writel_relaxed(val, addr); + raw_spin_unlock(&port->lock); +} + +static struct irq_chip mc_event_irq_chip = { + .name = "Microchip PCIe EVENT", + .irq_ack = mc_ack_event_irq, + .irq_mask = mc_mask_event_irq, + .irq_unmask = mc_unmask_event_irq, +}; + +static inline void mc_pcie_deinit_clk(void *data) +{ + struct clk *clk = data; + + clk_disable_unprepare(clk); +} + +static inline struct clk *mc_pcie_init_clk(struct device *dev, const char *id) +{ + struct clk *clk; + int ret; + + clk = devm_clk_get_optional(dev, id); + if (IS_ERR(clk)) + return clk; + if (!clk) + return clk; + + ret = clk_prepare_enable(clk); + if (ret) + return ERR_PTR(ret); + + devm_add_action_or_reset(dev, mc_pcie_deinit_clk, clk); + + return clk; +} + +static int mc_pcie_init_clks(struct device *dev) +{ + int i; + struct clk *fic; + + /* + * PCIe may be clocked via Fabric Interface using between 1 and 4 + * clocks. Scan DT for clocks and enable them if present + */ + for (i = 0; i < ARRAY_SIZE(poss_clks); i++) { + fic = mc_pcie_init_clk(dev, poss_clks[i]); + if (IS_ERR(fic)) + return PTR_ERR(fic); + } + + return 0; +} + +static int mc_request_event_irq(struct plda_pcie_rp *plda, int event_irq, + int event) +{ + return devm_request_irq(plda->dev, event_irq, mc_event_handler, + 0, event_cause[event].sym, plda); +} + +static const struct plda_event_ops mc_event_ops = { + .get_events = mc_get_events, +}; + +static const struct plda_event mc_event = { + .request_event_irq = mc_request_event_irq, + .intx_event = EVENT_LOCAL_PM_MSI_INT_INTX, + .msi_event = EVENT_LOCAL_PM_MSI_INT_MSI, +}; + +static inline void mc_clear_secs(struct mc_pcie *port) +{ + writel_relaxed(SEC_ERROR_INT_ALL_RAM_SEC_ERR_INT, + port->ctrl_base_addr + SEC_ERROR_INT); + writel_relaxed(0, port->ctrl_base_addr + SEC_ERROR_EVENT_CNT); +} + +static inline void mc_clear_deds(struct mc_pcie *port) +{ + writel_relaxed(DED_ERROR_INT_ALL_RAM_DED_ERR_INT, + port->ctrl_base_addr + DED_ERROR_INT); + writel_relaxed(0, port->ctrl_base_addr + DED_ERROR_EVENT_CNT); +} + +static void mc_disable_interrupts(struct mc_pcie *port) +{ + u32 val; + + /* Ensure ECC bypass is enabled */ + val = ECC_CONTROL_TX_RAM_ECC_BYPASS | + ECC_CONTROL_RX_RAM_ECC_BYPASS | + ECC_CONTROL_PCIE2AXI_RAM_ECC_BYPASS | + ECC_CONTROL_AXI2PCIE_RAM_ECC_BYPASS; + writel_relaxed(val, port->ctrl_base_addr + ECC_CONTROL); + + /* Disable SEC errors and clear any outstanding */ + writel_relaxed(SEC_ERROR_INT_ALL_RAM_SEC_ERR_INT, + port->ctrl_base_addr + SEC_ERROR_INT_MASK); + mc_clear_secs(port); + + /* Disable DED errors and clear any outstanding */ + writel_relaxed(DED_ERROR_INT_ALL_RAM_DED_ERR_INT, + port->ctrl_base_addr + DED_ERROR_INT_MASK); + mc_clear_deds(port); + + /* Disable local interrupts and clear any outstanding */ + writel_relaxed(0, port->bridge_base_addr + IMASK_LOCAL); + writel_relaxed(GENMASK(31, 0), port->bridge_base_addr + ISTATUS_LOCAL); + writel_relaxed(GENMASK(31, 0), port->bridge_base_addr + ISTATUS_MSI); + + /* Disable PCIe events and clear any outstanding */ + val = PCIE_EVENT_INT_L2_EXIT_INT | + PCIE_EVENT_INT_HOTRST_EXIT_INT | + PCIE_EVENT_INT_DLUP_EXIT_INT | + PCIE_EVENT_INT_L2_EXIT_INT_MASK | + PCIE_EVENT_INT_HOTRST_EXIT_INT_MASK | + PCIE_EVENT_INT_DLUP_EXIT_INT_MASK; + writel_relaxed(val, port->ctrl_base_addr + PCIE_EVENT_INT); + + /* Disable host interrupts and clear any outstanding */ + writel_relaxed(0, port->bridge_base_addr + IMASK_HOST); + writel_relaxed(GENMASK(31, 0), port->bridge_base_addr + ISTATUS_HOST); +} + +static void mc_pcie_setup_inbound_atr(struct mc_pcie *port, int window_index, + u64 axi_addr, u64 pcie_addr, u64 size) +{ + u32 table_offset = window_index * ATR_ENTRY_SIZE; + void __iomem *table_addr = port->bridge_base_addr + table_offset; + u32 atr_sz; + u32 val; + + atr_sz = ilog2(size) - 1; + + val = ALIGN_DOWN(lower_32_bits(pcie_addr), SZ_4K); + val |= FIELD_PREP(ATR_SIZE_MASK, atr_sz); + val |= ATR_IMPL_ENABLE; + + writel(val, table_addr + ATR0_PCIE_WIN0_SRCADDR_PARAM); + + writel(upper_32_bits(pcie_addr), table_addr + ATR0_PCIE_WIN0_SRC_ADDR); + + writel(lower_32_bits(axi_addr), table_addr + ATR0_PCIE_WIN0_TRSL_ADDR_LSB); + writel(upper_32_bits(axi_addr), table_addr + ATR0_PCIE_WIN0_TRSL_ADDR_UDW); + + writel(TRSL_ID_AXI4_MASTER_0, table_addr + ATR0_PCIE_WIN0_TRSL_PARAM); +} + +static int mc_pcie_setup_inbound_ranges(struct platform_device *pdev, + struct mc_pcie *port) +{ + struct device *dev = &pdev->dev; + struct device_node *dn = dev->of_node; + struct of_range_parser parser; + struct of_range range; + int atr_index = 0; + + /* + * MPFS PCIe Root Port is 32-bit only, behind a Fabric Interface + * Controller FPGA logic block which contains the AXI-S interface. + * + * From the point of view of the PCIe Root Port, there are only two + * supported Root Port configurations: + * + * Configuration 1: for use with fully coherent designs; supports a + * window from 0x0 (CPU space) to specified PCIe space. + * + * Configuration 2: for use with non-coherent designs; supports two + * 1 GB windows to CPU space; one mapping CPU space 0 to PCIe space + * 0x80000000 and a second mapping CPU space 0x40000000 to PCIe + * space 0xc0000000. This cfg needs two windows because of how the + * MSI space is allocated in the AXI-S range on MPFS. + * + * The FIC interface outside the PCIe block *must* complete the + * inbound address translation as per MCHP MPFS FPGA design + * guidelines. + */ + if (device_property_read_bool(dev, "dma-noncoherent")) { + /* + * Always need same two tables in this case. Need two tables + * due to hardware interactions between address and size. + */ + mc_pcie_setup_inbound_atr(port, 0, 0, + MPFS_NC_BOUNCE_ADDR, SZ_1G); + mc_pcie_setup_inbound_atr(port, 1, SZ_1G, + MPFS_NC_BOUNCE_ADDR + SZ_1G, SZ_1G); + } else { + /* Find any DMA ranges */ + if (of_pci_dma_range_parser_init(&parser, dn)) { + /* No DMA range property - setup default */ + mc_pcie_setup_inbound_atr(port, 0, 0, 0, SZ_4G); + return 0; + } + + for_each_of_range(&parser, &range) { + if (atr_index >= MC_MAX_NUM_INBOUND_WINDOWS) { + dev_err(dev, "too many inbound ranges; %d available tables\n", + MC_MAX_NUM_INBOUND_WINDOWS); + return -EINVAL; + } + mc_pcie_setup_inbound_atr(port, atr_index, 0, + range.pci_addr, range.size); + atr_index++; + } + } + + return 0; +} + +static int mc_platform_init(struct pci_config_window *cfg) +{ + struct device *dev = cfg->parent; + struct platform_device *pdev = to_platform_device(dev); + struct pci_host_bridge *bridge = platform_get_drvdata(pdev); + int ret; + + /* Configure address translation table 0 for PCIe config space */ + plda_pcie_setup_window(port->bridge_base_addr, 0, cfg->res.start, + cfg->res.start, + resource_size(&cfg->res)); + + /* Need some fixups in config space */ + mc_pcie_enable_msi(port, cfg->win); + + /* Configure non-config space outbound ranges */ + ret = plda_pcie_setup_iomems(bridge, &port->plda); + if (ret) + return ret; + + ret = mc_pcie_setup_inbound_ranges(pdev, port); + if (ret) + return ret; + + port->plda.event_ops = &mc_event_ops; + port->plda.event_irq_chip = &mc_event_irq_chip; + port->plda.events_bitmap = GENMASK(NUM_EVENTS - 1, 0); + + /* Address translation is up; safe to enable interrupts */ + ret = plda_init_interrupts(pdev, &port->plda, &mc_event); + if (ret) + return ret; + + return 0; +} + +static int mc_host_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + void __iomem *apb_base_addr; + struct plda_pcie_rp *plda; + int ret; + u32 val; + + port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); + if (!port) + return -ENOMEM; + + plda = &port->plda; + plda->dev = dev; + + port->bridge_base_addr = devm_platform_ioremap_resource_byname(pdev, + "bridge"); + port->ctrl_base_addr = devm_platform_ioremap_resource_byname(pdev, + "ctrl"); + if (!IS_ERR(port->bridge_base_addr) && !IS_ERR(port->ctrl_base_addr)) + goto addrs_set; + + /* + * The original, incorrect, binding that lumped the control and + * bridge addresses together still needs to be handled by the driver. + */ + apb_base_addr = devm_platform_ioremap_resource_byname(pdev, "apb"); + if (IS_ERR(apb_base_addr)) + return dev_err_probe(dev, PTR_ERR(apb_base_addr), + "both legacy apb register and ctrl/bridge regions missing"); + + port->bridge_base_addr = apb_base_addr + MC_PCIE1_BRIDGE_ADDR; + port->ctrl_base_addr = apb_base_addr + MC_PCIE1_CTRL_ADDR; + +addrs_set: + mc_disable_interrupts(port); + + plda->bridge_addr = port->bridge_base_addr; + plda->num_events = NUM_EVENTS; + + /* Allow enabling MSI by disabling MSI-X */ + val = readl(port->bridge_base_addr + PCIE_PCI_IRQ_DW0); + val &= ~MSIX_CAP_MASK; + writel(val, port->bridge_base_addr + PCIE_PCI_IRQ_DW0); + + /* Pick num vectors from bitfile programmed onto FPGA fabric */ + val = readl(port->bridge_base_addr + PCIE_PCI_IRQ_DW0); + val &= NUM_MSI_MSGS_MASK; + val >>= NUM_MSI_MSGS_SHIFT; + + plda->msi.num_vectors = 1 << val; + + /* Pick vector address from design */ + plda->msi.vector_phy = readl_relaxed(port->bridge_base_addr + IMSI_ADDR); + + ret = mc_pcie_init_clks(dev); + if (ret) { + dev_err(dev, "failed to get clock resources, error %d\n", ret); + return -ENODEV; + } + + return pci_host_common_probe(pdev); +} + +static const struct pci_ecam_ops mc_ecam_ops = { + .init = mc_platform_init, + .pci_ops = { + .map_bus = pci_ecam_map_bus, + .read = pci_generic_config_read, + .write = pci_generic_config_write, + } +}; + +static const struct of_device_id mc_pcie_of_match[] = { + { + .compatible = "microchip,pcie-host-1.0", + .data = &mc_ecam_ops, + }, + {}, +}; + +MODULE_DEVICE_TABLE(of, mc_pcie_of_match); + +static struct platform_driver mc_pcie_driver = { + .probe = mc_host_probe, + .driver = { + .name = "microchip-pcie", + .of_match_table = mc_pcie_of_match, + .suppress_bind_attrs = true, + }, +}; + +builtin_platform_driver(mc_pcie_driver); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Microchip PCIe host controller driver"); +MODULE_AUTHOR("Daire McNamara <daire.mcnamara@microchip.com>"); diff --git a/drivers/pci/controller/plda/pcie-plda-host.c b/drivers/pci/controller/plda/pcie-plda-host.c new file mode 100644 index 000000000..fd1a11b4c --- /dev/null +++ b/drivers/pci/controller/plda/pcie-plda-host.c @@ -0,0 +1,685 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PLDA PCIe XpressRich host controller driver + * + * Copyright (C) 2023 Microchip Co. Ltd + * StarFive Co. Ltd + * + * Author: Daire McNamara <daire.mcnamara@microchip.com> + */ + +#include <linux/align.h> +#include <linux/bitfield.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/irqdomain.h> +#include <linux/msi.h> +#include <linux/pci_regs.h> +#include <linux/pci-ecam.h> +#include <linux/wordpart.h> + +#include "pcie-plda.h" + +void __iomem *plda_pcie_map_bus(struct pci_bus *bus, unsigned int devfn, + int where) +{ + struct plda_pcie_rp *pcie = bus->sysdata; + + return pcie->config_base + PCIE_ECAM_OFFSET(bus->number, devfn, where); +} +EXPORT_SYMBOL_GPL(plda_pcie_map_bus); + +static void plda_handle_msi(struct irq_desc *desc) +{ + struct plda_pcie_rp *port = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + struct device *dev = port->dev; + struct plda_msi *msi = &port->msi; + void __iomem *bridge_base_addr = port->bridge_addr; + unsigned long status; + u32 bit; + int ret; + + chained_irq_enter(chip, desc); + + status = readl_relaxed(bridge_base_addr + ISTATUS_LOCAL); + if (status & PM_MSI_INT_MSI_MASK) { + writel_relaxed(status & PM_MSI_INT_MSI_MASK, + bridge_base_addr + ISTATUS_LOCAL); + status = readl_relaxed(bridge_base_addr + ISTATUS_MSI); + for_each_set_bit(bit, &status, msi->num_vectors) { + ret = generic_handle_domain_irq(msi->dev_domain, bit); + if (ret) + dev_err_ratelimited(dev, "bad MSI IRQ %d\n", + bit); + } + } + + chained_irq_exit(chip, desc); +} + +static void plda_msi_bottom_irq_ack(struct irq_data *data) +{ + struct plda_pcie_rp *port = irq_data_get_irq_chip_data(data); + void __iomem *bridge_base_addr = port->bridge_addr; + u32 bitpos = data->hwirq; + + writel_relaxed(BIT(bitpos), bridge_base_addr + ISTATUS_MSI); +} + +static void plda_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct plda_pcie_rp *port = irq_data_get_irq_chip_data(data); + phys_addr_t addr = port->msi.vector_phy; + + msg->address_lo = lower_32_bits(addr); + msg->address_hi = upper_32_bits(addr); + msg->data = data->hwirq; + + dev_dbg(port->dev, "msi#%x address_hi %#x address_lo %#x\n", + (int)data->hwirq, msg->address_hi, msg->address_lo); +} + +static struct irq_chip plda_msi_bottom_irq_chip = { + .name = "PLDA MSI", + .irq_ack = plda_msi_bottom_irq_ack, + .irq_compose_msi_msg = plda_compose_msi_msg, +}; + +static int plda_irq_msi_domain_alloc(struct irq_domain *domain, + unsigned int virq, + unsigned int nr_irqs, + void *args) +{ + struct plda_pcie_rp *port = domain->host_data; + struct plda_msi *msi = &port->msi; + unsigned long bit; + + mutex_lock(&msi->lock); + bit = find_first_zero_bit(msi->used, msi->num_vectors); + if (bit >= msi->num_vectors) { + mutex_unlock(&msi->lock); + return -ENOSPC; + } + + set_bit(bit, msi->used); + + irq_domain_set_info(domain, virq, bit, &plda_msi_bottom_irq_chip, + domain->host_data, handle_edge_irq, NULL, NULL); + + mutex_unlock(&msi->lock); + + return 0; +} + +static void plda_irq_msi_domain_free(struct irq_domain *domain, + unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct plda_pcie_rp *port = irq_data_get_irq_chip_data(d); + struct plda_msi *msi = &port->msi; + + mutex_lock(&msi->lock); + + if (test_bit(d->hwirq, msi->used)) + __clear_bit(d->hwirq, msi->used); + else + dev_err(port->dev, "trying to free unused MSI%lu\n", d->hwirq); + + mutex_unlock(&msi->lock); +} + +static const struct irq_domain_ops msi_domain_ops = { + .alloc = plda_irq_msi_domain_alloc, + .free = plda_irq_msi_domain_free, +}; + +#define PLDA_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_NO_AFFINITY) +#define PLDA_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_PCI_MSIX) + +static const struct msi_parent_ops plda_msi_parent_ops = { + .required_flags = PLDA_MSI_FLAGS_REQUIRED, + .supported_flags = PLDA_MSI_FLAGS_SUPPORTED, + .chip_flags = MSI_CHIP_FLAG_SET_ACK, + .bus_select_token = DOMAIN_BUS_PCI_MSI, + .prefix = "PLDA-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +static int plda_allocate_msi_domains(struct plda_pcie_rp *port) +{ + struct device *dev = port->dev; + struct plda_msi *msi = &port->msi; + + mutex_init(&port->msi.lock); + + struct irq_domain_info info = { + .fwnode = dev_fwnode(dev), + .ops = &msi_domain_ops, + .host_data = port, + .size = msi->num_vectors, + }; + + msi->dev_domain = msi_create_parent_irq_domain(&info, &plda_msi_parent_ops); + if (!msi->dev_domain) { + dev_err(dev, "failed to create IRQ domain\n"); + return -ENOMEM; + } + + return 0; +} + +static void plda_handle_intx(struct irq_desc *desc) +{ + struct plda_pcie_rp *port = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + struct device *dev = port->dev; + void __iomem *bridge_base_addr = port->bridge_addr; + unsigned long status; + u32 bit; + int ret; + + chained_irq_enter(chip, desc); + + status = readl_relaxed(bridge_base_addr + ISTATUS_LOCAL); + if (status & PM_MSI_INT_INTX_MASK) { + status &= PM_MSI_INT_INTX_MASK; + status >>= PM_MSI_INT_INTX_SHIFT; + for_each_set_bit(bit, &status, PCI_NUM_INTX) { + ret = generic_handle_domain_irq(port->intx_domain, bit); + if (ret) + dev_err_ratelimited(dev, "bad INTx IRQ %d\n", + bit); + } + } + + chained_irq_exit(chip, desc); +} + +static void plda_ack_intx_irq(struct irq_data *data) +{ + struct plda_pcie_rp *port = irq_data_get_irq_chip_data(data); + void __iomem *bridge_base_addr = port->bridge_addr; + u32 mask = BIT(data->hwirq + PM_MSI_INT_INTX_SHIFT); + + writel_relaxed(mask, bridge_base_addr + ISTATUS_LOCAL); +} + +static void plda_mask_intx_irq(struct irq_data *data) +{ + struct plda_pcie_rp *port = irq_data_get_irq_chip_data(data); + void __iomem *bridge_base_addr = port->bridge_addr; + unsigned long flags; + u32 mask = BIT(data->hwirq + PM_MSI_INT_INTX_SHIFT); + u32 val; + + raw_spin_lock_irqsave(&port->lock, flags); + val = readl_relaxed(bridge_base_addr + IMASK_LOCAL); + val &= ~mask; + writel_relaxed(val, bridge_base_addr + IMASK_LOCAL); + raw_spin_unlock_irqrestore(&port->lock, flags); +} + +static void plda_unmask_intx_irq(struct irq_data *data) +{ + struct plda_pcie_rp *port = irq_data_get_irq_chip_data(data); + void __iomem *bridge_base_addr = port->bridge_addr; + unsigned long flags; + u32 mask = BIT(data->hwirq + PM_MSI_INT_INTX_SHIFT); + u32 val; + + raw_spin_lock_irqsave(&port->lock, flags); + val = readl_relaxed(bridge_base_addr + IMASK_LOCAL); + val |= mask; + writel_relaxed(val, bridge_base_addr + IMASK_LOCAL); + raw_spin_unlock_irqrestore(&port->lock, flags); +} + +static struct irq_chip plda_intx_irq_chip = { + .name = "PLDA PCIe INTx", + .irq_ack = plda_ack_intx_irq, + .irq_mask = plda_mask_intx_irq, + .irq_unmask = plda_unmask_intx_irq, +}; + +static int plda_pcie_intx_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &plda_intx_irq_chip, handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + + return 0; +} + +static const struct irq_domain_ops intx_domain_ops = { + .map = plda_pcie_intx_map, +}; + +static u32 plda_get_events(struct plda_pcie_rp *port) +{ + u32 events, val, origin; + + origin = readl_relaxed(port->bridge_addr + ISTATUS_LOCAL); + + /* MSI event and sys events */ + val = (origin & SYS_AND_MSI_MASK) >> PM_MSI_INT_MSI_SHIFT; + events = val << (PM_MSI_INT_MSI_SHIFT - PCI_NUM_INTX + 1); + + /* INTx events */ + if (origin & PM_MSI_INT_INTX_MASK) + events |= BIT(PM_MSI_INT_INTX_SHIFT); + + /* remains are same with register */ + events |= origin & GENMASK(P_ATR_EVT_DOORBELL_SHIFT, 0); + + return events; +} + +static irqreturn_t plda_event_handler(int irq, void *dev_id) +{ + return IRQ_HANDLED; +} + +static void plda_handle_event(struct irq_desc *desc) +{ + struct plda_pcie_rp *port = irq_desc_get_handler_data(desc); + unsigned long events; + u32 bit; + struct irq_chip *chip = irq_desc_get_chip(desc); + + chained_irq_enter(chip, desc); + + events = port->event_ops->get_events(port); + + events &= port->events_bitmap; + for_each_set_bit(bit, &events, port->num_events) + generic_handle_domain_irq(port->event_domain, bit); + + chained_irq_exit(chip, desc); +} + +static u32 plda_hwirq_to_mask(int hwirq) +{ + u32 mask; + + /* hwirq 23 - 0 are the same with register */ + if (hwirq < EVENT_PM_MSI_INT_INTX) + mask = BIT(hwirq); + else if (hwirq == EVENT_PM_MSI_INT_INTX) + mask = PM_MSI_INT_INTX_MASK; + else + mask = BIT(hwirq + PCI_NUM_INTX - 1); + + return mask; +} + +static void plda_ack_event_irq(struct irq_data *data) +{ + struct plda_pcie_rp *port = irq_data_get_irq_chip_data(data); + + writel_relaxed(plda_hwirq_to_mask(data->hwirq), + port->bridge_addr + ISTATUS_LOCAL); +} + +static void plda_mask_event_irq(struct irq_data *data) +{ + struct plda_pcie_rp *port = irq_data_get_irq_chip_data(data); + u32 mask, val; + + mask = plda_hwirq_to_mask(data->hwirq); + + raw_spin_lock(&port->lock); + val = readl_relaxed(port->bridge_addr + IMASK_LOCAL); + val &= ~mask; + writel_relaxed(val, port->bridge_addr + IMASK_LOCAL); + raw_spin_unlock(&port->lock); +} + +static void plda_unmask_event_irq(struct irq_data *data) +{ + struct plda_pcie_rp *port = irq_data_get_irq_chip_data(data); + u32 mask, val; + + mask = plda_hwirq_to_mask(data->hwirq); + + raw_spin_lock(&port->lock); + val = readl_relaxed(port->bridge_addr + IMASK_LOCAL); + val |= mask; + writel_relaxed(val, port->bridge_addr + IMASK_LOCAL); + raw_spin_unlock(&port->lock); +} + +static struct irq_chip plda_event_irq_chip = { + .name = "PLDA PCIe EVENT", + .irq_ack = plda_ack_event_irq, + .irq_mask = plda_mask_event_irq, + .irq_unmask = plda_unmask_event_irq, +}; + +static const struct plda_event_ops plda_event_ops = { + .get_events = plda_get_events, +}; + +static int plda_pcie_event_map(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + struct plda_pcie_rp *port = (void *)domain->host_data; + + irq_set_chip_and_handler(irq, port->event_irq_chip, handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + + return 0; +} + +static const struct irq_domain_ops plda_event_domain_ops = { + .map = plda_pcie_event_map, +}; + +static int plda_pcie_init_irq_domains(struct plda_pcie_rp *port) +{ + struct device *dev = port->dev; + struct device_node *node = dev->of_node; + struct device_node *pcie_intc_node; + + /* Setup INTx */ + pcie_intc_node = of_get_next_child(node, NULL); + if (!pcie_intc_node) { + dev_err(dev, "failed to find PCIe Intc node\n"); + return -EINVAL; + } + + port->event_domain = irq_domain_create_linear(of_fwnode_handle(pcie_intc_node), + port->num_events, &plda_event_domain_ops, + port); + if (!port->event_domain) { + dev_err(dev, "failed to get event domain\n"); + of_node_put(pcie_intc_node); + return -ENOMEM; + } + + irq_domain_update_bus_token(port->event_domain, DOMAIN_BUS_NEXUS); + + port->intx_domain = irq_domain_create_linear(of_fwnode_handle(pcie_intc_node), PCI_NUM_INTX, + &intx_domain_ops, port); + if (!port->intx_domain) { + dev_err(dev, "failed to get an INTx IRQ domain\n"); + of_node_put(pcie_intc_node); + return -ENOMEM; + } + + irq_domain_update_bus_token(port->intx_domain, DOMAIN_BUS_WIRED); + + of_node_put(pcie_intc_node); + raw_spin_lock_init(&port->lock); + + return plda_allocate_msi_domains(port); +} + +static void plda_pcie_irq_domain_deinit(struct plda_pcie_rp *pcie); + +int plda_init_interrupts(struct platform_device *pdev, + struct plda_pcie_rp *port, + const struct plda_event *event) +{ + struct device *dev = &pdev->dev; + int event_irq, ret; + u32 i; + + if (!port->event_ops) + port->event_ops = &plda_event_ops; + + if (!port->event_irq_chip) + port->event_irq_chip = &plda_event_irq_chip; + + ret = plda_pcie_init_irq_domains(port); + if (ret) { + dev_err(dev, "failed creating IRQ domains\n"); + return ret; + } + + port->irq = platform_get_irq(pdev, 0); + if (port->irq < 0) { + ret = -ENODEV; + goto err_irq_domain_deinit; + } + + for_each_set_bit(i, &port->events_bitmap, port->num_events) { + event_irq = irq_create_mapping(port->event_domain, i); + if (!event_irq) { + dev_err(dev, "failed to map hwirq %d\n", i); + ret = -ENXIO; + goto err_irq_domain_deinit; + } + + if (event->request_event_irq) + ret = event->request_event_irq(port, event_irq, i); + else + ret = devm_request_irq(dev, event_irq, + plda_event_handler, + 0, NULL, port); + + if (ret) { + dev_err(dev, "failed to request IRQ %d\n", event_irq); + goto err_irq_domain_deinit; + } + } + + port->intx_irq = irq_create_mapping(port->event_domain, + event->intx_event); + if (!port->intx_irq) { + dev_err(dev, "failed to map INTx interrupt\n"); + ret = -ENXIO; + goto err_irq_domain_deinit; + } + + /* Plug the INTx chained handler */ + irq_set_chained_handler_and_data(port->intx_irq, plda_handle_intx, port); + + port->msi_irq = irq_create_mapping(port->event_domain, + event->msi_event); + if (!port->msi_irq) { + dev_err(dev, "failed to map MSI interrupt\n"); + ret = -ENXIO; + goto err_irq_domain_deinit; + } + + /* Plug the MSI chained handler */ + irq_set_chained_handler_and_data(port->msi_irq, plda_handle_msi, port); + + /* Plug the main event chained handler */ + irq_set_chained_handler_and_data(port->irq, plda_handle_event, port); + + return 0; + +err_irq_domain_deinit: + plda_pcie_irq_domain_deinit(port); + + return ret; +} +EXPORT_SYMBOL_GPL(plda_init_interrupts); + +void plda_pcie_setup_window(void __iomem *bridge_base_addr, u32 index, + phys_addr_t axi_addr, phys_addr_t pci_addr, + size_t size) +{ + u32 atr_sz = ilog2(size) - 1; + u32 val; + + if (index == 0) + val = PCIE_CONFIG_INTERFACE; + else + val = PCIE_TX_RX_INTERFACE; + + writel(val, bridge_base_addr + (index * ATR_ENTRY_SIZE) + + ATR0_AXI4_SLV0_TRSL_PARAM); + + val = ALIGN_DOWN(lower_32_bits(axi_addr), SZ_4K); + val |= FIELD_PREP(ATR_SIZE_MASK, atr_sz); + val |= ATR_IMPL_ENABLE; + writel(val, bridge_base_addr + (index * ATR_ENTRY_SIZE) + + ATR0_AXI4_SLV0_SRCADDR_PARAM); + + val = upper_32_bits(axi_addr); + writel(val, bridge_base_addr + (index * ATR_ENTRY_SIZE) + + ATR0_AXI4_SLV0_SRC_ADDR); + + val = lower_32_bits(pci_addr); + writel(val, bridge_base_addr + (index * ATR_ENTRY_SIZE) + + ATR0_AXI4_SLV0_TRSL_ADDR_LSB); + + val = upper_32_bits(pci_addr); + writel(val, bridge_base_addr + (index * ATR_ENTRY_SIZE) + + ATR0_AXI4_SLV0_TRSL_ADDR_UDW); +} +EXPORT_SYMBOL_GPL(plda_pcie_setup_window); + +void plda_pcie_setup_inbound_address_translation(struct plda_pcie_rp *port) +{ + void __iomem *bridge_base_addr = port->bridge_addr; + u32 val; + + val = readl(bridge_base_addr + ATR0_PCIE_WIN0_SRCADDR_PARAM); + val |= (ATR0_PCIE_ATR_SIZE << ATR0_PCIE_ATR_SIZE_SHIFT); + writel(val, bridge_base_addr + ATR0_PCIE_WIN0_SRCADDR_PARAM); + writel(0, bridge_base_addr + ATR0_PCIE_WIN0_SRC_ADDR); +} +EXPORT_SYMBOL_GPL(plda_pcie_setup_inbound_address_translation); + +int plda_pcie_setup_iomems(struct pci_host_bridge *bridge, + struct plda_pcie_rp *port) +{ + void __iomem *bridge_base_addr = port->bridge_addr; + struct resource_entry *entry; + u64 pci_addr; + u32 index = 1; + + resource_list_for_each_entry(entry, &bridge->windows) { + if (resource_type(entry->res) == IORESOURCE_MEM) { + pci_addr = entry->res->start - entry->offset; + plda_pcie_setup_window(bridge_base_addr, index, + entry->res->start, pci_addr, + resource_size(entry->res)); + index++; + } + } + + return 0; +} +EXPORT_SYMBOL_GPL(plda_pcie_setup_iomems); + +static void plda_pcie_irq_domain_deinit(struct plda_pcie_rp *pcie) +{ + u32 i, event_irq; + + if (pcie->irq > 0) + irq_set_chained_handler_and_data(pcie->irq, NULL, NULL); + if (pcie->msi_irq > 0) + irq_set_chained_handler_and_data(pcie->msi_irq, NULL, NULL); + if (pcie->intx_irq > 0) + irq_set_chained_handler_and_data(pcie->intx_irq, NULL, NULL); + + for_each_set_bit(i, &pcie->events_bitmap, pcie->num_events) { + event_irq = irq_find_mapping(pcie->event_domain, i); + if (event_irq) { + devm_free_irq(pcie->dev, event_irq, pcie); + irq_dispose_mapping(event_irq); + } + } + + if (pcie->intx_irq) + irq_dispose_mapping(pcie->intx_irq); + if (pcie->msi_irq) + irq_dispose_mapping(pcie->msi_irq); + + irq_domain_remove(pcie->msi.dev_domain); + + irq_domain_remove(pcie->intx_domain); + irq_domain_remove(pcie->event_domain); +} + +int plda_pcie_host_init(struct plda_pcie_rp *port, struct pci_ops *ops, + const struct plda_event *plda_event) +{ + struct device *dev = port->dev; + struct pci_host_bridge *bridge; + struct platform_device *pdev = to_platform_device(dev); + struct resource *cfg_res; + int ret; + + pdev = to_platform_device(dev); + + port->bridge_addr = + devm_platform_ioremap_resource_byname(pdev, "apb"); + + if (IS_ERR(port->bridge_addr)) + return dev_err_probe(dev, PTR_ERR(port->bridge_addr), + "failed to map reg memory\n"); + + cfg_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg"); + if (!cfg_res) + return dev_err_probe(dev, -ENODEV, + "failed to get config memory\n"); + + port->config_base = devm_ioremap_resource(dev, cfg_res); + if (IS_ERR(port->config_base)) + return dev_err_probe(dev, PTR_ERR(port->config_base), + "failed to map config memory\n"); + + bridge = devm_pci_alloc_host_bridge(dev, 0); + if (!bridge) + return -ENOMEM; + + if (port->host_ops && port->host_ops->host_init) { + ret = port->host_ops->host_init(port); + if (ret) + return ret; + } + + port->bridge = bridge; + plda_pcie_setup_window(port->bridge_addr, 0, cfg_res->start, 0, + resource_size(cfg_res)); + plda_pcie_setup_iomems(bridge, port); + plda_set_default_msi(&port->msi); + ret = plda_init_interrupts(pdev, port, plda_event); + if (ret) + goto err_host; + + /* Set default bus ops */ + bridge->ops = ops; + bridge->sysdata = port; + + ret = pci_host_probe(bridge); + if (ret < 0) { + dev_err_probe(dev, ret, "failed to probe pci host\n"); + goto err_probe; + } + + return ret; + +err_probe: + plda_pcie_irq_domain_deinit(port); +err_host: + if (port->host_ops && port->host_ops->host_deinit) + port->host_ops->host_deinit(port); + + return ret; +} +EXPORT_SYMBOL_GPL(plda_pcie_host_init); + +void plda_pcie_host_deinit(struct plda_pcie_rp *port) +{ + pci_lock_rescan_remove(); + pci_stop_root_bus(port->bridge->bus); + pci_remove_root_bus(port->bridge->bus); + pci_unlock_rescan_remove(); + + plda_pcie_irq_domain_deinit(port); + + if (port->host_ops && port->host_ops->host_deinit) + port->host_ops->host_deinit(port); +} +EXPORT_SYMBOL_GPL(plda_pcie_host_deinit); diff --git a/drivers/pci/controller/plda/pcie-plda.h b/drivers/pci/controller/plda/pcie-plda.h new file mode 100644 index 000000000..6b8665df7 --- /dev/null +++ b/drivers/pci/controller/plda/pcie-plda.h @@ -0,0 +1,274 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * PLDA PCIe host controller driver + */ + +#ifndef _PCIE_PLDA_H +#define _PCIE_PLDA_H + +/* Number of MSI IRQs */ +#define PLDA_MAX_NUM_MSI_IRQS 32 + +/* PCIe Bridge Phy Regs */ +#define GEN_SETTINGS 0x80 +#define RP_ENABLE 1 +#define PCIE_PCI_IDS_DW1 0x9c +#define IDS_CLASS_CODE_SHIFT 16 +#define REVISION_ID_MASK GENMASK(7, 0) +#define CLASS_CODE_ID_MASK GENMASK(31, 8) +#define PCIE_PCI_IRQ_DW0 0xa8 +#define MSIX_CAP_MASK BIT(31) +#define NUM_MSI_MSGS_MASK GENMASK(6, 4) +#define NUM_MSI_MSGS_SHIFT 4 +#define PCI_MISC 0xb4 +#define PHY_FUNCTION_DIS BIT(15) +#define PCIE_WINROM 0xfc +#define PREF_MEM_WIN_64_SUPPORT BIT(3) + +#define IMASK_LOCAL 0x180 +#define DMA_END_ENGINE_0_MASK 0x00000000u +#define DMA_END_ENGINE_0_SHIFT 0 +#define DMA_END_ENGINE_1_MASK 0x00000000u +#define DMA_END_ENGINE_1_SHIFT 1 +#define DMA_ERROR_ENGINE_0_MASK 0x00000100u +#define DMA_ERROR_ENGINE_0_SHIFT 8 +#define DMA_ERROR_ENGINE_1_MASK 0x00000200u +#define DMA_ERROR_ENGINE_1_SHIFT 9 +#define A_ATR_EVT_POST_ERR_MASK 0x00010000u +#define A_ATR_EVT_POST_ERR_SHIFT 16 +#define A_ATR_EVT_FETCH_ERR_MASK 0x00020000u +#define A_ATR_EVT_FETCH_ERR_SHIFT 17 +#define A_ATR_EVT_DISCARD_ERR_MASK 0x00040000u +#define A_ATR_EVT_DISCARD_ERR_SHIFT 18 +#define A_ATR_EVT_DOORBELL_MASK 0x00000000u +#define A_ATR_EVT_DOORBELL_SHIFT 19 +#define P_ATR_EVT_POST_ERR_MASK 0x00100000u +#define P_ATR_EVT_POST_ERR_SHIFT 20 +#define P_ATR_EVT_FETCH_ERR_MASK 0x00200000u +#define P_ATR_EVT_FETCH_ERR_SHIFT 21 +#define P_ATR_EVT_DISCARD_ERR_MASK 0x00400000u +#define P_ATR_EVT_DISCARD_ERR_SHIFT 22 +#define P_ATR_EVT_DOORBELL_MASK 0x00000000u +#define P_ATR_EVT_DOORBELL_SHIFT 23 +#define PM_MSI_INT_INTA_MASK 0x01000000u +#define PM_MSI_INT_INTA_SHIFT 24 +#define PM_MSI_INT_INTB_MASK 0x02000000u +#define PM_MSI_INT_INTB_SHIFT 25 +#define PM_MSI_INT_INTC_MASK 0x04000000u +#define PM_MSI_INT_INTC_SHIFT 26 +#define PM_MSI_INT_INTD_MASK 0x08000000u +#define PM_MSI_INT_INTD_SHIFT 27 +#define PM_MSI_INT_INTX_MASK 0x0f000000u +#define PM_MSI_INT_INTX_SHIFT 24 +#define PM_MSI_INT_MSI_MASK 0x10000000u +#define PM_MSI_INT_MSI_SHIFT 28 +#define PM_MSI_INT_AER_EVT_MASK 0x20000000u +#define PM_MSI_INT_AER_EVT_SHIFT 29 +#define PM_MSI_INT_EVENTS_MASK 0x40000000u +#define PM_MSI_INT_EVENTS_SHIFT 30 +#define PM_MSI_INT_SYS_ERR_MASK 0x80000000u +#define PM_MSI_INT_SYS_ERR_SHIFT 31 +#define SYS_AND_MSI_MASK GENMASK(31, 28) +#define NUM_LOCAL_EVENTS 15 +#define ISTATUS_LOCAL 0x184 +#define IMASK_HOST 0x188 +#define ISTATUS_HOST 0x18c +#define IMSI_ADDR 0x190 +#define ISTATUS_MSI 0x194 +#define PMSG_SUPPORT_RX 0x3f0 +#define PMSG_LTR_SUPPORT BIT(2) + +/* PCIe Master table init defines */ +#define ATR0_PCIE_WIN0_SRCADDR_PARAM 0x600u +#define ATR0_PCIE_ATR_SIZE 0x25 +#define ATR0_PCIE_ATR_SIZE_SHIFT 1 +#define ATR0_PCIE_WIN0_SRC_ADDR 0x604u +#define ATR0_PCIE_WIN0_TRSL_ADDR_LSB 0x608u +#define ATR0_PCIE_WIN0_TRSL_ADDR_UDW 0x60cu +#define ATR0_PCIE_WIN0_TRSL_PARAM 0x610u + +/* PCIe AXI slave table init defines */ +#define ATR0_AXI4_SLV0_SRCADDR_PARAM 0x800u +#define ATR_SIZE_MASK GENMASK(6, 1) +#define ATR_IMPL_ENABLE BIT(0) +#define ATR0_AXI4_SLV0_SRC_ADDR 0x804u +#define ATR0_AXI4_SLV0_TRSL_ADDR_LSB 0x808u +#define ATR0_AXI4_SLV0_TRSL_ADDR_UDW 0x80cu +#define ATR0_AXI4_SLV0_TRSL_PARAM 0x810u +#define PCIE_TX_RX_INTERFACE 0x00000000u +#define PCIE_CONFIG_INTERFACE 0x00000001u +#define TRSL_ID_AXI4_MASTER_0 0x00000004u + +#define CONFIG_SPACE_ADDR_OFFSET 0x1000u + +#define ATR_ENTRY_SIZE 32 + +enum plda_int_event { + PLDA_AXI_POST_ERR, + PLDA_AXI_FETCH_ERR, + PLDA_AXI_DISCARD_ERR, + PLDA_AXI_DOORBELL, + PLDA_PCIE_POST_ERR, + PLDA_PCIE_FETCH_ERR, + PLDA_PCIE_DISCARD_ERR, + PLDA_PCIE_DOORBELL, + PLDA_INTX, + PLDA_MSI, + PLDA_AER_EVENT, + PLDA_MISC_EVENTS, + PLDA_SYS_ERR, + PLDA_INT_EVENT_NUM +}; + +#define PLDA_NUM_DMA_EVENTS 16 + +#define EVENT_PM_MSI_INT_INTX (PLDA_NUM_DMA_EVENTS + PLDA_INTX) +#define EVENT_PM_MSI_INT_MSI (PLDA_NUM_DMA_EVENTS + PLDA_MSI) +#define PLDA_MAX_EVENT_NUM (PLDA_NUM_DMA_EVENTS + PLDA_INT_EVENT_NUM) + +/* + * PLDA interrupt register + * + * 31 27 23 15 7 0 + * +--+--+--+-+------+-+-+-+-+-+-+-+-+-----------+-----------+ + * |12|11|10|9| intx |7|6|5|4|3|2|1|0| DMA error | DMA end | + * +--+--+--+-+------+-+-+-+-+-+-+-+-+-----------+-----------+ + * event bit + * 0-7 (0-7) DMA interrupt end : reserved for vendor implement + * 8-15 (8-15) DMA error : reserved for vendor implement + * 16 (16) AXI post error (PLDA_AXI_POST_ERR) + * 17 (17) AXI fetch error (PLDA_AXI_FETCH_ERR) + * 18 (18) AXI discard error (PLDA_AXI_DISCARD_ERR) + * 19 (19) AXI doorbell (PLDA_PCIE_DOORBELL) + * 20 (20) PCIe post error (PLDA_PCIE_POST_ERR) + * 21 (21) PCIe fetch error (PLDA_PCIE_FETCH_ERR) + * 22 (22) PCIe discard error (PLDA_PCIE_DISCARD_ERR) + * 23 (23) PCIe doorbell (PLDA_PCIE_DOORBELL) + * 24 (27-24) INTx interruts (PLDA_INTX) + * 25 (28): MSI interrupt (PLDA_MSI) + * 26 (29): AER event (PLDA_AER_EVENT) + * 27 (30): PM/LTR/Hotplug (PLDA_MISC_EVENTS) + * 28 (31): System error (PLDA_SYS_ERR) + */ + +struct plda_pcie_rp; + +struct plda_event_ops { + u32 (*get_events)(struct plda_pcie_rp *pcie); +}; + +struct plda_pcie_host_ops { + int (*host_init)(struct plda_pcie_rp *pcie); + void (*host_deinit)(struct plda_pcie_rp *pcie); +}; + +struct plda_msi { + struct mutex lock; /* Protect used bitmap */ + struct irq_domain *dev_domain; + u32 num_vectors; + u64 vector_phy; + DECLARE_BITMAP(used, PLDA_MAX_NUM_MSI_IRQS); +}; + +struct plda_pcie_rp { + struct device *dev; + struct pci_host_bridge *bridge; + struct irq_domain *intx_domain; + struct irq_domain *event_domain; + raw_spinlock_t lock; + struct plda_msi msi; + const struct plda_event_ops *event_ops; + const struct irq_chip *event_irq_chip; + const struct plda_pcie_host_ops *host_ops; + void __iomem *bridge_addr; + void __iomem *config_base; + unsigned long events_bitmap; + int irq; + int msi_irq; + int intx_irq; + int num_events; +}; + +struct plda_event { + int (*request_event_irq)(struct plda_pcie_rp *pcie, + int event_irq, int event); + int intx_event; + int msi_event; +}; + +void __iomem *plda_pcie_map_bus(struct pci_bus *bus, unsigned int devfn, + int where); +int plda_init_interrupts(struct platform_device *pdev, + struct plda_pcie_rp *port, + const struct plda_event *event); +void plda_pcie_setup_window(void __iomem *bridge_base_addr, u32 index, + phys_addr_t axi_addr, phys_addr_t pci_addr, + size_t size); +void plda_pcie_setup_inbound_address_translation(struct plda_pcie_rp *port); +int plda_pcie_setup_iomems(struct pci_host_bridge *bridge, + struct plda_pcie_rp *port); +int plda_pcie_host_init(struct plda_pcie_rp *port, struct pci_ops *ops, + const struct plda_event *plda_event); +void plda_pcie_host_deinit(struct plda_pcie_rp *pcie); + +static inline void plda_set_default_msi(struct plda_msi *msi) +{ + msi->vector_phy = IMSI_ADDR; + msi->num_vectors = PLDA_MAX_NUM_MSI_IRQS; +} + +static inline void plda_pcie_enable_root_port(struct plda_pcie_rp *plda) +{ + u32 value; + + value = readl_relaxed(plda->bridge_addr + GEN_SETTINGS); + value |= RP_ENABLE; + writel_relaxed(value, plda->bridge_addr + GEN_SETTINGS); +} + +static inline void plda_pcie_set_standard_class(struct plda_pcie_rp *plda) +{ + u32 value; + + /* set class code and reserve revision id */ + value = readl_relaxed(plda->bridge_addr + PCIE_PCI_IDS_DW1); + value &= REVISION_ID_MASK; + value |= (PCI_CLASS_BRIDGE_PCI << IDS_CLASS_CODE_SHIFT); + writel_relaxed(value, plda->bridge_addr + PCIE_PCI_IDS_DW1); +} + +static inline void plda_pcie_set_pref_win_64bit(struct plda_pcie_rp *plda) +{ + u32 value; + + value = readl_relaxed(plda->bridge_addr + PCIE_WINROM); + value |= PREF_MEM_WIN_64_SUPPORT; + writel_relaxed(value, plda->bridge_addr + PCIE_WINROM); +} + +static inline void plda_pcie_disable_ltr(struct plda_pcie_rp *plda) +{ + u32 value; + + value = readl_relaxed(plda->bridge_addr + PMSG_SUPPORT_RX); + value &= ~PMSG_LTR_SUPPORT; + writel_relaxed(value, plda->bridge_addr + PMSG_SUPPORT_RX); +} + +static inline void plda_pcie_disable_func(struct plda_pcie_rp *plda) +{ + u32 value; + + value = readl_relaxed(plda->bridge_addr + PCI_MISC); + value |= PHY_FUNCTION_DIS; + writel_relaxed(value, plda->bridge_addr + PCI_MISC); +} + +static inline void plda_pcie_write_rc_bar(struct plda_pcie_rp *plda, u64 val) +{ + void __iomem *addr = plda->bridge_addr + CONFIG_SPACE_ADDR_OFFSET; + + writel_relaxed(lower_32_bits(val), addr + PCI_BASE_ADDRESS_0); + writel_relaxed(upper_32_bits(val), addr + PCI_BASE_ADDRESS_1); +} +#endif /* _PCIE_PLDA_H */ diff --git a/drivers/pci/controller/plda/pcie-starfive.c b/drivers/pci/controller/plda/pcie-starfive.c new file mode 100644 index 000000000..02d5aa6dc --- /dev/null +++ b/drivers/pci/controller/plda/pcie-starfive.c @@ -0,0 +1,505 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * PCIe host controller driver for StarFive JH7110 Soc. + * + * Copyright (C) 2023 StarFive Technology Co., Ltd. + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/gpio/consumer.h> +#include <linux/interrupt.h> +#include <linux/kernel.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/of_pci.h> +#include <linux/pci.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/regmap.h> +#include <linux/reset.h> +#include "../../pci.h" + +#include "pcie-plda.h" + +#define PCIE_FUNC_NUM 4 + +/* system control */ +#define STG_SYSCON_PCIE0_BASE 0x48 +#define STG_SYSCON_PCIE1_BASE 0x1f8 + +#define STG_SYSCON_AR_OFFSET 0x78 +#define STG_SYSCON_AXI4_SLVL_AR_MASK GENMASK(22, 8) +#define STG_SYSCON_AXI4_SLVL_PHY_AR(x) FIELD_PREP(GENMASK(20, 17), x) +#define STG_SYSCON_AW_OFFSET 0x7c +#define STG_SYSCON_AXI4_SLVL_AW_MASK GENMASK(14, 0) +#define STG_SYSCON_AXI4_SLVL_PHY_AW(x) FIELD_PREP(GENMASK(12, 9), x) +#define STG_SYSCON_CLKREQ BIT(22) +#define STG_SYSCON_CKREF_SRC_MASK GENMASK(19, 18) +#define STG_SYSCON_RP_NEP_OFFSET 0xe8 +#define STG_SYSCON_K_RP_NEP BIT(8) +#define STG_SYSCON_LNKSTA_OFFSET 0x170 +#define DATA_LINK_ACTIVE BIT(5) + +struct starfive_jh7110_pcie { + struct plda_pcie_rp plda; + struct reset_control *resets; + struct clk_bulk_data *clks; + struct regmap *reg_syscon; + struct regulator *vpcie3v3; + struct gpio_desc *reset_gpio; + struct phy *phy; + + unsigned int stg_pcie_base; + int num_clks; +}; + +/* + * JH7110 PCIe port BAR0/1 can be configured as 64-bit prefetchable memory + * space. PCIe read and write requests targeting BAR0/1 are routed to so called + * 'Bridge Configuration space' in PLDA IP datasheet, which contains the bridge + * internal registers, such as interrupt, DMA and ATU registers... + * JH7110 can access the Bridge Configuration space by local bus, and don`t + * want the bridge internal registers accessed by the DMA from EP devices. + * Thus, they are unimplemented and should be hidden here. + */ +static bool starfive_pcie_hide_rc_bar(struct pci_bus *bus, unsigned int devfn, + int offset) +{ + if (pci_is_root_bus(bus) && !devfn && + (offset == PCI_BASE_ADDRESS_0 || offset == PCI_BASE_ADDRESS_1)) + return true; + + return false; +} + +static int starfive_pcie_config_write(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 value) +{ + if (starfive_pcie_hide_rc_bar(bus, devfn, where)) + return PCIBIOS_SUCCESSFUL; + + return pci_generic_config_write(bus, devfn, where, size, value); +} + +static int starfive_pcie_config_read(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *value) +{ + if (starfive_pcie_hide_rc_bar(bus, devfn, where)) { + *value = 0; + return PCIBIOS_SUCCESSFUL; + } + + return pci_generic_config_read(bus, devfn, where, size, value); +} + +static int starfive_pcie_parse_dt(struct starfive_jh7110_pcie *pcie, + struct device *dev) +{ + int domain_nr; + + pcie->num_clks = devm_clk_bulk_get_all(dev, &pcie->clks); + if (pcie->num_clks < 0) + return dev_err_probe(dev, pcie->num_clks, + "failed to get pcie clocks\n"); + + pcie->resets = devm_reset_control_array_get_exclusive(dev); + if (IS_ERR(pcie->resets)) + return dev_err_probe(dev, PTR_ERR(pcie->resets), + "failed to get pcie resets"); + + pcie->reg_syscon = + syscon_regmap_lookup_by_phandle(dev->of_node, + "starfive,stg-syscon"); + + if (IS_ERR(pcie->reg_syscon)) + return dev_err_probe(dev, PTR_ERR(pcie->reg_syscon), + "failed to parse starfive,stg-syscon\n"); + + pcie->phy = devm_phy_optional_get(dev, NULL); + if (IS_ERR(pcie->phy)) + return dev_err_probe(dev, PTR_ERR(pcie->phy), + "failed to get pcie phy\n"); + + /* + * The PCIe domain numbers are set to be static in JH7110 DTS. + * As the STG system controller defines different bases in PCIe RP0 & + * RP1, we use them to identify which controller is doing the hardware + * initialization. + */ + domain_nr = of_get_pci_domain_nr(dev->of_node); + + if (domain_nr < 0 || domain_nr > 1) + return dev_err_probe(dev, -ENODEV, + "failed to get valid pcie domain\n"); + + if (domain_nr == 0) + pcie->stg_pcie_base = STG_SYSCON_PCIE0_BASE; + else + pcie->stg_pcie_base = STG_SYSCON_PCIE1_BASE; + + pcie->reset_gpio = devm_gpiod_get_optional(dev, "perst", + GPIOD_OUT_HIGH); + if (IS_ERR(pcie->reset_gpio)) + return dev_err_probe(dev, PTR_ERR(pcie->reset_gpio), + "failed to get perst-gpio\n"); + + pcie->vpcie3v3 = devm_regulator_get_optional(dev, "vpcie3v3"); + if (IS_ERR(pcie->vpcie3v3)) { + if (PTR_ERR(pcie->vpcie3v3) != -ENODEV) + return dev_err_probe(dev, PTR_ERR(pcie->vpcie3v3), + "failed to get vpcie3v3 regulator\n"); + pcie->vpcie3v3 = NULL; + } + + return 0; +} + +static struct pci_ops starfive_pcie_ops = { + .map_bus = plda_pcie_map_bus, + .read = starfive_pcie_config_read, + .write = starfive_pcie_config_write, +}; + +static int starfive_pcie_clk_rst_init(struct starfive_jh7110_pcie *pcie) +{ + struct device *dev = pcie->plda.dev; + int ret; + + ret = clk_bulk_prepare_enable(pcie->num_clks, pcie->clks); + if (ret) + return dev_err_probe(dev, ret, "failed to enable clocks\n"); + + ret = reset_control_deassert(pcie->resets); + if (ret) { + clk_bulk_disable_unprepare(pcie->num_clks, pcie->clks); + dev_err_probe(dev, ret, "failed to deassert resets\n"); + } + + return ret; +} + +static void starfive_pcie_clk_rst_deinit(struct starfive_jh7110_pcie *pcie) +{ + reset_control_assert(pcie->resets); + clk_bulk_disable_unprepare(pcie->num_clks, pcie->clks); +} + +static bool starfive_pcie_link_up(struct plda_pcie_rp *plda) +{ + struct starfive_jh7110_pcie *pcie = + container_of(plda, struct starfive_jh7110_pcie, plda); + int ret; + u32 stg_reg_val; + + ret = regmap_read(pcie->reg_syscon, + pcie->stg_pcie_base + STG_SYSCON_LNKSTA_OFFSET, + &stg_reg_val); + if (ret) { + dev_err(pcie->plda.dev, "failed to read link status\n"); + return false; + } + + return !!(stg_reg_val & DATA_LINK_ACTIVE); +} + +static int starfive_pcie_host_wait_for_link(struct starfive_jh7110_pcie *pcie) +{ + int retries; + + /* Check if the link is up or not */ + for (retries = 0; retries < PCIE_LINK_WAIT_MAX_RETRIES; retries++) { + if (starfive_pcie_link_up(&pcie->plda)) { + dev_info(pcie->plda.dev, "port link up\n"); + return 0; + } + msleep(PCIE_LINK_WAIT_SLEEP_MS); + } + + return -ETIMEDOUT; +} + +static int starfive_pcie_enable_phy(struct device *dev, + struct starfive_jh7110_pcie *pcie) +{ + int ret; + + if (!pcie->phy) + return 0; + + ret = phy_init(pcie->phy); + if (ret) + return dev_err_probe(dev, ret, + "failed to initialize pcie phy\n"); + + ret = phy_set_mode(pcie->phy, PHY_MODE_PCIE); + if (ret) { + dev_err_probe(dev, ret, "failed to set pcie mode\n"); + goto err_phy_on; + } + + ret = phy_power_on(pcie->phy); + if (ret) { + dev_err_probe(dev, ret, "failed to power on pcie phy\n"); + goto err_phy_on; + } + + return 0; + +err_phy_on: + phy_exit(pcie->phy); + return ret; +} + +static void starfive_pcie_disable_phy(struct starfive_jh7110_pcie *pcie) +{ + phy_power_off(pcie->phy); + phy_exit(pcie->phy); +} + +static void starfive_pcie_host_deinit(struct plda_pcie_rp *plda) +{ + struct starfive_jh7110_pcie *pcie = + container_of(plda, struct starfive_jh7110_pcie, plda); + + starfive_pcie_clk_rst_deinit(pcie); + if (pcie->vpcie3v3) + regulator_disable(pcie->vpcie3v3); + starfive_pcie_disable_phy(pcie); +} + +static int starfive_pcie_host_init(struct plda_pcie_rp *plda) +{ + struct starfive_jh7110_pcie *pcie = + container_of(plda, struct starfive_jh7110_pcie, plda); + struct device *dev = plda->dev; + int ret; + int i; + + ret = starfive_pcie_enable_phy(dev, pcie); + if (ret) + return ret; + + regmap_update_bits(pcie->reg_syscon, + pcie->stg_pcie_base + STG_SYSCON_RP_NEP_OFFSET, + STG_SYSCON_K_RP_NEP, STG_SYSCON_K_RP_NEP); + + regmap_update_bits(pcie->reg_syscon, + pcie->stg_pcie_base + STG_SYSCON_AW_OFFSET, + STG_SYSCON_CKREF_SRC_MASK, + FIELD_PREP(STG_SYSCON_CKREF_SRC_MASK, 2)); + + regmap_update_bits(pcie->reg_syscon, + pcie->stg_pcie_base + STG_SYSCON_AW_OFFSET, + STG_SYSCON_CLKREQ, STG_SYSCON_CLKREQ); + + ret = starfive_pcie_clk_rst_init(pcie); + if (ret) + goto err_disable_phy; + + if (pcie->vpcie3v3) { + ret = regulator_enable(pcie->vpcie3v3); + if (ret) { + dev_err_probe(dev, ret, "failed to enable vpcie3v3 regulator\n"); + goto err_clk_rst; + } + } + + if (pcie->reset_gpio) + gpiod_set_value_cansleep(pcie->reset_gpio, 1); + + /* Disable physical functions except #0 */ + for (i = 1; i < PCIE_FUNC_NUM; i++) { + regmap_update_bits(pcie->reg_syscon, + pcie->stg_pcie_base + STG_SYSCON_AR_OFFSET, + STG_SYSCON_AXI4_SLVL_AR_MASK, + STG_SYSCON_AXI4_SLVL_PHY_AR(i)); + + regmap_update_bits(pcie->reg_syscon, + pcie->stg_pcie_base + STG_SYSCON_AW_OFFSET, + STG_SYSCON_AXI4_SLVL_AW_MASK, + STG_SYSCON_AXI4_SLVL_PHY_AW(i)); + + plda_pcie_disable_func(plda); + } + + regmap_update_bits(pcie->reg_syscon, + pcie->stg_pcie_base + STG_SYSCON_AR_OFFSET, + STG_SYSCON_AXI4_SLVL_AR_MASK, 0); + regmap_update_bits(pcie->reg_syscon, + pcie->stg_pcie_base + STG_SYSCON_AW_OFFSET, + STG_SYSCON_AXI4_SLVL_AW_MASK, 0); + + plda_pcie_enable_root_port(plda); + plda_pcie_write_rc_bar(plda, 0); + + /* PCIe PCI Standard Configuration Identification Settings. */ + plda_pcie_set_standard_class(plda); + + /* + * The LTR message receiving is enabled by the register "PCIe Message + * Reception" as default, but the forward id & addr are uninitialized. + * If we do not disable LTR message forwarding here, or set a legal + * forwarding address, the kernel will get stuck. + * To workaround, disable the LTR message forwarding here before using + * this feature. + */ + plda_pcie_disable_ltr(plda); + + /* + * Enable the prefetchable memory window 64-bit addressing in JH7110. + * The 64-bits prefetchable address translation configurations in ATU + * can be work after enable the register setting below. + */ + plda_pcie_set_pref_win_64bit(plda); + + /* + * Ensure that PERST has been asserted for at least 100 ms, + * the sleep value is T_PVPERL from PCIe CEM spec r2.0 (Table 2-4) + */ + msleep(100); + if (pcie->reset_gpio) + gpiod_set_value_cansleep(pcie->reset_gpio, 0); + + /* + * With a Downstream Port (<=5GT/s), software must wait a minimum + * of 100ms following exit from a conventional reset before + * sending a configuration request to the device. + */ + msleep(PCIE_RESET_CONFIG_WAIT_MS); + + if (starfive_pcie_host_wait_for_link(pcie)) + dev_info(dev, "port link down\n"); + + return 0; + +err_clk_rst: + starfive_pcie_clk_rst_deinit(pcie); +err_disable_phy: + starfive_pcie_disable_phy(pcie); + + return ret; +} + +static const struct plda_pcie_host_ops sf_host_ops = { + .host_init = starfive_pcie_host_init, + .host_deinit = starfive_pcie_host_deinit, +}; + +static const struct plda_event stf_pcie_event = { + .intx_event = EVENT_PM_MSI_INT_INTX, + .msi_event = EVENT_PM_MSI_INT_MSI +}; + +static int starfive_pcie_probe(struct platform_device *pdev) +{ + struct starfive_jh7110_pcie *pcie; + struct device *dev = &pdev->dev; + struct plda_pcie_rp *plda; + int ret; + + pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL); + if (!pcie) + return -ENOMEM; + + plda = &pcie->plda; + plda->dev = dev; + + ret = starfive_pcie_parse_dt(pcie, dev); + if (ret) + return ret; + + pm_runtime_enable(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); + if (ret < 0) { + pm_runtime_disable(&pdev->dev); + return dev_err_probe(dev, ret, "failed to resume device\n"); + } + + plda->host_ops = &sf_host_ops; + plda->num_events = PLDA_MAX_EVENT_NUM; + /* mask doorbell event */ + plda->events_bitmap = GENMASK(PLDA_INT_EVENT_NUM - 1, 0) + & ~BIT(PLDA_AXI_DOORBELL) + & ~BIT(PLDA_PCIE_DOORBELL); + plda->events_bitmap <<= PLDA_NUM_DMA_EVENTS; + ret = plda_pcie_host_init(&pcie->plda, &starfive_pcie_ops, + &stf_pcie_event); + if (ret) { + pm_runtime_put_sync(&pdev->dev); + pm_runtime_disable(&pdev->dev); + return ret; + } + + platform_set_drvdata(pdev, pcie); + + return 0; +} + +static void starfive_pcie_remove(struct platform_device *pdev) +{ + struct starfive_jh7110_pcie *pcie = platform_get_drvdata(pdev); + + plda_pcie_host_deinit(&pcie->plda); + pm_runtime_put_sync(&pdev->dev); + pm_runtime_disable(&pdev->dev); + platform_set_drvdata(pdev, NULL); +} + +static int starfive_pcie_suspend_noirq(struct device *dev) +{ + struct starfive_jh7110_pcie *pcie = dev_get_drvdata(dev); + + clk_bulk_disable_unprepare(pcie->num_clks, pcie->clks); + starfive_pcie_disable_phy(pcie); + + return 0; +} + +static int starfive_pcie_resume_noirq(struct device *dev) +{ + struct starfive_jh7110_pcie *pcie = dev_get_drvdata(dev); + int ret; + + ret = starfive_pcie_enable_phy(dev, pcie); + if (ret) + return ret; + + ret = clk_bulk_prepare_enable(pcie->num_clks, pcie->clks); + if (ret) { + dev_err(dev, "failed to enable clocks\n"); + starfive_pcie_disable_phy(pcie); + return ret; + } + + return 0; +} + +static const struct dev_pm_ops starfive_pcie_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(starfive_pcie_suspend_noirq, + starfive_pcie_resume_noirq) +}; + +static const struct of_device_id starfive_pcie_of_match[] = { + { .compatible = "starfive,jh7110-pcie", }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, starfive_pcie_of_match); + +static struct platform_driver starfive_pcie_driver = { + .driver = { + .name = "pcie-starfive", + .of_match_table = of_match_ptr(starfive_pcie_of_match), + .pm = pm_sleep_ptr(&starfive_pcie_pm_ops), + }, + .probe = starfive_pcie_probe, + .remove = starfive_pcie_remove, +}; +module_platform_driver(starfive_pcie_driver); + +MODULE_DESCRIPTION("StarFive JH7110 PCIe host driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c new file mode 100644 index 000000000..241023ecf --- /dev/null +++ b/drivers/pci/controller/vmd.c @@ -0,0 +1,1287 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Volume Management Device driver + * Copyright (c) 2015, Intel Corporation. + */ + +#include <linux/device.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/irqchip/irq-msi-lib.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/pci.h> +#include <linux/pci-acpi.h> +#include <linux/pci-ecam.h> +#include <linux/srcu.h> +#include <linux/rculist.h> +#include <linux/rcupdate.h> + +#include <xen/xen.h> + +#include <asm/irqdomain.h> + +#define VMD_CFGBAR 0 +#define VMD_MEMBAR1 2 +#define VMD_MEMBAR2 4 + +#define PCI_REG_VMCAP 0x40 +#define BUS_RESTRICT_CAP(vmcap) (vmcap & 0x1) +#define PCI_REG_VMCONFIG 0x44 +#define BUS_RESTRICT_CFG(vmcfg) ((vmcfg >> 8) & 0x3) +#define VMCONFIG_MSI_REMAP 0x2 +#define PCI_REG_VMLOCK 0x70 +#define MB2_SHADOW_EN(vmlock) (vmlock & 0x2) + +#define MB2_SHADOW_OFFSET 0x2000 +#define MB2_SHADOW_SIZE 16 + +/* DMR BAR4 register offsets */ +#define SHADOW_MEMBAR1_28C1 0x2818 /* MEMBAR1 physical address */ +#define SHADOW_MEMBAR2_28C1 0x2820 /* MEMBAR2 physical address */ +#define BASE_ID_REG_28C1 0x2840 +#define MEMBAR2_OFFSET_28C1 0x30d0 + +enum vmd_features { + /* + * Device may contain registers which hint the physical location of the + * membars, in order to allow proper address translation during + * resource assignment to enable guest virtualization + */ + VMD_FEAT_HAS_MEMBAR_SHADOW = (1 << 0), + + /* + * Device may provide root port configuration information which limits + * bus numbering + */ + VMD_FEAT_HAS_BUS_RESTRICTIONS = (1 << 1), + + /* + * Device contains physical location shadow registers in + * vendor-specific capability space + */ + VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP = (1 << 2), + + /* + * Device may use MSI-X vector 0 for software triggering and will not + * be used for MSI remapping + */ + VMD_FEAT_OFFSET_FIRST_VECTOR = (1 << 3), + + /* + * Device can bypass remapping MSI-X transactions into its MSI-X table, + * avoiding the requirement of a VMD MSI domain for child device + * interrupt handling. + */ + VMD_FEAT_CAN_BYPASS_MSI_REMAP = (1 << 4), + + /* + * Enable ASPM on the PCIE root ports and set the default LTR of the + * storage devices on platforms where these values are not configured by + * BIOS. This is needed for laptops, which require these settings for + * proper power management of the SoC. + */ + VMD_FEAT_BIOS_PM_QUIRK = (1 << 5), + + /* + * Newer VMD with device ID 0x28C1 has unique settings compared to its + * predecessor where BIOS enumerates the entire VMD device tree and + * stores respective configurations including bus start range and + * shadow registers in VMD MMIO space in VMD BAR4/BAR5, otherwise + * referred to as MEMBAR2 or MSI-X BAR. + */ + VMD_FEAT_USE_BIOS_INFO = (1 << 6), +}; + +#define VMD_BIOS_PM_QUIRK_LTR 0x1003 /* 3145728 ns */ + +#define VMD_FEATS_CLIENT (VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP | \ + VMD_FEAT_HAS_BUS_RESTRICTIONS | \ + VMD_FEAT_OFFSET_FIRST_VECTOR | \ + VMD_FEAT_BIOS_PM_QUIRK) + +static DEFINE_IDA(vmd_instance_ida); + +/* + * Lock for manipulating VMD IRQ lists. + */ +static DEFINE_RAW_SPINLOCK(list_lock); + +/** + * struct vmd_irq - private data to map driver IRQ to the VMD shared vector + * @node: list item for parent traversal. + * @irq: back pointer to parent. + * @enabled: true if driver enabled IRQ + * @virq: the virtual IRQ value provided to the requesting driver. + * + * Every MSI/MSI-X IRQ requested for a device in a VMD domain will be mapped to + * a VMD IRQ using this structure. + */ +struct vmd_irq { + struct list_head node; + struct vmd_irq_list *irq; + bool enabled; + unsigned int virq; +}; + +/** + * struct vmd_irq_list - list of driver requested IRQs mapping to a VMD vector + * @irq_list: the list of irq's the VMD one demuxes to. + * @srcu: SRCU struct for local synchronization. + * @count: number of child IRQs assigned to this vector; used to track + * sharing. + * @virq: The underlying VMD Linux interrupt number + */ +struct vmd_irq_list { + struct list_head irq_list; + struct srcu_struct srcu; + unsigned int count; + unsigned int virq; +}; + +struct vmd_dev { + struct pci_dev *dev; + + raw_spinlock_t cfg_lock; + void __iomem *cfgbar; + + int msix_count; + struct vmd_irq_list *irqs; + + struct pci_sysdata sysdata; + struct resource resources[3]; + struct irq_domain *irq_domain; + struct pci_bus *bus; + u8 busn_start; + u8 first_vec; + char *name; + int instance; + unsigned long features; +}; + +static inline struct vmd_dev *vmd_from_bus(struct pci_bus *bus) +{ + return container_of(bus->sysdata, struct vmd_dev, sysdata); +} + +static inline unsigned int index_from_irqs(struct vmd_dev *vmd, + struct vmd_irq_list *irqs) +{ + return irqs - vmd->irqs; +} + +/* + * Drivers managing a device in a VMD domain allocate their own IRQs as before, + * but the MSI entry for the hardware it's driving will be programmed with a + * destination ID for the VMD MSI-X table. The VMD muxes interrupts in its + * domain into one of its own, and the VMD driver de-muxes these for the + * handlers sharing that VMD IRQ. The vmd irq_domain provides the operations + * and irq_chip to set this up. + */ +static void vmd_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +{ + struct vmd_irq *vmdirq = data->chip_data; + struct vmd_irq_list *irq = vmdirq->irq; + struct vmd_dev *vmd = irq_data_get_irq_handler_data(data); + + memset(msg, 0, sizeof(*msg)); + msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH; + msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW; + msg->arch_addr_lo.destid_0_7 = index_from_irqs(vmd, irq); +} + +static void vmd_irq_enable(struct irq_data *data) +{ + struct vmd_irq *vmdirq = data->chip_data; + + scoped_guard(raw_spinlock_irqsave, &list_lock) { + WARN_ON(vmdirq->enabled); + list_add_tail_rcu(&vmdirq->node, &vmdirq->irq->irq_list); + vmdirq->enabled = true; + } +} + +static void vmd_pci_msi_enable(struct irq_data *data) +{ + vmd_irq_enable(data->parent_data); + data->chip->irq_unmask(data); +} + +static unsigned int vmd_pci_msi_startup(struct irq_data *data) +{ + vmd_pci_msi_enable(data); + return 0; +} + +static void vmd_irq_disable(struct irq_data *data) +{ + struct vmd_irq *vmdirq = data->chip_data; + + scoped_guard(raw_spinlock_irqsave, &list_lock) { + if (vmdirq->enabled) { + list_del_rcu(&vmdirq->node); + vmdirq->enabled = false; + } + } +} + +static void vmd_pci_msi_disable(struct irq_data *data) +{ + data->chip->irq_mask(data); + vmd_irq_disable(data->parent_data); +} + +static void vmd_pci_msi_shutdown(struct irq_data *data) +{ + vmd_pci_msi_disable(data); +} + +static struct irq_chip vmd_msi_controller = { + .name = "VMD-MSI", + .irq_compose_msi_msg = vmd_compose_msi_msg, +}; + +/* + * XXX: We can be even smarter selecting the best IRQ once we solve the + * affinity problem. + */ +static struct vmd_irq_list *vmd_next_irq(struct vmd_dev *vmd, struct msi_desc *desc) +{ + int i, best; + + if (vmd->msix_count == 1 + vmd->first_vec) + return &vmd->irqs[vmd->first_vec]; + + /* + * White list for fast-interrupt handlers. All others will share the + * "slow" interrupt vector. + */ + switch (msi_desc_to_pci_dev(desc)->class) { + case PCI_CLASS_STORAGE_EXPRESS: + break; + default: + return &vmd->irqs[vmd->first_vec]; + } + + scoped_guard(raw_spinlock_irq, &list_lock) { + best = vmd->first_vec + 1; + for (i = best; i < vmd->msix_count; i++) + if (vmd->irqs[i].count < vmd->irqs[best].count) + best = i; + vmd->irqs[best].count++; + } + + return &vmd->irqs[best]; +} + +static void vmd_msi_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs); + +static int vmd_msi_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + struct msi_desc *desc = ((msi_alloc_info_t *)arg)->desc; + struct vmd_dev *vmd = domain->host_data; + struct vmd_irq *vmdirq; + + for (int i = 0; i < nr_irqs; ++i) { + vmdirq = kzalloc_obj(*vmdirq); + if (!vmdirq) { + vmd_msi_free(domain, virq, i); + return -ENOMEM; + } + + INIT_LIST_HEAD(&vmdirq->node); + vmdirq->irq = vmd_next_irq(vmd, desc); + vmdirq->virq = virq + i; + + irq_domain_set_info(domain, virq + i, vmdirq->irq->virq, + &vmd_msi_controller, vmdirq, + handle_untracked_irq, vmd, NULL); + } + + return 0; +} + +static void vmd_msi_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *irq_data; + struct vmd_irq *vmdirq; + + for (int i = 0; i < nr_irqs; ++i) { + irq_data = irq_domain_get_irq_data(domain, virq + i); + vmdirq = irq_data->chip_data; + + synchronize_srcu(&vmdirq->irq->srcu); + + /* XXX: Potential optimization to rebalance */ + scoped_guard(raw_spinlock_irq, &list_lock) + vmdirq->irq->count--; + + kfree(vmdirq); + } +} + +static const struct irq_domain_ops vmd_msi_domain_ops = { + .alloc = vmd_msi_alloc, + .free = vmd_msi_free, +}; + +static bool vmd_init_dev_msi_info(struct device *dev, struct irq_domain *domain, + struct irq_domain *real_parent, + struct msi_domain_info *info) +{ + if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info)) + return false; + + info->chip->irq_startup = vmd_pci_msi_startup; + info->chip->irq_shutdown = vmd_pci_msi_shutdown; + info->chip->irq_enable = vmd_pci_msi_enable; + info->chip->irq_disable = vmd_pci_msi_disable; + return true; +} + +#define VMD_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | MSI_FLAG_PCI_MSIX) +#define VMD_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_NO_AFFINITY) + +static const struct msi_parent_ops vmd_msi_parent_ops = { + .supported_flags = VMD_MSI_FLAGS_SUPPORTED, + .required_flags = VMD_MSI_FLAGS_REQUIRED, + .bus_select_token = DOMAIN_BUS_VMD_MSI, + .bus_select_mask = MATCH_PCI_MSI, + .prefix = "VMD-", + .init_dev_msi_info = vmd_init_dev_msi_info, +}; + +static int vmd_create_irq_domain(struct vmd_dev *vmd) +{ + struct irq_domain_info info = { + .size = vmd->msix_count, + .ops = &vmd_msi_domain_ops, + .host_data = vmd, + }; + + info.fwnode = irq_domain_alloc_named_id_fwnode("VMD-MSI", + vmd->sysdata.domain); + if (!info.fwnode) + return -ENODEV; + + vmd->irq_domain = msi_create_parent_irq_domain(&info, + &vmd_msi_parent_ops); + if (!vmd->irq_domain) { + irq_domain_free_fwnode(info.fwnode); + return -ENODEV; + } + + return 0; +} + +static void vmd_set_msi_remapping(struct vmd_dev *vmd, bool enable) +{ + u16 reg; + + if (!!(vmd->features & VMD_FEAT_USE_BIOS_INFO)) + return; + + pci_read_config_word(vmd->dev, PCI_REG_VMCONFIG, ®); + reg = enable ? (reg & ~VMCONFIG_MSI_REMAP) : + (reg | VMCONFIG_MSI_REMAP); + pci_write_config_word(vmd->dev, PCI_REG_VMCONFIG, reg); +} + +static void vmd_remove_irq_domain(struct vmd_dev *vmd) +{ + /* + * Some production BIOS won't enable remapping between soft reboots. + * Ensure remapping is restored before unloading the driver. + */ + if (!vmd->msix_count) + vmd_set_msi_remapping(vmd, true); + + if (vmd->irq_domain) { + struct fwnode_handle *fn = vmd->irq_domain->fwnode; + + irq_domain_remove(vmd->irq_domain); + irq_domain_free_fwnode(fn); + } +} + +static unsigned int vmd_bus_to_ecam(struct vmd_dev *vmd, unsigned int busnr) +{ + if (!!(vmd->features & VMD_FEAT_USE_BIOS_INFO)) + return busnr; + + return busnr - vmd->busn_start; +} + +static void __iomem *vmd_cfg_addr(struct vmd_dev *vmd, struct pci_bus *bus, + unsigned int devfn, int reg, int len) +{ + unsigned int busnr_ecam; + u32 offset; + + busnr_ecam = vmd_bus_to_ecam(vmd, bus->number); + offset = PCIE_ECAM_OFFSET(busnr_ecam, devfn, reg); + + if (offset + len >= resource_size(&vmd->dev->resource[VMD_CFGBAR])) + return NULL; + + return vmd->cfgbar + offset; +} + +/* + * CPU may deadlock if config space is not serialized on some versions of this + * hardware, so all config space access is done under a spinlock. + */ +static int vmd_pci_read(struct pci_bus *bus, unsigned int devfn, int reg, + int len, u32 *value) +{ + struct vmd_dev *vmd = vmd_from_bus(bus); + void __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len); + + if (!addr) + return -EFAULT; + + guard(raw_spinlock_irqsave)(&vmd->cfg_lock); + switch (len) { + case 1: + *value = readb(addr); + return 0; + case 2: + *value = readw(addr); + return 0; + case 4: + *value = readl(addr); + return 0; + default: + return -EINVAL; + } +} + +/* + * VMD h/w converts non-posted config writes to posted memory writes. The + * read-back in this function forces the completion so it returns only after + * the config space was written, as expected. + */ +static int vmd_pci_write(struct pci_bus *bus, unsigned int devfn, int reg, + int len, u32 value) +{ + struct vmd_dev *vmd = vmd_from_bus(bus); + void __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len); + + if (!addr) + return -EFAULT; + + guard(raw_spinlock_irqsave)(&vmd->cfg_lock); + switch (len) { + case 1: + writeb(value, addr); + readb(addr); + return 0; + case 2: + writew(value, addr); + readw(addr); + return 0; + case 4: + writel(value, addr); + readl(addr); + return 0; + default: + return -EINVAL; + } +} + +static struct pci_ops vmd_ops = { + .read = vmd_pci_read, + .write = vmd_pci_write, +}; + +#ifdef CONFIG_ACPI +static struct acpi_device *vmd_acpi_find_companion(struct pci_dev *pci_dev) +{ + struct pci_host_bridge *bridge; + u32 busnr, addr; + + if (pci_dev->bus->ops != &vmd_ops) + return NULL; + + bridge = pci_find_host_bridge(pci_dev->bus); + busnr = pci_dev->bus->number - bridge->bus->number; + /* + * The address computation below is only applicable to relative bus + * numbers below 32. + */ + if (busnr > 31) + return NULL; + + addr = (busnr << 24) | ((u32)pci_dev->devfn << 16) | 0x8000FFFFU; + + dev_dbg(&pci_dev->dev, "Looking for ACPI companion (address 0x%x)\n", + addr); + + return acpi_find_child_device(ACPI_COMPANION(bridge->dev.parent), addr, + false); +} + +static bool hook_installed; + +static void vmd_acpi_begin(void) +{ + if (pci_acpi_set_companion_lookup_hook(vmd_acpi_find_companion)) + return; + + hook_installed = true; +} + +static void vmd_acpi_end(void) +{ + if (!hook_installed) + return; + + pci_acpi_clear_companion_lookup_hook(); + hook_installed = false; +} +#else +static inline void vmd_acpi_begin(void) { } +static inline void vmd_acpi_end(void) { } +#endif /* CONFIG_ACPI */ + +static resource_size_t vmd_cfgbar_ecam_space(struct vmd_dev *vmd) +{ + resource_size_t cfgbar_buses; + unsigned int ecam_start; + + cfgbar_buses = resource_size(&vmd->dev->resource[VMD_CFGBAR]) >> 20; + ecam_start = vmd_bus_to_ecam(vmd, vmd->resources[0].start); + if (ecam_start >= cfgbar_buses) + return 0; + + return cfgbar_buses - ecam_start; +} +static void vmd_domain_reset(struct vmd_dev *vmd) +{ + u16 bus, max_buses = resource_size(&vmd->resources[0]); + u8 dev, functions, fn, hdr_type; + unsigned int ecam_bus; + char __iomem *base; + + max_buses = min_t(u16, max_buses, vmd_cfgbar_ecam_space(vmd)); + for (bus = 0; bus < max_buses; bus++) { + ecam_bus = vmd_bus_to_ecam(vmd, vmd->resources[0].start + bus); + for (dev = 0; dev < 32; dev++) { + base = vmd->cfgbar + PCIE_ECAM_OFFSET(ecam_bus, + PCI_DEVFN(dev, 0), 0); + + hdr_type = readb(base + PCI_HEADER_TYPE); + + functions = (hdr_type & PCI_HEADER_TYPE_MFD) ? 8 : 1; + for (fn = 0; fn < functions; fn++) { + base = vmd->cfgbar + PCIE_ECAM_OFFSET(ecam_bus, + PCI_DEVFN(dev, fn), 0); + + hdr_type = readb(base + PCI_HEADER_TYPE) & + PCI_HEADER_TYPE_MASK; + + if (hdr_type != PCI_HEADER_TYPE_BRIDGE || + (readw(base + PCI_CLASS_DEVICE) != + PCI_CLASS_BRIDGE_PCI)) + continue; + + /* + * Temporarily disable the I/O range before updating + * PCI_IO_BASE. + */ + writel(0x0000ffff, base + PCI_IO_BASE_UPPER16); + /* Update lower 16 bits of I/O base/limit */ + writew(0x00f0, base + PCI_IO_BASE); + /* Update upper 16 bits of I/O base/limit */ + writel(0, base + PCI_IO_BASE_UPPER16); + + /* MMIO Base/Limit */ + writel(0x0000fff0, base + PCI_MEMORY_BASE); + + /* Prefetchable MMIO Base/Limit */ + writel(0, base + PCI_PREF_LIMIT_UPPER32); + writel(0x0000fff0, base + PCI_PREF_MEMORY_BASE); + writel(0xffffffff, base + PCI_PREF_BASE_UPPER32); + } + } + } +} + +static void vmd_attach_resources(struct vmd_dev *vmd) +{ + vmd->dev->resource[VMD_MEMBAR1].child = &vmd->resources[1]; + vmd->dev->resource[VMD_MEMBAR2].child = &vmd->resources[2]; +} + +static void vmd_detach_resources(struct vmd_dev *vmd) +{ + vmd->dev->resource[VMD_MEMBAR1].child = NULL; + vmd->dev->resource[VMD_MEMBAR2].child = NULL; +} + +static int vmd_get_phys_offsets(struct vmd_dev *vmd, bool native_hint, + resource_size_t *offset1, + resource_size_t *offset2) +{ + struct pci_dev *dev = vmd->dev; + u64 phys1, phys2; + + if (native_hint) { + u32 vmlock; + int ret; + + ret = pci_read_config_dword(dev, PCI_REG_VMLOCK, &vmlock); + if (ret || PCI_POSSIBLE_ERROR(vmlock)) + return -ENODEV; + + if (MB2_SHADOW_EN(vmlock)) { + void __iomem *membar2; + + membar2 = pci_iomap(dev, VMD_MEMBAR2, 0); + if (!membar2) + return -ENOMEM; + phys1 = readq(membar2 + MB2_SHADOW_OFFSET); + phys2 = readq(membar2 + MB2_SHADOW_OFFSET + 8); + pci_iounmap(dev, membar2); + } else + return 0; + } else { + /* Hypervisor-Emulated Vendor-Specific Capability */ + int pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); + u32 reg, regu; + + pci_read_config_dword(dev, pos + 4, ®); + + /* "SHDW" */ + if (pos && reg == 0x53484457) { + pci_read_config_dword(dev, pos + 8, ®); + pci_read_config_dword(dev, pos + 12, ®u); + phys1 = (u64) regu << 32 | reg; + + pci_read_config_dword(dev, pos + 16, ®); + pci_read_config_dword(dev, pos + 20, ®u); + phys2 = (u64) regu << 32 | reg; + } else + return 0; + } + + *offset1 = dev->resource[VMD_MEMBAR1].start - + (phys1 & PCI_BASE_ADDRESS_MEM_MASK); + *offset2 = dev->resource[VMD_MEMBAR2].start - + (phys2 & PCI_BASE_ADDRESS_MEM_MASK); + + return 0; +} + +static int vmd_get_bus_number_start(struct vmd_dev *vmd) +{ + struct pci_dev *dev = vmd->dev; + u16 reg; + + pci_read_config_word(dev, PCI_REG_VMCAP, ®); + if (BUS_RESTRICT_CAP(reg)) { + pci_read_config_word(dev, PCI_REG_VMCONFIG, ®); + if (PCI_POSSIBLE_ERROR(reg)) + return -ENODEV; + + switch (BUS_RESTRICT_CFG(reg)) { + case 0: + vmd->busn_start = 0; + break; + case 1: + vmd->busn_start = 128; + break; + case 3: + case 2: + vmd->busn_start = 224; + break; + default: + pci_err(dev, "Unknown Bus Offset Setting (%d)\n", + BUS_RESTRICT_CFG(reg)); + return -ENODEV; + } + } + + return 0; +} + +static int vmd_get_bus_info_from_bar4(struct vmd_dev *vmd, + resource_size_t *offset1, + resource_size_t *offset2) +{ + u64 phys1, phys2, bar4_2840; + void __iomem *bar4; + u32 base_id; + u8 base_bus; + + bar4 = pci_ioremap_bar(vmd->dev, 4); + if (!bar4) + return -ENOMEM; + + /* Read shadow registers for MEMBAR1 and MEMBAR2 physical addresses */ + phys1 = readq(bar4 + SHADOW_MEMBAR1_28C1); + phys2 = readq(bar4 + SHADOW_MEMBAR2_28C1); + + /* + * Read and set bus start number from Base ID register. 24-bit Base ID + * register is part of 64-bit shadowed reqid hide range register and + * holds segment, bus, device and function. + */ + bar4_2840 = readq(bar4 + BASE_ID_REG_28C1); + base_id = bar4_2840 & 0xFFFFFF; + base_bus = base_id >> 8; + vmd->busn_start = base_bus; + + /* Calculate offsets like vmd_get_phys_offsets() does */ + if (phys1) + *offset1 = vmd->dev->resource[VMD_MEMBAR1].start - + (phys1 & PCI_BASE_ADDRESS_MEM_MASK); + if (phys2) + *offset2 = vmd->dev->resource[VMD_MEMBAR2].start - + (phys2 & PCI_BASE_ADDRESS_MEM_MASK); + + pci_iounmap(vmd->dev, bar4); + + return 0; +} + +static irqreturn_t vmd_irq(int irq, void *data) +{ + struct vmd_irq_list *irqs = data; + struct vmd_irq *vmdirq; + int idx; + + idx = srcu_read_lock(&irqs->srcu); + list_for_each_entry_rcu(vmdirq, &irqs->irq_list, node) + generic_handle_irq(vmdirq->virq); + srcu_read_unlock(&irqs->srcu, idx); + + return IRQ_HANDLED; +} + +static int vmd_alloc_irqs(struct vmd_dev *vmd) +{ + struct pci_dev *dev = vmd->dev; + int i, err; + + vmd->msix_count = pci_msix_vec_count(dev); + if (vmd->msix_count < 0) + return -ENODEV; + + vmd->msix_count = pci_alloc_irq_vectors(dev, vmd->first_vec + 1, + vmd->msix_count, PCI_IRQ_MSIX); + if (vmd->msix_count < 0) + return vmd->msix_count; + + vmd->irqs = devm_kcalloc(&dev->dev, vmd->msix_count, sizeof(*vmd->irqs), + GFP_KERNEL); + if (!vmd->irqs) + return -ENOMEM; + + for (i = 0; i < vmd->msix_count; i++) { + err = init_srcu_struct(&vmd->irqs[i].srcu); + if (err) + return err; + + INIT_LIST_HEAD(&vmd->irqs[i].irq_list); + vmd->irqs[i].virq = pci_irq_vector(dev, i); + err = devm_request_irq(&dev->dev, vmd->irqs[i].virq, + vmd_irq, IRQF_NO_THREAD, + vmd->name, &vmd->irqs[i]); + if (err) + return err; + } + + return 0; +} + +static int vmd_prepare_offsets_and_bus(struct vmd_dev *vmd, + unsigned long features, + resource_size_t *membar2_offset, + resource_size_t *offset1, + resource_size_t *offset2) +{ + int ret; + + /* + * Shadow registers may exist in certain VMD device IDs which allow + * guests to correctly assign host physical addresses to the root ports + * and child devices. These registers will either return the host value + * or 0, depending on an enable bit in the VMD device. + * + * For certain VMD devices (i.e. 0x28C1), BIOS places device info + * in BAR4 shadow registers to determine the base bus number and memory + * offsets. + */ + if (features & VMD_FEAT_USE_BIOS_INFO) { + *membar2_offset = MEMBAR2_OFFSET_28C1; + ret = vmd_get_bus_info_from_bar4(vmd, offset1, offset2); + if (ret) + return ret; + } else if (features & VMD_FEAT_HAS_MEMBAR_SHADOW) { + *membar2_offset = MB2_SHADOW_OFFSET + MB2_SHADOW_SIZE; + ret = vmd_get_phys_offsets(vmd, true, offset1, offset2); + if (ret) + return ret; + } else if (features & VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP) { + ret = vmd_get_phys_offsets(vmd, false, offset1, offset2); + if (ret) + return ret; + } + + /* + * Certain VMD devices may have a root port configuration option which + * limits the bus range to between 0-127, 128-255, or 224-255. + */ + if (features & VMD_FEAT_HAS_BUS_RESTRICTIONS) { + ret = vmd_get_bus_number_start(vmd); + if (ret) + return ret; + } + return 0; +} + +/* + * Since VMD is an aperture to regular PCIe root ports, only allow it to + * control features that the OS is allowed to control on the physical PCI bus. + */ +static void vmd_copy_host_bridge_flags(struct pci_host_bridge *root_bridge, + struct pci_host_bridge *vmd_bridge) +{ + vmd_bridge->native_pcie_hotplug = root_bridge->native_pcie_hotplug; + vmd_bridge->native_shpc_hotplug = root_bridge->native_shpc_hotplug; + vmd_bridge->native_aer = root_bridge->native_aer; + vmd_bridge->native_pme = root_bridge->native_pme; + vmd_bridge->native_ltr = root_bridge->native_ltr; + vmd_bridge->native_dpc = root_bridge->native_dpc; +} + +/* + * Enable ASPM and LTR settings on devices that aren't configured by BIOS. + */ +static int vmd_pm_enable_quirk(struct pci_dev *pdev, void *userdata) +{ + unsigned long features = *(unsigned long *)userdata; + u16 ltr = VMD_BIOS_PM_QUIRK_LTR; + u32 ltr_reg; + int pos; + + if (!(features & VMD_FEAT_BIOS_PM_QUIRK)) + return 0; + + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_LTR); + if (!pos) + goto out_state_change; + + /* + * Skip if the max snoop LTR is non-zero, indicating BIOS has set it + * so the LTR quirk is not needed. + */ + pci_read_config_dword(pdev, pos + PCI_LTR_MAX_SNOOP_LAT, <r_reg); + if (!!(ltr_reg & (PCI_LTR_VALUE_MASK | PCI_LTR_SCALE_MASK))) + goto out_state_change; + + /* + * Set the default values to the maximum required by the platform to + * allow the deepest power management savings. Write as a DWORD where + * the lower word is the max snoop latency and the upper word is the + * max non-snoop latency. + */ + ltr_reg = (ltr << 16) | ltr; + pci_write_config_dword(pdev, pos + PCI_LTR_MAX_SNOOP_LAT, ltr_reg); + pci_info(pdev, "VMD: Default LTR value set by driver\n"); + +out_state_change: + /* + * Ensure devices are in D0 before enabling PCI-PM L1 PM Substates, per + * PCIe r6.0, sec 5.5.4. + */ + pci_set_power_state_locked(pdev, PCI_D0); + pci_enable_link_state_locked(pdev, PCIE_LINK_STATE_ALL); + return 0; +} + +static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features) +{ + struct pci_sysdata *sd = &vmd->sysdata; + struct resource *res; + u32 upper_bits; + unsigned long flags; + LIST_HEAD(resources); + resource_size_t offset[2] = {0}; + resource_size_t membar2_offset = 0x2000; + resource_size_t busn_end; + struct pci_bus *child; + struct pci_dev *dev; + bool vmd_in_guest; + int ret; + + ret = vmd_prepare_offsets_and_bus(vmd, features, &membar2_offset, + &offset[0], &offset[1]); + if (ret) + return ret; + + /* Do not let resource[0] end go out of bounds */ + res = &vmd->dev->resource[VMD_CFGBAR]; + busn_end = vmd->busn_start + (resource_size(res) >> 20) - 1; + busn_end = min_t(resource_size_t, busn_end, 0xff); + vmd->resources[0] = (struct resource) { + .name = "VMD CFGBAR", + .start = vmd->busn_start, + .end = busn_end, + .flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED, + }; + + /* + * If the window is below 4GB, clear IORESOURCE_MEM_64 so we can + * put 32-bit resources in the window. + * + * There's no hardware reason why a 64-bit window *couldn't* + * contain a 32-bit resource, but pbus_size_mem() computes the + * bridge window size assuming a 64-bit window will contain no + * 32-bit resources. __pci_assign_resource() enforces that + * artificial restriction to make sure everything will fit. + * + * The only way we could use a 64-bit non-prefetchable MEMBAR is + * if its address is <4GB so that we can convert it to a 32-bit + * resource. To be visible to the host OS, all VMD endpoints must + * be initially configured by platform BIOS, which includes setting + * up these resources. We can assume the device is configured + * according to the platform needs. + */ + res = &vmd->dev->resource[VMD_MEMBAR1]; + upper_bits = upper_32_bits(res->end); + flags = res->flags & ~IORESOURCE_SIZEALIGN; + if (!upper_bits) + flags &= ~IORESOURCE_MEM_64; + vmd->resources[1] = (struct resource) { + .name = "VMD MEMBAR1", + .start = res->start, + .end = res->end, + .flags = flags, + .parent = res, + }; + + res = &vmd->dev->resource[VMD_MEMBAR2]; + upper_bits = upper_32_bits(res->end); + flags = res->flags & ~IORESOURCE_SIZEALIGN; + if (!upper_bits) + flags &= ~IORESOURCE_MEM_64; + vmd->resources[2] = (struct resource) { + .name = "VMD MEMBAR2", + .start = res->start + membar2_offset, + .end = res->end, + .flags = flags, + .parent = res, + }; + + /* Non-zero offset means guest/direct assign view. */ + vmd_in_guest = offset[0] || offset[1]; + + /* + * Currently MSI remapping must be enabled in guest passthrough mode + * due to some missing interrupt remapping plumbing. This is probably + * acceptable because the guest is usually CPU-limited and MSI + * remapping doesn't become a performance bottleneck. + */ + if (!(features & VMD_FEAT_CAN_BYPASS_MSI_REMAP) || vmd_in_guest) { + ret = vmd_alloc_irqs(vmd); + if (ret) + return ret; + + vmd_set_msi_remapping(vmd, true); + + ret = vmd_create_irq_domain(vmd); + if (ret) + return ret; + } else { + vmd_set_msi_remapping(vmd, false); + } + + pci_add_resource(&resources, &vmd->resources[0]); + pci_add_resource_offset(&resources, &vmd->resources[1], offset[0]); + pci_add_resource_offset(&resources, &vmd->resources[2], offset[1]); + + sd->vmd_dev = vmd->dev; + + /* + * Emulated domains start at 0x10000 to not clash with ACPI _SEG + * domains. Per ACPI r6.0, sec 6.5.6, _SEG returns an integer, of + * which the lower 16 bits are the PCI Segment Group (domain) number. + * Other bits are currently reserved. + */ + sd->domain = pci_bus_find_emul_domain_nr(0, 0x10000, INT_MAX); + if (sd->domain < 0) + return sd->domain; + + sd->node = pcibus_to_node(vmd->dev->bus); + + vmd->bus = pci_create_root_bus(&vmd->dev->dev, vmd->busn_start, + &vmd_ops, sd, &resources); + if (!vmd->bus) { + pci_bus_release_emul_domain_nr(sd->domain); + pci_free_resource_list(&resources); + vmd_remove_irq_domain(vmd); + return -ENODEV; + } + + /* + * Don't copy _OSC control flags from root bridge if running in a VM, as + * they don't reflect the physical root bridge capabilities. + */ + if (!vmd_in_guest) + vmd_copy_host_bridge_flags(pci_find_host_bridge(vmd->dev->bus), + to_pci_host_bridge(vmd->bus->bridge)); + + vmd_attach_resources(vmd); + if (vmd->irq_domain) + dev_set_msi_domain(&vmd->bus->dev, vmd->irq_domain); + else + dev_set_msi_domain(&vmd->bus->dev, + dev_get_msi_domain(&vmd->dev->dev)); + + WARN(sysfs_create_link(&vmd->dev->dev.kobj, &vmd->bus->dev.kobj, + "domain"), "Can't create symlink to domain\n"); + + vmd_acpi_begin(); + + pci_scan_child_bus(vmd->bus); + vmd_domain_reset(vmd); + + /* When Intel VMD is enabled, the OS does not discover the Root Ports + * owned by Intel VMD within the MMCFG space. pci_reset_bus() applies + * a reset to the parent of the PCI device supplied as argument. This + * is why we pass a child device, so the reset can be triggered at + * the Intel bridge level and propagated to all the children in the + * hierarchy. + */ + list_for_each_entry(child, &vmd->bus->children, node) { + if (!list_empty(&child->devices)) { + dev = list_first_entry(&child->devices, + struct pci_dev, bus_list); + ret = pci_reset_bus(dev); + if (ret) + pci_warn(dev, "can't reset device: %d\n", ret); + + break; + } + } + + pci_assign_unassigned_bus_resources(vmd->bus); + + pci_walk_bus(vmd->bus, vmd_pm_enable_quirk, &features); + + /* + * VMD root buses are virtual and don't return true on pci_is_pcie() + * and will fail pcie_bus_configure_settings() early. It can instead be + * run on each of the real root ports. + */ + list_for_each_entry(child, &vmd->bus->children, node) + pcie_bus_configure_settings(child); + + pci_bus_add_devices(vmd->bus); + + vmd_acpi_end(); + return 0; +} + +static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id) +{ + unsigned long features = (unsigned long) id->driver_data; + struct vmd_dev *vmd; + int err; + + if (xen_domain()) { + /* + * Xen doesn't have knowledge about devices in the VMD bus + * because the config space of devices behind the VMD bridge is + * not known to Xen, and hence Xen cannot discover or configure + * them in any way. + * + * Bypass of MSI remapping won't work in that case as direct + * write by Linux to the MSI entries won't result in functional + * interrupts, as Xen is the entity that manages the host + * interrupt controller and must configure interrupts. However + * multiplexing of interrupts by the VMD bridge will work under + * Xen, so force the usage of that mode which must always be + * supported by VMD bridges. + */ + features &= ~VMD_FEAT_CAN_BYPASS_MSI_REMAP; + } + + if (resource_size(&dev->resource[VMD_CFGBAR]) < (1 << 20)) + return -ENOMEM; + + vmd = devm_kzalloc(&dev->dev, sizeof(*vmd), GFP_KERNEL); + if (!vmd) + return -ENOMEM; + + vmd->dev = dev; + vmd->sysdata.domain = PCI_DOMAIN_NR_NOT_SET; + vmd->features = features; + vmd->instance = ida_alloc(&vmd_instance_ida, GFP_KERNEL); + if (vmd->instance < 0) + return vmd->instance; + + vmd->name = devm_kasprintf(&dev->dev, GFP_KERNEL, "vmd%d", + vmd->instance); + if (!vmd->name) { + err = -ENOMEM; + goto out_release_instance; + } + + err = pcim_enable_device(dev); + if (err < 0) + goto out_release_instance; + + vmd->cfgbar = pcim_iomap(dev, VMD_CFGBAR, 0); + if (!vmd->cfgbar) { + err = -ENOMEM; + goto out_release_instance; + } + + pci_set_master(dev); + if (dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(64)) && + dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(32))) { + err = -ENODEV; + goto out_release_instance; + } + + if (features & VMD_FEAT_OFFSET_FIRST_VECTOR) + vmd->first_vec = 1; + + raw_spin_lock_init(&vmd->cfg_lock); + pci_set_drvdata(dev, vmd); + err = vmd_enable_domain(vmd, features); + if (err) + goto out_release_instance; + + dev_info(&vmd->dev->dev, "Bound to PCI domain %04x\n", + vmd->sysdata.domain); + return 0; + + out_release_instance: + ida_free(&vmd_instance_ida, vmd->instance); + return err; +} + +static void vmd_cleanup_srcu(struct vmd_dev *vmd) +{ + int i; + + for (i = 0; i < vmd->msix_count; i++) + cleanup_srcu_struct(&vmd->irqs[i].srcu); +} + +static void vmd_remove(struct pci_dev *dev) +{ + struct vmd_dev *vmd = pci_get_drvdata(dev); + + pci_stop_root_bus(vmd->bus); + sysfs_remove_link(&vmd->dev->dev.kobj, "domain"); + pci_remove_root_bus(vmd->bus); + vmd_cleanup_srcu(vmd); + vmd_detach_resources(vmd); + vmd_remove_irq_domain(vmd); + ida_free(&vmd_instance_ida, vmd->instance); + pci_bus_release_emul_domain_nr(vmd->sysdata.domain); +} + +static void vmd_shutdown(struct pci_dev *dev) +{ + struct vmd_dev *vmd = pci_get_drvdata(dev); + + vmd_remove_irq_domain(vmd); +} + +#ifdef CONFIG_PM_SLEEP +static int vmd_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct vmd_dev *vmd = pci_get_drvdata(pdev); + int i; + + for (i = 0; i < vmd->msix_count; i++) + devm_free_irq(dev, vmd->irqs[i].virq, &vmd->irqs[i]); + + return 0; +} + +static int vmd_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct vmd_dev *vmd = pci_get_drvdata(pdev); + int err, i; + + vmd_set_msi_remapping(vmd, !!vmd->irq_domain); + + for (i = 0; i < vmd->msix_count; i++) { + err = devm_request_irq(dev, vmd->irqs[i].virq, + vmd_irq, IRQF_NO_THREAD, + vmd->name, &vmd->irqs[i]); + if (err) + return err; + } + + return 0; +} +#endif +static SIMPLE_DEV_PM_OPS(vmd_dev_pm_ops, vmd_suspend, vmd_resume); + +static const struct pci_device_id vmd_ids[] = { + {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_VMD_201D), + .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP,}, + {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_VMD_28C0), + .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW | + VMD_FEAT_HAS_BUS_RESTRICTIONS | + VMD_FEAT_CAN_BYPASS_MSI_REMAP,}, + {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_VMD_28C1), + .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW | + VMD_FEAT_CAN_BYPASS_MSI_REMAP | + VMD_FEAT_USE_BIOS_INFO,}, + {PCI_VDEVICE(INTEL, 0x467f), + .driver_data = VMD_FEATS_CLIENT,}, + {PCI_VDEVICE(INTEL, 0x4c3d), + .driver_data = VMD_FEATS_CLIENT,}, + {PCI_VDEVICE(INTEL, 0xa77f), + .driver_data = VMD_FEATS_CLIENT,}, + {PCI_VDEVICE(INTEL, 0x7d0b), + .driver_data = VMD_FEATS_CLIENT,}, + {PCI_VDEVICE(INTEL, 0xad0b), + .driver_data = VMD_FEATS_CLIENT,}, + {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_VMD_9A0B), + .driver_data = VMD_FEATS_CLIENT,}, + {PCI_VDEVICE(INTEL, 0xb60b), + .driver_data = VMD_FEATS_CLIENT,}, + {PCI_VDEVICE(INTEL, 0xb06f), + .driver_data = VMD_FEATS_CLIENT,}, + {PCI_VDEVICE(INTEL, 0xb07f), + .driver_data = VMD_FEATS_CLIENT,}, + {PCI_VDEVICE(INTEL, 0xd70b), + .driver_data = VMD_FEATS_CLIENT,}, + {PCI_VDEVICE(INTEL, 0xd73b), + .driver_data = VMD_FEATS_CLIENT,}, + {0,} +}; +MODULE_DEVICE_TABLE(pci, vmd_ids); + +static struct pci_driver vmd_drv = { + .name = "vmd", + .id_table = vmd_ids, + .probe = vmd_probe, + .remove = vmd_remove, + .shutdown = vmd_shutdown, + .driver = { + .pm = &vmd_dev_pm_ops, + }, +}; +module_pci_driver(vmd_drv); + +MODULE_AUTHOR("Intel Corporation"); +MODULE_DESCRIPTION("Volume Management Device driver"); +MODULE_LICENSE("GPL v2"); +MODULE_VERSION("0.6"); |