diff options
Diffstat (limited to 'drivers/ufs/host')
30 files changed, 14626 insertions, 0 deletions
diff --git a/drivers/ufs/host/Kconfig b/drivers/ufs/host/Kconfig new file mode 100644 index 000000000..ff170c0b6 --- /dev/null +++ b/drivers/ufs/host/Kconfig @@ -0,0 +1,170 @@ +# SPDX-License-Identifier: GPL-2.0+ +# +# Kernel configuration file for the UFS host controller drivers. +# +# Copyright (C) 2011-2013 Samsung India Software Operations +# +# Authors: +# Santosh Yaraganavi <santosh.sy@samsung.com> +# Vinayak Holikatti <h.vinayak@samsung.com> + +config SCSI_UFSHCD_PCI + tristate "PCI bus based UFS Controller support" + depends on PCI + help + This selects the PCI UFS Host Controller Interface. Select this if + you have UFS Host Controller with PCI Interface. + + If you have a controller with this interface, say Y or M here. + + If unsure, say N. + +config SCSI_UFS_DWC_TC_PCI + tristate "DesignWare pci support using a G210 Test Chip" + depends on SCSI_UFSHCD_PCI + help + Synopsys Test Chip is a PHY for prototyping purposes. + + If unsure, say N. + +config SCSI_UFSHCD_PLATFORM + tristate "Platform bus based UFS Controller support" + depends on HAS_IOMEM + help + This selects the UFS host controller support. Select this if + you have an UFS controller on Platform bus. + + If you have a controller with this interface, say Y or M here. + + If unsure, say N. + +config SCSI_UFS_CDNS_PLATFORM + tristate "Cadence UFS Controller platform driver" + depends on SCSI_UFSHCD_PLATFORM + help + This selects the Cadence-specific additions to UFSHCD platform driver. + + If unsure, say N. + +config SCSI_UFS_DWC_TC_PLATFORM + tristate "DesignWare platform support using a G210 Test Chip" + depends on OF && SCSI_UFSHCD_PLATFORM + help + Synopsys Test Chip is a PHY for prototyping purposes. + + If unsure, say N. + +config SCSI_UFS_QCOM + tristate "Qualcomm specific hooks to UFS controller platform driver" + depends on SCSI_UFSHCD_PLATFORM && ARCH_QCOM + depends on GENERIC_MSI_IRQ + depends on RESET_CONTROLLER + select QCOM_INLINE_CRYPTO_ENGINE if SCSI_UFS_CRYPTO + help + This selects the QCOM specific additions to UFSHCD platform driver. + UFS host on QCOM needs some vendor specific configuration before + accessing the hardware which includes PHY configuration and vendor + specific registers. + + Select this if you have UFS controller on QCOM chipset. + If unsure, say N. + +config SCSI_UFS_MEDIATEK + tristate "Mediatek specific hooks to UFS controller platform driver" + depends on SCSI_UFSHCD_PLATFORM && ARCH_MEDIATEK + depends on PM + depends on RESET_CONTROLLER + select PHY_MTK_UFS + select RESET_TI_SYSCON + help + This selects the Mediatek specific additions to UFSHCD platform driver. + UFS host on Mediatek needs some vendor specific configuration before + accessing the hardware which includes PHY configuration and vendor + specific registers. + + Select this if you have UFS controller on Mediatek chipset. + + If unsure, say N. + +config SCSI_UFS_HISI + tristate "Hisilicon specific hooks to UFS controller platform driver" + depends on (ARCH_HISI || COMPILE_TEST) && SCSI_UFSHCD_PLATFORM + help + This selects the Hisilicon specific additions to UFSHCD platform driver. + + Select this if you have UFS controller on Hisilicon chipset. + If unsure, say N. + +config SCSI_UFS_RENESAS + tristate "Renesas specific hooks to UFS controller platform driver" + depends on (ARCH_RENESAS || COMPILE_TEST) && SCSI_UFSHCD_PLATFORM + help + This selects the Renesas specific additions to UFSHCD platform driver. + UFS host on Renesas needs some vendor specific configuration before + accessing the hardware. + + Select this if you have UFS controller on Renesas chipset. + + If unsure, say N. + +config SCSI_UFS_TI_J721E + tristate "TI glue layer for Cadence UFS Controller" + depends on OF && HAS_IOMEM && (ARCH_K3 || COMPILE_TEST) + help + This selects driver for TI glue layer for Cadence UFS Host + Controller IP. + + Selects this if you have TI platform with UFS controller. + If unsure, say N. + +config SCSI_UFS_EXYNOS + tristate "Exynos specific hooks to UFS controller platform driver" + depends on SCSI_UFSHCD_PLATFORM && (ARCH_EXYNOS || COMPILE_TEST) + help + This selects the Samsung Exynos SoC specific additions to UFSHCD + platform driver. UFS host on Samsung Exynos SoC includes HCI and + UNIPRO layer, and associates with UFS-PHY driver. + + Select this if you have UFS host controller on Samsung Exynos SoC. + If unsure, say N. + +config SCSI_UFS_VARIABLE_SG_ENTRY_SIZE + bool + default y if SCSI_UFS_EXYNOS && SCSI_UFS_CRYPTO + +config SCSI_UFS_SPRD + tristate "Unisoc specific hooks to UFS controller platform driver" + depends on SCSI_UFSHCD_PLATFORM && (ARCH_SPRD || COMPILE_TEST) + help + This selects the Unisoc specific additions to UFSHCD platform driver. + UFS host on Unisoc needs some vendor specific configuration before + accessing the hardware which includes PHY configuration and vendor + specific registers. + + Select this if you have UFS controller on Unisoc chipset. + If unsure, say N. + +config SCSI_UFS_ROCKCHIP + tristate "Rockchip UFS host controller driver" + depends on SCSI_UFSHCD_PLATFORM && (ARCH_ROCKCHIP || COMPILE_TEST) + help + This selects the Rockchip specific additions to UFSHCD platform driver. + UFS host on Rockchip needs some vendor specific configuration before + accessing the hardware which includes PHY configuration and vendor + specific registers. + + Select this if you have UFS controller on Rockchip chipset. + If unsure, say N. + +config SCSI_UFS_AMD_VERSAL2 + tristate "AMD Versal Gen 2 UFS controller platform driver" + depends on SCSI_UFSHCD_PLATFORM && (ARCH_ZYNQMP || COMPILE_TEST) + help + This selects the AMD Versal Gen 2 specific additions on top of + the UFSHCD DWC and UFSHCD platform driver. UFS host on AMD + Versal Gen 2 needs some vendor specific configurations like PHY + and vendor specific register accesses before accessing the + hardware. + + Select this if you have UFS controller on AMD Versal Gen 2 SoC. + If unsure, say N. diff --git a/drivers/ufs/host/Makefile b/drivers/ufs/host/Makefile new file mode 100644 index 000000000..65d8bb23a --- /dev/null +++ b/drivers/ufs/host/Makefile @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_SCSI_UFS_DWC_TC_PCI) += tc-dwc-g210-pci.o ufshcd-dwc.o tc-dwc-g210.o +obj-$(CONFIG_SCSI_UFS_DWC_TC_PLATFORM) += tc-dwc-g210-pltfrm.o ufshcd-dwc.o tc-dwc-g210.o +obj-$(CONFIG_SCSI_UFS_CDNS_PLATFORM) += cdns-pltfrm.o +obj-$(CONFIG_SCSI_UFS_QCOM) += ufs-qcom.o +obj-$(CONFIG_SCSI_UFS_EXYNOS) += ufs-exynos.o +obj-$(CONFIG_SCSI_UFSHCD_PCI) += ufshcd-pci.o +obj-$(CONFIG_SCSI_UFSHCD_PLATFORM) += ufshcd-pltfrm.o +obj-$(CONFIG_SCSI_UFS_HISI) += ufs-hisi.o +obj-$(CONFIG_SCSI_UFS_MEDIATEK) += ufs-mediatek.o +obj-$(CONFIG_SCSI_UFS_RENESAS) += ufs-renesas.o +obj-$(CONFIG_SCSI_UFS_ROCKCHIP) += ufs-rockchip.o +obj-$(CONFIG_SCSI_UFS_SPRD) += ufs-sprd.o +obj-$(CONFIG_SCSI_UFS_TI_J721E) += ti-j721e-ufs.o +obj-$(CONFIG_SCSI_UFS_AMD_VERSAL2) += ufs-amd-versal2.o ufshcd-dwc.o diff --git a/drivers/ufs/host/cdns-pltfrm.c b/drivers/ufs/host/cdns-pltfrm.c new file mode 100644 index 000000000..e793e3538 --- /dev/null +++ b/drivers/ufs/host/cdns-pltfrm.c @@ -0,0 +1,334 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Platform UFS Host driver for Cadence controller + * + * Copyright (C) 2018 Cadence Design Systems, Inc. + * + * Authors: + * Jan Kotas <jank@cadence.com> + * + */ + +#include <linux/clk.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/of.h> +#include <linux/time.h> + +#include "ufshcd-pltfrm.h" + +#define CDNS_UFS_REG_HCLKDIV 0xFC +#define CDNS_UFS_REG_PHY_XCFGD1 0x113C +#define CDNS_UFS_MAX_L4_ATTRS 12 + +struct cdns_ufs_host { + /** + * cdns_ufs_dme_attr_val - for storing L4 attributes + */ + u32 cdns_ufs_dme_attr_val[CDNS_UFS_MAX_L4_ATTRS]; +}; + +/** + * cdns_ufs_get_l4_attr - get L4 attributes on local side + * @hba: per adapter instance + * + */ +static void cdns_ufs_get_l4_attr(struct ufs_hba *hba) +{ + struct cdns_ufs_host *host = ufshcd_get_variant(hba); + + ufshcd_dme_get(hba, UIC_ARG_MIB(T_PEERDEVICEID), + &host->cdns_ufs_dme_attr_val[0]); + ufshcd_dme_get(hba, UIC_ARG_MIB(T_PEERCPORTID), + &host->cdns_ufs_dme_attr_val[1]); + ufshcd_dme_get(hba, UIC_ARG_MIB(T_TRAFFICCLASS), + &host->cdns_ufs_dme_attr_val[2]); + ufshcd_dme_get(hba, UIC_ARG_MIB(T_PROTOCOLID), + &host->cdns_ufs_dme_attr_val[3]); + ufshcd_dme_get(hba, UIC_ARG_MIB(T_CPORTFLAGS), + &host->cdns_ufs_dme_attr_val[4]); + ufshcd_dme_get(hba, UIC_ARG_MIB(T_TXTOKENVALUE), + &host->cdns_ufs_dme_attr_val[5]); + ufshcd_dme_get(hba, UIC_ARG_MIB(T_RXTOKENVALUE), + &host->cdns_ufs_dme_attr_val[6]); + ufshcd_dme_get(hba, UIC_ARG_MIB(T_LOCALBUFFERSPACE), + &host->cdns_ufs_dme_attr_val[7]); + ufshcd_dme_get(hba, UIC_ARG_MIB(T_PEERBUFFERSPACE), + &host->cdns_ufs_dme_attr_val[8]); + ufshcd_dme_get(hba, UIC_ARG_MIB(T_CREDITSTOSEND), + &host->cdns_ufs_dme_attr_val[9]); + ufshcd_dme_get(hba, UIC_ARG_MIB(T_CPORTMODE), + &host->cdns_ufs_dme_attr_val[10]); + ufshcd_dme_get(hba, UIC_ARG_MIB(T_CONNECTIONSTATE), + &host->cdns_ufs_dme_attr_val[11]); +} + +/** + * cdns_ufs_set_l4_attr - set L4 attributes on local side + * @hba: per adapter instance + * + */ +static void cdns_ufs_set_l4_attr(struct ufs_hba *hba) +{ + struct cdns_ufs_host *host = ufshcd_get_variant(hba); + + ufshcd_dme_set(hba, UIC_ARG_MIB(T_CONNECTIONSTATE), 0); + ufshcd_dme_set(hba, UIC_ARG_MIB(T_PEERDEVICEID), + host->cdns_ufs_dme_attr_val[0]); + ufshcd_dme_set(hba, UIC_ARG_MIB(T_PEERCPORTID), + host->cdns_ufs_dme_attr_val[1]); + ufshcd_dme_set(hba, UIC_ARG_MIB(T_TRAFFICCLASS), + host->cdns_ufs_dme_attr_val[2]); + ufshcd_dme_set(hba, UIC_ARG_MIB(T_PROTOCOLID), + host->cdns_ufs_dme_attr_val[3]); + ufshcd_dme_set(hba, UIC_ARG_MIB(T_CPORTFLAGS), + host->cdns_ufs_dme_attr_val[4]); + ufshcd_dme_set(hba, UIC_ARG_MIB(T_TXTOKENVALUE), + host->cdns_ufs_dme_attr_val[5]); + ufshcd_dme_set(hba, UIC_ARG_MIB(T_RXTOKENVALUE), + host->cdns_ufs_dme_attr_val[6]); + ufshcd_dme_set(hba, UIC_ARG_MIB(T_LOCALBUFFERSPACE), + host->cdns_ufs_dme_attr_val[7]); + ufshcd_dme_set(hba, UIC_ARG_MIB(T_PEERBUFFERSPACE), + host->cdns_ufs_dme_attr_val[8]); + ufshcd_dme_set(hba, UIC_ARG_MIB(T_CREDITSTOSEND), + host->cdns_ufs_dme_attr_val[9]); + ufshcd_dme_set(hba, UIC_ARG_MIB(T_CPORTMODE), + host->cdns_ufs_dme_attr_val[10]); + ufshcd_dme_set(hba, UIC_ARG_MIB(T_CONNECTIONSTATE), + host->cdns_ufs_dme_attr_val[11]); +} + +/** + * cdns_ufs_set_hclkdiv() - set HCLKDIV register value based on the core_clk. + * @hba: host controller instance + * + * Return: zero for success and non-zero for failure. + */ +static int cdns_ufs_set_hclkdiv(struct ufs_hba *hba) +{ + struct ufs_clk_info *clki; + struct list_head *head = &hba->clk_list_head; + unsigned long core_clk_rate = 0; + u32 core_clk_div = 0; + + if (list_empty(head)) + return 0; + + list_for_each_entry(clki, head, list) { + if (IS_ERR_OR_NULL(clki->clk)) + continue; + if (!strcmp(clki->name, "core_clk")) + core_clk_rate = clk_get_rate(clki->clk); + } + + if (!core_clk_rate) { + dev_err(hba->dev, "%s: unable to find core_clk rate\n", + __func__); + return -EINVAL; + } + + core_clk_div = core_clk_rate / USEC_PER_SEC; + + ufshcd_writel(hba, core_clk_div, CDNS_UFS_REG_HCLKDIV); + /** + * Make sure the register was updated, + * UniPro layer will not work with an incorrect value. + */ + ufshcd_readl(hba, CDNS_UFS_REG_HCLKDIV); + + return 0; +} + +/** + * cdns_ufs_hce_enable_notify() - set HCLKDIV register + * @hba: host controller instance + * @status: notify stage (pre, post change) + * + * Return: zero for success and non-zero for failure. + */ +static int cdns_ufs_hce_enable_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status) +{ + if (status != PRE_CHANGE) + return 0; + + return cdns_ufs_set_hclkdiv(hba); +} + +/** + * cdns_ufs_hibern8_notify() - save and restore L4 attributes. + * @hba: host controller instance + * @cmd: UIC Command + * @status: notify stage (pre, post change) + */ +static void cdns_ufs_hibern8_notify(struct ufs_hba *hba, enum uic_cmd_dme cmd, + enum ufs_notify_change_status status) +{ + if (status == PRE_CHANGE && cmd == UIC_CMD_DME_HIBER_ENTER) + cdns_ufs_get_l4_attr(hba); + if (status == POST_CHANGE && cmd == UIC_CMD_DME_HIBER_EXIT) + cdns_ufs_set_l4_attr(hba); +} + +/** + * cdns_ufs_link_startup_notify() - handle link startup. + * @hba: host controller instance + * @status: notify stage (pre, post change) + * + * Return: zero for success and non-zero for failure. + */ +static int cdns_ufs_link_startup_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status) +{ + if (status != PRE_CHANGE) + return 0; + + /* + * Some UFS devices have issues if LCC is enabled. + * So we are setting PA_Local_TX_LCC_Enable to 0 + * before link startup which will make sure that both host + * and device TX LCC are disabled once link startup is + * completed. + */ + ufshcd_disable_host_tx_lcc(hba); + + /* + * Disabling Autohibern8 feature in cadence UFS + * to mask unexpected interrupt trigger. + */ + hba->ahit = 0; + + return 0; +} + +/** + * cdns_ufs_init - performs additional ufs initialization + * @hba: host controller instance + * + * Return: status of initialization. + */ +static int cdns_ufs_init(struct ufs_hba *hba) +{ + int status = 0; + struct cdns_ufs_host *host; + struct device *dev = hba->dev; + + host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL); + + if (!host) + return -ENOMEM; + ufshcd_set_variant(hba, host); + + status = ufshcd_vops_phy_initialization(hba); + + return status; +} + +/** + * cdns_ufs_m31_16nm_phy_initialization - performs m31 phy initialization + * @hba: host controller instance + * + * Return: 0 (success). + */ +static int cdns_ufs_m31_16nm_phy_initialization(struct ufs_hba *hba) +{ + u32 data; + + /* Increase RX_Advanced_Min_ActivateTime_Capability */ + data = ufshcd_readl(hba, CDNS_UFS_REG_PHY_XCFGD1); + data |= BIT(24); + ufshcd_writel(hba, data, CDNS_UFS_REG_PHY_XCFGD1); + + return 0; +} + +static const struct ufs_hba_variant_ops cdns_ufs_pltfm_hba_vops = { + .name = "cdns-ufs-pltfm", + .init = cdns_ufs_init, + .hce_enable_notify = cdns_ufs_hce_enable_notify, + .link_startup_notify = cdns_ufs_link_startup_notify, + .hibern8_notify = cdns_ufs_hibern8_notify, +}; + +static const struct ufs_hba_variant_ops cdns_ufs_m31_16nm_pltfm_hba_vops = { + .name = "cdns-ufs-pltfm", + .init = cdns_ufs_init, + .hce_enable_notify = cdns_ufs_hce_enable_notify, + .link_startup_notify = cdns_ufs_link_startup_notify, + .phy_initialization = cdns_ufs_m31_16nm_phy_initialization, + .hibern8_notify = cdns_ufs_hibern8_notify, +}; + +static const struct of_device_id cdns_ufs_of_match[] = { + { + .compatible = "cdns,ufshc", + .data = &cdns_ufs_pltfm_hba_vops, + }, + { + .compatible = "cdns,ufshc-m31-16nm", + .data = &cdns_ufs_m31_16nm_pltfm_hba_vops, + }, + { }, +}; + +MODULE_DEVICE_TABLE(of, cdns_ufs_of_match); + +/** + * cdns_ufs_pltfrm_probe - probe routine of the driver + * @pdev: pointer to platform device handle + * + * Return: zero for success and non-zero for failure. + */ +static int cdns_ufs_pltfrm_probe(struct platform_device *pdev) +{ + int err; + const struct of_device_id *of_id; + struct ufs_hba_variant_ops *vops; + struct device *dev = &pdev->dev; + + of_id = of_match_node(cdns_ufs_of_match, dev->of_node); + vops = (struct ufs_hba_variant_ops *)of_id->data; + + /* Perform generic probe */ + err = ufshcd_pltfrm_init(pdev, vops); + if (err) + dev_err(dev, "ufshcd_pltfrm_init() failed %d\n", err); + + return err; +} + +/** + * cdns_ufs_pltfrm_remove - removes the ufs driver + * @pdev: pointer to platform device handle + * + * Return: 0 (success). + */ +static void cdns_ufs_pltfrm_remove(struct platform_device *pdev) +{ + ufshcd_pltfrm_remove(pdev); +} + +static const struct dev_pm_ops cdns_ufs_dev_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(ufshcd_system_suspend, ufshcd_system_resume) + SET_RUNTIME_PM_OPS(ufshcd_runtime_suspend, ufshcd_runtime_resume, NULL) + .prepare = ufshcd_suspend_prepare, + .complete = ufshcd_resume_complete, +}; + +static struct platform_driver cdns_ufs_pltfrm_driver = { + .probe = cdns_ufs_pltfrm_probe, + .remove = cdns_ufs_pltfrm_remove, + .driver = { + .name = "cdns-ufshcd", + .pm = &cdns_ufs_dev_pm_ops, + .of_match_table = cdns_ufs_of_match, + }, +}; + +module_platform_driver(cdns_ufs_pltfrm_driver); + +MODULE_AUTHOR("Jan Kotas <jank@cadence.com>"); +MODULE_DESCRIPTION("Cadence UFS host controller platform driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/ufs/host/tc-dwc-g210-pci.c b/drivers/ufs/host/tc-dwc-g210-pci.c new file mode 100644 index 000000000..c6d89f9c4 --- /dev/null +++ b/drivers/ufs/host/tc-dwc-g210-pci.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Synopsys G210 Test Chip driver + * + * Copyright (C) 2015-2016 Synopsys, Inc. (www.synopsys.com) + * + * Authors: Joao Pinto <jpinto@synopsys.com> + */ + +#include <ufs/ufshcd.h> +#include "ufshcd-dwc.h" +#include "tc-dwc-g210.h" + +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/pm_runtime.h> + +/* Test Chip type expected values */ +#define TC_G210_20BIT 20 +#define TC_G210_40BIT 40 +#define TC_G210_INV 0 + +static int tc_type = TC_G210_INV; +module_param(tc_type, int, 0); +MODULE_PARM_DESC(tc_type, "Test Chip Type (20 = 20-bit, 40 = 40-bit)"); + +/* + * struct ufs_hba_dwc_vops - UFS DWC specific variant operations + */ +static struct ufs_hba_variant_ops tc_dwc_g210_pci_hba_vops = { + .name = "tc-dwc-g210-pci", + .link_startup_notify = ufshcd_dwc_link_startup_notify, +}; + +/** + * tc_dwc_g210_pci_remove - de-allocate PCI/SCSI host and host memory space + * data structure memory + * @pdev: pointer to PCI handle + */ +static void tc_dwc_g210_pci_remove(struct pci_dev *pdev) +{ + struct ufs_hba *hba = pci_get_drvdata(pdev); + + pm_runtime_forbid(&pdev->dev); + pm_runtime_get_noresume(&pdev->dev); + ufshcd_remove(hba); +} + +/** + * tc_dwc_g210_pci_probe - probe routine of the driver + * @pdev: pointer to PCI device handle + * @id: PCI device id + * + * Return: 0 on success, non-zero value on failure. + */ +static int +tc_dwc_g210_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct ufs_hba *hba; + void __iomem *mmio_base; + int err; + + /* Check Test Chip type and set the specific setup routine */ + if (tc_type == TC_G210_20BIT) { + tc_dwc_g210_pci_hba_vops.phy_initialization = + tc_dwc_g210_config_20_bit; + } else if (tc_type == TC_G210_40BIT) { + tc_dwc_g210_pci_hba_vops.phy_initialization = + tc_dwc_g210_config_40_bit; + } else { + dev_err(&pdev->dev, "test chip version not specified\n"); + return -EPERM; + } + + err = pcim_enable_device(pdev); + if (err) { + dev_err(&pdev->dev, "pcim_enable_device failed\n"); + return err; + } + + pci_set_master(pdev); + + mmio_base = pcim_iomap_region(pdev, 0, UFSHCD); + if (IS_ERR(mmio_base)) { + dev_err(&pdev->dev, "request and iomap failed\n"); + return PTR_ERR(mmio_base); + } + + err = ufshcd_alloc_host(&pdev->dev, &hba); + if (err) { + dev_err(&pdev->dev, "Allocation failed\n"); + return err; + } + + hba->vops = &tc_dwc_g210_pci_hba_vops; + + err = ufshcd_init(hba, mmio_base, pdev->irq); + if (err) { + dev_err(&pdev->dev, "Initialization failed\n"); + return err; + } + + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_allow(&pdev->dev); + + return 0; +} + +static const struct dev_pm_ops tc_dwc_g210_pci_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(ufshcd_system_suspend, ufshcd_system_resume) + SET_RUNTIME_PM_OPS(ufshcd_runtime_suspend, ufshcd_runtime_resume, NULL) + .prepare = ufshcd_suspend_prepare, + .complete = ufshcd_resume_complete, +}; + +static const struct pci_device_id tc_dwc_g210_pci_tbl[] = { + { PCI_VDEVICE(SYNOPSYS, 0xB101) }, + { PCI_VDEVICE(SYNOPSYS, 0xB102) }, + { } /* terminate list */ +}; + +MODULE_DEVICE_TABLE(pci, tc_dwc_g210_pci_tbl); + +static struct pci_driver tc_dwc_g210_pci_driver = { + .name = "tc-dwc-g210-pci", + .id_table = tc_dwc_g210_pci_tbl, + .probe = tc_dwc_g210_pci_probe, + .remove = tc_dwc_g210_pci_remove, + .driver = { + .pm = &tc_dwc_g210_pci_pm_ops + }, +}; + +module_pci_driver(tc_dwc_g210_pci_driver); + +MODULE_AUTHOR("Joao Pinto <Joao.Pinto@synopsys.com>"); +MODULE_DESCRIPTION("Synopsys Test Chip G210 PCI glue driver"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/ufs/host/tc-dwc-g210-pltfrm.c b/drivers/ufs/host/tc-dwc-g210-pltfrm.c new file mode 100644 index 000000000..454ac88c3 --- /dev/null +++ b/drivers/ufs/host/tc-dwc-g210-pltfrm.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Synopsys G210 Test Chip driver + * + * Copyright (C) 2015-2016 Synopsys, Inc. (www.synopsys.com) + * + * Authors: Joao Pinto <jpinto@synopsys.com> + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/of.h> +#include <linux/delay.h> +#include <linux/pm_runtime.h> + +#include "ufshcd-pltfrm.h" +#include "ufshcd-dwc.h" +#include "tc-dwc-g210.h" + +/* + * UFS DWC specific variant operations + */ +static struct ufs_hba_variant_ops tc_dwc_g210_20bit_pltfm_hba_vops = { + .name = "tc-dwc-g210-pltfm", + .link_startup_notify = ufshcd_dwc_link_startup_notify, + .phy_initialization = tc_dwc_g210_config_20_bit, +}; + +static struct ufs_hba_variant_ops tc_dwc_g210_40bit_pltfm_hba_vops = { + .name = "tc-dwc-g210-pltfm", + .link_startup_notify = ufshcd_dwc_link_startup_notify, + .phy_initialization = tc_dwc_g210_config_40_bit, +}; + +static const struct of_device_id tc_dwc_g210_pltfm_match[] = { + { + .compatible = "snps,g210-tc-6.00-20bit", + .data = &tc_dwc_g210_20bit_pltfm_hba_vops, + }, + { + .compatible = "snps,g210-tc-6.00-40bit", + .data = &tc_dwc_g210_40bit_pltfm_hba_vops, + }, + { }, +}; +MODULE_DEVICE_TABLE(of, tc_dwc_g210_pltfm_match); + +/** + * tc_dwc_g210_pltfm_probe() + * @pdev: pointer to platform device structure + * + */ +static int tc_dwc_g210_pltfm_probe(struct platform_device *pdev) +{ + int err; + const struct of_device_id *of_id; + struct ufs_hba_variant_ops *vops; + struct device *dev = &pdev->dev; + + of_id = of_match_node(tc_dwc_g210_pltfm_match, dev->of_node); + vops = (struct ufs_hba_variant_ops *)of_id->data; + + /* Perform generic probe */ + err = ufshcd_pltfrm_init(pdev, vops); + if (err) + dev_err(dev, "ufshcd_pltfrm_init() failed %d\n", err); + + return err; +} + +/** + * tc_dwc_g210_pltfm_remove() + * @pdev: pointer to platform device structure + * + */ +static void tc_dwc_g210_pltfm_remove(struct platform_device *pdev) +{ + ufshcd_pltfrm_remove(pdev); +} + +static const struct dev_pm_ops tc_dwc_g210_pltfm_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(ufshcd_system_suspend, ufshcd_system_resume) + SET_RUNTIME_PM_OPS(ufshcd_runtime_suspend, ufshcd_runtime_resume, NULL) +}; + +static struct platform_driver tc_dwc_g210_pltfm_driver = { + .probe = tc_dwc_g210_pltfm_probe, + .remove = tc_dwc_g210_pltfm_remove, + .driver = { + .name = "tc-dwc-g210-pltfm", + .pm = &tc_dwc_g210_pltfm_pm_ops, + .of_match_table = of_match_ptr(tc_dwc_g210_pltfm_match), + }, +}; + +module_platform_driver(tc_dwc_g210_pltfm_driver); + +MODULE_ALIAS("platform:tc-dwc-g210-pltfm"); +MODULE_DESCRIPTION("Synopsys Test Chip G210 platform glue driver"); +MODULE_AUTHOR("Joao Pinto <Joao.Pinto@synopsys.com>"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/ufs/host/tc-dwc-g210.c b/drivers/ufs/host/tc-dwc-g210.c new file mode 100644 index 000000000..0ac53cc84 --- /dev/null +++ b/drivers/ufs/host/tc-dwc-g210.c @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Synopsys G210 Test Chip driver + * + * Copyright (C) 2015-2016 Synopsys, Inc. (www.synopsys.com) + * + * Authors: Joao Pinto <jpinto@synopsys.com> + */ + +#include <linux/module.h> + +#include <ufs/ufshcd.h> +#include <ufs/unipro.h> + +#include "ufshcd-dwc.h" +#include "ufshci-dwc.h" +#include "tc-dwc-g210.h" + +/** + * tc_dwc_g210_setup_40bit_rmmi() - configure 40-bit RMMI. + * @hba: Pointer to drivers structure + * + * Return: 0 on success or non-zero value on failure. + */ +static int tc_dwc_g210_setup_40bit_rmmi(struct ufs_hba *hba) +{ + static const struct ufshcd_dme_attr_val setup_attrs[] = { + { UIC_ARG_MIB(TX_GLOBALHIBERNATE), 0x00, DME_LOCAL }, + { UIC_ARG_MIB(REFCLKMODE), 0x01, DME_LOCAL }, + { UIC_ARG_MIB(CDIRECTCTRL6), 0x80, DME_LOCAL }, + { UIC_ARG_MIB(CBDIVFACTOR), 0x08, DME_LOCAL }, + { UIC_ARG_MIB(CBDCOCTRL5), 0x64, DME_LOCAL }, + { UIC_ARG_MIB(CBPRGTUNING), 0x09, DME_LOCAL }, + { UIC_ARG_MIB(RTOBSERVESELECT), 0x00, DME_LOCAL }, + { UIC_ARG_MIB_SEL(TX_REFCLKFREQ, SELIND_LN0_TX), 0x01, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(TX_CFGCLKFREQVAL, SELIND_LN0_TX), 0x19, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGEXTRATTR, SELIND_LN0_TX), 0x14, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(DITHERCTRL2, SELIND_LN0_TX), 0xd6, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(RX_REFCLKFREQ, SELIND_LN0_RX), 0x01, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(RX_CFGCLKFREQVAL, SELIND_LN0_RX), 0x19, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGWIDEINLN, SELIND_LN0_RX), 4, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGRXCDR8, SELIND_LN0_RX), 0x80, + DME_LOCAL }, + { UIC_ARG_MIB(DIRECTCTRL10), 0x04, DME_LOCAL }, + { UIC_ARG_MIB(DIRECTCTRL19), 0x02, DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGRXCDR8, SELIND_LN0_RX), 0x80, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(ENARXDIRECTCFG4, SELIND_LN0_RX), 0x03, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGRXOVR8, SELIND_LN0_RX), 0x16, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(RXDIRECTCTRL2, SELIND_LN0_RX), 0x42, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(ENARXDIRECTCFG3, SELIND_LN0_RX), 0xa4, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(RXCALCTRL, SELIND_LN0_RX), 0x01, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(ENARXDIRECTCFG2, SELIND_LN0_RX), 0x01, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGRXOVR4, SELIND_LN0_RX), 0x28, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(RXSQCTRL, SELIND_LN0_RX), 0x1E, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGRXOVR6, SELIND_LN0_RX), 0x2f, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGRXOVR6, SELIND_LN0_RX), 0x2f, + DME_LOCAL }, + { UIC_ARG_MIB(CBPRGPLL2), 0x00, DME_LOCAL }, + }; + + return ufshcd_dwc_dme_set_attrs(hba, setup_attrs, + ARRAY_SIZE(setup_attrs)); +} + +/** + * tc_dwc_g210_setup_20bit_rmmi_lane0() - configure 20-bit RMMI Lane 0. + * @hba: Pointer to drivers structure + * + * Return: 0 on success or non-zero value on failure. + */ +static int tc_dwc_g210_setup_20bit_rmmi_lane0(struct ufs_hba *hba) +{ + static const struct ufshcd_dme_attr_val setup_attrs[] = { + { UIC_ARG_MIB_SEL(TX_REFCLKFREQ, SELIND_LN0_TX), 0x01, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(TX_CFGCLKFREQVAL, SELIND_LN0_TX), 0x19, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(RX_CFGCLKFREQVAL, SELIND_LN0_RX), 0x19, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGEXTRATTR, SELIND_LN0_TX), 0x12, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(DITHERCTRL2, SELIND_LN0_TX), 0xd6, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(RX_REFCLKFREQ, SELIND_LN0_RX), 0x01, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGWIDEINLN, SELIND_LN0_RX), 2, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGRXCDR8, SELIND_LN0_RX), 0x80, + DME_LOCAL }, + { UIC_ARG_MIB(DIRECTCTRL10), 0x04, DME_LOCAL }, + { UIC_ARG_MIB(DIRECTCTRL19), 0x02, DME_LOCAL }, + { UIC_ARG_MIB_SEL(ENARXDIRECTCFG4, SELIND_LN0_RX), 0x03, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGRXOVR8, SELIND_LN0_RX), 0x16, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(RXDIRECTCTRL2, SELIND_LN0_RX), 0x42, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(ENARXDIRECTCFG3, SELIND_LN0_RX), 0xa4, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(RXCALCTRL, SELIND_LN0_RX), 0x01, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(ENARXDIRECTCFG2, SELIND_LN0_RX), 0x01, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGRXOVR4, SELIND_LN0_RX), 0x28, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(RXSQCTRL, SELIND_LN0_RX), 0x1E, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGRXOVR6, SELIND_LN0_RX), 0x2f, + DME_LOCAL }, + { UIC_ARG_MIB(CBPRGPLL2), 0x00, DME_LOCAL }, + }; + + return ufshcd_dwc_dme_set_attrs(hba, setup_attrs, + ARRAY_SIZE(setup_attrs)); +} + +/** + * tc_dwc_g210_setup_20bit_rmmi_lane1() - configure 20-bit RMMI Lane 1. + * @hba: Pointer to drivers structure + * + * Return: 0 on success or non-zero value on failure. + */ +static int tc_dwc_g210_setup_20bit_rmmi_lane1(struct ufs_hba *hba) +{ + int connected_rx_lanes = 0; + int connected_tx_lanes = 0; + int ret = 0; + + static const struct ufshcd_dme_attr_val setup_tx_attrs[] = { + { UIC_ARG_MIB_SEL(TX_REFCLKFREQ, SELIND_LN1_TX), 0x0d, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(TX_CFGCLKFREQVAL, SELIND_LN1_TX), 0x19, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGEXTRATTR, SELIND_LN1_TX), 0x12, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(DITHERCTRL2, SELIND_LN0_TX), 0xd6, + DME_LOCAL }, + }; + + static const struct ufshcd_dme_attr_val setup_rx_attrs[] = { + { UIC_ARG_MIB_SEL(RX_REFCLKFREQ, SELIND_LN1_RX), 0x01, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(RX_CFGCLKFREQVAL, SELIND_LN1_RX), 0x19, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGWIDEINLN, SELIND_LN1_RX), 2, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGRXCDR8, SELIND_LN1_RX), 0x80, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(ENARXDIRECTCFG4, SELIND_LN1_RX), 0x03, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGRXOVR8, SELIND_LN1_RX), 0x16, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(RXDIRECTCTRL2, SELIND_LN1_RX), 0x42, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(ENARXDIRECTCFG3, SELIND_LN1_RX), 0xa4, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(RXCALCTRL, SELIND_LN1_RX), 0x01, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(ENARXDIRECTCFG2, SELIND_LN1_RX), 0x01, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGRXOVR4, SELIND_LN1_RX), 0x28, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(RXSQCTRL, SELIND_LN1_RX), 0x1E, + DME_LOCAL }, + { UIC_ARG_MIB_SEL(CFGRXOVR6, SELIND_LN1_RX), 0x2f, + DME_LOCAL }, + }; + + /* Get the available lane count */ + ufshcd_dme_get(hba, UIC_ARG_MIB(PA_AVAILRXDATALANES), + &connected_rx_lanes); + ufshcd_dme_get(hba, UIC_ARG_MIB(PA_AVAILTXDATALANES), + &connected_tx_lanes); + + if (connected_tx_lanes == 2) { + + ret = ufshcd_dwc_dme_set_attrs(hba, setup_tx_attrs, + ARRAY_SIZE(setup_tx_attrs)); + + if (ret) + goto out; + } + + if (connected_rx_lanes == 2) { + ret = ufshcd_dwc_dme_set_attrs(hba, setup_rx_attrs, + ARRAY_SIZE(setup_rx_attrs)); + } + +out: + return ret; +} + +/** + * tc_dwc_g210_setup_20bit_rmmi() - configure 20-bit RMMI. + * @hba: Pointer to drivers structure + * + * Return: 0 on success or non-zero value on failure. + */ +static int tc_dwc_g210_setup_20bit_rmmi(struct ufs_hba *hba) +{ + int ret = 0; + + static const struct ufshcd_dme_attr_val setup_attrs[] = { + { UIC_ARG_MIB(TX_GLOBALHIBERNATE), 0x00, DME_LOCAL }, + { UIC_ARG_MIB(REFCLKMODE), 0x01, DME_LOCAL }, + { UIC_ARG_MIB(CDIRECTCTRL6), 0xc0, DME_LOCAL }, + { UIC_ARG_MIB(CBDIVFACTOR), 0x44, DME_LOCAL }, + { UIC_ARG_MIB(CBDCOCTRL5), 0x64, DME_LOCAL }, + { UIC_ARG_MIB(CBPRGTUNING), 0x09, DME_LOCAL }, + { UIC_ARG_MIB(RTOBSERVESELECT), 0x00, DME_LOCAL }, + }; + + ret = ufshcd_dwc_dme_set_attrs(hba, setup_attrs, + ARRAY_SIZE(setup_attrs)); + if (ret) + goto out; + + /* Lane 0 configuration*/ + ret = tc_dwc_g210_setup_20bit_rmmi_lane0(hba); + if (ret) + goto out; + + /* Lane 1 configuration*/ + ret = tc_dwc_g210_setup_20bit_rmmi_lane1(hba); + if (ret) + goto out; + +out: + return ret; +} + +/** + * tc_dwc_g210_config_40_bit() - configure 40-bit TC specific attributes. + * @hba: Pointer to drivers structure + * + * Return: 0 on success non-zero value on failure. + */ +int tc_dwc_g210_config_40_bit(struct ufs_hba *hba) +{ + int ret = 0; + + dev_info(hba->dev, "Configuring Test Chip 40-bit RMMI\n"); + ret = tc_dwc_g210_setup_40bit_rmmi(hba); + if (ret) { + dev_err(hba->dev, "Configuration failed\n"); + goto out; + } + + /* To write Shadow register bank to effective configuration block */ + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(VS_MPHYCFGUPDT), 0x01); + if (ret) + goto out; + + /* To configure Debug OMC */ + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(VS_DEBUGOMC), 0x01); + +out: + return ret; +} +EXPORT_SYMBOL(tc_dwc_g210_config_40_bit); + +/** + * tc_dwc_g210_config_20_bit() - configure 20-bit TC specific attributes. + * @hba: Pointer to drivers structure + * + * Return: 0 on success non-zero value on failure. + */ +int tc_dwc_g210_config_20_bit(struct ufs_hba *hba) +{ + int ret = 0; + + dev_info(hba->dev, "Configuring Test Chip 20-bit RMMI\n"); + ret = tc_dwc_g210_setup_20bit_rmmi(hba); + if (ret) { + dev_err(hba->dev, "Configuration failed\n"); + goto out; + } + + /* To write Shadow register bank to effective configuration block */ + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(VS_MPHYCFGUPDT), 0x01); + if (ret) + goto out; + + /* To configure Debug OMC */ + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(VS_DEBUGOMC), 0x01); + +out: + return ret; +} +EXPORT_SYMBOL(tc_dwc_g210_config_20_bit); + +MODULE_AUTHOR("Joao Pinto <Joao.Pinto@synopsys.com>"); +MODULE_DESCRIPTION("Synopsys G210 Test Chip driver"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/ufs/host/tc-dwc-g210.h b/drivers/ufs/host/tc-dwc-g210.h new file mode 100644 index 000000000..f7154012f --- /dev/null +++ b/drivers/ufs/host/tc-dwc-g210.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Synopsys G210 Test Chip driver + * + * Copyright (C) 2015-2016 Synopsys, Inc. (www.synopsys.com) + * + * Authors: Joao Pinto <jpinto@synopsys.com> + */ + +#ifndef _TC_DWC_G210_H +#define _TC_DWC_G210_H + +struct ufs_hba; + +int tc_dwc_g210_config_40_bit(struct ufs_hba *hba); +int tc_dwc_g210_config_20_bit(struct ufs_hba *hba); + +#endif /* End of Header */ diff --git a/drivers/ufs/host/ti-j721e-ufs.c b/drivers/ufs/host/ti-j721e-ufs.c new file mode 100644 index 000000000..43781593b --- /dev/null +++ b/drivers/ufs/host/ti-j721e-ufs.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/ +// + +#include <linux/clk.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> + +#define TI_UFS_SS_CTRL 0x4 +#define TI_UFS_SS_RST_N_PCS BIT(0) +#define TI_UFS_SS_CLK_26MHZ BIT(4) + +struct ti_j721e_ufs { + void __iomem *regbase; + u32 reg; +}; + +static int ti_j721e_ufs_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct ti_j721e_ufs *ufs; + unsigned long clk_rate; + struct clk *clk; + int ret; + + ufs = devm_kzalloc(dev, sizeof(*ufs), GFP_KERNEL); + if (!ufs) + return -ENOMEM; + + ufs->regbase = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(ufs->regbase)) + return PTR_ERR(ufs->regbase); + + pm_runtime_enable(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) + goto disable_pm; + + /* Select MPHY refclk frequency */ + clk = devm_clk_get(dev, NULL); + if (IS_ERR(clk)) { + ret = PTR_ERR(clk); + dev_err(dev, "Cannot claim MPHY clock.\n"); + goto clk_err; + } + clk_rate = clk_get_rate(clk); + if (clk_rate == 26000000) + ufs->reg |= TI_UFS_SS_CLK_26MHZ; + devm_clk_put(dev, clk); + + /* Take UFS slave device out of reset */ + ufs->reg |= TI_UFS_SS_RST_N_PCS; + writel(ufs->reg, ufs->regbase + TI_UFS_SS_CTRL); + + dev_set_drvdata(dev, ufs); + + ret = of_platform_populate(pdev->dev.of_node, NULL, NULL, + dev); + if (ret) { + dev_err(dev, "failed to populate child nodes %d\n", ret); + goto clk_err; + } + + return ret; + +clk_err: + pm_runtime_put_sync(dev); +disable_pm: + pm_runtime_disable(dev); + return ret; +} + +static void ti_j721e_ufs_remove(struct platform_device *pdev) +{ + of_platform_depopulate(&pdev->dev); + pm_runtime_put_sync(&pdev->dev); + pm_runtime_disable(&pdev->dev); +} + +static int ti_j721e_ufs_resume(struct device *dev) +{ + struct ti_j721e_ufs *ufs = dev_get_drvdata(dev); + + writel(ufs->reg, ufs->regbase + TI_UFS_SS_CTRL); + return 0; +} + +static DEFINE_SIMPLE_DEV_PM_OPS(ti_j721e_ufs_pm_ops, NULL, ti_j721e_ufs_resume); + +static const struct of_device_id ti_j721e_ufs_of_match[] = { + { + .compatible = "ti,j721e-ufs", + }, + { }, +}; + +MODULE_DEVICE_TABLE(of, ti_j721e_ufs_of_match); + +static struct platform_driver ti_j721e_ufs_driver = { + .probe = ti_j721e_ufs_probe, + .remove = ti_j721e_ufs_remove, + .driver = { + .name = "ti-j721e-ufs", + .of_match_table = ti_j721e_ufs_of_match, + .pm = pm_sleep_ptr(&ti_j721e_ufs_pm_ops), + }, +}; +module_platform_driver(ti_j721e_ufs_driver); + +MODULE_AUTHOR("Vignesh Raghavendra <vigneshr@ti.com>"); +MODULE_DESCRIPTION("TI UFS host controller glue driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/ufs/host/ufs-amd-versal2.c b/drivers/ufs/host/ufs-amd-versal2.c new file mode 100644 index 000000000..2154d6286 --- /dev/null +++ b/drivers/ufs/host/ufs-amd-versal2.c @@ -0,0 +1,561 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2025 Advanced Micro Devices, Inc. + * + * Authors: Sai Krishna Potthuri <sai.krishna.potthuri@amd.com> + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/firmware/xlnx-zynqmp.h> +#include <linux/irqreturn.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/reset.h> +#include <ufs/unipro.h> + +#include "ufshcd-dwc.h" +#include "ufshcd-pltfrm.h" +#include "ufshci-dwc.h" + +/* PHY modes */ +#define UFSHCD_DWC_PHY_MODE_ROM 0 + +#define MPHY_FAST_RX_AFE_CAL BIT(2) +#define MPHY_FW_CALIB_CFG_VAL BIT(8) + +#define MPHY_RX_OVRD_EN BIT(3) +#define MPHY_RX_OVRD_VAL BIT(2) +#define MPHY_RX_ACK_MASK BIT(0) + +#define TIMEOUT_MICROSEC 1000000 + +struct ufs_versal2_host { + struct ufs_hba *hba; + struct reset_control *rstc; + struct reset_control *rstphy; + u32 phy_mode; + unsigned long host_clk; + u8 attcompval0; + u8 attcompval1; + u8 ctlecompval0; + u8 ctlecompval1; +}; + +static int ufs_versal2_phy_reg_write(struct ufs_hba *hba, u32 addr, u32 val) +{ + static struct ufshcd_dme_attr_val phy_write_attrs[] = { + { UIC_ARG_MIB(CBCREGADDRLSB), 0, DME_LOCAL }, + { UIC_ARG_MIB(CBCREGADDRMSB), 0, DME_LOCAL }, + { UIC_ARG_MIB(CBCREGWRLSB), 0, DME_LOCAL }, + { UIC_ARG_MIB(CBCREGWRMSB), 0, DME_LOCAL }, + { UIC_ARG_MIB(CBCREGRDWRSEL), 1, DME_LOCAL }, + { UIC_ARG_MIB(VS_MPHYCFGUPDT), 1, DME_LOCAL } + }; + + phy_write_attrs[0].mib_val = (u8)addr; + phy_write_attrs[1].mib_val = (u8)(addr >> 8); + phy_write_attrs[2].mib_val = (u8)val; + phy_write_attrs[3].mib_val = (u8)(val >> 8); + + return ufshcd_dwc_dme_set_attrs(hba, phy_write_attrs, ARRAY_SIZE(phy_write_attrs)); +} + +static int ufs_versal2_phy_reg_read(struct ufs_hba *hba, u32 addr, u32 *val) +{ + u32 mib_val; + int ret; + static struct ufshcd_dme_attr_val phy_read_attrs[] = { + { UIC_ARG_MIB(CBCREGADDRLSB), 0, DME_LOCAL }, + { UIC_ARG_MIB(CBCREGADDRMSB), 0, DME_LOCAL }, + { UIC_ARG_MIB(CBCREGRDWRSEL), 0, DME_LOCAL }, + { UIC_ARG_MIB(VS_MPHYCFGUPDT), 1, DME_LOCAL } + }; + + phy_read_attrs[0].mib_val = (u8)addr; + phy_read_attrs[1].mib_val = (u8)(addr >> 8); + + ret = ufshcd_dwc_dme_set_attrs(hba, phy_read_attrs, ARRAY_SIZE(phy_read_attrs)); + if (ret) + return ret; + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(CBCREGRDLSB), &mib_val); + if (ret) + return ret; + + *val = mib_val; + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(CBCREGRDMSB), &mib_val); + if (ret) + return ret; + + *val |= (mib_val << 8); + + return 0; +} + +static int ufs_versal2_enable_phy(struct ufs_hba *hba) +{ + u32 offset, reg; + int ret; + + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(VS_MPHYDISABLE), 0); + if (ret) + return ret; + + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(VS_MPHYCFGUPDT), 1); + if (ret) + return ret; + + /* Check Tx/Rx FSM states */ + for (offset = 0; offset < 2; offset++) { + u32 time_left, mibsel; + + time_left = TIMEOUT_MICROSEC; + mibsel = UIC_ARG_MIB_SEL(MTX_FSM_STATE, UIC_ARG_MPHY_TX_GEN_SEL_INDEX(offset)); + do { + ret = ufshcd_dme_get(hba, mibsel, ®); + if (ret) + return ret; + + if (reg == TX_STATE_HIBERN8 || reg == TX_STATE_SLEEP || + reg == TX_STATE_LSBURST) + break; + + time_left--; + usleep_range(1, 5); + } while (time_left); + + if (!time_left) { + dev_err(hba->dev, "Invalid Tx FSM state.\n"); + return -ETIMEDOUT; + } + + time_left = TIMEOUT_MICROSEC; + mibsel = UIC_ARG_MIB_SEL(MRX_FSM_STATE, UIC_ARG_MPHY_RX_GEN_SEL_INDEX(offset)); + do { + ret = ufshcd_dme_get(hba, mibsel, ®); + if (ret) + return ret; + + if (reg == RX_STATE_HIBERN8 || reg == RX_STATE_SLEEP || + reg == RX_STATE_LSBURST) + break; + + time_left--; + usleep_range(1, 5); + } while (time_left); + + if (!time_left) { + dev_err(hba->dev, "Invalid Rx FSM state.\n"); + return -ETIMEDOUT; + } + } + + return 0; +} + +static int ufs_versal2_setup_phy(struct ufs_hba *hba) +{ + struct ufs_versal2_host *host = ufshcd_get_variant(hba); + int ret; + u32 reg; + + /* Bypass RX-AFE offset calibrations (ATT/CTLE) */ + ret = ufs_versal2_phy_reg_read(hba, FAST_FLAGS(0), ®); + if (ret) + return ret; + + reg |= MPHY_FAST_RX_AFE_CAL; + ret = ufs_versal2_phy_reg_write(hba, FAST_FLAGS(0), reg); + if (ret) + return ret; + + ret = ufs_versal2_phy_reg_read(hba, FAST_FLAGS(1), ®); + if (ret) + return ret; + + reg |= MPHY_FAST_RX_AFE_CAL; + ret = ufs_versal2_phy_reg_write(hba, FAST_FLAGS(1), reg); + if (ret) + return ret; + + /* Program ATT and CTLE compensation values */ + if (host->attcompval0) { + ret = ufs_versal2_phy_reg_write(hba, RX_AFE_ATT_IDAC(0), host->attcompval0); + if (ret) + return ret; + } + + if (host->attcompval1) { + ret = ufs_versal2_phy_reg_write(hba, RX_AFE_ATT_IDAC(1), host->attcompval1); + if (ret) + return ret; + } + + if (host->ctlecompval0) { + ret = ufs_versal2_phy_reg_write(hba, RX_AFE_CTLE_IDAC(0), host->ctlecompval0); + if (ret) + return ret; + } + + if (host->ctlecompval1) { + ret = ufs_versal2_phy_reg_write(hba, RX_AFE_CTLE_IDAC(1), host->ctlecompval1); + if (ret) + return ret; + } + + ret = ufs_versal2_phy_reg_read(hba, FW_CALIB_CCFG(0), ®); + if (ret) + return ret; + + reg |= MPHY_FW_CALIB_CFG_VAL; + ret = ufs_versal2_phy_reg_write(hba, FW_CALIB_CCFG(0), reg); + if (ret) + return ret; + + ret = ufs_versal2_phy_reg_read(hba, FW_CALIB_CCFG(1), ®); + if (ret) + return ret; + + reg |= MPHY_FW_CALIB_CFG_VAL; + return ufs_versal2_phy_reg_write(hba, FW_CALIB_CCFG(1), reg); +} + +static int ufs_versal2_phy_init(struct ufs_hba *hba) +{ + struct ufs_versal2_host *host = ufshcd_get_variant(hba); + u32 time_left; + bool is_ready; + int ret; + static const struct ufshcd_dme_attr_val rmmi_attrs[] = { + { UIC_ARG_MIB(CBREFCLKCTRL2), CBREFREFCLK_GATE_OVR_EN, DME_LOCAL }, + { UIC_ARG_MIB(CBCRCTRL), 1, DME_LOCAL }, + { UIC_ARG_MIB(CBC10DIRECTCONF2), 1, DME_LOCAL }, + { UIC_ARG_MIB(VS_MPHYCFGUPDT), 1, DME_LOCAL } + }; + + /* Wait for Tx/Rx config_rdy */ + time_left = TIMEOUT_MICROSEC; + do { + time_left--; + ret = zynqmp_pm_is_mphy_tx_rx_config_ready(&is_ready); + if (ret) + return ret; + + if (!is_ready) + break; + + usleep_range(1, 5); + } while (time_left); + + if (!time_left) { + dev_err(hba->dev, "Tx/Rx configuration signal busy.\n"); + return -ETIMEDOUT; + } + + ret = ufshcd_dwc_dme_set_attrs(hba, rmmi_attrs, ARRAY_SIZE(rmmi_attrs)); + if (ret) + return ret; + + ret = reset_control_deassert(host->rstphy); + if (ret) { + dev_err(hba->dev, "ufsphy reset deassert failed, err = %d\n", ret); + return ret; + } + + /* Wait for SRAM init done */ + time_left = TIMEOUT_MICROSEC; + do { + time_left--; + ret = zynqmp_pm_is_sram_init_done(&is_ready); + if (ret) + return ret; + + if (is_ready) + break; + + usleep_range(1, 5); + } while (time_left); + + if (!time_left) { + dev_err(hba->dev, "SRAM initialization failed.\n"); + return -ETIMEDOUT; + } + + ret = ufs_versal2_setup_phy(hba); + if (ret) + return ret; + + return ufs_versal2_enable_phy(hba); +} + +static int ufs_versal2_init(struct ufs_hba *hba) +{ + struct ufs_versal2_host *host; + struct device *dev = hba->dev; + struct ufs_clk_info *clki; + int ret; + u32 cal; + + host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL); + if (!host) + return -ENOMEM; + + host->hba = hba; + ufshcd_set_variant(hba, host); + + host->phy_mode = UFSHCD_DWC_PHY_MODE_ROM; + + list_for_each_entry(clki, &hba->clk_list_head, list) { + if (!strcmp(clki->name, "core")) + host->host_clk = clk_get_rate(clki->clk); + } + + host->rstc = devm_reset_control_get_exclusive(dev, "host"); + if (IS_ERR(host->rstc)) { + dev_err(dev, "failed to get reset ctrl: host\n"); + return PTR_ERR(host->rstc); + } + + host->rstphy = devm_reset_control_get_exclusive(dev, "phy"); + if (IS_ERR(host->rstphy)) { + dev_err(dev, "failed to get reset ctrl: phy\n"); + return PTR_ERR(host->rstphy); + } + + ret = reset_control_assert(host->rstc); + if (ret) { + dev_err(hba->dev, "host reset assert failed, err = %d\n", ret); + return ret; + } + + ret = reset_control_assert(host->rstphy); + if (ret) { + dev_err(hba->dev, "phy reset assert failed, err = %d\n", ret); + return ret; + } + + ret = zynqmp_pm_set_sram_bypass(); + if (ret) { + dev_err(dev, "Bypass SRAM interface failed, err = %d\n", ret); + return ret; + } + + ret = reset_control_deassert(host->rstc); + if (ret) + dev_err(hba->dev, "host reset deassert failed, err = %d\n", ret); + + ret = zynqmp_pm_get_ufs_calibration_values(&cal); + if (ret) { + dev_err(dev, "failed to read calibration values\n"); + return ret; + } + + host->attcompval0 = (u8)cal; + host->attcompval1 = (u8)(cal >> 8); + host->ctlecompval0 = (u8)(cal >> 16); + host->ctlecompval1 = (u8)(cal >> 24); + + hba->quirks |= UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING; + + return 0; +} + +static int ufs_versal2_hce_enable_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status) +{ + int ret = 0; + + if (status == POST_CHANGE) { + ret = ufs_versal2_phy_init(hba); + if (ret) + dev_err(hba->dev, "Phy init failed (%d)\n", ret); + } + + return ret; +} + +static int ufs_versal2_link_startup_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status) +{ + struct ufs_versal2_host *host = ufshcd_get_variant(hba); + int ret = 0; + + switch (status) { + case PRE_CHANGE: + if (host->host_clk) + ufshcd_writel(hba, host->host_clk / 1000000, DWC_UFS_REG_HCLKDIV); + + break; + case POST_CHANGE: + ret = ufshcd_dwc_link_startup_notify(hba, status); + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + +static int ufs_versal2_phy_ratesel(struct ufs_hba *hba, u32 activelanes, u32 rx_req) +{ + u32 time_left, reg, lane; + int ret; + + for (lane = 0; lane < activelanes; lane++) { + time_left = TIMEOUT_MICROSEC; + ret = ufs_versal2_phy_reg_read(hba, RX_OVRD_IN_1(lane), ®); + if (ret) + return ret; + + reg |= MPHY_RX_OVRD_EN; + if (rx_req) + reg |= MPHY_RX_OVRD_VAL; + else + reg &= ~MPHY_RX_OVRD_VAL; + + ret = ufs_versal2_phy_reg_write(hba, RX_OVRD_IN_1(lane), reg); + if (ret) + return ret; + + do { + ret = ufs_versal2_phy_reg_read(hba, RX_PCS_OUT(lane), ®); + if (ret) + return ret; + + reg &= MPHY_RX_ACK_MASK; + if (reg == rx_req) + break; + + time_left--; + usleep_range(1, 5); + } while (time_left); + + if (!time_left) { + dev_err(hba->dev, "Invalid Rx Ack value.\n"); + return -ETIMEDOUT; + } + } + + return 0; +} + +static int ufs_versal2_pwr_change_notify(struct ufs_hba *hba, enum ufs_notify_change_status status, + struct ufs_pa_layer_attr *dev_req_params) +{ + struct ufs_versal2_host *host = ufshcd_get_variant(hba); + u32 lane, reg, rate = 0; + int ret = 0; + + if (status == PRE_CHANGE) { + /* If it is not a calibrated part, switch PWRMODE to SLOW_MODE */ + if (!host->attcompval0 && !host->attcompval1 && !host->ctlecompval0 && + !host->ctlecompval1) { + dev_req_params->pwr_rx = SLOW_MODE; + dev_req_params->pwr_tx = SLOW_MODE; + return 0; + } + + if (dev_req_params->pwr_rx == SLOW_MODE || dev_req_params->pwr_rx == SLOWAUTO_MODE) + return 0; + + if (dev_req_params->hs_rate == PA_HS_MODE_B) + rate = 1; + + /* Select the rate */ + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(CBRATESEL), rate); + if (ret) + return ret; + + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(VS_MPHYCFGUPDT), 1); + if (ret) + return ret; + + ret = ufs_versal2_phy_ratesel(hba, dev_req_params->lane_tx, 1); + if (ret) + return ret; + + ret = ufs_versal2_phy_ratesel(hba, dev_req_params->lane_tx, 0); + if (ret) + return ret; + + /* Remove rx_req override */ + for (lane = 0; lane < dev_req_params->lane_tx; lane++) { + ret = ufs_versal2_phy_reg_read(hba, RX_OVRD_IN_1(lane), ®); + if (ret) + return ret; + + reg &= ~MPHY_RX_OVRD_EN; + ret = ufs_versal2_phy_reg_write(hba, RX_OVRD_IN_1(lane), reg); + if (ret) + return ret; + } + + if (dev_req_params->lane_tx == UFS_LANE_2 && dev_req_params->lane_rx == UFS_LANE_2) + ret = ufshcd_dme_configure_adapt(hba, dev_req_params->gear_tx, + PA_INITIAL_ADAPT); + } + + return ret; +} + +static struct ufs_hba_variant_ops ufs_versal2_hba_vops = { + .name = "ufs-versal2-pltfm", + .init = ufs_versal2_init, + .link_startup_notify = ufs_versal2_link_startup_notify, + .hce_enable_notify = ufs_versal2_hce_enable_notify, + .pwr_change_notify = ufs_versal2_pwr_change_notify, +}; + +static const struct of_device_id ufs_versal2_pltfm_match[] = { + { + .compatible = "amd,versal2-ufs", + .data = &ufs_versal2_hba_vops, + }, + { }, +}; +MODULE_DEVICE_TABLE(of, ufs_versal2_pltfm_match); + +static int ufs_versal2_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + int ret; + + /* Perform generic probe */ + ret = ufshcd_pltfrm_init(pdev, &ufs_versal2_hba_vops); + if (ret) + dev_err(dev, "ufshcd_pltfrm_init() failed %d\n", ret); + + return ret; +} + +static void ufs_versal2_remove(struct platform_device *pdev) +{ + struct ufs_hba *hba = platform_get_drvdata(pdev); + + pm_runtime_get_sync(&(pdev)->dev); + ufshcd_remove(hba); +} + +static const struct dev_pm_ops ufs_versal2_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(ufshcd_system_suspend, ufshcd_system_resume) + SET_RUNTIME_PM_OPS(ufshcd_runtime_suspend, ufshcd_runtime_resume, NULL) +}; + +static struct platform_driver ufs_versal2_pltfm = { + .probe = ufs_versal2_probe, + .remove = ufs_versal2_remove, + .driver = { + .name = "ufshcd-versal2", + .pm = &ufs_versal2_pm_ops, + .of_match_table = of_match_ptr(ufs_versal2_pltfm_match), + }, +}; + +module_platform_driver(ufs_versal2_pltfm); + +MODULE_AUTHOR("Sai Krishna Potthuri <sai.krishna.potthuri@amd.com>"); +MODULE_DESCRIPTION("AMD Versal Gen 2 UFS Host Controller driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/ufs/host/ufs-exynos.c b/drivers/ufs/host/ufs-exynos.c new file mode 100644 index 000000000..b2f65c465 --- /dev/null +++ b/drivers/ufs/host/ufs-exynos.c @@ -0,0 +1,2350 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * UFS Host Controller driver for Exynos specific extensions + * + * Copyright (C) 2014-2015 Samsung Electronics Co., Ltd. + * Author: Seungwon Jeon <essuuj@gmail.com> + * Author: Alim Akhtar <alim.akhtar@samsung.com> + * + */ + +#include <linux/unaligned.h> +#include <crypto/aes.h> +#include <linux/arm-smccc.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/mfd/syscon.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> + +#include <ufs/ufshcd.h> +#include "ufshcd-pltfrm.h" +#include <ufs/ufshci.h> +#include <ufs/unipro.h> + +#include "ufs-exynos.h" + +#define DATA_UNIT_SIZE 4096 + +/* + * Exynos's Vendor specific registers for UFSHCI + */ +#define HCI_TXPRDT_ENTRY_SIZE 0x00 +#define PRDT_PREFETCH_EN BIT(31) +#define HCI_RXPRDT_ENTRY_SIZE 0x04 +#define HCI_1US_TO_CNT_VAL 0x0C +#define CNT_VAL_1US_MASK 0x3FF +#define HCI_UTRL_NEXUS_TYPE 0x40 +#define HCI_UTMRL_NEXUS_TYPE 0x44 +#define HCI_SW_RST 0x50 +#define UFS_LINK_SW_RST BIT(0) +#define UFS_UNIPRO_SW_RST BIT(1) +#define UFS_SW_RST_MASK (UFS_UNIPRO_SW_RST | UFS_LINK_SW_RST) +#define HCI_DATA_REORDER 0x60 +#define HCI_UNIPRO_APB_CLK_CTRL 0x68 +#define UNIPRO_APB_CLK(v, x) (((v) & ~0xF) | ((x) & 0xF)) +#define HCI_AXIDMA_RWDATA_BURST_LEN 0x6C +#define WLU_EN BIT(31) +#define WLU_BURST_LEN(x) ((x) << 27 | ((x) & 0xF)) +#define HCI_GPIO_OUT 0x70 +#define HCI_ERR_EN_PA_LAYER 0x78 +#define HCI_ERR_EN_DL_LAYER 0x7C +#define HCI_ERR_EN_N_LAYER 0x80 +#define HCI_ERR_EN_T_LAYER 0x84 +#define HCI_ERR_EN_DME_LAYER 0x88 +#define HCI_V2P1_CTRL 0x8C +#define IA_TICK_SEL BIT(16) +#define HCI_CLKSTOP_CTRL 0xB0 +#define REFCLKOUT_STOP BIT(4) +#define MPHY_APBCLK_STOP BIT(3) +#define REFCLK_STOP BIT(2) +#define UNIPRO_MCLK_STOP BIT(1) +#define UNIPRO_PCLK_STOP BIT(0) +#define CLK_STOP_MASK (REFCLKOUT_STOP | REFCLK_STOP |\ + UNIPRO_MCLK_STOP | MPHY_APBCLK_STOP|\ + UNIPRO_PCLK_STOP) +/* HCI_MISC is also known as HCI_FORCE_HCS */ +#define HCI_MISC 0xB4 +#define REFCLK_CTRL_EN BIT(7) +#define UNIPRO_PCLK_CTRL_EN BIT(6) +#define UNIPRO_MCLK_CTRL_EN BIT(5) +#define HCI_CORECLK_CTRL_EN BIT(4) +#define CLK_CTRL_EN_MASK (REFCLK_CTRL_EN |\ + UNIPRO_PCLK_CTRL_EN |\ + UNIPRO_MCLK_CTRL_EN) + +#define HCI_IOP_ACG_DISABLE 0x100 +#define HCI_IOP_ACG_DISABLE_EN BIT(0) + +/* Device fatal error */ +#define DFES_ERR_EN BIT(31) +#define DFES_DEF_L2_ERRS (UIC_DATA_LINK_LAYER_ERROR_RX_BUF_OF |\ + UIC_DATA_LINK_LAYER_ERROR_PA_INIT) +#define DFES_DEF_L3_ERRS (UIC_NETWORK_UNSUPPORTED_HEADER_TYPE |\ + UIC_NETWORK_BAD_DEVICEID_ENC |\ + UIC_NETWORK_LHDR_TRAP_PACKET_DROPPING) +#define DFES_DEF_L4_ERRS (UIC_TRANSPORT_UNSUPPORTED_HEADER_TYPE |\ + UIC_TRANSPORT_UNKNOWN_CPORTID |\ + UIC_TRANSPORT_NO_CONNECTION_RX |\ + UIC_TRANSPORT_BAD_TC) + +/* UFS Shareability */ +#define UFS_EXYNOSAUTO_WR_SHARABLE BIT(2) +#define UFS_EXYNOSAUTO_RD_SHARABLE BIT(1) +#define UFS_EXYNOSAUTO_SHARABLE (UFS_EXYNOSAUTO_WR_SHARABLE | \ + UFS_EXYNOSAUTO_RD_SHARABLE) +#define UFS_EXYNOSAUTOV920_WR_SHARABLE BIT(3) +#define UFS_EXYNOSAUTOV920_RD_SHARABLE BIT(2) +#define UFS_EXYNOSAUTOV920_SHARABLE (UFS_EXYNOSAUTOV920_WR_SHARABLE |\ + UFS_EXYNOSAUTOV920_RD_SHARABLE) +#define UFS_GS101_WR_SHARABLE BIT(1) +#define UFS_GS101_RD_SHARABLE BIT(0) +#define UFS_GS101_SHARABLE (UFS_GS101_WR_SHARABLE | \ + UFS_GS101_RD_SHARABLE) +#define UFS_SHAREABILITY_OFFSET 0x710 + +/* Multi-host registers */ +#define MHCTRL 0xC4 +#define MHCTRL_EN_VH_MASK (0xE) +#define MHCTRL_EN_VH(vh) (vh << 1) +#define PH2VH_MBOX 0xD8 + +#define MH_MSG_MASK (0xFF) + +#define MH_MSG(id, msg) ((id << 8) | (msg & 0xFF)) +#define MH_MSG_PH_READY 0x1 +#define MH_MSG_VH_READY 0x2 + +#define ALLOW_INQUIRY BIT(25) +#define ALLOW_MODE_SELECT BIT(24) +#define ALLOW_MODE_SENSE BIT(23) +#define ALLOW_PRE_FETCH GENMASK(22, 21) +#define ALLOW_READ_CMD_ALL GENMASK(20, 18) /* read_6/10/16 */ +#define ALLOW_READ_BUFFER BIT(17) +#define ALLOW_READ_CAPACITY GENMASK(16, 15) +#define ALLOW_REPORT_LUNS BIT(14) +#define ALLOW_REQUEST_SENSE BIT(13) +#define ALLOW_SYNCHRONIZE_CACHE GENMASK(8, 7) +#define ALLOW_TEST_UNIT_READY BIT(6) +#define ALLOW_UNMAP BIT(5) +#define ALLOW_VERIFY BIT(4) +#define ALLOW_WRITE_CMD_ALL GENMASK(3, 1) /* write_6/10/16 */ + +#define ALLOW_TRANS_VH_DEFAULT (ALLOW_INQUIRY | ALLOW_MODE_SELECT | \ + ALLOW_MODE_SENSE | ALLOW_PRE_FETCH | \ + ALLOW_READ_CMD_ALL | ALLOW_READ_BUFFER | \ + ALLOW_READ_CAPACITY | ALLOW_REPORT_LUNS | \ + ALLOW_REQUEST_SENSE | ALLOW_SYNCHRONIZE_CACHE | \ + ALLOW_TEST_UNIT_READY | ALLOW_UNMAP | \ + ALLOW_VERIFY | ALLOW_WRITE_CMD_ALL) + +#define HCI_MH_ALLOWABLE_TRAN_OF_VH 0x30C +#define HCI_MH_IID_IN_TASK_TAG 0X308 + +#define PH_READY_TIMEOUT_MS (5 * MSEC_PER_SEC) + +enum { + UNIPRO_L1_5 = 0,/* PHY Adapter */ + UNIPRO_L2, /* Data Link */ + UNIPRO_L3, /* Network */ + UNIPRO_L4, /* Transport */ + UNIPRO_DME, /* DME */ +}; + +/* + * UNIPRO registers + */ +#define UNIPRO_DME_POWERMODE_REQ_LOCALL2TIMER0 0x7888 +#define UNIPRO_DME_POWERMODE_REQ_LOCALL2TIMER1 0x788c +#define UNIPRO_DME_POWERMODE_REQ_LOCALL2TIMER2 0x7890 +#define UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER0 0x78B8 +#define UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER1 0x78BC +#define UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER2 0x78C0 + +/* + * UFS Protector registers + */ +#define UFSPRSECURITY 0x010 +#define NSSMU BIT(14) +#define UFSPSBEGIN0 0x200 +#define UFSPSEND0 0x204 +#define UFSPSLUN0 0x208 +#define UFSPSCTRL0 0x20C + +#define CNTR_DIV_VAL 40 + +static void exynos_ufs_auto_ctrl_hcc(struct exynos_ufs *ufs, bool en); +static void exynos_ufs_ctrl_clkstop(struct exynos_ufs *ufs, bool en); + +static inline void exynos_ufs_enable_auto_ctrl_hcc(struct exynos_ufs *ufs) +{ + exynos_ufs_auto_ctrl_hcc(ufs, true); +} + +static inline void exynos_ufs_disable_auto_ctrl_hcc(struct exynos_ufs *ufs) +{ + exynos_ufs_auto_ctrl_hcc(ufs, false); +} + +static inline void exynos_ufs_disable_auto_ctrl_hcc_save( + struct exynos_ufs *ufs, u32 *val) +{ + *val = hci_readl(ufs, HCI_MISC); + exynos_ufs_auto_ctrl_hcc(ufs, false); +} + +static inline void exynos_ufs_auto_ctrl_hcc_restore( + struct exynos_ufs *ufs, u32 *val) +{ + hci_writel(ufs, *val, HCI_MISC); +} + +static inline void exynos_ufs_gate_clks(struct exynos_ufs *ufs) +{ + exynos_ufs_ctrl_clkstop(ufs, true); +} + +static inline void exynos_ufs_ungate_clks(struct exynos_ufs *ufs) +{ + exynos_ufs_ctrl_clkstop(ufs, false); +} + +static int exynos_ufs_shareability(struct exynos_ufs *ufs) +{ + /* IO Coherency setting */ + if (ufs->sysreg) { + return regmap_update_bits(ufs->sysreg, + ufs->iocc_offset, + ufs->iocc_mask, ufs->iocc_val); + } + + return 0; +} + +static int gs101_ufs_drv_init(struct exynos_ufs *ufs) +{ + struct ufs_hba *hba = ufs->hba; + u32 reg; + + /* Enable WriteBooster */ + hba->caps |= UFSHCD_CAP_WB_EN; + + /* Enable clock gating and hibern8 */ + hba->caps |= UFSHCD_CAP_CLK_GATING | UFSHCD_CAP_HIBERN8_WITH_CLK_GATING; + + /* set ACG to be controlled by UFS_ACG_DISABLE */ + reg = hci_readl(ufs, HCI_IOP_ACG_DISABLE); + hci_writel(ufs, reg & (~HCI_IOP_ACG_DISABLE_EN), HCI_IOP_ACG_DISABLE); + + return exynos_ufs_shareability(ufs); +} + +static int exynosauto_ufs_drv_init(struct exynos_ufs *ufs) +{ + return exynos_ufs_shareability(ufs); +} + +static int exynosauto_ufs_post_hce_enable(struct exynos_ufs *ufs) +{ + struct ufs_hba *hba = ufs->hba; + + /* Enable Virtual Host #1 */ + ufshcd_rmwl(hba, MHCTRL_EN_VH_MASK, MHCTRL_EN_VH(1), MHCTRL); + /* Default VH Transfer permissions */ + hci_writel(ufs, ALLOW_TRANS_VH_DEFAULT, HCI_MH_ALLOWABLE_TRAN_OF_VH); + /* IID information is replaced in TASKTAG[7:5] instead of IID in UCD */ + hci_writel(ufs, 0x1, HCI_MH_IID_IN_TASK_TAG); + + return 0; +} + +static int exynosauto_ufs_pre_link(struct exynos_ufs *ufs) +{ + struct ufs_hba *hba = ufs->hba; + int i; + u32 tx_line_reset_period, rx_line_reset_period; + + rx_line_reset_period = (RX_LINE_RESET_TIME * ufs->mclk_rate) / NSEC_PER_MSEC; + tx_line_reset_period = (TX_LINE_RESET_TIME * ufs->mclk_rate) / NSEC_PER_MSEC; + + ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x40); + for_each_ufs_rx_lane(ufs, i) { + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_CLK_PRD, i), + DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_CLK_PRD_EN, i), 0x0); + + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_VALUE2, i), + (rx_line_reset_period >> 16) & 0xFF); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_VALUE1, i), + (rx_line_reset_period >> 8) & 0xFF); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_VALUE0, i), + (rx_line_reset_period) & 0xFF); + + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x2f, i), 0x79); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x84, i), 0x1); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x25, i), 0xf6); + } + + for_each_ufs_tx_lane(ufs, i) { + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_CLK_PRD, i), + DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate)); + /* Not to affect VND_TX_LINERESET_PVALUE to VND_TX_CLK_PRD */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_CLK_PRD_EN, i), + 0x02); + + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE2, i), + (tx_line_reset_period >> 16) & 0xFF); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE1, i), + (tx_line_reset_period >> 8) & 0xFF); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE0, i), + (tx_line_reset_period) & 0xFF); + + /* TX PWM Gear Capability / PWM_G1_ONLY */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x04, i), 0x1); + } + + ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x0); + + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_LOCAL_TX_LCC_ENABLE), 0x0); + + ufshcd_dme_set(hba, UIC_ARG_MIB(0xa011), 0x8000); + + return 0; +} + +static int exynosauto_ufs_pre_pwr_change(struct exynos_ufs *ufs, + struct ufs_pa_layer_attr *pwr) +{ + struct ufs_hba *hba = ufs->hba; + + /* PACP_PWR_req and delivered to the remote DME */ + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0), 12000); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1), 32000); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2), 16000); + + return 0; +} + +static int exynosauto_ufs_post_pwr_change(struct exynos_ufs *ufs, + const struct ufs_pa_layer_attr *pwr) +{ + struct ufs_hba *hba = ufs->hba; + u32 enabled_vh; + + enabled_vh = ufshcd_readl(hba, MHCTRL) & MHCTRL_EN_VH_MASK; + + /* Send physical host ready message to virtual hosts */ + ufshcd_writel(hba, MH_MSG(enabled_vh, MH_MSG_PH_READY), PH2VH_MBOX); + + return 0; +} + +static int exynos7_ufs_pre_link(struct exynos_ufs *ufs) +{ + struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr; + u32 val = attr->pa_dbg_opt_suite1_val; + struct ufs_hba *hba = ufs->hba; + int i; + + exynos_ufs_enable_ov_tm(hba); + for_each_ufs_tx_lane(ufs, i) + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x297, i), 0x17); + for_each_ufs_rx_lane(ufs, i) { + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x362, i), 0xff); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x363, i), 0x00); + } + exynos_ufs_disable_ov_tm(hba); + + for_each_ufs_tx_lane(ufs, i) + ufshcd_dme_set(hba, + UIC_ARG_MIB_SEL(TX_HIBERN8_CONTROL, i), 0x0); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_TXPHY_CFGUPDT), 0x1); + udelay(1); + ufshcd_dme_set(hba, UIC_ARG_MIB(attr->pa_dbg_opt_suite1_off), + val | (1 << 12)); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_SKIP_RESET_PHY), 0x1); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_SKIP_LINE_RESET), 0x1); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_LINE_RESET_REQ), 0x1); + udelay(1600); + ufshcd_dme_set(hba, UIC_ARG_MIB(attr->pa_dbg_opt_suite1_off), val); + + return 0; +} + +static int exynos7_ufs_post_link(struct exynos_ufs *ufs) +{ + struct ufs_hba *hba = ufs->hba; + int i; + + exynos_ufs_enable_ov_tm(hba); + for_each_ufs_tx_lane(ufs, i) { + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x28b, i), 0x83); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x29a, i), 0x07); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x277, i), + TX_LINERESET_N(exynos_ufs_calc_time_cntr(ufs, 200000))); + } + exynos_ufs_disable_ov_tm(hba); + + exynos_ufs_enable_dbg_mode(hba); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_SAVECONFIGTIME), 0xbb8); + exynos_ufs_disable_dbg_mode(hba); + + return 0; +} + +static int exynos7_ufs_pre_pwr_change(struct exynos_ufs *ufs, + struct ufs_pa_layer_attr *pwr) +{ + unipro_writel(ufs, 0x22, UNIPRO_DBG_FORCE_DME_CTRL_STATE); + + return 0; +} + +static int exynos7_ufs_post_pwr_change(struct exynos_ufs *ufs, + const struct ufs_pa_layer_attr *pwr) +{ + struct ufs_hba *hba = ufs->hba; + int lanes = max_t(u32, pwr->lane_rx, pwr->lane_tx); + + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_RXPHY_CFGUPDT), 0x1); + + if (lanes == 1) { + exynos_ufs_enable_dbg_mode(hba); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 0x1); + exynos_ufs_disable_dbg_mode(hba); + } + + return 0; +} + +static int exynosautov920_ufs_pre_link(struct exynos_ufs *ufs) +{ + struct ufs_hba *hba = ufs->hba; + int i; + u32 tx_line_reset_period, rx_line_reset_period; + + rx_line_reset_period = (RX_LINE_RESET_TIME * ufs->mclk_rate) + / NSEC_PER_MSEC; + tx_line_reset_period = (TX_LINE_RESET_TIME * ufs->mclk_rate) + / NSEC_PER_MSEC; + + unipro_writel(ufs, 0x5f, 0x44); + + ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x40); + ufshcd_dme_set(hba, UIC_ARG_MIB(0x202), 0x02); + + for_each_ufs_rx_lane(ufs, i) { + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_CLK_PRD, i), + DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_CLK_PRD_EN, i), 0x0); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_VALUE2, i), + (rx_line_reset_period >> 16) & 0xFF); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_VALUE1, i), + (rx_line_reset_period >> 8) & 0xFF); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_VALUE0, i), + (rx_line_reset_period) & 0xFF); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x2f, i), 0x69); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x84, i), 0x1); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x25, i), 0xf6); + } + + for_each_ufs_tx_lane(ufs, i) { + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_CLK_PRD, i), + DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_CLK_PRD_EN, i), + 0x02); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE2, i), + (tx_line_reset_period >> 16) & 0xFF); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE1, i), + (tx_line_reset_period >> 8) & 0xFF); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE0, i), + (tx_line_reset_period) & 0xFF); + + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x04, i), 0x1); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x7f, i), 0x0); + } + + ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x0); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_LOCAL_TX_LCC_ENABLE), 0x0); + ufshcd_dme_set(hba, UIC_ARG_MIB(0xa011), 0x8000); + + return 0; +} + +static int exynosautov920_ufs_post_link(struct exynos_ufs *ufs) +{ + struct ufs_hba *hba = ufs->hba; + + ufshcd_dme_set(hba, UIC_ARG_MIB(0x9529), 0x1); + ufshcd_dme_set(hba, UIC_ARG_MIB(0x15a4), 0x3e8); + ufshcd_dme_set(hba, UIC_ARG_MIB(0x9529), 0x0); + + return 0; +} + +static int exynosautov920_ufs_pre_pwr_change(struct exynos_ufs *ufs, + struct ufs_pa_layer_attr *pwr) +{ + struct ufs_hba *hba = ufs->hba; + + ufshcd_dme_set(hba, UIC_ARG_MIB(0x15d4), 0x1); + + ufshcd_dme_set(hba, UIC_ARG_MIB(DL_FC0PROTTIMEOUTVAL), 8064); + ufshcd_dme_set(hba, UIC_ARG_MIB(DL_TC0REPLAYTIMEOUTVAL), 28224); + ufshcd_dme_set(hba, UIC_ARG_MIB(DL_AFC0REQTIMEOUTVAL), 20160); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0), 12000); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1), 32000); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2), 16000); + + unipro_writel(ufs, 8064, UNIPRO_DME_POWERMODE_REQ_LOCALL2TIMER0); + unipro_writel(ufs, 28224, UNIPRO_DME_POWERMODE_REQ_LOCALL2TIMER1); + unipro_writel(ufs, 20160, UNIPRO_DME_POWERMODE_REQ_LOCALL2TIMER2); + unipro_writel(ufs, 12000, UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER0); + unipro_writel(ufs, 32000, UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER1); + unipro_writel(ufs, 16000, UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER2); + + return 0; +} + +/* + * exynos_ufs_auto_ctrl_hcc - HCI core clock control by h/w + * Control should be disabled in the below cases + * - Before host controller S/W reset + * - Access to UFS protector's register + */ +static void exynos_ufs_auto_ctrl_hcc(struct exynos_ufs *ufs, bool en) +{ + u32 misc = hci_readl(ufs, HCI_MISC); + + if (en) + hci_writel(ufs, misc | HCI_CORECLK_CTRL_EN, HCI_MISC); + else + hci_writel(ufs, misc & ~HCI_CORECLK_CTRL_EN, HCI_MISC); +} + +static void exynos_ufs_ctrl_clkstop(struct exynos_ufs *ufs, bool en) +{ + u32 ctrl = hci_readl(ufs, HCI_CLKSTOP_CTRL); + u32 misc = hci_readl(ufs, HCI_MISC); + + if (en) { + hci_writel(ufs, misc | CLK_CTRL_EN_MASK, HCI_MISC); + hci_writel(ufs, ctrl | CLK_STOP_MASK, HCI_CLKSTOP_CTRL); + } else { + hci_writel(ufs, ctrl & ~CLK_STOP_MASK, HCI_CLKSTOP_CTRL); + hci_writel(ufs, misc & ~CLK_CTRL_EN_MASK, HCI_MISC); + } +} + +static int exynos_ufs_get_clk_info(struct exynos_ufs *ufs) +{ + struct ufs_hba *hba = ufs->hba; + struct list_head *head = &hba->clk_list_head; + struct ufs_clk_info *clki; + unsigned long pclk_rate; + u32 f_min, f_max; + u8 div = 0; + int ret = 0; + + if (list_empty(head)) + goto out; + + list_for_each_entry(clki, head, list) { + if (!IS_ERR(clki->clk)) { + if (!strcmp(clki->name, "core_clk")) + ufs->clk_hci_core = clki->clk; + else if (!strcmp(clki->name, "sclk_unipro_main")) + ufs->clk_unipro_main = clki->clk; + } + } + + if (!ufs->clk_hci_core || !ufs->clk_unipro_main) { + dev_err(hba->dev, "failed to get clk info\n"); + ret = -EINVAL; + goto out; + } + + ufs->mclk_rate = clk_get_rate(ufs->clk_unipro_main); + pclk_rate = clk_get_rate(ufs->clk_hci_core); + f_min = ufs->pclk_avail_min; + f_max = ufs->pclk_avail_max; + + if (ufs->opts & EXYNOS_UFS_OPT_HAS_APB_CLK_CTRL) { + do { + pclk_rate /= (div + 1); + + if (pclk_rate <= f_max) + break; + div++; + } while (pclk_rate >= f_min); + } + + if (unlikely(pclk_rate < f_min || pclk_rate > f_max)) { + dev_err(hba->dev, "not available pclk range %lu\n", pclk_rate); + ret = -EINVAL; + goto out; + } + + ufs->pclk_rate = pclk_rate; + ufs->pclk_div = div; + +out: + return ret; +} + +static void exynos_ufs_set_unipro_pclk_div(struct exynos_ufs *ufs) +{ + if (ufs->opts & EXYNOS_UFS_OPT_HAS_APB_CLK_CTRL) { + u32 val; + + val = hci_readl(ufs, HCI_UNIPRO_APB_CLK_CTRL); + hci_writel(ufs, UNIPRO_APB_CLK(val, ufs->pclk_div), + HCI_UNIPRO_APB_CLK_CTRL); + } +} + +static void exynos_ufs_set_pwm_clk_div(struct exynos_ufs *ufs) +{ + struct ufs_hba *hba = ufs->hba; + struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr; + + ufshcd_dme_set(hba, + UIC_ARG_MIB(CMN_PWM_CLK_CTRL), attr->cmn_pwm_clk_ctrl); +} + +static void exynos_ufs_calc_pwm_clk_div(struct exynos_ufs *ufs) +{ + struct ufs_hba *hba = ufs->hba; + struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr; + const unsigned int div = 30, mult = 20; + const unsigned long pwm_min = 3 * 1000 * 1000; + const unsigned long pwm_max = 9 * 1000 * 1000; + const int divs[] = {32, 16, 8, 4}; + unsigned long clk = 0, _clk, clk_period; + int i = 0, clk_idx = -1; + + clk_period = UNIPRO_PCLK_PERIOD(ufs); + for (i = 0; i < ARRAY_SIZE(divs); i++) { + _clk = NSEC_PER_SEC * mult / (clk_period * divs[i] * div); + if (_clk >= pwm_min && _clk <= pwm_max) { + if (_clk > clk) { + clk_idx = i; + clk = _clk; + } + } + } + + if (clk_idx == -1) { + ufshcd_dme_get(hba, UIC_ARG_MIB(CMN_PWM_CLK_CTRL), &clk_idx); + dev_err(hba->dev, + "failed to decide pwm clock divider, will not change\n"); + } + + attr->cmn_pwm_clk_ctrl = clk_idx & PWM_CLK_CTRL_MASK; +} + +long exynos_ufs_calc_time_cntr(struct exynos_ufs *ufs, long period) +{ + const int precise = 10; + long pclk_rate = ufs->pclk_rate; + long clk_period, fraction; + + clk_period = UNIPRO_PCLK_PERIOD(ufs); + fraction = ((NSEC_PER_SEC % pclk_rate) * precise) / pclk_rate; + + return (period * precise) / ((clk_period * precise) + fraction); +} + +static void exynos_ufs_specify_phy_time_attr(struct exynos_ufs *ufs) +{ + struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr; + struct ufs_phy_time_cfg *t_cfg = &ufs->t_cfg; + + if (ufs->opts & EXYNOS_UFS_OPT_SKIP_CONFIG_PHY_ATTR) + return; + + t_cfg->tx_linereset_p = + exynos_ufs_calc_time_cntr(ufs, attr->tx_dif_p_nsec); + t_cfg->tx_linereset_n = + exynos_ufs_calc_time_cntr(ufs, attr->tx_dif_n_nsec); + t_cfg->tx_high_z_cnt = + exynos_ufs_calc_time_cntr(ufs, attr->tx_high_z_cnt_nsec); + t_cfg->tx_base_n_val = + exynos_ufs_calc_time_cntr(ufs, attr->tx_base_unit_nsec); + t_cfg->tx_gran_n_val = + exynos_ufs_calc_time_cntr(ufs, attr->tx_gran_unit_nsec); + t_cfg->tx_sleep_cnt = + exynos_ufs_calc_time_cntr(ufs, attr->tx_sleep_cnt); + + t_cfg->rx_linereset = + exynos_ufs_calc_time_cntr(ufs, attr->rx_dif_p_nsec); + t_cfg->rx_hibern8_wait = + exynos_ufs_calc_time_cntr(ufs, attr->rx_hibern8_wait_nsec); + t_cfg->rx_base_n_val = + exynos_ufs_calc_time_cntr(ufs, attr->rx_base_unit_nsec); + t_cfg->rx_gran_n_val = + exynos_ufs_calc_time_cntr(ufs, attr->rx_gran_unit_nsec); + t_cfg->rx_sleep_cnt = + exynos_ufs_calc_time_cntr(ufs, attr->rx_sleep_cnt); + t_cfg->rx_stall_cnt = + exynos_ufs_calc_time_cntr(ufs, attr->rx_stall_cnt); +} + +static void exynos_ufs_config_phy_time_attr(struct exynos_ufs *ufs) +{ + struct ufs_hba *hba = ufs->hba; + struct ufs_phy_time_cfg *t_cfg = &ufs->t_cfg; + int i; + + exynos_ufs_set_pwm_clk_div(ufs); + + exynos_ufs_enable_ov_tm(hba); + + for_each_ufs_rx_lane(ufs, i) { + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_FILLER_ENABLE, i), + ufs->drv_data->uic_attr->rx_filler_enable); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_LINERESET_VAL, i), + RX_LINERESET(t_cfg->rx_linereset)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_BASE_NVAL_07_00, i), + RX_BASE_NVAL_L(t_cfg->rx_base_n_val)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_BASE_NVAL_15_08, i), + RX_BASE_NVAL_H(t_cfg->rx_base_n_val)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_GRAN_NVAL_07_00, i), + RX_GRAN_NVAL_L(t_cfg->rx_gran_n_val)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_GRAN_NVAL_10_08, i), + RX_GRAN_NVAL_H(t_cfg->rx_gran_n_val)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_OV_SLEEP_CNT_TIMER, i), + RX_OV_SLEEP_CNT(t_cfg->rx_sleep_cnt)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_OV_STALL_CNT_TIMER, i), + RX_OV_STALL_CNT(t_cfg->rx_stall_cnt)); + } + + for_each_ufs_tx_lane(ufs, i) { + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_LINERESET_P_VAL, i), + TX_LINERESET_P(t_cfg->tx_linereset_p)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_HIGH_Z_CNT_07_00, i), + TX_HIGH_Z_CNT_L(t_cfg->tx_high_z_cnt)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_HIGH_Z_CNT_11_08, i), + TX_HIGH_Z_CNT_H(t_cfg->tx_high_z_cnt)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_BASE_NVAL_07_00, i), + TX_BASE_NVAL_L(t_cfg->tx_base_n_val)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_BASE_NVAL_15_08, i), + TX_BASE_NVAL_H(t_cfg->tx_base_n_val)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_GRAN_NVAL_07_00, i), + TX_GRAN_NVAL_L(t_cfg->tx_gran_n_val)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_GRAN_NVAL_10_08, i), + TX_GRAN_NVAL_H(t_cfg->tx_gran_n_val)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_OV_SLEEP_CNT_TIMER, i), + TX_OV_H8_ENTER_EN | + TX_OV_SLEEP_CNT(t_cfg->tx_sleep_cnt)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_MIN_ACTIVATETIME, i), + ufs->drv_data->uic_attr->tx_min_activatetime); + } + + exynos_ufs_disable_ov_tm(hba); +} + +static void exynos_ufs_config_phy_cap_attr(struct exynos_ufs *ufs) +{ + struct ufs_hba *hba = ufs->hba; + struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr; + int i; + + exynos_ufs_enable_ov_tm(hba); + + for_each_ufs_rx_lane(ufs, i) { + ufshcd_dme_set(hba, + UIC_ARG_MIB_SEL(RX_HS_G1_SYNC_LENGTH_CAP, i), + attr->rx_hs_g1_sync_len_cap); + ufshcd_dme_set(hba, + UIC_ARG_MIB_SEL(RX_HS_G2_SYNC_LENGTH_CAP, i), + attr->rx_hs_g2_sync_len_cap); + ufshcd_dme_set(hba, + UIC_ARG_MIB_SEL(RX_HS_G3_SYNC_LENGTH_CAP, i), + attr->rx_hs_g3_sync_len_cap); + ufshcd_dme_set(hba, + UIC_ARG_MIB_SEL(RX_HS_G1_PREP_LENGTH_CAP, i), + attr->rx_hs_g1_prep_sync_len_cap); + ufshcd_dme_set(hba, + UIC_ARG_MIB_SEL(RX_HS_G2_PREP_LENGTH_CAP, i), + attr->rx_hs_g2_prep_sync_len_cap); + ufshcd_dme_set(hba, + UIC_ARG_MIB_SEL(RX_HS_G3_PREP_LENGTH_CAP, i), + attr->rx_hs_g3_prep_sync_len_cap); + } + + if (attr->rx_adv_fine_gran_sup_en == 0) { + for_each_ufs_rx_lane(ufs, i) { + ufshcd_dme_set(hba, + UIC_ARG_MIB_SEL(RX_ADV_GRANULARITY_CAP, i), 0); + + if (attr->rx_min_actv_time_cap) + ufshcd_dme_set(hba, + UIC_ARG_MIB_SEL( + RX_MIN_ACTIVATETIME_CAPABILITY, i), + attr->rx_min_actv_time_cap); + + if (attr->rx_hibern8_time_cap) + ufshcd_dme_set(hba, + UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAP, i), + attr->rx_hibern8_time_cap); + } + } else if (attr->rx_adv_fine_gran_sup_en == 1) { + for_each_ufs_rx_lane(ufs, i) { + if (attr->rx_adv_fine_gran_step) + ufshcd_dme_set(hba, + UIC_ARG_MIB_SEL(RX_ADV_GRANULARITY_CAP, + i), RX_ADV_FINE_GRAN_STEP( + attr->rx_adv_fine_gran_step)); + + if (attr->rx_adv_min_actv_time_cap) + ufshcd_dme_set(hba, + UIC_ARG_MIB_SEL( + RX_ADV_MIN_ACTIVATETIME_CAP, i), + attr->rx_adv_min_actv_time_cap); + + if (attr->rx_adv_hibern8_time_cap) + ufshcd_dme_set(hba, + UIC_ARG_MIB_SEL(RX_ADV_HIBERN8TIME_CAP, + i), + attr->rx_adv_hibern8_time_cap); + } + } + + exynos_ufs_disable_ov_tm(hba); +} + +static void exynos_ufs_establish_connt(struct exynos_ufs *ufs) +{ + struct ufs_hba *hba = ufs->hba; + enum { + DEV_ID = 0x00, + PEER_DEV_ID = 0x01, + PEER_CPORT_ID = 0x00, + TRAFFIC_CLASS = 0x00, + }; + + /* allow cport attributes to be set */ + ufshcd_dme_set(hba, UIC_ARG_MIB(T_CONNECTIONSTATE), CPORT_IDLE); + + /* local unipro attributes */ + ufshcd_dme_set(hba, UIC_ARG_MIB(N_DEVICEID), DEV_ID); + ufshcd_dme_set(hba, UIC_ARG_MIB(N_DEVICEID_VALID), true); + ufshcd_dme_set(hba, UIC_ARG_MIB(T_PEERDEVICEID), PEER_DEV_ID); + ufshcd_dme_set(hba, UIC_ARG_MIB(T_PEERCPORTID), PEER_CPORT_ID); + ufshcd_dme_set(hba, UIC_ARG_MIB(T_CPORTFLAGS), CPORT_DEF_FLAGS); + ufshcd_dme_set(hba, UIC_ARG_MIB(T_TRAFFICCLASS), TRAFFIC_CLASS); + ufshcd_dme_set(hba, UIC_ARG_MIB(T_CONNECTIONSTATE), CPORT_CONNECTED); +} + +static void exynos_ufs_config_smu(struct exynos_ufs *ufs) +{ + u32 reg, val; + + if (ufs->opts & EXYNOS_UFS_OPT_UFSPR_SECURE) + return; + + exynos_ufs_disable_auto_ctrl_hcc_save(ufs, &val); + + /* make encryption disabled by default */ + reg = ufsp_readl(ufs, UFSPRSECURITY); + ufsp_writel(ufs, reg | NSSMU, UFSPRSECURITY); + ufsp_writel(ufs, 0x0, UFSPSBEGIN0); + ufsp_writel(ufs, 0xffffffff, UFSPSEND0); + ufsp_writel(ufs, 0xff, UFSPSLUN0); + ufsp_writel(ufs, 0xf1, UFSPSCTRL0); + + exynos_ufs_auto_ctrl_hcc_restore(ufs, &val); +} + +static void exynos_ufs_config_sync_pattern_mask(struct exynos_ufs *ufs, + struct ufs_pa_layer_attr *pwr) +{ + struct ufs_hba *hba = ufs->hba; + u8 g = max_t(u32, pwr->gear_rx, pwr->gear_tx); + u32 mask, sync_len; + enum { + SYNC_LEN_G1 = 80 * 1000, /* 80us */ + SYNC_LEN_G2 = 40 * 1000, /* 40us */ + SYNC_LEN_G3 = 20 * 1000, /* 20us */ + }; + int i; + + if (g == 1) + sync_len = SYNC_LEN_G1; + else if (g == 2) + sync_len = SYNC_LEN_G2; + else if (g == 3) + sync_len = SYNC_LEN_G3; + else + return; + + mask = exynos_ufs_calc_time_cntr(ufs, sync_len); + mask = (mask >> 8) & 0xff; + + exynos_ufs_enable_ov_tm(hba); + + for_each_ufs_rx_lane(ufs, i) + ufshcd_dme_set(hba, + UIC_ARG_MIB_SEL(RX_SYNC_MASK_LENGTH, i), mask); + + exynos_ufs_disable_ov_tm(hba); +} + +#define UFS_HW_VER_MAJOR_MASK GENMASK(15, 8) + +static u32 exynos_ufs_get_hs_gear(struct ufs_hba *hba) +{ + u8 major; + + major = FIELD_GET(UFS_HW_VER_MAJOR_MASK, hba->ufs_version); + + if (major >= 3) + return UFS_HS_G4; + + /* Default is HS-G3 */ + return UFS_HS_G3; +} + +static int exynos_ufs_pre_pwr_mode(struct ufs_hba *hba, + struct ufs_pa_layer_attr *dev_req_params) +{ + struct exynos_ufs *ufs = ufshcd_get_variant(hba); + struct phy *generic_phy = ufs->phy; + int ret; + + if (!dev_req_params) { + pr_err("%s: incoming dev_req_params is NULL\n", __func__); + ret = -EINVAL; + goto out; + } + + if (ufs->drv_data->pre_pwr_change) + ufs->drv_data->pre_pwr_change(ufs, dev_req_params); + + if (ufshcd_is_hs_mode(dev_req_params)) { + exynos_ufs_config_sync_pattern_mask(ufs, dev_req_params); + + switch (dev_req_params->hs_rate) { + case PA_HS_MODE_A: + case PA_HS_MODE_B: + phy_calibrate(generic_phy); + break; + } + } + + /* setting for three timeout values for traffic class #0 */ + ufshcd_dme_set(hba, UIC_ARG_MIB(DL_FC0PROTTIMEOUTVAL), 8064); + ufshcd_dme_set(hba, UIC_ARG_MIB(DL_TC0REPLAYTIMEOUTVAL), 28224); + ufshcd_dme_set(hba, UIC_ARG_MIB(DL_AFC0REQTIMEOUTVAL), 20160); + + return 0; +out: + return ret; +} + +#define PWR_MODE_STR_LEN 64 +static int exynos_ufs_post_pwr_mode(struct ufs_hba *hba, + const struct ufs_pa_layer_attr *pwr_req) +{ + struct exynos_ufs *ufs = ufshcd_get_variant(hba); + struct phy *generic_phy = ufs->phy; + int gear = max_t(u32, pwr_req->gear_rx, pwr_req->gear_tx); + int lanes = max_t(u32, pwr_req->lane_rx, pwr_req->lane_tx); + char pwr_str[PWR_MODE_STR_LEN] = ""; + + /* let default be PWM Gear 1, Lane 1 */ + if (!gear) + gear = 1; + + if (!lanes) + lanes = 1; + + if (ufs->drv_data->post_pwr_change) + ufs->drv_data->post_pwr_change(ufs, pwr_req); + + if ((ufshcd_is_hs_mode(pwr_req))) { + switch (pwr_req->hs_rate) { + case PA_HS_MODE_A: + case PA_HS_MODE_B: + phy_calibrate(generic_phy); + break; + } + + snprintf(pwr_str, PWR_MODE_STR_LEN, "%s series_%s G_%d L_%d", + "FAST", pwr_req->hs_rate == PA_HS_MODE_A ? "A" : "B", + gear, lanes); + } else { + snprintf(pwr_str, PWR_MODE_STR_LEN, "%s G_%d L_%d", + "SLOW", gear, lanes); + } + + dev_info(hba->dev, "Power mode changed to : %s\n", pwr_str); + + return 0; +} + +static void exynos_ufs_specify_nexus_t_xfer_req(struct ufs_hba *hba, + int tag, bool is_scsi_cmd) +{ + struct exynos_ufs *ufs = ufshcd_get_variant(hba); + u32 type; + + type = hci_readl(ufs, HCI_UTRL_NEXUS_TYPE); + + if (is_scsi_cmd) + hci_writel(ufs, type | (1 << tag), HCI_UTRL_NEXUS_TYPE); + else + hci_writel(ufs, type & ~(1 << tag), HCI_UTRL_NEXUS_TYPE); +} + +static void exynos_ufs_specify_nexus_t_tm_req(struct ufs_hba *hba, + int tag, u8 func) +{ + struct exynos_ufs *ufs = ufshcd_get_variant(hba); + u32 type; + + type = hci_readl(ufs, HCI_UTMRL_NEXUS_TYPE); + + switch (func) { + case UFS_ABORT_TASK: + case UFS_QUERY_TASK: + hci_writel(ufs, type | (1 << tag), HCI_UTMRL_NEXUS_TYPE); + break; + case UFS_ABORT_TASK_SET: + case UFS_CLEAR_TASK_SET: + case UFS_LOGICAL_RESET: + case UFS_QUERY_TASK_SET: + hci_writel(ufs, type & ~(1 << tag), HCI_UTMRL_NEXUS_TYPE); + break; + } +} + +static int exynos_ufs_phy_init(struct exynos_ufs *ufs) +{ + struct ufs_hba *hba = ufs->hba; + struct phy *generic_phy = ufs->phy; + int ret = 0; + + if (ufs->avail_ln_rx == 0 || ufs->avail_ln_tx == 0) { + ufshcd_dme_get(hba, UIC_ARG_MIB(PA_AVAILRXDATALANES), + &ufs->avail_ln_rx); + ufshcd_dme_get(hba, UIC_ARG_MIB(PA_AVAILTXDATALANES), + &ufs->avail_ln_tx); + WARN(ufs->avail_ln_rx != ufs->avail_ln_tx, + "available data lane is not equal(rx:%d, tx:%d)\n", + ufs->avail_ln_rx, ufs->avail_ln_tx); + } + + phy_set_bus_width(generic_phy, ufs->avail_ln_rx); + + if (generic_phy->power_count) { + phy_power_off(generic_phy); + phy_exit(generic_phy); + } + + ret = phy_init(generic_phy); + if (ret) { + dev_err(hba->dev, "%s: phy init failed, ret = %d\n", + __func__, ret); + return ret; + } + + ret = phy_power_on(generic_phy); + if (ret) + goto out_exit_phy; + + return 0; + +out_exit_phy: + phy_exit(generic_phy); + + return ret; +} + +static void exynos_ufs_config_unipro(struct exynos_ufs *ufs) +{ + struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr; + struct ufs_hba *hba = ufs->hba; + + if (attr->pa_dbg_clk_period_off) + ufshcd_dme_set(hba, UIC_ARG_MIB(attr->pa_dbg_clk_period_off), + DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate)); + + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTRAILINGCLOCKS), + ufs->drv_data->uic_attr->tx_trailingclks); + + if (attr->pa_dbg_opt_suite1_off) + ufshcd_dme_set(hba, UIC_ARG_MIB(attr->pa_dbg_opt_suite1_off), + attr->pa_dbg_opt_suite1_val); + + if (attr->pa_dbg_opt_suite2_off) + ufshcd_dme_set(hba, UIC_ARG_MIB(attr->pa_dbg_opt_suite2_off), + attr->pa_dbg_opt_suite2_val); +} + +static void exynos_ufs_config_intr(struct exynos_ufs *ufs, u32 errs, u8 index) +{ + switch (index) { + case UNIPRO_L1_5: + hci_writel(ufs, DFES_ERR_EN | errs, HCI_ERR_EN_PA_LAYER); + break; + case UNIPRO_L2: + hci_writel(ufs, DFES_ERR_EN | errs, HCI_ERR_EN_DL_LAYER); + break; + case UNIPRO_L3: + hci_writel(ufs, DFES_ERR_EN | errs, HCI_ERR_EN_N_LAYER); + break; + case UNIPRO_L4: + hci_writel(ufs, DFES_ERR_EN | errs, HCI_ERR_EN_T_LAYER); + break; + case UNIPRO_DME: + hci_writel(ufs, DFES_ERR_EN | errs, HCI_ERR_EN_DME_LAYER); + break; + } +} + +static int exynos_ufs_setup_clocks(struct ufs_hba *hba, bool on, + enum ufs_notify_change_status status) +{ + struct exynos_ufs *ufs = ufshcd_get_variant(hba); + + if (!ufs) + return 0; + + if (on && status == PRE_CHANGE) { + if (ufs->opts & EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL) + exynos_ufs_disable_auto_ctrl_hcc(ufs); + exynos_ufs_ungate_clks(ufs); + } else if (!on && status == POST_CHANGE) { + exynos_ufs_gate_clks(ufs); + if (ufs->opts & EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL) + exynos_ufs_enable_auto_ctrl_hcc(ufs); + } + + return 0; +} + +static int exynos_ufs_pre_link(struct ufs_hba *hba) +{ + struct exynos_ufs *ufs = ufshcd_get_variant(hba); + + /* hci */ + exynos_ufs_config_intr(ufs, DFES_DEF_L2_ERRS, UNIPRO_L2); + exynos_ufs_config_intr(ufs, DFES_DEF_L3_ERRS, UNIPRO_L3); + exynos_ufs_config_intr(ufs, DFES_DEF_L4_ERRS, UNIPRO_L4); + exynos_ufs_set_unipro_pclk_div(ufs); + + exynos_ufs_setup_clocks(hba, true, PRE_CHANGE); + + /* unipro */ + exynos_ufs_config_unipro(ufs); + + if (ufs->drv_data->pre_link) + ufs->drv_data->pre_link(ufs); + + /* m-phy */ + exynos_ufs_phy_init(ufs); + if (!(ufs->opts & EXYNOS_UFS_OPT_SKIP_CONFIG_PHY_ATTR)) { + exynos_ufs_config_phy_time_attr(ufs); + exynos_ufs_config_phy_cap_attr(ufs); + } + + return 0; +} + +static void exynos_ufs_fit_aggr_timeout(struct exynos_ufs *ufs) +{ + u32 val; + + /* Select function clock (mclk) for timer tick */ + if (ufs->opts & EXYNOS_UFS_OPT_TIMER_TICK_SELECT) { + val = hci_readl(ufs, HCI_V2P1_CTRL); + val |= IA_TICK_SEL; + hci_writel(ufs, val, HCI_V2P1_CTRL); + } + + val = exynos_ufs_calc_time_cntr(ufs, IATOVAL_NSEC / CNTR_DIV_VAL); + hci_writel(ufs, val & CNT_VAL_1US_MASK, HCI_1US_TO_CNT_VAL); +} + +static int exynos_ufs_post_link(struct ufs_hba *hba) +{ + struct exynos_ufs *ufs = ufshcd_get_variant(hba); + struct phy *generic_phy = ufs->phy; + struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr; + u32 val = ilog2(DATA_UNIT_SIZE); + + exynos_ufs_establish_connt(ufs); + exynos_ufs_fit_aggr_timeout(ufs); + + hci_writel(ufs, 0xa, HCI_DATA_REORDER); + + if (hba->caps & UFSHCD_CAP_CRYPTO) + val |= PRDT_PREFETCH_EN; + hci_writel(ufs, val, HCI_TXPRDT_ENTRY_SIZE); + + hci_writel(ufs, ilog2(DATA_UNIT_SIZE), HCI_RXPRDT_ENTRY_SIZE); + hci_writel(ufs, BIT(hba->nutrs) - 1, HCI_UTRL_NEXUS_TYPE); + hci_writel(ufs, BIT(hba->nutmrs) - 1, HCI_UTMRL_NEXUS_TYPE); + hci_writel(ufs, 0xf, HCI_AXIDMA_RWDATA_BURST_LEN); + + if (ufs->opts & EXYNOS_UFS_OPT_SKIP_CONNECTION_ESTAB) + ufshcd_dme_set(hba, + UIC_ARG_MIB(T_DBG_SKIP_INIT_HIBERN8_EXIT), true); + + if (attr->pa_granularity) { + exynos_ufs_enable_dbg_mode(hba); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_GRANULARITY), + attr->pa_granularity); + exynos_ufs_disable_dbg_mode(hba); + + if (attr->pa_tactivate) + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), + attr->pa_tactivate); + if (attr->pa_hibern8time && + !(ufs->opts & EXYNOS_UFS_OPT_USE_SW_HIBERN8_TIMER)) + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), + attr->pa_hibern8time); + } + + if (ufs->opts & EXYNOS_UFS_OPT_USE_SW_HIBERN8_TIMER) { + if (!attr->pa_granularity) + ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY), + &attr->pa_granularity); + if (!attr->pa_hibern8time) + ufshcd_dme_get(hba, UIC_ARG_MIB(PA_HIBERN8TIME), + &attr->pa_hibern8time); + /* + * not wait for HIBERN8 time to exit hibernation + */ + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), 0); + + if (attr->pa_granularity < 1 || attr->pa_granularity > 6) { + /* Valid range for granularity: 1 ~ 6 */ + dev_warn(hba->dev, + "%s: pa_granularity %d is invalid, assuming backwards compatibility\n", + __func__, + attr->pa_granularity); + attr->pa_granularity = 6; + } + } + + phy_calibrate(generic_phy); + + if (ufs->drv_data->post_link) + ufs->drv_data->post_link(ufs); + + return 0; +} + +static int exynos_ufs_parse_dt(struct device *dev, struct exynos_ufs *ufs) +{ + struct device_node *np = dev->of_node; + struct exynos_ufs_uic_attr *attr; + int ret = 0; + + ufs->drv_data = device_get_match_data(dev); + + if (ufs->drv_data && ufs->drv_data->uic_attr) { + attr = ufs->drv_data->uic_attr; + } else { + dev_err(dev, "failed to get uic attributes\n"); + ret = -EINVAL; + goto out; + } + + ufs->sysreg = syscon_regmap_lookup_by_phandle(np, "samsung,sysreg"); + if (IS_ERR(ufs->sysreg)) + ufs->sysreg = NULL; + else { + if (of_property_read_u32_index(np, "samsung,sysreg", 1, + &ufs->iocc_offset)) { + dev_warn(dev, "can't get an offset from sysreg. Set to default value\n"); + ufs->iocc_offset = UFS_SHAREABILITY_OFFSET; + } + } + + ufs->iocc_mask = ufs->drv_data->iocc_mask; + /* + * no 'dma-coherent' property means the descriptors are + * non-cacheable so iocc shareability should be disabled. + */ + if (of_dma_is_coherent(dev->of_node)) + ufs->iocc_val = ufs->iocc_mask; + else + ufs->iocc_val = 0; + + ufs->pclk_avail_min = PCLK_AVAIL_MIN; + ufs->pclk_avail_max = PCLK_AVAIL_MAX; + + attr->rx_adv_fine_gran_sup_en = RX_ADV_FINE_GRAN_SUP_EN; + attr->rx_adv_fine_gran_step = RX_ADV_FINE_GRAN_STEP_VAL; + attr->rx_adv_min_actv_time_cap = RX_ADV_MIN_ACTV_TIME_CAP; + attr->pa_granularity = PA_GRANULARITY_VAL; + attr->pa_tactivate = PA_TACTIVATE_VAL; + attr->pa_hibern8time = PA_HIBERN8TIME_VAL; + +out: + return ret; +} + +static inline void exynos_ufs_priv_init(struct ufs_hba *hba, + struct exynos_ufs *ufs) +{ + ufs->hba = hba; + ufs->opts = ufs->drv_data->opts; + ufs->rx_sel_idx = PA_MAXDATALANES; + if (ufs->opts & EXYNOS_UFS_OPT_BROKEN_RX_SEL_IDX) + ufs->rx_sel_idx = 0; + hba->priv = (void *)ufs; + hba->quirks = ufs->drv_data->quirks; +} + +#ifdef CONFIG_SCSI_UFS_CRYPTO + +/* + * Support for Flash Memory Protector (FMP), which is the inline encryption + * hardware on Exynos and Exynos-based SoCs. The interface to this hardware is + * not compatible with the standard UFS crypto. It requires that encryption be + * configured in the PRDT using a nonstandard extension. + */ + +enum fmp_crypto_algo_mode { + FMP_BYPASS_MODE = 0, + FMP_ALGO_MODE_AES_CBC = 1, + FMP_ALGO_MODE_AES_XTS = 2, +}; +enum fmp_crypto_key_length { + FMP_KEYLEN_256BIT = 1, +}; + +/** + * struct fmp_sg_entry - nonstandard format of PRDT entries when FMP is enabled + * + * @base: The standard PRDT entry, but with nonstandard bitfields in the high + * bits of the 'size' field, i.e. the last 32-bit word. When these + * nonstandard bitfields are zero, the data segment won't be encrypted or + * decrypted. Otherwise they specify the algorithm and key length with + * which the data segment will be encrypted or decrypted. + * @file_iv: The initialization vector (IV) with all bytes reversed + * @file_enckey: The first half of the AES-XTS key with all bytes reserved + * @file_twkey: The second half of the AES-XTS key with all bytes reserved + * @disk_iv: Unused + * @reserved: Unused + */ +struct fmp_sg_entry { + struct ufshcd_sg_entry base; + __be64 file_iv[2]; + __be64 file_enckey[4]; + __be64 file_twkey[4]; + __be64 disk_iv[2]; + __be64 reserved[2]; +}; + +#define SMC_CMD_FMP_SECURITY \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, ARM_SMCCC_SMC_64, \ + ARM_SMCCC_OWNER_SIP, 0x1810) +#define SMC_CMD_SMU \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, ARM_SMCCC_SMC_64, \ + ARM_SMCCC_OWNER_SIP, 0x1850) +#define SMC_CMD_FMP_SMU_RESUME \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, ARM_SMCCC_SMC_64, \ + ARM_SMCCC_OWNER_SIP, 0x1860) +#define SMU_EMBEDDED 0 +#define SMU_INIT 0 +#define CFG_DESCTYPE_3 3 + +static void exynos_ufs_fmp_init(struct ufs_hba *hba, struct exynos_ufs *ufs) +{ + struct blk_crypto_profile *profile = &hba->crypto_profile; + struct arm_smccc_res res; + int err; + + /* + * Check for the standard crypto support bit, since it's available even + * though the rest of the interface to FMP is nonstandard. + * + * This check should have the effect of preventing the driver from + * trying to use FMP on old Exynos SoCs that don't have FMP. + */ + if (!(ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES) & + MASK_CRYPTO_SUPPORT)) + return; + + /* + * The below sequence of SMC calls to enable FMP can be found in the + * downstream driver source for gs101 and other Exynos-based SoCs. It + * is the only way to enable FMP that works on SoCs such as gs101 that + * don't make the FMP registers accessible to Linux. It probably works + * on other Exynos-based SoCs too, and might even still be the only way + * that works. But this hasn't been properly tested, and this code is + * mutually exclusive with exynos_ufs_config_smu(). So for now only + * enable FMP support on SoCs with EXYNOS_UFS_OPT_UFSPR_SECURE. + */ + if (!(ufs->opts & EXYNOS_UFS_OPT_UFSPR_SECURE)) + return; + + /* + * This call (which sets DESCTYPE to 0x3 in the FMPSECURITY0 register) + * is needed to make the hardware use the larger PRDT entry size. + */ + BUILD_BUG_ON(sizeof(struct fmp_sg_entry) != 128); + arm_smccc_smc(SMC_CMD_FMP_SECURITY, 0, SMU_EMBEDDED, CFG_DESCTYPE_3, + 0, 0, 0, 0, &res); + if (res.a0) { + dev_warn(hba->dev, + "SMC_CMD_FMP_SECURITY failed on init: %ld. Disabling FMP support.\n", + res.a0); + return; + } + ufshcd_set_sg_entry_size(hba, sizeof(struct fmp_sg_entry)); + + /* + * This is needed to initialize FMP. Without it, errors occur when + * inline encryption is used. + */ + arm_smccc_smc(SMC_CMD_SMU, SMU_INIT, SMU_EMBEDDED, 0, 0, 0, 0, 0, &res); + if (res.a0) { + dev_err(hba->dev, + "SMC_CMD_SMU(SMU_INIT) failed: %ld. Disabling FMP support.\n", + res.a0); + return; + } + + /* Advertise crypto capabilities to the block layer. */ + err = devm_blk_crypto_profile_init(hba->dev, profile, 0); + if (err) { + /* Only ENOMEM should be possible here. */ + dev_err(hba->dev, "Failed to initialize crypto profile: %d\n", + err); + return; + } + profile->max_dun_bytes_supported = AES_BLOCK_SIZE; + profile->key_types_supported = BLK_CRYPTO_KEY_TYPE_RAW; + profile->dev = hba->dev; + profile->modes_supported[BLK_ENCRYPTION_MODE_AES_256_XTS] = + DATA_UNIT_SIZE; + + /* Advertise crypto support to ufshcd-core. */ + hba->caps |= UFSHCD_CAP_CRYPTO; + + /* Advertise crypto quirks to ufshcd-core. */ + hba->quirks |= UFSHCD_QUIRK_CUSTOM_CRYPTO_PROFILE | + UFSHCD_QUIRK_BROKEN_CRYPTO_ENABLE | + UFSHCD_QUIRK_KEYS_IN_PRDT; + +} + +static void exynos_ufs_fmp_resume(struct ufs_hba *hba) +{ + struct arm_smccc_res res; + + if (!(hba->caps & UFSHCD_CAP_CRYPTO)) + return; + + arm_smccc_smc(SMC_CMD_FMP_SECURITY, 0, SMU_EMBEDDED, CFG_DESCTYPE_3, + 0, 0, 0, 0, &res); + if (res.a0) + dev_err(hba->dev, + "SMC_CMD_FMP_SECURITY failed on resume: %ld\n", res.a0); + + arm_smccc_smc(SMC_CMD_FMP_SMU_RESUME, 0, SMU_EMBEDDED, 0, 0, 0, 0, 0, + &res); + if (res.a0) + dev_err(hba->dev, + "SMC_CMD_FMP_SMU_RESUME failed: %ld\n", res.a0); +} + +static inline __be64 fmp_key_word(const u8 *key, int j) +{ + return cpu_to_be64(get_unaligned_le64( + key + AES_KEYSIZE_256 - (j + 1) * sizeof(u64))); +} + +/* Fill the PRDT for a request according to the given encryption context. */ +static int exynos_ufs_fmp_fill_prdt(struct ufs_hba *hba, + const struct bio_crypt_ctx *crypt_ctx, + void *prdt, unsigned int num_segments) +{ + struct fmp_sg_entry *fmp_prdt = prdt; + const u8 *enckey = crypt_ctx->bc_key->bytes; + const u8 *twkey = enckey + AES_KEYSIZE_256; + u64 dun_lo = crypt_ctx->bc_dun[0]; + u64 dun_hi = crypt_ctx->bc_dun[1]; + unsigned int i; + + /* If FMP wasn't enabled, we shouldn't get any encrypted requests. */ + if (WARN_ON_ONCE(!(hba->caps & UFSHCD_CAP_CRYPTO))) + return -EIO; + + /* Configure FMP on each segment of the request. */ + for (i = 0; i < num_segments; i++) { + struct fmp_sg_entry *prd = &fmp_prdt[i]; + int j; + + /* Each segment must be exactly one data unit. */ + if (prd->base.size != cpu_to_le32(DATA_UNIT_SIZE - 1)) { + dev_err(hba->dev, + "data segment is misaligned for FMP\n"); + return -EIO; + } + + /* Set the algorithm and key length. */ + prd->base.size |= cpu_to_le32((FMP_ALGO_MODE_AES_XTS << 28) | + (FMP_KEYLEN_256BIT << 26)); + + /* Set the IV. */ + prd->file_iv[0] = cpu_to_be64(dun_hi); + prd->file_iv[1] = cpu_to_be64(dun_lo); + + /* Set the key. */ + for (j = 0; j < AES_KEYSIZE_256 / sizeof(u64); j++) { + prd->file_enckey[j] = fmp_key_word(enckey, j); + prd->file_twkey[j] = fmp_key_word(twkey, j); + } + + /* Increment the data unit number. */ + dun_lo++; + if (dun_lo == 0) + dun_hi++; + } + return 0; +} + +#else /* CONFIG_SCSI_UFS_CRYPTO */ + +static void exynos_ufs_fmp_init(struct ufs_hba *hba, struct exynos_ufs *ufs) +{ +} + +static void exynos_ufs_fmp_resume(struct ufs_hba *hba) +{ +} + +#define exynos_ufs_fmp_fill_prdt NULL + +#endif /* !CONFIG_SCSI_UFS_CRYPTO */ + +static int exynos_ufs_init(struct ufs_hba *hba) +{ + struct device *dev = hba->dev; + struct platform_device *pdev = to_platform_device(dev); + struct exynos_ufs *ufs; + int ret; + + ufs = devm_kzalloc(dev, sizeof(*ufs), GFP_KERNEL); + if (!ufs) + return -ENOMEM; + + /* exynos-specific hci */ + ufs->reg_hci = devm_platform_ioremap_resource_byname(pdev, "vs_hci"); + if (IS_ERR(ufs->reg_hci)) { + dev_err(dev, "cannot ioremap for hci vendor register\n"); + return PTR_ERR(ufs->reg_hci); + } + + /* unipro */ + ufs->reg_unipro = devm_platform_ioremap_resource_byname(pdev, "unipro"); + if (IS_ERR(ufs->reg_unipro)) { + dev_err(dev, "cannot ioremap for unipro register\n"); + return PTR_ERR(ufs->reg_unipro); + } + + /* ufs protector */ + ufs->reg_ufsp = devm_platform_ioremap_resource_byname(pdev, "ufsp"); + if (IS_ERR(ufs->reg_ufsp)) { + dev_err(dev, "cannot ioremap for ufs protector register\n"); + return PTR_ERR(ufs->reg_ufsp); + } + + ret = exynos_ufs_parse_dt(dev, ufs); + if (ret) { + dev_err(dev, "failed to get dt info.\n"); + goto out; + } + + ufs->phy = devm_phy_get(dev, "ufs-phy"); + if (IS_ERR(ufs->phy)) { + ret = PTR_ERR(ufs->phy); + dev_err(dev, "failed to get ufs-phy\n"); + goto out; + } + + exynos_ufs_priv_init(hba, ufs); + + exynos_ufs_fmp_init(hba, ufs); + + if (ufs->drv_data->drv_init) { + ret = ufs->drv_data->drv_init(ufs); + if (ret) { + dev_err(dev, "failed to init drv-data\n"); + goto out; + } + } + + ret = exynos_ufs_get_clk_info(ufs); + if (ret) + goto out; + exynos_ufs_specify_phy_time_attr(ufs); + + exynos_ufs_config_smu(ufs); + + hba->host->dma_alignment = DATA_UNIT_SIZE - 1; + return 0; + +out: + hba->priv = NULL; + return ret; +} + +static void exynos_ufs_exit(struct ufs_hba *hba) +{ + struct exynos_ufs *ufs = ufshcd_get_variant(hba); + + phy_power_off(ufs->phy); + phy_exit(ufs->phy); +} + +static int exynos_ufs_host_reset(struct ufs_hba *hba) +{ + struct exynos_ufs *ufs = ufshcd_get_variant(hba); + unsigned long timeout = jiffies + msecs_to_jiffies(1); + u32 val; + int ret = 0; + + exynos_ufs_disable_auto_ctrl_hcc_save(ufs, &val); + + hci_writel(ufs, UFS_SW_RST_MASK, HCI_SW_RST); + + do { + if (!(hci_readl(ufs, HCI_SW_RST) & UFS_SW_RST_MASK)) + goto out; + } while (time_before(jiffies, timeout)); + + dev_err(hba->dev, "timeout host sw-reset\n"); + ret = -ETIMEDOUT; + +out: + exynos_ufs_auto_ctrl_hcc_restore(ufs, &val); + return ret; +} + +static void exynos_ufs_dev_hw_reset(struct ufs_hba *hba) +{ + struct exynos_ufs *ufs = ufshcd_get_variant(hba); + + hci_writel(ufs, 0 << 0, HCI_GPIO_OUT); + udelay(5); + hci_writel(ufs, 1 << 0, HCI_GPIO_OUT); +} + +static void exynos_ufs_pre_hibern8(struct ufs_hba *hba, enum uic_cmd_dme cmd) +{ + struct exynos_ufs *ufs = ufshcd_get_variant(hba); + struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr; + static const union phy_notify phystate = { + .ufs_state = PHY_UFS_HIBERN8_EXIT + }; + + if (cmd == UIC_CMD_DME_HIBER_EXIT) { + if (ufs->opts & EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL) + exynos_ufs_disable_auto_ctrl_hcc(ufs); + exynos_ufs_ungate_clks(ufs); + + phy_notify_state(ufs->phy, phystate); + + if (ufs->opts & EXYNOS_UFS_OPT_USE_SW_HIBERN8_TIMER) { + static const unsigned int granularity_tbl[] = { + 1, 4, 8, 16, 32, 100 + }; + int h8_time = attr->pa_hibern8time * + granularity_tbl[attr->pa_granularity - 1]; + unsigned long us; + s64 delta; + + do { + delta = h8_time - ktime_us_delta(ktime_get(), + ufs->entry_hibern8_t); + if (delta <= 0) + break; + + us = min_t(s64, delta, USEC_PER_MSEC); + if (us >= 10) + usleep_range(us, us + 10); + } while (1); + } + } +} + +static void exynos_ufs_post_hibern8(struct ufs_hba *hba, enum uic_cmd_dme cmd) +{ + struct exynos_ufs *ufs = ufshcd_get_variant(hba); + static const union phy_notify phystate = { + .ufs_state = PHY_UFS_HIBERN8_ENTER + }; + + if (cmd == UIC_CMD_DME_HIBER_ENTER) { + ufs->entry_hibern8_t = ktime_get(); + exynos_ufs_gate_clks(ufs); + if (ufs->opts & EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL) + exynos_ufs_enable_auto_ctrl_hcc(ufs); + + phy_notify_state(ufs->phy, phystate); + } +} + +static int exynos_ufs_hce_enable_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status) +{ + struct exynos_ufs *ufs = ufshcd_get_variant(hba); + int ret = 0; + + switch (status) { + case PRE_CHANGE: + /* + * The maximum segment size must be set after scsi_host_alloc() + * has been called and before LUN scanning starts + * (ufshcd_async_scan()). Note: this callback may also be called + * from other functions than ufshcd_init(). + */ + hba->host->max_segment_size = DATA_UNIT_SIZE; + + if (ufs->drv_data->pre_hce_enable) { + ret = ufs->drv_data->pre_hce_enable(ufs); + if (ret) + return ret; + } + + ret = exynos_ufs_host_reset(hba); + if (ret) + return ret; + exynos_ufs_dev_hw_reset(hba); + break; + case POST_CHANGE: + exynos_ufs_calc_pwm_clk_div(ufs); + if (!(ufs->opts & EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL)) + exynos_ufs_enable_auto_ctrl_hcc(ufs); + + if (ufs->drv_data->post_hce_enable) + ret = ufs->drv_data->post_hce_enable(ufs); + + break; + } + + return ret; +} + +static int exynos_ufs_link_startup_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status) +{ + int ret = 0; + + switch (status) { + case PRE_CHANGE: + ret = exynos_ufs_pre_link(hba); + break; + case POST_CHANGE: + ret = exynos_ufs_post_link(hba); + break; + } + + return ret; +} + +static int exynos_ufs_negotiate_pwr_mode(struct ufs_hba *hba, + const struct ufs_pa_layer_attr *dev_max_params, + struct ufs_pa_layer_attr *dev_req_params) +{ + struct ufs_host_params host_params; + + ufshcd_init_host_params(&host_params); + + /* This driver only support symmetric gear setting e.g. hs_tx_gear == hs_rx_gear */ + host_params.hs_tx_gear = exynos_ufs_get_hs_gear(hba); + host_params.hs_rx_gear = exynos_ufs_get_hs_gear(hba); + + return ufshcd_negotiate_pwr_params(&host_params, dev_max_params, dev_req_params); +} + +static int exynos_ufs_pwr_change_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status, + struct ufs_pa_layer_attr *dev_req_params) +{ + int ret = 0; + + switch (status) { + case PRE_CHANGE: + ret = exynos_ufs_pre_pwr_mode(hba, dev_req_params); + break; + case POST_CHANGE: + ret = exynos_ufs_post_pwr_mode(hba, dev_req_params); + break; + } + + return ret; +} + +static void exynos_ufs_hibern8_notify(struct ufs_hba *hba, + enum uic_cmd_dme cmd, + enum ufs_notify_change_status notify) +{ + switch ((u8)notify) { + case PRE_CHANGE: + exynos_ufs_pre_hibern8(hba, cmd); + break; + case POST_CHANGE: + exynos_ufs_post_hibern8(hba, cmd); + break; + } +} + +static int gs101_ufs_suspend(struct exynos_ufs *ufs) +{ + hci_writel(ufs, 0 << 0, HCI_GPIO_OUT); + return 0; +} + +static int exynos_ufs_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op, + enum ufs_notify_change_status status) +{ + struct exynos_ufs *ufs = ufshcd_get_variant(hba); + + if (status == PRE_CHANGE) + return 0; + + if (ufs->drv_data->suspend) + ufs->drv_data->suspend(ufs); + + if (!ufshcd_is_link_active(hba)) + phy_power_off(ufs->phy); + + return 0; +} + +static int exynos_ufs_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) +{ + struct exynos_ufs *ufs = ufshcd_get_variant(hba); + + if (!ufshcd_is_link_active(hba)) + phy_power_on(ufs->phy); + + exynos_ufs_config_smu(ufs); + exynos_ufs_fmp_resume(hba); + return 0; +} + +static int exynosauto_ufs_vh_link_startup_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status) +{ + if (status == POST_CHANGE) { + ufshcd_set_link_active(hba); + ufshcd_set_ufs_dev_active(hba); + } + + return 0; +} + +static int exynosauto_ufs_vh_wait_ph_ready(struct ufs_hba *hba) +{ + u32 mbox; + ktime_t start, stop; + + start = ktime_get(); + stop = ktime_add(start, ms_to_ktime(PH_READY_TIMEOUT_MS)); + + do { + mbox = ufshcd_readl(hba, PH2VH_MBOX); + /* TODO: Mailbox message protocols between the PH and VHs are + * not implemented yet. This will be supported later + */ + if ((mbox & MH_MSG_MASK) == MH_MSG_PH_READY) + return 0; + + usleep_range(40, 50); + } while (ktime_before(ktime_get(), stop)); + + return -ETIME; +} + +static int exynosauto_ufs_vh_init(struct ufs_hba *hba) +{ + struct device *dev = hba->dev; + struct platform_device *pdev = to_platform_device(dev); + struct exynos_ufs *ufs; + int ret; + + ufs = devm_kzalloc(dev, sizeof(*ufs), GFP_KERNEL); + if (!ufs) + return -ENOMEM; + + /* exynos-specific hci */ + ufs->reg_hci = devm_platform_ioremap_resource_byname(pdev, "vs_hci"); + if (IS_ERR(ufs->reg_hci)) { + dev_err(dev, "cannot ioremap for hci vendor register\n"); + return PTR_ERR(ufs->reg_hci); + } + + ret = exynosauto_ufs_vh_wait_ph_ready(hba); + if (ret) + return ret; + + ufs->drv_data = device_get_match_data(dev); + if (!ufs->drv_data) + return -ENODEV; + + exynos_ufs_priv_init(hba, ufs); + + return 0; +} + +static int fsd_ufs_pre_link(struct exynos_ufs *ufs) +{ + struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr; + struct ufs_hba *hba = ufs->hba; + int i; + + ufshcd_dme_set(hba, UIC_ARG_MIB(attr->pa_dbg_clk_period_off), + DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate)); + ufshcd_dme_set(hba, UIC_ARG_MIB(0x201), 0x12); + ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x40); + + for_each_ufs_tx_lane(ufs, i) { + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0xAA, i), + DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x8F, i), 0x3F); + } + + for_each_ufs_rx_lane(ufs, i) { + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x12, i), + DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x5C, i), 0x38); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x0F, i), 0x0); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x65, i), 0x1); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x69, i), 0x1); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x21, i), 0x0); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x22, i), 0x0); + } + + ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x0); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_AUTOMODE_THLD), 0x4E20); + + ufshcd_dme_set(hba, UIC_ARG_MIB(attr->pa_dbg_opt_suite1_off), + 0x2e820183); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_LOCAL_TX_LCC_ENABLE), 0x0); + + exynos_ufs_establish_connt(ufs); + + return 0; +} + +static int fsd_ufs_post_link(struct exynos_ufs *ufs) +{ + int i; + struct ufs_hba *hba = ufs->hba; + u32 hw_cap_min_tactivate; + u32 peer_rx_min_actv_time_cap; + u32 max_rx_hibern8_time_cap; + + ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(0x8F, 4), + &hw_cap_min_tactivate); /* HW Capability of MIN_TACTIVATE */ + ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), + &peer_rx_min_actv_time_cap); /* PA_TActivate */ + ufshcd_dme_get(hba, UIC_ARG_MIB(PA_HIBERN8TIME), + &max_rx_hibern8_time_cap); /* PA_Hibern8Time */ + + if (peer_rx_min_actv_time_cap >= hw_cap_min_tactivate) + ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE), + peer_rx_min_actv_time_cap + 1); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), max_rx_hibern8_time_cap + 1); + + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_MODE), 0x01); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_SAVECONFIGTIME), 0xFA); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_MODE), 0x00); + + ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x40); + + for_each_ufs_rx_lane(ufs, i) { + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x35, i), 0x05); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x73, i), 0x01); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x41, i), 0x02); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x42, i), 0xAC); + } + + ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x0); + + return 0; +} + +static int fsd_ufs_pre_pwr_change(struct exynos_ufs *ufs, + struct ufs_pa_layer_attr *pwr) +{ + struct ufs_hba *hba = ufs->hba; + + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), 0x1); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), 0x1); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0), 12000); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1), 32000); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2), 16000); + + unipro_writel(ufs, 12000, UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER0); + unipro_writel(ufs, 32000, UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER1); + unipro_writel(ufs, 16000, UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER2); + + return 0; +} + +static int fsd_ufs_suspend(struct exynos_ufs *ufs) +{ + exynos_ufs_gate_clks(ufs); + hci_writel(ufs, 0, HCI_GPIO_OUT); + return 0; +} + +static inline u32 get_mclk_period_unipro_18(struct exynos_ufs *ufs) +{ + return (16 * 1000 * 1000000UL / ufs->mclk_rate); +} + +static int gs101_ufs_pre_link(struct exynos_ufs *ufs) +{ + struct ufs_hba *hba = ufs->hba; + int i; + u32 tx_line_reset_period, rx_line_reset_period; + + rx_line_reset_period = (RX_LINE_RESET_TIME * ufs->mclk_rate) + / NSEC_PER_MSEC; + tx_line_reset_period = (TX_LINE_RESET_TIME * ufs->mclk_rate) + / NSEC_PER_MSEC; + + unipro_writel(ufs, get_mclk_period_unipro_18(ufs), COMP_CLK_PERIOD); + + ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x40); + + for_each_ufs_rx_lane(ufs, i) { + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_CLK_PRD, i), + DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_CLK_PRD_EN, i), 0x0); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_VALUE2, i), + (rx_line_reset_period >> 16) & 0xFF); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_VALUE1, i), + (rx_line_reset_period >> 8) & 0xFF); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_VALUE0, i), + (rx_line_reset_period) & 0xFF); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x2f, i), 0x69); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x84, i), 0x1); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x25, i), 0xf6); + } + + for_each_ufs_tx_lane(ufs, i) { + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_CLK_PRD, i), + DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate)); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_CLK_PRD_EN, i), + 0x02); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE2, i), + (tx_line_reset_period >> 16) & 0xFF); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE1, i), + (tx_line_reset_period >> 8) & 0xFF); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE0, i), + (tx_line_reset_period) & 0xFF); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x04, i), 1); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x7F, i), 0); + } + + ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x0); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_LOCAL_TX_LCC_ENABLE), 0x0); + ufshcd_dme_set(hba, UIC_ARG_MIB(N_DEVICEID), 0x0); + ufshcd_dme_set(hba, UIC_ARG_MIB(N_DEVICEID_VALID), 0x1); + ufshcd_dme_set(hba, UIC_ARG_MIB(T_PEERDEVICEID), 0x1); + ufshcd_dme_set(hba, UIC_ARG_MIB(T_CONNECTIONSTATE), CPORT_CONNECTED); + ufshcd_dme_set(hba, UIC_ARG_MIB(0xA006), 0x8000); + + return 0; +} + +static int gs101_ufs_post_link(struct exynos_ufs *ufs) +{ + struct ufs_hba *hba = ufs->hba; + + /* + * Enable Write Line Unique. This field has to be 0x3 + * to support Write Line Unique transaction on gs101. + */ + hci_writel(ufs, WLU_EN | WLU_BURST_LEN(3), HCI_AXIDMA_RWDATA_BURST_LEN); + + exynos_ufs_enable_dbg_mode(hba); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_SAVECONFIGTIME), 0x3e8); + exynos_ufs_disable_dbg_mode(hba); + + return 0; +} + +static int gs101_ufs_pre_pwr_change(struct exynos_ufs *ufs, + struct ufs_pa_layer_attr *pwr) +{ + struct ufs_hba *hba = ufs->hba; + + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0), 12000); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1), 32000); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2), 16000); + unipro_writel(ufs, 8064, UNIPRO_DME_POWERMODE_REQ_LOCALL2TIMER0); + unipro_writel(ufs, 28224, UNIPRO_DME_POWERMODE_REQ_LOCALL2TIMER1); + unipro_writel(ufs, 20160, UNIPRO_DME_POWERMODE_REQ_LOCALL2TIMER2); + unipro_writel(ufs, 12000, UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER0); + unipro_writel(ufs, 32000, UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER1); + unipro_writel(ufs, 16000, UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER2); + + return 0; +} + +static const struct ufs_hba_variant_ops ufs_hba_exynos_ops = { + .name = "exynos_ufs", + .init = exynos_ufs_init, + .exit = exynos_ufs_exit, + .hce_enable_notify = exynos_ufs_hce_enable_notify, + .link_startup_notify = exynos_ufs_link_startup_notify, + .negotiate_pwr_mode = exynos_ufs_negotiate_pwr_mode, + .pwr_change_notify = exynos_ufs_pwr_change_notify, + .setup_clocks = exynos_ufs_setup_clocks, + .setup_xfer_req = exynos_ufs_specify_nexus_t_xfer_req, + .setup_task_mgmt = exynos_ufs_specify_nexus_t_tm_req, + .hibern8_notify = exynos_ufs_hibern8_notify, + .suspend = exynos_ufs_suspend, + .resume = exynos_ufs_resume, + .fill_crypto_prdt = exynos_ufs_fmp_fill_prdt, +}; + +static struct ufs_hba_variant_ops ufs_hba_exynosauto_vh_ops = { + .name = "exynosauto_ufs_vh", + .init = exynosauto_ufs_vh_init, + .link_startup_notify = exynosauto_ufs_vh_link_startup_notify, +}; + +static int exynos_ufs_probe(struct platform_device *pdev) +{ + int err; + struct device *dev = &pdev->dev; + const struct ufs_hba_variant_ops *vops = &ufs_hba_exynos_ops; + const struct exynos_ufs_drv_data *drv_data = + device_get_match_data(dev); + + if (drv_data && drv_data->vops) + vops = drv_data->vops; + + err = ufshcd_pltfrm_init(pdev, vops); + if (err) + dev_err(dev, "ufshcd_pltfrm_init() failed %d\n", err); + + return err; +} + +static void exynos_ufs_remove(struct platform_device *pdev) +{ + ufshcd_pltfrm_remove(pdev); +} + +static struct exynos_ufs_uic_attr exynos7_uic_attr = { + .tx_trailingclks = 0x10, + .tx_dif_p_nsec = 3000000, /* unit: ns */ + .tx_dif_n_nsec = 1000000, /* unit: ns */ + .tx_high_z_cnt_nsec = 20000, /* unit: ns */ + .tx_base_unit_nsec = 100000, /* unit: ns */ + .tx_gran_unit_nsec = 4000, /* unit: ns */ + .tx_sleep_cnt = 1000, /* unit: ns */ + .tx_min_activatetime = 0xa, + .rx_filler_enable = 0x2, + .rx_dif_p_nsec = 1000000, /* unit: ns */ + .rx_hibern8_wait_nsec = 4000000, /* unit: ns */ + .rx_base_unit_nsec = 100000, /* unit: ns */ + .rx_gran_unit_nsec = 4000, /* unit: ns */ + .rx_sleep_cnt = 1280, /* unit: ns */ + .rx_stall_cnt = 320, /* unit: ns */ + .rx_hs_g1_sync_len_cap = SYNC_LEN_COARSE(0xf), + .rx_hs_g2_sync_len_cap = SYNC_LEN_COARSE(0xf), + .rx_hs_g3_sync_len_cap = SYNC_LEN_COARSE(0xf), + .rx_hs_g1_prep_sync_len_cap = PREP_LEN(0xf), + .rx_hs_g2_prep_sync_len_cap = PREP_LEN(0xf), + .rx_hs_g3_prep_sync_len_cap = PREP_LEN(0xf), + .pa_dbg_clk_period_off = PA_DBG_CLK_PERIOD, + .pa_dbg_opt_suite1_val = 0x30103, + .pa_dbg_opt_suite1_off = PA_DBG_OPTION_SUITE, +}; + +static const struct exynos_ufs_drv_data exynosauto_ufs_drvs = { + .uic_attr = &exynos7_uic_attr, + .quirks = UFSHCD_QUIRK_PRDT_BYTE_GRAN | + UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR | + UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR | + UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING, + .opts = EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL | + EXYNOS_UFS_OPT_SKIP_CONFIG_PHY_ATTR | + EXYNOS_UFS_OPT_BROKEN_RX_SEL_IDX, + .iocc_mask = UFS_EXYNOSAUTO_SHARABLE, + .drv_init = exynosauto_ufs_drv_init, + .post_hce_enable = exynosauto_ufs_post_hce_enable, + .pre_link = exynosauto_ufs_pre_link, + .pre_pwr_change = exynosauto_ufs_pre_pwr_change, + .post_pwr_change = exynosauto_ufs_post_pwr_change, +}; + +static const struct exynos_ufs_drv_data exynosauto_ufs_vh_drvs = { + .vops = &ufs_hba_exynosauto_vh_ops, + .quirks = UFSHCD_QUIRK_PRDT_BYTE_GRAN | + UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR | + UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR | + UFSHCI_QUIRK_BROKEN_HCE | + UFSHCD_QUIRK_BROKEN_UIC_CMD | + UFSHCD_QUIRK_SKIP_PH_CONFIGURATION | + UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING, + .opts = EXYNOS_UFS_OPT_BROKEN_RX_SEL_IDX, +}; + +static const struct exynos_ufs_drv_data exynos_ufs_drvs = { + .uic_attr = &exynos7_uic_attr, + .quirks = UFSHCD_QUIRK_PRDT_BYTE_GRAN | + UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR | + UFSHCI_QUIRK_BROKEN_HCE | + UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR | + UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR | + UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL | + UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING, + .opts = EXYNOS_UFS_OPT_HAS_APB_CLK_CTRL | + EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL | + EXYNOS_UFS_OPT_BROKEN_RX_SEL_IDX | + EXYNOS_UFS_OPT_SKIP_CONNECTION_ESTAB | + EXYNOS_UFS_OPT_USE_SW_HIBERN8_TIMER, + .pre_link = exynos7_ufs_pre_link, + .post_link = exynos7_ufs_post_link, + .pre_pwr_change = exynos7_ufs_pre_pwr_change, + .post_pwr_change = exynos7_ufs_post_pwr_change, +}; + +static struct exynos_ufs_uic_attr gs101_uic_attr = { + .tx_trailingclks = 0xff, + .pa_dbg_opt_suite1_val = 0x90913C1C, + .pa_dbg_opt_suite1_off = PA_GS101_DBG_OPTION_SUITE1, + .pa_dbg_opt_suite2_val = 0xE01C115F, + .pa_dbg_opt_suite2_off = PA_GS101_DBG_OPTION_SUITE2, +}; + +static struct exynos_ufs_uic_attr fsd_uic_attr = { + .tx_trailingclks = 0x10, + .tx_dif_p_nsec = 3000000, /* unit: ns */ + .tx_dif_n_nsec = 1000000, /* unit: ns */ + .tx_high_z_cnt_nsec = 20000, /* unit: ns */ + .tx_base_unit_nsec = 100000, /* unit: ns */ + .tx_gran_unit_nsec = 4000, /* unit: ns */ + .tx_sleep_cnt = 1000, /* unit: ns */ + .tx_min_activatetime = 0xa, + .rx_filler_enable = 0x2, + .rx_dif_p_nsec = 1000000, /* unit: ns */ + .rx_hibern8_wait_nsec = 4000000, /* unit: ns */ + .rx_base_unit_nsec = 100000, /* unit: ns */ + .rx_gran_unit_nsec = 4000, /* unit: ns */ + .rx_sleep_cnt = 1280, /* unit: ns */ + .rx_stall_cnt = 320, /* unit: ns */ + .rx_hs_g1_sync_len_cap = SYNC_LEN_COARSE(0xf), + .rx_hs_g2_sync_len_cap = SYNC_LEN_COARSE(0xf), + .rx_hs_g3_sync_len_cap = SYNC_LEN_COARSE(0xf), + .rx_hs_g1_prep_sync_len_cap = PREP_LEN(0xf), + .rx_hs_g2_prep_sync_len_cap = PREP_LEN(0xf), + .rx_hs_g3_prep_sync_len_cap = PREP_LEN(0xf), + .pa_dbg_clk_period_off = PA_DBG_CLK_PERIOD, + .pa_dbg_opt_suite1_val = 0x2E820183, + .pa_dbg_opt_suite1_off = PA_DBG_OPTION_SUITE, +}; + +static const struct exynos_ufs_drv_data fsd_ufs_drvs = { + .uic_attr = &fsd_uic_attr, + .quirks = UFSHCD_QUIRK_PRDT_BYTE_GRAN | + UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR | + UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR | + UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING | + UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR, + .opts = EXYNOS_UFS_OPT_HAS_APB_CLK_CTRL | + EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL | + EXYNOS_UFS_OPT_SKIP_CONFIG_PHY_ATTR | + EXYNOS_UFS_OPT_BROKEN_RX_SEL_IDX, + .pre_link = fsd_ufs_pre_link, + .post_link = fsd_ufs_post_link, + .pre_pwr_change = fsd_ufs_pre_pwr_change, + .suspend = fsd_ufs_suspend, +}; + +static const struct exynos_ufs_drv_data gs101_ufs_drvs = { + .uic_attr = &gs101_uic_attr, + .quirks = UFSHCD_QUIRK_PRDT_BYTE_GRAN | + UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR | + UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR | + UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR | + UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL | + UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING, + .opts = EXYNOS_UFS_OPT_SKIP_CONFIG_PHY_ATTR | + EXYNOS_UFS_OPT_UFSPR_SECURE | + EXYNOS_UFS_OPT_TIMER_TICK_SELECT, + .iocc_mask = UFS_GS101_SHARABLE, + .drv_init = gs101_ufs_drv_init, + .pre_link = gs101_ufs_pre_link, + .post_link = gs101_ufs_post_link, + .pre_pwr_change = gs101_ufs_pre_pwr_change, + .suspend = gs101_ufs_suspend, +}; + +static const struct exynos_ufs_drv_data exynosautov920_ufs_drvs = { + .uic_attr = &exynos7_uic_attr, + .quirks = UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING, + .opts = EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL | + EXYNOS_UFS_OPT_SKIP_CONFIG_PHY_ATTR | + EXYNOS_UFS_OPT_BROKEN_RX_SEL_IDX | + EXYNOS_UFS_OPT_TIMER_TICK_SELECT, + .iocc_mask = UFS_EXYNOSAUTOV920_SHARABLE, + .drv_init = exynosauto_ufs_drv_init, + .post_hce_enable = exynosauto_ufs_post_hce_enable, + .pre_link = exynosautov920_ufs_pre_link, + .post_link = exynosautov920_ufs_post_link, + .pre_pwr_change = exynosautov920_ufs_pre_pwr_change, +}; + +static const struct of_device_id exynos_ufs_of_match[] = { + { .compatible = "google,gs101-ufs", + .data = &gs101_ufs_drvs }, + { .compatible = "samsung,exynos7-ufs", + .data = &exynos_ufs_drvs }, + { .compatible = "samsung,exynosautov9-ufs", + .data = &exynosauto_ufs_drvs }, + { .compatible = "samsung,exynosautov9-ufs-vh", + .data = &exynosauto_ufs_vh_drvs }, + { .compatible = "samsung,exynosautov920-ufs", + .data = &exynosautov920_ufs_drvs }, + { .compatible = "tesla,fsd-ufs", + .data = &fsd_ufs_drvs }, + {}, +}; +MODULE_DEVICE_TABLE(of, exynos_ufs_of_match); + +static const struct dev_pm_ops exynos_ufs_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(ufshcd_system_suspend, ufshcd_system_resume) + SET_RUNTIME_PM_OPS(ufshcd_runtime_suspend, ufshcd_runtime_resume, NULL) + .prepare = ufshcd_suspend_prepare, + .complete = ufshcd_resume_complete, +}; + +static struct platform_driver exynos_ufs_pltform = { + .probe = exynos_ufs_probe, + .remove = exynos_ufs_remove, + .driver = { + .name = "exynos-ufshc", + .pm = &exynos_ufs_pm_ops, + .of_match_table = exynos_ufs_of_match, + }, +}; +module_platform_driver(exynos_ufs_pltform); + +MODULE_AUTHOR("Alim Akhtar <alim.akhtar@samsung.com>"); +MODULE_AUTHOR("Seungwon Jeon <essuuj@gmail.com>"); +MODULE_DESCRIPTION("Exynos UFS HCI Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/ufs/host/ufs-exynos.h b/drivers/ufs/host/ufs-exynos.h new file mode 100644 index 000000000..abe7e4727 --- /dev/null +++ b/drivers/ufs/host/ufs-exynos.h @@ -0,0 +1,294 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * UFS Host Controller driver for Exynos specific extensions + * + * Copyright (C) 2014-2015 Samsung Electronics Co., Ltd. + * + */ + +#ifndef _UFS_EXYNOS_H_ +#define _UFS_EXYNOS_H_ + +/* + * Component registers + */ + +#define COMP_CLK_PERIOD 0x44 + +/* + * UNIPRO registers + */ +#define UNIPRO_DBG_FORCE_DME_CTRL_STATE 0x150 + +/* + * MIBs for PA debug registers + */ +#define PA_DBG_CLK_PERIOD 0x9514 +#define PA_DBG_TXPHY_CFGUPDT 0x9518 +#define PA_DBG_RXPHY_CFGUPDT 0x9519 +#define PA_DBG_MODE 0x9529 +#define PA_DBG_SKIP_RESET_PHY 0x9539 +#define PA_DBG_AUTOMODE_THLD 0x9536 +#define PA_DBG_OV_TM 0x9540 +#define PA_DBG_SKIP_LINE_RESET 0x9541 +#define PA_DBG_LINE_RESET_REQ 0x9543 +#define PA_DBG_OPTION_SUITE 0x9564 +#define PA_DBG_OPTION_SUITE_DYN 0x9565 + +/* + * Note: GS101_DBG_OPTION offsets below differ from the TRM + * but match the downstream driver. Following the TRM + * results in non-functioning UFS. + */ +#define PA_GS101_DBG_OPTION_SUITE1 0x956a +#define PA_GS101_DBG_OPTION_SUITE2 0x956d + +/* + * MIBs for Transport Layer debug registers + */ +#define T_DBG_SKIP_INIT_HIBERN8_EXIT 0xc001 + +/* + * Exynos MPHY attributes + */ +#define TX_LINERESET_N_VAL 0x0277 +#define TX_LINERESET_N(v) (((v) >> 10) & 0xFF) +#define TX_LINERESET_P_VAL 0x027D +#define TX_LINERESET_P(v) (((v) >> 12) & 0xFF) +#define TX_OV_SLEEP_CNT_TIMER 0x028E +#define TX_OV_H8_ENTER_EN (1 << 7) +#define TX_OV_SLEEP_CNT(v) (((v) >> 5) & 0x7F) +#define TX_HIGH_Z_CNT_11_08 0x028C +#define TX_HIGH_Z_CNT_H(v) (((v) >> 8) & 0xF) +#define TX_HIGH_Z_CNT_07_00 0x028D +#define TX_HIGH_Z_CNT_L(v) ((v) & 0xFF) +#define TX_BASE_NVAL_07_00 0x0293 +#define TX_BASE_NVAL_L(v) ((v) & 0xFF) +#define TX_BASE_NVAL_15_08 0x0294 +#define TX_BASE_NVAL_H(v) (((v) >> 8) & 0xFF) +#define TX_GRAN_NVAL_07_00 0x0295 +#define TX_GRAN_NVAL_L(v) ((v) & 0xFF) +#define TX_GRAN_NVAL_10_08 0x0296 +#define TX_GRAN_NVAL_H(v) (((v) >> 8) & 0x3) + +#define VND_TX_CLK_PRD 0xAA +#define VND_TX_CLK_PRD_EN 0xA9 +#define VND_TX_LINERESET_PVALUE0 0xAD +#define VND_TX_LINERESET_PVALUE1 0xAC +#define VND_TX_LINERESET_PVALUE2 0xAB + +#define TX_LINE_RESET_TIME 3200 + +#define VND_RX_CLK_PRD 0x12 +#define VND_RX_CLK_PRD_EN 0x11 +#define VND_RX_LINERESET_VALUE0 0x1D +#define VND_RX_LINERESET_VALUE1 0x1C +#define VND_RX_LINERESET_VALUE2 0x1B + +#define RX_LINE_RESET_TIME 1000 + +#define RX_FILLER_ENABLE 0x0316 +#define RX_FILLER_EN (1 << 1) +#define RX_LINERESET_VAL 0x0317 +#define RX_LINERESET(v) (((v) >> 12) & 0xFF) +#define RX_LCC_IGNORE 0x0318 +#define RX_SYNC_MASK_LENGTH 0x0321 +#define RX_HIBERN8_WAIT_VAL_BIT_20_16 0x0331 +#define RX_HIBERN8_WAIT_VAL_BIT_15_08 0x0332 +#define RX_HIBERN8_WAIT_VAL_BIT_07_00 0x0333 +#define RX_OV_SLEEP_CNT_TIMER 0x0340 +#define RX_OV_SLEEP_CNT(v) (((v) >> 6) & 0x1F) +#define RX_OV_STALL_CNT_TIMER 0x0341 +#define RX_OV_STALL_CNT(v) (((v) >> 4) & 0xFF) +#define RX_BASE_NVAL_07_00 0x0355 +#define RX_BASE_NVAL_L(v) ((v) & 0xFF) +#define RX_BASE_NVAL_15_08 0x0354 +#define RX_BASE_NVAL_H(v) (((v) >> 8) & 0xFF) +#define RX_GRAN_NVAL_07_00 0x0353 +#define RX_GRAN_NVAL_L(v) ((v) & 0xFF) +#define RX_GRAN_NVAL_10_08 0x0352 +#define RX_GRAN_NVAL_H(v) (((v) >> 8) & 0x3) + +#define CMN_PWM_CLK_CTRL 0x0402 +#define PWM_CLK_CTRL_MASK 0x3 + +#define IATOVAL_NSEC 20000 /* unit: ns */ +#define UNIPRO_PCLK_PERIOD(ufs) (NSEC_PER_SEC / ufs->pclk_rate) + +struct exynos_ufs; + +/* vendor specific pre-defined parameters */ +#define SLOW 1 +#define FAST 2 + +#define RX_ADV_FINE_GRAN_SUP_EN 0x1 +#define RX_ADV_FINE_GRAN_STEP_VAL 0x3 +#define RX_ADV_MIN_ACTV_TIME_CAP 0x9 + +#define PA_GRANULARITY_VAL 0x6 +#define PA_TACTIVATE_VAL 0x3 +#define PA_HIBERN8TIME_VAL 0x20 + +#define PCLK_AVAIL_MIN 70000000 +#define PCLK_AVAIL_MAX 267000000 + +struct exynos_ufs_uic_attr { + /* TX Attributes */ + unsigned int tx_trailingclks; + unsigned int tx_dif_p_nsec; + unsigned int tx_dif_n_nsec; + unsigned int tx_high_z_cnt_nsec; + unsigned int tx_base_unit_nsec; + unsigned int tx_gran_unit_nsec; + unsigned int tx_sleep_cnt; + unsigned int tx_min_activatetime; + /* RX Attributes */ + unsigned int rx_filler_enable; + unsigned int rx_dif_p_nsec; + unsigned int rx_hibern8_wait_nsec; + unsigned int rx_base_unit_nsec; + unsigned int rx_gran_unit_nsec; + unsigned int rx_sleep_cnt; + unsigned int rx_stall_cnt; + unsigned int rx_hs_g1_sync_len_cap; + unsigned int rx_hs_g2_sync_len_cap; + unsigned int rx_hs_g3_sync_len_cap; + unsigned int rx_hs_g1_prep_sync_len_cap; + unsigned int rx_hs_g2_prep_sync_len_cap; + unsigned int rx_hs_g3_prep_sync_len_cap; + /* Common Attributes */ + unsigned int cmn_pwm_clk_ctrl; + /* Internal Attributes */ + unsigned int pa_dbg_clk_period_off; + unsigned int pa_dbg_opt_suite1_val; + unsigned int pa_dbg_opt_suite1_off; + unsigned int pa_dbg_opt_suite2_val; + unsigned int pa_dbg_opt_suite2_off; + /* Changeable Attributes */ + unsigned int rx_adv_fine_gran_sup_en; + unsigned int rx_adv_fine_gran_step; + unsigned int rx_min_actv_time_cap; + unsigned int rx_hibern8_time_cap; + unsigned int rx_adv_min_actv_time_cap; + unsigned int rx_adv_hibern8_time_cap; + unsigned int pa_granularity; + unsigned int pa_tactivate; + unsigned int pa_hibern8time; +}; + +struct exynos_ufs_drv_data { + const struct ufs_hba_variant_ops *vops; + struct exynos_ufs_uic_attr *uic_attr; + unsigned int quirks; + unsigned int opts; + u32 iocc_mask; + /* SoC's specific operations */ + int (*drv_init)(struct exynos_ufs *ufs); + int (*pre_link)(struct exynos_ufs *ufs); + int (*post_link)(struct exynos_ufs *ufs); + int (*pre_pwr_change)(struct exynos_ufs *ufs, + struct ufs_pa_layer_attr *pwr); + int (*post_pwr_change)(struct exynos_ufs *ufs, + const struct ufs_pa_layer_attr *pwr); + int (*pre_hce_enable)(struct exynos_ufs *ufs); + int (*post_hce_enable)(struct exynos_ufs *ufs); + int (*suspend)(struct exynos_ufs *ufs); +}; + +struct ufs_phy_time_cfg { + u32 tx_linereset_p; + u32 tx_linereset_n; + u32 tx_high_z_cnt; + u32 tx_base_n_val; + u32 tx_gran_n_val; + u32 tx_sleep_cnt; + u32 rx_linereset; + u32 rx_hibern8_wait; + u32 rx_base_n_val; + u32 rx_gran_n_val; + u32 rx_sleep_cnt; + u32 rx_stall_cnt; +}; + +struct exynos_ufs { + struct ufs_hba *hba; + struct phy *phy; + void __iomem *reg_hci; + void __iomem *reg_unipro; + void __iomem *reg_ufsp; + struct clk *clk_hci_core; + struct clk *clk_unipro_main; + struct clk *clk_apb; + u32 pclk_rate; + u32 pclk_div; + u32 pclk_avail_min; + u32 pclk_avail_max; + unsigned long mclk_rate; + int avail_ln_rx; + int avail_ln_tx; + int rx_sel_idx; + struct ufs_pa_layer_attr dev_req_params; + struct ufs_phy_time_cfg t_cfg; + ktime_t entry_hibern8_t; + const struct exynos_ufs_drv_data *drv_data; + struct regmap *sysreg; + u32 iocc_offset; + u32 iocc_mask; + u32 iocc_val; + + u32 opts; +#define EXYNOS_UFS_OPT_HAS_APB_CLK_CTRL BIT(0) +#define EXYNOS_UFS_OPT_SKIP_CONNECTION_ESTAB BIT(1) +#define EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL BIT(2) +#define EXYNOS_UFS_OPT_BROKEN_RX_SEL_IDX BIT(3) +#define EXYNOS_UFS_OPT_USE_SW_HIBERN8_TIMER BIT(4) +#define EXYNOS_UFS_OPT_SKIP_CONFIG_PHY_ATTR BIT(5) +#define EXYNOS_UFS_OPT_UFSPR_SECURE BIT(6) +#define EXYNOS_UFS_OPT_TIMER_TICK_SELECT BIT(7) +}; + +#define for_each_ufs_rx_lane(ufs, i) \ + for (i = (ufs)->rx_sel_idx; \ + i < (ufs)->rx_sel_idx + (ufs)->avail_ln_rx; i++) +#define for_each_ufs_tx_lane(ufs, i) \ + for (i = 0; i < (ufs)->avail_ln_tx; i++) + +#define EXYNOS_UFS_MMIO_FUNC(name) \ +static inline void name##_writel(struct exynos_ufs *ufs, u32 val, u32 reg)\ +{ \ + writel(val, ufs->reg_##name + reg); \ +} \ + \ +static inline u32 name##_readl(struct exynos_ufs *ufs, u32 reg) \ +{ \ + return readl(ufs->reg_##name + reg); \ +} + +EXYNOS_UFS_MMIO_FUNC(hci); +EXYNOS_UFS_MMIO_FUNC(unipro); +EXYNOS_UFS_MMIO_FUNC(ufsp); +#undef EXYNOS_UFS_MMIO_FUNC + +long exynos_ufs_calc_time_cntr(struct exynos_ufs *, long); + +static inline void exynos_ufs_enable_ov_tm(struct ufs_hba *hba) +{ + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_OV_TM), true); +} + +static inline void exynos_ufs_disable_ov_tm(struct ufs_hba *hba) +{ + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_OV_TM), false); +} + +static inline void exynos_ufs_enable_dbg_mode(struct ufs_hba *hba) +{ + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_MODE), true); +} + +static inline void exynos_ufs_disable_dbg_mode(struct ufs_hba *hba) +{ + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_MODE), false); +} + +#endif /* _UFS_EXYNOS_H_ */ diff --git a/drivers/ufs/host/ufs-hisi.c b/drivers/ufs/host/ufs-hisi.c new file mode 100644 index 000000000..993e20ac2 --- /dev/null +++ b/drivers/ufs/host/ufs-hisi.c @@ -0,0 +1,605 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * HiSilicon Hixxxx UFS Driver + * + * Copyright (c) 2016-2017 Linaro Ltd. + * Copyright (c) 2016-2017 HiSilicon Technologies Co., Ltd. + */ + +#include <linux/time.h> +#include <linux/delay.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/dma-mapping.h> +#include <linux/platform_device.h> +#include <linux/reset.h> + +#include <ufs/ufshcd.h> +#include "ufshcd-pltfrm.h" +#include <ufs/unipro.h> +#include "ufs-hisi.h" +#include <ufs/ufshci.h> +#include <ufs/ufs_quirks.h> + +static int ufs_hisi_check_hibern8(struct ufs_hba *hba) +{ + int err = 0; + u32 tx_fsm_val_0 = 0; + u32 tx_fsm_val_1 = 0; + unsigned long timeout = jiffies + msecs_to_jiffies(HBRN8_POLL_TOUT_MS); + + do { + err = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(MPHY_TX_FSM_STATE, 0), + &tx_fsm_val_0); + err |= ufshcd_dme_get(hba, + UIC_ARG_MIB_SEL(MPHY_TX_FSM_STATE, 1), &tx_fsm_val_1); + if (err || (tx_fsm_val_0 == TX_FSM_HIBERN8 && + tx_fsm_val_1 == TX_FSM_HIBERN8)) + break; + + /* sleep for max. 200us */ + usleep_range(100, 200); + } while (time_before(jiffies, timeout)); + + /* + * we might have scheduled out for long during polling so + * check the state again. + */ + if (time_after(jiffies, timeout)) { + err = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(MPHY_TX_FSM_STATE, 0), + &tx_fsm_val_0); + err |= ufshcd_dme_get(hba, + UIC_ARG_MIB_SEL(MPHY_TX_FSM_STATE, 1), &tx_fsm_val_1); + } + + if (err) { + dev_err(hba->dev, "%s: unable to get TX_FSM_STATE, err %d\n", + __func__, err); + } else if (tx_fsm_val_0 != TX_FSM_HIBERN8 || + tx_fsm_val_1 != TX_FSM_HIBERN8) { + err = -1; + dev_err(hba->dev, "%s: invalid TX_FSM_STATE, lane0 = %d, lane1 = %d\n", + __func__, tx_fsm_val_0, tx_fsm_val_1); + } + + return err; +} + +static void ufs_hisi_clk_init(struct ufs_hba *hba) +{ + struct ufs_hisi_host *host = ufshcd_get_variant(hba); + + ufs_sys_ctrl_clr_bits(host, BIT_SYSCTRL_REF_CLOCK_EN, PHY_CLK_CTRL); + if (ufs_sys_ctrl_readl(host, PHY_CLK_CTRL) & BIT_SYSCTRL_REF_CLOCK_EN) + mdelay(1); + /* use abb clk */ + ufs_sys_ctrl_clr_bits(host, BIT_UFS_REFCLK_SRC_SEl, UFS_SYSCTRL); + ufs_sys_ctrl_clr_bits(host, BIT_UFS_REFCLK_ISO_EN, PHY_ISO_EN); + /* open mphy ref clk */ + ufs_sys_ctrl_set_bits(host, BIT_SYSCTRL_REF_CLOCK_EN, PHY_CLK_CTRL); +} + +static void ufs_hisi_soc_init(struct ufs_hba *hba) +{ + struct ufs_hisi_host *host = ufshcd_get_variant(hba); + u32 reg; + + if (!IS_ERR(host->rst)) + reset_control_assert(host->rst); + + /* HC_PSW powerup */ + ufs_sys_ctrl_set_bits(host, BIT_UFS_PSW_MTCMOS_EN, PSW_POWER_CTRL); + udelay(10); + /* notify PWR ready */ + ufs_sys_ctrl_set_bits(host, BIT_SYSCTRL_PWR_READY, HC_LP_CTRL); + ufs_sys_ctrl_writel(host, MASK_UFS_DEVICE_RESET | 0, + UFS_DEVICE_RESET_CTRL); + + reg = ufs_sys_ctrl_readl(host, PHY_CLK_CTRL); + reg = (reg & ~MASK_SYSCTRL_CFG_CLOCK_FREQ) | UFS_FREQ_CFG_CLK; + /* set cfg clk freq */ + ufs_sys_ctrl_writel(host, reg, PHY_CLK_CTRL); + /* set ref clk freq */ + ufs_sys_ctrl_clr_bits(host, MASK_SYSCTRL_REF_CLOCK_SEL, PHY_CLK_CTRL); + /* bypass ufs clk gate */ + ufs_sys_ctrl_set_bits(host, MASK_UFS_CLK_GATE_BYPASS, + CLOCK_GATE_BYPASS); + ufs_sys_ctrl_set_bits(host, MASK_UFS_SYSCRTL_BYPASS, UFS_SYSCTRL); + + /* open psw clk */ + ufs_sys_ctrl_set_bits(host, BIT_SYSCTRL_PSW_CLK_EN, PSW_CLK_CTRL); + /* disable ufshc iso */ + ufs_sys_ctrl_clr_bits(host, BIT_UFS_PSW_ISO_CTRL, PSW_POWER_CTRL); + /* disable phy iso */ + ufs_sys_ctrl_clr_bits(host, BIT_UFS_PHY_ISO_CTRL, PHY_ISO_EN); + /* notice iso disable */ + ufs_sys_ctrl_clr_bits(host, BIT_SYSCTRL_LP_ISOL_EN, HC_LP_CTRL); + + /* disable lp_reset_n */ + ufs_sys_ctrl_set_bits(host, BIT_SYSCTRL_LP_RESET_N, RESET_CTRL_EN); + mdelay(1); + + ufs_sys_ctrl_writel(host, MASK_UFS_DEVICE_RESET | BIT_UFS_DEVICE_RESET, + UFS_DEVICE_RESET_CTRL); + + msleep(20); + + /* + * enable the fix of linereset recovery, + * and enable rx_reset/tx_rest beat + * enable ref_clk_en override(bit5) & + * override value = 1(bit4), with mask + */ + ufs_sys_ctrl_writel(host, 0x03300330, UFS_DEVICE_RESET_CTRL); + + if (!IS_ERR(host->rst)) + reset_control_deassert(host->rst); +} + +static int ufs_hisi_link_startup_pre_change(struct ufs_hba *hba) +{ + struct ufs_hisi_host *host = ufshcd_get_variant(hba); + int err; + uint32_t value; + uint32_t reg; + + /* Unipro VS_mphy_disable */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0xD0C1, 0x0), 0x1); + /* PA_HSSeries */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x156A, 0x0), 0x2); + /* MPHY CBRATESEL */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x8114, 0x0), 0x1); + /* MPHY CBOVRCTRL2 */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x8121, 0x0), 0x2D); + /* MPHY CBOVRCTRL3 */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x8122, 0x0), 0x1); + + if (host->caps & UFS_HISI_CAP_PHY10nm) { + /* MPHY CBOVRCTRL4 */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x8127, 0x0), 0x98); + /* MPHY CBOVRCTRL5 */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x8128, 0x0), 0x1); + } + + /* Unipro VS_MphyCfgUpdt */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0xD085, 0x0), 0x1); + /* MPHY RXOVRCTRL4 rx0 */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x800D, 0x4), 0x58); + /* MPHY RXOVRCTRL4 rx1 */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x800D, 0x5), 0x58); + /* MPHY RXOVRCTRL5 rx0 */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x800E, 0x4), 0xB); + /* MPHY RXOVRCTRL5 rx1 */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x800E, 0x5), 0xB); + /* MPHY RXSQCONTROL rx0 */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x8009, 0x4), 0x1); + /* MPHY RXSQCONTROL rx1 */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x8009, 0x5), 0x1); + /* Unipro VS_MphyCfgUpdt */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0xD085, 0x0), 0x1); + + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x8113, 0x0), 0x1); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0xD085, 0x0), 0x1); + + if (host->caps & UFS_HISI_CAP_PHY10nm) { + /* RX_Hibern8Time_Capability*/ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x0092, 0x4), 0xA); + /* RX_Hibern8Time_Capability*/ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x0092, 0x5), 0xA); + /* RX_Min_ActivateTime */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x008f, 0x4), 0xA); + /* RX_Min_ActivateTime*/ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x008f, 0x5), 0xA); + } else { + /* Tactive RX */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x008F, 0x4), 0x7); + /* Tactive RX */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x008F, 0x5), 0x7); + } + + /* Gear3 Synclength */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x0095, 0x4), 0x4F); + /* Gear3 Synclength */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x0095, 0x5), 0x4F); + /* Gear2 Synclength */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x0094, 0x4), 0x4F); + /* Gear2 Synclength */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x0094, 0x5), 0x4F); + /* Gear1 Synclength */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x008B, 0x4), 0x4F); + /* Gear1 Synclength */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x008B, 0x5), 0x4F); + /* Thibernate Tx */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x000F, 0x0), 0x5); + /* Thibernate Tx */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x000F, 0x1), 0x5); + + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0xD085, 0x0), 0x1); + /* Unipro VS_mphy_disable */ + ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(0xD0C1, 0x0), &value); + if (value != 0x1) + dev_info(hba->dev, + "Warring!!! Unipro VS_mphy_disable is 0x%x\n", value); + + /* Unipro VS_mphy_disable */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0xD0C1, 0x0), 0x0); + err = ufs_hisi_check_hibern8(hba); + if (err) + dev_err(hba->dev, "ufs_hisi_check_hibern8 error\n"); + + if (!(host->caps & UFS_HISI_CAP_PHY10nm)) + ufshcd_writel(hba, UFS_HCLKDIV_NORMAL_VALUE, UFS_REG_HCLKDIV); + + /* disable auto H8 */ + reg = ufshcd_readl(hba, REG_AUTO_HIBERNATE_IDLE_TIMER); + reg = reg & (~UFS_AHIT_AH8ITV_MASK); + ufshcd_writel(hba, reg, REG_AUTO_HIBERNATE_IDLE_TIMER); + + /* Unipro PA_Local_TX_LCC_Enable */ + ufshcd_disable_host_tx_lcc(hba); + /* close Unipro VS_Mk2ExtnSupport */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0xD0AB, 0x0), 0x0); + ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(0xD0AB, 0x0), &value); + if (value != 0) { + /* Ensure close success */ + dev_info(hba->dev, "WARN: close VS_Mk2ExtnSupport failed\n"); + } + + return err; +} + +static int ufs_hisi_link_startup_post_change(struct ufs_hba *hba) +{ + struct ufs_hisi_host *host = ufshcd_get_variant(hba); + + /* Unipro DL_AFC0CreditThreshold */ + ufshcd_dme_set(hba, UIC_ARG_MIB(0x2044), 0x0); + /* Unipro DL_TC0OutAckThreshold */ + ufshcd_dme_set(hba, UIC_ARG_MIB(0x2045), 0x0); + /* Unipro DL_TC0TXFCThreshold */ + ufshcd_dme_set(hba, UIC_ARG_MIB(0x2040), 0x9); + + /* not bypass ufs clk gate */ + ufs_sys_ctrl_clr_bits(host, MASK_UFS_CLK_GATE_BYPASS, + CLOCK_GATE_BYPASS); + ufs_sys_ctrl_clr_bits(host, MASK_UFS_SYSCRTL_BYPASS, + UFS_SYSCTRL); + + /* select received symbol cnt */ + ufshcd_dme_set(hba, UIC_ARG_MIB(0xd09a), 0x80000000); + /* reset counter0 and enable */ + ufshcd_dme_set(hba, UIC_ARG_MIB(0xd09c), 0x00000005); + + return 0; +} + +static int ufs_hisi_link_startup_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status) +{ + int err = 0; + + switch (status) { + case PRE_CHANGE: + err = ufs_hisi_link_startup_pre_change(hba); + break; + case POST_CHANGE: + err = ufs_hisi_link_startup_post_change(hba); + break; + default: + break; + } + + return err; +} + +static void ufs_hisi_set_dev_cap(struct ufs_host_params *host_params) +{ + ufshcd_init_host_params(host_params); +} + +static int ufs_hisi_negotiate_pwr_mode(struct ufs_hba *hba, + const struct ufs_pa_layer_attr *dev_max_params, + struct ufs_pa_layer_attr *dev_req_params) +{ + struct ufs_host_params host_params; + + ufs_hisi_set_dev_cap(&host_params); + + return ufshcd_negotiate_pwr_params(&host_params, dev_max_params, dev_req_params); +} + +static void ufs_hisi_pwr_change_pre_change(struct ufs_hba *hba) +{ + struct ufs_hisi_host *host = ufshcd_get_variant(hba); + + if (host->caps & UFS_HISI_CAP_PHY10nm) { + /* + * Boston platform need to set SaveConfigTime to 0x13, + * and change sync length to maximum value + */ + /* VS_DebugSaveConfigTime */ + ufshcd_dme_set(hba, UIC_ARG_MIB((u32)0xD0A0), 0x13); + /* g1 sync length */ + ufshcd_dme_set(hba, UIC_ARG_MIB((u32)0x1552), 0x4f); + /* g2 sync length */ + ufshcd_dme_set(hba, UIC_ARG_MIB((u32)0x1554), 0x4f); + /* g3 sync length */ + ufshcd_dme_set(hba, UIC_ARG_MIB((u32)0x1556), 0x4f); + /* PA_Hibern8Time */ + ufshcd_dme_set(hba, UIC_ARG_MIB((u32)0x15a7), 0xA); + /* PA_Tactivate */ + ufshcd_dme_set(hba, UIC_ARG_MIB((u32)0x15a8), 0xA); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0xd085, 0x0), 0x01); + } + + if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME) { + pr_info("ufs flash device must set VS_DebugSaveConfigTime 0x10\n"); + /* VS_DebugSaveConfigTime */ + ufshcd_dme_set(hba, UIC_ARG_MIB(0xD0A0), 0x10); + /* sync length */ + ufshcd_dme_set(hba, UIC_ARG_MIB(0x1556), 0x48); + } + + /* update */ + ufshcd_dme_set(hba, UIC_ARG_MIB(0x15A8), 0x1); + /* PA_TxSkip */ + ufshcd_dme_set(hba, UIC_ARG_MIB(0x155c), 0x0); + /*PA_PWRModeUserData0 = 8191, default is 0*/ + ufshcd_dme_set(hba, UIC_ARG_MIB(0x15b0), SZ_8K - 1); + /*PA_PWRModeUserData1 = 65535, default is 0*/ + ufshcd_dme_set(hba, UIC_ARG_MIB(0x15b1), SZ_64K - 1); + /*PA_PWRModeUserData2 = 32767, default is 0*/ + ufshcd_dme_set(hba, UIC_ARG_MIB(0x15b2), SZ_32K - 1); + /*DME_FC0ProtectionTimeOutVal = 8191, default is 0*/ + ufshcd_dme_set(hba, UIC_ARG_MIB(0xd041), SZ_8K - 1); + /*DME_TC0ReplayTimeOutVal = 65535, default is 0*/ + ufshcd_dme_set(hba, UIC_ARG_MIB(0xd042), SZ_64K - 1); + /*DME_AFC0ReqTimeOutVal = 32767, default is 0*/ + ufshcd_dme_set(hba, UIC_ARG_MIB(0xd043), SZ_32K - 1); + /*PA_PWRModeUserData3 = 8191, default is 0*/ + ufshcd_dme_set(hba, UIC_ARG_MIB(0x15b3), SZ_8K - 1); + /*PA_PWRModeUserData4 = 65535, default is 0*/ + ufshcd_dme_set(hba, UIC_ARG_MIB(0x15b4), SZ_64K - 1); + /*PA_PWRModeUserData5 = 32767, default is 0*/ + ufshcd_dme_set(hba, UIC_ARG_MIB(0x15b5), SZ_32K - 1); + /*DME_FC1ProtectionTimeOutVal = 8191, default is 0*/ + ufshcd_dme_set(hba, UIC_ARG_MIB(0xd044), SZ_8K - 1); + /*DME_TC1ReplayTimeOutVal = 65535, default is 0*/ + ufshcd_dme_set(hba, UIC_ARG_MIB(0xd045), SZ_64K - 1); + /*DME_AFC1ReqTimeOutVal = 32767, default is 0*/ + ufshcd_dme_set(hba, UIC_ARG_MIB(0xd046), SZ_32K - 1); +} + +static int ufs_hisi_pwr_change_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status, + struct ufs_pa_layer_attr *dev_req_params) +{ + int ret = 0; + + if (!dev_req_params) { + dev_err(hba->dev, + "%s: incoming dev_req_params is NULL\n", __func__); + ret = -EINVAL; + goto out; + } + + switch (status) { + case PRE_CHANGE: + ufs_hisi_pwr_change_pre_change(hba); + break; + case POST_CHANGE: + break; + default: + ret = -EINVAL; + break; + } +out: + return ret; +} + +static int ufs_hisi_suspend_prepare(struct device *dev) +{ + /* RPM and SPM are different. Refer ufs_hisi_suspend() */ + return __ufshcd_suspend_prepare(dev, false); +} + +static int ufs_hisi_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op, + enum ufs_notify_change_status status) +{ + struct ufs_hisi_host *host = ufshcd_get_variant(hba); + + if (status == PRE_CHANGE) + return 0; + + if (pm_op == UFS_RUNTIME_PM) + return 0; + + if (host->in_suspend) { + WARN_ON(1); + return 0; + } + + ufs_sys_ctrl_clr_bits(host, BIT_SYSCTRL_REF_CLOCK_EN, PHY_CLK_CTRL); + udelay(10); + /* set ref_dig_clk override of PHY PCS to 0 */ + ufs_sys_ctrl_writel(host, 0x00100000, UFS_DEVICE_RESET_CTRL); + + host->in_suspend = true; + + return 0; +} + +static int ufs_hisi_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) +{ + struct ufs_hisi_host *host = ufshcd_get_variant(hba); + + if (!host->in_suspend) + return 0; + + /* set ref_dig_clk override of PHY PCS to 1 */ + ufs_sys_ctrl_writel(host, 0x00100010, UFS_DEVICE_RESET_CTRL); + udelay(10); + ufs_sys_ctrl_set_bits(host, BIT_SYSCTRL_REF_CLOCK_EN, PHY_CLK_CTRL); + + host->in_suspend = false; + return 0; +} + +static int ufs_hisi_get_resource(struct ufs_hisi_host *host) +{ + struct device *dev = host->hba->dev; + struct platform_device *pdev = to_platform_device(dev); + + /* get resource of ufs sys ctrl */ + host->ufs_sys_ctrl = devm_platform_ioremap_resource(pdev, 1); + return PTR_ERR_OR_ZERO(host->ufs_sys_ctrl); +} + +static void ufs_hisi_set_pm_lvl(struct ufs_hba *hba) +{ + hba->rpm_lvl = UFS_PM_LVL_1; + hba->spm_lvl = UFS_PM_LVL_3; +} + +/** + * ufs_hisi_init_common + * @hba: host controller instance + */ +static int ufs_hisi_init_common(struct ufs_hba *hba) +{ + int err = 0; + struct device *dev = hba->dev; + struct ufs_hisi_host *host; + + host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL); + if (!host) + return -ENOMEM; + + host->hba = hba; + ufshcd_set_variant(hba, host); + + host->rst = devm_reset_control_get(dev, "rst"); + if (IS_ERR(host->rst)) { + dev_err(dev, "%s: failed to get reset control\n", __func__); + err = PTR_ERR(host->rst); + goto error; + } + + ufs_hisi_set_pm_lvl(hba); + + err = ufs_hisi_get_resource(host); + if (err) + goto error; + + return 0; + +error: + ufshcd_set_variant(hba, NULL); + return err; +} + +static int ufs_hi3660_init(struct ufs_hba *hba) +{ + int ret = 0; + struct device *dev = hba->dev; + + ret = ufs_hisi_init_common(hba); + if (ret) { + dev_err(dev, "%s: ufs common init fail\n", __func__); + return ret; + } + + ufs_hisi_clk_init(hba); + + ufs_hisi_soc_init(hba); + + return 0; +} + +static int ufs_hi3670_init(struct ufs_hba *hba) +{ + int ret = 0; + struct device *dev = hba->dev; + struct ufs_hisi_host *host; + + ret = ufs_hisi_init_common(hba); + if (ret) { + dev_err(dev, "%s: ufs common init fail\n", __func__); + return ret; + } + + ufs_hisi_clk_init(hba); + + ufs_hisi_soc_init(hba); + + /* Add cap for 10nm PHY variant on HI3670 SoC */ + host = ufshcd_get_variant(hba); + host->caps |= UFS_HISI_CAP_PHY10nm; + + return 0; +} + +static const struct ufs_hba_variant_ops ufs_hba_hi3660_vops = { + .name = "hi3660", + .init = ufs_hi3660_init, + .link_startup_notify = ufs_hisi_link_startup_notify, + .negotiate_pwr_mode = ufs_hisi_negotiate_pwr_mode, + .pwr_change_notify = ufs_hisi_pwr_change_notify, + .suspend = ufs_hisi_suspend, + .resume = ufs_hisi_resume, +}; + +static const struct ufs_hba_variant_ops ufs_hba_hi3670_vops = { + .name = "hi3670", + .init = ufs_hi3670_init, + .link_startup_notify = ufs_hisi_link_startup_notify, + .negotiate_pwr_mode = ufs_hisi_negotiate_pwr_mode, + .pwr_change_notify = ufs_hisi_pwr_change_notify, + .suspend = ufs_hisi_suspend, + .resume = ufs_hisi_resume, +}; + +static const struct of_device_id ufs_hisi_of_match[] = { + { .compatible = "hisilicon,hi3660-ufs", .data = &ufs_hba_hi3660_vops }, + { .compatible = "hisilicon,hi3670-ufs", .data = &ufs_hba_hi3670_vops }, + {}, +}; + +MODULE_DEVICE_TABLE(of, ufs_hisi_of_match); + +static int ufs_hisi_probe(struct platform_device *pdev) +{ + const struct of_device_id *of_id; + + of_id = of_match_node(ufs_hisi_of_match, pdev->dev.of_node); + + return ufshcd_pltfrm_init(pdev, of_id->data); +} + +static void ufs_hisi_remove(struct platform_device *pdev) +{ + ufshcd_pltfrm_remove(pdev); +} + +static const struct dev_pm_ops ufs_hisi_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(ufshcd_system_suspend, ufshcd_system_resume) + SET_RUNTIME_PM_OPS(ufshcd_runtime_suspend, ufshcd_runtime_resume, NULL) + .prepare = ufs_hisi_suspend_prepare, + .complete = ufshcd_resume_complete, +}; + +static struct platform_driver ufs_hisi_pltform = { + .probe = ufs_hisi_probe, + .remove = ufs_hisi_remove, + .driver = { + .name = "ufshcd-hisi", + .pm = &ufs_hisi_pm_ops, + .of_match_table = ufs_hisi_of_match, + }, +}; +module_platform_driver(ufs_hisi_pltform); + +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:ufshcd-hisi"); +MODULE_DESCRIPTION("HiSilicon Hixxxx UFS Driver"); diff --git a/drivers/ufs/host/ufs-hisi.h b/drivers/ufs/host/ufs-hisi.h new file mode 100644 index 000000000..5a90c0f4e --- /dev/null +++ b/drivers/ufs/host/ufs-hisi.h @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2017, HiSilicon. All rights reserved. + */ + +#ifndef UFS_HISI_H_ +#define UFS_HISI_H_ + +#define HBRN8_POLL_TOUT_MS 1000 + +/* + * ufs sysctrl specific define + */ +#define PSW_POWER_CTRL (0x04) +#define PHY_ISO_EN (0x08) +#define HC_LP_CTRL (0x0C) +#define PHY_CLK_CTRL (0x10) +#define PSW_CLK_CTRL (0x14) +#define CLOCK_GATE_BYPASS (0x18) +#define RESET_CTRL_EN (0x1C) +#define UFS_SYSCTRL (0x5C) +#define UFS_DEVICE_RESET_CTRL (0x60) + +#define BIT_UFS_PSW_ISO_CTRL (1 << 16) +#define BIT_UFS_PSW_MTCMOS_EN (1 << 0) +#define BIT_UFS_REFCLK_ISO_EN (1 << 16) +#define BIT_UFS_PHY_ISO_CTRL (1 << 0) +#define BIT_SYSCTRL_LP_ISOL_EN (1 << 16) +#define BIT_SYSCTRL_PWR_READY (1 << 8) +#define BIT_SYSCTRL_REF_CLOCK_EN (1 << 24) +#define MASK_SYSCTRL_REF_CLOCK_SEL (0x3 << 8) +#define MASK_SYSCTRL_CFG_CLOCK_FREQ (0xFF) +#define UFS_FREQ_CFG_CLK (0x39) +#define BIT_SYSCTRL_PSW_CLK_EN (1 << 4) +#define MASK_UFS_CLK_GATE_BYPASS (0x3F) +#define BIT_SYSCTRL_LP_RESET_N (1 << 0) +#define BIT_UFS_REFCLK_SRC_SEl (1 << 0) +#define MASK_UFS_SYSCRTL_BYPASS (0x3F << 16) +#define MASK_UFS_DEVICE_RESET (0x1 << 16) +#define BIT_UFS_DEVICE_RESET (0x1) + +/* + * M-TX Configuration Attributes for Hixxxx + */ +#define MPHY_TX_FSM_STATE 0x41 +#define TX_FSM_HIBERN8 0x1 + +/* + * Hixxxx UFS HC specific Registers + */ +enum { + UFS_REG_OCPTHRTL = 0xc0, + UFS_REG_OOCPR = 0xc4, + + UFS_REG_CDACFG = 0xd0, + UFS_REG_CDATX1 = 0xd4, + UFS_REG_CDATX2 = 0xd8, + UFS_REG_CDARX1 = 0xdc, + UFS_REG_CDARX2 = 0xe0, + UFS_REG_CDASTA = 0xe4, + + UFS_REG_LBMCFG = 0xf0, + UFS_REG_LBMSTA = 0xf4, + UFS_REG_UFSMODE = 0xf8, + + UFS_REG_HCLKDIV = 0xfc, +}; + +/* AHIT - Auto-Hibernate Idle Timer */ +#define UFS_AHIT_AH8ITV_MASK 0x3FF + +/* REG UFS_REG_OCPTHRTL definition */ +#define UFS_HCLKDIV_NORMAL_VALUE 0xE4 + +/* vendor specific pre-defined parameters */ +#define SLOW 1 +#define FAST 2 + +#define UFS_HISI_CAP_RESERVED BIT(0) +#define UFS_HISI_CAP_PHY10nm BIT(1) + +struct ufs_hisi_host { + struct ufs_hba *hba; + void __iomem *ufs_sys_ctrl; + + struct reset_control *rst; + + uint64_t caps; + + bool in_suspend; +}; + +#define ufs_sys_ctrl_writel(host, val, reg) \ + writel((val), (host)->ufs_sys_ctrl + (reg)) +#define ufs_sys_ctrl_readl(host, reg) readl((host)->ufs_sys_ctrl + (reg)) +#define ufs_sys_ctrl_set_bits(host, mask, reg) \ + ufs_sys_ctrl_writel( \ + (host), ((mask) | (ufs_sys_ctrl_readl((host), (reg)))), (reg)) +#define ufs_sys_ctrl_clr_bits(host, mask, reg) \ + ufs_sys_ctrl_writel((host), \ + ((~(mask)) & (ufs_sys_ctrl_readl((host), (reg)))), \ + (reg)) + +#endif /* UFS_HISI_H_ */ diff --git a/drivers/ufs/host/ufs-mediatek-sip.h b/drivers/ufs/host/ufs-mediatek-sip.h new file mode 100644 index 000000000..7d17aedf6 --- /dev/null +++ b/drivers/ufs/host/ufs-mediatek-sip.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2022 MediaTek Inc. + */ + +#ifndef _UFS_MEDIATEK_SIP_H +#define _UFS_MEDIATEK_SIP_H + +#include <linux/soc/mediatek/mtk_sip_svc.h> + +/* + * SiP (Slicon Partner) commands + */ +#define MTK_SIP_UFS_CONTROL MTK_SIP_SMC_CMD(0x276) +#define UFS_MTK_SIP_VA09_PWR_CTRL BIT(0) +#define UFS_MTK_SIP_DEVICE_RESET BIT(1) +#define UFS_MTK_SIP_CRYPTO_CTRL BIT(2) +#define UFS_MTK_SIP_REF_CLK_NOTIFICATION BIT(3) +#define UFS_MTK_SIP_SRAM_PWR_CTRL BIT(5) +#define UFS_MTK_SIP_GET_VCC_NUM BIT(6) +#define UFS_MTK_SIP_DEVICE_PWR_CTRL BIT(7) +#define UFS_MTK_SIP_MPHY_CTRL BIT(8) +#define UFS_MTK_SIP_MTCMOS_CTRL BIT(9) + +/* + * Multi-VCC by Numbering + */ +enum ufs_mtk_vcc_num { + UFS_VCC_NONE = 0, + UFS_VCC_1, + UFS_VCC_2, + UFS_VCC_MAX +}; + +enum ufs_mtk_mphy_op { + UFS_MPHY_BACKUP = 0, + UFS_MPHY_RESTORE +}; + +/* + * SMC call wrapper function + */ +struct ufs_mtk_smc_arg { + unsigned long cmd; + struct arm_smccc_res *res; + unsigned long v1; + unsigned long v2; + unsigned long v3; + unsigned long v4; + unsigned long v5; + unsigned long v6; + unsigned long v7; +}; + + +static inline void _ufs_mtk_smc(struct ufs_mtk_smc_arg s) +{ + arm_smccc_smc(MTK_SIP_UFS_CONTROL, + s.cmd, + s.v1, s.v2, s.v3, s.v4, s.v5, s.v6, s.res); +} + +#define ufs_mtk_smc(...) \ + _ufs_mtk_smc((struct ufs_mtk_smc_arg) {__VA_ARGS__}) + +/* Sip kernel interface */ +#define ufs_mtk_va09_pwr_ctrl(res, on) \ + ufs_mtk_smc(UFS_MTK_SIP_VA09_PWR_CTRL, &(res), on) + +#define ufs_mtk_crypto_ctrl(res, enable) \ + ufs_mtk_smc(UFS_MTK_SIP_CRYPTO_CTRL, &(res), enable) + +#define ufs_mtk_ref_clk_notify(on, stage, res) \ + ufs_mtk_smc(UFS_MTK_SIP_REF_CLK_NOTIFICATION, &(res), on, stage) + +#define ufs_mtk_device_reset_ctrl(high, res) \ + ufs_mtk_smc(UFS_MTK_SIP_DEVICE_RESET, &(res), high) + +#define ufs_mtk_sram_pwr_ctrl(on, res) \ + ufs_mtk_smc(UFS_MTK_SIP_SRAM_PWR_CTRL, &(res), on) + +#define ufs_mtk_get_vcc_num(res) \ + ufs_mtk_smc(UFS_MTK_SIP_GET_VCC_NUM, &(res)) + +#define ufs_mtk_device_pwr_ctrl(on, ufs_version, res) \ + ufs_mtk_smc(UFS_MTK_SIP_DEVICE_PWR_CTRL, &(res), on, ufs_version) + +#define ufs_mtk_mphy_ctrl(op, res) \ + ufs_mtk_smc(UFS_MTK_SIP_MPHY_CTRL, &(res), op) + +#define ufs_mtk_mtcmos_ctrl(op, res) \ + ufs_mtk_smc(UFS_MTK_SIP_MTCMOS_CTRL, &(res), op) + +#endif /* !_UFS_MEDIATEK_SIP_H */ diff --git a/drivers/ufs/host/ufs-mediatek-trace.h b/drivers/ufs/host/ufs-mediatek-trace.h new file mode 100644 index 000000000..0df8ac843 --- /dev/null +++ b/drivers/ufs/host/ufs-mediatek-trace.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020 MediaTek Inc. + */ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM ufs_mtk + +#if !defined(_TRACE_EVENT_UFS_MEDIATEK_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_EVENT_UFS_MEDIATEK_H + +#include <linux/tracepoint.h> + +TRACE_EVENT(ufs_mtk_event, + TP_PROTO(unsigned int type, unsigned int data), + TP_ARGS(type, data), + + TP_STRUCT__entry( + __field(unsigned int, type) + __field(unsigned int, data) + ), + + TP_fast_assign( + __entry->type = type; + __entry->data = data; + ), + + TP_printk("ufs: event=%u data=%u", + __entry->type, __entry->data) +); + +TRACE_EVENT(ufs_mtk_clk_scale, + TP_PROTO(const char *name, bool scale_up, unsigned long clk_rate), + TP_ARGS(name, scale_up, clk_rate), + + TP_STRUCT__entry( + __string(name, name) + __field(bool, scale_up) + __field(unsigned long, clk_rate) + ), + + TP_fast_assign( + __assign_str(name); + __entry->scale_up = scale_up; + __entry->clk_rate = clk_rate; + ), + + TP_printk("ufs: clk (%s) scaled %s @ %lu", + __get_str(name), + __entry->scale_up ? "up" : "down", + __entry->clk_rate) +); + +#endif + +#undef TRACE_INCLUDE_PATH +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_PATH ../../drivers/ufs/host +#define TRACE_INCLUDE_FILE ufs-mediatek-trace +#include <trace/define_trace.h> diff --git a/drivers/ufs/host/ufs-mediatek.c b/drivers/ufs/host/ufs-mediatek.c new file mode 100644 index 000000000..3991a5126 --- /dev/null +++ b/drivers/ufs/host/ufs-mediatek.c @@ -0,0 +1,2559 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2019 MediaTek Inc. + * Authors: + * Stanley Chu <stanley.chu@mediatek.com> + * Peter Wang <peter.wang@mediatek.com> + */ + +#include <linux/arm-smccc.h> +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_device.h> +#include <linux/of_platform.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/regulator/consumer.h> +#include <linux/reset.h> + +#include <ufs/ufshcd.h> +#include "ufshcd-pltfrm.h" +#include <ufs/ufs_quirks.h> +#include <ufs/unipro.h> + +#include "ufs-mediatek.h" +#include "ufs-mediatek-sip.h" + +static int ufs_mtk_config_mcq(struct ufs_hba *hba, bool irq); +static void _ufs_mtk_clk_scale(struct ufs_hba *hba, bool scale_up); + +#define CREATE_TRACE_POINTS +#include "ufs-mediatek-trace.h" +#undef CREATE_TRACE_POINTS + +#define MAX_SUPP_MAC 64 +#define MCQ_QUEUE_OFFSET(c) ((((c) >> 16) & 0xFF) * 0x200) + +static const struct ufs_dev_quirk ufs_mtk_dev_fixups[] = { + { .wmanufacturerid = UFS_ANY_VENDOR, + .model = UFS_ANY_MODEL, + .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM }, + { .wmanufacturerid = UFS_VENDOR_SKHYNIX, + .model = "H9HQ21AFAMZDAR", + .quirk = UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES }, + {} +}; + +static const struct of_device_id ufs_mtk_of_match[] = { + { .compatible = "mediatek,mt8183-ufshci" }, + { .compatible = "mediatek,mt8195-ufshci" }, + {}, +}; +MODULE_DEVICE_TABLE(of, ufs_mtk_of_match); + +/* + * Details of UIC Errors + */ +static const char *const ufs_uic_err_str[] = { + "PHY Adapter Layer", + "Data Link Layer", + "Network Link Layer", + "Transport Link Layer", + "DME" +}; + +static const char *const ufs_uic_pa_err_str[] = { + "PHY error on Lane 0", + "PHY error on Lane 1", + "PHY error on Lane 2", + "PHY error on Lane 3", + "Generic PHY Adapter Error. This should be the LINERESET indication" +}; + +static const char *const ufs_uic_dl_err_str[] = { + "NAC_RECEIVED", + "TCx_REPLAY_TIMER_EXPIRED", + "AFCx_REQUEST_TIMER_EXPIRED", + "FCx_PROTECTION_TIMER_EXPIRED", + "CRC_ERROR", + "RX_BUFFER_OVERFLOW", + "MAX_FRAME_LENGTH_EXCEEDED", + "WRONG_SEQUENCE_NUMBER", + "AFC_FRAME_SYNTAX_ERROR", + "NAC_FRAME_SYNTAX_ERROR", + "EOF_SYNTAX_ERROR", + "FRAME_SYNTAX_ERROR", + "BAD_CTRL_SYMBOL_TYPE", + "PA_INIT_ERROR", + "PA_ERROR_IND_RECEIVED", + "PA_INIT" +}; + +static bool ufs_mtk_is_boost_crypt_enabled(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + return host->caps & UFS_MTK_CAP_BOOST_CRYPT_ENGINE; +} + +static bool ufs_mtk_is_va09_supported(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + return host->caps & UFS_MTK_CAP_VA09_PWR_CTRL; +} + +static bool ufs_mtk_is_broken_vcc(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + return host->caps & UFS_MTK_CAP_BROKEN_VCC; +} + +static bool ufs_mtk_is_pmc_via_fastauto(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + return host->caps & UFS_MTK_CAP_PMC_VIA_FASTAUTO; +} + +static bool ufs_mtk_is_tx_skew_fix(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + return host->caps & UFS_MTK_CAP_TX_SKEW_FIX; +} + +static bool ufs_mtk_is_rtff_mtcmos(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + return host->caps & UFS_MTK_CAP_RTFF_MTCMOS; +} + +static bool ufs_mtk_is_allow_vccqx_lpm(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + return host->caps & UFS_MTK_CAP_ALLOW_VCCQX_LPM; +} + +static bool ufs_mtk_is_clk_scale_ready(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + struct ufs_mtk_clk *mclk = &host->mclk; + + return mclk->ufs_sel_clki && + mclk->ufs_sel_max_clki && + mclk->ufs_sel_min_clki; +} + +static void ufs_mtk_cfg_unipro_cg(struct ufs_hba *hba, bool enable) +{ + u32 tmp; + + if (enable) { + ufshcd_dme_get(hba, + UIC_ARG_MIB(VS_SAVEPOWERCONTROL), &tmp); + tmp = tmp | + (1 << RX_SYMBOL_CLK_GATE_EN) | + (1 << SYS_CLK_GATE_EN) | + (1 << TX_CLK_GATE_EN); + ufshcd_dme_set(hba, + UIC_ARG_MIB(VS_SAVEPOWERCONTROL), tmp); + + ufshcd_dme_get(hba, + UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), &tmp); + tmp = tmp & ~(1 << TX_SYMBOL_CLK_REQ_FORCE); + ufshcd_dme_set(hba, + UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), tmp); + } else { + ufshcd_dme_get(hba, + UIC_ARG_MIB(VS_SAVEPOWERCONTROL), &tmp); + tmp = tmp & ~((1 << RX_SYMBOL_CLK_GATE_EN) | + (1 << SYS_CLK_GATE_EN) | + (1 << TX_CLK_GATE_EN)); + ufshcd_dme_set(hba, + UIC_ARG_MIB(VS_SAVEPOWERCONTROL), tmp); + + ufshcd_dme_get(hba, + UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), &tmp); + tmp = tmp | (1 << TX_SYMBOL_CLK_REQ_FORCE); + ufshcd_dme_set(hba, + UIC_ARG_MIB(VS_DEBUGCLOCKENABLE), tmp); + } +} + +static void ufs_mtk_crypto_enable(struct ufs_hba *hba) +{ + struct arm_smccc_res res; + + ufs_mtk_crypto_ctrl(res, 1); + if (res.a0) { + dev_info(hba->dev, "%s: crypto enable failed, err: %lu\n", + __func__, res.a0); + hba->caps &= ~UFSHCD_CAP_CRYPTO; + } +} + +static void ufs_mtk_host_reset(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + struct arm_smccc_res res; + + reset_control_assert(host->hci_reset); + reset_control_assert(host->crypto_reset); + reset_control_assert(host->unipro_reset); + reset_control_assert(host->mphy_reset); + + usleep_range(100, 110); + + reset_control_deassert(host->unipro_reset); + reset_control_deassert(host->crypto_reset); + reset_control_deassert(host->hci_reset); + reset_control_deassert(host->mphy_reset); + + /* restore mphy setting aftre mphy reset */ + if (host->mphy_reset) + ufs_mtk_mphy_ctrl(UFS_MPHY_RESTORE, res); +} + +static void ufs_mtk_init_reset_control(struct ufs_hba *hba, + struct reset_control **rc, + char *str) +{ + *rc = devm_reset_control_get(hba->dev, str); + if (IS_ERR(*rc)) { + dev_info(hba->dev, "Failed to get reset control %s: %ld\n", + str, PTR_ERR(*rc)); + *rc = NULL; + } +} + +static void ufs_mtk_init_reset(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + ufs_mtk_init_reset_control(hba, &host->hci_reset, + "hci_rst"); + ufs_mtk_init_reset_control(hba, &host->unipro_reset, + "unipro_rst"); + ufs_mtk_init_reset_control(hba, &host->crypto_reset, + "crypto_rst"); + ufs_mtk_init_reset_control(hba, &host->mphy_reset, + "mphy_rst"); +} + +static int ufs_mtk_hce_enable_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + if (status == PRE_CHANGE) { + if (host->unipro_lpm) { + hba->vps->hba_enable_delay_us = 0; + } else { + hba->vps->hba_enable_delay_us = 600; + ufs_mtk_host_reset(hba); + } + + if (hba->caps & UFSHCD_CAP_CRYPTO) + ufs_mtk_crypto_enable(hba); + + if (host->caps & UFS_MTK_CAP_DISABLE_AH8) { + ufshcd_writel(hba, 0, + REG_AUTO_HIBERNATE_IDLE_TIMER); + hba->capabilities &= ~MASK_AUTO_HIBERN8_SUPPORT; + hba->ahit = 0; + } + + /* + * Turn on CLK_CG early to bypass abnormal ERR_CHK signal + * to prevent host hang issue + */ + ufshcd_writel(hba, + ufshcd_readl(hba, REG_UFS_XOUFS_CTRL) | 0x80, + REG_UFS_XOUFS_CTRL); + + if (host->legacy_ip_ver) + return 0; + + /* DDR_EN setting */ + if (host->ip_ver >= IP_VER_MT6989) { + ufshcd_rmwl(hba, UFS_MASK(0x7FFF, 8), + 0x453000, REG_UFS_MMIO_OPT_CTRL_0); + } + + if (host->ip_ver >= IP_VER_MT6991_A0) { + /* Enable multi-rtt */ + ufshcd_rmwl(hba, MRTT_EN, MRTT_EN, REG_UFS_MMIO_OPT_CTRL_0); + /* Enable random performance improvement */ + ufshcd_rmwl(hba, RDN_PFM_IMPV_DIS, 0, REG_UFS_MMIO_OPT_CTRL_0); + } + } + + return 0; +} + +static int ufs_mtk_bind_mphy(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + struct device *dev = hba->dev; + struct device_node *np = dev->of_node; + int err = 0; + + host->mphy = devm_of_phy_get_by_index(dev, np, 0); + + if (host->mphy == ERR_PTR(-EPROBE_DEFER)) { + /* + * UFS driver might be probed before the phy driver does. + * In that case we would like to return EPROBE_DEFER code. + */ + err = -EPROBE_DEFER; + dev_info(dev, + "%s: required phy hasn't probed yet. err = %d\n", + __func__, err); + } else if (IS_ERR(host->mphy)) { + err = PTR_ERR(host->mphy); + if (err != -ENODEV) { + dev_info(dev, "%s: PHY get failed %d\n", __func__, + err); + } + } + + if (err) + host->mphy = NULL; + /* + * Allow unbound mphy because not every platform needs specific + * mphy control. + */ + if (err == -ENODEV) + err = 0; + + return err; +} + +static int ufs_mtk_setup_ref_clk(struct ufs_hba *hba, bool on) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + struct arm_smccc_res res; + ktime_t timeout, time_checked; + u32 value; + + if (host->ref_clk_enabled == on) + return 0; + + ufs_mtk_ref_clk_notify(on, PRE_CHANGE, res); + + if (on) { + ufshcd_writel(hba, REFCLK_REQUEST, REG_UFS_REFCLK_CTRL); + } else { + ufshcd_delay_us(host->ref_clk_gating_wait_us, 10); + ufshcd_writel(hba, REFCLK_RELEASE, REG_UFS_REFCLK_CTRL); + } + + /* Wait for ack */ + timeout = ktime_add_us(ktime_get(), REFCLK_REQ_TIMEOUT_US); + do { + time_checked = ktime_get(); + value = ufshcd_readl(hba, REG_UFS_REFCLK_CTRL); + + /* Wait until ack bit equals to req bit */ + if (((value & REFCLK_ACK) >> 1) == (value & REFCLK_REQUEST)) + goto out; + + usleep_range(100, 200); + } while (ktime_before(time_checked, timeout)); + + dev_err(hba->dev, "missing ack of refclk req, reg: 0x%x\n", value); + + /* + * If clock on timeout, assume clock is off, notify tfa do clock + * off setting.(keep DIFN disable, release resource) + * If clock off timeout, assume clock will off finally, + * set ref_clk_enabled directly.(keep DIFN disable, keep resource) + */ + if (on) + ufs_mtk_ref_clk_notify(false, POST_CHANGE, res); + else + host->ref_clk_enabled = false; + + return -ETIMEDOUT; + +out: + host->ref_clk_enabled = on; + if (on) + ufshcd_delay_us(host->ref_clk_ungating_wait_us, 10); + + ufs_mtk_ref_clk_notify(on, POST_CHANGE, res); + + return 0; +} + +static void ufs_mtk_setup_ref_clk_wait_us(struct ufs_hba *hba, + u16 gating_us) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + if (hba->dev_info.clk_gating_wait_us) { + host->ref_clk_gating_wait_us = + hba->dev_info.clk_gating_wait_us; + } else { + host->ref_clk_gating_wait_us = gating_us; + } + + host->ref_clk_ungating_wait_us = REFCLK_DEFAULT_WAIT_US; +} + +static void ufs_mtk_dbg_sel(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + if (!host->legacy_ip_ver && host->ip_ver >= IP_VER_MT6983) { + ufshcd_writel(hba, 0x820820, REG_UFS_DEBUG_SEL); + ufshcd_writel(hba, 0x0, REG_UFS_DEBUG_SEL_B0); + ufshcd_writel(hba, 0x55555555, REG_UFS_DEBUG_SEL_B1); + ufshcd_writel(hba, 0xaaaaaaaa, REG_UFS_DEBUG_SEL_B2); + ufshcd_writel(hba, 0xffffffff, REG_UFS_DEBUG_SEL_B3); + } else { + ufshcd_writel(hba, 0x20, REG_UFS_DEBUG_SEL); + } +} + +static int ufs_mtk_wait_idle_state(struct ufs_hba *hba, + unsigned long retry_ms) +{ + u64 timeout, time_checked; + u32 val, sm; + bool wait_idle; + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + /* cannot use plain ktime_get() in suspend */ + timeout = ktime_get_mono_fast_ns() + retry_ms * 1000000UL; + + /* wait a specific time after check base */ + udelay(10); + wait_idle = false; + + do { + time_checked = ktime_get_mono_fast_ns(); + if (host->legacy_ip_ver || host->ip_ver < IP_VER_MT6899) { + ufs_mtk_dbg_sel(hba); + val = ufshcd_readl(hba, REG_UFS_PROBE); + } else { + val = ufshcd_readl(hba, REG_UFS_UFS_MMIO_OTSD_CTRL); + val = val >> 16; + } + + sm = val & 0x1f; + + /* + * if state is in H8 enter and H8 enter confirm + * wait until return to idle state. + */ + if ((sm >= VS_HIB_ENTER) && (sm <= VS_HIB_EXIT)) { + wait_idle = true; + udelay(50); + continue; + } else if (!wait_idle) + break; + + if (wait_idle && (sm == VS_HCE_BASE)) + break; + } while (time_checked < timeout); + + if (wait_idle && sm != VS_HCE_BASE) { + dev_info(hba->dev, "wait idle tmo: 0x%x\n", val); + return -ETIMEDOUT; + } + + return 0; +} + +static int ufs_mtk_wait_link_state(struct ufs_hba *hba, u32 state, + unsigned long max_wait_ms) +{ + ktime_t timeout, time_checked; + u32 val; + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + timeout = ktime_add_ms(ktime_get(), max_wait_ms); + do { + time_checked = ktime_get(); + + if (host->legacy_ip_ver || host->ip_ver < IP_VER_MT6899) { + ufs_mtk_dbg_sel(hba); + val = ufshcd_readl(hba, REG_UFS_PROBE); + val = val >> 28; + } else { + val = ufshcd_readl(hba, REG_UFS_UFS_MMIO_OTSD_CTRL); + val = val >> 24; + } + + if (val == state) + return 0; + + /* Sleep for max. 200us */ + usleep_range(100, 200); + } while (ktime_before(time_checked, timeout)); + + return -ETIMEDOUT; +} + +static int ufs_mtk_mphy_power_on(struct ufs_hba *hba, bool on) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + struct phy *mphy = host->mphy; + struct arm_smccc_res res; + int ret = 0; + + if (!mphy || !(on ^ host->mphy_powered_on)) + return 0; + + if (on) { + if (ufs_mtk_is_va09_supported(hba)) { + ret = regulator_enable(host->reg_va09); + if (ret < 0) + goto out; + /* wait 200 us to stablize VA09 */ + usleep_range(200, 210); + ufs_mtk_va09_pwr_ctrl(res, 1); + } + phy_power_on(mphy); + } else { + phy_power_off(mphy); + if (ufs_mtk_is_va09_supported(hba)) { + ufs_mtk_va09_pwr_ctrl(res, 0); + ret = regulator_disable(host->reg_va09); + } + } +out: + if (ret) { + dev_info(hba->dev, + "failed to %s va09: %d\n", + on ? "enable" : "disable", + ret); + } else { + host->mphy_powered_on = on; + } + + return ret; +} + +static int ufs_mtk_get_host_clk(struct device *dev, const char *name, + struct clk **clk_out) +{ + struct clk *clk; + int err = 0; + + clk = devm_clk_get(dev, name); + if (IS_ERR(clk)) + err = PTR_ERR(clk); + else + *clk_out = clk; + + return err; +} + +static void ufs_mtk_boost_crypt(struct ufs_hba *hba, bool boost) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + struct ufs_mtk_crypt_cfg *cfg; + struct regulator *reg; + int volt, ret; + + if (!ufs_mtk_is_boost_crypt_enabled(hba)) + return; + + cfg = host->crypt; + volt = cfg->vcore_volt; + reg = cfg->reg_vcore; + + ret = clk_prepare_enable(cfg->clk_crypt_mux); + if (ret) { + dev_info(hba->dev, "clk_prepare_enable(): %d\n", + ret); + return; + } + + if (boost) { + ret = regulator_set_voltage(reg, volt, INT_MAX); + if (ret) { + dev_info(hba->dev, + "failed to set vcore to %d\n", volt); + goto out; + } + + ret = clk_set_parent(cfg->clk_crypt_mux, + cfg->clk_crypt_perf); + if (ret) { + dev_info(hba->dev, + "failed to set clk_crypt_perf\n"); + regulator_set_voltage(reg, 0, INT_MAX); + goto out; + } + } else { + ret = clk_set_parent(cfg->clk_crypt_mux, + cfg->clk_crypt_lp); + if (ret) { + dev_info(hba->dev, + "failed to set clk_crypt_lp\n"); + goto out; + } + + ret = regulator_set_voltage(reg, 0, INT_MAX); + if (ret) { + dev_info(hba->dev, + "failed to set vcore to MIN\n"); + } + } +out: + clk_disable_unprepare(cfg->clk_crypt_mux); +} + +static int ufs_mtk_init_host_clk(struct ufs_hba *hba, const char *name, + struct clk **clk) +{ + int ret; + + ret = ufs_mtk_get_host_clk(hba->dev, name, clk); + if (ret) { + dev_info(hba->dev, "%s: failed to get %s: %d", __func__, + name, ret); + } + + return ret; +} + +static void ufs_mtk_init_boost_crypt(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + struct ufs_mtk_crypt_cfg *cfg; + struct device *dev = hba->dev; + struct regulator *reg; + u32 volt; + + host->crypt = devm_kzalloc(dev, sizeof(*(host->crypt)), + GFP_KERNEL); + if (!host->crypt) + goto disable_caps; + + reg = devm_regulator_get_optional(dev, "dvfsrc-vcore"); + if (IS_ERR(reg)) { + dev_info(dev, "failed to get dvfsrc-vcore: %ld", + PTR_ERR(reg)); + goto disable_caps; + } + + if (of_property_read_u32(dev->of_node, "boost-crypt-vcore-min", + &volt)) { + dev_info(dev, "failed to get boost-crypt-vcore-min"); + goto disable_caps; + } + + cfg = host->crypt; + if (ufs_mtk_init_host_clk(hba, "crypt_mux", + &cfg->clk_crypt_mux)) + goto disable_caps; + + if (ufs_mtk_init_host_clk(hba, "crypt_lp", + &cfg->clk_crypt_lp)) + goto disable_caps; + + if (ufs_mtk_init_host_clk(hba, "crypt_perf", + &cfg->clk_crypt_perf)) + goto disable_caps; + + cfg->reg_vcore = reg; + cfg->vcore_volt = volt; + host->caps |= UFS_MTK_CAP_BOOST_CRYPT_ENGINE; + +disable_caps: + return; +} + +static void ufs_mtk_init_va09_pwr_ctrl(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + host->reg_va09 = regulator_get(hba->dev, "va09"); + if (IS_ERR(host->reg_va09)) + dev_info(hba->dev, "failed to get va09"); + else + host->caps |= UFS_MTK_CAP_VA09_PWR_CTRL; +} + +static void ufs_mtk_init_host_caps(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + struct device_node *np = hba->dev->of_node; + + if (of_property_read_bool(np, "mediatek,ufs-boost-crypt")) + ufs_mtk_init_boost_crypt(hba); + + if (of_property_read_bool(np, "mediatek,ufs-support-va09")) + ufs_mtk_init_va09_pwr_ctrl(hba); + + if (of_property_read_bool(np, "mediatek,ufs-disable-ah8")) + host->caps |= UFS_MTK_CAP_DISABLE_AH8; + + if (of_property_read_bool(np, "mediatek,ufs-broken-vcc")) + host->caps |= UFS_MTK_CAP_BROKEN_VCC; + + if (of_property_read_bool(np, "mediatek,ufs-pmc-via-fastauto")) + host->caps |= UFS_MTK_CAP_PMC_VIA_FASTAUTO; + + if (of_property_read_bool(np, "mediatek,ufs-tx-skew-fix")) + host->caps |= UFS_MTK_CAP_TX_SKEW_FIX; + + if (of_property_read_bool(np, "mediatek,ufs-disable-mcq")) + host->caps |= UFS_MTK_CAP_DISABLE_MCQ; + + if (of_property_read_bool(np, "mediatek,ufs-rtff-mtcmos")) + host->caps |= UFS_MTK_CAP_RTFF_MTCMOS; + + if (of_property_read_bool(np, "mediatek,ufs-broken-rtc")) + host->caps |= UFS_MTK_CAP_MCQ_BROKEN_RTC; + + dev_info(hba->dev, "caps: 0x%x", host->caps); +} + +static void ufs_mtk_scale_perf(struct ufs_hba *hba, bool scale_up) +{ + ufs_mtk_boost_crypt(hba, scale_up); +} + +static void ufs_mtk_pwr_ctrl(struct ufs_hba *hba, bool on) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + if (on) { + phy_power_on(host->mphy); + ufs_mtk_setup_ref_clk(hba, on); + if (!ufshcd_is_clkscaling_supported(hba)) + ufs_mtk_scale_perf(hba, on); + } else { + if (!ufshcd_is_clkscaling_supported(hba)) + ufs_mtk_scale_perf(hba, on); + ufs_mtk_setup_ref_clk(hba, on); + phy_power_off(host->mphy); + } +} + +static void ufs_mtk_mcq_disable_irq(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + u32 irq, i; + + if (!hba->mcq_enabled) + return; + + if (host->mcq_nr_intr == 0) + return; + + for (i = 0; i < host->mcq_nr_intr; i++) { + irq = host->mcq_intr_info[i].irq; + disable_irq(irq); + } + host->is_mcq_intr_enabled = false; +} + +static void ufs_mtk_mcq_enable_irq(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + u32 irq, i; + + if (!hba->mcq_enabled) + return; + + if (host->mcq_nr_intr == 0) + return; + + if (host->is_mcq_intr_enabled == true) + return; + + for (i = 0; i < host->mcq_nr_intr; i++) { + irq = host->mcq_intr_info[i].irq; + enable_irq(irq); + } + host->is_mcq_intr_enabled = true; +} + +/** + * ufs_mtk_setup_clocks - enables/disable clocks + * @hba: host controller instance + * @on: If true, enable clocks else disable them. + * @status: PRE_CHANGE or POST_CHANGE notify + * + * Return: 0 on success, non-zero on failure. + */ +static int ufs_mtk_setup_clocks(struct ufs_hba *hba, bool on, + enum ufs_notify_change_status status) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + bool clk_pwr_off = false; + int ret = 0; + + /* + * In case ufs_mtk_init() is not yet done, simply ignore. + * This ufs_mtk_setup_clocks() shall be called from + * ufs_mtk_init() after init is done. + */ + if (!host) + return 0; + + if (!on && status == PRE_CHANGE) { + if (ufshcd_is_link_off(hba)) { + clk_pwr_off = true; + } else if (ufshcd_is_link_hibern8(hba) || + (!ufshcd_can_hibern8_during_gating(hba) && + ufshcd_is_auto_hibern8_enabled(hba))) { + /* + * Gate ref-clk and poweroff mphy if link state is in + * OFF or Hibern8 by either Auto-Hibern8 or + * ufshcd_link_state_transition(). + */ + ret = ufs_mtk_wait_link_state(hba, + VS_LINK_HIBERN8, + 15); + if (!ret) + clk_pwr_off = true; + } + + if (clk_pwr_off) { + ufs_mtk_pwr_ctrl(hba, false); + } else { + dev_warn(hba->dev, "Clock is not turned off, hba->ahit = 0x%x, AHIT = 0x%x\n", + hba->ahit, + ufshcd_readl(hba, + REG_AUTO_HIBERNATE_IDLE_TIMER)); + } + ufs_mtk_mcq_disable_irq(hba); + } else if (on && status == POST_CHANGE) { + ufs_mtk_pwr_ctrl(hba, true); + ufs_mtk_mcq_enable_irq(hba); + } + + return ret; +} + +static u32 ufs_mtk_mcq_get_irq(struct ufs_hba *hba, unsigned int cpu) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + struct blk_mq_tag_set *tag_set = &hba->host->tag_set; + struct blk_mq_queue_map *map = &tag_set->map[HCTX_TYPE_DEFAULT]; + unsigned int nr = map->nr_queues; + unsigned int q_index; + + q_index = map->mq_map[cpu]; + if (q_index >= nr) { + dev_err(hba->dev, "hwq index %d exceed %d\n", + q_index, nr); + return MTK_MCQ_INVALID_IRQ; + } + + return host->mcq_intr_info[q_index].irq; +} + +static void ufs_mtk_mcq_set_irq_affinity(struct ufs_hba *hba, unsigned int cpu) +{ + unsigned int irq, _cpu; + int ret; + + irq = ufs_mtk_mcq_get_irq(hba, cpu); + if (irq == MTK_MCQ_INVALID_IRQ) { + dev_err(hba->dev, "invalid irq. unable to bind irq to cpu%d", cpu); + return; + } + + /* force migrate irq of cpu0 to cpu3 */ + _cpu = (cpu == 0) ? 3 : cpu; + ret = irq_set_affinity(irq, cpumask_of(_cpu)); + if (ret) { + dev_err(hba->dev, "set irq %d affinity to CPU %d failed\n", + irq, _cpu); + return; + } + dev_info(hba->dev, "set irq %d affinity to CPU: %d\n", irq, _cpu); +} + +static bool ufs_mtk_is_legacy_chipset(struct ufs_hba *hba, u32 hw_ip_ver) +{ + bool is_legacy = false; + + switch (hw_ip_ver) { + case IP_LEGACY_VER_MT6893: + case IP_LEGACY_VER_MT6781: + /* can add other legacy chipset ID here accordingly */ + is_legacy = true; + break; + default: + break; + } + dev_info(hba->dev, "legacy IP version - 0x%x, is legacy : %d", hw_ip_ver, is_legacy); + + return is_legacy; +} + +/* + * HW version format has been changed from 01MMmmmm to 1MMMmmmm, since + * project MT6878. In order to perform correct version comparison, + * version number is changed by SW for the following projects. + * IP_VER_MT6983 0x00360000 to 0x10360000 + * IP_VER_MT6897 0x01440000 to 0x10440000 + * IP_VER_MT6989 0x01450000 to 0x10450000 + * IP_VER_MT6991 0x01460000 to 0x10460000 + */ +static void ufs_mtk_get_hw_ip_version(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + u32 hw_ip_ver; + + hw_ip_ver = ufshcd_readl(hba, REG_UFS_MTK_IP_VER); + + if (((hw_ip_ver & (0xFF << 24)) == (0x1 << 24)) || + ((hw_ip_ver & (0xFF << 24)) == 0)) { + hw_ip_ver &= ~(0xFF << 24); + hw_ip_ver |= (0x1 << 28); + } + + host->ip_ver = hw_ip_ver; + + host->legacy_ip_ver = ufs_mtk_is_legacy_chipset(hba, hw_ip_ver); +} + +static void ufs_mtk_get_controller_version(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + int ret, ver = 0; + + if (host->hw_ver.major) + return; + + /* Set default (minimum) version anyway */ + host->hw_ver.major = 2; + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_LOCALVERINFO), &ver); + if (!ret) { + if (ver >= UFS_UNIPRO_VER_1_8) { + host->hw_ver.major = 3; + /* + * Fix HCI version for some platforms with + * incorrect version + */ + if (hba->ufs_version < ufshci_version(3, 0)) + hba->ufs_version = ufshci_version(3, 0); + } + } +} + +static u32 ufs_mtk_get_ufs_hci_version(struct ufs_hba *hba) +{ + return hba->ufs_version; +} + +/** + * ufs_mtk_init_clocks - Init mtk driver private clocks + * + * @hba: per adapter instance + */ +static void ufs_mtk_init_clocks(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + struct list_head *head = &hba->clk_list_head; + struct ufs_clk_info *clki, *clki_tmp; + struct device *dev = hba->dev; + struct regulator *reg; + u32 volt; + + /* + * Find private clocks and store them in struct ufs_mtk_clk. + * Remove "ufs_sel_min_src" and "ufs_sel_min_src" from list to avoid + * being switched on/off in clock gating. + */ + list_for_each_entry_safe(clki, clki_tmp, head, list) { + if (!strcmp(clki->name, "ufs_sel")) { + host->mclk.ufs_sel_clki = clki; + } else if (!strcmp(clki->name, "ufs_sel_max_src")) { + host->mclk.ufs_sel_max_clki = clki; + clk_disable_unprepare(clki->clk); + list_del(&clki->list); + } else if (!strcmp(clki->name, "ufs_sel_min_src")) { + host->mclk.ufs_sel_min_clki = clki; + clk_disable_unprepare(clki->clk); + list_del(&clki->list); + } else if (!strcmp(clki->name, "ufs_fde")) { + host->mclk.ufs_fde_clki = clki; + } else if (!strcmp(clki->name, "ufs_fde_max_src")) { + host->mclk.ufs_fde_max_clki = clki; + clk_disable_unprepare(clki->clk); + list_del(&clki->list); + } else if (!strcmp(clki->name, "ufs_fde_min_src")) { + host->mclk.ufs_fde_min_clki = clki; + clk_disable_unprepare(clki->clk); + list_del(&clki->list); + } + } + + list_for_each_entry(clki, head, list) { + dev_info(hba->dev, "clk \"%s\" present", clki->name); + } + + if (!ufs_mtk_is_clk_scale_ready(hba)) { + hba->caps &= ~UFSHCD_CAP_CLK_SCALING; + dev_info(hba->dev, + "%s: Clk-scaling not ready. Feature disabled.", + __func__); + return; + } + + /* + * Default get vcore if dts have these settings. + * No matter clock scaling support or not. (may disable by customer) + */ + reg = devm_regulator_get_optional(dev, "dvfsrc-vcore"); + if (IS_ERR(reg)) { + dev_info(dev, "failed to get dvfsrc-vcore: %ld", + PTR_ERR(reg)); + return; + } + + if (of_property_read_u32(dev->of_node, "clk-scale-up-vcore-min", + &volt)) { + dev_info(dev, "failed to get clk-scale-up-vcore-min"); + return; + } + + host->mclk.reg_vcore = reg; + host->mclk.vcore_volt = volt; + + /* If default boot is max gear, request vcore */ + if (reg && volt && host->clk_scale_up) { + if (regulator_set_voltage(reg, volt, INT_MAX)) { + dev_info(hba->dev, + "Failed to set vcore to %d\n", volt); + } + } +} + +#define MAX_VCC_NAME 30 +static int ufs_mtk_vreg_fix_vcc(struct ufs_hba *hba) +{ + struct ufs_vreg_info *info = &hba->vreg_info; + struct device_node *np = hba->dev->of_node; + struct device *dev = hba->dev; + char vcc_name[MAX_VCC_NAME]; + struct arm_smccc_res res; + int err, ver; + + if (info->vcc) + return 0; + + if (of_property_read_bool(np, "mediatek,ufs-vcc-by-num")) { + ufs_mtk_get_vcc_num(res); + if (res.a1 > UFS_VCC_NONE && res.a1 < UFS_VCC_MAX) + snprintf(vcc_name, MAX_VCC_NAME, "vcc-opt%lu", res.a1); + else + return -ENODEV; + } else if (of_property_read_bool(np, "mediatek,ufs-vcc-by-ver")) { + ver = (hba->dev_info.wspecversion & 0xF00) >> 8; + snprintf(vcc_name, MAX_VCC_NAME, "vcc-ufs%u", ver); + } else { + return 0; + } + + err = ufshcd_populate_vreg(dev, vcc_name, &info->vcc, false); + if (err) + return err; + + err = ufshcd_get_vreg(dev, info->vcc); + if (err) + return err; + + err = regulator_enable(info->vcc->reg); + if (!err) { + info->vcc->enabled = true; + dev_info(dev, "%s: %s enabled\n", __func__, vcc_name); + } + + return err; +} + +static void ufs_mtk_vreg_fix_vccqx(struct ufs_hba *hba) +{ + struct ufs_vreg_info *info = &hba->vreg_info; + struct ufs_vreg **vreg_on, **vreg_off; + + if (hba->dev_info.wspecversion >= 0x0300) { + vreg_on = &info->vccq; + vreg_off = &info->vccq2; + } else { + vreg_on = &info->vccq2; + vreg_off = &info->vccq; + } + + if (*vreg_on) + (*vreg_on)->always_on = true; + + if (*vreg_off) { + regulator_disable((*vreg_off)->reg); + devm_kfree(hba->dev, (*vreg_off)->name); + devm_kfree(hba->dev, *vreg_off); + *vreg_off = NULL; + } +} + +static void ufs_mtk_setup_clk_gating(struct ufs_hba *hba) +{ + unsigned long flags; + u32 ah_ms = 10; + u32 ah_scale, ah_timer; + static const u32 scale_us[] = {1, 10, 100, 1000, 10000, 100000}; + + if (ufshcd_is_clkgating_allowed(hba)) { + if (ufshcd_is_auto_hibern8_supported(hba) && hba->ahit) { + ah_scale = FIELD_GET(UFSHCI_AHIBERN8_SCALE_MASK, + hba->ahit); + ah_timer = FIELD_GET(UFSHCI_AHIBERN8_TIMER_MASK, + hba->ahit); + if (ah_scale <= 5) + ah_ms = ah_timer * scale_us[ah_scale] / 1000; + } + + spin_lock_irqsave(hba->host->host_lock, flags); + hba->clk_gating.delay_ms = max(ah_ms, 10U); + spin_unlock_irqrestore(hba->host->host_lock, flags); + } +} + +static void ufs_mtk_fix_ahit(struct ufs_hba *hba) +{ + unsigned int us; + + if (ufshcd_is_auto_hibern8_supported(hba)) { + switch (hba->dev_info.wmanufacturerid) { + case UFS_VENDOR_SAMSUNG: + /* configure auto-hibern8 timer to 3.5 ms */ + us = 3500; + break; + + case UFS_VENDOR_MICRON: + /* configure auto-hibern8 timer to 2 ms */ + us = 2000; + break; + + default: + /* configure auto-hibern8 timer to 1 ms */ + us = 1000; + break; + } + + hba->ahit = ufshcd_us_to_ahit(us); + } + + ufs_mtk_setup_clk_gating(hba); +} + +static void ufs_mtk_fix_clock_scaling(struct ufs_hba *hba) +{ + /* UFS version is below 4.0, clock scaling is not necessary */ + if ((hba->dev_info.wspecversion < 0x0400) && + ufs_mtk_is_clk_scale_ready(hba)) { + hba->caps &= ~UFSHCD_CAP_CLK_SCALING; + + _ufs_mtk_clk_scale(hba, false); + } +} + +static void ufs_mtk_init_mcq_irq(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + struct platform_device *pdev; + int i; + int irq; + + host->mcq_nr_intr = UFSHCD_MAX_Q_NR; + pdev = container_of(hba->dev, struct platform_device, dev); + + if (host->caps & UFS_MTK_CAP_DISABLE_MCQ) + goto failed; + + for (i = 0; i < host->mcq_nr_intr; i++) { + /* irq index 0 is legacy irq, sq/cq irq start from index 1 */ + irq = platform_get_irq(pdev, i + 1); + if (irq < 0) { + host->mcq_intr_info[i].irq = MTK_MCQ_INVALID_IRQ; + goto failed; + } + host->mcq_intr_info[i].hba = hba; + host->mcq_intr_info[i].irq = irq; + dev_info(hba->dev, "get platform mcq irq: %d, %d\n", i, irq); + } + + return; +failed: + /* invalidate irq info */ + for (i = 0; i < host->mcq_nr_intr; i++) + host->mcq_intr_info[i].irq = MTK_MCQ_INVALID_IRQ; + + host->mcq_nr_intr = 0; +} + +/** + * ufs_mtk_init - find other essential mmio bases + * @hba: host controller instance + * + * Binds PHY with controller and powers up PHY enabling clocks + * and regulators. + * + * Return: -EPROBE_DEFER if binding fails, returns negative error + * on phy power up failure and returns zero on success. + */ +static int ufs_mtk_init(struct ufs_hba *hba) +{ + const struct of_device_id *id; + struct device *dev = hba->dev; + struct ufs_mtk_host *host; + struct Scsi_Host *shost = hba->host; + int err = 0; + struct arm_smccc_res res; + + host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL); + if (!host) { + err = -ENOMEM; + dev_info(dev, "%s: no memory for mtk ufs host\n", __func__); + goto out; + } + + host->hba = hba; + ufshcd_set_variant(hba, host); + + id = of_match_device(ufs_mtk_of_match, dev); + if (!id) { + err = -EINVAL; + goto out; + } + + /* Initialize host capability */ + ufs_mtk_init_host_caps(hba); + + ufs_mtk_init_mcq_irq(hba); + + err = ufs_mtk_bind_mphy(hba); + if (err) + goto out_variant_clear; + + ufs_mtk_init_reset(hba); + + /* backup mphy setting if mphy can reset */ + if (host->mphy_reset) + ufs_mtk_mphy_ctrl(UFS_MPHY_BACKUP, res); + + /* Enable runtime autosuspend */ + hba->caps |= UFSHCD_CAP_RPM_AUTOSUSPEND; + + /* Enable clock-gating */ + hba->caps |= UFSHCD_CAP_CLK_GATING; + + /* Enable inline encryption */ + hba->caps |= UFSHCD_CAP_CRYPTO; + + /* Enable WriteBooster */ + hba->caps |= UFSHCD_CAP_WB_EN; + + /* Enable clk scaling*/ + hba->caps |= UFSHCD_CAP_CLK_SCALING; + host->clk_scale_up = true; /* default is max freq */ + + /* Set runtime pm delay to replace default */ + shost->rpm_autosuspend_delay = MTK_RPM_AUTOSUSPEND_DELAY_MS; + + hba->quirks |= UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL; + + hba->quirks |= UFSHCD_QUIRK_MCQ_BROKEN_INTR; + if (host->caps & UFS_MTK_CAP_MCQ_BROKEN_RTC) + hba->quirks |= UFSHCD_QUIRK_MCQ_BROKEN_RTC; + + hba->vps->wb_flush_threshold = UFS_WB_BUF_REMAIN_PERCENT(80); + + if (host->caps & UFS_MTK_CAP_DISABLE_AH8) + hba->caps |= UFSHCD_CAP_HIBERN8_WITH_CLK_GATING; + + if (host->caps & UFS_MTK_CAP_DISABLE_MCQ) + hba->quirks |= UFSHCD_QUIRK_BROKEN_LSDBS_CAP; + + ufs_mtk_init_clocks(hba); + + /* + * ufshcd_vops_init() is invoked after + * ufshcd_setup_clock(true) in ufshcd_hba_init() thus + * phy clock setup is skipped. + * + * Enable phy clocks specifically here. + */ + ufs_mtk_mphy_power_on(hba, true); + + if (ufs_mtk_is_rtff_mtcmos(hba)) { + /* First Restore here, to avoid backup unexpected value */ + ufs_mtk_mtcmos_ctrl(false, res); + + /* Power on to init */ + ufs_mtk_mtcmos_ctrl(true, res); + } + + ufs_mtk_setup_clocks(hba, true, POST_CHANGE); + + ufs_mtk_get_hw_ip_version(hba); + + goto out; + +out_variant_clear: + ufshcd_set_variant(hba, NULL); +out: + return err; +} + +static int ufs_mtk_negotiate_pwr_mode(struct ufs_hba *hba, + const struct ufs_pa_layer_attr *dev_max_params, + struct ufs_pa_layer_attr *dev_req_params) +{ + struct ufs_host_params host_params; + + ufshcd_init_host_params(&host_params); + host_params.hs_rx_gear = UFS_HS_G5; + host_params.hs_tx_gear = UFS_HS_G5; + + if (dev_max_params->pwr_rx == SLOW_MODE || + dev_max_params->pwr_tx == SLOW_MODE) + host_params.desired_working_mode = UFS_PWM_MODE; + + return ufshcd_negotiate_pwr_params(&host_params, dev_max_params, dev_req_params); +} + +static bool ufs_mtk_pmc_via_fastauto(struct ufs_hba *hba, + struct ufs_pa_layer_attr *dev_req_params) +{ + if (!ufs_mtk_is_pmc_via_fastauto(hba)) + return false; + + if (dev_req_params->hs_rate == hba->pwr_info.hs_rate) + return false; + + if (dev_req_params->pwr_tx != FAST_MODE && + dev_req_params->gear_tx < UFS_HS_G4) + return false; + + if (dev_req_params->pwr_rx != FAST_MODE && + dev_req_params->gear_rx < UFS_HS_G4) + return false; + + if (dev_req_params->pwr_tx == SLOW_MODE || + dev_req_params->pwr_rx == SLOW_MODE) + return false; + + return true; +} + +static void ufs_mtk_adjust_sync_length(struct ufs_hba *hba) +{ + int i; + u32 value; + u32 cnt, att, min; + struct attr_min { + u32 attr; + u32 min_value; + } pa_min_sync_length[] = { + {PA_TXHSG1SYNCLENGTH, 0x48}, + {PA_TXHSG2SYNCLENGTH, 0x48}, + {PA_TXHSG3SYNCLENGTH, 0x48}, + {PA_TXHSG4SYNCLENGTH, 0x48}, + {PA_TXHSG5SYNCLENGTH, 0x48} + }; + + cnt = sizeof(pa_min_sync_length) / sizeof(struct attr_min); + for (i = 0; i < cnt; i++) { + att = pa_min_sync_length[i].attr; + min = pa_min_sync_length[i].min_value; + ufshcd_dme_get(hba, UIC_ARG_MIB(att), &value); + if (value < min) + ufshcd_dme_set(hba, UIC_ARG_MIB(att), min); + + ufshcd_dme_peer_get(hba, UIC_ARG_MIB(att), &value); + if (value < min) + ufshcd_dme_peer_set(hba, UIC_ARG_MIB(att), min); + } +} + +static int ufs_mtk_pre_pwr_change(struct ufs_hba *hba, + struct ufs_pa_layer_attr *dev_req_params) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + int ret = 0; + + if (ufs_mtk_pmc_via_fastauto(hba, dev_req_params)) { + ufs_mtk_adjust_sync_length(hba); + + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), true); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), UFS_HS_G1); + + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), true); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), UFS_HS_G1); + + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES), + dev_req_params->lane_tx); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES), + dev_req_params->lane_rx); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES), + dev_req_params->hs_rate); + + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXHSADAPTTYPE), + PA_NO_ADAPT); + + if (!(hba->quirks & UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING)) { + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0), + DL_FC0ProtectionTimeOutVal_Default); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1), + DL_TC0ReplayTimeOutVal_Default); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2), + DL_AFC0ReqTimeOutVal_Default); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3), + DL_FC1ProtectionTimeOutVal_Default); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4), + DL_TC1ReplayTimeOutVal_Default); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5), + DL_AFC1ReqTimeOutVal_Default); + + ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal), + DL_FC0ProtectionTimeOutVal_Default); + ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal), + DL_TC0ReplayTimeOutVal_Default); + ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal), + DL_AFC0ReqTimeOutVal_Default); + } + + ret = ufshcd_uic_change_pwr_mode(hba, + FASTAUTO_MODE << 4 | FASTAUTO_MODE); + + if (ret) { + dev_err(hba->dev, "%s: HSG1B FASTAUTO failed ret=%d\n", + __func__, ret); + } + } + + /* if already configured to the requested pwr_mode, skip adapt */ + if (dev_req_params->gear_rx == hba->pwr_info.gear_rx && + dev_req_params->gear_tx == hba->pwr_info.gear_tx && + dev_req_params->lane_rx == hba->pwr_info.lane_rx && + dev_req_params->lane_tx == hba->pwr_info.lane_tx && + dev_req_params->pwr_rx == hba->pwr_info.pwr_rx && + dev_req_params->pwr_tx == hba->pwr_info.pwr_tx && + dev_req_params->hs_rate == hba->pwr_info.hs_rate) { + return ret; + } + + if (dev_req_params->pwr_rx == FAST_MODE || + dev_req_params->pwr_rx == FASTAUTO_MODE) { + if (host->hw_ver.major >= 3) { + ret = ufshcd_dme_configure_adapt(hba, + dev_req_params->gear_tx, + PA_INITIAL_ADAPT); + } else { + ret = ufshcd_dme_configure_adapt(hba, + dev_req_params->gear_tx, + PA_NO_ADAPT); + } + } else { + ret = ufshcd_dme_configure_adapt(hba, + dev_req_params->gear_tx, + PA_NO_ADAPT); + } + + return ret; +} + +static int ufs_mtk_auto_hibern8_disable(struct ufs_hba *hba) +{ + int ret; + + /* disable auto-hibern8 */ + ufshcd_writel(hba, 0, REG_AUTO_HIBERNATE_IDLE_TIMER); + + /* wait host return to idle state when auto-hibern8 off */ + ret = ufs_mtk_wait_idle_state(hba, 5); + if (ret) + goto out; + + ret = ufs_mtk_wait_link_state(hba, VS_LINK_UP, 100); + +out: + if (ret) { + dev_warn(hba->dev, "exit h8 state fail, ret=%d\n", ret); + + ufshcd_force_error_recovery(hba); + + /* trigger error handler and break suspend */ + ret = -EBUSY; + } + + return ret; +} + +static int ufs_mtk_pwr_change_notify(struct ufs_hba *hba, + enum ufs_notify_change_status stage, + struct ufs_pa_layer_attr *dev_req_params) +{ + int ret = 0; + static u32 reg; + + switch (stage) { + case PRE_CHANGE: + if (ufshcd_is_auto_hibern8_supported(hba)) { + reg = ufshcd_readl(hba, REG_AUTO_HIBERNATE_IDLE_TIMER); + ufs_mtk_auto_hibern8_disable(hba); + } + ret = ufs_mtk_pre_pwr_change(hba, dev_req_params); + break; + case POST_CHANGE: + if (ufshcd_is_auto_hibern8_supported(hba)) + ufshcd_writel(hba, reg, REG_AUTO_HIBERNATE_IDLE_TIMER); + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + +static int ufs_mtk_unipro_set_lpm(struct ufs_hba *hba, bool lpm) +{ + int ret; + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + ret = ufshcd_dme_set(hba, + UIC_ARG_MIB_SEL(VS_UNIPROPOWERDOWNCONTROL, 0), + lpm ? 1 : 0); + if (!ret || !lpm) { + /* + * Forcibly set as non-LPM mode if UIC commands is failed + * to use default hba_enable_delay_us value for re-enabling + * the host. + */ + host->unipro_lpm = lpm; + } + + return ret; +} + +static int ufs_mtk_pre_link(struct ufs_hba *hba) +{ + int ret; + u32 tmp; + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + ufs_mtk_get_controller_version(hba); + + ret = ufs_mtk_unipro_set_lpm(hba, false); + if (ret) + return ret; + + /* + * Setting PA_Local_TX_LCC_Enable to 0 before link startup + * to make sure that both host and device TX LCC are disabled + * once link startup is completed. + */ + ret = ufshcd_disable_host_tx_lcc(hba); + if (ret) + return ret; + + /* disable deep stall */ + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(VS_SAVEPOWERCONTROL), &tmp); + if (ret) + return ret; + + tmp &= ~(1 << 6); + + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(VS_SAVEPOWERCONTROL), tmp); + + /* Enable the 1144 functions setting */ + if (host->ip_ver == IP_VER_MT6989) { + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(VS_DEBUGOMC), &tmp); + if (ret) + return ret; + + tmp |= 0x10; + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(VS_DEBUGOMC), tmp); + } + + return ret; +} + +static void ufs_mtk_post_link(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + u32 tmp; + + /* fix device PA_INIT no adapt */ + if (host->ip_ver >= IP_VER_MT6899) { + ufshcd_dme_get(hba, UIC_ARG_MIB(VS_DEBUGOMC), &tmp); + tmp |= 0x100; + ufshcd_dme_set(hba, UIC_ARG_MIB(VS_DEBUGOMC), tmp); + } + + /* enable unipro clock gating feature */ + ufs_mtk_cfg_unipro_cg(hba, true); +} + +static int ufs_mtk_link_startup_notify(struct ufs_hba *hba, + enum ufs_notify_change_status stage) +{ + int ret = 0; + + switch (stage) { + case PRE_CHANGE: + ret = ufs_mtk_pre_link(hba); + break; + case POST_CHANGE: + ufs_mtk_post_link(hba); + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + +static int ufs_mtk_device_reset(struct ufs_hba *hba) +{ + struct arm_smccc_res res; + + ufs_mtk_device_reset_ctrl(0, res); + + /* disable hba in middle of device reset */ + ufshcd_hba_stop(hba); + + /* + * The reset signal is active low. UFS devices shall detect + * more than or equal to 1us of positive or negative RST_n + * pulse width. + * + * To be on safe side, keep the reset low for at least 10us. + */ + usleep_range(10, 15); + + ufs_mtk_device_reset_ctrl(1, res); + + /* Some devices may need time to respond to rst_n */ + usleep_range(10000, 15000); + + dev_info(hba->dev, "device reset done\n"); + + return 0; +} + +static int ufs_mtk_link_set_hpm(struct ufs_hba *hba) +{ + int err; + u32 val; + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + err = ufshcd_hba_enable(hba); + if (err) + return err; + + err = ufs_mtk_unipro_set_lpm(hba, false); + if (err) { + if (host->ip_ver < IP_VER_MT6899) { + ufs_mtk_dbg_sel(hba); + val = ufshcd_readl(hba, REG_UFS_PROBE); + } else { + val = ufshcd_readl(hba, REG_UFS_UFS_MMIO_OTSD_CTRL); + } + ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)val); + val = ufshcd_readl(hba, REG_INTERRUPT_STATUS); + ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)val); + return err; + } + + err = ufshcd_uic_hibern8_exit(hba); + if (err) + return err; + + /* Check link state to make sure exit h8 success */ + err = ufs_mtk_wait_idle_state(hba, 5); + if (err) { + dev_warn(hba->dev, "wait idle fail, err=%d\n", err); + return err; + } + err = ufs_mtk_wait_link_state(hba, VS_LINK_UP, 100); + if (err) { + dev_warn(hba->dev, "exit h8 state fail, err=%d\n", err); + return err; + } + ufshcd_set_link_active(hba); + + err = ufshcd_make_hba_operational(hba); + if (err) + return err; + + if (hba->mcq_enabled) { + ufs_mtk_config_mcq(hba, false); + ufshcd_mcq_make_queues_operational(hba); + ufshcd_mcq_config_mac(hba, hba->nutrs); + ufshcd_mcq_enable(hba); + } + + return 0; +} + +static int ufs_mtk_link_set_lpm(struct ufs_hba *hba) +{ + int err; + + /* Disable reset confirm feature by UniPro */ + ufshcd_writel(hba, + (ufshcd_readl(hba, REG_UFS_XOUFS_CTRL) & ~0x100), + REG_UFS_XOUFS_CTRL); + + err = ufs_mtk_unipro_set_lpm(hba, true); + if (err) { + /* Resume UniPro state for following error recovery */ + ufs_mtk_unipro_set_lpm(hba, false); + return err; + } + + return 0; +} + +static void ufs_mtk_vccqx_set_lpm(struct ufs_hba *hba, bool lpm) +{ + struct ufs_vreg *vccqx = NULL; + + if (!hba->vreg_info.vccq && !hba->vreg_info.vccq2) + return; + + if (hba->vreg_info.vccq) + vccqx = hba->vreg_info.vccq; + else + vccqx = hba->vreg_info.vccq2; + + regulator_set_mode(vccqx->reg, + lpm ? REGULATOR_MODE_IDLE : REGULATOR_MODE_NORMAL); +} + +static void ufs_mtk_vsx_set_lpm(struct ufs_hba *hba, bool lpm) +{ + struct arm_smccc_res res; + + ufs_mtk_device_pwr_ctrl(!lpm, + (unsigned long)hba->dev_info.wspecversion, + res); +} + +static void ufs_mtk_dev_vreg_set_lpm(struct ufs_hba *hba, bool lpm) +{ + bool skip_vccqx = false; + + /* Prevent entering LPM when device is still active */ + if (lpm && ufshcd_is_ufs_dev_active(hba)) + return; + + /* Skip vccqx lpm control and control vsx only */ + if (!hba->vreg_info.vccq && !hba->vreg_info.vccq2) + skip_vccqx = true; + + /* VCC is always-on, control vsx only */ + if (!hba->vreg_info.vcc) + skip_vccqx = true; + + /* Broken vcc keep vcc always on, most case control vsx only */ + if (lpm && hba->vreg_info.vcc && hba->vreg_info.vcc->enabled) { + /* Some device vccqx/vsx can enter lpm */ + if (ufs_mtk_is_allow_vccqx_lpm(hba)) + skip_vccqx = false; + else /* control vsx only */ + skip_vccqx = true; + } + + if (lpm) { + if (!skip_vccqx) + ufs_mtk_vccqx_set_lpm(hba, lpm); + ufs_mtk_vsx_set_lpm(hba, lpm); + } else { + ufs_mtk_vsx_set_lpm(hba, lpm); + if (!skip_vccqx) + ufs_mtk_vccqx_set_lpm(hba, lpm); + } +} + +static int ufs_mtk_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op, + enum ufs_notify_change_status status) +{ + int err; + struct arm_smccc_res res; + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + if (status == PRE_CHANGE) { + if (ufshcd_is_auto_hibern8_supported(hba)) + return ufs_mtk_auto_hibern8_disable(hba); + return 0; + } + + if (ufshcd_is_link_hibern8(hba)) { + err = ufs_mtk_link_set_lpm(hba); + if (err) + goto fail; + } + + if (!ufshcd_is_link_active(hba)) { + /* + * Make sure no error will be returned to prevent + * ufshcd_suspend() re-enabling regulators while vreg is still + * in low-power mode. + */ + err = ufs_mtk_mphy_power_on(hba, false); + if (err) + goto fail; + } + + if (ufshcd_is_link_off(hba)) + ufs_mtk_device_reset_ctrl(0, res); + + ufs_mtk_sram_pwr_ctrl(false, res); + + /* Release pm_qos/clk if in scale-up mode during suspend */ + if (ufshcd_is_clkscaling_supported(hba) && (host->clk_scale_up)) { + ufshcd_pm_qos_update(hba, false); + _ufs_mtk_clk_scale(hba, false); + } else if ((!ufshcd_is_clkscaling_supported(hba) && + hba->pwr_info.gear_rx >= UFS_HS_G5)) { + _ufs_mtk_clk_scale(hba, false); + } + + return 0; +fail: + /* + * Set link as off state enforcedly to trigger + * ufshcd_host_reset_and_restore() in ufshcd_suspend() + * for completed host reset. + */ + ufshcd_set_link_off(hba); + return -EAGAIN; +} + +static int ufs_mtk_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) +{ + int err; + struct arm_smccc_res res; + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) + ufs_mtk_dev_vreg_set_lpm(hba, false); + + ufs_mtk_sram_pwr_ctrl(true, res); + + err = ufs_mtk_mphy_power_on(hba, true); + if (err) + goto fail; + + /* Request pm_qos/clk if in scale-up mode after resume */ + if (ufshcd_is_clkscaling_supported(hba) && (host->clk_scale_up)) { + ufshcd_pm_qos_update(hba, true); + _ufs_mtk_clk_scale(hba, true); + } else if ((!ufshcd_is_clkscaling_supported(hba) && + hba->pwr_info.gear_rx >= UFS_HS_G5)) { + _ufs_mtk_clk_scale(hba, true); + } + + if (ufshcd_is_link_hibern8(hba)) { + err = ufs_mtk_link_set_hpm(hba); + if (err) + goto fail; + } + + return 0; + +fail: + /* + * Check if the platform (parent) device has resumed, and ensure that + * power, clock, and MTCMOS are all turned on. + */ + err = ufshcd_link_recovery(hba); + if (err) { + dev_err(hba->dev, "Device PM: req=%d, status:%d, err:%d\n", + hba->dev->power.request, + hba->dev->power.runtime_status, + hba->dev->power.runtime_error); + } + + return 0; /* Cannot return a failure, otherwise, the I/O will hang. */ +} + +static void ufs_mtk_dbg_register_dump(struct ufs_hba *hba) +{ + /* Dump ufshci register 0x140 ~ 0x14C */ + ufshcd_dump_regs(hba, REG_UFS_XOUFS_CTRL, 0x10, + "XOUFS Ctrl (0x140): "); + + ufshcd_dump_regs(hba, REG_UFS_EXTREG, 0x4, "Ext Reg "); + + /* Dump ufshci register 0x2200 ~ 0x22AC */ + ufshcd_dump_regs(hba, REG_UFS_MPHYCTRL, + REG_UFS_REJECT_MON - REG_UFS_MPHYCTRL + 4, + "MPHY Ctrl (0x2200): "); + + /* Direct debugging information to REG_MTK_PROBE */ + ufs_mtk_dbg_sel(hba); + ufshcd_dump_regs(hba, REG_UFS_PROBE, 0x4, "Debug Probe "); +} + +static int ufs_mtk_apply_dev_quirks(struct ufs_hba *hba) +{ + struct ufs_dev_info *dev_info = &hba->dev_info; + u16 mid = dev_info->wmanufacturerid; + unsigned int cpu; + + if (hba->mcq_enabled) { + /* Iterate all cpus to set affinity for mcq irqs */ + for (cpu = 0; cpu < nr_cpu_ids; cpu++) + ufs_mtk_mcq_set_irq_affinity(hba, cpu); + } + + if (mid == UFS_VENDOR_SAMSUNG) { + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 6); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), 10); + } else if (mid == UFS_VENDOR_MICRON) { + /* Only for the host which have TX skew issue */ + if (ufs_mtk_is_tx_skew_fix(hba) && + (STR_PRFX_EQUAL("MT128GBCAV2U31", dev_info->model) || + STR_PRFX_EQUAL("MT256GBCAV4U31", dev_info->model) || + STR_PRFX_EQUAL("MT512GBCAV8U31", dev_info->model) || + STR_PRFX_EQUAL("MT256GBEAX4U40", dev_info->model) || + STR_PRFX_EQUAL("MT512GAYAX4U40", dev_info->model) || + STR_PRFX_EQUAL("MT001TAYAX8U40", dev_info->model))) { + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 8); + } + } + + /* + * Decide waiting time before gating reference clock and + * after ungating reference clock according to vendors' + * requirements. + */ + if (mid == UFS_VENDOR_SAMSUNG) + ufs_mtk_setup_ref_clk_wait_us(hba, 1); + else if (mid == UFS_VENDOR_SKHYNIX) + ufs_mtk_setup_ref_clk_wait_us(hba, 30); + else if (mid == UFS_VENDOR_TOSHIBA) + ufs_mtk_setup_ref_clk_wait_us(hba, 100); + else + ufs_mtk_setup_ref_clk_wait_us(hba, + REFCLK_DEFAULT_WAIT_US); + return 0; +} + +static void ufs_mtk_fixup_dev_quirks(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + ufshcd_fixup_dev_quirks(hba, ufs_mtk_dev_fixups); + + if (ufs_mtk_is_broken_vcc(hba) && hba->vreg_info.vcc) { + hba->vreg_info.vcc->always_on = true; + /* + * VCC will be kept always-on thus we don't + * need any delay before putting device's VCC in LPM mode. + */ + hba->dev_quirks &= ~UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM; + } + + /* + * Add a delay after enabling UFS5 VCC to ensure the voltage + * is stable before the refclk is enabled. + */ + if (hba->dev_info.wspecversion >= 0x0500 && + (host->ip_ver == IP_VER_MT6995_A0 || + host->ip_ver == IP_VER_MT6995_B0)) + hba->quirks |= UFSHCD_QUIRK_VCC_ON_DELAY; + + ufs_mtk_vreg_fix_vcc(hba); + ufs_mtk_vreg_fix_vccqx(hba); + ufs_mtk_fix_ahit(hba); + ufs_mtk_fix_clock_scaling(hba); +} + +static void ufs_mtk_event_notify(struct ufs_hba *hba, + enum ufs_event_type evt, void *data) +{ + unsigned int val = *(u32 *)data; + unsigned long reg; + u8 bit; + + trace_ufs_mtk_event(evt, val); + + /* Print details of UIC Errors */ + if (evt <= UFS_EVT_DME_ERR) { + dev_info(hba->dev, + "Host UIC Error Code (%s): %08x\n", + ufs_uic_err_str[evt], val); + reg = val; + } + + if (evt == UFS_EVT_PA_ERR) { + for_each_set_bit(bit, ®, ARRAY_SIZE(ufs_uic_pa_err_str)) + dev_info(hba->dev, "%s\n", ufs_uic_pa_err_str[bit]); + } + + if (evt == UFS_EVT_DL_ERR) { + for_each_set_bit(bit, ®, ARRAY_SIZE(ufs_uic_dl_err_str)) + dev_info(hba->dev, "%s\n", ufs_uic_dl_err_str[bit]); + } +} + +static void ufs_mtk_config_scaling_param(struct ufs_hba *hba, + struct devfreq_dev_profile *profile, + struct devfreq_simple_ondemand_data *data) +{ + /* Customize min gear in clk scaling */ + hba->clk_scaling.min_gear = UFS_HS_G4; + + hba->vps->devfreq_profile.polling_ms = 200; + hba->vps->ondemand_data.upthreshold = 50; + hba->vps->ondemand_data.downdifferential = 20; +} + +static void _ufs_mtk_clk_scale(struct ufs_hba *hba, bool scale_up) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + struct ufs_mtk_clk *mclk = &host->mclk; + struct ufs_clk_info *clki = mclk->ufs_sel_clki; + struct ufs_clk_info *fde_clki = mclk->ufs_fde_clki; + struct regulator *reg; + int volt, ret = 0; + bool clk_bind_vcore = false; + bool clk_fde_scale = false; + + if (!hba->clk_scaling.is_initialized) + return; + + if (!clki || !fde_clki) + return; + + reg = host->mclk.reg_vcore; + volt = host->mclk.vcore_volt; + if (reg && volt != 0) + clk_bind_vcore = true; + + if (mclk->ufs_fde_max_clki && mclk->ufs_fde_min_clki) + clk_fde_scale = true; + + ret = clk_prepare_enable(clki->clk); + if (ret) { + dev_info(hba->dev, + "clk_prepare_enable() fail, ret: %d\n", ret); + return; + } + + if (clk_fde_scale) { + ret = clk_prepare_enable(fde_clki->clk); + if (ret) { + dev_info(hba->dev, + "fde clk_prepare_enable() fail, ret: %d\n", ret); + return; + } + } + + if (scale_up) { + if (clk_bind_vcore) { + ret = regulator_set_voltage(reg, volt, INT_MAX); + if (ret) { + dev_info(hba->dev, + "Failed to set vcore to %d\n", volt); + goto out; + } + } + + ret = clk_set_parent(clki->clk, mclk->ufs_sel_max_clki->clk); + if (ret) { + dev_info(hba->dev, "Failed to set clk mux, ret = %d\n", + ret); + } + + if (clk_fde_scale) { + ret = clk_set_parent(fde_clki->clk, + mclk->ufs_fde_max_clki->clk); + if (ret) { + dev_info(hba->dev, + "Failed to set fde clk mux, ret = %d\n", + ret); + } + } + } else { + if (clk_fde_scale) { + ret = clk_set_parent(fde_clki->clk, + mclk->ufs_fde_min_clki->clk); + if (ret) { + dev_info(hba->dev, + "Failed to set fde clk mux, ret = %d\n", + ret); + goto out; + } + } + + ret = clk_set_parent(clki->clk, mclk->ufs_sel_min_clki->clk); + if (ret) { + dev_info(hba->dev, "Failed to set clk mux, ret = %d\n", + ret); + goto out; + } + + if (clk_bind_vcore) { + ret = regulator_set_voltage(reg, 0, INT_MAX); + if (ret) { + dev_info(hba->dev, + "failed to set vcore to MIN\n"); + } + } + } + +out: + clk_disable_unprepare(clki->clk); + + if (clk_fde_scale) + clk_disable_unprepare(fde_clki->clk); +} + +/** + * ufs_mtk_clk_scale - Internal clk scaling operation + * + * MTK platform supports clk scaling by switching parent of ufs_sel(mux). + * The ufs_sel downstream to ufs_ck which feeds directly to UFS hardware. + * Max and min clocks rate of ufs_sel defined in dts should match rate of + * "ufs_sel_max_src" and "ufs_sel_min_src" respectively. + * This prevent changing rate of pll clock that is shared between modules. + * + * @hba: per adapter instance + * @scale_up: True for scaling up and false for scaling down + */ +static void ufs_mtk_clk_scale(struct ufs_hba *hba, bool scale_up) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + struct ufs_mtk_clk *mclk = &host->mclk; + struct ufs_clk_info *clki = mclk->ufs_sel_clki; + + if (host->clk_scale_up == scale_up) + goto out; + + if (scale_up) + _ufs_mtk_clk_scale(hba, true); + else + _ufs_mtk_clk_scale(hba, false); + + host->clk_scale_up = scale_up; + + /* Must always set before clk_set_rate() */ + if (scale_up) + clki->curr_freq = clki->max_freq; + else + clki->curr_freq = clki->min_freq; +out: + trace_ufs_mtk_clk_scale(clki->name, scale_up, clk_get_rate(clki->clk)); +} + +static int ufs_mtk_clk_scale_notify(struct ufs_hba *hba, bool scale_up, + unsigned long target_freq, + enum ufs_notify_change_status status) +{ + if (!ufshcd_is_clkscaling_supported(hba)) + return 0; + + if (status == PRE_CHANGE) { + /* Switch parent before clk_set_rate() */ + ufs_mtk_clk_scale(hba, scale_up); + } else { + /* Request interrupt latency QoS accordingly */ + ufs_mtk_scale_perf(hba, scale_up); + } + + return 0; +} + +static int ufs_mtk_get_hba_mac(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + /* MCQ operation not permitted */ + if (host->caps & UFS_MTK_CAP_DISABLE_MCQ) + return -EPERM; + + return MAX_SUPP_MAC; +} + +static int ufs_mtk_op_runtime_config(struct ufs_hba *hba) +{ + struct ufshcd_mcq_opr_info_t *opr; + int i; + + hba->mcq_opr[OPR_SQD].offset = REG_UFS_MTK_SQD; + hba->mcq_opr[OPR_SQIS].offset = REG_UFS_MTK_SQIS; + hba->mcq_opr[OPR_CQD].offset = REG_UFS_MTK_CQD; + hba->mcq_opr[OPR_CQIS].offset = REG_UFS_MTK_CQIS; + + for (i = 0; i < OPR_MAX; i++) { + opr = &hba->mcq_opr[i]; + opr->stride = REG_UFS_MCQ_STRIDE; + opr->base = hba->mmio_base + opr->offset; + } + + return 0; +} + +static int ufs_mtk_mcq_config_resource(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + + /* fail mcq initialization if interrupt is not filled properly */ + if (!host->mcq_nr_intr) { + dev_info(hba->dev, "IRQs not ready. MCQ disabled."); + return -EINVAL; + } + + hba->mcq_base = hba->mmio_base + MCQ_QUEUE_OFFSET(hba->mcq_capabilities); + return 0; +} + +static irqreturn_t ufs_mtk_mcq_intr(int irq, void *__intr_info) +{ + struct ufs_mtk_mcq_intr_info *mcq_intr_info = __intr_info; + struct ufs_hba *hba = mcq_intr_info->hba; + struct ufs_hw_queue *hwq; + u32 events; + int qid = mcq_intr_info->qid; + + hwq = &hba->uhq[qid]; + + events = ufshcd_mcq_read_cqis(hba, qid); + if (events) + ufshcd_mcq_write_cqis(hba, events, qid); + + if (events & UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS) + ufshcd_mcq_poll_cqe_lock(hba, hwq); + + return IRQ_HANDLED; +} + +static int ufs_mtk_config_mcq_irq(struct ufs_hba *hba) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + u32 irq, i; + int ret; + + for (i = 0; i < host->mcq_nr_intr; i++) { + irq = host->mcq_intr_info[i].irq; + if (irq == MTK_MCQ_INVALID_IRQ) { + dev_err(hba->dev, "invalid irq. %d\n", i); + return -ENOPARAM; + } + + host->mcq_intr_info[i].qid = i; + ret = devm_request_irq(hba->dev, irq, ufs_mtk_mcq_intr, 0, UFSHCD, + &host->mcq_intr_info[i]); + + dev_dbg(hba->dev, "request irq %d intr %s\n", irq, ret ? "failed" : ""); + + if (ret) { + dev_err(hba->dev, "Cannot request irq %d\n", ret); + return ret; + } + } + host->is_mcq_intr_enabled = true; + + return 0; +} + +static int ufs_mtk_config_mcq(struct ufs_hba *hba, bool irq) +{ + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + int ret = 0; + + if (!host->mcq_set_intr) { + /* Disable irq option register */ + ufshcd_rmwl(hba, MCQ_INTR_EN_MSK, 0, REG_UFS_MMIO_OPT_CTRL_0); + + if (irq) { + ret = ufs_mtk_config_mcq_irq(hba); + if (ret) + return ret; + } + + host->mcq_set_intr = true; + } + + ufshcd_rmwl(hba, MCQ_AH8, MCQ_AH8, REG_UFS_MMIO_OPT_CTRL_0); + ufshcd_rmwl(hba, MCQ_INTR_EN_MSK, MCQ_MULTI_INTR_EN, REG_UFS_MMIO_OPT_CTRL_0); + + return 0; +} + +static int ufs_mtk_config_esi(struct ufs_hba *hba) +{ + return ufs_mtk_config_mcq(hba, true); +} + +static void ufs_mtk_config_scsi_dev(struct scsi_device *sdev) +{ + struct ufs_hba *hba = shost_priv(sdev->host); + + dev_dbg(hba->dev, "lu %llu scsi device configured", sdev->lun); + if (sdev->lun == 2) + blk_queue_flag_set(QUEUE_FLAG_SAME_FORCE, sdev->request_queue); +} + +/* + * struct ufs_hba_mtk_vops - UFS MTK specific variant operations + * + * The variant operations configure the necessary controller and PHY + * handshake during initialization. + */ +static const struct ufs_hba_variant_ops ufs_hba_mtk_vops = { + .name = "mediatek.ufshci", + .max_num_rtt = MTK_MAX_NUM_RTT, + .init = ufs_mtk_init, + .get_ufs_hci_version = ufs_mtk_get_ufs_hci_version, + .setup_clocks = ufs_mtk_setup_clocks, + .hce_enable_notify = ufs_mtk_hce_enable_notify, + .link_startup_notify = ufs_mtk_link_startup_notify, + .negotiate_pwr_mode = ufs_mtk_negotiate_pwr_mode, + .pwr_change_notify = ufs_mtk_pwr_change_notify, + .apply_dev_quirks = ufs_mtk_apply_dev_quirks, + .fixup_dev_quirks = ufs_mtk_fixup_dev_quirks, + .suspend = ufs_mtk_suspend, + .resume = ufs_mtk_resume, + .dbg_register_dump = ufs_mtk_dbg_register_dump, + .device_reset = ufs_mtk_device_reset, + .event_notify = ufs_mtk_event_notify, + .config_scaling_param = ufs_mtk_config_scaling_param, + .clk_scale_notify = ufs_mtk_clk_scale_notify, + /* mcq vops */ + .get_hba_mac = ufs_mtk_get_hba_mac, + .op_runtime_config = ufs_mtk_op_runtime_config, + .mcq_config_resource = ufs_mtk_mcq_config_resource, + .config_esi = ufs_mtk_config_esi, + .config_scsi_dev = ufs_mtk_config_scsi_dev, +}; + +/** + * ufs_mtk_probe - probe routine of the driver + * @pdev: pointer to Platform device handle + * + * Return: zero for success and non-zero for failure. + */ +static int ufs_mtk_probe(struct platform_device *pdev) +{ + int err; + struct device *dev = &pdev->dev, *phy_dev = NULL; + struct device_node *reset_node, *phy_node = NULL; + struct platform_device *reset_pdev, *phy_pdev = NULL; + struct device_link *link; + struct ufs_hba *hba; + struct ufs_mtk_host *host; + + reset_node = of_find_compatible_node(NULL, NULL, + "ti,syscon-reset"); + if (!reset_node) { + dev_notice(dev, "find ti,syscon-reset fail\n"); + goto skip_reset; + } + reset_pdev = of_find_device_by_node(reset_node); + if (!reset_pdev) { + dev_notice(dev, "find reset_pdev fail\n"); + goto skip_reset; + } + link = device_link_add(dev, &reset_pdev->dev, + DL_FLAG_AUTOPROBE_CONSUMER); + put_device(&reset_pdev->dev); + if (!link) { + dev_notice(dev, "add reset device_link fail\n"); + goto skip_reset; + } + /* supplier is not probed */ + if (link->status == DL_STATE_DORMANT) { + err = -EPROBE_DEFER; + goto out; + } + +skip_reset: + /* find phy node */ + phy_node = of_parse_phandle(dev->of_node, "phys", 0); + + if (phy_node) { + phy_pdev = of_find_device_by_node(phy_node); + if (!phy_pdev) + goto skip_phy; + phy_dev = &phy_pdev->dev; + + pm_runtime_set_active(phy_dev); + pm_runtime_enable(phy_dev); + pm_runtime_get_sync(phy_dev); + + put_device(phy_dev); + dev_info(dev, "phys node found\n"); + } else { + dev_notice(dev, "phys node not found\n"); + } + +skip_phy: + /* perform generic probe */ + err = ufshcd_pltfrm_init(pdev, &ufs_hba_mtk_vops); + if (err) { + dev_err(dev, "probe failed %d\n", err); + goto out; + } + + hba = platform_get_drvdata(pdev); + if (!hba) + goto out; + + if (phy_node && phy_dev) { + host = ufshcd_get_variant(hba); + host->phy_dev = phy_dev; + } + + /* + * Because the default power setting of VSx (the upper layer of + * VCCQ/VCCQ2) is HWLP, we need to prevent VCCQ/VCCQ2 from + * entering LPM. + */ + ufs_mtk_dev_vreg_set_lpm(hba, false); + +out: + of_node_put(phy_node); + of_node_put(reset_node); + return err; +} + +/** + * ufs_mtk_remove - set driver_data of the device to NULL + * @pdev: pointer to platform device handle + * + * Always return 0 + */ +static void ufs_mtk_remove(struct platform_device *pdev) +{ + ufshcd_pltfrm_remove(pdev); +} + +static int ufs_mtk_system_suspend(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + struct arm_smccc_res res; + int ret; + + if (hba->shutting_down) { + ret = -EBUSY; + goto out; + } + + ret = ufshcd_system_suspend(dev); + if (ret) + goto out; + + if (pm_runtime_suspended(hba->dev)) + goto out; + + ufs_mtk_dev_vreg_set_lpm(hba, true); + + if (ufs_mtk_is_rtff_mtcmos(hba)) + ufs_mtk_mtcmos_ctrl(false, res); + +out: + return ret; +} + +static int ufs_mtk_system_resume(struct device *dev) +{ + int ret = 0; + struct ufs_hba *hba = dev_get_drvdata(dev); + struct arm_smccc_res res; + + if (pm_runtime_suspended(hba->dev)) + goto out; + + if (ufs_mtk_is_rtff_mtcmos(hba)) + ufs_mtk_mtcmos_ctrl(true, res); + + ufs_mtk_dev_vreg_set_lpm(hba, false); + +out: + ret = ufshcd_system_resume(dev); + + return ret; +} + +static int ufs_mtk_runtime_suspend(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + struct arm_smccc_res res; + int ret = 0; + + ret = ufshcd_runtime_suspend(dev); + if (ret) + return ret; + + ufs_mtk_dev_vreg_set_lpm(hba, true); + + if (ufs_mtk_is_rtff_mtcmos(hba)) + ufs_mtk_mtcmos_ctrl(false, res); + + if (host->phy_dev) + pm_runtime_put_sync(host->phy_dev); + + return 0; +} + +static int ufs_mtk_runtime_resume(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + struct ufs_mtk_host *host = ufshcd_get_variant(hba); + struct arm_smccc_res res; + + if (ufs_mtk_is_rtff_mtcmos(hba)) + ufs_mtk_mtcmos_ctrl(true, res); + + if (host->phy_dev) + pm_runtime_get_sync(host->phy_dev); + + ufs_mtk_dev_vreg_set_lpm(hba, false); + + return ufshcd_runtime_resume(dev); +} + +static const struct dev_pm_ops ufs_mtk_pm_ops = { + SYSTEM_SLEEP_PM_OPS(ufs_mtk_system_suspend, ufs_mtk_system_resume) + RUNTIME_PM_OPS(ufs_mtk_runtime_suspend, ufs_mtk_runtime_resume, NULL) + .prepare = ufshcd_suspend_prepare, + .complete = ufshcd_resume_complete, +}; + +static struct platform_driver ufs_mtk_pltform = { + .probe = ufs_mtk_probe, + .remove = ufs_mtk_remove, + .driver = { + .name = "ufshcd-mtk", + .pm = pm_ptr(&ufs_mtk_pm_ops), + .of_match_table = ufs_mtk_of_match, + }, +}; + +MODULE_AUTHOR("Stanley Chu <stanley.chu@mediatek.com>"); +MODULE_AUTHOR("Peter Wang <peter.wang@mediatek.com>"); +MODULE_DESCRIPTION("MediaTek UFS Host Driver"); +MODULE_LICENSE("GPL v2"); + +module_platform_driver(ufs_mtk_pltform); diff --git a/drivers/ufs/host/ufs-mediatek.h b/drivers/ufs/host/ufs-mediatek.h new file mode 100644 index 000000000..8547a6f04 --- /dev/null +++ b/drivers/ufs/host/ufs-mediatek.h @@ -0,0 +1,236 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2019 MediaTek Inc. + */ + +#ifndef _UFS_MEDIATEK_H +#define _UFS_MEDIATEK_H + +#include <linux/bitops.h> + +/* + * MCQ define and struct + */ +#define UFSHCD_MAX_Q_NR 8 +#define MTK_MCQ_INVALID_IRQ 0xFFFF + +/* REG_UFS_MMIO_OPT_CTRL_0 160h */ +#define EHS_EN BIT(0) +#define PFM_IMPV BIT(1) +#define MCQ_MULTI_INTR_EN BIT(2) +#define MCQ_CMB_INTR_EN BIT(3) +#define MCQ_AH8 BIT(4) +#define MON_EN BIT(5) +#define MRTT_EN BIT(25) +#define RDN_PFM_IMPV_DIS BIT(28) + +#define MCQ_INTR_EN_MSK (MCQ_MULTI_INTR_EN | MCQ_CMB_INTR_EN) + +/* + * Vendor specific UFSHCI Registers + */ +#define REG_UFS_XOUFS_CTRL 0x140 +#define REG_UFS_REFCLK_CTRL 0x144 +#define REG_UFS_UFS_MMIO_OTSD_CTRL 0x14C +#define REG_UFS_MMIO_OPT_CTRL_0 0x160 +#define REG_UFS_EXTREG 0x2100 +#define REG_UFS_MPHYCTRL 0x2200 +#define REG_UFS_MTK_IP_VER 0x2240 +#define REG_UFS_REJECT_MON 0x22AC +#define REG_UFS_DEBUG_SEL 0x22C0 +#define REG_UFS_PROBE 0x22C8 +#define REG_UFS_DEBUG_SEL_B0 0x22D0 +#define REG_UFS_DEBUG_SEL_B1 0x22D4 +#define REG_UFS_DEBUG_SEL_B2 0x22D8 +#define REG_UFS_DEBUG_SEL_B3 0x22DC + +#define REG_UFS_MTK_SQD 0x2800 +#define REG_UFS_MTK_SQIS 0x2814 +#define REG_UFS_MTK_CQD 0x281C +#define REG_UFS_MTK_CQIS 0x2824 + +#define REG_UFS_MCQ_STRIDE 0x30 + +/* + * Ref-clk control + * + * Values for register REG_UFS_REFCLK_CTRL + */ +#define REFCLK_RELEASE 0x0 +#define REFCLK_REQUEST BIT(0) +#define REFCLK_ACK BIT(1) + +#define REFCLK_REQ_TIMEOUT_US 3000 +#define REFCLK_DEFAULT_WAIT_US 32 + +/* + * Other attributes + */ +#define VS_DEBUGCLOCKENABLE 0xD0A1 +#define VS_SAVEPOWERCONTROL 0xD0A6 +#define VS_UNIPROPOWERDOWNCONTROL 0xD0A8 + +/* + * Vendor specific link state + */ +enum { + VS_LINK_DISABLED = 0, + VS_LINK_DOWN = 1, + VS_LINK_UP = 2, + VS_LINK_HIBERN8 = 3, + VS_LINK_LOST = 4, + VS_LINK_CFG = 5, +}; + +/* + * Vendor specific host controller state + */ +enum { + VS_HCE_RESET = 0, + VS_HCE_BASE = 1, + VS_HCE_OOCPR_WAIT = 2, + VS_HCE_DME_RESET = 3, + VS_HCE_MIDDLE = 4, + VS_HCE_DME_ENABLE = 5, + VS_HCE_DEFAULTS = 6, + VS_HIB_IDLEEN = 7, + VS_HIB_ENTER = 8, + VS_HIB_ENTER_CONF = 9, + VS_HIB_MIDDLE = 10, + VS_HIB_WAITTIMER = 11, + VS_HIB_EXIT_CONF = 12, + VS_HIB_EXIT = 13, +}; + +/* + * VS_DEBUGCLOCKENABLE + */ +enum { + TX_SYMBOL_CLK_REQ_FORCE = 5, +}; + +/* + * VS_SAVEPOWERCONTROL + */ +enum { + RX_SYMBOL_CLK_GATE_EN = 0, + SYS_CLK_GATE_EN = 2, + TX_CLK_GATE_EN = 3, +}; + +/* + * Host capability + */ +enum ufs_mtk_host_caps { + UFS_MTK_CAP_BOOST_CRYPT_ENGINE = 1 << 0, + UFS_MTK_CAP_VA09_PWR_CTRL = 1 << 1, + UFS_MTK_CAP_DISABLE_AH8 = 1 << 2, + UFS_MTK_CAP_BROKEN_VCC = 1 << 3, + + /* + * Override UFS_MTK_CAP_BROKEN_VCC's behavior to + * allow vccqx upstream to enter LPM + */ + UFS_MTK_CAP_ALLOW_VCCQX_LPM = 1 << 5, + UFS_MTK_CAP_PMC_VIA_FASTAUTO = 1 << 6, + UFS_MTK_CAP_TX_SKEW_FIX = 1 << 7, + UFS_MTK_CAP_DISABLE_MCQ = 1 << 8, + /* Control MTCMOS with RTFF */ + UFS_MTK_CAP_RTFF_MTCMOS = 1 << 9, + + UFS_MTK_CAP_MCQ_BROKEN_RTC = 1 << 10, +}; + +struct ufs_mtk_crypt_cfg { + struct regulator *reg_vcore; + struct clk *clk_crypt_perf; + struct clk *clk_crypt_mux; + struct clk *clk_crypt_lp; + int vcore_volt; +}; + +struct ufs_mtk_clk { + struct ufs_clk_info *ufs_sel_clki; /* Mux */ + struct ufs_clk_info *ufs_sel_max_clki; /* Max src */ + struct ufs_clk_info *ufs_sel_min_clki; /* Min src */ + struct ufs_clk_info *ufs_fde_clki; /* Mux */ + struct ufs_clk_info *ufs_fde_max_clki; /* Max src */ + struct ufs_clk_info *ufs_fde_min_clki; /* Min src */ + struct regulator *reg_vcore; + int vcore_volt; +}; + +struct ufs_mtk_hw_ver { + u8 step; + u8 minor; + u8 major; +}; + +struct ufs_mtk_mcq_intr_info { + struct ufs_hba *hba; + u32 irq; + u8 qid; +}; + +struct ufs_mtk_host { + struct phy *mphy; + struct regulator *reg_va09; + struct reset_control *hci_reset; + struct reset_control *unipro_reset; + struct reset_control *crypto_reset; + struct reset_control *mphy_reset; + struct ufs_hba *hba; + struct ufs_mtk_crypt_cfg *crypt; + struct ufs_mtk_clk mclk; + struct ufs_mtk_hw_ver hw_ver; + enum ufs_mtk_host_caps caps; + bool mphy_powered_on; + bool unipro_lpm; + bool ref_clk_enabled; + bool clk_scale_up; + u16 ref_clk_ungating_wait_us; + u16 ref_clk_gating_wait_us; + u32 ip_ver; + bool legacy_ip_ver; + + bool mcq_set_intr; + bool is_mcq_intr_enabled; + int mcq_nr_intr; + struct ufs_mtk_mcq_intr_info mcq_intr_info[UFSHCD_MAX_Q_NR]; + struct device *phy_dev; +}; + +/* MTK delay of autosuspend: 500 ms */ +#define MTK_RPM_AUTOSUSPEND_DELAY_MS 500 + +/* MTK RTT support number */ +#define MTK_MAX_NUM_RTT 2 + +/* UFSHCI MTK ip version value */ +enum { + /* UFSHCI 3.1 */ + IP_VER_MT6983 = 0x10360000, + IP_VER_MT6878 = 0x10420200, + + /* UFSHCI 4.0 */ + IP_VER_MT6897 = 0x10440000, + IP_VER_MT6989 = 0x10450000, + IP_VER_MT6899 = 0x10450100, + IP_VER_MT6991_A0 = 0x10460000, + IP_VER_MT6991_B0 = 0x10470000, + IP_VER_MT6993 = 0x10480000, + + /* UFSHCI 5.0 */ + IP_VER_MT6995_A0 = 0x10490000, + IP_VER_MT6995_B0 = 0x10500000, + + IP_VER_NONE = 0xFFFFFFFF +}; + +enum ip_ver_legacy { + IP_LEGACY_VER_MT6781 = 0x10380000, + IP_LEGACY_VER_MT6879 = 0x10360000, + IP_LEGACY_VER_MT6893 = 0x20160706 +}; + +#endif /* !_UFS_MEDIATEK_H */ diff --git a/drivers/ufs/host/ufs-qcom.c b/drivers/ufs/host/ufs-qcom.c new file mode 100644 index 000000000..291c43448 --- /dev/null +++ b/drivers/ufs/host/ufs-qcom.c @@ -0,0 +1,3050 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2013-2016, Linux Foundation. All rights reserved. + */ + +#include <linux/acpi.h> +#include <linux/clk.h> +#include <linux/cleanup.h> +#include <linux/delay.h> +#include <linux/devfreq.h> +#include <linux/gpio/consumer.h> +#include <linux/interconnect.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/phy/phy.h> +#include <linux/platform_device.h> +#include <linux/pm_domain.h> +#include <linux/reset-controller.h> +#include <linux/time.h> +#include <linux/unaligned.h> +#include <linux/units.h> + +#include <soc/qcom/ice.h> + +#include <ufs/ufshcd.h> +#include <ufs/ufshci.h> +#include <ufs/ufs_quirks.h> +#include <ufs/unipro.h> +#include "ufshcd-pltfrm.h" +#include "ufs-qcom.h" + +#define MCQ_QCFGPTR_MASK GENMASK(7, 0) +#define MCQ_QCFGPTR_UNIT 0x200 +#define MCQ_SQATTR_OFFSET(c) \ + ((((c) >> 16) & MCQ_QCFGPTR_MASK) * MCQ_QCFGPTR_UNIT) +#define MCQ_QCFG_SIZE 0x40 + +/* De-emphasis for gear-5 */ +#define DEEMPHASIS_3_5_dB 0x04 +#define NO_DEEMPHASIS 0x0 + +#define UFS_ICE_SYNC_RST_SEL BIT(3) +#define UFS_ICE_SYNC_RST_SW BIT(4) + +enum { + TSTBUS_UAWM, + TSTBUS_UARM, + TSTBUS_TXUC, + TSTBUS_RXUC, + TSTBUS_DFC, + TSTBUS_TRLUT, + TSTBUS_TMRLUT, + TSTBUS_OCSC, + TSTBUS_UTP_HCI, + TSTBUS_COMBINED, + TSTBUS_WRAPPER, + TSTBUS_UNIPRO, + TSTBUS_MAX, +}; + +#define QCOM_UFS_MAX_GEAR 5 +#define QCOM_UFS_MAX_LANE 2 + +enum { + MODE_MIN, + MODE_PWM, + MODE_HS_RA, + MODE_HS_RB, + MODE_MAX, +}; + +static const struct __ufs_qcom_bw_table { + u32 mem_bw; + u32 cfg_bw; +} ufs_qcom_bw_table[MODE_MAX + 1][QCOM_UFS_MAX_GEAR + 1][QCOM_UFS_MAX_LANE + 1] = { + [MODE_MIN][0][0] = { 0, 0 }, /* Bandwidth values in KB/s */ + [MODE_PWM][UFS_PWM_G1][UFS_LANE_1] = { 922, 1000 }, + [MODE_PWM][UFS_PWM_G2][UFS_LANE_1] = { 1844, 1000 }, + [MODE_PWM][UFS_PWM_G3][UFS_LANE_1] = { 3688, 1000 }, + [MODE_PWM][UFS_PWM_G4][UFS_LANE_1] = { 7376, 1000 }, + [MODE_PWM][UFS_PWM_G5][UFS_LANE_1] = { 14752, 1000 }, + [MODE_PWM][UFS_PWM_G1][UFS_LANE_2] = { 1844, 1000 }, + [MODE_PWM][UFS_PWM_G2][UFS_LANE_2] = { 3688, 1000 }, + [MODE_PWM][UFS_PWM_G3][UFS_LANE_2] = { 7376, 1000 }, + [MODE_PWM][UFS_PWM_G4][UFS_LANE_2] = { 14752, 1000 }, + [MODE_PWM][UFS_PWM_G5][UFS_LANE_2] = { 29504, 1000 }, + [MODE_HS_RA][UFS_HS_G1][UFS_LANE_1] = { 127796, 1000 }, + [MODE_HS_RA][UFS_HS_G2][UFS_LANE_1] = { 255591, 1000 }, + [MODE_HS_RA][UFS_HS_G3][UFS_LANE_1] = { 1492582, 102400 }, + [MODE_HS_RA][UFS_HS_G4][UFS_LANE_1] = { 2915200, 204800 }, + [MODE_HS_RA][UFS_HS_G5][UFS_LANE_1] = { 5836800, 409600 }, + [MODE_HS_RA][UFS_HS_G1][UFS_LANE_2] = { 255591, 1000 }, + [MODE_HS_RA][UFS_HS_G2][UFS_LANE_2] = { 511181, 1000 }, + [MODE_HS_RA][UFS_HS_G3][UFS_LANE_2] = { 1492582, 204800 }, + [MODE_HS_RA][UFS_HS_G4][UFS_LANE_2] = { 2915200, 409600 }, + [MODE_HS_RA][UFS_HS_G5][UFS_LANE_2] = { 5836800, 819200 }, + [MODE_HS_RB][UFS_HS_G1][UFS_LANE_1] = { 149422, 1000 }, + [MODE_HS_RB][UFS_HS_G2][UFS_LANE_1] = { 298189, 1000 }, + [MODE_HS_RB][UFS_HS_G3][UFS_LANE_1] = { 1492582, 102400 }, + [MODE_HS_RB][UFS_HS_G4][UFS_LANE_1] = { 2915200, 204800 }, + [MODE_HS_RB][UFS_HS_G5][UFS_LANE_1] = { 5836800, 409600 }, + [MODE_HS_RB][UFS_HS_G1][UFS_LANE_2] = { 298189, 1000 }, + [MODE_HS_RB][UFS_HS_G2][UFS_LANE_2] = { 596378, 1000 }, + [MODE_HS_RB][UFS_HS_G3][UFS_LANE_2] = { 1492582, 204800 }, + [MODE_HS_RB][UFS_HS_G4][UFS_LANE_2] = { 2915200, 409600 }, + [MODE_HS_RB][UFS_HS_G5][UFS_LANE_2] = { 5836800, 819200 }, + [MODE_MAX][0][0] = { 7643136, 819200 }, +}; + +static const struct { + int nminor; + char *prefix; +} testbus_info[TSTBUS_MAX] = { + [TSTBUS_UAWM] = {32, "TSTBUS_UAWM"}, + [TSTBUS_UARM] = {32, "TSTBUS_UARM"}, + [TSTBUS_TXUC] = {32, "TSTBUS_TXUC"}, + [TSTBUS_RXUC] = {32, "TSTBUS_RXUC"}, + [TSTBUS_DFC] = {32, "TSTBUS_DFC"}, + [TSTBUS_TRLUT] = {32, "TSTBUS_TRLUT"}, + [TSTBUS_TMRLUT] = {32, "TSTBUS_TMRLUT"}, + [TSTBUS_OCSC] = {32, "TSTBUS_OCSC"}, + [TSTBUS_UTP_HCI] = {32, "TSTBUS_UTP_HCI"}, + [TSTBUS_COMBINED] = {32, "TSTBUS_COMBINED"}, + [TSTBUS_WRAPPER] = {32, "TSTBUS_WRAPPER"}, + [TSTBUS_UNIPRO] = {256, "TSTBUS_UNIPRO"}, +}; + +static void ufs_qcom_get_default_testbus_cfg(struct ufs_qcom_host *host); +static unsigned long ufs_qcom_opp_freq_to_clk_freq(struct ufs_hba *hba, + unsigned long freq, char *name); +static int ufs_qcom_set_core_clk_ctrl(struct ufs_hba *hba, bool is_scale_up, unsigned long freq); + +static struct ufs_qcom_host *rcdev_to_ufs_host(struct reset_controller_dev *rcd) +{ + return container_of(rcd, struct ufs_qcom_host, rcdev); +} + +#ifdef CONFIG_SCSI_UFS_CRYPTO +/** + * ufs_qcom_config_ice_allocator() - ICE core allocator configuration + * + * @host: pointer to qcom specific variant structure. + */ +static void ufs_qcom_config_ice_allocator(struct ufs_qcom_host *host) +{ + struct ufs_hba *hba = host->hba; + static const uint8_t val[4] = { NUM_RX_R1W0, NUM_TX_R0W1, NUM_RX_R1W1, NUM_TX_R1W1 }; + u32 config; + + if (!(host->caps & UFS_QCOM_CAP_ICE_CONFIG) || + !(host->hba->caps & UFSHCD_CAP_CRYPTO)) + return; + + config = get_unaligned_le32(val); + + ufshcd_writel(hba, ICE_ALLOCATOR_TYPE, REG_UFS_MEM_ICE_CONFIG); + ufshcd_writel(hba, config, REG_UFS_MEM_ICE_NUM_CORE); +} + +static inline void ufs_qcom_ice_enable(struct ufs_qcom_host *host) +{ + if (host->hba->caps & UFSHCD_CAP_CRYPTO) + qcom_ice_enable(host->ice); +} + +static const struct blk_crypto_ll_ops ufs_qcom_crypto_ops; /* forward decl */ + +static int ufs_qcom_ice_init(struct ufs_qcom_host *host) +{ + struct ufs_hba *hba = host->hba; + struct blk_crypto_profile *profile = &hba->crypto_profile; + struct device *dev = hba->dev; + struct qcom_ice *ice; + union ufs_crypto_capabilities caps; + union ufs_crypto_cap_entry cap; + int err; + int i; + + ice = devm_of_qcom_ice_get(dev); + if (IS_ERR(ice)) { + if (ice != ERR_PTR(-EOPNOTSUPP)) + return PTR_ERR(ice); + + dev_warn(dev, "Disabling inline encryption support\n"); + return 0; + } + + host->ice = ice; + + /* Initialize the blk_crypto_profile */ + + caps.reg_val = cpu_to_le32(ufshcd_readl(hba, REG_UFS_CCAP)); + + /* The number of keyslots supported is (CFGC+1) */ + err = devm_blk_crypto_profile_init(dev, profile, caps.config_count + 1); + if (err) + return err; + + profile->ll_ops = ufs_qcom_crypto_ops; + profile->max_dun_bytes_supported = 8; + profile->key_types_supported = qcom_ice_get_supported_key_type(ice); + profile->dev = dev; + + /* + * Currently this driver only supports AES-256-XTS. All known versions + * of ICE support it, but to be safe make sure it is really declared in + * the crypto capability registers. The crypto capability registers + * also give the supported data unit size(s). + */ + for (i = 0; i < caps.num_crypto_cap; i++) { + cap.reg_val = cpu_to_le32(ufshcd_readl(hba, + REG_UFS_CRYPTOCAP + + i * sizeof(__le32))); + if (cap.algorithm_id == UFS_CRYPTO_ALG_AES_XTS && + cap.key_size == UFS_CRYPTO_KEY_SIZE_256) + profile->modes_supported[BLK_ENCRYPTION_MODE_AES_256_XTS] |= + cap.sdus_mask * 512; + } + + hba->caps |= UFSHCD_CAP_CRYPTO; + hba->quirks |= UFSHCD_QUIRK_CUSTOM_CRYPTO_PROFILE; + return 0; +} + +static inline int ufs_qcom_ice_resume(struct ufs_qcom_host *host) +{ + if (host->hba->caps & UFSHCD_CAP_CRYPTO) + return qcom_ice_resume(host->ice); + + return 0; +} + +static inline int ufs_qcom_ice_suspend(struct ufs_qcom_host *host) +{ + if (host->hba->caps & UFSHCD_CAP_CRYPTO) + return qcom_ice_suspend(host->ice); + + return 0; +} + +static int ufs_qcom_ice_keyslot_program(struct blk_crypto_profile *profile, + const struct blk_crypto_key *key, + unsigned int slot) +{ + struct ufs_hba *hba = ufs_hba_from_crypto_profile(profile); + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + int err; + + ufshcd_hold(hba); + err = qcom_ice_program_key(host->ice, slot, key); + ufshcd_release(hba); + return err; +} + +static int ufs_qcom_ice_keyslot_evict(struct blk_crypto_profile *profile, + const struct blk_crypto_key *key, + unsigned int slot) +{ + struct ufs_hba *hba = ufs_hba_from_crypto_profile(profile); + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + int err; + + ufshcd_hold(hba); + err = qcom_ice_evict_key(host->ice, slot); + ufshcd_release(hba); + return err; +} + +static int ufs_qcom_ice_derive_sw_secret(struct blk_crypto_profile *profile, + const u8 *eph_key, size_t eph_key_size, + u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE]) +{ + struct ufs_hba *hba = ufs_hba_from_crypto_profile(profile); + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + + return qcom_ice_derive_sw_secret(host->ice, eph_key, eph_key_size, + sw_secret); +} + +static int ufs_qcom_ice_import_key(struct blk_crypto_profile *profile, + const u8 *raw_key, size_t raw_key_size, + u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]) +{ + struct ufs_hba *hba = ufs_hba_from_crypto_profile(profile); + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + + return qcom_ice_import_key(host->ice, raw_key, raw_key_size, lt_key); +} + +static int ufs_qcom_ice_generate_key(struct blk_crypto_profile *profile, + u8 lt_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]) +{ + struct ufs_hba *hba = ufs_hba_from_crypto_profile(profile); + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + + return qcom_ice_generate_key(host->ice, lt_key); +} + +static int ufs_qcom_ice_prepare_key(struct blk_crypto_profile *profile, + const u8 *lt_key, size_t lt_key_size, + u8 eph_key[BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE]) +{ + struct ufs_hba *hba = ufs_hba_from_crypto_profile(profile); + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + + return qcom_ice_prepare_key(host->ice, lt_key, lt_key_size, eph_key); +} + +static const struct blk_crypto_ll_ops ufs_qcom_crypto_ops = { + .keyslot_program = ufs_qcom_ice_keyslot_program, + .keyslot_evict = ufs_qcom_ice_keyslot_evict, + .derive_sw_secret = ufs_qcom_ice_derive_sw_secret, + .import_key = ufs_qcom_ice_import_key, + .generate_key = ufs_qcom_ice_generate_key, + .prepare_key = ufs_qcom_ice_prepare_key, +}; + +#else + +static inline void ufs_qcom_ice_enable(struct ufs_qcom_host *host) +{ +} + +static int ufs_qcom_ice_init(struct ufs_qcom_host *host) +{ + return 0; +} + +static inline int ufs_qcom_ice_resume(struct ufs_qcom_host *host) +{ + return 0; +} + +static inline int ufs_qcom_ice_suspend(struct ufs_qcom_host *host) +{ + return 0; +} + +static void ufs_qcom_config_ice_allocator(struct ufs_qcom_host *host) +{ +} + +#endif + +static void ufs_qcom_disable_lane_clks(struct ufs_qcom_host *host) +{ + if (!host->is_lane_clks_enabled) + return; + + clk_bulk_disable_unprepare(host->num_clks, host->clks); + + host->is_lane_clks_enabled = false; +} + +static int ufs_qcom_enable_lane_clks(struct ufs_qcom_host *host) +{ + int err; + + err = clk_bulk_prepare_enable(host->num_clks, host->clks); + if (err) + return err; + + host->is_lane_clks_enabled = true; + + return 0; +} + +static int ufs_qcom_init_lane_clks(struct ufs_qcom_host *host) +{ + int err; + struct device *dev = host->hba->dev; + + if (has_acpi_companion(dev)) + return 0; + + err = devm_clk_bulk_get_all(dev, &host->clks); + if (err <= 0) + return err; + + host->num_clks = err; + + return 0; +} + +static int ufs_qcom_check_hibern8(struct ufs_hba *hba) +{ + int err; + u32 tx_fsm_val; + unsigned long timeout = jiffies + msecs_to_jiffies(HBRN8_POLL_TOUT_MS); + + do { + err = ufshcd_dme_get(hba, + UIC_ARG_MIB_SEL(MPHY_TX_FSM_STATE, + UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)), + &tx_fsm_val); + if (err || tx_fsm_val == TX_FSM_HIBERN8) + break; + + /* sleep for max. 200us */ + usleep_range(100, 200); + } while (time_before(jiffies, timeout)); + + /* + * we might have scheduled out for long during polling so + * check the state again. + */ + if (time_after(jiffies, timeout)) + err = ufshcd_dme_get(hba, + UIC_ARG_MIB_SEL(MPHY_TX_FSM_STATE, + UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)), + &tx_fsm_val); + + if (err) { + dev_err(hba->dev, "%s: unable to get TX_FSM_STATE, err %d\n", + __func__, err); + } else if (tx_fsm_val != TX_FSM_HIBERN8) { + err = tx_fsm_val; + dev_err(hba->dev, "%s: invalid TX_FSM_STATE = %d\n", + __func__, err); + } + + return err; +} + +static void ufs_qcom_select_unipro_mode(struct ufs_qcom_host *host) +{ + ufshcd_rmwl(host->hba, QUNIPRO_SEL, QUNIPRO_SEL, REG_UFS_CFG1); + + if (host->hw_ver.major >= 0x05) + ufshcd_rmwl(host->hba, QUNIPRO_G4_SEL, 0, REG_UFS_CFG0); +} + +/* + * ufs_qcom_host_reset - reset host controller and PHY + */ +static int ufs_qcom_host_reset(struct ufs_hba *hba) +{ + int ret; + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + bool reenable_intr; + + if (!host->core_reset) + return 0; + + reenable_intr = hba->is_irq_enabled; + ufshcd_disable_irq(hba); + + ret = reset_control_assert(host->core_reset); + if (ret) { + dev_err(hba->dev, "%s: core_reset assert failed, err = %d\n", + __func__, ret); + return ret; + } + + /* + * The hardware requirement for delay between assert/deassert + * is at least 3-4 sleep clock (32.7KHz) cycles, which comes to + * ~125us (4/32768). To be on the safe side add 200us delay. + */ + usleep_range(200, 210); + + ret = reset_control_deassert(host->core_reset); + if (ret) { + dev_err(hba->dev, "%s: core_reset deassert failed, err = %d\n", + __func__, ret); + return ret; + } + + usleep_range(1000, 1100); + + if (reenable_intr) + ufshcd_enable_irq(hba); + + return 0; +} + +static u32 ufs_qcom_get_hs_gear(struct ufs_hba *hba) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + + if (host->hw_ver.major >= 0x4) + return UFS_QCOM_MAX_GEAR(ufshcd_readl(hba, REG_UFS_PARAM0)); + + /* Default is HS-G3 */ + return UFS_HS_G3; +} + +static int ufs_qcom_power_up_sequence(struct ufs_hba *hba) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + struct ufs_host_params *host_params = &host->host_params; + struct phy *phy = host->generic_phy; + enum phy_mode mode; + int ret; + + /* + * HW ver 5 can only support up to HS-G5 Rate-A due to HW limitations. + * If the HS-G5 PHY gear is used, update host_params->hs_rate to Rate-A, + * so that the subsequent power mode change shall stick to Rate-A. + */ + if (host->hw_ver.major == 0x5 && host->phy_gear == UFS_HS_G5) + host_params->hs_rate = PA_HS_MODE_A; + + mode = host_params->hs_rate == PA_HS_MODE_B ? PHY_MODE_UFS_HS_B : PHY_MODE_UFS_HS_A; + + /* Reset UFS Host Controller and PHY */ + ret = ufs_qcom_host_reset(hba); + if (ret) + return ret; + + if (phy->power_count) + phy_power_off(phy); + + + /* phy initialization - calibrate the phy */ + ret = phy_init(phy); + if (ret) { + dev_err(hba->dev, "%s: phy init failed, ret = %d\n", + __func__, ret); + return ret; + } + + ret = phy_set_mode_ext(phy, mode, host->phy_gear); + if (ret) + goto out_disable_phy; + + /* power on phy - start serdes and phy's power and clocks */ + ret = phy_power_on(phy); + if (ret) { + dev_err(hba->dev, "%s: phy power on failed, ret = %d\n", + __func__, ret); + goto out_disable_phy; + } + + ret = phy_calibrate(phy); + if (ret) { + dev_err(hba->dev, "Failed to calibrate PHY: %d\n", ret); + goto out_disable_phy; + } + + ufs_qcom_select_unipro_mode(host); + + return 0; + +out_disable_phy: + phy_exit(phy); + + return ret; +} + +/* + * The UTP controller has a number of internal clock gating cells (CGCs). + * Internal hardware sub-modules within the UTP controller control the CGCs. + * Hardware CGCs disable the clock to inactivate UTP sub-modules not involved + * in a specific operation, UTP controller CGCs are by default disabled and + * this function enables them (after every UFS link startup) to save some power + * leakage. + */ +static void ufs_qcom_enable_hw_clk_gating(struct ufs_hba *hba) +{ + int err; + + /* Enable UTP internal clock gating */ + ufshcd_rmwl(hba, REG_UFS_CFG2_CGC_EN_ALL, REG_UFS_CFG2_CGC_EN_ALL, + REG_UFS_CFG2); + + /* Ensure that HW clock gating is enabled before next operations */ + ufshcd_readl(hba, REG_UFS_CFG2); + + /* Enable Unipro internal clock gating */ + err = ufshcd_dme_rmw(hba, DL_VS_CLK_CFG_MASK, + DL_VS_CLK_CFG_MASK, DL_VS_CLK_CFG); + if (err) + goto out; + + err = ufshcd_dme_rmw(hba, PA_VS_CLK_CFG_REG_MASK, + PA_VS_CLK_CFG_REG_MASK, PA_VS_CLK_CFG_REG); + if (err) + goto out; + + err = ufshcd_dme_rmw(hba, DME_VS_CORE_CLK_CTRL_DME_HW_CGC_EN, + DME_VS_CORE_CLK_CTRL_DME_HW_CGC_EN, + DME_VS_CORE_CLK_CTRL); +out: + if (err) + dev_err(hba->dev, "hw clk gating enabled failed\n"); +} + +static int ufs_qcom_hce_enable_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + int err; + + switch (status) { + case PRE_CHANGE: + err = ufs_qcom_power_up_sequence(hba); + if (err) + return err; + + /* + * The PHY PLL output is the source of tx/rx lane symbol + * clocks, hence, enable the lane clocks only after PHY + * is initialized. + */ + err = ufs_qcom_enable_lane_clks(host); + break; + case POST_CHANGE: + /* check if UFS PHY moved from DISABLED to HIBERN8 */ + err = ufs_qcom_check_hibern8(hba); + ufs_qcom_enable_hw_clk_gating(hba); + ufs_qcom_ice_enable(host); + ufs_qcom_config_ice_allocator(host); + break; + default: + dev_err(hba->dev, "%s: invalid status %d\n", __func__, status); + err = -EINVAL; + break; + } + return err; +} + +static int ufs_qcom_fw_managed_hce_enable_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + + switch (status) { + case PRE_CHANGE: + ufs_qcom_select_unipro_mode(host); + break; + case POST_CHANGE: + ufs_qcom_enable_hw_clk_gating(hba); + ufs_qcom_ice_enable(host); + break; + default: + dev_err(hba->dev, "Invalid status %d\n", status); + return -EINVAL; + } + + return 0; +} + +/** + * ufs_qcom_cfg_timers - Configure ufs qcom cfg timers + * + * @hba: host controller instance + * @is_pre_scale_up: flag to check if pre scale up condition. + * @freq: target opp freq + * Return: zero for success and non-zero in case of a failure. + */ +static int ufs_qcom_cfg_timers(struct ufs_hba *hba, bool is_pre_scale_up, unsigned long freq) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + struct ufs_clk_info *clki; + unsigned long clk_freq = 0; + u32 core_clk_cycles_per_us; + + /* + * UTP controller uses SYS1CLK_1US_REG register for Interrupt + * Aggregation logic. + * It is mandatory to write SYS1CLK_1US_REG register on UFS host + * controller V4.0.0 onwards. + */ + if (host->hw_ver.major < 4 && !ufshcd_is_intr_aggr_allowed(hba)) + return 0; + + if (hba->use_pm_opp && freq != ULONG_MAX) { + clk_freq = ufs_qcom_opp_freq_to_clk_freq(hba, freq, "core_clk"); + if (clk_freq) + goto cfg_timers; + } + + list_for_each_entry(clki, &hba->clk_list_head, list) { + if (!strcmp(clki->name, "core_clk")) { + if (freq == ULONG_MAX) { + clk_freq = clki->max_freq; + break; + } + + if (is_pre_scale_up) + clk_freq = clki->max_freq; + else + clk_freq = clk_get_rate(clki->clk); + break; + } + + } + +cfg_timers: + /* If frequency is smaller than 1MHz, set to 1MHz */ + if (clk_freq < DEFAULT_CLK_RATE_HZ) + clk_freq = DEFAULT_CLK_RATE_HZ; + + core_clk_cycles_per_us = clk_freq / USEC_PER_SEC; + if (ufshcd_readl(hba, REG_UFS_SYS1CLK_1US) != core_clk_cycles_per_us) { + ufshcd_writel(hba, core_clk_cycles_per_us, REG_UFS_SYS1CLK_1US); + /* + * make sure above write gets applied before we return from + * this function. + */ + ufshcd_readl(hba, REG_UFS_SYS1CLK_1US); + } + + return 0; +} + +static void ufs_qcom_link_startup_post_change(struct ufs_hba *hba) +{ + if (ufshcd_is_auto_hibern8_supported(hba)) + ufshcd_rmwl(hba, UFS_HW_CLK_CTRL_EN, UFS_HW_CLK_CTRL_EN, + UFS_AH8_CFG); +} + +static int ufs_qcom_link_startup_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status) +{ + int err = 0; + + switch (status) { + case PRE_CHANGE: + if (ufs_qcom_cfg_timers(hba, false, ULONG_MAX)) { + dev_err(hba->dev, "%s: ufs_qcom_cfg_timers() failed\n", + __func__); + return -EINVAL; + } + + err = ufs_qcom_set_core_clk_ctrl(hba, true, ULONG_MAX); + if (err) + dev_err(hba->dev, "cfg core clk ctrl failed\n"); + /* + * Some UFS devices (and may be host) have issues if LCC is + * enabled. So we are setting PA_Local_TX_LCC_Enable to 0 + * before link startup which will make sure that both host + * and device TX LCC are disabled once link startup is + * completed. + */ + err = ufshcd_disable_host_tx_lcc(hba); + + break; + case POST_CHANGE: + ufs_qcom_link_startup_post_change(hba); + break; + default: + break; + } + + return err; +} + +static void ufs_qcom_device_reset_ctrl(struct ufs_hba *hba, bool asserted) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + + /* reset gpio is optional */ + if (!host->device_reset) + return; + + gpiod_set_value_cansleep(host->device_reset, asserted); +} + +static int ufs_qcom_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op, + enum ufs_notify_change_status status) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + + if (status == PRE_CHANGE) + return 0; + + if (!ufs_qcom_is_link_active(hba)) + ufs_qcom_disable_lane_clks(host); + + + /* reset the connected UFS device during power down */ + if (ufs_qcom_is_link_off(hba) && host->device_reset) { + ufs_qcom_device_reset_ctrl(hba, true); + /* + * After sending the SSU command, asserting the rst_n + * line causes the device firmware to wake up and + * execute its reset routine. + * + * During this process, the device may draw current + * beyond the permissible limit for low-power mode (LPM). + * A 10ms delay, based on experimental observations, + * allows the UFS device to complete its hardware reset + * before transitioning the power rail to LPM. + */ + usleep_range(10000, 11000); + } + + return ufs_qcom_ice_suspend(host); +} + +static int ufs_qcom_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + int err; + u32 reg_val; + + err = ufs_qcom_enable_lane_clks(host); + if (err) + return err; + + if ((!ufs_qcom_is_link_active(hba)) && + host->hw_ver.major == 5 && + host->hw_ver.minor == 0 && + host->hw_ver.step == 0) { + ufshcd_writel(hba, UFS_ICE_SYNC_RST_SEL | UFS_ICE_SYNC_RST_SW, UFS_MEM_ICE_CFG); + reg_val = ufshcd_readl(hba, UFS_MEM_ICE_CFG); + reg_val &= ~(UFS_ICE_SYNC_RST_SEL | UFS_ICE_SYNC_RST_SW); + /* + * HW documentation doesn't recommend any delay between the + * reset set and clear. But we are enforcing an arbitrary delay + * to give flops enough time to settle in. + */ + usleep_range(50, 100); + ufshcd_writel(hba, reg_val, UFS_MEM_ICE_CFG); + ufshcd_readl(hba, UFS_MEM_ICE_CFG); + } + + return ufs_qcom_ice_resume(host); +} + +static int ufs_qcom_fw_managed_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op, + enum ufs_notify_change_status status) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + + if (status == PRE_CHANGE) + return 0; + + pm_runtime_put_sync(hba->dev); + + return ufs_qcom_ice_suspend(host); +} + +static int ufs_qcom_fw_managed_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + int err; + + err = pm_runtime_resume_and_get(hba->dev); + if (err) { + dev_err(hba->dev, "PM runtime resume failed: %d\n", err); + return err; + } + + return ufs_qcom_ice_resume(host); +} + +static void ufs_qcom_dev_ref_clk_ctrl(struct ufs_qcom_host *host, bool enable) +{ + if (host->dev_ref_clk_ctrl_mmio && + (enable ^ host->is_dev_ref_clk_enabled)) { + u32 temp = readl_relaxed(host->dev_ref_clk_ctrl_mmio); + + if (enable) + temp |= host->dev_ref_clk_en_mask; + else + temp &= ~host->dev_ref_clk_en_mask; + + /* + * If we are here to disable this clock it might be immediately + * after entering into hibern8 in which case we need to make + * sure that device ref_clk is active for specific time after + * hibern8 enter. + */ + if (!enable) { + unsigned long gating_wait; + + gating_wait = host->hba->dev_info.clk_gating_wait_us; + if (!gating_wait) { + udelay(1); + } else { + /* + * bRefClkGatingWaitTime defines the minimum + * time for which the reference clock is + * required by device during transition from + * HS-MODE to LS-MODE or HIBERN8 state. Give it + * more delay to be on the safe side. + */ + gating_wait += 10; + usleep_range(gating_wait, gating_wait + 10); + } + } + + writel_relaxed(temp, host->dev_ref_clk_ctrl_mmio); + + /* + * Make sure the write to ref_clk reaches the destination and + * not stored in a Write Buffer (WB). + */ + readl(host->dev_ref_clk_ctrl_mmio); + + /* + * If we call hibern8 exit after this, we need to make sure that + * device ref_clk is stable for at least 1us before the hibern8 + * exit command. + */ + if (enable) + udelay(1); + + host->is_dev_ref_clk_enabled = enable; + } +} + +static int ufs_qcom_icc_set_bw(struct ufs_qcom_host *host, u32 mem_bw, u32 cfg_bw) +{ + struct device *dev = host->hba->dev; + int ret; + + ret = icc_set_bw(host->icc_ddr, 0, mem_bw); + if (ret < 0) { + dev_err(dev, "failed to set bandwidth request: %d\n", ret); + return ret; + } + + ret = icc_set_bw(host->icc_cpu, 0, cfg_bw); + if (ret < 0) { + dev_err(dev, "failed to set bandwidth request: %d\n", ret); + return ret; + } + + return 0; +} + +static struct __ufs_qcom_bw_table ufs_qcom_get_bw_table(struct ufs_qcom_host *host) +{ + struct ufs_pa_layer_attr *p = &host->dev_req_params; + int gear = max_t(u32, p->gear_rx, p->gear_tx); + int lane = max_t(u32, p->lane_rx, p->lane_tx); + + if (WARN_ONCE(gear > QCOM_UFS_MAX_GEAR, + "ICC scaling for UFS Gear (%d) not supported. Using Gear (%d) bandwidth\n", + gear, QCOM_UFS_MAX_GEAR)) + gear = QCOM_UFS_MAX_GEAR; + + if (WARN_ONCE(lane > QCOM_UFS_MAX_LANE, + "ICC scaling for UFS Lane (%d) not supported. Using Lane (%d) bandwidth\n", + lane, QCOM_UFS_MAX_LANE)) + lane = QCOM_UFS_MAX_LANE; + + if (ufshcd_is_hs_mode(p)) { + if (p->hs_rate == PA_HS_MODE_B) + return ufs_qcom_bw_table[MODE_HS_RB][gear][lane]; + else + return ufs_qcom_bw_table[MODE_HS_RA][gear][lane]; + } else { + return ufs_qcom_bw_table[MODE_PWM][gear][lane]; + } +} + +static int ufs_qcom_icc_update_bw(struct ufs_qcom_host *host) +{ + struct __ufs_qcom_bw_table bw_table; + + bw_table = ufs_qcom_get_bw_table(host); + + return ufs_qcom_icc_set_bw(host, bw_table.mem_bw, bw_table.cfg_bw); +} + +static void ufs_qcom_set_tx_hs_equalizer(struct ufs_hba *hba, u32 gear, u32 tx_lanes) +{ + u32 equalizer_val; + int ret, i; + + /* Determine the equalizer value based on the gear */ + equalizer_val = (gear == 5) ? DEEMPHASIS_3_5_dB : NO_DEEMPHASIS; + + for (i = 0; i < tx_lanes; i++) { + ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_HS_EQUALIZER, i), + equalizer_val); + if (ret) + dev_err(hba->dev, "%s: failed equalizer lane %d\n", + __func__, i); + } +} + +static int ufs_qcom_negotiate_pwr_mode(struct ufs_hba *hba, + const struct ufs_pa_layer_attr *dev_max_params, + struct ufs_pa_layer_attr *dev_req_params) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + struct ufs_host_params *host_params = &host->host_params; + + return ufshcd_negotiate_pwr_params(host_params, dev_max_params, dev_req_params); +} + +static int ufs_qcom_pwr_change_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status, + struct ufs_pa_layer_attr *dev_req_params) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + int ret = 0; + + if (!dev_req_params) { + pr_err("%s: incoming dev_req_params is NULL\n", __func__); + return -EINVAL; + } + + switch (status) { + case PRE_CHANGE: + /* + * During UFS driver probe, always update the PHY gear to match the negotiated + * gear, so that, if quirk UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH is enabled, + * the second init can program the optimal PHY settings. This allows one to start + * the first init with either the minimum or the maximum support gear. + */ + if (hba->ufshcd_state == UFSHCD_STATE_RESET) { + /* + * Skip REINIT if the negotiated gear matches with the + * initial phy_gear. Otherwise, update the phy_gear to + * program the optimal gear setting during REINIT. + */ + if (host->phy_gear == dev_req_params->gear_tx) + hba->quirks &= ~UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH; + else + host->phy_gear = dev_req_params->gear_tx; + } + + /* enable the device ref clock before changing to HS mode */ + if (!ufshcd_is_hs_mode(&hba->pwr_info) && + ufshcd_is_hs_mode(dev_req_params)) + ufs_qcom_dev_ref_clk_ctrl(host, true); + + if (host->hw_ver.major >= 0x4) { + ufshcd_dme_configure_adapt(hba, + dev_req_params->gear_tx, + PA_INITIAL_ADAPT); + } + + if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TX_DEEMPHASIS_TUNING) + ufs_qcom_set_tx_hs_equalizer(hba, + dev_req_params->gear_tx, dev_req_params->lane_tx); + + break; + case POST_CHANGE: + /* cache the power mode parameters to use internally */ + memcpy(&host->dev_req_params, + dev_req_params, sizeof(*dev_req_params)); + + ufs_qcom_icc_update_bw(host); + + /* disable the device ref clock if entered PWM mode */ + if (ufshcd_is_hs_mode(&hba->pwr_info) && + !ufshcd_is_hs_mode(dev_req_params)) + ufs_qcom_dev_ref_clk_ctrl(host, false); + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + +static int ufs_qcom_quirk_host_pa_saveconfigtime(struct ufs_hba *hba) +{ + int err; + u32 pa_vs_config_reg1; + + err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_VS_CONFIG_REG1), + &pa_vs_config_reg1); + if (err) + return err; + + /* Allow extension of MSB bits of PA_SaveConfigTime attribute */ + return ufshcd_dme_set(hba, UIC_ARG_MIB(PA_VS_CONFIG_REG1), + (pa_vs_config_reg1 | (1 << 12))); +} + +static void ufs_qcom_override_pa_tx_hsg1_sync_len(struct ufs_hba *hba) +{ + int err; + + err = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TX_HSG1_SYNC_LENGTH), + PA_TX_HSG1_SYNC_LENGTH_VAL); + if (err) + dev_err(hba->dev, "Failed (%d) set PA_TX_HSG1_SYNC_LENGTH\n", err); +} + +/** + * ufs_qcom_double_t_adapt_l0l1l2l3 - Create a new adapt that doubles the + * adaptation duration TADAPT_L0_L1_L2_L3 derived from the old adapt. + * + * @old_adapt: Original ADAPT_L0_L1_L2_L3 capability + * + * ADAPT_length_L0_L1_L2_L3 formula from M-PHY spec: + * if (ADAPT_range_L0_L1_L2_L3 == COARSE) { + * ADAPT_length_L0_L1_L2_L3 = [0, 12] + * ADAPT_L0_L1_L2_L3 = 215 x 2^ADAPT_length_L0_L1_L2_L3 + * } else if (ADAPT_range_L0_L1_L2_L3 == FINE) { + * ADAPT_length_L0_L1_L2_L3 = [0, 127] + * TADAPT_L0_L1_L2_L3 = 215 x (ADAPT_length_L0_L1_L2_L3 + 1) + * } + * + * To double the adaptation duration TADAPT_L0_L1_L2_L3: + * 1. If adapt range is COARSE (1'b1), new adapt = old adapt + 1. + * 2. If adapt range is FINE (1'b0): + * a) If old adapt length is < 64, (new adapt + 1) = 2 * (old adapt + 1). + * b) If old adapt length is >= 64, set new adapt to 0x88 using COARSE + * range, because new adapt get from equation in a) shall exceed 127. + * + * Examples: + * ADAPT_range_L0_L1_L2_L3 | ADAPT_length_L0_L1_L2_L3 | TADAPT_L0_L1_L2_L3 (PAM-4 UI) + * 0 3 131072 + * 0 7 262144 + * 0 63 2097152 + * 0 64 2129920 + * 0 127 4194304 + * 1 8 8388608 + * 1 9 16777216 + * 1 10 33554432 + * 1 11 67108864 + * 1 12 134217728 + * + * Return: new adapt. + */ +static u32 ufs_qcom_double_t_adapt_l0l1l2l3(u32 old_adapt) +{ + u32 adapt_length = old_adapt & ADAPT_LENGTH_MASK; + u32 new_adapt; + + if (IS_ADAPT_RANGE_COARSE(old_adapt)) { + new_adapt = (adapt_length + 1) | ADAPT_RANGE_BIT; + } else { + if (adapt_length < 64) + new_adapt = (adapt_length << 1) + 1; + else + /* + * 0x88 is the very coarse Adapt value which is two + * times of the largest fine Adapt value (0x7F) + */ + new_adapt = 0x88; + } + + return new_adapt; +} + +static void ufs_qcom_limit_max_gear(struct ufs_hba *hba, + enum ufs_hs_gear_tag gear) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; + struct ufs_host_params *host_params = &host->host_params; + + host_params->hs_tx_gear = gear; + host_params->hs_rx_gear = gear; + pwr_info->gear_tx = gear; + pwr_info->gear_rx = gear; + + dev_warn(hba->dev, "Limited max gear of host and device to HS-G%d\n", gear); +} + +static void ufs_qcom_fixup_tx_adapt_l0l1l2l3(struct ufs_hba *hba) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; + struct ufs_host_params *host_params = &host->host_params; + u32 old_adapt, new_adapt, actual_adapt; + bool limit_speed = false; + int err; + + if (host->hw_ver.major != 0x7 || host->hw_ver.minor > 0x1 || + host_params->hs_tx_gear <= UFS_HS_G5 || + pwr_info->gear_tx <= UFS_HS_G5) + return; + + err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTINITIALL0L1L2L3), &old_adapt); + if (err) + goto out; + + if (old_adapt > ADAPT_L0L1L2L3_LENGTH_MAX) { + dev_err(hba->dev, "PA_PeerRxHsG6AdaptInitialL0L1L2L3 value (0x%x) exceeds MAX\n", + old_adapt); + err = -ERANGE; + goto out; + } + + new_adapt = ufs_qcom_double_t_adapt_l0l1l2l3(old_adapt); + dev_dbg(hba->dev, "Original PA_PeerRxHsG6AdaptInitialL0L1L2L3 = 0x%x, new value = 0x%x\n", + old_adapt, new_adapt); + + /* + * 0x8C is the max possible value allowed by UniPro v3.0 spec, some HWs + * can accept 0x8D but some cannot. + */ + if (new_adapt <= ADAPT_L0L1L2L3_LENGTH_MAX || + (new_adapt == ADAPT_L0L1L2L3_LENGTH_MAX + 1 && host->hw_ver.minor == 0x1)) { + err = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTINITIALL0L1L2L3), + new_adapt); + if (err) + goto out; + + err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTINITIALL0L1L2L3), + &actual_adapt); + if (err) + goto out; + + if (actual_adapt != new_adapt) { + limit_speed = true; + dev_warn(hba->dev, "PA_PeerRxHsG6AdaptInitialL0L1L2L3 0x%x, expect 0x%x\n", + actual_adapt, new_adapt); + } + } else { + limit_speed = true; + dev_warn(hba->dev, "New PA_PeerRxHsG6AdaptInitialL0L1L2L3 (0x%x) is too large!\n", + new_adapt); + } + + err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTREFRESHL0L1L2L3), &old_adapt); + if (err) + goto out; + + if (old_adapt > ADAPT_L0L1L2L3_LENGTH_MAX) { + dev_err(hba->dev, "PA_PeerRxHsG6AdaptRefreshL0L1L2L3 value (0x%x) exceeds MAX\n", + old_adapt); + err = -ERANGE; + goto out; + } + + new_adapt = ufs_qcom_double_t_adapt_l0l1l2l3(old_adapt); + dev_dbg(hba->dev, "Original PA_PeerRxHsG6AdaptRefreshL0L1L2L3 = 0x%x, new value = 0x%x\n", + old_adapt, new_adapt); + + /* + * 0x8C is the max possible value allowed by UniPro v3.0 spec, some HWs + * can accept 0x8D but some cannot. + */ + if (new_adapt <= ADAPT_L0L1L2L3_LENGTH_MAX || + (new_adapt == ADAPT_L0L1L2L3_LENGTH_MAX + 1 && host->hw_ver.minor == 0x1)) { + err = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTREFRESHL0L1L2L3), + new_adapt); + if (err) + goto out; + + err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PEERRXHSG6ADAPTREFRESHL0L1L2L3), + &actual_adapt); + if (err) + goto out; + + if (actual_adapt != new_adapt) { + limit_speed = true; + dev_warn(hba->dev, "PA_PeerRxHsG6AdaptRefreshL0L1L2L3 0x%x, expect 0x%x\n", + new_adapt, actual_adapt); + } + } else { + limit_speed = true; + dev_warn(hba->dev, "New PA_PeerRxHsG6AdaptRefreshL0L1L2L3 (0x%x) is too large!\n", + new_adapt); + } + +out: + if (limit_speed || err) + ufs_qcom_limit_max_gear(hba, UFS_HS_G5); +} + +static int ufs_qcom_apply_dev_quirks(struct ufs_hba *hba) +{ + int err = 0; + + ufs_qcom_fixup_tx_adapt_l0l1l2l3(hba); + + if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME) + err = ufs_qcom_quirk_host_pa_saveconfigtime(hba); + + if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TX_HSG1_SYNC_LENGTH) + ufs_qcom_override_pa_tx_hsg1_sync_len(hba); + + return err; +} + +/* UFS device-specific quirks */ +static struct ufs_dev_quirk ufs_qcom_dev_fixups[] = { + { .wmanufacturerid = UFS_VENDOR_SKHYNIX, + .model = UFS_ANY_MODEL, + .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM }, + { .wmanufacturerid = UFS_VENDOR_WDC, + .model = UFS_ANY_MODEL, + .quirk = UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE }, + { .wmanufacturerid = UFS_VENDOR_SAMSUNG, + .model = UFS_ANY_MODEL, + .quirk = UFS_DEVICE_QUIRK_PA_TX_HSG1_SYNC_LENGTH | + UFS_DEVICE_QUIRK_PA_TX_DEEMPHASIS_TUNING }, + {} +}; + +static void ufs_qcom_fixup_dev_quirks(struct ufs_hba *hba) +{ + ufshcd_fixup_dev_quirks(hba, ufs_qcom_dev_fixups); +} + +static u32 ufs_qcom_get_ufs_hci_version(struct ufs_hba *hba) +{ + return ufshci_version(2, 0); +} + +/** + * ufs_qcom_advertise_quirks - advertise the known QCOM UFS controller quirks + * @hba: host controller instance + * + * QCOM UFS host controller might have some non standard behaviours (quirks) + * than what is specified by UFSHCI specification. Advertise all such + * quirks to standard UFS host controller driver so standard takes them into + * account. + */ +static void ufs_qcom_advertise_quirks(struct ufs_hba *hba) +{ + const struct ufs_qcom_drvdata *drvdata = of_device_get_match_data(hba->dev); + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + + if (host->hw_ver.major == 0x2) + hba->quirks |= UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION; + + if (host->hw_ver.major > 0x3) + hba->quirks |= UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH; + + if (host->hw_ver.major == 0x7 && host->hw_ver.minor == 0x1) + hba->quirks |= UFSHCD_QUIRK_EXTENDED_TX_EQTR_ADAPT_LENGTH_L0L1L2L3; + + if (drvdata && drvdata->quirks) + hba->quirks |= drvdata->quirks; +} + +static void ufs_qcom_set_phy_gear(struct ufs_qcom_host *host) +{ + struct ufs_host_params *host_params = &host->host_params; + u32 val, dev_major; + + /* + * Default to powering up the PHY to the max gear possible, which is + * backwards compatible with lower gears but not optimal from + * a power usage point of view. After device negotiation, if the + * gear is lower a reinit will be performed to program the PHY + * to the ideal gear for this combo of controller and device. + */ + host->phy_gear = host_params->hs_tx_gear; + + if (host->hw_ver.major < 0x4) { + /* + * These controllers only have one PHY init sequence, + * let's power up the PHY using that (the minimum supported + * gear, UFS_HS_G2). + */ + host->phy_gear = UFS_HS_G2; + } else if (host->hw_ver.major >= 0x5) { + val = ufshcd_readl(host->hba, REG_UFS_DEBUG_SPARE_CFG); + dev_major = FIELD_GET(UFS_DEV_VER_MAJOR_MASK, val); + + /* + * Since the UFS device version is populated, let's remove the + * REINIT quirk as the negotiated gear won't change during boot. + * So there is no need to do reinit. + */ + if (dev_major != 0x0) + host->hba->quirks &= ~UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH; + + /* + * For UFS 3.1 device and older, power up the PHY using HS-G4 + * PHY gear to save power. + */ + if (dev_major > 0x0 && dev_major < 0x4) + host->phy_gear = UFS_HS_G4; + } +} + +static void ufs_qcom_parse_gear_limits(struct ufs_hba *hba) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + struct ufs_host_params *host_params = &host->host_params; + u32 hs_gear_old = host_params->hs_tx_gear; + + ufshcd_parse_gear_limits(hba, host_params); + if (host_params->hs_tx_gear != hs_gear_old) { + host->phy_gear = host_params->hs_tx_gear; + } +} + +static void ufs_qcom_set_host_params(struct ufs_hba *hba) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + struct ufs_host_params *host_params = &host->host_params; + + ufshcd_init_host_params(host_params); + + /* This driver only supports symmetic gear setting i.e., hs_tx_gear == hs_rx_gear */ + host_params->hs_tx_gear = host_params->hs_rx_gear = ufs_qcom_get_hs_gear(hba); +} + +static void ufs_qcom_set_host_caps(struct ufs_hba *hba) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + + if (host->hw_ver.major >= 0x5) + host->caps |= UFS_QCOM_CAP_ICE_CONFIG; +} + +static void ufs_qcom_set_caps(struct ufs_hba *hba) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + + hba->caps |= UFSHCD_CAP_CLK_GATING | UFSHCD_CAP_HIBERN8_WITH_CLK_GATING; + hba->caps |= UFSHCD_CAP_CLK_SCALING | UFSHCD_CAP_WB_WITH_CLK_SCALING; + hba->caps |= UFSHCD_CAP_AUTO_BKOPS_SUSPEND; + hba->caps |= UFSHCD_CAP_WB_EN; + hba->caps |= UFSHCD_CAP_AGGR_POWER_COLLAPSE; + hba->caps |= UFSHCD_CAP_RPM_AUTOSUSPEND; + + if (host->hw_ver.major >= 0x7) + hba->caps |= UFSHCD_CAP_TX_EQUALIZATION; + + ufs_qcom_set_host_caps(hba); +} + +/** + * ufs_qcom_setup_clocks - enables/disable clocks + * @hba: host controller instance + * @on: If true, enable clocks else disable them. + * @status: PRE_CHANGE or POST_CHANGE notify + * + * There are certain clocks which comes from the PHY so it needs + * to be managed together along with controller clocks which also + * provides a better power saving. Hence keep phy_power_off/on calls + * in ufs_qcom_setup_clocks, so that PHY's regulators & clks can be + * turned on/off along with UFS's clocks. + * + * Return: 0 on success, non-zero on failure. + */ +static int ufs_qcom_setup_clocks(struct ufs_hba *hba, bool on, + enum ufs_notify_change_status status) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + struct phy *phy; + int err; + + /* + * In case ufs_qcom_init() is not yet done, simply ignore. + * This ufs_qcom_setup_clocks() shall be called from + * ufs_qcom_init() after init is done. + */ + if (!host) + return 0; + + phy = host->generic_phy; + + switch (status) { + case PRE_CHANGE: + if (on) { + ufs_qcom_icc_update_bw(host); + if (ufs_qcom_is_link_hibern8(hba)) { + err = ufs_qcom_enable_lane_clks(host); + if (err) { + dev_err(hba->dev, "enable lane clks failed, ret=%d\n", err); + return err; + } + } + } else { + if (!ufs_qcom_is_link_active(hba)) { + /* disable device ref_clk */ + ufs_qcom_dev_ref_clk_ctrl(host, false); + } + + err = phy_power_off(phy); + if (err) { + dev_err(hba->dev, "phy power off failed, ret=%d\n", err); + return err; + } + } + break; + case POST_CHANGE: + if (on) { + err = phy_power_on(phy); + if (err) { + dev_err(hba->dev, "phy power on failed, ret = %d\n", err); + return err; + } + + /* enable the device ref clock for HS mode*/ + if (ufshcd_is_hs_mode(&hba->pwr_info)) + ufs_qcom_dev_ref_clk_ctrl(host, true); + } else { + if (ufs_qcom_is_link_hibern8(hba)) + ufs_qcom_disable_lane_clks(host); + + ufs_qcom_icc_set_bw(host, ufs_qcom_bw_table[MODE_MIN][0][0].mem_bw, + ufs_qcom_bw_table[MODE_MIN][0][0].cfg_bw); + } + break; + } + + return 0; +} + +static int +ufs_qcom_reset_assert(struct reset_controller_dev *rcdev, unsigned long id) +{ + struct ufs_qcom_host *host = rcdev_to_ufs_host(rcdev); + + ufs_qcom_assert_reset(host->hba); + /* provide 1ms delay to let the reset pulse propagate. */ + usleep_range(1000, 1100); + return 0; +} + +static int +ufs_qcom_reset_deassert(struct reset_controller_dev *rcdev, unsigned long id) +{ + struct ufs_qcom_host *host = rcdev_to_ufs_host(rcdev); + + ufs_qcom_deassert_reset(host->hba); + + /* + * after reset deassertion, phy will need all ref clocks, + * voltage, current to settle down before starting serdes. + */ + usleep_range(1000, 1100); + return 0; +} + +static const struct reset_control_ops ufs_qcom_reset_ops = { + .assert = ufs_qcom_reset_assert, + .deassert = ufs_qcom_reset_deassert, +}; + +static int ufs_qcom_icc_init(struct ufs_qcom_host *host) +{ + struct device *dev = host->hba->dev; + int ret; + + host->icc_ddr = devm_of_icc_get(dev, "ufs-ddr"); + if (IS_ERR(host->icc_ddr)) + return dev_err_probe(dev, PTR_ERR(host->icc_ddr), + "failed to acquire interconnect path\n"); + + host->icc_cpu = devm_of_icc_get(dev, "cpu-ufs"); + if (IS_ERR(host->icc_cpu)) + return dev_err_probe(dev, PTR_ERR(host->icc_cpu), + "failed to acquire interconnect path\n"); + + /* + * Set Maximum bandwidth vote before initializing the UFS controller and + * device. Ideally, a minimal interconnect vote would suffice for the + * initialization, but a max vote would allow faster initialization. + */ + ret = ufs_qcom_icc_set_bw(host, ufs_qcom_bw_table[MODE_MAX][0][0].mem_bw, + ufs_qcom_bw_table[MODE_MAX][0][0].cfg_bw); + if (ret < 0) + return dev_err_probe(dev, ret, "failed to set bandwidth request\n"); + + return 0; +} + +/** + * ufs_qcom_init - bind phy with controller + * @hba: host controller instance + * + * Binds PHY with controller and powers up PHY enabling clocks + * and regulators. + * + * Return: -EPROBE_DEFER if binding fails, returns negative error + * on phy power up failure and returns zero on success. + */ +static int ufs_qcom_init(struct ufs_hba *hba) +{ + int err; + struct device *dev = hba->dev; + struct ufs_qcom_host *host; + struct ufs_clk_info *clki; + const struct ufs_qcom_drvdata *drvdata = of_device_get_match_data(hba->dev); + + host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL); + if (!host) + return -ENOMEM; + + /* Make a two way bind between the qcom host and the hba */ + host->hba = hba; + ufshcd_set_variant(hba, host); + + /* Setup the optional reset control of HCI */ + host->core_reset = devm_reset_control_get_optional(hba->dev, "rst"); + if (IS_ERR(host->core_reset)) { + err = dev_err_probe(dev, PTR_ERR(host->core_reset), + "Failed to get reset control\n"); + goto out_variant_clear; + } + + /* Fire up the reset controller. Failure here is non-fatal. */ + host->rcdev.of_node = dev->of_node; + host->rcdev.ops = &ufs_qcom_reset_ops; + host->rcdev.owner = dev->driver->owner; + host->rcdev.nr_resets = 1; + err = devm_reset_controller_register(dev, &host->rcdev); + if (err) + dev_warn(dev, "Failed to register reset controller\n"); + + if (!has_acpi_companion(dev)) { + host->generic_phy = devm_phy_get(dev, "ufsphy"); + if (IS_ERR(host->generic_phy)) { + err = dev_err_probe(dev, PTR_ERR(host->generic_phy), "Failed to get PHY\n"); + goto out_variant_clear; + } + } + + err = ufs_qcom_icc_init(host); + if (err) + goto out_variant_clear; + + host->device_reset = devm_gpiod_get_optional(dev, "reset", + GPIOD_OUT_HIGH); + if (IS_ERR(host->device_reset)) { + err = dev_err_probe(dev, PTR_ERR(host->device_reset), + "Failed to acquire device reset gpio\n"); + goto out_variant_clear; + } + + ufs_qcom_get_controller_revision(hba, &host->hw_ver.major, + &host->hw_ver.minor, &host->hw_ver.step); + + host->dev_ref_clk_ctrl_mmio = hba->mmio_base + REG_UFS_CFG1; + host->dev_ref_clk_en_mask = BIT(26); + + list_for_each_entry(clki, &hba->clk_list_head, list) { + if (!strcmp(clki->name, "core_clk_unipro")) + clki->keep_link_active = true; + } + + err = ufs_qcom_init_lane_clks(host); + if (err) + goto out_variant_clear; + + ufs_qcom_set_caps(hba); + ufs_qcom_advertise_quirks(hba); + ufs_qcom_set_host_params(hba); + ufs_qcom_set_phy_gear(host); + ufs_qcom_parse_gear_limits(hba); + + err = ufs_qcom_ice_init(host); + if (err) + goto out_variant_clear; + + ufs_qcom_setup_clocks(hba, true, POST_CHANGE); + + ufs_qcom_get_default_testbus_cfg(host); + err = ufs_qcom_testbus_config(host); + if (err) + /* Failure is non-fatal */ + dev_warn(dev, "%s: failed to configure the testbus %d\n", + __func__, err); + + if (drvdata && drvdata->no_phy_retention) + hba->spm_lvl = UFS_PM_LVL_5; + + return 0; + +out_variant_clear: + ufshcd_set_variant(hba, NULL); + + return err; +} + +static void ufs_qcom_exit(struct ufs_hba *hba) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + + ufs_qcom_disable_lane_clks(host); + phy_power_off(host->generic_phy); + phy_exit(host->generic_phy); +} + +static int ufs_qcom_fw_managed_init(struct ufs_hba *hba) +{ + struct device *dev = hba->dev; + struct ufs_qcom_host *host; + int err; + + host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL); + if (!host) + return -ENOMEM; + + host->hba = hba; + ufshcd_set_variant(hba, host); + + ufs_qcom_get_controller_revision(hba, &host->hw_ver.major, + &host->hw_ver.minor, &host->hw_ver.step); + + err = ufs_qcom_ice_init(host); + if (err) + goto out_variant_clear; + + ufs_qcom_get_default_testbus_cfg(host); + err = ufs_qcom_testbus_config(host); + if (err) + /* Failure is non-fatal */ + dev_warn(dev, "Failed to configure the testbus %d\n", err); + + hba->caps |= UFSHCD_CAP_WB_EN; + + ufs_qcom_advertise_quirks(hba); + host->hba->quirks &= ~UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH; + + hba->spm_lvl = hba->rpm_lvl = hba->pm_lvl_min = UFS_PM_LVL_5; + + ufs_qcom_set_host_params(hba); + ufs_qcom_parse_gear_limits(hba); + + return 0; + +out_variant_clear: + ufshcd_set_variant(hba, NULL); + return err; +} + +static void ufs_qcom_fw_managed_exit(struct ufs_hba *hba) +{ + pm_runtime_put_sync(hba->dev); +} + +/** + * ufs_qcom_set_clk_40ns_cycles - Configure 40ns clk cycles + * + * @hba: host controller instance + * @cycles_in_1us: No of cycles in 1us to be configured + * + * Returns error if dme get/set configuration for 40ns fails + * and returns zero on success. + */ +static int ufs_qcom_set_clk_40ns_cycles(struct ufs_hba *hba, + u32 cycles_in_1us) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + u32 cycles_in_40ns; + u32 reg; + int err; + + /* + * UFS host controller V4.0.0 onwards needs to program + * PA_VS_CORE_CLK_40NS_CYCLES attribute per programmed + * frequency of unipro core clk of UFS host controller. + */ + if (host->hw_ver.major < 4) + return 0; + + /* + * Generic formulae for cycles_in_40ns = (freq_unipro/25) is not + * applicable for all frequencies. For ex: ceil(37.5 MHz/25) will + * be 2 and ceil(403 MHZ/25) will be 17 whereas Hardware + * specification expect to be 16. Hence use exact hardware spec + * mandated value for cycles_in_40ns instead of calculating using + * generic formulae. + */ + switch (cycles_in_1us) { + case UNIPRO_CORE_CLK_FREQ_403_MHZ: + cycles_in_40ns = 16; + break; + case UNIPRO_CORE_CLK_FREQ_300_MHZ: + cycles_in_40ns = 12; + break; + case UNIPRO_CORE_CLK_FREQ_201_5_MHZ: + cycles_in_40ns = 8; + break; + case UNIPRO_CORE_CLK_FREQ_150_MHZ: + cycles_in_40ns = 6; + break; + case UNIPRO_CORE_CLK_FREQ_100_MHZ: + cycles_in_40ns = 4; + break; + case UNIPRO_CORE_CLK_FREQ_75_MHZ: + cycles_in_40ns = 3; + break; + case UNIPRO_CORE_CLK_FREQ_37_5_MHZ: + cycles_in_40ns = 2; + break; + default: + dev_err(hba->dev, "UNIPRO clk freq %u MHz not supported\n", + cycles_in_1us); + return -EINVAL; + } + + err = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_VS_CORE_CLK_40NS_CYCLES), ®); + if (err) + return err; + + reg &= ~PA_VS_CORE_CLK_40NS_CYCLES_MASK; + reg |= cycles_in_40ns; + + return ufshcd_dme_set(hba, UIC_ARG_MIB(PA_VS_CORE_CLK_40NS_CYCLES), reg); +} + +static int ufs_qcom_set_core_clk_ctrl(struct ufs_hba *hba, bool is_scale_up, unsigned long freq) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + struct list_head *head = &hba->clk_list_head; + struct ufs_clk_info *clki; + u32 cycles_in_1us = 0; + u32 core_clk_ctrl_reg; + unsigned long clk_freq; + int err; + + if (hba->use_pm_opp && freq != ULONG_MAX) { + clk_freq = ufs_qcom_opp_freq_to_clk_freq(hba, freq, "core_clk_unipro"); + if (clk_freq) { + cycles_in_1us = ceil(clk_freq, HZ_PER_MHZ); + goto set_core_clk_ctrl; + } + } + + list_for_each_entry(clki, head, list) { + if (!IS_ERR_OR_NULL(clki->clk) && + !strcmp(clki->name, "core_clk_unipro")) { + if (!clki->max_freq) { + cycles_in_1us = 150; /* default for backwards compatibility */ + break; + } + + if (freq == ULONG_MAX) { + cycles_in_1us = ceil(clki->max_freq, HZ_PER_MHZ); + break; + } + + if (is_scale_up) + cycles_in_1us = ceil(clki->max_freq, HZ_PER_MHZ); + else + cycles_in_1us = ceil(clk_get_rate(clki->clk), HZ_PER_MHZ); + break; + } + } + +set_core_clk_ctrl: + err = ufshcd_dme_get(hba, + UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL), + &core_clk_ctrl_reg); + if (err) + return err; + + /* Bit mask is different for UFS host controller V4.0.0 onwards */ + if (host->hw_ver.major >= 4) { + if (!FIELD_FIT(CLK_1US_CYCLES_MASK_V4, cycles_in_1us)) + return -ERANGE; + core_clk_ctrl_reg &= ~CLK_1US_CYCLES_MASK_V4; + core_clk_ctrl_reg |= FIELD_PREP(CLK_1US_CYCLES_MASK_V4, cycles_in_1us); + } else { + if (!FIELD_FIT(CLK_1US_CYCLES_MASK, cycles_in_1us)) + return -ERANGE; + core_clk_ctrl_reg &= ~CLK_1US_CYCLES_MASK; + core_clk_ctrl_reg |= FIELD_PREP(CLK_1US_CYCLES_MASK, cycles_in_1us); + } + + /* Clear CORE_CLK_DIV_EN */ + core_clk_ctrl_reg &= ~DME_VS_CORE_CLK_CTRL_CORE_CLK_DIV_EN_BIT; + + err = ufshcd_dme_set(hba, + UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL), + core_clk_ctrl_reg); + if (err) + return err; + + /* Configure unipro core clk 40ns attribute */ + return ufs_qcom_set_clk_40ns_cycles(hba, cycles_in_1us); +} + +static int ufs_qcom_clk_scale_up_pre_change(struct ufs_hba *hba, unsigned long freq) +{ + int ret; + + ret = ufs_qcom_cfg_timers(hba, true, freq); + if (ret) { + dev_err(hba->dev, "%s ufs cfg timer failed\n", __func__); + return ret; + } + /* set unipro core clock attributes and clear clock divider */ + return ufs_qcom_set_core_clk_ctrl(hba, true, freq); +} + +static int ufs_qcom_clk_scale_up_post_change(struct ufs_hba *hba) +{ + return 0; +} + +static int ufs_qcom_clk_scale_down_pre_change(struct ufs_hba *hba) +{ + int err; + u32 core_clk_ctrl_reg; + + err = ufshcd_dme_get(hba, + UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL), + &core_clk_ctrl_reg); + + /* make sure CORE_CLK_DIV_EN is cleared */ + if (!err && + (core_clk_ctrl_reg & DME_VS_CORE_CLK_CTRL_CORE_CLK_DIV_EN_BIT)) { + core_clk_ctrl_reg &= ~DME_VS_CORE_CLK_CTRL_CORE_CLK_DIV_EN_BIT; + err = ufshcd_dme_set(hba, + UIC_ARG_MIB(DME_VS_CORE_CLK_CTRL), + core_clk_ctrl_reg); + } + + return err; +} + +static int ufs_qcom_clk_scale_down_post_change(struct ufs_hba *hba, unsigned long freq) +{ + int ret; + + ret = ufs_qcom_cfg_timers(hba, false, freq); + if (ret) { + dev_err(hba->dev, "%s: ufs_qcom_cfg_timers() failed\n", __func__); + return ret; + } + /* set unipro core clock attributes and clear clock divider */ + return ufs_qcom_set_core_clk_ctrl(hba, false, freq); +} + +static int ufs_qcom_clk_scale_notify(struct ufs_hba *hba, bool scale_up, + unsigned long target_freq, + enum ufs_notify_change_status status) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + int err; + + /* check the host controller state before sending hibern8 cmd */ + if (!ufshcd_is_hba_active(hba)) + return 0; + + if (status == PRE_CHANGE) { + err = ufshcd_uic_hibern8_enter(hba); + if (err) + return err; + if (scale_up) + err = ufs_qcom_clk_scale_up_pre_change(hba, target_freq); + else + err = ufs_qcom_clk_scale_down_pre_change(hba); + + if (err) { + ufshcd_uic_hibern8_exit(hba); + return err; + } + } else { + if (scale_up) + err = ufs_qcom_clk_scale_up_post_change(hba); + else + err = ufs_qcom_clk_scale_down_post_change(hba, target_freq); + + + if (err) { + ufshcd_uic_hibern8_exit(hba); + return err; + } + + ufs_qcom_icc_update_bw(host); + ufshcd_uic_hibern8_exit(hba); + } + + return 0; +} + +static void ufs_qcom_enable_test_bus(struct ufs_qcom_host *host) +{ + ufshcd_rmwl(host->hba, UFS_REG_TEST_BUS_EN, + UFS_REG_TEST_BUS_EN, REG_UFS_CFG1); + ufshcd_rmwl(host->hba, TEST_BUS_EN, TEST_BUS_EN, REG_UFS_CFG1); +} + +static void ufs_qcom_get_default_testbus_cfg(struct ufs_qcom_host *host) +{ + /* provide a legal default configuration */ + host->testbus.select_major = TSTBUS_UNIPRO; + host->testbus.select_minor = 37; +} + +static bool ufs_qcom_testbus_cfg_is_ok(struct ufs_qcom_host *host) +{ + if (host->testbus.select_major >= TSTBUS_MAX) { + dev_err(host->hba->dev, + "%s: UFS_CFG1[TEST_BUS_SEL} may not equal 0x%05X\n", + __func__, host->testbus.select_major); + return false; + } + + return true; +} + +int ufs_qcom_testbus_config(struct ufs_qcom_host *host) +{ + int reg; + int offset; + u32 mask = TEST_BUS_SUB_SEL_MASK; + + if (!host) + return -EINVAL; + + if (!ufs_qcom_testbus_cfg_is_ok(host)) + return -EPERM; + + switch (host->testbus.select_major) { + case TSTBUS_UAWM: + reg = UFS_TEST_BUS_CTRL_0; + offset = 24; + break; + case TSTBUS_UARM: + reg = UFS_TEST_BUS_CTRL_0; + offset = 16; + break; + case TSTBUS_TXUC: + reg = UFS_TEST_BUS_CTRL_0; + offset = 8; + break; + case TSTBUS_RXUC: + reg = UFS_TEST_BUS_CTRL_0; + offset = 0; + break; + case TSTBUS_DFC: + reg = UFS_TEST_BUS_CTRL_1; + offset = 24; + break; + case TSTBUS_TRLUT: + reg = UFS_TEST_BUS_CTRL_1; + offset = 16; + break; + case TSTBUS_TMRLUT: + reg = UFS_TEST_BUS_CTRL_1; + offset = 8; + break; + case TSTBUS_OCSC: + reg = UFS_TEST_BUS_CTRL_1; + offset = 0; + break; + case TSTBUS_WRAPPER: + reg = UFS_TEST_BUS_CTRL_2; + offset = 16; + break; + case TSTBUS_COMBINED: + reg = UFS_TEST_BUS_CTRL_2; + offset = 8; + break; + case TSTBUS_UTP_HCI: + reg = UFS_TEST_BUS_CTRL_2; + offset = 0; + break; + case TSTBUS_UNIPRO: + reg = UFS_UNIPRO_CFG; + offset = 20; + mask = 0xFFF; + break; + /* + * No need for a default case, since + * ufs_qcom_testbus_cfg_is_ok() checks that the configuration + * is legal + */ + } + mask <<= offset; + ufshcd_rmwl(host->hba, TEST_BUS_SEL, + (u32)host->testbus.select_major << 19, + REG_UFS_CFG1); + ufshcd_rmwl(host->hba, mask, + (u32)host->testbus.select_minor << offset, + reg); + ufs_qcom_enable_test_bus(host); + + return 0; +} + +static void ufs_qcom_dump_testbus(struct ufs_hba *hba) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + int i, j, nminor = 0, testbus_len = 0; + char *prefix; + + u32 *testbus __free(kfree) = kmalloc_array(256, sizeof(u32), GFP_KERNEL); + if (!testbus) + return; + + for (j = 0; j < TSTBUS_MAX; j++) { + nminor = testbus_info[j].nminor; + prefix = testbus_info[j].prefix; + host->testbus.select_major = j; + testbus_len = nminor * sizeof(u32); + for (i = 0; i < nminor; i++) { + host->testbus.select_minor = i; + ufs_qcom_testbus_config(host); + testbus[i] = ufshcd_readl(hba, UFS_TEST_BUS); + } + print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, + 16, 4, testbus, testbus_len, false); + } +} + +static int ufs_qcom_dump_regs(struct ufs_hba *hba, size_t offset, size_t len, + const char *prefix, void __iomem *base) +{ + size_t pos; + + if (offset % 4 != 0 || len % 4 != 0) + return -EINVAL; + + u32 *regs __free(kfree) = kzalloc(len, GFP_ATOMIC); + if (!regs) + return -ENOMEM; + + for (pos = 0; pos < len; pos += 4) + regs[pos / 4] = readl(base + offset + pos); + + print_hex_dump(KERN_ERR, prefix, + len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE, + 16, 4, regs, len, false); + + return 0; +} + +static void ufs_qcom_dump_mcq_hci_regs(struct ufs_hba *hba) +{ + struct ufshcd_mcq_opr_info_t *opr = &hba->mcq_opr[0]; + void __iomem *mcq_vs_base = hba->mcq_base + UFS_MEM_VS_BASE; + + struct dump_info { + void __iomem *base; + size_t offset; + size_t len; + const char *prefix; + }; + + struct dump_info mcq_dumps[] = { + {hba->mcq_base, 0x0, 256 * 4, "MCQ HCI-0 "}, + {hba->mcq_base, 0x400, 256 * 4, "MCQ HCI-1 "}, + {mcq_vs_base, 0x0, 5 * 4, "MCQ VS-0 "}, + {opr->base, 0x0, 256 * 4, "MCQ SQD-0 "}, + {opr->base, 0x400, 256 * 4, "MCQ SQD-1 "}, + {opr->base, 0x800, 256 * 4, "MCQ SQD-2 "}, + {opr->base, 0xc00, 256 * 4, "MCQ SQD-3 "}, + {opr->base, 0x1000, 256 * 4, "MCQ SQD-4 "}, + {opr->base, 0x1400, 256 * 4, "MCQ SQD-5 "}, + {opr->base, 0x1800, 256 * 4, "MCQ SQD-6 "}, + {opr->base, 0x1c00, 256 * 4, "MCQ SQD-7 "}, + + }; + + for (int i = 0; i < ARRAY_SIZE(mcq_dumps); i++) { + ufs_qcom_dump_regs(hba, mcq_dumps[i].offset, mcq_dumps[i].len, + mcq_dumps[i].prefix, mcq_dumps[i].base); + cond_resched(); + } +} + +static void ufs_qcom_dump_dbg_regs(struct ufs_hba *hba) +{ + u32 reg; + struct ufs_qcom_host *host; + + host = ufshcd_get_variant(hba); + + dev_err(hba->dev, "HW_H8_ENTER_CNT=%d\n", ufshcd_readl(hba, REG_UFS_HW_H8_ENTER_CNT)); + dev_err(hba->dev, "HW_H8_EXIT_CNT=%d\n", ufshcd_readl(hba, REG_UFS_HW_H8_EXIT_CNT)); + + dev_err(hba->dev, "SW_H8_ENTER_CNT=%d\n", ufshcd_readl(hba, REG_UFS_SW_H8_ENTER_CNT)); + dev_err(hba->dev, "SW_H8_EXIT_CNT=%d\n", ufshcd_readl(hba, REG_UFS_SW_H8_EXIT_CNT)); + + dev_err(hba->dev, "SW_AFTER_HW_H8_ENTER_CNT=%d\n", + ufshcd_readl(hba, REG_UFS_SW_AFTER_HW_H8_ENTER_CNT)); + + ufshcd_dump_regs(hba, REG_UFS_SYS1CLK_1US, 16 * 4, + "HCI Vendor Specific Registers "); + + reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_REG_OCSC); + ufshcd_dump_regs(hba, reg, 44 * 4, "UFS_UFS_DBG_RD_REG_OCSC "); + + reg = ufshcd_readl(hba, REG_UFS_CFG1); + reg |= UTP_DBG_RAMS_EN; + ufshcd_writel(hba, reg, REG_UFS_CFG1); + + reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_EDTL_RAM); + ufshcd_dump_regs(hba, reg, 32 * 4, "UFS_UFS_DBG_RD_EDTL_RAM "); + + reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_DESC_RAM); + ufshcd_dump_regs(hba, reg, 128 * 4, "UFS_UFS_DBG_RD_DESC_RAM "); + + reg = ufs_qcom_get_debug_reg_offset(host, UFS_UFS_DBG_RD_PRDT_RAM); + ufshcd_dump_regs(hba, reg, 64 * 4, "UFS_UFS_DBG_RD_PRDT_RAM "); + + /* clear bit 17 - UTP_DBG_RAMS_EN */ + ufshcd_rmwl(hba, UTP_DBG_RAMS_EN, 0, REG_UFS_CFG1); + + reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_UAWM); + ufshcd_dump_regs(hba, reg, 4 * 4, "UFS_DBG_RD_REG_UAWM "); + + reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_UARM); + ufshcd_dump_regs(hba, reg, 4 * 4, "UFS_DBG_RD_REG_UARM "); + + reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_TXUC); + ufshcd_dump_regs(hba, reg, 48 * 4, "UFS_DBG_RD_REG_TXUC "); + + reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_RXUC); + ufshcd_dump_regs(hba, reg, 27 * 4, "UFS_DBG_RD_REG_RXUC "); + + reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_DFC); + ufshcd_dump_regs(hba, reg, 19 * 4, "UFS_DBG_RD_REG_DFC "); + + reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_TRLUT); + ufshcd_dump_regs(hba, reg, 34 * 4, "UFS_DBG_RD_REG_TRLUT "); + + reg = ufs_qcom_get_debug_reg_offset(host, UFS_DBG_RD_REG_TMRLUT); + ufshcd_dump_regs(hba, reg, 9 * 4, "UFS_DBG_RD_REG_TMRLUT "); + + if (hba->mcq_enabled) { + reg = ufs_qcom_get_debug_reg_offset(host, UFS_RD_REG_MCQ); + ufshcd_dump_regs(hba, reg, 64 * 4, "HCI MCQ Debug Registers "); + } + + /* ensure below dumps occur only in task context due to blocking calls. */ + if (in_task()) { + /* Dump MCQ Host Vendor Specific Registers */ + if (hba->mcq_enabled) + ufs_qcom_dump_mcq_hci_regs(hba); + + /* voluntarily yield the CPU as we are dumping too much data */ + ufshcd_dump_regs(hba, UFS_TEST_BUS, 4, "UFS_TEST_BUS "); + cond_resched(); + ufs_qcom_dump_testbus(hba); + } +} + +/** + * ufs_qcom_device_reset() - toggle the (optional) device reset line + * @hba: per-adapter instance + * + * Toggles the (optional) reset line to reset the attached device. + */ +static int ufs_qcom_device_reset(struct ufs_hba *hba) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + + /* reset gpio is optional */ + if (!host->device_reset) + return -EOPNOTSUPP; + + /* + * The UFS device shall detect reset pulses of 1us, sleep for 10us to + * be on the safe side. + */ + ufs_qcom_device_reset_ctrl(hba, true); + usleep_range(10, 15); + + ufs_qcom_device_reset_ctrl(hba, false); + usleep_range(10, 15); + + return 0; +} + +/** + * ufs_qcom_fw_managed_device_reset - Reset UFS device under FW-managed design + * @hba: pointer to UFS host bus adapter + * + * In the firmware-managed reset model, the power domain is powered on by genpd + * before the UFS controller driver probes. For subsequent resets (such as + * suspend/resume or recovery), the UFS driver must explicitly invoke PM runtime + * + * Return: 0 on success or a negative error code on failure. + */ +static int ufs_qcom_fw_managed_device_reset(struct ufs_hba *hba) +{ + static bool is_boot = true; + int err; + + /* Skip reset on cold boot; perform it on subsequent calls */ + if (is_boot) { + is_boot = false; + return 0; + } + + pm_runtime_put_sync(hba->dev); + err = pm_runtime_resume_and_get(hba->dev); + if (err < 0) { + dev_err(hba->dev, "PM runtime resume failed: %d\n", err); + return err; + } + + return 0; +} + +static void ufs_qcom_config_scaling_param(struct ufs_hba *hba, + struct devfreq_dev_profile *p, + struct devfreq_simple_ondemand_data *d) +{ + p->polling_ms = 60; + p->timer = DEVFREQ_TIMER_DELAYED; + d->upthreshold = 70; + d->downdifferential = 5; + + hba->clk_scaling.suspend_on_no_request = true; +} + +static int ufs_qcom_mcq_config_resource(struct ufs_hba *hba) +{ + struct platform_device *pdev = to_platform_device(hba->dev); + struct resource *res; + + /* Map the MCQ configuration region */ + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "mcq"); + if (!res) { + dev_err(hba->dev, "MCQ resource not found in device tree\n"); + return -ENODEV; + } + + hba->mcq_base = devm_ioremap_resource(hba->dev, res); + if (IS_ERR(hba->mcq_base)) { + dev_err(hba->dev, "Failed to map MCQ region: %ld\n", + PTR_ERR(hba->mcq_base)); + return PTR_ERR(hba->mcq_base); + } + + return 0; +} + +static int ufs_qcom_op_runtime_config(struct ufs_hba *hba) +{ + struct ufshcd_mcq_opr_info_t *opr; + int i; + u32 doorbell_offsets[OPR_MAX]; + + /* + * Configure doorbell address offsets in MCQ configuration registers. + * These values are offsets relative to mmio_base (UFS_HCI_BASE). + * + * Memory Layout: + * - mmio_base = UFS_HCI_BASE + * - mcq_base = MCQ_CONFIG_BASE = mmio_base + (UFS_QCOM_MCQCAP_QCFGPTR * 0x200) + * - Doorbell registers are at: mmio_base + (UFS_QCOM_MCQCAP_QCFGPTR * 0x200) + + * - UFS_QCOM_MCQ_SQD_OFFSET + * - Which is also: mcq_base + UFS_QCOM_MCQ_SQD_OFFSET + */ + + doorbell_offsets[OPR_SQD] = UFS_QCOM_SQD_ADDR_OFFSET; + doorbell_offsets[OPR_SQIS] = UFS_QCOM_SQIS_ADDR_OFFSET; + doorbell_offsets[OPR_CQD] = UFS_QCOM_CQD_ADDR_OFFSET; + doorbell_offsets[OPR_CQIS] = UFS_QCOM_CQIS_ADDR_OFFSET; + + /* + * Configure MCQ operation registers. + * + * The doorbell registers are physically located within the MCQ region: + * - doorbell_physical_addr = mmio_base + doorbell_offset + * - doorbell_physical_addr = mcq_base + (doorbell_offset - MCQ_CONFIG_OFFSET) + */ + for (i = 0; i < OPR_MAX; i++) { + opr = &hba->mcq_opr[i]; + opr->offset = doorbell_offsets[i]; /* Offset relative to mmio_base */ + opr->stride = UFS_QCOM_MCQ_STRIDE; /* 256 bytes between queues */ + + /* + * Calculate the actual doorbell base address within MCQ region: + * base = mcq_base + (doorbell_offset - MCQ_CONFIG_OFFSET) + */ + opr->base = hba->mcq_base + (opr->offset - UFS_QCOM_MCQ_CONFIG_OFFSET); + } + + return 0; +} + +static int ufs_qcom_get_hba_mac(struct ufs_hba *hba) +{ + /* Qualcomm HC supports up to 64 */ + return MAX_SUPP_MAC; +} + +static int ufs_qcom_get_outstanding_cqs(struct ufs_hba *hba, + unsigned long *ocqs) +{ + /* Read from MCQ vendor-specific register in MCQ region */ + *ocqs = readl(hba->mcq_base + UFS_MEM_CQIS_VS); + + return 0; +} + +static void ufs_qcom_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) +{ + struct device *dev = msi_desc_to_dev(desc); + struct ufs_hba *hba = dev_get_drvdata(dev); + + ufshcd_mcq_config_esi(hba, msg); +} + +struct ufs_qcom_irq { + unsigned int irq; + unsigned int idx; + struct ufs_hba *hba; +}; + +static irqreturn_t ufs_qcom_mcq_esi_handler(int irq, void *data) +{ + struct ufs_qcom_irq *qi = data; + struct ufs_hba *hba = qi->hba; + struct ufs_hw_queue *hwq = &hba->uhq[qi->idx]; + + ufshcd_mcq_write_cqis(hba, 0x1, qi->idx); + ufshcd_mcq_poll_cqe_lock(hba, hwq); + + return IRQ_HANDLED; +} + +static int ufs_qcom_config_esi(struct ufs_hba *hba) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + int nr_irqs, ret; + + if (host->esi_enabled) + return 0; + + /* + * 1. We only handle CQs as of now. + * 2. Poll queues do not need ESI. + */ + nr_irqs = hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL]; + + ret = platform_device_msi_init_and_alloc_irqs(hba->dev, nr_irqs, + ufs_qcom_write_msi_msg); + if (ret) { + dev_warn(hba->dev, "Platform MSI not supported or failed, continuing without ESI\n"); + return ret; /* Continue without ESI */ + } + + struct ufs_qcom_irq *qi = devm_kcalloc(hba->dev, nr_irqs, sizeof(*qi), GFP_KERNEL); + + if (!qi) { + platform_device_msi_free_irqs_all(hba->dev); + return -ENOMEM; + } + + for (int idx = 0; idx < nr_irqs; idx++) { + qi[idx].irq = msi_get_virq(hba->dev, idx); + qi[idx].idx = idx; + qi[idx].hba = hba; + + ret = devm_request_irq(hba->dev, qi[idx].irq, ufs_qcom_mcq_esi_handler, + IRQF_SHARED, "qcom-mcq-esi", qi + idx); + if (ret) { + dev_err(hba->dev, "%s: Failed to request IRQ for %d, err = %d\n", + __func__, qi[idx].irq, ret); + /* Free previously allocated IRQs */ + for (int j = 0; j < idx; j++) + devm_free_irq(hba->dev, qi[j].irq, qi + j); + platform_device_msi_free_irqs_all(hba->dev); + devm_kfree(hba->dev, qi); + return ret; + } + } + + if (host->hw_ver.major >= 6) { + ufshcd_rmwl(hba, ESI_VEC_MASK, FIELD_PREP(ESI_VEC_MASK, MAX_ESI_VEC - 1), + REG_UFS_CFG3); + } + ufshcd_mcq_enable_esi(hba); + host->esi_enabled = true; + return 0; +} + +static unsigned long ufs_qcom_opp_freq_to_clk_freq(struct ufs_hba *hba, + unsigned long freq, char *name) +{ + struct ufs_clk_info *clki; + struct dev_pm_opp *opp; + unsigned long clk_freq; + int idx = 0; + bool found = false; + + opp = dev_pm_opp_find_freq_exact_indexed(hba->dev, freq, 0, true); + if (IS_ERR(opp)) { + dev_err(hba->dev, "Failed to find OPP for exact frequency %lu\n", freq); + return 0; + } + + list_for_each_entry(clki, &hba->clk_list_head, list) { + if (!strcmp(clki->name, name)) { + found = true; + break; + } + + idx++; + } + + if (!found) { + dev_err(hba->dev, "Failed to find clock '%s' in clk list\n", name); + dev_pm_opp_put(opp); + return 0; + } + + clk_freq = dev_pm_opp_get_freq_indexed(opp, idx); + + dev_pm_opp_put(opp); + + return clk_freq; +} + +static u32 ufs_qcom_freq_to_gear_speed(struct ufs_hba *hba, unsigned long freq) +{ + u32 gear = UFS_HS_DONT_CHANGE; + unsigned long unipro_freq; + + if (!hba->use_pm_opp) + return gear; + + unipro_freq = ufs_qcom_opp_freq_to_clk_freq(hba, freq, "core_clk_unipro"); + switch (unipro_freq) { + case 403000000: + gear = UFS_HS_G5; + break; + case 300000000: + gear = UFS_HS_G4; + break; + case 201500000: + gear = UFS_HS_G3; + break; + case 150000000: + case 100000000: + gear = UFS_HS_G2; + break; + case 75000000: + case 37500000: + gear = UFS_HS_G1; + break; + default: + dev_err(hba->dev, "%s: Unsupported clock freq : %lu\n", __func__, freq); + return UFS_HS_DONT_CHANGE; + } + + return min_t(u32, gear, hba->max_pwr_info.info.gear_rx); +} + +static int ufs_qcom_host_eom_config(struct ufs_hba *hba, int lane, + const struct ufs_eom_coord *eom_coord, + u32 target_test_count) +{ + enum ufs_eom_eye_mask eye_mask = eom_coord->eye_mask; + int v_step = eom_coord->v_step; + int t_step = eom_coord->t_step; + u32 volt_step, timing_step; + int ret; + + if (abs(v_step) > UFS_QCOM_EOM_VOLTAGE_STEPS_MAX) { + dev_err(hba->dev, "Invalid EOM Voltage Step: %d\n", v_step); + return -ERANGE; + } + + if (abs(t_step) > UFS_QCOM_EOM_TIMING_STEPS_MAX) { + dev_err(hba->dev, "Invalid EOM Timing Step: %d\n", t_step); + return -ERANGE; + } + + if (v_step < 0) + volt_step = RX_EYEMON_NEGATIVE_STEP_BIT | (u32)(-v_step); + else + volt_step = (u32)v_step; + + if (t_step < 0) + timing_step = RX_EYEMON_NEGATIVE_STEP_BIT | (u32)(-t_step); + else + timing_step = (u32)t_step; + + ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_EYEMON_ENABLE, + UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)), + BIT(eye_mask) | RX_EYEMON_EXTENDED_VRANGE_BIT); + if (ret) { + dev_err(hba->dev, "Failed to enable Host EOM on Lane %d: %d\n", + lane, ret); + return ret; + } + + ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_EYEMON_TIMING_STEPS, + UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)), + timing_step); + if (ret) { + dev_err(hba->dev, "Failed to set Host EOM timing step on Lane %d: %d\n", + lane, ret); + return ret; + } + + ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_EYEMON_VOLTAGE_STEPS, + UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)), + volt_step); + if (ret) { + dev_err(hba->dev, "Failed to set Host EOM voltage step on Lane %d: %d\n", + lane, ret); + return ret; + } + + ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_EYEMON_TARGET_TEST_COUNT, + UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)), + target_test_count); + if (ret) + dev_err(hba->dev, "Failed to set Host EOM target test count on Lane %d: %d\n", + lane, ret); + + return ret; +} + +static int ufs_qcom_host_eom_may_stop(struct ufs_hba *hba, int lane, + u32 target_test_count, u32 *err_count) +{ + u32 start, tested_count, error_count; + int ret; + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(RX_EYEMON_START, + UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)), + &start); + if (ret) { + dev_err(hba->dev, "Failed to get Host EOM start status on Lane %d: %d\n", + lane, ret); + return ret; + } + + if (start & 0x1) + return -EAGAIN; + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(RX_EYEMON_TESTED_COUNT, + UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)), + &tested_count); + if (ret) { + dev_err(hba->dev, "Failed to get Host EOM tested count on Lane %d: %d\n", + lane, ret); + return ret; + } + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(RX_EYEMON_ERROR_COUNT, + UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)), + &error_count); + if (ret) { + dev_err(hba->dev, "Failed to get Host EOM error count on Lane %d: %d\n", + lane, ret); + return ret; + } + + /* EOM can stop */ + if ((tested_count >= target_test_count - 3) || error_count > 0) { + *err_count = error_count; + + /* Disable EOM */ + ret = ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_EYEMON_ENABLE, + UIC_ARG_MPHY_RX_GEN_SEL_INDEX(lane)), + 0x0); + if (ret) { + dev_err(hba->dev, "Failed to disable Host EOM on Lane %d: %d\n", + lane, ret); + return ret; + } + } else { + return -EAGAIN; + } + + return 0; +} + +static int ufs_qcom_host_eom_scan(struct ufs_hba *hba, int num_lanes, + const struct ufs_eom_coord *eom_coord, + u32 target_test_count, u32 *err_count) +{ + bool eom_stopped[PA_MAXDATALANES] = { 0 }; + int lane, ret; + u32 setting; + + if (!err_count || !eom_coord) + return -EINVAL; + + if (target_test_count < UFS_QCOM_EOM_TARGET_TEST_COUNT_MIN) { + dev_err(hba->dev, "Target test count (%u) too small for Host EOM\n", + target_test_count); + return -ERANGE; + } + + for (lane = 0; lane < num_lanes; lane++) { + ret = ufs_qcom_host_eom_config(hba, lane, eom_coord, + target_test_count); + if (ret) { + dev_err(hba->dev, "Failed to config Host RX EOM: %d\n", ret); + return ret; + } + } + + /* + * Trigger a PACP_PWR_req to kick start EOM, but not to really change + * the Power Mode. + */ + ret = ufshcd_uic_change_pwr_mode(hba, FAST_MODE << 4 | FAST_MODE); + if (ret) { + dev_err(hba->dev, "Failed to change power mode to kick start Host EOM: %d\n", + ret); + return ret; + } + +more_burst: + /* Create burst on Host RX Lane. */ + ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_LOCALVERINFO), &setting); + + for (lane = 0; lane < num_lanes; lane++) { + if (eom_stopped[lane]) + continue; + + ret = ufs_qcom_host_eom_may_stop(hba, lane, target_test_count, + &err_count[lane]); + if (!ret) { + eom_stopped[lane] = true; + } else if (ret == -EAGAIN) { + /* Need more burst to excercise EOM */ + goto more_burst; + } else { + dev_err(hba->dev, "Failed to stop Host EOM: %d\n", ret); + return ret; + } + + dev_dbg(hba->dev, "Host RX Lane %d EOM, v_step %d, t_step %d, error count %u\n", + lane, eom_coord->v_step, eom_coord->t_step, + err_count[lane]); + } + + return 0; +} + +static int ufs_qcom_host_sw_rx_fom(struct ufs_hba *hba, int num_lanes, u32 *fom) +{ + const struct ufs_eom_coord *eom_coord = sw_rx_fom_eom_coords_g6; + u32 eom_err_count[PA_MAXDATALANES] = { 0 }; + u32 curr_ahit; + int lane, i, ret; + + if (!fom) + return -EINVAL; + + /* Stop the auto hibernate idle timer */ + curr_ahit = ufshcd_readl(hba, REG_AUTO_HIBERNATE_IDLE_TIMER); + if (curr_ahit) + ufshcd_writel(hba, 0, REG_AUTO_HIBERNATE_IDLE_TIMER); + + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXHSADAPTTYPE), PA_NO_ADAPT); + if (ret) { + dev_err(hba->dev, "Failed to select NO_ADAPT before starting Host EOM: %d\n", ret); + goto out; + } + + for (i = 0; i < SW_RX_FOM_EOM_COORDS; i++, eom_coord++) { + ret = ufs_qcom_host_eom_scan(hba, num_lanes, eom_coord, + UFS_QCOM_EOM_TARGET_TEST_COUNT_G6, + eom_err_count); + if (ret) { + dev_err(hba->dev, "Failed to run Host EOM scan: %d\n", ret); + break; + } + + for (lane = 0; lane < num_lanes; lane++) { + /* Bad coordinates have no weights */ + if (eom_err_count[lane]) + continue; + fom[lane] += SW_RX_FOM_EOM_COORDS_WEIGHT; + } + } + +out: + /* Restore the auto hibernate idle timer */ + if (curr_ahit) + ufshcd_writel(hba, curr_ahit, REG_AUTO_HIBERNATE_IDLE_TIMER); + + return ret; +} + +static int ufs_qcom_get_rx_fom(struct ufs_hba *hba, + struct ufs_pa_layer_attr *pwr_mode, + struct tx_eqtr_iter *h_iter, + struct tx_eqtr_iter *d_iter) +{ + struct ufshcd_tx_eq_params *params __free(kfree) = + kzalloc(sizeof(*params), GFP_KERNEL); + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + struct ufs_pa_layer_attr old_pwr_info; + u32 fom[PA_MAXDATALANES] = { 0 }; + u32 gear = pwr_mode->gear_tx; + u32 rate = pwr_mode->hs_rate; + int lane, ret; + + if (host->hw_ver.major != 0x7 || host->hw_ver.minor > 0x1 || + gear <= UFS_HS_G5 || !d_iter || !d_iter->is_updated) + return 0; + + if (gear < UFS_HS_G1 || gear > UFS_HS_GEAR_MAX) + return -ERANGE; + + if (!params) + return -ENOMEM; + + memcpy(&old_pwr_info, &hba->pwr_info, sizeof(struct ufs_pa_layer_attr)); + + memcpy(params, &hba->tx_eq_params[gear - 1], sizeof(struct ufshcd_tx_eq_params)); + for (lane = 0; lane < pwr_mode->lane_rx; lane++) { + params->device[lane].preshoot = d_iter->preshoot; + params->device[lane].deemphasis = d_iter->deemphasis; + } + + /* Use TX EQTR settings as Device's TX Equalization settings. */ + ret = ufshcd_apply_tx_eq_settings(hba, params, gear); + if (ret) { + dev_err(hba->dev, "%s: Failed to apply TX EQ settings for HS-G%u: %d\n", + __func__, gear, ret); + return ret; + } + + /* Force PMC to target HS Gear to use new TX Equalization settings. */ + ret = ufshcd_change_power_mode(hba, pwr_mode, UFSHCD_PMC_POLICY_FORCE); + if (ret) { + dev_err(hba->dev, "%s: Failed to change power mode to HS-G%u, Rate-%s: %d\n", + __func__, gear, ufs_hs_rate_to_str(rate), ret); + return ret; + } + + ret = ufs_qcom_host_sw_rx_fom(hba, pwr_mode->lane_rx, fom); + if (ret) { + dev_err(hba->dev, "Failed to get SW FOM of TX (PreShoot: %u, DeEmphasis: %u): %d\n", + d_iter->preshoot, d_iter->deemphasis, ret); + return ret; + } + + /* Restore Device's TX Equalization settings. */ + ret = ufshcd_apply_tx_eq_settings(hba, &hba->tx_eq_params[gear - 1], gear); + if (ret) { + dev_err(hba->dev, "%s: Failed to apply TX EQ settings for HS-G%u: %d\n", + __func__, gear, ret); + return ret; + } + + /* Restore Power Mode. */ + ret = ufshcd_change_power_mode(hba, &old_pwr_info, UFSHCD_PMC_POLICY_FORCE); + if (ret) { + dev_err(hba->dev, "%s: Failed to restore power mode to HS-G%u: %d\n", + __func__, old_pwr_info.gear_tx, ret); + return ret; + } + + for (lane = 0; lane < pwr_mode->lane_rx; lane++) + d_iter->fom[lane] = fom[lane]; + + return 0; +} + +static int ufs_qcom_apply_tx_eqtr_settings(struct ufs_hba *hba, + struct ufs_pa_layer_attr *pwr_mode, + struct tx_eqtr_iter *h_iter, + struct tx_eqtr_iter *d_iter) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + u32 setting = 0; + int lane; + + if (host->hw_ver.major != 0x7 || host->hw_ver.minor > 0x1) + return 0; + + for (lane = 0; lane < pwr_mode->lane_tx; lane++) { + setting |= TX_HS_PRESHOOT_BITS(lane, h_iter->preshoot); + setting |= TX_HS_DEEMPHASIS_BITS(lane, h_iter->deemphasis); + } + + return ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXEQG1SETTING), setting); +} + +static int ufs_qcom_tx_eqtr_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status, + struct ufs_pa_layer_attr *pwr_mode) +{ + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + struct ufs_pa_layer_attr pwr_mode_hs_g1 = { + .gear_rx = UFS_HS_G1, + .gear_tx = UFS_HS_G1, + .lane_rx = pwr_mode->lane_rx, + .lane_tx = pwr_mode->lane_tx, + .pwr_rx = FAST_MODE, + .pwr_tx = FAST_MODE, + .hs_rate = pwr_mode->hs_rate, + }; + u32 gear = pwr_mode->gear_tx; + u32 rate = pwr_mode->hs_rate; + int ret; + + if (host->hw_ver.major != 0x7 || host->hw_ver.minor > 0x1) + return 0; + + if (status == PRE_CHANGE) { + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TXEQG1SETTING), + &host->saved_tx_eq_g1_setting); + if (ret) + return ret; + + /* PMC to target HS Gear. */ + ret = ufshcd_change_power_mode(hba, pwr_mode, + UFSHCD_PMC_POLICY_DONT_FORCE); + if (ret) + dev_err(hba->dev, "%s: Failed to PMC to target HS-G%u, Rate-%s: %d\n", + __func__, gear, ufs_hs_rate_to_str(rate), ret); + } else { + ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXEQG1SETTING), + host->saved_tx_eq_g1_setting); + if (ret) + return ret; + + /* PMC back to HS-G1. */ + ret = ufshcd_change_power_mode(hba, &pwr_mode_hs_g1, + UFSHCD_PMC_POLICY_DONT_FORCE); + if (ret) + dev_err(hba->dev, "%s: Failed to PMC to HS-G1, Rate-%s: %d\n", + __func__, ufs_hs_rate_to_str(rate), ret); + } + + return ret; +} + +/* + * struct ufs_hba_qcom_vops - UFS QCOM specific variant operations + * + * The variant operations configure the necessary controller and PHY + * handshake during initialization. + */ +static const struct ufs_hba_variant_ops ufs_hba_qcom_vops = { + .name = "qcom", + .init = ufs_qcom_init, + .exit = ufs_qcom_exit, + .get_ufs_hci_version = ufs_qcom_get_ufs_hci_version, + .clk_scale_notify = ufs_qcom_clk_scale_notify, + .setup_clocks = ufs_qcom_setup_clocks, + .hce_enable_notify = ufs_qcom_hce_enable_notify, + .link_startup_notify = ufs_qcom_link_startup_notify, + .negotiate_pwr_mode = ufs_qcom_negotiate_pwr_mode, + .pwr_change_notify = ufs_qcom_pwr_change_notify, + .apply_dev_quirks = ufs_qcom_apply_dev_quirks, + .fixup_dev_quirks = ufs_qcom_fixup_dev_quirks, + .suspend = ufs_qcom_suspend, + .resume = ufs_qcom_resume, + .dbg_register_dump = ufs_qcom_dump_dbg_regs, + .device_reset = ufs_qcom_device_reset, + .config_scaling_param = ufs_qcom_config_scaling_param, + .mcq_config_resource = ufs_qcom_mcq_config_resource, + .get_hba_mac = ufs_qcom_get_hba_mac, + .op_runtime_config = ufs_qcom_op_runtime_config, + .get_outstanding_cqs = ufs_qcom_get_outstanding_cqs, + .config_esi = ufs_qcom_config_esi, + .freq_to_gear_speed = ufs_qcom_freq_to_gear_speed, + .get_rx_fom = ufs_qcom_get_rx_fom, + .apply_tx_eqtr_settings = ufs_qcom_apply_tx_eqtr_settings, + .tx_eqtr_notify = ufs_qcom_tx_eqtr_notify, +}; + +static const struct ufs_hba_variant_ops ufs_hba_qcom_sa8255p_vops = { + .name = "qcom-sa8255p", + .init = ufs_qcom_fw_managed_init, + .exit = ufs_qcom_fw_managed_exit, + .hce_enable_notify = ufs_qcom_fw_managed_hce_enable_notify, + .pwr_change_notify = ufs_qcom_pwr_change_notify, + .apply_dev_quirks = ufs_qcom_apply_dev_quirks, + .fixup_dev_quirks = ufs_qcom_fixup_dev_quirks, + .suspend = ufs_qcom_fw_managed_suspend, + .resume = ufs_qcom_fw_managed_resume, + .dbg_register_dump = ufs_qcom_dump_dbg_regs, + .device_reset = ufs_qcom_fw_managed_device_reset, +}; + +/** + * ufs_qcom_probe - probe routine of the driver + * @pdev: pointer to Platform device handle + * + * Return: zero for success and non-zero for failure. + */ +static int ufs_qcom_probe(struct platform_device *pdev) +{ + int err; + struct device *dev = &pdev->dev; + const struct ufs_hba_variant_ops *vops; + const struct ufs_qcom_drvdata *drvdata = device_get_match_data(dev); + + if (drvdata && drvdata->vops) + vops = drvdata->vops; + else + vops = &ufs_hba_qcom_vops; + + /* Perform generic probe */ + err = ufshcd_pltfrm_init(pdev, vops); + if (err) + return dev_err_probe(dev, err, "ufshcd_pltfrm_init() failed\n"); + + return 0; +} + +/** + * ufs_qcom_remove - set driver_data of the device to NULL + * @pdev: pointer to platform device handle + * + * Always returns 0 + */ +static void ufs_qcom_remove(struct platform_device *pdev) +{ + struct ufs_hba *hba = platform_get_drvdata(pdev); + struct ufs_qcom_host *host = ufshcd_get_variant(hba); + + ufshcd_pltfrm_remove(pdev); + if (host->esi_enabled) + platform_device_msi_free_irqs_all(hba->dev); +} + +static const struct ufs_qcom_drvdata ufs_qcom_sm8550_drvdata = { + .quirks = UFSHCD_QUIRK_BROKEN_LSDBS_CAP, + .no_phy_retention = true, +}; + +static const struct ufs_qcom_drvdata ufs_qcom_sa8255p_drvdata = { + .vops = &ufs_hba_qcom_sa8255p_vops +}; + +static const struct of_device_id ufs_qcom_of_match[] __maybe_unused = { + { .compatible = "qcom,ufshc" }, + { .compatible = "qcom,sm8550-ufshc", .data = &ufs_qcom_sm8550_drvdata }, + { .compatible = "qcom,sm8650-ufshc", .data = &ufs_qcom_sm8550_drvdata }, + { .compatible = "qcom,sa8255p-ufshc", .data = &ufs_qcom_sa8255p_drvdata }, + {}, +}; +MODULE_DEVICE_TABLE(of, ufs_qcom_of_match); + +#ifdef CONFIG_ACPI +static const struct acpi_device_id ufs_qcom_acpi_match[] = { + { "QCOM24A5" }, + { }, +}; +MODULE_DEVICE_TABLE(acpi, ufs_qcom_acpi_match); +#endif + +static const struct dev_pm_ops ufs_qcom_pm_ops = { + SET_RUNTIME_PM_OPS(ufshcd_runtime_suspend, ufshcd_runtime_resume, NULL) + .prepare = ufshcd_suspend_prepare, + .complete = ufshcd_resume_complete, +#ifdef CONFIG_PM_SLEEP + .suspend = ufshcd_system_suspend, + .resume = ufshcd_system_resume, + .freeze = ufshcd_system_freeze, + .restore = ufshcd_system_restore, + .thaw = ufshcd_system_thaw, +#endif +}; + +static struct platform_driver ufs_qcom_pltform = { + .probe = ufs_qcom_probe, + .remove = ufs_qcom_remove, + .driver = { + .name = "ufshcd-qcom", + .pm = &ufs_qcom_pm_ops, + .of_match_table = of_match_ptr(ufs_qcom_of_match), + .acpi_match_table = ACPI_PTR(ufs_qcom_acpi_match), + }, +}; +module_platform_driver(ufs_qcom_pltform); + +MODULE_DESCRIPTION("Qualcomm UFS host controller driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/ufs/host/ufs-qcom.h b/drivers/ufs/host/ufs-qcom.h new file mode 100644 index 000000000..e20b3ca50 --- /dev/null +++ b/drivers/ufs/host/ufs-qcom.h @@ -0,0 +1,388 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2013-2015, The Linux Foundation. All rights reserved. + */ + +#ifndef UFS_QCOM_H_ +#define UFS_QCOM_H_ + +#include <linux/reset-controller.h> +#include <linux/reset.h> +#include <soc/qcom/ice.h> +#include <ufs/ufshcd.h> + +#define MPHY_TX_FSM_STATE 0x41 +#define TX_FSM_HIBERN8 0x1 +#define HBRN8_POLL_TOUT_MS 100 +#define DEFAULT_CLK_RATE_HZ 1000000 +#define MAX_SUPP_MAC 64 +#define MAX_ESI_VEC 32 + +#define UFS_HW_VER_MAJOR_MASK GENMASK(31, 28) +#define UFS_HW_VER_MINOR_MASK GENMASK(27, 16) +#define UFS_HW_VER_STEP_MASK GENMASK(15, 0) +#define UFS_DEV_VER_MAJOR_MASK GENMASK(7, 4) + +#define UFS_QCOM_LIMIT_HS_RATE PA_HS_MODE_B + +/* bit and mask definitions for PA_VS_CLK_CFG_REG attribute */ +#define PA_VS_CLK_CFG_REG 0x9004 +#define PA_VS_CLK_CFG_REG_MASK GENMASK(8, 0) + +/* bit and mask definitions for DL_VS_CLK_CFG attribute */ +#define DL_VS_CLK_CFG 0xA00B +#define DL_VS_CLK_CFG_MASK GENMASK(9, 0) +#define DME_VS_CORE_CLK_CTRL_DME_HW_CGC_EN BIT(9) + +#define UFS_QCOM_EOM_VOLTAGE_STEPS_MAX 127 +#define UFS_QCOM_EOM_TIMING_STEPS_MAX 63 +#define UFS_QCOM_EOM_TARGET_TEST_COUNT_MIN 8 +#define UFS_QCOM_EOM_TARGET_TEST_COUNT_G6 0x3F + +#define SW_RX_FOM_EOM_COORDS 23 +#define SW_RX_FOM_EOM_COORDS_WEIGHT (127 / SW_RX_FOM_EOM_COORDS) + +struct ufs_eom_coord { + int t_step; + int v_step; + u8 eye_mask; +}; + +static const struct ufs_eom_coord sw_rx_fom_eom_coords_g6[SW_RX_FOM_EOM_COORDS] = { + [0] = { -2, -15, UFS_EOM_EYE_MASK_M }, + [1] = { 0, -15, UFS_EOM_EYE_MASK_M }, + [2] = { 2, -15, UFS_EOM_EYE_MASK_M }, + [3] = { -4, -10, UFS_EOM_EYE_MASK_M }, + [4] = { -2, -10, UFS_EOM_EYE_MASK_M }, + [5] = { 0, -10, UFS_EOM_EYE_MASK_M }, + [6] = { 2, -10, UFS_EOM_EYE_MASK_M }, + [7] = { 4, -10, UFS_EOM_EYE_MASK_M }, + [8] = { -6, 0, UFS_EOM_EYE_MASK_M }, + [9] = { -4, 0, UFS_EOM_EYE_MASK_M }, + [10] = { -2, 0, UFS_EOM_EYE_MASK_M }, + [11] = { 0, 0, UFS_EOM_EYE_MASK_M }, + [12] = { 2, 0, UFS_EOM_EYE_MASK_M }, + [13] = { 4, 0, UFS_EOM_EYE_MASK_M }, + [14] = { 6, 0, UFS_EOM_EYE_MASK_M }, + [15] = { -4, 10, UFS_EOM_EYE_MASK_M }, + [16] = { -2, 10, UFS_EOM_EYE_MASK_M }, + [17] = { 0, 10, UFS_EOM_EYE_MASK_M }, + [18] = { 2, 10, UFS_EOM_EYE_MASK_M }, + [19] = { 4, 10, UFS_EOM_EYE_MASK_M }, + [20] = { -2, 15, UFS_EOM_EYE_MASK_M }, + [21] = { 0, 15, UFS_EOM_EYE_MASK_M }, + [22] = { 2, 15, UFS_EOM_EYE_MASK_M }, +}; + +/* Qualcomm MCQ Configuration */ +#define UFS_QCOM_MCQCAP_QCFGPTR 224 /* 0xE0 in hex */ +#define UFS_QCOM_MCQ_CONFIG_OFFSET (UFS_QCOM_MCQCAP_QCFGPTR * 0x200) /* 0x1C000 */ + +/* Doorbell offsets within MCQ region (relative to MCQ_CONFIG_BASE) */ +#define UFS_QCOM_MCQ_SQD_OFFSET 0x5000 +#define UFS_QCOM_MCQ_CQD_OFFSET 0x5080 +#define UFS_QCOM_MCQ_SQIS_OFFSET 0x5040 +#define UFS_QCOM_MCQ_CQIS_OFFSET 0x50C0 +#define UFS_QCOM_MCQ_STRIDE 0x100 + +/* Calculated doorbell address offsets (relative to mmio_base) */ +#define UFS_QCOM_SQD_ADDR_OFFSET (UFS_QCOM_MCQ_CONFIG_OFFSET + UFS_QCOM_MCQ_SQD_OFFSET) +#define UFS_QCOM_CQD_ADDR_OFFSET (UFS_QCOM_MCQ_CONFIG_OFFSET + UFS_QCOM_MCQ_CQD_OFFSET) +#define UFS_QCOM_SQIS_ADDR_OFFSET (UFS_QCOM_MCQ_CONFIG_OFFSET + UFS_QCOM_MCQ_SQIS_OFFSET) +#define UFS_QCOM_CQIS_ADDR_OFFSET (UFS_QCOM_MCQ_CONFIG_OFFSET + UFS_QCOM_MCQ_CQIS_OFFSET) +#define REG_UFS_MCQ_STRIDE UFS_QCOM_MCQ_STRIDE + +/* MCQ Vendor specific address offsets (relative to MCQ_CONFIG_BASE) */ +#define UFS_MEM_VS_BASE 0x4000 +#define UFS_MEM_CQIS_VS 0x4008 + +/* QCOM UFS host controller vendor specific registers */ +enum { + REG_UFS_SYS1CLK_1US = 0xC0, + REG_UFS_TX_SYMBOL_CLK_NS_US = 0xC4, + REG_UFS_LOCAL_PORT_ID_REG = 0xC8, + REG_UFS_PA_ERR_CODE = 0xCC, + /* On older UFS revisions, this register is called "RETRY_TIMER_REG" */ + REG_UFS_PARAM0 = 0xD0, + /* On older UFS revisions, this register is called "REG_UFS_PA_LINK_STARTUP_TIMER" */ + REG_UFS_CFG0 = 0xD8, + REG_UFS_CFG1 = 0xDC, + REG_UFS_CFG2 = 0xE0, + REG_UFS_HW_VERSION = 0xE4, + + UFS_TEST_BUS = 0xE8, + UFS_TEST_BUS_CTRL_0 = 0xEC, + UFS_TEST_BUS_CTRL_1 = 0xF0, + UFS_TEST_BUS_CTRL_2 = 0xF4, + UFS_UNIPRO_CFG = 0xF8, + + /* + * QCOM UFS host controller vendor specific registers + * added in HW Version 3.0.0 + */ + UFS_AH8_CFG = 0xFC, + + UFS_RD_REG_MCQ = 0xD00, + UFS_MEM_ICE_CFG = 0x2600, + REG_UFS_MEM_ICE_CONFIG = 0x260C, + REG_UFS_MEM_ICE_NUM_CORE = 0x2664, + + REG_UFS_CFG3 = 0x271C, + + REG_UFS_DEBUG_SPARE_CFG = 0x284C, +}; + +/* QCOM UFS host controller vendor specific debug registers */ +enum { + UFS_DBG_RD_REG_UAWM = 0x100, + UFS_DBG_RD_REG_UARM = 0x200, + UFS_DBG_RD_REG_TXUC = 0x300, + UFS_DBG_RD_REG_RXUC = 0x400, + UFS_DBG_RD_REG_DFC = 0x500, + UFS_DBG_RD_REG_TRLUT = 0x600, + UFS_DBG_RD_REG_TMRLUT = 0x700, + UFS_UFS_DBG_RD_REG_OCSC = 0x800, + + UFS_UFS_DBG_RD_DESC_RAM = 0x1500, + UFS_UFS_DBG_RD_PRDT_RAM = 0x1700, + UFS_UFS_DBG_RD_RESP_RAM = 0x1800, + UFS_UFS_DBG_RD_EDTL_RAM = 0x1900, +}; + +/* QCOM UFS HC vendor specific Hibern8 count registers */ +enum { + REG_UFS_HW_H8_ENTER_CNT = 0x2700, + REG_UFS_SW_H8_ENTER_CNT = 0x2704, + REG_UFS_SW_AFTER_HW_H8_ENTER_CNT = 0x2708, + REG_UFS_HW_H8_EXIT_CNT = 0x270C, + REG_UFS_SW_H8_EXIT_CNT = 0x2710, +}; + +#define UFS_CNTLR_2_x_x_VEN_REGS_OFFSET(x) (0x000 + x) +#define UFS_CNTLR_3_x_x_VEN_REGS_OFFSET(x) (0x400 + x) + +/* bit definitions for REG_UFS_CFG0 register */ +#define QUNIPRO_G4_SEL BIT(5) + +/* bit definitions for REG_UFS_CFG1 register */ +#define QUNIPRO_SEL BIT(0) +#define UFS_PHY_SOFT_RESET BIT(1) +#define UTP_DBG_RAMS_EN BIT(17) +#define TEST_BUS_EN BIT(18) +#define TEST_BUS_SEL GENMASK(22, 19) +#define UFS_REG_TEST_BUS_EN BIT(30) + +/* bit definitions for REG_UFS_CFG2 register */ +#define UAWM_HW_CGC_EN BIT(0) +#define UARM_HW_CGC_EN BIT(1) +#define TXUC_HW_CGC_EN BIT(2) +#define RXUC_HW_CGC_EN BIT(3) +#define DFC_HW_CGC_EN BIT(4) +#define TRLUT_HW_CGC_EN BIT(5) +#define TMRLUT_HW_CGC_EN BIT(6) +#define OCSC_HW_CGC_EN BIT(7) + +/* bit definitions for REG_UFS_CFG3 register */ +#define ESI_VEC_MASK GENMASK(22, 12) + +/* bit definitions for REG_UFS_PARAM0 */ +#define MAX_HS_GEAR_MASK GENMASK(6, 4) +#define UFS_QCOM_MAX_GEAR(x) FIELD_GET(MAX_HS_GEAR_MASK, (x)) + +/* bit definition for UFS_UFS_TEST_BUS_CTRL_n */ +#define TEST_BUS_SUB_SEL_MASK GENMASK(4, 0) /* All XXX_SEL fields are 5 bits wide */ + +/* bit definition for UFS Shared ICE config */ +#define UFS_QCOM_CAP_ICE_CONFIG BIT(0) + +#define REG_UFS_CFG2_CGC_EN_ALL (UAWM_HW_CGC_EN | UARM_HW_CGC_EN |\ + TXUC_HW_CGC_EN | RXUC_HW_CGC_EN |\ + DFC_HW_CGC_EN | TRLUT_HW_CGC_EN |\ + TMRLUT_HW_CGC_EN | OCSC_HW_CGC_EN) + +/* QUniPro Vendor specific attributes */ +#define PA_TX_HSG1_SYNC_LENGTH 0x1552 +#define PA_VS_CONFIG_REG1 0x9000 +#define DME_VS_CORE_CLK_CTRL 0xD002 +#define TX_HS_EQUALIZER 0x0037 + +/* bit and mask definitions for DME_VS_CORE_CLK_CTRL attribute */ +#define CLK_1US_CYCLES_MASK_V4 GENMASK(27, 16) +#define CLK_1US_CYCLES_MASK GENMASK(7, 0) +#define DME_VS_CORE_CLK_CTRL_CORE_CLK_DIV_EN_BIT BIT(8) +#define PA_VS_CORE_CLK_40NS_CYCLES 0x9007 +#define PA_VS_CORE_CLK_40NS_CYCLES_MASK GENMASK(6, 0) + + +/* QCOM UFS host controller core clk frequencies */ +#define UNIPRO_CORE_CLK_FREQ_37_5_MHZ 38 +#define UNIPRO_CORE_CLK_FREQ_75_MHZ 75 +#define UNIPRO_CORE_CLK_FREQ_100_MHZ 100 +#define UNIPRO_CORE_CLK_FREQ_150_MHZ 150 +#define UNIPRO_CORE_CLK_FREQ_300_MHZ 300 +#define UNIPRO_CORE_CLK_FREQ_201_5_MHZ 202 +#define UNIPRO_CORE_CLK_FREQ_403_MHZ 403 + +/* TX_HSG1_SYNC_LENGTH attr value */ +#define PA_TX_HSG1_SYNC_LENGTH_VAL 0x4A + +/* + * Some ufs device vendors need a different TSync length. + * Enable this quirk to give an additional TX_HS_SYNC_LENGTH. + */ +#define UFS_DEVICE_QUIRK_PA_TX_HSG1_SYNC_LENGTH BIT(16) + +/* + * Some ufs device vendors need a different Deemphasis setting. + * Enable this quirk to tune TX Deemphasis parameters. + */ +#define UFS_DEVICE_QUIRK_PA_TX_DEEMPHASIS_TUNING BIT(17) + +/* ICE allocator type to share AES engines among TX stream and RX stream */ +#define ICE_ALLOCATOR_TYPE 2 + +/* + * Number of cores allocated for RX stream when Read data block received and + * Write data block is not in progress + */ +#define NUM_RX_R1W0 28 + +/* + * Number of cores allocated for TX stream when Device asked to send write + * data block and Read data block is not in progress + */ +#define NUM_TX_R0W1 28 + +/* + * Number of cores allocated for RX stream when Read data block received and + * Write data block is in progress + * OR + * Device asked to send write data block and Read data block is in progress + */ +#define NUM_RX_R1W1 15 + +/* + * Number of cores allocated for TX stream (UFS write) when Read data block + * received and Write data block is in progress + * OR + * Device asked to send write data block and Read data block is in progress + */ +#define NUM_TX_R1W1 13 + +/* bit definitions for UFS_AH8_CFG register */ +#define CC_UFS_SYS_CLK_REQ_EN BIT(2) +#define CC_UFS_ICE_CORE_CLK_REQ_EN BIT(3) +#define CC_UFS_UNIPRO_CORE_CLK_REQ_EN BIT(4) +#define CC_UFS_AUXCLK_REQ_EN BIT(5) + +#define UFS_HW_CLK_CTRL_EN (CC_UFS_SYS_CLK_REQ_EN |\ + CC_UFS_ICE_CORE_CLK_REQ_EN |\ + CC_UFS_UNIPRO_CORE_CLK_REQ_EN |\ + CC_UFS_AUXCLK_REQ_EN) + +static inline void +ufs_qcom_get_controller_revision(struct ufs_hba *hba, + u8 *major, u16 *minor, u16 *step) +{ + u32 ver = ufshcd_readl(hba, REG_UFS_HW_VERSION); + + *major = FIELD_GET(UFS_HW_VER_MAJOR_MASK, ver); + *minor = FIELD_GET(UFS_HW_VER_MINOR_MASK, ver); + *step = FIELD_GET(UFS_HW_VER_STEP_MASK, ver); +}; + +static inline void ufs_qcom_assert_reset(struct ufs_hba *hba) +{ + ufshcd_rmwl(hba, UFS_PHY_SOFT_RESET, UFS_PHY_SOFT_RESET, REG_UFS_CFG1); + + /* + * Dummy read to ensure the write takes effect before doing any sort + * of delay + */ + ufshcd_readl(hba, REG_UFS_CFG1); +} + +static inline void ufs_qcom_deassert_reset(struct ufs_hba *hba) +{ + ufshcd_rmwl(hba, UFS_PHY_SOFT_RESET, 0, REG_UFS_CFG1); + + /* + * Dummy read to ensure the write takes effect before doing any sort + * of delay + */ + ufshcd_readl(hba, REG_UFS_CFG1); +} + +/* Host controller hardware version: major.minor.step */ +struct ufs_hw_version { + u16 step; + u16 minor; + u8 major; +}; + +struct ufs_qcom_testbus { + u8 select_major; + u8 select_minor; +}; + +struct gpio_desc; + +struct ufs_qcom_host { + struct phy *generic_phy; + struct ufs_hba *hba; + struct ufs_pa_layer_attr dev_req_params; + struct clk_bulk_data *clks; + u32 num_clks; + bool is_lane_clks_enabled; + + struct icc_path *icc_ddr; + struct icc_path *icc_cpu; + +#ifdef CONFIG_SCSI_UFS_CRYPTO + struct qcom_ice *ice; +#endif + u32 caps; + void __iomem *dev_ref_clk_ctrl_mmio; + bool is_dev_ref_clk_enabled; + struct ufs_hw_version hw_ver; + + u32 dev_ref_clk_en_mask; + + struct ufs_qcom_testbus testbus; + + /* Reset control of HCI */ + struct reset_control *core_reset; + struct reset_controller_dev rcdev; + + struct gpio_desc *device_reset; + + struct ufs_host_params host_params; + u32 phy_gear; + + bool esi_enabled; + + u32 saved_tx_eq_g1_setting; +}; + +struct ufs_qcom_drvdata { + enum ufshcd_quirks quirks; + bool no_phy_retention; + const struct ufs_hba_variant_ops *vops; +}; + +static inline u32 +ufs_qcom_get_debug_reg_offset(struct ufs_qcom_host *host, u32 reg) +{ + if (host->hw_ver.major <= 0x02) + return UFS_CNTLR_2_x_x_VEN_REGS_OFFSET(reg); + + return UFS_CNTLR_3_x_x_VEN_REGS_OFFSET(reg); +}; + +#define ufs_qcom_is_link_off(hba) ufshcd_is_link_off(hba) +#define ufs_qcom_is_link_active(hba) ufshcd_is_link_active(hba) +#define ufs_qcom_is_link_hibern8(hba) ufshcd_is_link_hibern8(hba) +#define ceil(freq, div) ((freq) % (div) == 0 ? ((freq)/(div)) : ((freq)/(div) + 1)) + +int ufs_qcom_testbus_config(struct ufs_qcom_host *host); + +#endif /* UFS_QCOM_H_ */ diff --git a/drivers/ufs/host/ufs-renesas.c b/drivers/ufs/host/ufs-renesas.c new file mode 100644 index 000000000..5bf7d0e77 --- /dev/null +++ b/drivers/ufs/host/ufs-renesas.c @@ -0,0 +1,558 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Renesas UFS host controller driver + * + * Copyright (C) 2022 Renesas Electronics Corporation + */ + +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/err.h> +#include <linux/firmware.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/nvmem-consumer.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/sys_soc.h> +#include <ufs/ufshcd.h> + +#include "ufshcd-pltfrm.h" + +#define EFUSE_CALIB_SIZE 8 + +struct ufs_renesas_priv { + const struct firmware *fw; + void (*pre_init)(struct ufs_hba *hba); + bool initialized; /* The hardware needs initialization once */ + u8 calib[EFUSE_CALIB_SIZE]; +}; + +#define UFS_RENESAS_FIRMWARE_NAME "r8a779f0_ufs.bin" +MODULE_FIRMWARE(UFS_RENESAS_FIRMWARE_NAME); + +static void ufs_renesas_dbg_register_dump(struct ufs_hba *hba) +{ + ufshcd_dump_regs(hba, 0xc0, 0x40, "regs: 0xc0 + "); +} + +static void ufs_renesas_poll(struct ufs_hba *hba, u32 reg, u32 expected, u32 mask) +{ + int ret; + u32 val; + + ret = readl_poll_timeout_atomic(hba->mmio_base + reg, + val, (val & mask) == expected, + 10, 1000); + if (ret) + dev_err(hba->dev, "%s: poll failed %d (%08x, %08x, %08x)\n", + __func__, ret, val, mask, expected); +} + +static u32 ufs_renesas_read(struct ufs_hba *hba, u32 reg) +{ + return ufshcd_readl(hba, reg); +} + +static void ufs_renesas_write(struct ufs_hba *hba, u32 reg, u32 value) +{ + ufshcd_writel(hba, value, reg); +} + +static void ufs_renesas_write_d0_d4(struct ufs_hba *hba, u32 data_d0, u32 data_d4) +{ + ufs_renesas_write(hba, 0xd0, data_d0); + ufs_renesas_write(hba, 0xd4, data_d4); +} + +static void ufs_renesas_write_800_80c_poll(struct ufs_hba *hba, u32 addr, + u32 data_800) +{ + ufs_renesas_write_d0_d4(hba, 0x0000080c, 0x00000100); + ufs_renesas_write_d0_d4(hba, 0x00000800, (data_800 << 16) | BIT(8) | addr); + ufs_renesas_write(hba, 0xd0, 0x0000080c); + ufs_renesas_poll(hba, 0xd4, BIT(8), BIT(8)); +} + +static void ufs_renesas_write_804_80c_poll(struct ufs_hba *hba, u32 addr, u32 data_804) +{ + ufs_renesas_write_d0_d4(hba, 0x0000080c, 0x00000100); + ufs_renesas_write_d0_d4(hba, 0x00000804, (data_804 << 16) | BIT(8) | addr); + ufs_renesas_write(hba, 0xd0, 0x0000080c); + ufs_renesas_poll(hba, 0xd4, BIT(8), BIT(8)); +} + +static void ufs_renesas_write_828_82c_poll(struct ufs_hba *hba, u32 data_828) +{ + ufs_renesas_write_d0_d4(hba, 0x0000082c, 0x0f000000); + ufs_renesas_write_d0_d4(hba, 0x00000828, data_828); + ufs_renesas_write(hba, 0xd0, 0x0000082c); + ufs_renesas_poll(hba, 0xd4, data_828, data_828); +} + +static void ufs_renesas_write_phy(struct ufs_hba *hba, u32 addr16, u32 data16) +{ + ufs_renesas_write(hba, 0xf0, 1); + ufs_renesas_write_800_80c_poll(hba, 0x16, addr16 & 0xff); + ufs_renesas_write_800_80c_poll(hba, 0x17, (addr16 >> 8) & 0xff); + ufs_renesas_write_800_80c_poll(hba, 0x18, data16 & 0xff); + ufs_renesas_write_800_80c_poll(hba, 0x19, (data16 >> 8) & 0xff); + ufs_renesas_write_800_80c_poll(hba, 0x1c, 0x01); + ufs_renesas_write_828_82c_poll(hba, 0x0f000000); + ufs_renesas_write(hba, 0xf0, 0); +} + +static void ufs_renesas_set_phy(struct ufs_hba *hba, u32 addr16, u32 data16) +{ + u32 low, high; + + ufs_renesas_write(hba, 0xf0, 1); + ufs_renesas_write_800_80c_poll(hba, 0x16, addr16 & 0xff); + ufs_renesas_write_800_80c_poll(hba, 0x17, (addr16 >> 8) & 0xff); + ufs_renesas_write_800_80c_poll(hba, 0x1c, 0x01); + ufs_renesas_write_828_82c_poll(hba, 0x0f000000); + ufs_renesas_write_804_80c_poll(hba, 0x1a, 0); + ufs_renesas_write(hba, 0xd0, 0x00000808); + low = ufs_renesas_read(hba, 0xd4) & 0xff; + ufs_renesas_write_804_80c_poll(hba, 0x1b, 0); + ufs_renesas_write(hba, 0xd0, 0x00000808); + high = ufs_renesas_read(hba, 0xd4) & 0xff; + ufs_renesas_write_828_82c_poll(hba, 0x0f000000); + ufs_renesas_write(hba, 0xf0, 0); + + data16 |= (high << 8) | low; + ufs_renesas_write_phy(hba, addr16, data16); +} + +static void ufs_renesas_reset_indirect_write(struct ufs_hba *hba, int gpio, + u32 addr, u32 data) +{ + ufs_renesas_write(hba, 0xf0, gpio); + ufs_renesas_write_800_80c_poll(hba, addr, data); +} + +static void ufs_renesas_reset_indirect_update(struct ufs_hba *hba) +{ + ufs_renesas_write_d0_d4(hba, 0x0000082c, 0x0f000000); + ufs_renesas_write_d0_d4(hba, 0x00000828, 0x0f000000); + ufs_renesas_write(hba, 0xd0, 0x0000082c); + ufs_renesas_poll(hba, 0xd4, BIT(27) | BIT(26) | BIT(24), BIT(27) | BIT(26) | BIT(24)); + ufs_renesas_write(hba, 0xf0, 0); +} + +static void ufs_renesas_indirect_write(struct ufs_hba *hba, u32 gpio, u32 addr, + u32 data_800) +{ + ufs_renesas_write(hba, 0xf0, gpio); + ufs_renesas_write_800_80c_poll(hba, addr, data_800); + ufs_renesas_write_828_82c_poll(hba, 0x0f000000); + ufs_renesas_write(hba, 0xf0, 0); +} + +static void ufs_renesas_indirect_poll(struct ufs_hba *hba, u32 gpio, u32 addr, + u32 expected, u32 mask) +{ + ufs_renesas_write(hba, 0xf0, gpio); + ufs_renesas_write_800_80c_poll(hba, addr, 0); + ufs_renesas_write(hba, 0xd0, 0x00000808); + ufs_renesas_poll(hba, 0xd4, expected, mask); + ufs_renesas_write(hba, 0xf0, 0); +} + +static void ufs_renesas_init_step1_to_3(struct ufs_hba *hba, bool init108) +{ + ufs_renesas_write(hba, 0xc0, 0x49425308); + ufs_renesas_write_d0_d4(hba, 0x00000104, 0x00000002); + if (init108) + ufs_renesas_write_d0_d4(hba, 0x00000108, 0x00000002); + udelay(1); + ufs_renesas_write_d0_d4(hba, 0x00000828, 0x00000200); + udelay(1); + ufs_renesas_write_d0_d4(hba, 0x00000828, 0x00000000); + ufs_renesas_write_d0_d4(hba, 0x00000104, 0x00000001); + if (init108) + ufs_renesas_write_d0_d4(hba, 0x00000108, 0x00000001); + ufs_renesas_write_d0_d4(hba, 0x00000940, 0x00000001); + udelay(1); + ufs_renesas_write_d0_d4(hba, 0x00000940, 0x00000000); + + ufs_renesas_write(hba, 0xc0, 0x49425308); + ufs_renesas_write(hba, 0xc0, 0x41584901); +} + +static void ufs_renesas_init_step4_to_6(struct ufs_hba *hba) +{ + ufs_renesas_write_d0_d4(hba, 0x0000080c, 0x00000100); + ufs_renesas_write_d0_d4(hba, 0x00000804, 0x00000000); + ufs_renesas_write(hba, 0xd0, 0x0000080c); + ufs_renesas_poll(hba, 0xd4, BIT(8), BIT(8)); + + ufs_renesas_write(hba, REG_CONTROLLER_ENABLE, 0x00000001); + + ufs_renesas_write(hba, 0xd0, 0x00000804); + ufs_renesas_poll(hba, 0xd4, BIT(8) | BIT(6) | BIT(0), BIT(8) | BIT(6) | BIT(0)); +} + +static u32 ufs_renesas_init_disable_timer(struct ufs_hba *hba) +{ + u32 timer_val; + + ufs_renesas_write(hba, 0xd0, 0x00000d00); + timer_val = ufs_renesas_read(hba, 0xd4) & 0x0000ffff; + ufs_renesas_write(hba, 0xd4, 0x00000000); + ufs_renesas_write_d0_d4(hba, 0x0000082c, 0x0f000000); + ufs_renesas_write_d0_d4(hba, 0x00000828, 0x08000000); + ufs_renesas_write(hba, 0xd0, 0x0000082c); + ufs_renesas_poll(hba, 0xd4, BIT(27), BIT(27)); + ufs_renesas_write(hba, 0xd0, 0x00000d2c); + ufs_renesas_poll(hba, 0xd4, BIT(0), BIT(0)); + + return timer_val; +} + +static void ufs_renesas_init_enable_timer(struct ufs_hba *hba, u32 timer_val) +{ + ufs_renesas_write(hba, 0xf0, 0); + ufs_renesas_write(hba, 0xd0, 0x00000d00); + ufs_renesas_write(hba, 0xd4, timer_val); +} + +static void ufs_renesas_write_phy_10ad_10af(struct ufs_hba *hba, + u32 data_10ad, u32 data_10af) +{ + ufs_renesas_write_phy(hba, 0x10ae, 0x0001); + ufs_renesas_write_phy(hba, 0x10ad, data_10ad); + ufs_renesas_write_phy(hba, 0x10af, data_10af); + ufs_renesas_write_phy(hba, 0x10b6, 0x0001); + ufs_renesas_write_phy(hba, 0x10ae, 0x0000); +} + +static void ufs_renesas_init_compensation_and_slicers(struct ufs_hba *hba) +{ + ufs_renesas_write_phy_10ad_10af(hba, 0x0000, 0x0001); + ufs_renesas_write_phy_10ad_10af(hba, 0x0000, 0x0002); + ufs_renesas_write_phy_10ad_10af(hba, 0x0080, 0x0000); + ufs_renesas_write_phy_10ad_10af(hba, 0x0080, 0x001a); +} + +static void ufs_renesas_r8a779f0_es10_pre_init(struct ufs_hba *hba) +{ + u32 timer_val; + + /* This setting is for SERIES B */ + ufs_renesas_init_step1_to_3(hba, false); + + ufs_renesas_init_step4_to_6(hba); + + timer_val = ufs_renesas_init_disable_timer(hba); + + /* phy setup */ + ufs_renesas_indirect_write(hba, 1, 0x01, 0x001f); + ufs_renesas_indirect_write(hba, 7, 0x5d, 0x0014); + ufs_renesas_indirect_write(hba, 7, 0x5e, 0x0014); + ufs_renesas_indirect_write(hba, 7, 0x0d, 0x0003); + ufs_renesas_indirect_write(hba, 7, 0x0e, 0x0007); + ufs_renesas_indirect_write(hba, 7, 0x5f, 0x0003); + ufs_renesas_indirect_write(hba, 7, 0x60, 0x0003); + ufs_renesas_indirect_write(hba, 7, 0x5b, 0x00a6); + ufs_renesas_indirect_write(hba, 7, 0x5c, 0x0003); + + ufs_renesas_indirect_poll(hba, 7, 0x3c, 0, BIT(7)); + ufs_renesas_indirect_poll(hba, 7, 0x4c, 0, BIT(4)); + + ufs_renesas_indirect_write(hba, 1, 0x32, 0x0080); + ufs_renesas_indirect_write(hba, 1, 0x1f, 0x0001); + ufs_renesas_indirect_write(hba, 0, 0x2c, 0x0001); + ufs_renesas_indirect_write(hba, 0, 0x32, 0x0087); + + ufs_renesas_indirect_write(hba, 1, 0x4d, 0x0061); + ufs_renesas_indirect_write(hba, 4, 0x9b, 0x0009); + ufs_renesas_indirect_write(hba, 4, 0xa6, 0x0005); + ufs_renesas_indirect_write(hba, 4, 0xa5, 0x0058); + ufs_renesas_indirect_write(hba, 1, 0x39, 0x0027); + ufs_renesas_indirect_write(hba, 1, 0x47, 0x004c); + + ufs_renesas_indirect_write(hba, 7, 0x0d, 0x0002); + ufs_renesas_indirect_write(hba, 7, 0x0e, 0x0007); + + ufs_renesas_write_phy(hba, 0x0028, 0x0061); + ufs_renesas_write_phy(hba, 0x4014, 0x0061); + ufs_renesas_set_phy(hba, 0x401c, BIT(2)); + ufs_renesas_write_phy(hba, 0x4000, 0x0000); + ufs_renesas_write_phy(hba, 0x4001, 0x0000); + + ufs_renesas_init_compensation_and_slicers(hba); + + ufs_renesas_indirect_write(hba, 7, 0x70, 0x0016); + ufs_renesas_indirect_write(hba, 7, 0x71, 0x0016); + ufs_renesas_indirect_write(hba, 7, 0x72, 0x0014); + ufs_renesas_indirect_write(hba, 7, 0x73, 0x0014); + ufs_renesas_indirect_write(hba, 7, 0x74, 0x0000); + ufs_renesas_indirect_write(hba, 7, 0x75, 0x0000); + ufs_renesas_indirect_write(hba, 7, 0x76, 0x0010); + ufs_renesas_indirect_write(hba, 7, 0x77, 0x0010); + ufs_renesas_indirect_write(hba, 7, 0x78, 0x00ff); + ufs_renesas_indirect_write(hba, 7, 0x79, 0x0000); + + ufs_renesas_indirect_write(hba, 7, 0x19, 0x0007); + ufs_renesas_indirect_write(hba, 7, 0x1a, 0x0007); + ufs_renesas_indirect_write(hba, 7, 0x24, 0x000c); + ufs_renesas_indirect_write(hba, 7, 0x25, 0x000c); + ufs_renesas_indirect_write(hba, 7, 0x62, 0x0000); + ufs_renesas_indirect_write(hba, 7, 0x63, 0x0000); + ufs_renesas_indirect_write(hba, 7, 0x5d, 0x0014); + ufs_renesas_indirect_write(hba, 7, 0x5e, 0x0017); + ufs_renesas_indirect_write(hba, 7, 0x5d, 0x0004); + ufs_renesas_indirect_write(hba, 7, 0x5e, 0x0017); + ufs_renesas_indirect_poll(hba, 7, 0x55, 0, BIT(6)); + ufs_renesas_indirect_poll(hba, 7, 0x41, 0, BIT(7)); + /* end of phy setup */ + + ufs_renesas_init_enable_timer(hba, timer_val); +} + +static void ufs_renesas_r8a779f0_init_step3_add(struct ufs_hba *hba, bool assert) +{ + u32 val_2x = 0, val_3x = 0, val_4x = 0; + + if (assert) { + val_2x = 0x0001; + val_3x = 0x0003; + val_4x = 0x0001; + } + + ufs_renesas_reset_indirect_write(hba, 7, 0x20, val_2x); + ufs_renesas_reset_indirect_write(hba, 7, 0x4a, val_4x); + ufs_renesas_reset_indirect_write(hba, 7, 0x35, val_3x); + ufs_renesas_reset_indirect_update(hba); + ufs_renesas_reset_indirect_write(hba, 7, 0x21, val_2x); + ufs_renesas_reset_indirect_write(hba, 7, 0x4b, val_4x); + ufs_renesas_reset_indirect_write(hba, 7, 0x36, val_3x); + ufs_renesas_reset_indirect_update(hba); +} + +static void ufs_renesas_r8a779f0_pre_init(struct ufs_hba *hba) +{ + struct ufs_renesas_priv *priv = ufshcd_get_variant(hba); + u32 timer_val; + u32 data; + int i; + + /* This setting is for SERIES B */ + ufs_renesas_init_step1_to_3(hba, true); + + ufs_renesas_r8a779f0_init_step3_add(hba, true); + ufs_renesas_reset_indirect_write(hba, 7, 0x5f, 0x0063); + ufs_renesas_reset_indirect_update(hba); + ufs_renesas_reset_indirect_write(hba, 7, 0x60, 0x0003); + ufs_renesas_reset_indirect_update(hba); + ufs_renesas_reset_indirect_write(hba, 7, 0x5b, 0x00a6); + ufs_renesas_reset_indirect_update(hba); + ufs_renesas_reset_indirect_write(hba, 7, 0x5c, 0x0003); + ufs_renesas_reset_indirect_update(hba); + ufs_renesas_r8a779f0_init_step3_add(hba, false); + + ufs_renesas_init_step4_to_6(hba); + + timer_val = ufs_renesas_init_disable_timer(hba); + + ufs_renesas_indirect_write(hba, 1, 0x01, 0x001f); + ufs_renesas_indirect_write(hba, 7, 0x5d, 0x0014); + ufs_renesas_indirect_write(hba, 7, 0x5e, 0x0014); + ufs_renesas_indirect_write(hba, 7, 0x0d, 0x0007); + ufs_renesas_indirect_write(hba, 7, 0x0e, 0x0007); + + ufs_renesas_indirect_poll(hba, 7, 0x3c, 0, BIT(7)); + ufs_renesas_indirect_poll(hba, 7, 0x4c, 0, BIT(4)); + + ufs_renesas_indirect_write(hba, 1, 0x32, 0x0080); + ufs_renesas_indirect_write(hba, 1, 0x1f, 0x0001); + ufs_renesas_indirect_write(hba, 1, 0x2c, 0x0001); + ufs_renesas_indirect_write(hba, 1, 0x32, 0x0087); + + ufs_renesas_indirect_write(hba, 1, 0x4d, priv->calib[2]); + ufs_renesas_indirect_write(hba, 1, 0x4e, priv->calib[3]); + ufs_renesas_indirect_write(hba, 1, 0x0d, 0x0006); + ufs_renesas_indirect_write(hba, 1, 0x0e, 0x0007); + ufs_renesas_write_phy(hba, 0x0028, priv->calib[3]); + ufs_renesas_write_phy(hba, 0x4014, priv->calib[3]); + + ufs_renesas_set_phy(hba, 0x401c, BIT(2)); + + ufs_renesas_write_phy(hba, 0x4000, priv->calib[6]); + ufs_renesas_write_phy(hba, 0x4001, priv->calib[7]); + + ufs_renesas_indirect_write(hba, 1, 0x14, 0x0001); + + ufs_renesas_init_compensation_and_slicers(hba); + + ufs_renesas_indirect_write(hba, 7, 0x79, 0x0000); + ufs_renesas_indirect_write(hba, 7, 0x24, 0x000c); + ufs_renesas_indirect_write(hba, 7, 0x25, 0x000c); + ufs_renesas_indirect_write(hba, 7, 0x62, 0x00c0); + ufs_renesas_indirect_write(hba, 7, 0x63, 0x0001); + + for (i = 0; i < priv->fw->size / 2; i++) { + data = (priv->fw->data[i * 2 + 1] << 8) | priv->fw->data[i * 2]; + ufs_renesas_write_phy(hba, 0xc000 + i, data); + } + + ufs_renesas_indirect_write(hba, 7, 0x0d, 0x0002); + ufs_renesas_indirect_write(hba, 7, 0x0e, 0x0007); + + ufs_renesas_indirect_write(hba, 7, 0x5d, 0x0014); + ufs_renesas_indirect_write(hba, 7, 0x5e, 0x0017); + ufs_renesas_indirect_write(hba, 7, 0x5d, 0x0004); + ufs_renesas_indirect_write(hba, 7, 0x5e, 0x0017); + ufs_renesas_indirect_poll(hba, 7, 0x55, 0, BIT(6)); + ufs_renesas_indirect_poll(hba, 7, 0x41, 0, BIT(7)); + + ufs_renesas_init_enable_timer(hba, timer_val); +} + +static int ufs_renesas_hce_enable_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status) +{ + struct ufs_renesas_priv *priv = ufshcd_get_variant(hba); + + if (priv->initialized) + return 0; + + if (status == PRE_CHANGE) + priv->pre_init(hba); + + priv->initialized = true; + + return 0; +} + +static int ufs_renesas_setup_clocks(struct ufs_hba *hba, bool on, + enum ufs_notify_change_status status) +{ + if (on && status == PRE_CHANGE) + pm_runtime_get_sync(hba->dev); + else if (!on && status == POST_CHANGE) + pm_runtime_put(hba->dev); + + return 0; +} + +static const struct soc_device_attribute ufs_fallback[] = { + { .soc_id = "r8a779f0", .revision = "ES1.[01]" }, + { /* Sentinel */ } +}; + +static int ufs_renesas_init(struct ufs_hba *hba) +{ + const struct soc_device_attribute *attr; + struct nvmem_cell *cell = NULL; + struct device *dev = hba->dev; + struct ufs_renesas_priv *priv; + u8 *data = NULL; + size_t len; + int ret; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + ufshcd_set_variant(hba, priv); + + hba->quirks |= UFSHCD_QUIRK_HIBERN_FASTAUTO; + + attr = soc_device_match(ufs_fallback); + if (attr) + goto fallback; + + ret = request_firmware(&priv->fw, UFS_RENESAS_FIRMWARE_NAME, dev); + if (ret) { + dev_warn(dev, "Failed to load firmware\n"); + goto fallback; + } + + cell = nvmem_cell_get(dev, "calibration"); + if (IS_ERR(cell)) { + dev_warn(dev, "No calibration data specified\n"); + goto fallback; + } + + data = nvmem_cell_read(cell, &len); + if (IS_ERR(data)) { + dev_warn(dev, "Failed to read calibration data: %pe\n", data); + goto fallback; + } + + if (len != EFUSE_CALIB_SIZE) { + dev_warn(dev, "Invalid calibration data size %zu\n", len); + goto fallback; + } + + memcpy(priv->calib, data, EFUSE_CALIB_SIZE); + priv->pre_init = ufs_renesas_r8a779f0_pre_init; + goto out; + +fallback: + dev_info(dev, "Using ES1.0 init code\n"); + priv->pre_init = ufs_renesas_r8a779f0_es10_pre_init; + +out: + kfree(data); + if (!IS_ERR_OR_NULL(cell)) + nvmem_cell_put(cell); + + return 0; +} + +static void ufs_renesas_exit(struct ufs_hba *hba) +{ + struct ufs_renesas_priv *priv = ufshcd_get_variant(hba); + + release_firmware(priv->fw); +} + +static int ufs_renesas_set_dma_mask(struct ufs_hba *hba) +{ + return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32)); +} + +static const struct ufs_hba_variant_ops ufs_renesas_vops = { + .name = "renesas", + .init = ufs_renesas_init, + .exit = ufs_renesas_exit, + .set_dma_mask = ufs_renesas_set_dma_mask, + .setup_clocks = ufs_renesas_setup_clocks, + .hce_enable_notify = ufs_renesas_hce_enable_notify, + .dbg_register_dump = ufs_renesas_dbg_register_dump, +}; + +static const struct of_device_id __maybe_unused ufs_renesas_of_match[] = { + { .compatible = "renesas,r8a779f0-ufs" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, ufs_renesas_of_match); + +static int ufs_renesas_probe(struct platform_device *pdev) +{ + return ufshcd_pltfrm_init(pdev, &ufs_renesas_vops); +} + +static void ufs_renesas_remove(struct platform_device *pdev) +{ + ufshcd_pltfrm_remove(pdev); +} + +static struct platform_driver ufs_renesas_platform = { + .probe = ufs_renesas_probe, + .remove = ufs_renesas_remove, + .driver = { + .name = "ufshcd-renesas", + .of_match_table = of_match_ptr(ufs_renesas_of_match), + }, +}; +module_platform_driver(ufs_renesas_platform); + +MODULE_AUTHOR("Yoshihiro Shimoda <yoshihiro.shimoda.uh@renesas.com>"); +MODULE_DESCRIPTION("Renesas UFS host controller driver"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/ufs/host/ufs-rockchip.c b/drivers/ufs/host/ufs-rockchip.c new file mode 100644 index 000000000..bac68f238 --- /dev/null +++ b/drivers/ufs/host/ufs-rockchip.c @@ -0,0 +1,361 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Rockchip UFS Host Controller driver + * + * Copyright (C) 2025 Rockchip Electronics Co., Ltd. + */ + +#include <linux/clk.h> +#include <linux/gpio/consumer.h> +#include <linux/mfd/syscon.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/pm_domain.h> +#include <linux/pm_wakeup.h> +#include <linux/regmap.h> +#include <linux/reset.h> + +#include <ufs/ufshcd.h> +#include <ufs/unipro.h> +#include "ufshcd-pltfrm.h" +#include "ufs-rockchip.h" + +static void ufs_rockchip_controller_reset(struct ufs_rockchip_host *host) +{ + reset_control_assert(host->rst); + udelay(1); + reset_control_deassert(host->rst); +} + +static int ufs_rockchip_hce_enable_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status) +{ + struct ufs_rockchip_host *host = ufshcd_get_variant(hba); + int err = 0; + + if (status == POST_CHANGE) { + err = ufshcd_dme_reset(hba); + if (err) + return err; + + err = ufshcd_dme_enable(hba); + if (err) + return err; + + return ufshcd_vops_phy_initialization(hba); + } + + /* PRE_CHANGE */ + ufs_rockchip_controller_reset(host); + + return 0; +} + +static void ufs_rockchip_set_pm_lvl(struct ufs_hba *hba) +{ + hba->rpm_lvl = UFS_PM_LVL_5; + hba->spm_lvl = UFS_PM_LVL_5; +} + +static int ufs_rockchip_rk3576_phy_init(struct ufs_hba *hba) +{ + struct ufs_rockchip_host *host = ufshcd_get_variant(hba); + + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(PA_LOCAL_TX_LCC_ENABLE, 0x0), 0x0); + /* enable the mphy DME_SET cfg */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(MPHY_CFG, 0x0), MPHY_CFG_ENABLE); + for (int i = 0; i < 2; i++) { + /* Configuration M - TX */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_CLK_PRD, SEL_TX_LANE0 + i), 0x06); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_CLK_PRD_EN, SEL_TX_LANE0 + i), 0x02); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_VALUE, SEL_TX_LANE0 + i), 0x44); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE1, SEL_TX_LANE0 + i), 0xe6); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE2, SEL_TX_LANE0 + i), 0x07); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_TASE_VALUE, SEL_TX_LANE0 + i), 0x93); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_BASE_NVALUE, SEL_TX_LANE0 + i), 0xc9); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_POWER_SAVING_CTRL, SEL_TX_LANE0 + i), 0x00); + /* Configuration M - RX */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_CLK_PRD, SEL_RX_LANE0 + i), 0x06); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_CLK_PRD_EN, SEL_RX_LANE0 + i), 0x00); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_VALUE, SEL_RX_LANE0 + i), 0x58); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_PVALUE1, SEL_RX_LANE0 + i), 0x8c); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_PVALUE2, SEL_RX_LANE0 + i), 0x02); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_OPTION, SEL_RX_LANE0 + i), 0xf6); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_POWER_SAVING_CTRL, SEL_RX_LANE0 + i), 0x69); + } + + /* disable the mphy DME_SET cfg */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(MPHY_CFG, 0x0), MPHY_CFG_DISABLE); + + ufs_sys_writel(host->mphy_base, 0x80, CMN_REG23); + ufs_sys_writel(host->mphy_base, 0xB5, TRSV0_REG14); + ufs_sys_writel(host->mphy_base, 0xB5, TRSV1_REG14); + + ufs_sys_writel(host->mphy_base, 0x03, TRSV0_REG15); + ufs_sys_writel(host->mphy_base, 0x03, TRSV1_REG15); + + ufs_sys_writel(host->mphy_base, 0x38, TRSV0_REG08); + ufs_sys_writel(host->mphy_base, 0x38, TRSV1_REG08); + + ufs_sys_writel(host->mphy_base, 0x50, TRSV0_REG29); + ufs_sys_writel(host->mphy_base, 0x50, TRSV1_REG29); + + ufs_sys_writel(host->mphy_base, 0x80, TRSV0_REG2E); + ufs_sys_writel(host->mphy_base, 0x80, TRSV1_REG2E); + + ufs_sys_writel(host->mphy_base, 0x18, TRSV0_REG3C); + ufs_sys_writel(host->mphy_base, 0x18, TRSV1_REG3C); + + ufs_sys_writel(host->mphy_base, 0x03, TRSV0_REG16); + ufs_sys_writel(host->mphy_base, 0x03, TRSV1_REG16); + + ufs_sys_writel(host->mphy_base, 0x20, TRSV0_REG17); + ufs_sys_writel(host->mphy_base, 0x20, TRSV1_REG17); + + ufs_sys_writel(host->mphy_base, 0xC0, TRSV0_REG18); + ufs_sys_writel(host->mphy_base, 0xC0, TRSV1_REG18); + + ufs_sys_writel(host->mphy_base, 0x03, CMN_REG25); + + ufs_sys_writel(host->mphy_base, 0x03, TRSV0_REG3D); + ufs_sys_writel(host->mphy_base, 0x03, TRSV1_REG3D); + + ufs_sys_writel(host->mphy_base, 0xC0, CMN_REG23); + udelay(1); + ufs_sys_writel(host->mphy_base, 0x00, CMN_REG23); + + usleep_range(200, 250); + /* start link up */ + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(MIB_T_DBG_CPORT_TX_ENDIAN, 0), 0x0); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(MIB_T_DBG_CPORT_RX_ENDIAN, 0), 0x0); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(N_DEVICEID, 0), 0x0); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(N_DEVICEID_VALID, 0), 0x1); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(T_PEERDEVICEID, 0), 0x1); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(T_CONNECTIONSTATE, 0), 0x1); + + return 0; +} + +static int ufs_rockchip_common_init(struct ufs_hba *hba) +{ + struct device *dev = hba->dev; + struct platform_device *pdev = to_platform_device(dev); + struct ufs_rockchip_host *host; + int err; + + host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL); + if (!host) + return -ENOMEM; + + host->ufs_sys_ctrl = devm_platform_ioremap_resource_byname(pdev, "hci_grf"); + if (IS_ERR(host->ufs_sys_ctrl)) + return dev_err_probe(dev, PTR_ERR(host->ufs_sys_ctrl), + "Failed to map HCI system control registers\n"); + + host->ufs_phy_ctrl = devm_platform_ioremap_resource_byname(pdev, "mphy_grf"); + if (IS_ERR(host->ufs_phy_ctrl)) + return dev_err_probe(dev, PTR_ERR(host->ufs_phy_ctrl), + "Failed to map mphy system control registers\n"); + + host->mphy_base = devm_platform_ioremap_resource_byname(pdev, "mphy"); + if (IS_ERR(host->mphy_base)) + return dev_err_probe(dev, PTR_ERR(host->mphy_base), + "Failed to map mphy base registers\n"); + + host->rst = devm_reset_control_array_get_exclusive(dev); + if (IS_ERR(host->rst)) + return dev_err_probe(dev, PTR_ERR(host->rst), + "failed to get reset control\n"); + + ufs_rockchip_controller_reset(host); + + host->ref_out_clk = devm_clk_get_enabled(dev, "ref_out"); + if (IS_ERR(host->ref_out_clk)) + return dev_err_probe(dev, PTR_ERR(host->ref_out_clk), + "ref_out clock unavailable\n"); + + host->rst_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW); + if (IS_ERR(host->rst_gpio)) + return dev_err_probe(dev, PTR_ERR(host->rst_gpio), + "failed to get reset gpio\n"); + + err = devm_clk_bulk_get_all_enabled(dev, &host->clks); + if (err < 0) + return dev_err_probe(dev, err, "failed to enable clocks\n"); + + host->hba = hba; + + ufshcd_set_variant(hba, host); + + return 0; +} + +static int ufs_rockchip_rk3576_init(struct ufs_hba *hba) +{ + struct device *dev = hba->dev; + int ret; + + hba->quirks = UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING; + + /* Enable BKOPS when suspend */ + hba->caps |= UFSHCD_CAP_AUTO_BKOPS_SUSPEND; + /* Enable putting device into deep sleep */ + hba->caps |= UFSHCD_CAP_DEEPSLEEP; + /* Enable devfreq of UFS */ + hba->caps |= UFSHCD_CAP_CLK_SCALING; + /* Enable WriteBooster */ + hba->caps |= UFSHCD_CAP_WB_EN; + + /* Set the default desired pm level in case no users set via sysfs */ + ufs_rockchip_set_pm_lvl(hba); + + ret = ufs_rockchip_common_init(hba); + if (ret) + return dev_err_probe(dev, ret, "ufs common init fail\n"); + + return 0; +} + +static int ufs_rockchip_device_reset(struct ufs_hba *hba) +{ + struct ufs_rockchip_host *host = ufshcd_get_variant(hba); + + gpiod_set_value_cansleep(host->rst_gpio, 1); + usleep_range(20, 25); + + gpiod_set_value_cansleep(host->rst_gpio, 0); + usleep_range(20, 25); + + return 0; +} + +static const struct ufs_hba_variant_ops ufs_hba_rk3576_vops = { + .name = "rk3576", + .init = ufs_rockchip_rk3576_init, + .device_reset = ufs_rockchip_device_reset, + .hce_enable_notify = ufs_rockchip_hce_enable_notify, + .phy_initialization = ufs_rockchip_rk3576_phy_init, +}; + +static const struct of_device_id ufs_rockchip_of_match[] = { + { .compatible = "rockchip,rk3576-ufshc", .data = &ufs_hba_rk3576_vops }, + { }, +}; +MODULE_DEVICE_TABLE(of, ufs_rockchip_of_match); + +static int ufs_rockchip_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + const struct ufs_hba_variant_ops *vops; + int err; + + vops = device_get_match_data(dev); + if (!vops) + return dev_err_probe(dev, -ENODATA, "ufs_hba_variant_ops not defined.\n"); + + err = ufshcd_pltfrm_init(pdev, vops); + if (err) + return dev_err_probe(dev, err, "ufshcd_pltfrm_init failed\n"); + + return 0; +} + +static void ufs_rockchip_remove(struct platform_device *pdev) +{ + ufshcd_pltfrm_remove(pdev); +} + +#ifdef CONFIG_PM +static int ufs_rockchip_runtime_suspend(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + struct ufs_rockchip_host *host = ufshcd_get_variant(hba); + + clk_disable_unprepare(host->ref_out_clk); + + /* Do not power down the genpd if rpm_lvl is less than level 5 */ + dev_pm_genpd_rpm_always_on(dev, hba->rpm_lvl < UFS_PM_LVL_5); + + return ufshcd_runtime_suspend(dev); +} + +static int ufs_rockchip_runtime_resume(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + struct ufs_rockchip_host *host = ufshcd_get_variant(hba); + int err; + + err = clk_prepare_enable(host->ref_out_clk); + if (err) { + dev_err(hba->dev, "failed to enable ref_out clock %d\n", err); + return err; + } + + ufs_rockchip_controller_reset(host); + + return ufshcd_runtime_resume(dev); +} +#endif + +#ifdef CONFIG_PM_SLEEP +static int ufs_rockchip_system_suspend(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + struct ufs_rockchip_host *host = ufshcd_get_variant(hba); + int err; + + /* + * If spm_lvl is less than level 5, it means we need to keep the host + * controller in powered-on state. So device_set_awake_path() is + * calling pm core to notify the genpd provider to meet this requirement + */ + if (hba->spm_lvl < UFS_PM_LVL_5) + device_set_awake_path(dev); + + err = ufshcd_system_suspend(dev); + if (err) { + dev_err(hba->dev, "UFSHCD system suspend failed %d\n", err); + return err; + } + + clk_disable_unprepare(host->ref_out_clk); + + return 0; +} + +static int ufs_rockchip_system_resume(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + struct ufs_rockchip_host *host = ufshcd_get_variant(hba); + int err; + + err = clk_prepare_enable(host->ref_out_clk); + if (err) { + dev_err(hba->dev, "failed to enable ref_out clock %d\n", err); + return err; + } + + return ufshcd_system_resume(dev); +} +#endif + +static const struct dev_pm_ops ufs_rockchip_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(ufs_rockchip_system_suspend, ufs_rockchip_system_resume) + SET_RUNTIME_PM_OPS(ufs_rockchip_runtime_suspend, ufs_rockchip_runtime_resume, NULL) + .prepare = ufshcd_suspend_prepare, + .complete = ufshcd_resume_complete, +}; + +static struct platform_driver ufs_rockchip_pltform = { + .probe = ufs_rockchip_probe, + .remove = ufs_rockchip_remove, + .driver = { + .name = "ufshcd-rockchip", + .pm = &ufs_rockchip_pm_ops, + .of_match_table = ufs_rockchip_of_match, + }, +}; +module_platform_driver(ufs_rockchip_pltform); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Rockchip UFS Host Driver"); diff --git a/drivers/ufs/host/ufs-rockchip.h b/drivers/ufs/host/ufs-rockchip.h new file mode 100644 index 000000000..3ba6fb9f7 --- /dev/null +++ b/drivers/ufs/host/ufs-rockchip.h @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Rockchip UFS Host Controller driver + * + * Copyright (C) 2025 Rockchip Electronics Co., Ltd. + */ + +#ifndef _UFS_ROCKCHIP_H_ +#define _UFS_ROCKCHIP_H_ + +#define SEL_TX_LANE0 0x0 +#define SEL_TX_LANE1 0x1 +#define SEL_TX_LANE2 0x2 +#define SEL_TX_LANE3 0x3 +#define SEL_RX_LANE0 0x4 +#define SEL_RX_LANE1 0x5 +#define SEL_RX_LANE2 0x6 +#define SEL_RX_LANE3 0x7 + +#define VND_TX_CLK_PRD 0xAA +#define VND_TX_CLK_PRD_EN 0xA9 +#define VND_TX_LINERESET_PVALUE2 0xAB +#define VND_TX_LINERESET_PVALUE1 0xAC +#define VND_TX_LINERESET_VALUE 0xAD +#define VND_TX_BASE_NVALUE 0x93 +#define VND_TX_TASE_VALUE 0x94 +#define VND_TX_POWER_SAVING_CTRL 0x7F +#define VND_RX_CLK_PRD 0x12 +#define VND_RX_CLK_PRD_EN 0x11 +#define VND_RX_LINERESET_PVALUE2 0x1B +#define VND_RX_LINERESET_PVALUE1 0x1C +#define VND_RX_LINERESET_VALUE 0x1D +#define VND_RX_LINERESET_OPTION 0x25 +#define VND_RX_POWER_SAVING_CTRL 0x2F +#define VND_RX_SAVE_DET_CTRL 0x1E + +#define CMN_REG23 0x8C +#define CMN_REG25 0x94 +#define TRSV0_REG08 0xE0 +#define TRSV1_REG08 0x220 +#define TRSV0_REG14 0x110 +#define TRSV1_REG14 0x250 +#define TRSV0_REG15 0x134 +#define TRSV1_REG15 0x274 +#define TRSV0_REG16 0x128 +#define TRSV1_REG16 0x268 +#define TRSV0_REG17 0x12C +#define TRSV1_REG17 0x26c +#define TRSV0_REG18 0x120 +#define TRSV1_REG18 0x260 +#define TRSV0_REG29 0x164 +#define TRSV1_REG29 0x2A4 +#define TRSV0_REG2E 0x178 +#define TRSV1_REG2E 0x2B8 +#define TRSV0_REG3C 0x1B0 +#define TRSV1_REG3C 0x2F0 +#define TRSV0_REG3D 0x1B4 +#define TRSV1_REG3D 0x2F4 + +#define MPHY_CFG 0x200 +#define MPHY_CFG_ENABLE 0x40 +#define MPHY_CFG_DISABLE 0x0 + +#define MIB_T_DBG_CPORT_TX_ENDIAN 0xc022 +#define MIB_T_DBG_CPORT_RX_ENDIAN 0xc023 + +struct ufs_rockchip_host { + struct ufs_hba *hba; + void __iomem *ufs_phy_ctrl; + void __iomem *ufs_sys_ctrl; + void __iomem *mphy_base; + struct gpio_desc *rst_gpio; + struct reset_control *rst; + struct clk *ref_out_clk; + struct clk_bulk_data *clks; + uint64_t caps; +}; + +#define ufs_sys_writel(base, val, reg) \ + writel((val), (base) + (reg)) +#define ufs_sys_readl(base, reg) readl((base) + (reg)) +#define ufs_sys_set_bits(base, mask, reg) \ + ufs_sys_writel( \ + (base), ((mask) | (ufs_sys_readl((base), (reg)))), (reg)) +#define ufs_sys_ctrl_clr_bits(base, mask, reg) \ + ufs_sys_writel((base), \ + ((~(mask)) & (ufs_sys_readl((base), (reg)))), \ + (reg)) + +#endif /* _UFS_ROCKCHIP_H_ */ diff --git a/drivers/ufs/host/ufs-sprd.c b/drivers/ufs/host/ufs-sprd.c new file mode 100644 index 000000000..a5e8c591b --- /dev/null +++ b/drivers/ufs/host/ufs-sprd.c @@ -0,0 +1,450 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * UNISOC UFS Host Controller driver + * + * Copyright (C) 2022 Unisoc, Inc. + * Author: Zhe Wang <zhe.wang1@unisoc.com> + */ + +#include <linux/arm-smccc.h> +#include <linux/mfd/syscon.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/reset.h> +#include <linux/regulator/consumer.h> + +#include <ufs/ufshcd.h> +#include "ufshcd-pltfrm.h" +#include "ufs-sprd.h" + +static const struct of_device_id ufs_sprd_of_match[]; + +static struct ufs_sprd_priv *ufs_sprd_get_priv_data(struct ufs_hba *hba) +{ + struct ufs_sprd_host *host = ufshcd_get_variant(hba); + + WARN_ON(!host->priv); + return host->priv; +} + +static void ufs_sprd_regmap_update(struct ufs_sprd_priv *priv, unsigned int index, + unsigned int reg, unsigned int bits, unsigned int val) +{ + regmap_update_bits(priv->sysci[index].regmap, reg, bits, val); +} + +static void ufs_sprd_regmap_read(struct ufs_sprd_priv *priv, unsigned int index, + unsigned int reg, unsigned int *val) +{ + regmap_read(priv->sysci[index].regmap, reg, val); +} + +static void ufs_sprd_get_unipro_ver(struct ufs_hba *hba) +{ + struct ufs_sprd_host *host = ufshcd_get_variant(hba); + + if (ufshcd_dme_get(hba, UIC_ARG_MIB(PA_LOCALVERINFO), &host->unipro_ver)) + host->unipro_ver = 0; +} + +static void ufs_sprd_ctrl_uic_compl(struct ufs_hba *hba, bool enable) +{ + u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); + + if (enable == true) + set |= UIC_COMMAND_COMPL; + else + set &= ~UIC_COMMAND_COMPL; + ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE); +} + +static int ufs_sprd_get_reset_ctrl(struct device *dev, struct ufs_sprd_rst *rci) +{ + rci->rc = devm_reset_control_get(dev, rci->name); + if (IS_ERR(rci->rc)) { + dev_err(dev, "failed to get reset ctrl:%s\n", rci->name); + return PTR_ERR(rci->rc); + } + + return 0; +} + +static int ufs_sprd_get_syscon_reg(struct device *dev, struct ufs_sprd_syscon *sysci) +{ + sysci->regmap = syscon_regmap_lookup_by_phandle(dev->of_node, sysci->name); + if (IS_ERR(sysci->regmap)) { + dev_err(dev, "failed to get ufs syscon:%s\n", sysci->name); + return PTR_ERR(sysci->regmap); + } + + return 0; +} + +static int ufs_sprd_get_vreg(struct device *dev, struct ufs_sprd_vreg *vregi) +{ + vregi->vreg = devm_regulator_get(dev, vregi->name); + if (IS_ERR(vregi->vreg)) { + dev_err(dev, "failed to get vreg:%s\n", vregi->name); + return PTR_ERR(vregi->vreg); + } + + return 0; +} + +static int ufs_sprd_parse_dt(struct device *dev, struct ufs_hba *hba, struct ufs_sprd_host *host) +{ + u32 i; + struct ufs_sprd_priv *priv = host->priv; + int ret = 0; + + /* Parse UFS reset ctrl info */ + for (i = 0; i < SPRD_UFS_RST_MAX; i++) { + if (!priv->rci[i].name) + continue; + ret = ufs_sprd_get_reset_ctrl(dev, &priv->rci[i]); + if (ret) + goto out; + } + + /* Parse UFS syscon reg info */ + for (i = 0; i < SPRD_UFS_SYSCON_MAX; i++) { + if (!priv->sysci[i].name) + continue; + ret = ufs_sprd_get_syscon_reg(dev, &priv->sysci[i]); + if (ret) + goto out; + } + + /* Parse UFS vreg info */ + for (i = 0; i < SPRD_UFS_VREG_MAX; i++) { + if (!priv->vregi[i].name) + continue; + ret = ufs_sprd_get_vreg(dev, &priv->vregi[i]); + if (ret) + goto out; + } + +out: + return ret; +} + +static int ufs_sprd_common_init(struct ufs_hba *hba) +{ + struct device *dev = hba->dev; + struct ufs_sprd_host *host; + struct platform_device __maybe_unused *pdev = to_platform_device(dev); + const struct of_device_id *of_id; + int ret = 0; + + host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL); + if (!host) + return -ENOMEM; + + of_id = of_match_node(ufs_sprd_of_match, pdev->dev.of_node); + if (of_id->data != NULL) + host->priv = container_of(of_id->data, struct ufs_sprd_priv, + ufs_hba_sprd_vops); + + host->hba = hba; + ufshcd_set_variant(hba, host); + + hba->caps |= UFSHCD_CAP_CLK_GATING | + UFSHCD_CAP_CRYPTO | + UFSHCD_CAP_WB_EN; + hba->quirks |= UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS; + + ret = ufs_sprd_parse_dt(dev, hba, host); + + return ret; +} + +static int sprd_ufs_pwr_change_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status, + struct ufs_pa_layer_attr *dev_req_params) +{ + struct ufs_sprd_host *host = ufshcd_get_variant(hba); + + if (status == PRE_CHANGE) { + if (host->unipro_ver >= UFS_UNIPRO_VER_1_8) + ufshcd_dme_configure_adapt(hba, dev_req_params->gear_tx, + PA_INITIAL_ADAPT); + } + + return 0; +} + +static int ufs_sprd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op, + enum ufs_notify_change_status status) +{ + unsigned long flags; + + if (status == PRE_CHANGE) { + if (ufshcd_is_auto_hibern8_supported(hba)) { + spin_lock_irqsave(hba->host->host_lock, flags); + ufshcd_writel(hba, 0, REG_AUTO_HIBERNATE_IDLE_TIMER); + spin_unlock_irqrestore(hba->host->host_lock, flags); + } + } + + return 0; +} + +static void ufs_sprd_n6_host_reset(struct ufs_hba *hba) +{ + struct ufs_sprd_priv *priv = ufs_sprd_get_priv_data(hba); + + dev_info(hba->dev, "ufs host reset!\n"); + + reset_control_assert(priv->rci[SPRD_UFSHCI_SOFT_RST].rc); + usleep_range(1000, 1100); + reset_control_deassert(priv->rci[SPRD_UFSHCI_SOFT_RST].rc); +} + +static int ufs_sprd_n6_device_reset(struct ufs_hba *hba) +{ + struct ufs_sprd_priv *priv = ufs_sprd_get_priv_data(hba); + + dev_info(hba->dev, "ufs device reset!\n"); + + reset_control_assert(priv->rci[SPRD_UFS_DEV_RST].rc); + usleep_range(1000, 1100); + reset_control_deassert(priv->rci[SPRD_UFS_DEV_RST].rc); + + return 0; +} + +static void ufs_sprd_n6_key_acc_enable(struct ufs_hba *hba) +{ + u32 val; + u32 retry = 10; + struct arm_smccc_res res; + +check_hce: + /* Key access only can be enabled under HCE enable */ + val = ufshcd_readl(hba, REG_CONTROLLER_ENABLE); + if (!(val & CONTROLLER_ENABLE)) { + ufs_sprd_n6_host_reset(hba); + val |= CONTROLLER_ENABLE; + ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE); + usleep_range(1000, 1100); + if (retry) { + retry--; + goto check_hce; + } + goto disable_crypto; + } + + arm_smccc_smc(SPRD_SIP_SVC_STORAGE_UFS_CRYPTO_ENABLE, + 0, 0, 0, 0, 0, 0, 0, &res); + if (!res.a0) + return; + +disable_crypto: + dev_err(hba->dev, "key reg access enable fail, disable crypto\n"); + hba->caps &= ~UFSHCD_CAP_CRYPTO; +} + +static int ufs_sprd_n6_init(struct ufs_hba *hba) +{ + struct ufs_sprd_priv *priv; + int ret = 0; + + ret = ufs_sprd_common_init(hba); + if (ret != 0) + return ret; + + priv = ufs_sprd_get_priv_data(hba); + + ret = regulator_enable(priv->vregi[SPRD_UFS_VDD_MPHY].vreg); + if (ret) + return -ENODEV; + + if (hba->caps & UFSHCD_CAP_CRYPTO) + ufs_sprd_n6_key_acc_enable(hba); + + return 0; +} + +static int ufs_sprd_n6_phy_init(struct ufs_hba *hba) +{ + int ret = 0; + uint32_t val = 0; + uint32_t retry = 10; + uint32_t offset; + struct ufs_sprd_priv *priv = ufs_sprd_get_priv_data(hba); + + ufshcd_dme_set(hba, UIC_ARG_MIB(CBREFCLKCTRL2), 0x90); + ufshcd_dme_set(hba, UIC_ARG_MIB(CBCRCTRL), 0x01); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RXSQCONTROL, + UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)), 0x01); + ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RXSQCONTROL, + UIC_ARG_MPHY_RX_GEN_SEL_INDEX(1)), 0x01); + ufshcd_dme_set(hba, UIC_ARG_MIB(VS_MPHYCFGUPDT), 0x01); + ufshcd_dme_set(hba, UIC_ARG_MIB(CBRATESEL), 0x01); + + do { + /* phy_sram_init_done */ + ufs_sprd_regmap_read(priv, SPRD_UFS_ANLG, 0xc, &val); + if ((val & 0x1) == 0x1) { + for (offset = 0x40; offset < 0x42; offset++) { + /* Lane afe calibration */ + ufshcd_dme_set(hba, UIC_ARG_MIB(CBCREGADDRLSB), 0x1c); + ufshcd_dme_set(hba, UIC_ARG_MIB(CBCREGADDRMSB), offset); + ufshcd_dme_set(hba, UIC_ARG_MIB(CBCREGWRLSB), 0x04); + ufshcd_dme_set(hba, UIC_ARG_MIB(CBCREGWRMSB), 0x00); + ufshcd_dme_set(hba, UIC_ARG_MIB(CBCREGRDWRSEL), 0x01); + ufshcd_dme_set(hba, UIC_ARG_MIB(VS_MPHYCFGUPDT), 0x01); + } + + goto update_phy; + } + udelay(1000); + retry--; + } while (retry > 0); + + ret = -ETIMEDOUT; + goto out; + +update_phy: + /* phy_sram_ext_ld_done */ + ufs_sprd_regmap_update(priv, SPRD_UFS_ANLG, 0xc, 0x2, 0); + ufshcd_dme_set(hba, UIC_ARG_MIB(VS_MPHYCFGUPDT), 0x01); + ufshcd_dme_set(hba, UIC_ARG_MIB(VS_MPHYDISABLE), 0x0); +out: + return ret; +} + + +static int sprd_ufs_n6_hce_enable_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status) +{ + int err = 0; + struct ufs_sprd_priv *priv = ufs_sprd_get_priv_data(hba); + + if (status == PRE_CHANGE) { + /* phy_sram_ext_ld_done */ + ufs_sprd_regmap_update(priv, SPRD_UFS_ANLG, 0xc, 0x2, 0x2); + /* phy_sram_bypass */ + ufs_sprd_regmap_update(priv, SPRD_UFS_ANLG, 0xc, 0x4, 0x4); + + ufs_sprd_n6_host_reset(hba); + + if (hba->caps & UFSHCD_CAP_CRYPTO) + ufs_sprd_n6_key_acc_enable(hba); + } + + if (status == POST_CHANGE) { + err = ufs_sprd_n6_phy_init(hba); + if (err) { + dev_err(hba->dev, "Phy setup failed (%d)\n", err); + goto out; + } + + ufs_sprd_get_unipro_ver(hba); + } +out: + return err; +} + +static void sprd_ufs_n6_h8_notify(struct ufs_hba *hba, + enum uic_cmd_dme cmd, + enum ufs_notify_change_status status) +{ + struct ufs_sprd_priv *priv = ufs_sprd_get_priv_data(hba); + + if (status == PRE_CHANGE) { + if (cmd == UIC_CMD_DME_HIBER_ENTER) + /* + * Disable UIC COMPL INTR to prevent access to UFSHCI after + * checking HCS.UPMCRS + */ + ufs_sprd_ctrl_uic_compl(hba, false); + + if (cmd == UIC_CMD_DME_HIBER_EXIT) { + ufs_sprd_regmap_update(priv, SPRD_UFS_AON_APB, APB_UFSDEV_REG, + APB_UFSDEV_REFCLK_EN, APB_UFSDEV_REFCLK_EN); + ufs_sprd_regmap_update(priv, SPRD_UFS_AON_APB, APB_USB31PLL_CTRL, + APB_USB31PLLV_REF2MPHY, APB_USB31PLLV_REF2MPHY); + } + } + + if (status == POST_CHANGE) { + if (cmd == UIC_CMD_DME_HIBER_EXIT) + ufs_sprd_ctrl_uic_compl(hba, true); + + if (cmd == UIC_CMD_DME_HIBER_ENTER) { + ufs_sprd_regmap_update(priv, SPRD_UFS_AON_APB, APB_UFSDEV_REG, + APB_UFSDEV_REFCLK_EN, 0); + ufs_sprd_regmap_update(priv, SPRD_UFS_AON_APB, APB_USB31PLL_CTRL, + APB_USB31PLLV_REF2MPHY, 0); + } + } +} + +static struct ufs_sprd_priv n6_ufs = { + .rci[SPRD_UFSHCI_SOFT_RST] = { .name = "controller", }, + .rci[SPRD_UFS_DEV_RST] = { .name = "device", }, + + .sysci[SPRD_UFS_ANLG] = { .name = "sprd,ufs-anlg-syscon", }, + .sysci[SPRD_UFS_AON_APB] = { .name = "sprd,aon-apb-syscon", }, + + .vregi[SPRD_UFS_VDD_MPHY] = { .name = "vdd-mphy", }, + + .ufs_hba_sprd_vops = { + .name = "sprd,ums9620-ufs", + .init = ufs_sprd_n6_init, + .hce_enable_notify = sprd_ufs_n6_hce_enable_notify, + .pwr_change_notify = sprd_ufs_pwr_change_notify, + .hibern8_notify = sprd_ufs_n6_h8_notify, + .device_reset = ufs_sprd_n6_device_reset, + .suspend = ufs_sprd_suspend, + }, +}; + +static const struct of_device_id __maybe_unused ufs_sprd_of_match[] = { + { .compatible = "sprd,ums9620-ufs", .data = &n6_ufs.ufs_hba_sprd_vops}, + {}, +}; +MODULE_DEVICE_TABLE(of, ufs_sprd_of_match); + +static int ufs_sprd_probe(struct platform_device *pdev) +{ + int err; + struct device *dev = &pdev->dev; + const struct of_device_id *of_id; + + of_id = of_match_node(ufs_sprd_of_match, dev->of_node); + err = ufshcd_pltfrm_init(pdev, of_id->data); + if (err) + dev_err(dev, "ufshcd_pltfrm_init() failed %d\n", err); + + return err; +} + +static void ufs_sprd_remove(struct platform_device *pdev) +{ + ufshcd_pltfrm_remove(pdev); +} + +static const struct dev_pm_ops ufs_sprd_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(ufshcd_system_suspend, ufshcd_system_resume) + SET_RUNTIME_PM_OPS(ufshcd_runtime_suspend, ufshcd_runtime_resume, NULL) + .prepare = ufshcd_suspend_prepare, + .complete = ufshcd_resume_complete, +}; + +static struct platform_driver ufs_sprd_pltform = { + .probe = ufs_sprd_probe, + .remove = ufs_sprd_remove, + .driver = { + .name = "ufshcd-sprd", + .pm = &ufs_sprd_pm_ops, + .of_match_table = of_match_ptr(ufs_sprd_of_match), + }, +}; +module_platform_driver(ufs_sprd_pltform); + +MODULE_AUTHOR("Zhe Wang <zhe.wang1@unisoc.com>"); +MODULE_DESCRIPTION("Unisoc UFS Host Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/ufs/host/ufs-sprd.h b/drivers/ufs/host/ufs-sprd.h new file mode 100644 index 000000000..26ad5c3af --- /dev/null +++ b/drivers/ufs/host/ufs-sprd.h @@ -0,0 +1,85 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * UNISOC UFS Host Controller driver + * + * Copyright (C) 2022 Unisoc, Inc. + * Author: Zhe Wang <zhe.wang1@unisoc.com> + */ + +#ifndef _UFS_SPRD_H_ +#define _UFS_SPRD_H_ + +/* Vendor specific attributes */ +#define RXSQCONTROL 0x8009 +#define CBRATESEL 0x8114 +#define CBCREGADDRLSB 0x8116 +#define CBCREGADDRMSB 0x8117 +#define CBCREGWRLSB 0x8118 +#define CBCREGWRMSB 0x8119 +#define CBCREGRDWRSEL 0x811C +#define CBCRCTRL 0x811F +#define CBREFCLKCTRL2 0x8132 +#define VS_MPHYDISABLE 0xD0C1 + +#define APB_UFSDEV_REG 0xCE8 +#define APB_UFSDEV_REFCLK_EN 0x2 +#define APB_USB31PLL_CTRL 0xCFC +#define APB_USB31PLLV_REF2MPHY 0x1 + +#define SPRD_SIP_SVC_STORAGE_UFS_CRYPTO_ENABLE \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + ARM_SMCCC_OWNER_SIP, \ + 0x0301) + +enum SPRD_UFS_RST_INDEX { + SPRD_UFSHCI_SOFT_RST, + SPRD_UFS_DEV_RST, + + SPRD_UFS_RST_MAX +}; + +enum SPRD_UFS_SYSCON_INDEX { + SPRD_UFS_ANLG, + SPRD_UFS_AON_APB, + + SPRD_UFS_SYSCON_MAX +}; + +enum SPRD_UFS_VREG_INDEX { + SPRD_UFS_VDD_MPHY, + + SPRD_UFS_VREG_MAX +}; + +struct ufs_sprd_rst { + const char *name; + struct reset_control *rc; +}; + +struct ufs_sprd_syscon { + const char *name; + struct regmap *regmap; +}; + +struct ufs_sprd_vreg { + const char *name; + struct regulator *vreg; +}; + +struct ufs_sprd_priv { + struct ufs_sprd_rst rci[SPRD_UFS_RST_MAX]; + struct ufs_sprd_syscon sysci[SPRD_UFS_SYSCON_MAX]; + struct ufs_sprd_vreg vregi[SPRD_UFS_VREG_MAX]; + const struct ufs_hba_variant_ops ufs_hba_sprd_vops; +}; + +struct ufs_sprd_host { + struct ufs_hba *hba; + struct ufs_sprd_priv *priv; + void __iomem *ufs_dbg_mmio; + + enum ufs_unipro_ver unipro_ver; +}; + +#endif /* _UFS_SPRD_H_ */ diff --git a/drivers/ufs/host/ufshcd-dwc.c b/drivers/ufs/host/ufshcd-dwc.c new file mode 100644 index 000000000..21b1cf912 --- /dev/null +++ b/drivers/ufs/host/ufshcd-dwc.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * UFS Host driver for Synopsys Designware Core + * + * Copyright (C) 2015-2016 Synopsys, Inc. (www.synopsys.com) + * + * Authors: Joao Pinto <jpinto@synopsys.com> + */ + +#include <linux/module.h> + +#include <ufs/ufshcd.h> +#include <ufs/unipro.h> + +#include "ufshcd-dwc.h" +#include "ufshci-dwc.h" + +int ufshcd_dwc_dme_set_attrs(struct ufs_hba *hba, + const struct ufshcd_dme_attr_val *v, int n) +{ + int ret = 0; + int attr_node = 0; + + for (attr_node = 0; attr_node < n; attr_node++) { + ret = ufshcd_dme_set_attr(hba, v[attr_node].attr_sel, + ATTR_SET_NOR, v[attr_node].mib_val, v[attr_node].peer); + + if (ret) + return ret; + } + + return 0; +} +EXPORT_SYMBOL(ufshcd_dwc_dme_set_attrs); + +/** + * ufshcd_dwc_program_clk_div() - program clock divider. + * @hba: Private Structure pointer + * @divider_val: clock divider value to be programmed + * + */ +static void ufshcd_dwc_program_clk_div(struct ufs_hba *hba, u32 divider_val) +{ + ufshcd_writel(hba, divider_val, DWC_UFS_REG_HCLKDIV); +} + +/** + * ufshcd_dwc_link_is_up() - check if link is up. + * @hba: private structure pointer + * + * Return: 0 on success, non-zero value on failure. + */ +static int ufshcd_dwc_link_is_up(struct ufs_hba *hba) +{ + int dme_result = 0; + + ufshcd_dme_get(hba, UIC_ARG_MIB(VS_POWERSTATE), &dme_result); + + if (dme_result == UFSHCD_LINK_IS_UP) { + ufshcd_set_link_active(hba); + return 0; + } + + return 1; +} + +/** + * ufshcd_dwc_connection_setup() - configure unipro attributes. + * @hba: pointer to drivers private data + * + * This function configures both the local side (host) and the peer side + * (device) unipro attributes to establish the connection to application/ + * cport. + * This function is not required if the hardware is properly configured to + * have this connection setup on reset. But invoking this function does no + * harm and should be fine even working with any ufs device. + * + * Return: 0 on success non-zero value on failure. + */ +static int ufshcd_dwc_connection_setup(struct ufs_hba *hba) +{ + static const struct ufshcd_dme_attr_val setup_attrs[] = { + { UIC_ARG_MIB(T_CONNECTIONSTATE), 0, DME_LOCAL }, + { UIC_ARG_MIB(N_DEVICEID), 0, DME_LOCAL }, + { UIC_ARG_MIB(N_DEVICEID_VALID), 0, DME_LOCAL }, + { UIC_ARG_MIB(T_PEERDEVICEID), 1, DME_LOCAL }, + { UIC_ARG_MIB(T_PEERCPORTID), 0, DME_LOCAL }, + { UIC_ARG_MIB(T_TRAFFICCLASS), 0, DME_LOCAL }, + { UIC_ARG_MIB(T_CPORTFLAGS), 0x6, DME_LOCAL }, + { UIC_ARG_MIB(T_CPORTMODE), 1, DME_LOCAL }, + { UIC_ARG_MIB(T_CONNECTIONSTATE), 1, DME_LOCAL }, + { UIC_ARG_MIB(T_CONNECTIONSTATE), 0, DME_PEER }, + { UIC_ARG_MIB(N_DEVICEID), 1, DME_PEER }, + { UIC_ARG_MIB(N_DEVICEID_VALID), 1, DME_PEER }, + { UIC_ARG_MIB(T_PEERDEVICEID), 1, DME_PEER }, + { UIC_ARG_MIB(T_PEERCPORTID), 0, DME_PEER }, + { UIC_ARG_MIB(T_TRAFFICCLASS), 0, DME_PEER }, + { UIC_ARG_MIB(T_CPORTFLAGS), 0x6, DME_PEER }, + { UIC_ARG_MIB(T_CPORTMODE), 1, DME_PEER }, + { UIC_ARG_MIB(T_CONNECTIONSTATE), 1, DME_PEER } + }; + + return ufshcd_dwc_dme_set_attrs(hba, setup_attrs, ARRAY_SIZE(setup_attrs)); +} + +/** + * ufshcd_dwc_link_startup_notify() - program clock divider. + * @hba: private structure pointer + * @status: Callback notify status + * + * Return: 0 on success, non-zero value on failure. + */ +int ufshcd_dwc_link_startup_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status) +{ + int err = 0; + + if (status == PRE_CHANGE) { + ufshcd_dwc_program_clk_div(hba, DWC_UFS_REG_HCLKDIV_DIV_125); + + err = ufshcd_vops_phy_initialization(hba); + if (err) { + dev_err(hba->dev, "Phy setup failed (%d)\n", err); + goto out; + } + } else { /* POST_CHANGE */ + err = ufshcd_dwc_link_is_up(hba); + if (err) { + dev_err(hba->dev, "Link is not up\n"); + goto out; + } + + err = ufshcd_dwc_connection_setup(hba); + if (err) + dev_err(hba->dev, "Connection setup failed (%d)\n", + err); + } + +out: + return err; +} +EXPORT_SYMBOL(ufshcd_dwc_link_startup_notify); + +MODULE_AUTHOR("Joao Pinto <Joao.Pinto@synopsys.com>"); +MODULE_DESCRIPTION("UFS Host driver for Synopsys Designware Core"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/ufs/host/ufshcd-dwc.h b/drivers/ufs/host/ufshcd-dwc.h new file mode 100644 index 000000000..c618bb914 --- /dev/null +++ b/drivers/ufs/host/ufshcd-dwc.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * UFS Host driver for Synopsys Designware Core + * + * Copyright (C) 2015-2016 Synopsys, Inc. (www.synopsys.com) + * + * Authors: Joao Pinto <jpinto@synopsys.com> + */ + +#ifndef _UFSHCD_DWC_H +#define _UFSHCD_DWC_H + +#include <ufs/ufshcd.h> + +/* RMMI Attributes */ +#define CBREFCLKCTRL2 0x8132 +#define CBCRCTRL 0x811F +#define CBC10DIRECTCONF2 0x810E +#define CBRATESEL 0x8114 +#define CBCREGADDRLSB 0x8116 +#define CBCREGADDRMSB 0x8117 +#define CBCREGWRLSB 0x8118 +#define CBCREGWRMSB 0x8119 +#define CBCREGRDLSB 0x811A +#define CBCREGRDMSB 0x811B +#define CBCREGRDWRSEL 0x811C + +#define CBREFREFCLK_GATE_OVR_EN BIT(7) + +/* M-PHY Attributes */ +#define MTX_FSM_STATE 0x41 +#define MRX_FSM_STATE 0xC1 + +/* M-PHY registers */ +#define RX_OVRD_IN_1(n) (0x3006 + ((n) * 0x100)) +#define RX_PCS_OUT(n) (0x300F + ((n) * 0x100)) +#define FAST_FLAGS(n) (0x401C + ((n) * 0x100)) +#define RX_AFE_ATT_IDAC(n) (0x4000 + ((n) * 0x100)) +#define RX_AFE_CTLE_IDAC(n) (0x4001 + ((n) * 0x100)) +#define FW_CALIB_CCFG(n) (0x404D + ((n) * 0x100)) + +/* Tx/Rx FSM state */ +enum rx_fsm_state { + RX_STATE_DISABLED = 0, + RX_STATE_HIBERN8 = 1, + RX_STATE_SLEEP = 2, + RX_STATE_STALL = 3, + RX_STATE_LSBURST = 4, + RX_STATE_HSBURST = 5, +}; + +enum tx_fsm_state { + TX_STATE_DISABLED = 0, + TX_STATE_HIBERN8 = 1, + TX_STATE_SLEEP = 2, + TX_STATE_STALL = 3, + TX_STATE_LSBURST = 4, + TX_STATE_HSBURST = 5, +}; + +struct ufshcd_dme_attr_val { + u32 attr_sel; + u32 mib_val; + u8 peer; +}; + +int ufshcd_dwc_link_startup_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status); +int ufshcd_dwc_dme_set_attrs(struct ufs_hba *hba, + const struct ufshcd_dme_attr_val *v, int n); +#endif /* End of Header */ diff --git a/drivers/ufs/host/ufshcd-pci.c b/drivers/ufs/host/ufshcd-pci.c new file mode 100644 index 000000000..f2433879b --- /dev/null +++ b/drivers/ufs/host/ufshcd-pci.c @@ -0,0 +1,718 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Universal Flash Storage Host controller PCI glue driver + * + * Copyright (C) 2011-2013 Samsung India Software Operations + * + * Authors: + * Santosh Yaraganavi <santosh.sy@samsung.com> + * Vinayak Holikatti <h.vinayak@samsung.com> + */ + +#include <ufs/ufshcd.h> +#include <linux/delay.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/pm_runtime.h> +#include <linux/pm_qos.h> +#include <linux/suspend.h> +#include <linux/debugfs.h> +#include <linux/uuid.h> +#include <linux/acpi.h> +#include <linux/gpio/consumer.h> + +#define MAX_SUPP_MAC 64 + +enum intel_ufs_dsm_func_id { + INTEL_DSM_FNS = 0, + INTEL_DSM_RESET = 1, +}; + +struct intel_host { + u32 dsm_fns; + u32 active_ltr; + u32 idle_ltr; + int saved_spm_lvl; + struct dentry *debugfs_root; + struct gpio_desc *reset_gpio; +}; + +static const guid_t intel_dsm_guid = + GUID_INIT(0x1A4832A0, 0x7D03, 0x43CA, + 0xB0, 0x20, 0xF6, 0xDC, 0xD1, 0x2A, 0x19, 0x50); + +static bool __intel_dsm_supported(struct intel_host *host, + enum intel_ufs_dsm_func_id fn) +{ + return fn < 32 && fn >= 0 && (host->dsm_fns & (1u << fn)); +} + +#define INTEL_DSM_SUPPORTED(host, name) \ + __intel_dsm_supported(host, INTEL_DSM_##name) + +static int __intel_dsm(struct intel_host *intel_host, struct device *dev, + unsigned int fn, u32 *result) +{ + union acpi_object *obj; + int err = 0; + size_t len; + + obj = acpi_evaluate_dsm_typed(ACPI_HANDLE(dev), &intel_dsm_guid, 0, fn, NULL, + ACPI_TYPE_BUFFER); + if (!obj) + return -EOPNOTSUPP; + + if (obj->buffer.length < 1) { + err = -EINVAL; + goto out; + } + + len = min_t(size_t, obj->buffer.length, 4); + + *result = 0; + memcpy(result, obj->buffer.pointer, len); +out: + ACPI_FREE(obj); + + return err; +} + +static int intel_dsm(struct intel_host *intel_host, struct device *dev, + unsigned int fn, u32 *result) +{ + if (!__intel_dsm_supported(intel_host, fn)) + return -EOPNOTSUPP; + + return __intel_dsm(intel_host, dev, fn, result); +} + +static void intel_dsm_init(struct intel_host *intel_host, struct device *dev) +{ + int err; + + err = __intel_dsm(intel_host, dev, INTEL_DSM_FNS, &intel_host->dsm_fns); + dev_dbg(dev, "DSM fns %#x, error %d\n", intel_host->dsm_fns, err); +} + +static int ufs_intel_hce_enable_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status) +{ + /* Cannot enable ICE until after HC enable */ + if (status == POST_CHANGE && hba->caps & UFSHCD_CAP_CRYPTO) { + u32 hce = ufshcd_readl(hba, REG_CONTROLLER_ENABLE); + + hce |= CRYPTO_GENERAL_ENABLE; + ufshcd_writel(hba, hce, REG_CONTROLLER_ENABLE); + } + + return 0; +} + +static int ufs_intel_disable_lcc(struct ufs_hba *hba) +{ + u32 attr = UIC_ARG_MIB(PA_LOCAL_TX_LCC_ENABLE); + u32 lcc_enable = 0; + + ufshcd_dme_get(hba, attr, &lcc_enable); + if (lcc_enable) + ufshcd_disable_host_tx_lcc(hba); + + return 0; +} + +static int ufs_intel_link_startup_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status) +{ + int err = 0; + + switch (status) { + case PRE_CHANGE: + err = ufs_intel_disable_lcc(hba); + break; + case POST_CHANGE: + break; + default: + break; + } + + return err; +} + +static int ufs_intel_set_lanes(struct ufs_hba *hba, u32 lanes) +{ + struct ufs_pa_layer_attr pwr_info = hba->pwr_info; + int ret; + + pwr_info.lane_rx = lanes; + pwr_info.lane_tx = lanes; + ret = ufshcd_change_power_mode(hba, &pwr_info, + UFSHCD_PMC_POLICY_DONT_FORCE); + if (ret) + dev_err(hba->dev, "%s: Setting %u lanes, err = %d\n", + __func__, lanes, ret); + return ret; +} + +static int ufs_intel_lkf_pwr_change_notify(struct ufs_hba *hba, + enum ufs_notify_change_status status, + struct ufs_pa_layer_attr *dev_req_params) +{ + int err = 0; + + switch (status) { + case PRE_CHANGE: + if (ufshcd_is_hs_mode(dev_req_params) && + (hba->pwr_info.lane_rx != 2 || hba->pwr_info.lane_tx != 2)) + ufs_intel_set_lanes(hba, 2); + break; + case POST_CHANGE: + if (ufshcd_is_hs_mode(dev_req_params)) { + u32 peer_granularity; + + usleep_range(1000, 1250); + err = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY), + &peer_granularity); + } + break; + default: + break; + } + + return err; +} + +static int ufs_intel_lkf_apply_dev_quirks(struct ufs_hba *hba) +{ + u32 granularity, peer_granularity; + u32 pa_tactivate, peer_pa_tactivate; + int ret; + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY), &granularity); + if (ret) + goto out; + + ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY), &peer_granularity); + if (ret) + goto out; + + ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate); + if (ret) + goto out; + + ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &peer_pa_tactivate); + if (ret) + goto out; + + if (granularity == peer_granularity) { + u32 new_peer_pa_tactivate = pa_tactivate + 2; + + ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE), new_peer_pa_tactivate); + } +out: + return ret; +} + +#define INTEL_ACTIVELTR 0x804 +#define INTEL_IDLELTR 0x808 + +#define INTEL_LTR_REQ BIT(15) +#define INTEL_LTR_SCALE_MASK GENMASK(11, 10) +#define INTEL_LTR_SCALE_1US (2 << 10) +#define INTEL_LTR_SCALE_32US (3 << 10) +#define INTEL_LTR_VALUE_MASK GENMASK(9, 0) + +static void intel_cache_ltr(struct ufs_hba *hba) +{ + struct intel_host *host = ufshcd_get_variant(hba); + + host->active_ltr = readl(hba->mmio_base + INTEL_ACTIVELTR); + host->idle_ltr = readl(hba->mmio_base + INTEL_IDLELTR); +} + +static void intel_ltr_set(struct device *dev, s32 val) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + struct intel_host *host = ufshcd_get_variant(hba); + u32 ltr; + + pm_runtime_get_sync(dev); + + /* + * Program latency tolerance (LTR) accordingly what has been asked + * by the PM QoS layer or disable it in case we were passed + * negative value or PM_QOS_LATENCY_ANY. + */ + ltr = readl(hba->mmio_base + INTEL_ACTIVELTR); + + if (val == PM_QOS_LATENCY_ANY || val < 0) { + ltr &= ~INTEL_LTR_REQ; + } else { + ltr |= INTEL_LTR_REQ; + ltr &= ~INTEL_LTR_SCALE_MASK; + ltr &= ~INTEL_LTR_VALUE_MASK; + + if (val > INTEL_LTR_VALUE_MASK) { + val >>= 5; + if (val > INTEL_LTR_VALUE_MASK) + val = INTEL_LTR_VALUE_MASK; + ltr |= INTEL_LTR_SCALE_32US | val; + } else { + ltr |= INTEL_LTR_SCALE_1US | val; + } + } + + if (ltr == host->active_ltr) + goto out; + + writel(ltr, hba->mmio_base + INTEL_ACTIVELTR); + writel(ltr, hba->mmio_base + INTEL_IDLELTR); + + /* Cache the values into intel_host structure */ + intel_cache_ltr(hba); +out: + pm_runtime_put(dev); +} + +static void intel_ltr_expose(struct device *dev) +{ + dev->power.set_latency_tolerance = intel_ltr_set; + dev_pm_qos_expose_latency_tolerance(dev); +} + +static void intel_ltr_hide(struct device *dev) +{ + dev_pm_qos_hide_latency_tolerance(dev); + dev->power.set_latency_tolerance = NULL; +} + +static void intel_add_debugfs(struct ufs_hba *hba) +{ + struct dentry *dir = debugfs_create_dir(dev_name(hba->dev), NULL); + struct intel_host *host = ufshcd_get_variant(hba); + + intel_cache_ltr(hba); + + host->debugfs_root = dir; + debugfs_create_x32("active_ltr", 0444, dir, &host->active_ltr); + debugfs_create_x32("idle_ltr", 0444, dir, &host->idle_ltr); +} + +static void intel_remove_debugfs(struct ufs_hba *hba) +{ + struct intel_host *host = ufshcd_get_variant(hba); + + debugfs_remove_recursive(host->debugfs_root); +} + +static int ufs_intel_device_reset(struct ufs_hba *hba) +{ + struct intel_host *host = ufshcd_get_variant(hba); + + if (INTEL_DSM_SUPPORTED(host, RESET)) { + u32 result = 0; + int err; + + err = intel_dsm(host, hba->dev, INTEL_DSM_RESET, &result); + if (!err && !result) + err = -EIO; + if (err) + dev_err(hba->dev, "%s: DSM error %d result %u\n", + __func__, err, result); + return err; + } + + if (!host->reset_gpio) + return -EOPNOTSUPP; + + gpiod_set_value_cansleep(host->reset_gpio, 1); + usleep_range(10, 15); + + gpiod_set_value_cansleep(host->reset_gpio, 0); + usleep_range(10, 15); + + return 0; +} + +static struct gpio_desc *ufs_intel_get_reset_gpio(struct device *dev) +{ + /* GPIO in _DSD has active low setting */ + return devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW); +} + +static int ufs_intel_common_init(struct ufs_hba *hba) +{ + struct intel_host *host; + + hba->caps |= UFSHCD_CAP_RPM_AUTOSUSPEND; + + host = devm_kzalloc(hba->dev, sizeof(*host), GFP_KERNEL); + if (!host) + return -ENOMEM; + host->saved_spm_lvl = -1; + ufshcd_set_variant(hba, host); + intel_dsm_init(host, hba->dev); + if (INTEL_DSM_SUPPORTED(host, RESET)) { + if (hba->vops->device_reset) + hba->caps |= UFSHCD_CAP_DEEPSLEEP; + } else { + if (hba->vops->device_reset) + host->reset_gpio = ufs_intel_get_reset_gpio(hba->dev); + if (IS_ERR(host->reset_gpio)) { + dev_err(hba->dev, "%s: failed to get reset GPIO, error %ld\n", + __func__, PTR_ERR(host->reset_gpio)); + host->reset_gpio = NULL; + } + if (host->reset_gpio) { + gpiod_set_value_cansleep(host->reset_gpio, 0); + hba->caps |= UFSHCD_CAP_DEEPSLEEP; + } + } + intel_ltr_expose(hba->dev); + intel_add_debugfs(hba); + return 0; +} + +static void ufs_intel_common_exit(struct ufs_hba *hba) +{ + intel_remove_debugfs(hba); + intel_ltr_hide(hba->dev); +} + +static int ufs_intel_resume(struct ufs_hba *hba, enum ufs_pm_op op) +{ + if (ufshcd_is_link_hibern8(hba)) { + int ret = ufshcd_uic_hibern8_exit(hba); + + if (!ret) { + ufshcd_set_link_active(hba); + } else { + dev_err(hba->dev, "%s: hibern8 exit failed %d\n", + __func__, ret); + /* + * Force reset and restore. Any other actions can lead + * to an unrecoverable state. + */ + ufshcd_set_link_off(hba); + } + } + + return 0; +} + +static int ufs_intel_ehl_init(struct ufs_hba *hba) +{ + hba->quirks |= UFSHCD_QUIRK_BROKEN_AUTO_HIBERN8; + return ufs_intel_common_init(hba); +} + +static int ufs_intel_lkf_init(struct ufs_hba *hba) +{ + int err; + + hba->nop_out_timeout = 200; + hba->quirks |= UFSHCD_QUIRK_BROKEN_AUTO_HIBERN8; + hba->caps |= UFSHCD_CAP_CRYPTO; + err = ufs_intel_common_init(hba); + /* LKF always needs a full reset, so set PM accordingly */ + if (hba->caps & UFSHCD_CAP_DEEPSLEEP) { + hba->spm_lvl = UFS_PM_LVL_6; + hba->rpm_lvl = UFS_PM_LVL_6; + } else { + hba->spm_lvl = UFS_PM_LVL_5; + hba->rpm_lvl = UFS_PM_LVL_5; + } + return err; +} + +static int ufs_intel_adl_init(struct ufs_hba *hba) +{ + hba->nop_out_timeout = 200; + hba->quirks |= UFSHCD_QUIRK_BROKEN_AUTO_HIBERN8 | + UFSHCD_QUIRK_PERFORM_LINK_STARTUP_ONCE; + hba->caps |= UFSHCD_CAP_WB_EN; + return ufs_intel_common_init(hba); +} + +static int ufs_intel_mtl_init(struct ufs_hba *hba) +{ + hba->rpm_lvl = UFS_PM_LVL_2; + hba->spm_lvl = UFS_PM_LVL_2; + hba->caps |= UFSHCD_CAP_CRYPTO | UFSHCD_CAP_WB_EN; + return ufs_intel_common_init(hba); +} + +static int ufs_qemu_get_hba_mac(struct ufs_hba *hba) +{ + return MAX_SUPP_MAC; +} + +static int ufs_qemu_mcq_config_resource(struct ufs_hba *hba) +{ + hba->mcq_base = hba->mmio_base + ufshcd_mcq_queue_cfg_addr(hba); + + return 0; +} + +static int ufs_qemu_op_runtime_config(struct ufs_hba *hba) +{ + struct ufshcd_mcq_opr_info_t *opr; + int i; + + u32 sqdao = ufsmcq_readl(hba, ufshcd_mcq_cfg_offset(REG_SQDAO, 0)); + u32 sqisao = ufsmcq_readl(hba, ufshcd_mcq_cfg_offset(REG_SQISAO, 0)); + u32 cqdao = ufsmcq_readl(hba, ufshcd_mcq_cfg_offset(REG_CQDAO, 0)); + u32 cqisao = ufsmcq_readl(hba, ufshcd_mcq_cfg_offset(REG_CQISAO, 0)); + + hba->mcq_opr[OPR_SQD].offset = sqdao; + hba->mcq_opr[OPR_SQIS].offset = sqisao; + hba->mcq_opr[OPR_CQD].offset = cqdao; + hba->mcq_opr[OPR_CQIS].offset = cqisao; + + for (i = 0; i < OPR_MAX; i++) { + opr = &hba->mcq_opr[i]; + opr->stride = 48; + opr->base = hba->mmio_base + opr->offset; + } + + return 0; +} + +static struct ufs_hba_variant_ops ufs_qemu_hba_vops = { + .name = "qemu-pci", + .get_hba_mac = ufs_qemu_get_hba_mac, + .mcq_config_resource = ufs_qemu_mcq_config_resource, + .op_runtime_config = ufs_qemu_op_runtime_config, +}; + +static struct ufs_hba_variant_ops ufs_intel_cnl_hba_vops = { + .name = "intel-pci", + .init = ufs_intel_common_init, + .exit = ufs_intel_common_exit, + .link_startup_notify = ufs_intel_link_startup_notify, + .resume = ufs_intel_resume, +}; + +static struct ufs_hba_variant_ops ufs_intel_ehl_hba_vops = { + .name = "intel-pci", + .init = ufs_intel_ehl_init, + .exit = ufs_intel_common_exit, + .link_startup_notify = ufs_intel_link_startup_notify, + .resume = ufs_intel_resume, +}; + +static struct ufs_hba_variant_ops ufs_intel_lkf_hba_vops = { + .name = "intel-pci", + .init = ufs_intel_lkf_init, + .exit = ufs_intel_common_exit, + .hce_enable_notify = ufs_intel_hce_enable_notify, + .link_startup_notify = ufs_intel_link_startup_notify, + .pwr_change_notify = ufs_intel_lkf_pwr_change_notify, + .apply_dev_quirks = ufs_intel_lkf_apply_dev_quirks, + .resume = ufs_intel_resume, + .device_reset = ufs_intel_device_reset, +}; + +static struct ufs_hba_variant_ops ufs_intel_adl_hba_vops = { + .name = "intel-pci", + .init = ufs_intel_adl_init, + .exit = ufs_intel_common_exit, + .link_startup_notify = ufs_intel_link_startup_notify, + .resume = ufs_intel_resume, + .device_reset = ufs_intel_device_reset, +}; + +static struct ufs_hba_variant_ops ufs_intel_mtl_hba_vops = { + .name = "intel-pci", + .init = ufs_intel_mtl_init, + .exit = ufs_intel_common_exit, + .hce_enable_notify = ufs_intel_hce_enable_notify, + .link_startup_notify = ufs_intel_link_startup_notify, + .resume = ufs_intel_resume, + .device_reset = ufs_intel_device_reset, +}; + +#ifdef CONFIG_PM_SLEEP +static int ufshcd_pci_restore(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + + /* Force a full reset and restore */ + ufshcd_set_link_off(hba); + + return ufshcd_system_resume(dev); +} + +static int ufs_intel_suspend_prepare(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + struct intel_host *host = ufshcd_get_variant(hba); + int err; + + /* + * Only s2idle (S0ix) retains link state. Force power-off + * (UFS_PM_LVL_5) for any other case. + */ + if (pm_suspend_target_state != PM_SUSPEND_TO_IDLE && hba->spm_lvl < UFS_PM_LVL_5) { + host->saved_spm_lvl = hba->spm_lvl; + hba->spm_lvl = UFS_PM_LVL_5; + } + + err = ufshcd_suspend_prepare(dev); + + if (err < 0 && host->saved_spm_lvl != -1) { + hba->spm_lvl = host->saved_spm_lvl; + host->saved_spm_lvl = -1; + } + + return err; +} + +static void ufs_intel_resume_complete(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + struct intel_host *host = ufshcd_get_variant(hba); + + ufshcd_resume_complete(dev); + + if (host->saved_spm_lvl != -1) { + hba->spm_lvl = host->saved_spm_lvl; + host->saved_spm_lvl = -1; + } +} + +static int ufshcd_pci_suspend_prepare(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + + if (!strcmp(hba->vops->name, "intel-pci")) + return ufs_intel_suspend_prepare(dev); + + return ufshcd_suspend_prepare(dev); +} + +static void ufshcd_pci_resume_complete(struct device *dev) +{ + struct ufs_hba *hba = dev_get_drvdata(dev); + + if (!strcmp(hba->vops->name, "intel-pci")) { + ufs_intel_resume_complete(dev); + return; + } + + ufshcd_resume_complete(dev); +} +#endif + +/** + * ufshcd_pci_remove - de-allocate PCI/SCSI host and host memory space + * data structure memory + * @pdev: pointer to PCI handle + */ +static void ufshcd_pci_remove(struct pci_dev *pdev) +{ + struct ufs_hba *hba = pci_get_drvdata(pdev); + + pm_runtime_forbid(&pdev->dev); + pm_runtime_get_noresume(&pdev->dev); + ufshcd_remove(hba); +} + +/** + * ufshcd_pci_probe - probe routine of the driver + * @pdev: pointer to PCI device handle + * @id: PCI device id + * + * Return: 0 on success, non-zero value on failure. + */ +static int +ufshcd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct ufs_hba *hba; + void __iomem *mmio_base; + int err; + + err = pcim_enable_device(pdev); + if (err) { + dev_err(&pdev->dev, "pcim_enable_device failed\n"); + return err; + } + + pci_set_master(pdev); + + mmio_base = pcim_iomap_region(pdev, 0, UFSHCD); + if (IS_ERR(mmio_base)) { + dev_err(&pdev->dev, "request and iomap failed\n"); + return PTR_ERR(mmio_base); + } + + err = ufshcd_alloc_host(&pdev->dev, &hba); + if (err) { + dev_err(&pdev->dev, "Allocation failed\n"); + return err; + } + + hba->vops = (struct ufs_hba_variant_ops *)id->driver_data; + + err = ufshcd_init(hba, mmio_base, pdev->irq); + if (err) { + dev_err(&pdev->dev, "Initialization failed\n"); + return err; + } + + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_allow(&pdev->dev); + + return 0; +} + +static const struct dev_pm_ops ufshcd_pci_pm_ops = { + SET_RUNTIME_PM_OPS(ufshcd_runtime_suspend, ufshcd_runtime_resume, NULL) +#ifdef CONFIG_PM_SLEEP + .suspend = ufshcd_system_suspend, + .resume = ufshcd_system_resume, + .freeze = ufshcd_system_suspend, + .thaw = ufshcd_system_resume, + .poweroff = ufshcd_system_suspend, + .restore = ufshcd_pci_restore, + .prepare = ufshcd_pci_suspend_prepare, + .complete = ufshcd_pci_resume_complete, +#endif +}; + +static const struct pci_device_id ufshcd_pci_tbl[] = { + { PCI_VDEVICE(REDHAT, 0x0013), .driver_data = (kernel_ulong_t)&ufs_qemu_hba_vops }, + { PCI_VDEVICE(SAMSUNG, 0xC00C), .driver_data = 0 }, + { PCI_VDEVICE(INTEL, 0x9DFA), .driver_data = (kernel_ulong_t)&ufs_intel_cnl_hba_vops }, + { PCI_VDEVICE(INTEL, 0x4B41), .driver_data = (kernel_ulong_t)&ufs_intel_ehl_hba_vops }, + { PCI_VDEVICE(INTEL, 0x4B43), .driver_data = (kernel_ulong_t)&ufs_intel_ehl_hba_vops }, + { PCI_VDEVICE(INTEL, 0x98FA), .driver_data = (kernel_ulong_t)&ufs_intel_lkf_hba_vops }, + { PCI_VDEVICE(INTEL, 0x51FF), .driver_data = (kernel_ulong_t)&ufs_intel_adl_hba_vops }, + { PCI_VDEVICE(INTEL, 0x54FF), .driver_data = (kernel_ulong_t)&ufs_intel_adl_hba_vops }, + { PCI_VDEVICE(INTEL, 0x7E47), .driver_data = (kernel_ulong_t)&ufs_intel_mtl_hba_vops }, + { PCI_VDEVICE(INTEL, 0xA847), .driver_data = (kernel_ulong_t)&ufs_intel_mtl_hba_vops }, + { PCI_VDEVICE(INTEL, 0x7747), .driver_data = (kernel_ulong_t)&ufs_intel_mtl_hba_vops }, + { PCI_VDEVICE(INTEL, 0xE447), .driver_data = (kernel_ulong_t)&ufs_intel_mtl_hba_vops }, + { PCI_VDEVICE(INTEL, 0x4D47), .driver_data = (kernel_ulong_t)&ufs_intel_mtl_hba_vops }, + { PCI_VDEVICE(INTEL, 0xD335), .driver_data = (kernel_ulong_t)&ufs_intel_mtl_hba_vops }, + { PCI_VDEVICE(AMD, 0x1B29), .driver_data = 0 }, + { } /* terminate list */ +}; + +MODULE_DEVICE_TABLE(pci, ufshcd_pci_tbl); + +static struct pci_driver ufshcd_pci_driver = { + .name = UFSHCD, + .id_table = ufshcd_pci_tbl, + .probe = ufshcd_pci_probe, + .remove = ufshcd_pci_remove, + .driver = { + .pm = &ufshcd_pci_pm_ops + }, +}; + +module_pci_driver(ufshcd_pci_driver); + +MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>"); +MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>"); +MODULE_DESCRIPTION("UFS host controller PCI glue driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/ufs/host/ufshcd-pltfrm.c b/drivers/ufs/host/ufshcd-pltfrm.c new file mode 100644 index 000000000..c2dafb583 --- /dev/null +++ b/drivers/ufs/host/ufshcd-pltfrm.c @@ -0,0 +1,569 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Universal Flash Storage Host controller Platform bus based glue driver + * Copyright (C) 2011-2013 Samsung India Software Operations + * + * Authors: + * Santosh Yaraganavi <santosh.sy@samsung.com> + * Vinayak Holikatti <h.vinayak@samsung.com> + */ + +#include <linux/clk.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/pm_opp.h> +#include <linux/pm_runtime.h> +#include <linux/of.h> + +#include <ufs/ufshcd.h> +#include "ufshcd-pltfrm.h" +#include <ufs/unipro.h> + +#define UFSHCD_DEFAULT_LANES_PER_DIRECTION 2 + +static int ufshcd_parse_clock_info(struct ufs_hba *hba) +{ + int ret = 0; + int cnt; + int i; + struct device *dev = hba->dev; + struct device_node *np = dev->of_node; + const char *name; + u32 *clkfreq = NULL; + struct ufs_clk_info *clki; + ssize_t sz = 0; + + if (!np) + goto out; + + cnt = of_property_count_strings(np, "clock-names"); + if (!cnt || (cnt == -EINVAL)) { + dev_info(dev, "%s: Unable to find clocks, assuming enabled\n", + __func__); + } else if (cnt < 0) { + dev_err(dev, "%s: count clock strings failed, err %d\n", + __func__, cnt); + ret = cnt; + } + + if (cnt <= 0) + goto out; + + sz = of_property_count_u32_elems(np, "freq-table-hz"); + if (sz <= 0) { + dev_info(dev, "freq-table-hz property not specified\n"); + goto out; + } + + if (sz != 2 * cnt) { + dev_err(dev, "%s len mismatch\n", "freq-table-hz"); + ret = -EINVAL; + goto out; + } + + clkfreq = devm_kcalloc(dev, sz, sizeof(*clkfreq), + GFP_KERNEL); + if (!clkfreq) { + ret = -ENOMEM; + goto out; + } + + ret = of_property_read_u32_array(np, "freq-table-hz", + clkfreq, sz); + if (ret && (ret != -EINVAL)) { + dev_err(dev, "%s: error reading array %d\n", + "freq-table-hz", ret); + return ret; + } + + for (i = 0; i < sz; i += 2) { + ret = of_property_read_string_index(np, "clock-names", i/2, + &name); + if (ret) + goto out; + + clki = devm_kzalloc(dev, sizeof(*clki), GFP_KERNEL); + if (!clki) { + ret = -ENOMEM; + goto out; + } + + clki->min_freq = clkfreq[i]; + clki->max_freq = clkfreq[i+1]; + clki->name = devm_kstrdup(dev, name, GFP_KERNEL); + if (!clki->name) { + ret = -ENOMEM; + goto out; + } + + if (!strcmp(name, "ref_clk")) + clki->keep_link_active = true; + dev_dbg(dev, "%s: min %u max %u name %s\n", "freq-table-hz", + clki->min_freq, clki->max_freq, clki->name); + list_add_tail(&clki->list, &hba->clk_list_head); + } +out: + return ret; +} + +static bool phandle_exists(const struct device_node *np, + const char *phandle_name, int index) +{ + struct device_node *parse_np = of_parse_phandle(np, phandle_name, index); + + if (parse_np) + of_node_put(parse_np); + + return parse_np != NULL; +} + +#define MAX_PROP_SIZE 32 +int ufshcd_populate_vreg(struct device *dev, const char *name, + struct ufs_vreg **out_vreg, bool skip_current) +{ + char prop_name[MAX_PROP_SIZE]; + struct ufs_vreg *vreg = NULL; + struct device_node *np = dev->of_node; + + if (!np) { + dev_err(dev, "%s: non DT initialization\n", __func__); + goto out; + } + + snprintf(prop_name, MAX_PROP_SIZE, "%s-supply", name); + if (!phandle_exists(np, prop_name, 0)) { + dev_info(dev, "%s: Unable to find %s regulator, assuming enabled\n", + __func__, prop_name); + goto out; + } + + vreg = devm_kzalloc(dev, sizeof(*vreg), GFP_KERNEL); + if (!vreg) + return -ENOMEM; + + vreg->name = devm_kstrdup(dev, name, GFP_KERNEL); + if (!vreg->name) + return -ENOMEM; + + if (skip_current) { + vreg->max_uA = 0; + goto out; + } + + snprintf(prop_name, MAX_PROP_SIZE, "%s-max-microamp", name); + if (of_property_read_u32(np, prop_name, &vreg->max_uA)) { + dev_info(dev, "%s: unable to find %s\n", __func__, prop_name); + vreg->max_uA = 0; + } +out: + *out_vreg = vreg; + return 0; +} +EXPORT_SYMBOL_GPL(ufshcd_populate_vreg); + +/** + * ufshcd_parse_regulator_info - get regulator info from device tree + * @hba: per adapter instance + * + * Get regulator info from device tree for vcc, vccq, vccq2 power supplies. + * If any of the supplies are not defined it is assumed that they are always-on + * and hence return zero. If the property is defined but parsing is failed + * then return corresponding error. + * + * Return: 0 upon success; < 0 upon failure. + */ +static int ufshcd_parse_regulator_info(struct ufs_hba *hba) +{ + int err; + struct device *dev = hba->dev; + struct ufs_vreg_info *info = &hba->vreg_info; + + err = ufshcd_populate_vreg(dev, "vdd-hba", &info->vdd_hba, true); + if (err) + goto out; + + err = ufshcd_populate_vreg(dev, "vcc", &info->vcc, false); + if (err) + goto out; + + err = ufshcd_populate_vreg(dev, "vccq", &info->vccq, false); + if (err) + goto out; + + err = ufshcd_populate_vreg(dev, "vccq2", &info->vccq2, false); +out: + return err; +} + +static void ufshcd_init_lanes_per_dir(struct ufs_hba *hba) +{ + struct device *dev = hba->dev; + int ret; + + ret = of_property_read_u32(dev->of_node, "lanes-per-direction", + &hba->lanes_per_direction); + if (ret) { + dev_dbg(hba->dev, + "%s: failed to read lanes-per-direction, ret=%d\n", + __func__, ret); + hba->lanes_per_direction = UFSHCD_DEFAULT_LANES_PER_DIRECTION; + } +} + +/** + * ufshcd_parse_clock_min_max_freq - Parse MIN and MAX clocks freq + * @hba: per adapter instance + * + * This function parses MIN and MAX frequencies of all clocks required + * by the host drivers. + * + * Returns 0 for success and non-zero for failure + */ +static int ufshcd_parse_clock_min_max_freq(struct ufs_hba *hba) +{ + struct list_head *head = &hba->clk_list_head; + struct ufs_clk_info *clki; + struct dev_pm_opp *opp; + unsigned long freq; + u8 idx = 0; + + list_for_each_entry(clki, head, list) { + if (!clki->name) + continue; + + clki->clk = devm_clk_get(hba->dev, clki->name); + if (IS_ERR(clki->clk)) + continue; + + /* Find Max Freq */ + freq = ULONG_MAX; + opp = dev_pm_opp_find_freq_floor_indexed(hba->dev, &freq, idx); + if (IS_ERR(opp)) { + dev_err(hba->dev, "Failed to find OPP for MAX frequency\n"); + return PTR_ERR(opp); + } + clki->max_freq = dev_pm_opp_get_freq_indexed(opp, idx); + dev_pm_opp_put(opp); + + /* Find Min Freq */ + freq = 0; + opp = dev_pm_opp_find_freq_ceil_indexed(hba->dev, &freq, idx); + if (IS_ERR(opp)) { + dev_err(hba->dev, "Failed to find OPP for MIN frequency\n"); + return PTR_ERR(opp); + } + clki->min_freq = dev_pm_opp_get_freq_indexed(opp, idx++); + dev_pm_opp_put(opp); + } + + return 0; +} + +static int ufshcd_parse_operating_points(struct ufs_hba *hba) +{ + struct device *dev = hba->dev; + struct device_node *np = dev->of_node; + struct dev_pm_opp_config config = {}; + struct ufs_clk_info *clki; + const char **clk_names; + int cnt, i, ret; + + if (!of_property_present(np, "operating-points-v2")) + return 0; + + if (of_property_present(np, "freq-table-hz")) { + dev_err(dev, "%s: operating-points and freq-table-hz are incompatible\n", + __func__); + return -EINVAL; + } + + cnt = of_property_count_strings(np, "clock-names"); + if (cnt <= 0) { + dev_err(dev, "%s: Missing clock-names\n", __func__); + return -ENODEV; + } + + /* OPP expects clk_names to be NULL terminated */ + clk_names = devm_kcalloc(dev, cnt + 1, sizeof(*clk_names), GFP_KERNEL); + if (!clk_names) + return -ENOMEM; + + /* + * We still need to get reference to all clocks as the UFS core uses + * them separately. + */ + for (i = 0; i < cnt; i++) { + ret = of_property_read_string_index(np, "clock-names", i, + &clk_names[i]); + if (ret) + return ret; + + clki = devm_kzalloc(dev, sizeof(*clki), GFP_KERNEL); + if (!clki) + return -ENOMEM; + + clki->name = devm_kstrdup(dev, clk_names[i], GFP_KERNEL); + if (!clki->name) + return -ENOMEM; + + if (!strcmp(clk_names[i], "ref_clk")) + clki->keep_link_active = true; + + list_add_tail(&clki->list, &hba->clk_list_head); + } + + config.clk_names = clk_names, + config.config_clks = ufshcd_opp_config_clks; + + ret = devm_pm_opp_set_config(dev, &config); + if (ret) + return ret; + + ret = devm_pm_opp_of_add_table(dev); + if (ret) { + dev_err(dev, "Failed to add OPP table: %d\n", ret); + return ret; + } + + ret = ufshcd_parse_clock_min_max_freq(hba); + if (ret) + return ret; + + hba->use_pm_opp = true; + + return 0; +} + +/** + * ufshcd_negotiate_pwr_params - find power mode settings that are supported by + * both the controller and the device + * @host_params: pointer to host parameters + * @dev_max: pointer to device attributes + * @agreed_pwr: returned agreed attributes + * + * Return: 0 on success, non-zero value on failure. + */ +int ufshcd_negotiate_pwr_params(const struct ufs_host_params *host_params, + const struct ufs_pa_layer_attr *dev_max, + struct ufs_pa_layer_attr *agreed_pwr) +{ + int min_host_gear; + int min_dev_gear; + bool is_dev_sup_hs = false; + bool is_host_max_hs = false; + + if (dev_max->pwr_rx == FAST_MODE) + is_dev_sup_hs = true; + + if (host_params->desired_working_mode == UFS_HS_MODE) { + is_host_max_hs = true; + min_host_gear = min_t(u32, host_params->hs_rx_gear, + host_params->hs_tx_gear); + } else { + min_host_gear = min_t(u32, host_params->pwm_rx_gear, + host_params->pwm_tx_gear); + } + + /* + * device doesn't support HS but host_params->desired_working_mode is HS, + * thus device and host_params don't agree + */ + if (!is_dev_sup_hs && is_host_max_hs) { + pr_info("%s: device doesn't support HS\n", + __func__); + return -ENOTSUPP; + } else if (is_dev_sup_hs && is_host_max_hs) { + /* + * since device supports HS, it supports FAST_MODE. + * since host_params->desired_working_mode is also HS + * then final decision (FAST/FASTAUTO) is done according + * to pltfrm_params as it is the restricting factor + */ + agreed_pwr->pwr_rx = host_params->rx_pwr_hs; + agreed_pwr->pwr_tx = agreed_pwr->pwr_rx; + } else { + /* + * here host_params->desired_working_mode is PWM. + * it doesn't matter whether device supports HS or PWM, + * in both cases host_params->desired_working_mode will + * determine the mode + */ + agreed_pwr->pwr_rx = host_params->rx_pwr_pwm; + agreed_pwr->pwr_tx = agreed_pwr->pwr_rx; + } + + /* + * we would like tx to work in the minimum number of lanes + * between device capability and vendor preferences. + * the same decision will be made for rx + */ + agreed_pwr->lane_tx = min_t(u32, dev_max->lane_tx, + host_params->tx_lanes); + agreed_pwr->lane_rx = min_t(u32, dev_max->lane_rx, + host_params->rx_lanes); + + /* device maximum gear is the minimum between device rx and tx gears */ + min_dev_gear = min_t(u32, dev_max->gear_rx, dev_max->gear_tx); + + /* + * if both device capabilities and vendor pre-defined preferences are + * both HS or both PWM then set the minimum gear to be the chosen + * working gear. + * if one is PWM and one is HS then the one that is PWM get to decide + * what is the gear, as it is the one that also decided previously what + * pwr the device will be configured to. + */ + if ((is_dev_sup_hs && is_host_max_hs) || + (!is_dev_sup_hs && !is_host_max_hs)) { + agreed_pwr->gear_rx = + min_t(u32, min_dev_gear, min_host_gear); + } else if (!is_dev_sup_hs) { + agreed_pwr->gear_rx = min_dev_gear; + } else { + agreed_pwr->gear_rx = min_host_gear; + } + agreed_pwr->gear_tx = agreed_pwr->gear_rx; + + agreed_pwr->hs_rate = host_params->hs_rate; + + return 0; +} +EXPORT_SYMBOL_GPL(ufshcd_negotiate_pwr_params); + +/** + * ufshcd_parse_gear_limits - Parse DT-based gear and rate limits for UFS + * @hba: Pointer to UFS host bus adapter instance + * @host_params: Pointer to UFS host parameters structure to be updated + * + * This function reads optional device tree properties to apply + * platform-specific constraints. + * + * "limit-hs-gear": Specifies the max HS gear. + * "limit-gear-rate": Specifies the max High-Speed rate. + */ +void ufshcd_parse_gear_limits(struct ufs_hba *hba, struct ufs_host_params *host_params) +{ + struct device_node *np = hba->dev->of_node; + u32 hs_gear; + const char *hs_rate; + + if (!of_property_read_u32(np, "limit-hs-gear", &hs_gear)) { + host_params->hs_tx_gear = hs_gear; + host_params->hs_rx_gear = hs_gear; + } + + if (!of_property_read_string(np, "limit-gear-rate", &hs_rate)) { + if (!strcmp(hs_rate, "rate-a")) + host_params->hs_rate = PA_HS_MODE_A; + else if (!strcmp(hs_rate, "rate-b")) + host_params->hs_rate = PA_HS_MODE_B; + else + dev_warn(hba->dev, "Invalid rate: %s\n", hs_rate); + } +} +EXPORT_SYMBOL_GPL(ufshcd_parse_gear_limits); + +void ufshcd_init_host_params(struct ufs_host_params *host_params) +{ + *host_params = (struct ufs_host_params){ + .tx_lanes = UFS_LANE_2, + .rx_lanes = UFS_LANE_2, + .hs_rx_gear = UFS_HS_G3, + .hs_tx_gear = UFS_HS_G3, + .pwm_rx_gear = UFS_PWM_G4, + .pwm_tx_gear = UFS_PWM_G4, + .rx_pwr_pwm = SLOW_MODE, + .tx_pwr_pwm = SLOW_MODE, + .rx_pwr_hs = FAST_MODE, + .tx_pwr_hs = FAST_MODE, + .hs_rate = PA_HS_MODE_B, + .desired_working_mode = UFS_HS_MODE, + }; +} +EXPORT_SYMBOL_GPL(ufshcd_init_host_params); + +/** + * ufshcd_pltfrm_init - probe routine of the driver + * @pdev: pointer to Platform device handle + * @vops: pointer to variant ops + * + * Return: 0 on success, non-zero value on failure. + */ +int ufshcd_pltfrm_init(struct platform_device *pdev, + const struct ufs_hba_variant_ops *vops) +{ + struct ufs_hba *hba; + void __iomem *mmio_base; + int irq, err; + struct device *dev = &pdev->dev; + + mmio_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(mmio_base)) + return PTR_ERR(mmio_base); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + err = ufshcd_alloc_host(dev, &hba); + if (err) { + dev_err(dev, "Allocation failed\n"); + return err; + } + + hba->vops = vops; + + err = ufshcd_parse_clock_info(hba); + if (err) { + dev_err(dev, "%s: clock parse failed %d\n", + __func__, err); + return err; + } + err = ufshcd_parse_regulator_info(hba); + if (err) { + dev_err(dev, "%s: regulator init failed %d\n", + __func__, err); + return err; + } + + ufshcd_init_lanes_per_dir(hba); + + err = ufshcd_parse_operating_points(hba); + if (err) { + dev_err(dev, "%s: OPP parse failed %d\n", __func__, err); + return err; + } + + err = ufshcd_init(hba, mmio_base, irq); + if (err) { + dev_err_probe(dev, err, "Initialization failed with error %d\n", + err); + return err; + } + + pm_runtime_set_active(dev); + pm_runtime_enable(dev); + + return 0; +} +EXPORT_SYMBOL_GPL(ufshcd_pltfrm_init); + +/** + * ufshcd_pltfrm_remove - Remove ufshcd platform + * @pdev: pointer to Platform device handle + */ +void ufshcd_pltfrm_remove(struct platform_device *pdev) +{ + struct ufs_hba *hba = platform_get_drvdata(pdev); + + pm_runtime_get_sync(&pdev->dev); + ufshcd_remove(hba); + pm_runtime_disable(&pdev->dev); + pm_runtime_put_noidle(&pdev->dev); +} +EXPORT_SYMBOL_GPL(ufshcd_pltfrm_remove); + +MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>"); +MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>"); +MODULE_DESCRIPTION("UFS host controller Platform bus based glue driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/ufs/host/ufshcd-pltfrm.h b/drivers/ufs/host/ufshcd-pltfrm.h new file mode 100644 index 000000000..0a18a8aed --- /dev/null +++ b/drivers/ufs/host/ufshcd-pltfrm.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2015, The Linux Foundation. All rights reserved. + */ + +#ifndef UFSHCD_PLTFRM_H_ +#define UFSHCD_PLTFRM_H_ + +#include <ufs/ufshcd.h> + +#define UFS_PWM_MODE 1 +#define UFS_HS_MODE 2 + +struct ufs_host_params { + u32 pwm_rx_gear; /* pwm rx gear to work in */ + u32 pwm_tx_gear; /* pwm tx gear to work in */ + u32 hs_rx_gear; /* hs rx gear to work in */ + u32 hs_tx_gear; /* hs tx gear to work in */ + u32 rx_lanes; /* number of rx lanes */ + u32 tx_lanes; /* number of tx lanes */ + u32 rx_pwr_pwm; /* rx pwm working pwr */ + u32 tx_pwr_pwm; /* tx pwm working pwr */ + u32 rx_pwr_hs; /* rx hs working pwr */ + u32 tx_pwr_hs; /* tx hs working pwr */ + u32 hs_rate; /* rate A/B to work in HS */ + u32 desired_working_mode; +}; + +int ufshcd_negotiate_pwr_params(const struct ufs_host_params *host_params, + const struct ufs_pa_layer_attr *dev_max, + struct ufs_pa_layer_attr *agreed_pwr); +void ufshcd_init_host_params(struct ufs_host_params *host_params); +void ufshcd_parse_gear_limits(struct ufs_hba *hba, struct ufs_host_params *host_params); +int ufshcd_pltfrm_init(struct platform_device *pdev, + const struct ufs_hba_variant_ops *vops); +void ufshcd_pltfrm_remove(struct platform_device *pdev); +int ufshcd_populate_vreg(struct device *dev, const char *name, + struct ufs_vreg **out_vreg, bool skip_current); + +#endif /* UFSHCD_PLTFRM_H_ */ diff --git a/drivers/ufs/host/ufshci-dwc.h b/drivers/ufs/host/ufshci-dwc.h new file mode 100644 index 000000000..6c290e272 --- /dev/null +++ b/drivers/ufs/host/ufshci-dwc.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * UFS Host driver for Synopsys Designware Core + * + * Copyright (C) 2015-2016 Synopsys, Inc. (www.synopsys.com) + * + * Authors: Joao Pinto <jpinto@synopsys.com> + */ + +#ifndef _UFSHCI_DWC_H +#define _UFSHCI_DWC_H + +/* DWC HC UFSHCI specific Registers */ +enum dwc_specific_registers { + DWC_UFS_REG_HCLKDIV = 0xFC, +}; + +/* Clock Divider Values: Hex equivalent of frequency in MHz */ +enum clk_div_values { + DWC_UFS_REG_HCLKDIV_DIV_62_5 = 0x3e, + DWC_UFS_REG_HCLKDIV_DIV_125 = 0x7d, + DWC_UFS_REG_HCLKDIV_DIV_200 = 0xc8, +}; + +/* Selector Index */ +enum selector_index { + SELIND_LN0_TX = 0x00, + SELIND_LN1_TX = 0x01, + SELIND_LN0_RX = 0x04, + SELIND_LN1_RX = 0x05, +}; + +#endif /* End of Header */ |