diff options
Diffstat (limited to 'drivers/cache')
| -rw-r--r-- | drivers/cache/Kconfig | 56 | ||||
| -rw-r--r-- | drivers/cache/Makefile | 7 | ||||
| -rw-r--r-- | drivers/cache/ax45mp_cache.c | 217 | ||||
| -rw-r--r-- | drivers/cache/hisi_soc_hha.c | 193 | ||||
| -rw-r--r-- | drivers/cache/sifive_ccache.c | 354 | ||||
| -rw-r--r-- | drivers/cache/starfive_starlink_cache.c | 130 |
6 files changed, 957 insertions, 0 deletions
diff --git a/drivers/cache/Kconfig b/drivers/cache/Kconfig new file mode 100644 index 000000000..1518449d4 --- /dev/null +++ b/drivers/cache/Kconfig @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: GPL-2.0 + +menuconfig CACHEMAINT_FOR_DMA + bool "Cache management for noncoherent DMA" + depends on RISCV + default y + help + These drivers implement support for noncoherent DMA master devices + on platforms that lack the standard CPU interfaces for this. + +if CACHEMAINT_FOR_DMA + +config AX45MP_L2_CACHE + bool "Andes Technology AX45MP L2 Cache controller" + select RISCV_NONSTANDARD_CACHE_OPS + help + Support for the L2 cache controller on Andes Technology AX45MP platforms. + +config SIFIVE_CCACHE + bool "Sifive Composable Cache controller" + depends on ARCH_SIFIVE || ARCH_STARFIVE + help + Support for the composable cache controller on SiFive platforms. + +config STARFIVE_STARLINK_CACHE + bool "StarFive StarLink Cache controller" + depends on ARCH_STARFIVE + depends on 64BIT + select RISCV_DMA_NONCOHERENT + select RISCV_NONSTANDARD_CACHE_OPS + help + Support for the StarLink cache controller IP from StarFive. + +endif #CACHEMAINT_FOR_DMA + +menuconfig CACHEMAINT_FOR_HOTPLUG + bool "Cache management for memory hot plug like operations" + depends on GENERIC_CPU_CACHE_MAINTENANCE + help + These drivers implement cache management for flows where it is necessary + to flush data from all host caches. + +if CACHEMAINT_FOR_HOTPLUG + +config HISI_SOC_HHA + tristate "HiSilicon Hydra Home Agent (HHA) device driver" + depends on (ARM64 && ACPI) || COMPILE_TEST + help + The Hydra Home Agent (HHA) is responsible for cache coherency + on the SoC. This drivers enables the cache maintenance functions of + the HHA. + + This driver can be built as a module. If so, the module will be + called hisi_soc_hha. + +endif #CACHEMAINT_FOR_HOTPLUG diff --git a/drivers/cache/Makefile b/drivers/cache/Makefile new file mode 100644 index 000000000..b3362b15d --- /dev/null +++ b/drivers/cache/Makefile @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_AX45MP_L2_CACHE) += ax45mp_cache.o +obj-$(CONFIG_SIFIVE_CCACHE) += sifive_ccache.o +obj-$(CONFIG_STARFIVE_STARLINK_CACHE) += starfive_starlink_cache.o + +obj-$(CONFIG_HISI_SOC_HHA) += hisi_soc_hha.o diff --git a/drivers/cache/ax45mp_cache.c b/drivers/cache/ax45mp_cache.c new file mode 100644 index 000000000..934c5087e --- /dev/null +++ b/drivers/cache/ax45mp_cache.c @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * non-coherent cache functions for Andes AX45MP + * + * Copyright (C) 2023 Renesas Electronics Corp. + */ + +#include <linux/cacheflush.h> +#include <linux/cacheinfo.h> +#include <linux/dma-direction.h> +#include <linux/of_address.h> +#include <linux/of_platform.h> + +#include <asm/dma-noncoherent.h> + +/* L2 cache registers */ +#define AX45MP_L2C_REG_CTL_OFFSET 0x8 + +#define AX45MP_L2C_REG_C0_CMD_OFFSET 0x40 +#define AX45MP_L2C_REG_C0_ACC_OFFSET 0x48 +#define AX45MP_L2C_REG_STATUS_OFFSET 0x80 + +/* D-cache operation */ +#define AX45MP_CCTL_L1D_VA_INVAL 0 /* Invalidate an L1 cache entry */ +#define AX45MP_CCTL_L1D_VA_WB 1 /* Write-back an L1 cache entry */ + +/* L2 CCTL status */ +#define AX45MP_CCTL_L2_STATUS_IDLE 0 + +/* L2 CCTL status cores mask */ +#define AX45MP_CCTL_L2_STATUS_C0_MASK 0xf + +/* L2 cache operation */ +#define AX45MP_CCTL_L2_PA_INVAL 0x8 /* Invalidate an L2 cache entry */ +#define AX45MP_CCTL_L2_PA_WB 0x9 /* Write-back an L2 cache entry */ + +#define AX45MP_L2C_REG_PER_CORE_OFFSET 0x10 +#define AX45MP_CCTL_L2_STATUS_PER_CORE_OFFSET 4 + +#define AX45MP_L2C_REG_CN_CMD_OFFSET(n) \ + (AX45MP_L2C_REG_C0_CMD_OFFSET + ((n) * AX45MP_L2C_REG_PER_CORE_OFFSET)) +#define AX45MP_L2C_REG_CN_ACC_OFFSET(n) \ + (AX45MP_L2C_REG_C0_ACC_OFFSET + ((n) * AX45MP_L2C_REG_PER_CORE_OFFSET)) +#define AX45MP_CCTL_L2_STATUS_CN_MASK(n) \ + (AX45MP_CCTL_L2_STATUS_C0_MASK << ((n) * AX45MP_CCTL_L2_STATUS_PER_CORE_OFFSET)) + +#define AX45MP_CCTL_REG_UCCTLBEGINADDR_NUM 0x80b +#define AX45MP_CCTL_REG_UCCTLCOMMAND_NUM 0x80c + +#define AX45MP_CACHE_LINE_SIZE 64 + +struct ax45mp_priv { + void __iomem *l2c_base; + u32 ax45mp_cache_line_size; +}; + +static struct ax45mp_priv ax45mp_priv; + +/* L2 Cache operations */ +static inline uint32_t ax45mp_cpu_l2c_get_cctl_status(void) +{ + return readl(ax45mp_priv.l2c_base + AX45MP_L2C_REG_STATUS_OFFSET); +} + +static void ax45mp_cpu_cache_operation(unsigned long start, unsigned long end, + unsigned int l1_op, unsigned int l2_op) +{ + unsigned long line_size = ax45mp_priv.ax45mp_cache_line_size; + void __iomem *base = ax45mp_priv.l2c_base; + int mhartid = smp_processor_id(); + unsigned long pa; + + while (end > start) { + csr_write(AX45MP_CCTL_REG_UCCTLBEGINADDR_NUM, start); + csr_write(AX45MP_CCTL_REG_UCCTLCOMMAND_NUM, l1_op); + + pa = virt_to_phys((void *)start); + writel(pa, base + AX45MP_L2C_REG_CN_ACC_OFFSET(mhartid)); + writel(l2_op, base + AX45MP_L2C_REG_CN_CMD_OFFSET(mhartid)); + while ((ax45mp_cpu_l2c_get_cctl_status() & + AX45MP_CCTL_L2_STATUS_CN_MASK(mhartid)) != + AX45MP_CCTL_L2_STATUS_IDLE) + ; + + start += line_size; + } +} + +/* Write-back L1 and L2 cache entry */ +static inline void ax45mp_cpu_dcache_wb_range(unsigned long start, unsigned long end) +{ + ax45mp_cpu_cache_operation(start, end, AX45MP_CCTL_L1D_VA_WB, + AX45MP_CCTL_L2_PA_WB); +} + +/* Invalidate the L1 and L2 cache entry */ +static inline void ax45mp_cpu_dcache_inval_range(unsigned long start, unsigned long end) +{ + ax45mp_cpu_cache_operation(start, end, AX45MP_CCTL_L1D_VA_INVAL, + AX45MP_CCTL_L2_PA_INVAL); +} + +static void ax45mp_dma_cache_inv(phys_addr_t paddr, size_t size) +{ + unsigned long start = (unsigned long)phys_to_virt(paddr); + unsigned long end = start + size; + unsigned long line_size; + unsigned long flags; + + if (unlikely(start == end)) + return; + + line_size = ax45mp_priv.ax45mp_cache_line_size; + + start = start & (~(line_size - 1)); + end = ((end + line_size - 1) & (~(line_size - 1))); + + local_irq_save(flags); + + ax45mp_cpu_dcache_inval_range(start, end); + + local_irq_restore(flags); +} + +static void ax45mp_dma_cache_wback(phys_addr_t paddr, size_t size) +{ + unsigned long start = (unsigned long)phys_to_virt(paddr); + unsigned long end = start + size; + unsigned long line_size; + unsigned long flags; + + if (unlikely(start == end)) + return; + + line_size = ax45mp_priv.ax45mp_cache_line_size; + start = start & (~(line_size - 1)); + end = ((end + line_size - 1) & (~(line_size - 1))); + local_irq_save(flags); + ax45mp_cpu_dcache_wb_range(start, end); + local_irq_restore(flags); +} + +static void ax45mp_dma_cache_wback_inv(phys_addr_t paddr, size_t size) +{ + ax45mp_dma_cache_wback(paddr, size); + ax45mp_dma_cache_inv(paddr, size); +} + +static int ax45mp_get_l2_line_size(struct device_node *np) +{ + int ret; + + ret = of_property_read_u32(np, "cache-line-size", &ax45mp_priv.ax45mp_cache_line_size); + if (ret) { + pr_err("Failed to get cache-line-size, defaulting to 64 bytes\n"); + return ret; + } + + if (ax45mp_priv.ax45mp_cache_line_size != AX45MP_CACHE_LINE_SIZE) { + pr_err("Expected cache-line-size to be 64 bytes (found:%u)\n", + ax45mp_priv.ax45mp_cache_line_size); + return -EINVAL; + } + + return 0; +} + +static const struct riscv_nonstd_cache_ops ax45mp_cmo_ops __initdata = { + .wback = &ax45mp_dma_cache_wback, + .inv = &ax45mp_dma_cache_inv, + .wback_inv = &ax45mp_dma_cache_wback_inv, +}; + +static const struct of_device_id ax45mp_cache_ids[] = { + { .compatible = "andestech,ax45mp-cache" }, + { /* sentinel */ } +}; + +static int __init ax45mp_cache_init(void) +{ + struct resource res; + int ret; + + struct device_node *np __free(device_node) = + of_find_matching_node(NULL, ax45mp_cache_ids); + if (!of_device_is_available(np)) + return -ENODEV; + + ret = of_address_to_resource(np, 0, &res); + if (ret) + return ret; + + /* + * If IOCP is present on the Andes AX45MP core riscv_cbom_block_size + * will be 0 for sure, so we can definitely rely on it. If + * riscv_cbom_block_size = 0 we don't need to handle CMO using SW any + * more so we just return success here and only if its being set we + * continue further in the probe path. + */ + if (!riscv_cbom_block_size) + return 0; + + ax45mp_priv.l2c_base = ioremap(res.start, resource_size(&res)); + if (!ax45mp_priv.l2c_base) + return -ENOMEM; + + ret = ax45mp_get_l2_line_size(np); + if (ret) { + iounmap(ax45mp_priv.l2c_base); + return ret; + } + + riscv_noncoherent_register_cache_ops(&ax45mp_cmo_ops); + + return 0; +} +early_initcall(ax45mp_cache_init); diff --git a/drivers/cache/hisi_soc_hha.c b/drivers/cache/hisi_soc_hha.c new file mode 100644 index 000000000..756c43398 --- /dev/null +++ b/drivers/cache/hisi_soc_hha.c @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Driver for HiSilicon Hydra Home Agent (HHA). + * + * Copyright (c) 2025 HiSilicon Technologies Co., Ltd. + * Author: Yicong Yang <yangyicong@hisilicon.com> + * Yushan Wang <wangyushan12@huawei.com> + * + * A system typically contains multiple HHAs. Each is responsible for a subset + * of the physical addresses in the system, but interleave can make the mapping + * from a particular cache line to a responsible HHA complex. As such no + * filtering is done in the driver, with the hardware being responsible for + * responding with success for even if it was not responsible for any addresses + * in the range on which the operation was requested. + */ + +#include <linux/bitfield.h> +#include <linux/cache_coherency.h> +#include <linux/dev_printk.h> +#include <linux/init.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/memregion.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/platform_device.h> + +#define HISI_HHA_CTRL 0x5004 +#define HISI_HHA_CTRL_EN BIT(0) +#define HISI_HHA_CTRL_RANGE BIT(1) +#define HISI_HHA_CTRL_TYPE GENMASK(3, 2) +#define HISI_HHA_START_L 0x5008 +#define HISI_HHA_START_H 0x500c +#define HISI_HHA_LEN_L 0x5010 +#define HISI_HHA_LEN_H 0x5014 + +/* The maintain operation performs in a 128 Byte granularity */ +#define HISI_HHA_MAINT_ALIGN 128 + +#define HISI_HHA_POLL_GAP_US 10 +#define HISI_HHA_POLL_TIMEOUT_US 50000 + +struct hisi_soc_hha { + /* Must be first element */ + struct cache_coherency_ops_inst cci; + /* Locks HHA instance to forbid overlapping access. */ + struct mutex lock; + void __iomem *base; +}; + +static bool hisi_hha_cache_maintain_wait_finished(struct hisi_soc_hha *soc_hha) +{ + u32 val; + + return !readl_poll_timeout_atomic(soc_hha->base + HISI_HHA_CTRL, val, + !(val & HISI_HHA_CTRL_EN), + HISI_HHA_POLL_GAP_US, + HISI_HHA_POLL_TIMEOUT_US); +} + +static int hisi_soc_hha_wbinv(struct cache_coherency_ops_inst *cci, + struct cc_inval_params *invp) +{ + struct hisi_soc_hha *soc_hha = + container_of(cci, struct hisi_soc_hha, cci); + phys_addr_t top, addr = invp->addr; + size_t size = invp->size; + u32 reg; + + if (!size) + return -EINVAL; + + addr = ALIGN_DOWN(addr, HISI_HHA_MAINT_ALIGN); + top = ALIGN(addr + size, HISI_HHA_MAINT_ALIGN); + size = top - addr; + + guard(mutex)(&soc_hha->lock); + + if (!hisi_hha_cache_maintain_wait_finished(soc_hha)) + return -EBUSY; + + /* + * Hardware will search for addresses ranging [addr, addr + size - 1], + * last byte included, and perform maintenance in 128 byte granules + * on those cachelines which contain the addresses. If a given instance + * is either not responsible for a cacheline or that cacheline is not + * currently present then the search will fail, no operation will be + * necessary and the device will report success. + */ + size -= 1; + + writel(lower_32_bits(addr), soc_hha->base + HISI_HHA_START_L); + writel(upper_32_bits(addr), soc_hha->base + HISI_HHA_START_H); + writel(lower_32_bits(size), soc_hha->base + HISI_HHA_LEN_L); + writel(upper_32_bits(size), soc_hha->base + HISI_HHA_LEN_H); + + reg = FIELD_PREP(HISI_HHA_CTRL_TYPE, 1); /* Clean Invalid */ + reg |= HISI_HHA_CTRL_RANGE | HISI_HHA_CTRL_EN; + writel(reg, soc_hha->base + HISI_HHA_CTRL); + + return 0; +} + +static int hisi_soc_hha_done(struct cache_coherency_ops_inst *cci) +{ + struct hisi_soc_hha *soc_hha = + container_of(cci, struct hisi_soc_hha, cci); + + guard(mutex)(&soc_hha->lock); + if (!hisi_hha_cache_maintain_wait_finished(soc_hha)) + return -ETIMEDOUT; + + return 0; +} + +static const struct cache_coherency_ops hha_ops = { + .wbinv = hisi_soc_hha_wbinv, + .done = hisi_soc_hha_done, +}; + +static int hisi_soc_hha_probe(struct platform_device *pdev) +{ + struct hisi_soc_hha *soc_hha; + struct resource *mem; + int ret; + + soc_hha = cache_coherency_ops_instance_alloc(&hha_ops, + struct hisi_soc_hha, cci); + if (!soc_hha) + return -ENOMEM; + + platform_set_drvdata(pdev, soc_hha); + + mutex_init(&soc_hha->lock); + + mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!mem) { + ret = -ENOMEM; + goto err_free_cci; + } + + soc_hha->base = ioremap(mem->start, resource_size(mem)); + if (!soc_hha->base) { + ret = dev_err_probe(&pdev->dev, -ENOMEM, + "failed to remap io memory"); + goto err_free_cci; + } + + ret = cache_coherency_ops_instance_register(&soc_hha->cci); + if (ret) + goto err_iounmap; + + return 0; + +err_iounmap: + iounmap(soc_hha->base); +err_free_cci: + cache_coherency_ops_instance_put(&soc_hha->cci); + return ret; +} + +static void hisi_soc_hha_remove(struct platform_device *pdev) +{ + struct hisi_soc_hha *soc_hha = platform_get_drvdata(pdev); + + cache_coherency_ops_instance_unregister(&soc_hha->cci); + iounmap(soc_hha->base); + cache_coherency_ops_instance_put(&soc_hha->cci); +} + +static const struct acpi_device_id hisi_soc_hha_ids[] = { + { "HISI0511", }, + { } +}; +MODULE_DEVICE_TABLE(acpi, hisi_soc_hha_ids); + +static struct platform_driver hisi_soc_hha_driver = { + .driver = { + .name = "hisi_soc_hha", + .acpi_match_table = hisi_soc_hha_ids, + }, + .probe = hisi_soc_hha_probe, + .remove = hisi_soc_hha_remove, +}; + +module_platform_driver(hisi_soc_hha_driver); + +MODULE_IMPORT_NS("CACHE_COHERENCY"); +MODULE_DESCRIPTION("HiSilicon Hydra Home Agent driver supporting cache maintenance"); +MODULE_AUTHOR("Yicong Yang <yangyicong@hisilicon.com>"); +MODULE_AUTHOR("Yushan Wang <wangyushan12@huawei.com>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/cache/sifive_ccache.c b/drivers/cache/sifive_ccache.c new file mode 100644 index 000000000..2acb8bdf0 --- /dev/null +++ b/drivers/cache/sifive_ccache.c @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * SiFive composable cache controller Driver + * + * Copyright (C) 2018-2022 SiFive, Inc. + * + */ + +#define pr_fmt(fmt) "CCACHE: " fmt + +#include <linux/align.h> +#include <linux/debugfs.h> +#include <linux/interrupt.h> +#include <linux/of_irq.h> +#include <linux/of_address.h> +#include <linux/device.h> +#include <linux/bitfield.h> +#include <linux/platform_device.h> +#include <linux/property.h> +#include <asm/cacheflush.h> +#include <asm/cacheinfo.h> +#include <asm/dma-noncoherent.h> +#include <soc/sifive/sifive_ccache.h> + +#define SIFIVE_CCACHE_DIRECCFIX_LOW 0x100 +#define SIFIVE_CCACHE_DIRECCFIX_HIGH 0x104 +#define SIFIVE_CCACHE_DIRECCFIX_COUNT 0x108 + +#define SIFIVE_CCACHE_DIRECCFAIL_LOW 0x120 +#define SIFIVE_CCACHE_DIRECCFAIL_HIGH 0x124 +#define SIFIVE_CCACHE_DIRECCFAIL_COUNT 0x128 + +#define SIFIVE_CCACHE_DATECCFIX_LOW 0x140 +#define SIFIVE_CCACHE_DATECCFIX_HIGH 0x144 +#define SIFIVE_CCACHE_DATECCFIX_COUNT 0x148 + +#define SIFIVE_CCACHE_DATECCFAIL_LOW 0x160 +#define SIFIVE_CCACHE_DATECCFAIL_HIGH 0x164 +#define SIFIVE_CCACHE_DATECCFAIL_COUNT 0x168 + +#define SIFIVE_CCACHE_CONFIG 0x00 +#define SIFIVE_CCACHE_CONFIG_BANK_MASK GENMASK_ULL(7, 0) +#define SIFIVE_CCACHE_CONFIG_WAYS_MASK GENMASK_ULL(15, 8) +#define SIFIVE_CCACHE_CONFIG_SETS_MASK GENMASK_ULL(23, 16) +#define SIFIVE_CCACHE_CONFIG_BLKS_MASK GENMASK_ULL(31, 24) + +#define SIFIVE_CCACHE_FLUSH64 0x200 +#define SIFIVE_CCACHE_FLUSH32 0x240 + +#define SIFIVE_CCACHE_WAYENABLE 0x08 +#define SIFIVE_CCACHE_ECCINJECTERR 0x40 + +#define SIFIVE_CCACHE_MAX_ECCINTR 4 +#define SIFIVE_CCACHE_LINE_SIZE 64 + +static void __iomem *ccache_base; +static int g_irq[SIFIVE_CCACHE_MAX_ECCINTR]; +static struct riscv_cacheinfo_ops ccache_cache_ops; +static int level; + +enum { + DIR_CORR = 0, + DATA_CORR, + DATA_UNCORR, + DIR_UNCORR, +}; + +enum { + QUIRK_NONSTANDARD_CACHE_OPS = BIT(0), + QUIRK_BROKEN_DATA_UNCORR = BIT(1), +}; + +#ifdef CONFIG_DEBUG_FS +static struct dentry *sifive_test; + +static ssize_t ccache_write(struct file *file, const char __user *data, + size_t count, loff_t *ppos) +{ + unsigned int val; + + if (kstrtouint_from_user(data, count, 0, &val)) + return -EINVAL; + if ((val < 0xFF) || (val >= 0x10000 && val < 0x100FF)) + writel(val, ccache_base + SIFIVE_CCACHE_ECCINJECTERR); + else + return -EINVAL; + return count; +} + +static const struct file_operations ccache_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .write = ccache_write +}; + +static void setup_sifive_debug(void) +{ + sifive_test = debugfs_create_dir("sifive_ccache_cache", NULL); + + debugfs_create_file("sifive_debug_inject_error", 0200, + sifive_test, NULL, &ccache_fops); +} +#endif + +static void ccache_config_read(void) +{ + u32 cfg; + + cfg = readl(ccache_base + SIFIVE_CCACHE_CONFIG); + pr_info("%llu banks, %llu ways, sets/bank=%llu, bytes/block=%llu\n", + FIELD_GET(SIFIVE_CCACHE_CONFIG_BANK_MASK, cfg), + FIELD_GET(SIFIVE_CCACHE_CONFIG_WAYS_MASK, cfg), + BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_SETS_MASK, cfg)), + BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_BLKS_MASK, cfg))); + + cfg = readl(ccache_base + SIFIVE_CCACHE_WAYENABLE); + pr_info("Index of the largest way enabled: %u\n", cfg); +} + +static const struct of_device_id sifive_ccache_ids[] = { + { .compatible = "eswin,eic7700-l3-cache", + .data = (void *)(QUIRK_NONSTANDARD_CACHE_OPS) }, + { .compatible = "sifive,fu540-c000-ccache" }, + { .compatible = "sifive,fu740-c000-ccache" }, + { .compatible = "starfive,jh7100-ccache", + .data = (void *)(QUIRK_NONSTANDARD_CACHE_OPS | QUIRK_BROKEN_DATA_UNCORR) }, + { .compatible = "starfive,jh7110-ccache", + .data = (void *)(QUIRK_NONSTANDARD_CACHE_OPS) }, + { .compatible = "sifive,ccache0" }, + { /* end of table */ } +}; + +static ATOMIC_NOTIFIER_HEAD(ccache_err_chain); + +int register_sifive_ccache_error_notifier(struct notifier_block *nb) +{ + return atomic_notifier_chain_register(&ccache_err_chain, nb); +} +EXPORT_SYMBOL_GPL(register_sifive_ccache_error_notifier); + +int unregister_sifive_ccache_error_notifier(struct notifier_block *nb) +{ + return atomic_notifier_chain_unregister(&ccache_err_chain, nb); +} +EXPORT_SYMBOL_GPL(unregister_sifive_ccache_error_notifier); + +#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS +static void ccache_flush_range(phys_addr_t start, size_t len) +{ + phys_addr_t end = start + len; + phys_addr_t line; + + if (!len) + return; + + mb(); /* complete earlier memory accesses before the cache flush */ + for (line = ALIGN_DOWN(start, SIFIVE_CCACHE_LINE_SIZE); line < end; + line += SIFIVE_CCACHE_LINE_SIZE) { +#ifdef CONFIG_32BIT + writel_relaxed(line >> 4, ccache_base + SIFIVE_CCACHE_FLUSH32); +#else + writeq_relaxed(line, ccache_base + SIFIVE_CCACHE_FLUSH64); +#endif + } + mb(); /* issue later memory accesses after the cache flush */ +} + +static const struct riscv_nonstd_cache_ops ccache_mgmt_ops __initconst = { + .wback = &ccache_flush_range, + .inv = &ccache_flush_range, + .wback_inv = &ccache_flush_range, +}; +#endif /* CONFIG_RISCV_NONSTANDARD_CACHE_OPS */ + +static int ccache_largest_wayenabled(void) +{ + return readl(ccache_base + SIFIVE_CCACHE_WAYENABLE) & 0xFF; +} + +static ssize_t number_of_ways_enabled_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + return sprintf(buf, "%u\n", ccache_largest_wayenabled()); +} + +static DEVICE_ATTR_RO(number_of_ways_enabled); + +static struct attribute *priv_attrs[] = { + &dev_attr_number_of_ways_enabled.attr, + NULL, +}; + +static const struct attribute_group priv_attr_group = { + .attrs = priv_attrs, +}; + +static const struct attribute_group *ccache_get_priv_group(struct cacheinfo + *this_leaf) +{ + /* We want to use private group for composable cache only */ + if (this_leaf->level == level) + return &priv_attr_group; + else + return NULL; +} + +static irqreturn_t ccache_int_handler(int irq, void *device) +{ + unsigned int add_h, add_l; + + if (irq == g_irq[DIR_CORR]) { + add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_HIGH); + add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_LOW); + pr_err("DirError @ 0x%08X.%08X\n", add_h, add_l); + /* Reading this register clears the DirError interrupt sig */ + readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_COUNT); + atomic_notifier_call_chain(&ccache_err_chain, + SIFIVE_CCACHE_ERR_TYPE_CE, + "DirECCFix"); + } + if (irq == g_irq[DIR_UNCORR]) { + add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_HIGH); + add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_LOW); + /* Reading this register clears the DirFail interrupt sig */ + readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_COUNT); + atomic_notifier_call_chain(&ccache_err_chain, + SIFIVE_CCACHE_ERR_TYPE_UE, + "DirECCFail"); + panic("CCACHE: DirFail @ 0x%08X.%08X\n", add_h, add_l); + } + if (irq == g_irq[DATA_CORR]) { + add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_HIGH); + add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_LOW); + pr_err("DataError @ 0x%08X.%08X\n", add_h, add_l); + /* Reading this register clears the DataError interrupt sig */ + readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_COUNT); + atomic_notifier_call_chain(&ccache_err_chain, + SIFIVE_CCACHE_ERR_TYPE_CE, + "DatECCFix"); + } + if (irq == g_irq[DATA_UNCORR]) { + add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_HIGH); + add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_LOW); + pr_err("DataFail @ 0x%08X.%08X\n", add_h, add_l); + /* Reading this register clears the DataFail interrupt sig */ + readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_COUNT); + atomic_notifier_call_chain(&ccache_err_chain, + SIFIVE_CCACHE_ERR_TYPE_UE, + "DatECCFail"); + } + + return IRQ_HANDLED; +} + +static int sifive_ccache_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + unsigned long quirks; + int intr_num, rc; + + quirks = (unsigned long)device_get_match_data(dev); + + intr_num = platform_irq_count(pdev); + if (!intr_num) + return dev_err_probe(dev, -ENODEV, "No interrupts property\n"); + + for (int i = 0; i < intr_num; i++) { + if (i == DATA_UNCORR && (quirks & QUIRK_BROKEN_DATA_UNCORR)) + continue; + + g_irq[i] = platform_get_irq(pdev, i); + if (g_irq[i] < 0) + return g_irq[i]; + + rc = devm_request_irq(dev, g_irq[i], ccache_int_handler, 0, "ccache_ecc", NULL); + if (rc) + return dev_err_probe(dev, rc, "Could not request IRQ %d\n", g_irq[i]); + } + + return 0; +} + +static struct platform_driver sifive_ccache_driver = { + .probe = sifive_ccache_probe, + .driver = { + .name = "sifive_ccache", + .of_match_table = sifive_ccache_ids, + }, +}; + +static int __init sifive_ccache_init(void) +{ + struct device_node *np; + struct resource res; + const struct of_device_id *match; + unsigned long quirks __maybe_unused; + int rc; + + np = of_find_matching_node_and_match(NULL, sifive_ccache_ids, &match); + if (!np) + return -ENODEV; + + quirks = (uintptr_t)match->data; + + if (of_address_to_resource(np, 0, &res)) { + rc = -ENODEV; + goto err_node_put; + } + + ccache_base = ioremap(res.start, resource_size(&res)); + if (!ccache_base) { + rc = -ENOMEM; + goto err_node_put; + } + + if (of_property_read_u32(np, "cache-level", &level)) { + rc = -ENOENT; + goto err_unmap; + } + +#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS + if (quirks & QUIRK_NONSTANDARD_CACHE_OPS) { + riscv_cbom_block_size = SIFIVE_CCACHE_LINE_SIZE; + riscv_noncoherent_supported(); + riscv_noncoherent_register_cache_ops(&ccache_mgmt_ops); + } +#endif + + ccache_config_read(); + + ccache_cache_ops.get_priv_group = ccache_get_priv_group; + riscv_set_cacheinfo_ops(&ccache_cache_ops); + +#ifdef CONFIG_DEBUG_FS + setup_sifive_debug(); +#endif + + rc = platform_driver_register(&sifive_ccache_driver); + if (rc) + goto err_unmap; + + of_node_put(np); + + return 0; + +err_unmap: + iounmap(ccache_base); +err_node_put: + of_node_put(np); + return rc; +} + +arch_initcall(sifive_ccache_init); diff --git a/drivers/cache/starfive_starlink_cache.c b/drivers/cache/starfive_starlink_cache.c new file mode 100644 index 000000000..3a25d2d7c --- /dev/null +++ b/drivers/cache/starfive_starlink_cache.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Cache Management Operations for StarFive's Starlink cache controller + * + * Copyright (C) 2024 Shanghai StarFive Technology Co., Ltd. + * + * Author: Joshua Yeong <joshua.yeong@starfivetech.com> + */ + +#include <linux/bitfield.h> +#include <linux/cacheflush.h> +#include <linux/iopoll.h> +#include <linux/of_address.h> + +#include <asm/dma-noncoherent.h> + +#define STARLINK_CACHE_FLUSH_START_ADDR 0x0 +#define STARLINK_CACHE_FLUSH_END_ADDR 0x8 +#define STARLINK_CACHE_FLUSH_CTL 0x10 +#define STARLINK_CACHE_ALIGN 0x40 + +#define STARLINK_CACHE_ADDRESS_RANGE_MASK GENMASK(39, 0) +#define STARLINK_CACHE_FLUSH_CTL_MODE_MASK GENMASK(2, 1) +#define STARLINK_CACHE_FLUSH_CTL_ENABLE_MASK BIT(0) + +#define STARLINK_CACHE_FLUSH_CTL_CLEAN_INVALIDATE 0 +#define STARLINK_CACHE_FLUSH_CTL_MAKE_INVALIDATE 1 +#define STARLINK_CACHE_FLUSH_CTL_CLEAN_SHARED 2 +#define STARLINK_CACHE_FLUSH_POLL_DELAY_US 1 +#define STARLINK_CACHE_FLUSH_TIMEOUT_US 5000000 + +static void __iomem *starlink_cache_base; + +static void starlink_cache_flush_complete(void) +{ + volatile void __iomem *ctl = starlink_cache_base + STARLINK_CACHE_FLUSH_CTL; + u64 v; + int ret; + + ret = readq_poll_timeout_atomic(ctl, v, !(v & STARLINK_CACHE_FLUSH_CTL_ENABLE_MASK), + STARLINK_CACHE_FLUSH_POLL_DELAY_US, + STARLINK_CACHE_FLUSH_TIMEOUT_US); + if (ret) + WARN(1, "StarFive Starlink cache flush operation timeout\n"); +} + +static void starlink_cache_dma_cache_wback(phys_addr_t paddr, unsigned long size) +{ + writeq(FIELD_PREP(STARLINK_CACHE_ADDRESS_RANGE_MASK, paddr), + starlink_cache_base + STARLINK_CACHE_FLUSH_START_ADDR); + writeq(FIELD_PREP(STARLINK_CACHE_ADDRESS_RANGE_MASK, paddr + size), + starlink_cache_base + STARLINK_CACHE_FLUSH_END_ADDR); + + mb(); + writeq(FIELD_PREP(STARLINK_CACHE_FLUSH_CTL_MODE_MASK, + STARLINK_CACHE_FLUSH_CTL_CLEAN_SHARED), + starlink_cache_base + STARLINK_CACHE_FLUSH_CTL); + + starlink_cache_flush_complete(); +} + +static void starlink_cache_dma_cache_invalidate(phys_addr_t paddr, unsigned long size) +{ + writeq(FIELD_PREP(STARLINK_CACHE_ADDRESS_RANGE_MASK, paddr), + starlink_cache_base + STARLINK_CACHE_FLUSH_START_ADDR); + writeq(FIELD_PREP(STARLINK_CACHE_ADDRESS_RANGE_MASK, paddr + size), + starlink_cache_base + STARLINK_CACHE_FLUSH_END_ADDR); + + mb(); + writeq(FIELD_PREP(STARLINK_CACHE_FLUSH_CTL_MODE_MASK, + STARLINK_CACHE_FLUSH_CTL_MAKE_INVALIDATE), + starlink_cache_base + STARLINK_CACHE_FLUSH_CTL); + + starlink_cache_flush_complete(); +} + +static void starlink_cache_dma_cache_wback_inv(phys_addr_t paddr, unsigned long size) +{ + writeq(FIELD_PREP(STARLINK_CACHE_ADDRESS_RANGE_MASK, paddr), + starlink_cache_base + STARLINK_CACHE_FLUSH_START_ADDR); + writeq(FIELD_PREP(STARLINK_CACHE_ADDRESS_RANGE_MASK, paddr + size), + starlink_cache_base + STARLINK_CACHE_FLUSH_END_ADDR); + + mb(); + writeq(FIELD_PREP(STARLINK_CACHE_FLUSH_CTL_MODE_MASK, + STARLINK_CACHE_FLUSH_CTL_CLEAN_INVALIDATE), + starlink_cache_base + STARLINK_CACHE_FLUSH_CTL); + + starlink_cache_flush_complete(); +} + +static const struct riscv_nonstd_cache_ops starlink_cache_ops = { + .wback = &starlink_cache_dma_cache_wback, + .inv = &starlink_cache_dma_cache_invalidate, + .wback_inv = &starlink_cache_dma_cache_wback_inv, +}; + +static const struct of_device_id starlink_cache_ids[] = { + { .compatible = "starfive,jh8100-starlink-cache" }, + { /* sentinel */ } +}; + +static int __init starlink_cache_init(void) +{ + u32 block_size; + int ret; + + struct device_node *np __free(device_node) = + of_find_matching_node(NULL, starlink_cache_ids); + if (!of_device_is_available(np)) + return -ENODEV; + + ret = of_property_read_u32(np, "cache-block-size", &block_size); + if (ret) + return ret; + + if (block_size % STARLINK_CACHE_ALIGN) + return -EINVAL; + + starlink_cache_base = of_iomap(np, 0); + if (!starlink_cache_base) + return -ENOMEM; + + riscv_cbom_block_size = block_size; + riscv_noncoherent_supported(); + riscv_noncoherent_register_cache_ops(&starlink_cache_ops); + + return 0; +} +arch_initcall(starlink_cache_init); |