351 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * SiFive composable cache controller Driver4 *5 * Copyright (C) 2018-2022 SiFive, Inc.6 *7 */8 9#define pr_fmt(fmt) "CCACHE: " fmt10 11#include <linux/align.h>12#include <linux/debugfs.h>13#include <linux/interrupt.h>14#include <linux/of_irq.h>15#include <linux/of_address.h>16#include <linux/device.h>17#include <linux/bitfield.h>18#include <linux/platform_device.h>19#include <linux/property.h>20#include <asm/cacheflush.h>21#include <asm/cacheinfo.h>22#include <asm/dma-noncoherent.h>23#include <soc/sifive/sifive_ccache.h>24 25#define SIFIVE_CCACHE_DIRECCFIX_LOW 0x10026#define SIFIVE_CCACHE_DIRECCFIX_HIGH 0x10427#define SIFIVE_CCACHE_DIRECCFIX_COUNT 0x10828 29#define SIFIVE_CCACHE_DIRECCFAIL_LOW 0x12030#define SIFIVE_CCACHE_DIRECCFAIL_HIGH 0x12431#define SIFIVE_CCACHE_DIRECCFAIL_COUNT 0x12832 33#define SIFIVE_CCACHE_DATECCFIX_LOW 0x14034#define SIFIVE_CCACHE_DATECCFIX_HIGH 0x14435#define SIFIVE_CCACHE_DATECCFIX_COUNT 0x14836 37#define SIFIVE_CCACHE_DATECCFAIL_LOW 0x16038#define SIFIVE_CCACHE_DATECCFAIL_HIGH 0x16439#define SIFIVE_CCACHE_DATECCFAIL_COUNT 0x16840 41#define SIFIVE_CCACHE_CONFIG 0x0042#define SIFIVE_CCACHE_CONFIG_BANK_MASK GENMASK_ULL(7, 0)43#define SIFIVE_CCACHE_CONFIG_WAYS_MASK GENMASK_ULL(15, 8)44#define SIFIVE_CCACHE_CONFIG_SETS_MASK GENMASK_ULL(23, 16)45#define SIFIVE_CCACHE_CONFIG_BLKS_MASK GENMASK_ULL(31, 24)46 47#define SIFIVE_CCACHE_FLUSH64 0x20048#define SIFIVE_CCACHE_FLUSH32 0x24049 50#define SIFIVE_CCACHE_WAYENABLE 0x0851#define SIFIVE_CCACHE_ECCINJECTERR 0x4052 53#define SIFIVE_CCACHE_MAX_ECCINTR 454#define SIFIVE_CCACHE_LINE_SIZE 6455 56static void __iomem *ccache_base;57static int g_irq[SIFIVE_CCACHE_MAX_ECCINTR];58static struct riscv_cacheinfo_ops ccache_cache_ops;59static int level;60 61enum {62 DIR_CORR = 0,63 DATA_CORR,64 DATA_UNCORR,65 DIR_UNCORR,66};67 68enum {69 QUIRK_NONSTANDARD_CACHE_OPS = BIT(0),70 QUIRK_BROKEN_DATA_UNCORR = BIT(1),71};72 73#ifdef CONFIG_DEBUG_FS74static struct dentry *sifive_test;75 76static ssize_t ccache_write(struct file *file, const char __user *data,77 size_t count, loff_t *ppos)78{79 unsigned int val;80 81 if (kstrtouint_from_user(data, count, 0, &val))82 return -EINVAL;83 if ((val < 0xFF) || (val >= 0x10000 && val < 0x100FF))84 writel(val, ccache_base + SIFIVE_CCACHE_ECCINJECTERR);85 else86 return -EINVAL;87 return count;88}89 90static const struct file_operations ccache_fops = {91 .owner = THIS_MODULE,92 .open = simple_open,93 .write = ccache_write94};95 96static void setup_sifive_debug(void)97{98 sifive_test = debugfs_create_dir("sifive_ccache_cache", NULL);99 100 debugfs_create_file("sifive_debug_inject_error", 0200,101 sifive_test, NULL, &ccache_fops);102}103#endif104 105static void ccache_config_read(void)106{107 u32 cfg;108 109 cfg = readl(ccache_base + SIFIVE_CCACHE_CONFIG);110 pr_info("%llu banks, %llu ways, sets/bank=%llu, bytes/block=%llu\n",111 FIELD_GET(SIFIVE_CCACHE_CONFIG_BANK_MASK, cfg),112 FIELD_GET(SIFIVE_CCACHE_CONFIG_WAYS_MASK, cfg),113 BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_SETS_MASK, cfg)),114 BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_BLKS_MASK, cfg)));115 116 cfg = readl(ccache_base + SIFIVE_CCACHE_WAYENABLE);117 pr_info("Index of the largest way enabled: %u\n", cfg);118}119 120static const struct of_device_id sifive_ccache_ids[] = {121 { .compatible = "sifive,fu540-c000-ccache" },122 { .compatible = "sifive,fu740-c000-ccache" },123 { .compatible = "starfive,jh7100-ccache",124 .data = (void *)(QUIRK_NONSTANDARD_CACHE_OPS | QUIRK_BROKEN_DATA_UNCORR) },125 { .compatible = "sifive,ccache0" },126 { /* end of table */ }127};128 129static ATOMIC_NOTIFIER_HEAD(ccache_err_chain);130 131int register_sifive_ccache_error_notifier(struct notifier_block *nb)132{133 return atomic_notifier_chain_register(&ccache_err_chain, nb);134}135EXPORT_SYMBOL_GPL(register_sifive_ccache_error_notifier);136 137int unregister_sifive_ccache_error_notifier(struct notifier_block *nb)138{139 return atomic_notifier_chain_unregister(&ccache_err_chain, nb);140}141EXPORT_SYMBOL_GPL(unregister_sifive_ccache_error_notifier);142 143#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS144static void ccache_flush_range(phys_addr_t start, size_t len)145{146 phys_addr_t end = start + len;147 phys_addr_t line;148 149 if (!len)150 return;151 152 mb();153 for (line = ALIGN_DOWN(start, SIFIVE_CCACHE_LINE_SIZE); line < end;154 line += SIFIVE_CCACHE_LINE_SIZE) {155#ifdef CONFIG_32BIT156 writel(line >> 4, ccache_base + SIFIVE_CCACHE_FLUSH32);157#else158 writeq(line, ccache_base + SIFIVE_CCACHE_FLUSH64);159#endif160 mb();161 }162}163 164static const struct riscv_nonstd_cache_ops ccache_mgmt_ops __initconst = {165 .wback = &ccache_flush_range,166 .inv = &ccache_flush_range,167 .wback_inv = &ccache_flush_range,168};169#endif /* CONFIG_RISCV_NONSTANDARD_CACHE_OPS */170 171static int ccache_largest_wayenabled(void)172{173 return readl(ccache_base + SIFIVE_CCACHE_WAYENABLE) & 0xFF;174}175 176static ssize_t number_of_ways_enabled_show(struct device *dev,177 struct device_attribute *attr,178 char *buf)179{180 return sprintf(buf, "%u\n", ccache_largest_wayenabled());181}182 183static DEVICE_ATTR_RO(number_of_ways_enabled);184 185static struct attribute *priv_attrs[] = {186 &dev_attr_number_of_ways_enabled.attr,187 NULL,188};189 190static const struct attribute_group priv_attr_group = {191 .attrs = priv_attrs,192};193 194static const struct attribute_group *ccache_get_priv_group(struct cacheinfo195 *this_leaf)196{197 /* We want to use private group for composable cache only */198 if (this_leaf->level == level)199 return &priv_attr_group;200 else201 return NULL;202}203 204static irqreturn_t ccache_int_handler(int irq, void *device)205{206 unsigned int add_h, add_l;207 208 if (irq == g_irq[DIR_CORR]) {209 add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_HIGH);210 add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_LOW);211 pr_err("DirError @ 0x%08X.%08X\n", add_h, add_l);212 /* Reading this register clears the DirError interrupt sig */213 readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_COUNT);214 atomic_notifier_call_chain(&ccache_err_chain,215 SIFIVE_CCACHE_ERR_TYPE_CE,216 "DirECCFix");217 }218 if (irq == g_irq[DIR_UNCORR]) {219 add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_HIGH);220 add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_LOW);221 /* Reading this register clears the DirFail interrupt sig */222 readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_COUNT);223 atomic_notifier_call_chain(&ccache_err_chain,224 SIFIVE_CCACHE_ERR_TYPE_UE,225 "DirECCFail");226 panic("CCACHE: DirFail @ 0x%08X.%08X\n", add_h, add_l);227 }228 if (irq == g_irq[DATA_CORR]) {229 add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_HIGH);230 add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_LOW);231 pr_err("DataError @ 0x%08X.%08X\n", add_h, add_l);232 /* Reading this register clears the DataError interrupt sig */233 readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_COUNT);234 atomic_notifier_call_chain(&ccache_err_chain,235 SIFIVE_CCACHE_ERR_TYPE_CE,236 "DatECCFix");237 }238 if (irq == g_irq[DATA_UNCORR]) {239 add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_HIGH);240 add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_LOW);241 pr_err("DataFail @ 0x%08X.%08X\n", add_h, add_l);242 /* Reading this register clears the DataFail interrupt sig */243 readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_COUNT);244 atomic_notifier_call_chain(&ccache_err_chain,245 SIFIVE_CCACHE_ERR_TYPE_UE,246 "DatECCFail");247 }248 249 return IRQ_HANDLED;250}251 252static int sifive_ccache_probe(struct platform_device *pdev)253{254 struct device *dev = &pdev->dev;255 unsigned long quirks;256 int intr_num, rc;257 258 quirks = (unsigned long)device_get_match_data(dev);259 260 intr_num = platform_irq_count(pdev);261 if (!intr_num)262 return dev_err_probe(dev, -ENODEV, "No interrupts property\n");263 264 for (int i = 0; i < intr_num; i++) {265 if (i == DATA_UNCORR && (quirks & QUIRK_BROKEN_DATA_UNCORR))266 continue;267 268 g_irq[i] = platform_get_irq(pdev, i);269 if (g_irq[i] < 0)270 return g_irq[i];271 272 rc = devm_request_irq(dev, g_irq[i], ccache_int_handler, 0, "ccache_ecc", NULL);273 if (rc)274 return dev_err_probe(dev, rc, "Could not request IRQ %d\n", g_irq[i]);275 }276 277 return 0;278}279 280static struct platform_driver sifive_ccache_driver = {281 .probe = sifive_ccache_probe,282 .driver = {283 .name = "sifive_ccache",284 .of_match_table = sifive_ccache_ids,285 },286};287 288static int __init sifive_ccache_init(void)289{290 struct device_node *np;291 struct resource res;292 const struct of_device_id *match;293 unsigned long quirks __maybe_unused;294 int rc;295 296 np = of_find_matching_node_and_match(NULL, sifive_ccache_ids, &match);297 if (!np)298 return -ENODEV;299 300 quirks = (uintptr_t)match->data;301 302 if (of_address_to_resource(np, 0, &res)) {303 rc = -ENODEV;304 goto err_node_put;305 }306 307 ccache_base = ioremap(res.start, resource_size(&res));308 if (!ccache_base) {309 rc = -ENOMEM;310 goto err_node_put;311 }312 313 if (of_property_read_u32(np, "cache-level", &level)) {314 rc = -ENOENT;315 goto err_unmap;316 }317 318#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS319 if (quirks & QUIRK_NONSTANDARD_CACHE_OPS) {320 riscv_cbom_block_size = SIFIVE_CCACHE_LINE_SIZE;321 riscv_noncoherent_supported();322 riscv_noncoherent_register_cache_ops(&ccache_mgmt_ops);323 }324#endif325 326 ccache_config_read();327 328 ccache_cache_ops.get_priv_group = ccache_get_priv_group;329 riscv_set_cacheinfo_ops(&ccache_cache_ops);330 331#ifdef CONFIG_DEBUG_FS332 setup_sifive_debug();333#endif334 335 rc = platform_driver_register(&sifive_ccache_driver);336 if (rc)337 goto err_unmap;338 339 of_node_put(np);340 341 return 0;342 343err_unmap:344 iounmap(ccache_base);345err_node_put:346 of_node_put(np);347 return rc;348}349 350arch_initcall(sifive_ccache_init);351