453 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * The On Chip Memory (OCMEM) allocator allows various clients to allocate4 * memory from OCMEM based on performance, latency and power requirements.5 * This is typically used by the GPU, camera/video, and audio components on6 * some Snapdragon SoCs.7 *8 * Copyright (C) 2019 Brian Masney <masneyb@onstation.org>9 * Copyright (C) 2015 Red Hat. Author: Rob Clark <robdclark@gmail.com>10 */11 12#include <linux/bitfield.h>13#include <linux/cleanup.h>14#include <linux/clk.h>15#include <linux/io.h>16#include <linux/kernel.h>17#include <linux/module.h>18#include <linux/of.h>19#include <linux/of_platform.h>20#include <linux/platform_device.h>21#include <linux/firmware/qcom/qcom_scm.h>22#include <linux/sizes.h>23#include <linux/slab.h>24#include <linux/types.h>25#include <soc/qcom/ocmem.h>26 27enum region_mode {28 WIDE_MODE = 0x0,29 THIN_MODE,30 MODE_DEFAULT = WIDE_MODE,31};32 33enum ocmem_macro_state {34 PASSTHROUGH = 0,35 PERI_ON = 1,36 CORE_ON = 2,37 CLK_OFF = 4,38};39 40struct ocmem_region {41 bool interleaved;42 enum region_mode mode;43 unsigned int num_macros;44 enum ocmem_macro_state macro_state[4];45 unsigned long macro_size;46 unsigned long region_size;47};48 49struct ocmem_config {50 uint8_t num_regions;51 unsigned long macro_size;52};53 54struct ocmem {55 struct device *dev;56 const struct ocmem_config *config;57 struct resource *memory;58 void __iomem *mmio;59 struct clk *core_clk;60 struct clk *iface_clk;61 unsigned int num_ports;62 unsigned int num_macros;63 bool interleaved;64 struct ocmem_region *regions;65 unsigned long active_allocations;66};67 68#define OCMEM_MIN_ALIGN SZ_64K69#define OCMEM_MIN_ALLOC SZ_64K70 71#define OCMEM_REG_HW_VERSION 0x0000000072#define OCMEM_REG_HW_PROFILE 0x0000000473 74#define OCMEM_REG_REGION_MODE_CTL 0x0000100075#define OCMEM_REGION_MODE_CTL_REG0_THIN 0x0000000176#define OCMEM_REGION_MODE_CTL_REG1_THIN 0x0000000277#define OCMEM_REGION_MODE_CTL_REG2_THIN 0x0000000478#define OCMEM_REGION_MODE_CTL_REG3_THIN 0x0000000879 80#define OCMEM_REG_GFX_MPU_START 0x0000100481#define OCMEM_REG_GFX_MPU_END 0x0000100882 83#define OCMEM_HW_VERSION_MAJOR(val) FIELD_GET(GENMASK(31, 28), val)84#define OCMEM_HW_VERSION_MINOR(val) FIELD_GET(GENMASK(27, 16), val)85#define OCMEM_HW_VERSION_STEP(val) FIELD_GET(GENMASK(15, 0), val)86 87#define OCMEM_HW_PROFILE_NUM_PORTS(val) FIELD_GET(0x0000000f, (val))88#define OCMEM_HW_PROFILE_NUM_MACROS(val) FIELD_GET(0x00003f00, (val))89 90#define OCMEM_HW_PROFILE_LAST_REGN_HALFSIZE 0x0001000091#define OCMEM_HW_PROFILE_INTERLEAVING 0x0002000092#define OCMEM_REG_GEN_STATUS 0x0000000c93 94#define OCMEM_REG_PSGSC_STATUS 0x0000003895#define OCMEM_REG_PSGSC_CTL(i0) (0x0000003c + 0x1*(i0))96 97#define OCMEM_PSGSC_CTL_MACRO0_MODE(val) FIELD_PREP(0x00000007, (val))98#define OCMEM_PSGSC_CTL_MACRO1_MODE(val) FIELD_PREP(0x00000070, (val))99#define OCMEM_PSGSC_CTL_MACRO2_MODE(val) FIELD_PREP(0x00000700, (val))100#define OCMEM_PSGSC_CTL_MACRO3_MODE(val) FIELD_PREP(0x00007000, (val))101 102static inline void ocmem_write(struct ocmem *ocmem, u32 reg, u32 data)103{104 writel(data, ocmem->mmio + reg);105}106 107static inline u32 ocmem_read(struct ocmem *ocmem, u32 reg)108{109 return readl(ocmem->mmio + reg);110}111 112static void update_ocmem(struct ocmem *ocmem)113{114 uint32_t region_mode_ctrl = 0x0;115 int i;116 117 if (!qcom_scm_ocmem_lock_available()) {118 for (i = 0; i < ocmem->config->num_regions; i++) {119 struct ocmem_region *region = &ocmem->regions[i];120 121 if (region->mode == THIN_MODE)122 region_mode_ctrl |= BIT(i);123 }124 125 dev_dbg(ocmem->dev, "ocmem_region_mode_control %x\n",126 region_mode_ctrl);127 ocmem_write(ocmem, OCMEM_REG_REGION_MODE_CTL, region_mode_ctrl);128 }129 130 for (i = 0; i < ocmem->config->num_regions; i++) {131 struct ocmem_region *region = &ocmem->regions[i];132 u32 data;133 134 data = OCMEM_PSGSC_CTL_MACRO0_MODE(region->macro_state[0]) |135 OCMEM_PSGSC_CTL_MACRO1_MODE(region->macro_state[1]) |136 OCMEM_PSGSC_CTL_MACRO2_MODE(region->macro_state[2]) |137 OCMEM_PSGSC_CTL_MACRO3_MODE(region->macro_state[3]);138 139 ocmem_write(ocmem, OCMEM_REG_PSGSC_CTL(i), data);140 }141}142 143static unsigned long phys_to_offset(struct ocmem *ocmem,144 unsigned long addr)145{146 if (addr < ocmem->memory->start || addr >= ocmem->memory->end)147 return 0;148 149 return addr - ocmem->memory->start;150}151 152static unsigned long device_address(struct ocmem *ocmem,153 enum ocmem_client client,154 unsigned long addr)155{156 WARN_ON(client != OCMEM_GRAPHICS);157 158 /* TODO: gpu uses phys_to_offset, but others do not.. */159 return phys_to_offset(ocmem, addr);160}161 162static void update_range(struct ocmem *ocmem, struct ocmem_buf *buf,163 enum ocmem_macro_state mstate, enum region_mode rmode)164{165 unsigned long offset = 0;166 int i, j;167 168 for (i = 0; i < ocmem->config->num_regions; i++) {169 struct ocmem_region *region = &ocmem->regions[i];170 171 if (buf->offset <= offset && offset < buf->offset + buf->len)172 region->mode = rmode;173 174 for (j = 0; j < region->num_macros; j++) {175 if (buf->offset <= offset &&176 offset < buf->offset + buf->len)177 region->macro_state[j] = mstate;178 179 offset += region->macro_size;180 }181 }182 183 update_ocmem(ocmem);184}185 186struct ocmem *of_get_ocmem(struct device *dev)187{188 struct platform_device *pdev;189 struct ocmem *ocmem;190 191 struct device_node *devnode __free(device_node) = of_parse_phandle(dev->of_node,192 "sram", 0);193 if (!devnode || !devnode->parent) {194 dev_err(dev, "Cannot look up sram phandle\n");195 return ERR_PTR(-ENODEV);196 }197 198 pdev = of_find_device_by_node(devnode->parent);199 if (!pdev) {200 dev_err(dev, "Cannot find device node %s\n", devnode->name);201 return ERR_PTR(-EPROBE_DEFER);202 }203 204 ocmem = platform_get_drvdata(pdev);205 if (!ocmem) {206 dev_err(dev, "Cannot get ocmem\n");207 put_device(&pdev->dev);208 return ERR_PTR(-ENODEV);209 }210 return ocmem;211}212EXPORT_SYMBOL_GPL(of_get_ocmem);213 214struct ocmem_buf *ocmem_allocate(struct ocmem *ocmem, enum ocmem_client client,215 unsigned long size)216{217 int ret;218 219 /* TODO: add support for other clients... */220 if (WARN_ON(client != OCMEM_GRAPHICS))221 return ERR_PTR(-ENODEV);222 223 if (size < OCMEM_MIN_ALLOC || !IS_ALIGNED(size, OCMEM_MIN_ALIGN))224 return ERR_PTR(-EINVAL);225 226 if (test_and_set_bit_lock(BIT(client), &ocmem->active_allocations))227 return ERR_PTR(-EBUSY);228 229 struct ocmem_buf *buf __free(kfree) = kzalloc(sizeof(*buf), GFP_KERNEL);230 if (!buf) {231 ret = -ENOMEM;232 goto err_unlock;233 }234 235 buf->offset = 0;236 buf->addr = device_address(ocmem, client, buf->offset);237 buf->len = size;238 239 update_range(ocmem, buf, CORE_ON, WIDE_MODE);240 241 if (qcom_scm_ocmem_lock_available()) {242 ret = qcom_scm_ocmem_lock(QCOM_SCM_OCMEM_GRAPHICS_ID,243 buf->offset, buf->len, WIDE_MODE);244 if (ret) {245 dev_err(ocmem->dev, "could not lock: %d\n", ret);246 ret = -EINVAL;247 goto err_unlock;248 }249 } else {250 ocmem_write(ocmem, OCMEM_REG_GFX_MPU_START, buf->offset);251 ocmem_write(ocmem, OCMEM_REG_GFX_MPU_END,252 buf->offset + buf->len);253 }254 255 dev_dbg(ocmem->dev, "using %ldK of OCMEM at 0x%08lx for client %d\n",256 size / 1024, buf->addr, client);257 258 return_ptr(buf);259 260err_unlock:261 clear_bit_unlock(BIT(client), &ocmem->active_allocations);262 263 return ERR_PTR(ret);264}265EXPORT_SYMBOL_GPL(ocmem_allocate);266 267void ocmem_free(struct ocmem *ocmem, enum ocmem_client client,268 struct ocmem_buf *buf)269{270 /* TODO: add support for other clients... */271 if (WARN_ON(client != OCMEM_GRAPHICS))272 return;273 274 update_range(ocmem, buf, CLK_OFF, MODE_DEFAULT);275 276 if (qcom_scm_ocmem_lock_available()) {277 int ret;278 279 ret = qcom_scm_ocmem_unlock(QCOM_SCM_OCMEM_GRAPHICS_ID,280 buf->offset, buf->len);281 if (ret)282 dev_err(ocmem->dev, "could not unlock: %d\n", ret);283 } else {284 ocmem_write(ocmem, OCMEM_REG_GFX_MPU_START, 0x0);285 ocmem_write(ocmem, OCMEM_REG_GFX_MPU_END, 0x0);286 }287 288 kfree(buf);289 290 clear_bit_unlock(BIT(client), &ocmem->active_allocations);291}292EXPORT_SYMBOL_GPL(ocmem_free);293 294static int ocmem_dev_probe(struct platform_device *pdev)295{296 struct device *dev = &pdev->dev;297 unsigned long reg, region_size;298 int i, j, ret, num_banks;299 struct ocmem *ocmem;300 301 if (!qcom_scm_is_available())302 return -EPROBE_DEFER;303 304 ocmem = devm_kzalloc(dev, sizeof(*ocmem), GFP_KERNEL);305 if (!ocmem)306 return -ENOMEM;307 308 ocmem->dev = dev;309 ocmem->config = device_get_match_data(dev);310 311 ocmem->core_clk = devm_clk_get(dev, "core");312 if (IS_ERR(ocmem->core_clk))313 return dev_err_probe(dev, PTR_ERR(ocmem->core_clk),314 "Unable to get core clock\n");315 316 ocmem->iface_clk = devm_clk_get_optional(dev, "iface");317 if (IS_ERR(ocmem->iface_clk))318 return dev_err_probe(dev, PTR_ERR(ocmem->iface_clk),319 "Unable to get iface clock\n");320 321 ocmem->mmio = devm_platform_ioremap_resource_byname(pdev, "ctrl");322 if (IS_ERR(ocmem->mmio))323 return dev_err_probe(&pdev->dev, PTR_ERR(ocmem->mmio),324 "Failed to ioremap ocmem_ctrl resource\n");325 326 ocmem->memory = platform_get_resource_byname(pdev, IORESOURCE_MEM,327 "mem");328 if (!ocmem->memory) {329 dev_err(dev, "Could not get mem region\n");330 return -ENXIO;331 }332 333 /* The core clock is synchronous with graphics */334 WARN_ON(clk_set_rate(ocmem->core_clk, 1000) < 0);335 336 ret = clk_prepare_enable(ocmem->core_clk);337 if (ret)338 return dev_err_probe(ocmem->dev, ret, "Failed to enable core clock\n");339 340 ret = clk_prepare_enable(ocmem->iface_clk);341 if (ret) {342 clk_disable_unprepare(ocmem->core_clk);343 return dev_err_probe(ocmem->dev, ret, "Failed to enable iface clock\n");344 }345 346 if (qcom_scm_restore_sec_cfg_available()) {347 dev_dbg(dev, "configuring scm\n");348 ret = qcom_scm_restore_sec_cfg(QCOM_SCM_OCMEM_DEV_ID, 0);349 if (ret) {350 dev_err_probe(dev, ret, "Could not enable secure configuration\n");351 goto err_clk_disable;352 }353 }354 355 reg = ocmem_read(ocmem, OCMEM_REG_HW_VERSION);356 dev_dbg(dev, "OCMEM hardware version: %lu.%lu.%lu\n",357 OCMEM_HW_VERSION_MAJOR(reg),358 OCMEM_HW_VERSION_MINOR(reg),359 OCMEM_HW_VERSION_STEP(reg));360 361 reg = ocmem_read(ocmem, OCMEM_REG_HW_PROFILE);362 ocmem->num_ports = OCMEM_HW_PROFILE_NUM_PORTS(reg);363 ocmem->num_macros = OCMEM_HW_PROFILE_NUM_MACROS(reg);364 ocmem->interleaved = !!(reg & OCMEM_HW_PROFILE_INTERLEAVING);365 366 num_banks = ocmem->num_ports / 2;367 region_size = ocmem->config->macro_size * num_banks;368 369 dev_info(dev, "%u ports, %u regions, %u macros, %sinterleaved\n",370 ocmem->num_ports, ocmem->config->num_regions,371 ocmem->num_macros, ocmem->interleaved ? "" : "not ");372 373 ocmem->regions = devm_kcalloc(dev, ocmem->config->num_regions,374 sizeof(struct ocmem_region), GFP_KERNEL);375 if (!ocmem->regions) {376 ret = -ENOMEM;377 goto err_clk_disable;378 }379 380 for (i = 0; i < ocmem->config->num_regions; i++) {381 struct ocmem_region *region = &ocmem->regions[i];382 383 if (WARN_ON(num_banks > ARRAY_SIZE(region->macro_state))) {384 ret = -EINVAL;385 goto err_clk_disable;386 }387 388 region->mode = MODE_DEFAULT;389 region->num_macros = num_banks;390 391 if (i == (ocmem->config->num_regions - 1) &&392 reg & OCMEM_HW_PROFILE_LAST_REGN_HALFSIZE) {393 region->macro_size = ocmem->config->macro_size / 2;394 region->region_size = region_size / 2;395 } else {396 region->macro_size = ocmem->config->macro_size;397 region->region_size = region_size;398 }399 400 for (j = 0; j < ARRAY_SIZE(region->macro_state); j++)401 region->macro_state[j] = CLK_OFF;402 }403 404 platform_set_drvdata(pdev, ocmem);405 406 return 0;407 408err_clk_disable:409 clk_disable_unprepare(ocmem->core_clk);410 clk_disable_unprepare(ocmem->iface_clk);411 return ret;412}413 414static void ocmem_dev_remove(struct platform_device *pdev)415{416 struct ocmem *ocmem = platform_get_drvdata(pdev);417 418 clk_disable_unprepare(ocmem->core_clk);419 clk_disable_unprepare(ocmem->iface_clk);420}421 422static const struct ocmem_config ocmem_8226_config = {423 .num_regions = 1,424 .macro_size = SZ_128K,425};426 427static const struct ocmem_config ocmem_8974_config = {428 .num_regions = 3,429 .macro_size = SZ_128K,430};431 432static const struct of_device_id ocmem_of_match[] = {433 { .compatible = "qcom,msm8226-ocmem", .data = &ocmem_8226_config },434 { .compatible = "qcom,msm8974-ocmem", .data = &ocmem_8974_config },435 { }436};437 438MODULE_DEVICE_TABLE(of, ocmem_of_match);439 440static struct platform_driver ocmem_driver = {441 .probe = ocmem_dev_probe,442 .remove_new = ocmem_dev_remove,443 .driver = {444 .name = "ocmem",445 .of_match_table = ocmem_of_match,446 },447};448 449module_platform_driver(ocmem_driver);450 451MODULE_DESCRIPTION("On Chip Memory (OCMEM) allocator for some Snapdragon SoCs");452MODULE_LICENSE("GPL v2");453