1273 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright 2021 NXP */3 4#include <dt-bindings/firmware/imx/rsrc.h>5#include <linux/arm-smccc.h>6#include <linux/clk.h>7#include <linux/err.h>8#include <linux/firmware.h>9#include <linux/firmware/imx/sci.h>10#include <linux/interrupt.h>11#include <linux/kernel.h>12#include <linux/mailbox_client.h>13#include <linux/mfd/syscon.h>14#include <linux/module.h>15#include <linux/of.h>16#include <linux/of_reserved_mem.h>17#include <linux/platform_device.h>18#include <linux/pm_domain.h>19#include <linux/pm_runtime.h>20#include <linux/regmap.h>21#include <linux/remoteproc.h>22#include <linux/slab.h>23 24#include "imx_rproc.h"25#include "remoteproc_elf_helpers.h"26#include "remoteproc_internal.h"27 28#define DSP_RPROC_CLK_MAX 529 30/*31 * Module parameters32 */33static unsigned int no_mailboxes;34module_param_named(no_mailboxes, no_mailboxes, int, 0644);35MODULE_PARM_DESC(no_mailboxes,36 "There is no mailbox between cores, so ignore remote proc reply after start, default is 0 (off).");37 38#define REMOTE_IS_READY BIT(0)39#define REMOTE_READY_WAIT_MAX_RETRIES 50040 41/* att flags */42/* DSP own area */43#define ATT_OWN BIT(31)44/* DSP instruction area */45#define ATT_IRAM BIT(30)46 47/* Definitions for i.MX8MP */48/* DAP registers */49#define IMX8M_DAP_DEBUG 0x2880000050#define IMX8M_DAP_DEBUG_SIZE (64 * 1024)51#define IMX8M_DAP_PWRCTL (0x4000 + 0x3020)52#define IMX8M_PWRCTL_CORERESET BIT(16)53 54/* DSP audio mix registers */55#define IMX8M_AudioDSP_REG0 0x10056#define IMX8M_AudioDSP_REG1 0x10457#define IMX8M_AudioDSP_REG2 0x10858#define IMX8M_AudioDSP_REG3 0x10c59 60#define IMX8M_AudioDSP_REG2_RUNSTALL BIT(5)61#define IMX8M_AudioDSP_REG2_PWAITMODE BIT(1)62 63/* Definitions for i.MX8ULP */64#define IMX8ULP_SIM_LPAV_REG_SYSCTRL0 0x865#define IMX8ULP_SYSCTRL0_DSP_DBG_RST BIT(25)66#define IMX8ULP_SYSCTRL0_DSP_PLAT_CLK_EN BIT(19)67#define IMX8ULP_SYSCTRL0_DSP_PBCLK_EN BIT(18)68#define IMX8ULP_SYSCTRL0_DSP_CLK_EN BIT(17)69#define IMX8ULP_SYSCTRL0_DSP_RST BIT(16)70#define IMX8ULP_SYSCTRL0_DSP_OCD_HALT BIT(14)71#define IMX8ULP_SYSCTRL0_DSP_STALL BIT(13)72 73#define IMX8ULP_SIP_HIFI_XRDC 0xc200000e74 75/*76 * enum - Predefined Mailbox Messages77 *78 * @RP_MBOX_SUSPEND_SYSTEM: system suspend request for the remote processor79 *80 * @RP_MBOX_SUSPEND_ACK: successful response from remote processor for a81 * suspend request82 *83 * @RP_MBOX_RESUME_SYSTEM: system resume request for the remote processor84 *85 * @RP_MBOX_RESUME_ACK: successful response from remote processor for a86 * resume request87 */88enum imx_dsp_rp_mbox_messages {89 RP_MBOX_SUSPEND_SYSTEM = 0xFF11,90 RP_MBOX_SUSPEND_ACK = 0xFF12,91 RP_MBOX_RESUME_SYSTEM = 0xFF13,92 RP_MBOX_RESUME_ACK = 0xFF14,93};94 95/**96 * struct imx_dsp_rproc - DSP remote processor state97 * @regmap: regmap handler98 * @rproc: rproc handler99 * @dsp_dcfg: device configuration pointer100 * @clks: clocks needed by this device101 * @cl: mailbox client to request the mailbox channel102 * @cl_rxdb: mailbox client to request the mailbox channel for doorbell103 * @tx_ch: mailbox tx channel handle104 * @rx_ch: mailbox rx channel handle105 * @rxdb_ch: mailbox rx doorbell channel handle106 * @pd_list: power domain list107 * @ipc_handle: System Control Unit ipc handle108 * @rproc_work: work for processing virtio interrupts109 * @pm_comp: completion primitive to sync for suspend response110 * @flags: control flags111 */112struct imx_dsp_rproc {113 struct regmap *regmap;114 struct rproc *rproc;115 const struct imx_dsp_rproc_dcfg *dsp_dcfg;116 struct clk_bulk_data clks[DSP_RPROC_CLK_MAX];117 struct mbox_client cl;118 struct mbox_client cl_rxdb;119 struct mbox_chan *tx_ch;120 struct mbox_chan *rx_ch;121 struct mbox_chan *rxdb_ch;122 struct dev_pm_domain_list *pd_list;123 struct imx_sc_ipc *ipc_handle;124 struct work_struct rproc_work;125 struct completion pm_comp;126 u32 flags;127};128 129/**130 * struct imx_dsp_rproc_dcfg - DSP remote processor configuration131 * @dcfg: imx_rproc_dcfg handler132 * @reset: reset callback function133 */134struct imx_dsp_rproc_dcfg {135 const struct imx_rproc_dcfg *dcfg;136 int (*reset)(struct imx_dsp_rproc *priv);137};138 139static const struct imx_rproc_att imx_dsp_rproc_att_imx8qm[] = {140 /* dev addr , sys addr , size , flags */141 { 0x596e8000, 0x556e8000, 0x00008000, ATT_OWN },142 { 0x596f0000, 0x556f0000, 0x00008000, ATT_OWN },143 { 0x596f8000, 0x556f8000, 0x00000800, ATT_OWN | ATT_IRAM},144 { 0x55700000, 0x55700000, 0x00070000, ATT_OWN },145 /* DDR (Data) */146 { 0x80000000, 0x80000000, 0x60000000, 0},147};148 149static const struct imx_rproc_att imx_dsp_rproc_att_imx8qxp[] = {150 /* dev addr , sys addr , size , flags */151 { 0x596e8000, 0x596e8000, 0x00008000, ATT_OWN },152 { 0x596f0000, 0x596f0000, 0x00008000, ATT_OWN },153 { 0x596f8000, 0x596f8000, 0x00000800, ATT_OWN | ATT_IRAM},154 { 0x59700000, 0x59700000, 0x00070000, ATT_OWN },155 /* DDR (Data) */156 { 0x80000000, 0x80000000, 0x60000000, 0},157};158 159static const struct imx_rproc_att imx_dsp_rproc_att_imx8mp[] = {160 /* dev addr , sys addr , size , flags */161 { 0x3b6e8000, 0x3b6e8000, 0x00008000, ATT_OWN },162 { 0x3b6f0000, 0x3b6f0000, 0x00008000, ATT_OWN },163 { 0x3b6f8000, 0x3b6f8000, 0x00000800, ATT_OWN | ATT_IRAM},164 { 0x3b700000, 0x3b700000, 0x00040000, ATT_OWN },165 /* DDR (Data) */166 { 0x40000000, 0x40000000, 0x80000000, 0},167};168 169static const struct imx_rproc_att imx_dsp_rproc_att_imx8ulp[] = {170 /* dev addr , sys addr , size , flags */171 { 0x21170000, 0x21170000, 0x00010000, ATT_OWN | ATT_IRAM},172 { 0x21180000, 0x21180000, 0x00010000, ATT_OWN },173 /* DDR (Data) */174 { 0x0c000000, 0x80000000, 0x10000000, 0},175 { 0x30000000, 0x90000000, 0x10000000, 0},176};177 178/* Initialize the mailboxes between cores, if exists */179static int (*imx_dsp_rproc_mbox_init)(struct imx_dsp_rproc *priv);180 181/* Reset function for DSP on i.MX8MP */182static int imx8mp_dsp_reset(struct imx_dsp_rproc *priv)183{184 void __iomem *dap = ioremap_wc(IMX8M_DAP_DEBUG, IMX8M_DAP_DEBUG_SIZE);185 int pwrctl;186 187 /* Put DSP into reset and stall */188 pwrctl = readl(dap + IMX8M_DAP_PWRCTL);189 pwrctl |= IMX8M_PWRCTL_CORERESET;190 writel(pwrctl, dap + IMX8M_DAP_PWRCTL);191 192 /* Keep reset asserted for 10 cycles */193 usleep_range(1, 2);194 195 regmap_update_bits(priv->regmap, IMX8M_AudioDSP_REG2,196 IMX8M_AudioDSP_REG2_RUNSTALL,197 IMX8M_AudioDSP_REG2_RUNSTALL);198 199 /* Take the DSP out of reset and keep stalled for FW loading */200 pwrctl = readl(dap + IMX8M_DAP_PWRCTL);201 pwrctl &= ~IMX8M_PWRCTL_CORERESET;202 writel(pwrctl, dap + IMX8M_DAP_PWRCTL);203 204 iounmap(dap);205 return 0;206}207 208/* Reset function for DSP on i.MX8ULP */209static int imx8ulp_dsp_reset(struct imx_dsp_rproc *priv)210{211 struct arm_smccc_res res;212 213 /* Put DSP into reset and stall */214 regmap_update_bits(priv->regmap, IMX8ULP_SIM_LPAV_REG_SYSCTRL0,215 IMX8ULP_SYSCTRL0_DSP_RST, IMX8ULP_SYSCTRL0_DSP_RST);216 regmap_update_bits(priv->regmap, IMX8ULP_SIM_LPAV_REG_SYSCTRL0,217 IMX8ULP_SYSCTRL0_DSP_STALL,218 IMX8ULP_SYSCTRL0_DSP_STALL);219 220 /* Configure resources of DSP through TFA */221 arm_smccc_smc(IMX8ULP_SIP_HIFI_XRDC, 0, 0, 0, 0, 0, 0, 0, &res);222 223 /* Take the DSP out of reset and keep stalled for FW loading */224 regmap_update_bits(priv->regmap, IMX8ULP_SIM_LPAV_REG_SYSCTRL0,225 IMX8ULP_SYSCTRL0_DSP_RST, 0);226 regmap_update_bits(priv->regmap, IMX8ULP_SIM_LPAV_REG_SYSCTRL0,227 IMX8ULP_SYSCTRL0_DSP_DBG_RST, 0);228 229 return 0;230}231 232/* Specific configuration for i.MX8MP */233static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8mp = {234 .src_reg = IMX8M_AudioDSP_REG2,235 .src_mask = IMX8M_AudioDSP_REG2_RUNSTALL,236 .src_start = 0,237 .src_stop = IMX8M_AudioDSP_REG2_RUNSTALL,238 .att = imx_dsp_rproc_att_imx8mp,239 .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8mp),240 .method = IMX_RPROC_MMIO,241};242 243static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8mp = {244 .dcfg = &dsp_rproc_cfg_imx8mp,245 .reset = imx8mp_dsp_reset,246};247 248/* Specific configuration for i.MX8ULP */249static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8ulp = {250 .src_reg = IMX8ULP_SIM_LPAV_REG_SYSCTRL0,251 .src_mask = IMX8ULP_SYSCTRL0_DSP_STALL,252 .src_start = 0,253 .src_stop = IMX8ULP_SYSCTRL0_DSP_STALL,254 .att = imx_dsp_rproc_att_imx8ulp,255 .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8ulp),256 .method = IMX_RPROC_MMIO,257};258 259static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8ulp = {260 .dcfg = &dsp_rproc_cfg_imx8ulp,261 .reset = imx8ulp_dsp_reset,262};263 264/* Specific configuration for i.MX8QXP */265static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8qxp = {266 .att = imx_dsp_rproc_att_imx8qxp,267 .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8qxp),268 .method = IMX_RPROC_SCU_API,269};270 271static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8qxp = {272 .dcfg = &dsp_rproc_cfg_imx8qxp,273};274 275/* Specific configuration for i.MX8QM */276static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8qm = {277 .att = imx_dsp_rproc_att_imx8qm,278 .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8qm),279 .method = IMX_RPROC_SCU_API,280};281 282static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8qm = {283 .dcfg = &dsp_rproc_cfg_imx8qm,284};285 286static int imx_dsp_rproc_ready(struct rproc *rproc)287{288 struct imx_dsp_rproc *priv = rproc->priv;289 int i;290 291 if (!priv->rxdb_ch)292 return 0;293 294 for (i = 0; i < REMOTE_READY_WAIT_MAX_RETRIES; i++) {295 if (priv->flags & REMOTE_IS_READY)296 return 0;297 usleep_range(100, 200);298 }299 300 return -ETIMEDOUT;301}302 303/*304 * Start function for rproc_ops305 *306 * There is a handshake for start procedure: when DSP starts, it307 * will send a doorbell message to this driver, then the308 * REMOTE_IS_READY flags is set, then driver will kick309 * a message to DSP.310 */311static int imx_dsp_rproc_start(struct rproc *rproc)312{313 struct imx_dsp_rproc *priv = rproc->priv;314 const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg;315 const struct imx_rproc_dcfg *dcfg = dsp_dcfg->dcfg;316 struct device *dev = rproc->dev.parent;317 int ret;318 319 switch (dcfg->method) {320 case IMX_RPROC_MMIO:321 ret = regmap_update_bits(priv->regmap,322 dcfg->src_reg,323 dcfg->src_mask,324 dcfg->src_start);325 break;326 case IMX_RPROC_SCU_API:327 ret = imx_sc_pm_cpu_start(priv->ipc_handle,328 IMX_SC_R_DSP,329 true,330 rproc->bootaddr);331 break;332 default:333 return -EOPNOTSUPP;334 }335 336 if (ret)337 dev_err(dev, "Failed to enable remote core!\n");338 else339 ret = imx_dsp_rproc_ready(rproc);340 341 return ret;342}343 344/*345 * Stop function for rproc_ops346 * It clears the REMOTE_IS_READY flags347 */348static int imx_dsp_rproc_stop(struct rproc *rproc)349{350 struct imx_dsp_rproc *priv = rproc->priv;351 const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg;352 const struct imx_rproc_dcfg *dcfg = dsp_dcfg->dcfg;353 struct device *dev = rproc->dev.parent;354 int ret = 0;355 356 if (rproc->state == RPROC_CRASHED) {357 priv->flags &= ~REMOTE_IS_READY;358 return 0;359 }360 361 switch (dcfg->method) {362 case IMX_RPROC_MMIO:363 ret = regmap_update_bits(priv->regmap, dcfg->src_reg, dcfg->src_mask,364 dcfg->src_stop);365 break;366 case IMX_RPROC_SCU_API:367 ret = imx_sc_pm_cpu_start(priv->ipc_handle,368 IMX_SC_R_DSP,369 false,370 rproc->bootaddr);371 break;372 default:373 return -EOPNOTSUPP;374 }375 376 if (ret)377 dev_err(dev, "Failed to stop remote core\n");378 else379 priv->flags &= ~REMOTE_IS_READY;380 381 return ret;382}383 384/**385 * imx_dsp_rproc_sys_to_da() - internal memory translation helper386 * @priv: private data pointer387 * @sys: system address (DDR address)388 * @len: length of the memory buffer389 * @da: device address to translate390 *391 * Convert system address (DDR address) to device address (DSP)392 * for there may be memory remap for device.393 */394static int imx_dsp_rproc_sys_to_da(struct imx_dsp_rproc *priv, u64 sys,395 size_t len, u64 *da)396{397 const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg;398 const struct imx_rproc_dcfg *dcfg = dsp_dcfg->dcfg;399 int i;400 401 /* Parse address translation table */402 for (i = 0; i < dcfg->att_size; i++) {403 const struct imx_rproc_att *att = &dcfg->att[i];404 405 if (sys >= att->sa && sys + len <= att->sa + att->size) {406 unsigned int offset = sys - att->sa;407 408 *da = att->da + offset;409 return 0;410 }411 }412 413 return -ENOENT;414}415 416/* Main virtqueue message work function417 *418 * This function is executed upon scheduling of the i.MX DSP remoteproc419 * driver's workqueue. The workqueue is scheduled by the mailbox rx420 * handler.421 *422 * This work function processes both the Tx and Rx virtqueue indices on423 * every invocation. The rproc_vq_interrupt function can detect if there424 * are new unprocessed messages or not (returns IRQ_NONE vs IRQ_HANDLED),425 * but there is no need to check for these return values. The index 0426 * triggering will process all pending Rx buffers, and the index 1 triggering427 * will process all newly available Tx buffers and will wakeup any potentially428 * blocked senders.429 *430 * NOTE:431 * The current logic is based on an inherent design assumption of supporting432 * only 2 vrings, but this can be changed if needed.433 */434static void imx_dsp_rproc_vq_work(struct work_struct *work)435{436 struct imx_dsp_rproc *priv = container_of(work, struct imx_dsp_rproc,437 rproc_work);438 struct rproc *rproc = priv->rproc;439 440 mutex_lock(&rproc->lock);441 442 if (rproc->state != RPROC_RUNNING)443 goto unlock_mutex;444 445 rproc_vq_interrupt(priv->rproc, 0);446 rproc_vq_interrupt(priv->rproc, 1);447 448unlock_mutex:449 mutex_unlock(&rproc->lock);450}451 452/**453 * imx_dsp_rproc_rx_tx_callback() - inbound mailbox message handler454 * @cl: mailbox client pointer used for requesting the mailbox channel455 * @data: mailbox payload456 *457 * This handler is invoked by mailbox driver whenever a mailbox458 * message is received. Usually, the SUSPEND and RESUME related messages459 * are handled in this function, other messages are handled by remoteproc core460 */461static void imx_dsp_rproc_rx_tx_callback(struct mbox_client *cl, void *data)462{463 struct rproc *rproc = dev_get_drvdata(cl->dev);464 struct imx_dsp_rproc *priv = rproc->priv;465 struct device *dev = rproc->dev.parent;466 u32 message = (u32)(*(u32 *)data);467 468 dev_dbg(dev, "mbox msg: 0x%x\n", message);469 470 switch (message) {471 case RP_MBOX_SUSPEND_ACK:472 complete(&priv->pm_comp);473 break;474 case RP_MBOX_RESUME_ACK:475 complete(&priv->pm_comp);476 break;477 default:478 schedule_work(&priv->rproc_work);479 break;480 }481}482 483/**484 * imx_dsp_rproc_rxdb_callback() - inbound mailbox message handler485 * @cl: mailbox client pointer used for requesting the mailbox channel486 * @data: mailbox payload487 *488 * For doorbell, there is no message specified, just set REMOTE_IS_READY489 * flag.490 */491static void imx_dsp_rproc_rxdb_callback(struct mbox_client *cl, void *data)492{493 struct rproc *rproc = dev_get_drvdata(cl->dev);494 struct imx_dsp_rproc *priv = rproc->priv;495 496 /* Remote is ready after firmware is loaded and running */497 priv->flags |= REMOTE_IS_READY;498}499 500/**501 * imx_dsp_rproc_mbox_alloc() - request mailbox channels502 * @priv: private data pointer503 *504 * Request three mailbox channels (tx, rx, rxdb).505 */506static int imx_dsp_rproc_mbox_alloc(struct imx_dsp_rproc *priv)507{508 struct device *dev = priv->rproc->dev.parent;509 struct mbox_client *cl;510 int ret;511 512 if (!of_property_present(dev->of_node, "mbox-names"))513 return 0;514 515 cl = &priv->cl;516 cl->dev = dev;517 cl->tx_block = true;518 cl->tx_tout = 100;519 cl->knows_txdone = false;520 cl->rx_callback = imx_dsp_rproc_rx_tx_callback;521 522 /* Channel for sending message */523 priv->tx_ch = mbox_request_channel_byname(cl, "tx");524 if (IS_ERR(priv->tx_ch)) {525 ret = PTR_ERR(priv->tx_ch);526 dev_dbg(cl->dev, "failed to request tx mailbox channel: %d\n",527 ret);528 return ret;529 }530 531 /* Channel for receiving message */532 priv->rx_ch = mbox_request_channel_byname(cl, "rx");533 if (IS_ERR(priv->rx_ch)) {534 ret = PTR_ERR(priv->rx_ch);535 dev_dbg(cl->dev, "failed to request rx mailbox channel: %d\n",536 ret);537 goto free_channel_tx;538 }539 540 cl = &priv->cl_rxdb;541 cl->dev = dev;542 cl->rx_callback = imx_dsp_rproc_rxdb_callback;543 544 /*545 * RX door bell is used to receive the ready signal from remote546 * after firmware loaded.547 */548 priv->rxdb_ch = mbox_request_channel_byname(cl, "rxdb");549 if (IS_ERR(priv->rxdb_ch)) {550 ret = PTR_ERR(priv->rxdb_ch);551 dev_dbg(cl->dev, "failed to request mbox chan rxdb, ret %d\n",552 ret);553 goto free_channel_rx;554 }555 556 return 0;557 558free_channel_rx:559 mbox_free_channel(priv->rx_ch);560free_channel_tx:561 mbox_free_channel(priv->tx_ch);562 return ret;563}564 565/*566 * imx_dsp_rproc_mbox_no_alloc()567 *568 * Empty function for no mailbox between cores569 *570 * Always return 0571 */572static int imx_dsp_rproc_mbox_no_alloc(struct imx_dsp_rproc *priv)573{574 return 0;575}576 577static void imx_dsp_rproc_free_mbox(struct imx_dsp_rproc *priv)578{579 mbox_free_channel(priv->tx_ch);580 mbox_free_channel(priv->rx_ch);581 mbox_free_channel(priv->rxdb_ch);582}583 584/**585 * imx_dsp_rproc_add_carveout() - request mailbox channels586 * @priv: private data pointer587 *588 * This function registers specified memory entry in @rproc carveouts list589 * The carveouts can help to mapping the memory address for DSP.590 */591static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv)592{593 const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg;594 const struct imx_rproc_dcfg *dcfg = dsp_dcfg->dcfg;595 struct rproc *rproc = priv->rproc;596 struct device *dev = rproc->dev.parent;597 struct device_node *np = dev->of_node;598 struct of_phandle_iterator it;599 struct rproc_mem_entry *mem;600 struct reserved_mem *rmem;601 void __iomem *cpu_addr;602 int a;603 u64 da;604 605 /* Remap required addresses */606 for (a = 0; a < dcfg->att_size; a++) {607 const struct imx_rproc_att *att = &dcfg->att[a];608 609 if (!(att->flags & ATT_OWN))610 continue;611 612 if (imx_dsp_rproc_sys_to_da(priv, att->sa, att->size, &da))613 return -EINVAL;614 615 cpu_addr = devm_ioremap_wc(dev, att->sa, att->size);616 if (!cpu_addr) {617 dev_err(dev, "failed to map memory %p\n", &att->sa);618 return -ENOMEM;619 }620 621 /* Register memory region */622 mem = rproc_mem_entry_init(dev, (void __force *)cpu_addr, (dma_addr_t)att->sa,623 att->size, da, NULL, NULL, "dsp_mem");624 625 if (mem)626 rproc_coredump_add_segment(rproc, da, att->size);627 else628 return -ENOMEM;629 630 rproc_add_carveout(rproc, mem);631 }632 633 of_phandle_iterator_init(&it, np, "memory-region", NULL, 0);634 while (of_phandle_iterator_next(&it) == 0) {635 /*636 * Ignore the first memory region which will be used vdev buffer.637 * No need to do extra handlings, rproc_add_virtio_dev will handle it.638 */639 if (!strcmp(it.node->name, "vdev0buffer"))640 continue;641 642 rmem = of_reserved_mem_lookup(it.node);643 if (!rmem) {644 of_node_put(it.node);645 dev_err(dev, "unable to acquire memory-region\n");646 return -EINVAL;647 }648 649 if (imx_dsp_rproc_sys_to_da(priv, rmem->base, rmem->size, &da)) {650 of_node_put(it.node);651 return -EINVAL;652 }653 654 cpu_addr = devm_ioremap_wc(dev, rmem->base, rmem->size);655 if (!cpu_addr) {656 of_node_put(it.node);657 dev_err(dev, "failed to map memory %p\n", &rmem->base);658 return -ENOMEM;659 }660 661 /* Register memory region */662 mem = rproc_mem_entry_init(dev, (void __force *)cpu_addr, (dma_addr_t)rmem->base,663 rmem->size, da, NULL, NULL, it.node->name);664 665 if (mem) {666 rproc_coredump_add_segment(rproc, da, rmem->size);667 } else {668 of_node_put(it.node);669 return -ENOMEM;670 }671 672 rproc_add_carveout(rproc, mem);673 }674 675 return 0;676}677 678/* Prepare function for rproc_ops */679static int imx_dsp_rproc_prepare(struct rproc *rproc)680{681 struct imx_dsp_rproc *priv = rproc->priv;682 struct device *dev = rproc->dev.parent;683 struct rproc_mem_entry *carveout;684 int ret;685 686 ret = imx_dsp_rproc_add_carveout(priv);687 if (ret) {688 dev_err(dev, "failed on imx_dsp_rproc_add_carveout\n");689 return ret;690 }691 692 pm_runtime_get_sync(dev);693 694 /*695 * Clear buffers after pm rumtime for internal ocram is not696 * accessible if power and clock are not enabled.697 */698 list_for_each_entry(carveout, &rproc->carveouts, node) {699 if (carveout->va)700 memset(carveout->va, 0, carveout->len);701 }702 703 return 0;704}705 706/* Unprepare function for rproc_ops */707static int imx_dsp_rproc_unprepare(struct rproc *rproc)708{709 pm_runtime_put_sync(rproc->dev.parent);710 711 return 0;712}713 714/* Kick function for rproc_ops */715static void imx_dsp_rproc_kick(struct rproc *rproc, int vqid)716{717 struct imx_dsp_rproc *priv = rproc->priv;718 struct device *dev = rproc->dev.parent;719 int err;720 __u32 mmsg;721 722 if (!priv->tx_ch) {723 dev_err(dev, "No initialized mbox tx channel\n");724 return;725 }726 727 /*728 * Send the index of the triggered virtqueue as the mu payload.729 * Let remote processor know which virtqueue is used.730 */731 mmsg = vqid;732 733 err = mbox_send_message(priv->tx_ch, (void *)&mmsg);734 if (err < 0)735 dev_err(dev, "%s: failed (%d, err:%d)\n", __func__, vqid, err);736}737 738/*739 * Custom memory copy implementation for i.MX DSP Cores740 *741 * The IRAM is part of the HiFi DSP.742 * According to hw specs only 32-bits writes are allowed.743 */744static int imx_dsp_rproc_memcpy(void *dst, const void *src, size_t size)745{746 void __iomem *dest = (void __iomem *)dst;747 const u8 *src_byte = src;748 const u32 *source = src;749 u32 affected_mask;750 int i, q, r;751 u32 tmp;752 753 /* destination must be 32bit aligned */754 if (!IS_ALIGNED((uintptr_t)dest, 4))755 return -EINVAL;756 757 q = size / 4;758 r = size % 4;759 760 /* copy data in units of 32 bits at a time */761 for (i = 0; i < q; i++)762 writel(source[i], dest + i * 4);763 764 if (r) {765 affected_mask = GENMASK(8 * r, 0);766 767 /*768 * first read the 32bit data of dest, then change affected769 * bytes, and write back to dest.770 * For unaffected bytes, it should not be changed771 */772 tmp = readl(dest + q * 4);773 tmp &= ~affected_mask;774 775 /* avoid reading after end of source */776 for (i = 0; i < r; i++)777 tmp |= (src_byte[q * 4 + i] << (8 * i));778 779 writel(tmp, dest + q * 4);780 }781 782 return 0;783}784 785/*786 * Custom memset implementation for i.MX DSP Cores787 *788 * The IRAM is part of the HiFi DSP.789 * According to hw specs only 32-bits writes are allowed.790 */791static int imx_dsp_rproc_memset(void *addr, u8 value, size_t size)792{793 void __iomem *tmp_dst = (void __iomem *)addr;794 u32 tmp_val = value;795 u32 affected_mask;796 int q, r;797 u32 tmp;798 799 /* destination must be 32bit aligned */800 if (!IS_ALIGNED((uintptr_t)addr, 4))801 return -EINVAL;802 803 tmp_val |= tmp_val << 8;804 tmp_val |= tmp_val << 16;805 806 q = size / 4;807 r = size % 4;808 809 while (q--)810 writel(tmp_val, tmp_dst++);811 812 if (r) {813 affected_mask = GENMASK(8 * r, 0);814 815 /*816 * first read the 32bit data of addr, then change affected817 * bytes, and write back to addr.818 * For unaffected bytes, it should not be changed819 */820 tmp = readl(tmp_dst);821 tmp &= ~affected_mask;822 823 tmp |= (tmp_val & affected_mask);824 writel(tmp, tmp_dst);825 }826 827 return 0;828}829 830/*831 * imx_dsp_rproc_elf_load_segments() - load firmware segments to memory832 * @rproc: remote processor which will be booted using these fw segments833 * @fw: the ELF firmware image834 *835 * This function loads the firmware segments to memory, where the remote836 * processor expects them.837 *838 * Return: 0 on success and an appropriate error code otherwise839 */840static int imx_dsp_rproc_elf_load_segments(struct rproc *rproc, const struct firmware *fw)841{842 struct device *dev = &rproc->dev;843 const void *ehdr, *phdr;844 int i, ret = 0;845 u16 phnum;846 const u8 *elf_data = fw->data;847 u8 class = fw_elf_get_class(fw);848 u32 elf_phdr_get_size = elf_size_of_phdr(class);849 850 ehdr = elf_data;851 phnum = elf_hdr_get_e_phnum(class, ehdr);852 phdr = elf_data + elf_hdr_get_e_phoff(class, ehdr);853 854 /* go through the available ELF segments */855 for (i = 0; i < phnum; i++, phdr += elf_phdr_get_size) {856 u64 da = elf_phdr_get_p_paddr(class, phdr);857 u64 memsz = elf_phdr_get_p_memsz(class, phdr);858 u64 filesz = elf_phdr_get_p_filesz(class, phdr);859 u64 offset = elf_phdr_get_p_offset(class, phdr);860 u32 type = elf_phdr_get_p_type(class, phdr);861 void *ptr;862 863 if (type != PT_LOAD || !memsz)864 continue;865 866 dev_dbg(dev, "phdr: type %d da 0x%llx memsz 0x%llx filesz 0x%llx\n",867 type, da, memsz, filesz);868 869 if (filesz > memsz) {870 dev_err(dev, "bad phdr filesz 0x%llx memsz 0x%llx\n",871 filesz, memsz);872 ret = -EINVAL;873 break;874 }875 876 if (offset + filesz > fw->size) {877 dev_err(dev, "truncated fw: need 0x%llx avail 0x%zx\n",878 offset + filesz, fw->size);879 ret = -EINVAL;880 break;881 }882 883 if (!rproc_u64_fit_in_size_t(memsz)) {884 dev_err(dev, "size (%llx) does not fit in size_t type\n",885 memsz);886 ret = -EOVERFLOW;887 break;888 }889 890 /* grab the kernel address for this device address */891 ptr = rproc_da_to_va(rproc, da, memsz, NULL);892 if (!ptr) {893 dev_err(dev, "bad phdr da 0x%llx mem 0x%llx\n", da,894 memsz);895 ret = -EINVAL;896 break;897 }898 899 /* put the segment where the remote processor expects it */900 if (filesz) {901 ret = imx_dsp_rproc_memcpy(ptr, elf_data + offset, filesz);902 if (ret) {903 dev_err(dev, "memory copy failed for da 0x%llx memsz 0x%llx\n",904 da, memsz);905 break;906 }907 }908 909 /* zero out remaining memory for this segment */910 if (memsz > filesz) {911 ret = imx_dsp_rproc_memset(ptr + filesz, 0, memsz - filesz);912 if (ret) {913 dev_err(dev, "memset failed for da 0x%llx memsz 0x%llx\n",914 da, memsz);915 break;916 }917 }918 }919 920 return ret;921}922 923static int imx_dsp_rproc_parse_fw(struct rproc *rproc, const struct firmware *fw)924{925 if (rproc_elf_load_rsc_table(rproc, fw))926 dev_warn(&rproc->dev, "no resource table found for this firmware\n");927 928 return 0;929}930 931static const struct rproc_ops imx_dsp_rproc_ops = {932 .prepare = imx_dsp_rproc_prepare,933 .unprepare = imx_dsp_rproc_unprepare,934 .start = imx_dsp_rproc_start,935 .stop = imx_dsp_rproc_stop,936 .kick = imx_dsp_rproc_kick,937 .load = imx_dsp_rproc_elf_load_segments,938 .parse_fw = imx_dsp_rproc_parse_fw,939 .find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table,940 .sanity_check = rproc_elf_sanity_check,941 .get_boot_addr = rproc_elf_get_boot_addr,942};943 944/**945 * imx_dsp_attach_pm_domains() - attach the power domains946 * @priv: private data pointer947 *948 * On i.MX8QM and i.MX8QXP there is multiple power domains949 * required, so need to link them.950 */951static int imx_dsp_attach_pm_domains(struct imx_dsp_rproc *priv)952{953 struct device *dev = priv->rproc->dev.parent;954 int ret;955 956 /* A single PM domain is already attached. */957 if (dev->pm_domain)958 return 0;959 960 ret = dev_pm_domain_attach_list(dev, NULL, &priv->pd_list);961 return ret < 0 ? ret : 0;962}963 964/**965 * imx_dsp_rproc_detect_mode() - detect DSP control mode966 * @priv: private data pointer967 *968 * Different platform has different control method for DSP, which depends969 * on how the DSP is integrated in platform.970 *971 * For i.MX8QXP and i.MX8QM, DSP should be started and stopped by System972 * Control Unit.973 * For i.MX8MP and i.MX8ULP, DSP should be started and stopped by system974 * integration module.975 */976static int imx_dsp_rproc_detect_mode(struct imx_dsp_rproc *priv)977{978 const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg;979 struct device *dev = priv->rproc->dev.parent;980 struct regmap *regmap;981 int ret = 0;982 983 switch (dsp_dcfg->dcfg->method) {984 case IMX_RPROC_SCU_API:985 ret = imx_scu_get_handle(&priv->ipc_handle);986 if (ret)987 return ret;988 break;989 case IMX_RPROC_MMIO:990 regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "fsl,dsp-ctrl");991 if (IS_ERR(regmap)) {992 dev_err(dev, "failed to find syscon\n");993 return PTR_ERR(regmap);994 }995 996 priv->regmap = regmap;997 break;998 default:999 ret = -EOPNOTSUPP;1000 break;1001 }1002 1003 return ret;1004}1005 1006static const char *imx_dsp_clks_names[DSP_RPROC_CLK_MAX] = {1007 /* DSP clocks */1008 "core", "ocram", "debug", "ipg", "mu",1009};1010 1011static int imx_dsp_rproc_clk_get(struct imx_dsp_rproc *priv)1012{1013 struct device *dev = priv->rproc->dev.parent;1014 struct clk_bulk_data *clks = priv->clks;1015 int i;1016 1017 for (i = 0; i < DSP_RPROC_CLK_MAX; i++)1018 clks[i].id = imx_dsp_clks_names[i];1019 1020 return devm_clk_bulk_get_optional(dev, DSP_RPROC_CLK_MAX, clks);1021}1022 1023static int imx_dsp_rproc_probe(struct platform_device *pdev)1024{1025 const struct imx_dsp_rproc_dcfg *dsp_dcfg;1026 struct device *dev = &pdev->dev;1027 struct imx_dsp_rproc *priv;1028 struct rproc *rproc;1029 const char *fw_name;1030 int ret;1031 1032 dsp_dcfg = of_device_get_match_data(dev);1033 if (!dsp_dcfg)1034 return -ENODEV;1035 1036 ret = rproc_of_parse_firmware(dev, 0, &fw_name);1037 if (ret) {1038 dev_err(dev, "failed to parse firmware-name property, ret = %d\n",1039 ret);1040 return ret;1041 }1042 1043 rproc = devm_rproc_alloc(dev, "imx-dsp-rproc", &imx_dsp_rproc_ops,1044 fw_name, sizeof(*priv));1045 if (!rproc)1046 return -ENOMEM;1047 1048 priv = rproc->priv;1049 priv->rproc = rproc;1050 priv->dsp_dcfg = dsp_dcfg;1051 1052 if (no_mailboxes)1053 imx_dsp_rproc_mbox_init = imx_dsp_rproc_mbox_no_alloc;1054 else1055 imx_dsp_rproc_mbox_init = imx_dsp_rproc_mbox_alloc;1056 1057 dev_set_drvdata(dev, rproc);1058 1059 INIT_WORK(&priv->rproc_work, imx_dsp_rproc_vq_work);1060 1061 ret = imx_dsp_rproc_detect_mode(priv);1062 if (ret) {1063 dev_err(dev, "failed on imx_dsp_rproc_detect_mode\n");1064 return ret;1065 }1066 1067 /* There are multiple power domains required by DSP on some platform */1068 ret = imx_dsp_attach_pm_domains(priv);1069 if (ret) {1070 dev_err(dev, "failed on imx_dsp_attach_pm_domains\n");1071 return ret;1072 }1073 /* Get clocks */1074 ret = imx_dsp_rproc_clk_get(priv);1075 if (ret) {1076 dev_err(dev, "failed on imx_dsp_rproc_clk_get\n");1077 goto err_detach_domains;1078 }1079 1080 init_completion(&priv->pm_comp);1081 rproc->auto_boot = false;1082 ret = rproc_add(rproc);1083 if (ret) {1084 dev_err(dev, "rproc_add failed\n");1085 goto err_detach_domains;1086 }1087 1088 pm_runtime_enable(dev);1089 1090 return 0;1091 1092err_detach_domains:1093 dev_pm_domain_detach_list(priv->pd_list);1094 1095 return ret;1096}1097 1098static void imx_dsp_rproc_remove(struct platform_device *pdev)1099{1100 struct rproc *rproc = platform_get_drvdata(pdev);1101 struct imx_dsp_rproc *priv = rproc->priv;1102 1103 pm_runtime_disable(&pdev->dev);1104 rproc_del(rproc);1105 dev_pm_domain_detach_list(priv->pd_list);1106}1107 1108/* pm runtime functions */1109static int imx_dsp_runtime_resume(struct device *dev)1110{1111 struct rproc *rproc = dev_get_drvdata(dev);1112 struct imx_dsp_rproc *priv = rproc->priv;1113 const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg;1114 int ret;1115 1116 /*1117 * There is power domain attached with mailbox, if setup mailbox1118 * in probe(), then the power of mailbox is always enabled,1119 * the power can't be saved.1120 * So move setup of mailbox to runtime resume.1121 */1122 ret = imx_dsp_rproc_mbox_init(priv);1123 if (ret) {1124 dev_err(dev, "failed on imx_dsp_rproc_mbox_init\n");1125 return ret;1126 }1127 1128 ret = clk_bulk_prepare_enable(DSP_RPROC_CLK_MAX, priv->clks);1129 if (ret) {1130 dev_err(dev, "failed on clk_bulk_prepare_enable\n");1131 return ret;1132 }1133 1134 /* Reset DSP if needed */1135 if (dsp_dcfg->reset)1136 dsp_dcfg->reset(priv);1137 1138 return 0;1139}1140 1141static int imx_dsp_runtime_suspend(struct device *dev)1142{1143 struct rproc *rproc = dev_get_drvdata(dev);1144 struct imx_dsp_rproc *priv = rproc->priv;1145 1146 clk_bulk_disable_unprepare(DSP_RPROC_CLK_MAX, priv->clks);1147 1148 imx_dsp_rproc_free_mbox(priv);1149 1150 return 0;1151}1152 1153static void imx_dsp_load_firmware(const struct firmware *fw, void *context)1154{1155 struct rproc *rproc = context;1156 int ret;1157 1158 /*1159 * Same flow as start procedure.1160 * Load the ELF segments to memory firstly.1161 */1162 ret = rproc_load_segments(rproc, fw);1163 if (ret)1164 goto out;1165 1166 /* Start the remote processor */1167 ret = rproc->ops->start(rproc);1168 if (ret)1169 goto out;1170 1171 rproc->ops->kick(rproc, 0);1172 1173out:1174 release_firmware(fw);1175}1176 1177static int imx_dsp_suspend(struct device *dev)1178{1179 struct rproc *rproc = dev_get_drvdata(dev);1180 struct imx_dsp_rproc *priv = rproc->priv;1181 __u32 mmsg = RP_MBOX_SUSPEND_SYSTEM;1182 int ret;1183 1184 if (rproc->state != RPROC_RUNNING)1185 goto out;1186 1187 reinit_completion(&priv->pm_comp);1188 1189 /* Tell DSP that suspend is happening */1190 ret = mbox_send_message(priv->tx_ch, (void *)&mmsg);1191 if (ret < 0) {1192 dev_err(dev, "PM mbox_send_message failed: %d\n", ret);1193 return ret;1194 }1195 1196 /*1197 * DSP need to save the context at suspend.1198 * Here waiting the response for DSP, then power can be disabled.1199 */1200 if (!wait_for_completion_timeout(&priv->pm_comp, msecs_to_jiffies(100)))1201 return -EBUSY;1202 1203out:1204 /*1205 * The power of DSP is disabled in suspend, so force pm runtime1206 * to be suspend, then we can reenable the power and clocks at1207 * resume stage.1208 */1209 return pm_runtime_force_suspend(dev);1210}1211 1212static int imx_dsp_resume(struct device *dev)1213{1214 struct rproc *rproc = dev_get_drvdata(dev);1215 int ret = 0;1216 1217 ret = pm_runtime_force_resume(dev);1218 if (ret)1219 return ret;1220 1221 if (rproc->state != RPROC_RUNNING)1222 return 0;1223 1224 /*1225 * The power of DSP is disabled at suspend, the memory of dsp1226 * is reset, the image segments are lost. So need to reload1227 * firmware and restart the DSP if it is in running state.1228 */1229 ret = request_firmware_nowait(THIS_MODULE, FW_ACTION_UEVENT,1230 rproc->firmware, dev, GFP_KERNEL,1231 rproc, imx_dsp_load_firmware);1232 if (ret < 0) {1233 dev_err(dev, "load firmware failed: %d\n", ret);1234 goto err;1235 }1236 1237 return 0;1238 1239err:1240 pm_runtime_force_suspend(dev);1241 1242 return ret;1243}1244 1245static const struct dev_pm_ops imx_dsp_rproc_pm_ops = {1246 SYSTEM_SLEEP_PM_OPS(imx_dsp_suspend, imx_dsp_resume)1247 RUNTIME_PM_OPS(imx_dsp_runtime_suspend, imx_dsp_runtime_resume, NULL)1248};1249 1250static const struct of_device_id imx_dsp_rproc_of_match[] = {1251 { .compatible = "fsl,imx8qxp-hifi4", .data = &imx_dsp_rproc_cfg_imx8qxp },1252 { .compatible = "fsl,imx8qm-hifi4", .data = &imx_dsp_rproc_cfg_imx8qm },1253 { .compatible = "fsl,imx8mp-hifi4", .data = &imx_dsp_rproc_cfg_imx8mp },1254 { .compatible = "fsl,imx8ulp-hifi4", .data = &imx_dsp_rproc_cfg_imx8ulp },1255 {},1256};1257MODULE_DEVICE_TABLE(of, imx_dsp_rproc_of_match);1258 1259static struct platform_driver imx_dsp_rproc_driver = {1260 .probe = imx_dsp_rproc_probe,1261 .remove_new = imx_dsp_rproc_remove,1262 .driver = {1263 .name = "imx-dsp-rproc",1264 .of_match_table = imx_dsp_rproc_of_match,1265 .pm = pm_ptr(&imx_dsp_rproc_pm_ops),1266 },1267};1268module_platform_driver(imx_dsp_rproc_driver);1269 1270MODULE_LICENSE("GPL v2");1271MODULE_DESCRIPTION("i.MX HiFi Core Remote Processor Control Driver");1272MODULE_AUTHOR("Shengjiu Wang <shengjiu.wang@nxp.com>");1273