577 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2//3// Copyright(c) 2021-2022 Intel Corporation4//5// Authors: Cezary Rojewski <cezary.rojewski@intel.com>6// Amadeusz Slawinski <amadeuszx.slawinski@linux.intel.com>7//8 9#include <linux/io-64-nonatomic-lo-hi.h>10#include <linux/slab.h>11#include <sound/hdaudio_ext.h>12#include "avs.h"13#include "messages.h"14#include "registers.h"15#include "trace.h"16 17#define AVS_IPC_TIMEOUT_MS 30018#define AVS_D0IX_DELAY_MS 30019 20static int21avs_dsp_set_d0ix(struct avs_dev *adev, bool enable)22{23 struct avs_ipc *ipc = adev->ipc;24 int ret;25 26 /* Is transition required? */27 if (ipc->in_d0ix == enable)28 return 0;29 30 ret = avs_dsp_op(adev, set_d0ix, enable);31 if (ret) {32 /* Prevent further d0ix attempts on conscious IPC failure. */33 if (ret == -AVS_EIPC)34 atomic_inc(&ipc->d0ix_disable_depth);35 36 ipc->in_d0ix = false;37 return ret;38 }39 40 ipc->in_d0ix = enable;41 return 0;42}43 44static void avs_dsp_schedule_d0ix(struct avs_dev *adev, struct avs_ipc_msg *tx)45{46 if (atomic_read(&adev->ipc->d0ix_disable_depth))47 return;48 49 mod_delayed_work(system_power_efficient_wq, &adev->ipc->d0ix_work,50 msecs_to_jiffies(AVS_D0IX_DELAY_MS));51}52 53static void avs_dsp_d0ix_work(struct work_struct *work)54{55 struct avs_ipc *ipc = container_of(work, struct avs_ipc, d0ix_work.work);56 57 avs_dsp_set_d0ix(to_avs_dev(ipc->dev), true);58}59 60static int avs_dsp_wake_d0i0(struct avs_dev *adev, struct avs_ipc_msg *tx)61{62 struct avs_ipc *ipc = adev->ipc;63 64 if (!atomic_read(&ipc->d0ix_disable_depth)) {65 cancel_delayed_work_sync(&ipc->d0ix_work);66 return avs_dsp_set_d0ix(adev, false);67 }68 69 return 0;70}71 72int avs_dsp_disable_d0ix(struct avs_dev *adev)73{74 struct avs_ipc *ipc = adev->ipc;75 76 /* Prevent PG only on the first disable. */77 if (atomic_inc_return(&ipc->d0ix_disable_depth) == 1) {78 cancel_delayed_work_sync(&ipc->d0ix_work);79 return avs_dsp_set_d0ix(adev, false);80 }81 82 return 0;83}84 85int avs_dsp_enable_d0ix(struct avs_dev *adev)86{87 struct avs_ipc *ipc = adev->ipc;88 89 if (atomic_dec_and_test(&ipc->d0ix_disable_depth))90 queue_delayed_work(system_power_efficient_wq, &ipc->d0ix_work,91 msecs_to_jiffies(AVS_D0IX_DELAY_MS));92 return 0;93}94 95static void avs_dsp_recovery(struct avs_dev *adev)96{97 struct avs_soc_component *acomp;98 unsigned int core_mask;99 int ret;100 101 mutex_lock(&adev->comp_list_mutex);102 /* disconnect all running streams */103 list_for_each_entry(acomp, &adev->comp_list, node) {104 struct snd_soc_pcm_runtime *rtd;105 struct snd_soc_card *card;106 107 card = acomp->base.card;108 if (!card)109 continue;110 111 for_each_card_rtds(card, rtd) {112 struct snd_pcm *pcm;113 int dir;114 115 pcm = rtd->pcm;116 if (!pcm || rtd->dai_link->no_pcm)117 continue;118 119 for_each_pcm_streams(dir) {120 struct snd_pcm_substream *substream;121 122 substream = pcm->streams[dir].substream;123 if (!substream || !substream->runtime)124 continue;125 126 /* No need for _irq() as we are in nonatomic context. */127 snd_pcm_stream_lock(substream);128 snd_pcm_stop(substream, SNDRV_PCM_STATE_DISCONNECTED);129 snd_pcm_stream_unlock(substream);130 }131 }132 }133 mutex_unlock(&adev->comp_list_mutex);134 135 /* forcibly shutdown all cores */136 core_mask = GENMASK(adev->hw_cfg.dsp_cores - 1, 0);137 avs_dsp_core_disable(adev, core_mask);138 139 /* attempt dsp reboot */140 ret = avs_dsp_boot_firmware(adev, true);141 if (ret < 0)142 dev_err(adev->dev, "dsp reboot failed: %d\n", ret);143 144 pm_runtime_mark_last_busy(adev->dev);145 pm_runtime_enable(adev->dev);146 pm_request_autosuspend(adev->dev);147 148 atomic_set(&adev->ipc->recovering, 0);149}150 151static void avs_dsp_recovery_work(struct work_struct *work)152{153 struct avs_ipc *ipc = container_of(work, struct avs_ipc, recovery_work);154 155 avs_dsp_recovery(to_avs_dev(ipc->dev));156}157 158static void avs_dsp_exception_caught(struct avs_dev *adev, union avs_notify_msg *msg)159{160 struct avs_ipc *ipc = adev->ipc;161 162 /* Account for the double-exception case. */163 ipc->ready = false;164 165 if (!atomic_add_unless(&ipc->recovering, 1, 1)) {166 dev_err(adev->dev, "dsp recovery is already in progress\n");167 return;168 }169 170 dev_crit(adev->dev, "communication severed, rebooting dsp..\n");171 172 cancel_delayed_work_sync(&ipc->d0ix_work);173 ipc->in_d0ix = false;174 /* Re-enabled on recovery completion. */175 pm_runtime_disable(adev->dev);176 177 /* Process received notification. */178 avs_dsp_op(adev, coredump, msg);179 180 schedule_work(&ipc->recovery_work);181}182 183static void avs_dsp_receive_rx(struct avs_dev *adev, u64 header)184{185 struct avs_ipc *ipc = adev->ipc;186 union avs_reply_msg msg = AVS_MSG(header);187 u64 reg;188 189 reg = readq(avs_sram_addr(adev, AVS_FW_REGS_WINDOW));190 trace_avs_ipc_reply_msg(header, reg);191 192 ipc->rx.header = header;193 /* Abort copying payload if request processing was unsuccessful. */194 if (!msg.status) {195 /* update size in case of LARGE_CONFIG_GET */196 if (msg.msg_target == AVS_MOD_MSG &&197 msg.global_msg_type == AVS_MOD_LARGE_CONFIG_GET)198 ipc->rx.size = min_t(u32, AVS_MAILBOX_SIZE,199 msg.ext.large_config.data_off_size);200 201 memcpy_fromio(ipc->rx.data, avs_uplink_addr(adev), ipc->rx.size);202 trace_avs_msg_payload(ipc->rx.data, ipc->rx.size);203 }204}205 206static void avs_dsp_process_notification(struct avs_dev *adev, u64 header)207{208 struct avs_notify_mod_data mod_data;209 union avs_notify_msg msg = AVS_MSG(header);210 size_t data_size = 0;211 void *data = NULL;212 u64 reg;213 214 reg = readq(avs_sram_addr(adev, AVS_FW_REGS_WINDOW));215 trace_avs_ipc_notify_msg(header, reg);216 217 /* Ignore spurious notifications until handshake is established. */218 if (!adev->ipc->ready && msg.notify_msg_type != AVS_NOTIFY_FW_READY) {219 dev_dbg(adev->dev, "FW not ready, skip notification: 0x%08x\n", msg.primary);220 return;221 }222 223 /* Calculate notification payload size. */224 switch (msg.notify_msg_type) {225 case AVS_NOTIFY_FW_READY:226 break;227 228 case AVS_NOTIFY_PHRASE_DETECTED:229 data_size = sizeof(struct avs_notify_voice_data);230 break;231 232 case AVS_NOTIFY_RESOURCE_EVENT:233 data_size = sizeof(struct avs_notify_res_data);234 break;235 236 case AVS_NOTIFY_LOG_BUFFER_STATUS:237 case AVS_NOTIFY_EXCEPTION_CAUGHT:238 break;239 240 case AVS_NOTIFY_MODULE_EVENT:241 /* To know the total payload size, header needs to be read first. */242 memcpy_fromio(&mod_data, avs_uplink_addr(adev), sizeof(mod_data));243 data_size = sizeof(mod_data) + mod_data.data_size;244 break;245 246 default:247 dev_info(adev->dev, "unknown notification: 0x%08x\n", msg.primary);248 break;249 }250 251 if (data_size) {252 data = kmalloc(data_size, GFP_KERNEL);253 if (!data)254 return;255 256 memcpy_fromio(data, avs_uplink_addr(adev), data_size);257 trace_avs_msg_payload(data, data_size);258 }259 260 /* Perform notification-specific operations. */261 switch (msg.notify_msg_type) {262 case AVS_NOTIFY_FW_READY:263 dev_dbg(adev->dev, "FW READY 0x%08x\n", msg.primary);264 adev->ipc->ready = true;265 complete(&adev->fw_ready);266 break;267 268 case AVS_NOTIFY_LOG_BUFFER_STATUS:269 avs_log_buffer_status_locked(adev, &msg);270 break;271 272 case AVS_NOTIFY_EXCEPTION_CAUGHT:273 avs_dsp_exception_caught(adev, &msg);274 break;275 276 default:277 break;278 }279 280 kfree(data);281}282 283void avs_dsp_process_response(struct avs_dev *adev, u64 header)284{285 struct avs_ipc *ipc = adev->ipc;286 287 /*288 * Response may either be solicited - a reply for a request that has289 * been sent beforehand - or unsolicited (notification).290 */291 if (avs_msg_is_reply(header)) {292 /* Response processing is invoked from IRQ thread. */293 spin_lock_irq(&ipc->rx_lock);294 avs_dsp_receive_rx(adev, header);295 ipc->rx_completed = true;296 spin_unlock_irq(&ipc->rx_lock);297 } else {298 avs_dsp_process_notification(adev, header);299 }300 301 complete(&ipc->busy_completion);302}303 304static bool avs_ipc_is_busy(struct avs_ipc *ipc)305{306 struct avs_dev *adev = to_avs_dev(ipc->dev);307 const struct avs_spec *const spec = adev->spec;308 u32 hipc_rsp;309 310 hipc_rsp = snd_hdac_adsp_readl(adev, spec->hipc->rsp_offset);311 return hipc_rsp & spec->hipc->rsp_busy_mask;312}313 314static int avs_ipc_wait_busy_completion(struct avs_ipc *ipc, int timeout)315{316 u32 repeats_left = 128; /* to avoid infinite looping */317 int ret;318 319again:320 ret = wait_for_completion_timeout(&ipc->busy_completion, msecs_to_jiffies(timeout));321 322 /* DSP could be unresponsive at this point. */323 if (!ipc->ready)324 return -EPERM;325 326 if (!ret) {327 if (!avs_ipc_is_busy(ipc))328 return -ETIMEDOUT;329 /*330 * Firmware did its job, either notification or reply331 * has been received - now wait until it's processed.332 */333 wait_for_completion_killable(&ipc->busy_completion);334 }335 336 /* Ongoing notification's bottom-half may cause early wakeup */337 spin_lock(&ipc->rx_lock);338 if (!ipc->rx_completed) {339 if (repeats_left) {340 /* Reply delayed due to notification. */341 repeats_left--;342 reinit_completion(&ipc->busy_completion);343 spin_unlock(&ipc->rx_lock);344 goto again;345 }346 347 spin_unlock(&ipc->rx_lock);348 return -ETIMEDOUT;349 }350 351 spin_unlock(&ipc->rx_lock);352 return 0;353}354 355static void avs_ipc_msg_init(struct avs_ipc *ipc, struct avs_ipc_msg *reply)356{357 lockdep_assert_held(&ipc->rx_lock);358 359 ipc->rx.header = 0;360 ipc->rx.size = reply ? reply->size : 0;361 ipc->rx_completed = false;362 363 reinit_completion(&ipc->done_completion);364 reinit_completion(&ipc->busy_completion);365}366 367static void avs_dsp_send_tx(struct avs_dev *adev, struct avs_ipc_msg *tx, bool read_fwregs)368{369 const struct avs_spec *const spec = adev->spec;370 u64 reg = ULONG_MAX;371 372 tx->header |= spec->hipc->req_busy_mask;373 if (read_fwregs)374 reg = readq(avs_sram_addr(adev, AVS_FW_REGS_WINDOW));375 376 trace_avs_request(tx, reg);377 378 if (tx->size)379 memcpy_toio(avs_downlink_addr(adev), tx->data, tx->size);380 snd_hdac_adsp_writel(adev, spec->hipc->req_ext_offset, tx->header >> 32);381 snd_hdac_adsp_writel(adev, spec->hipc->req_offset, tx->header & UINT_MAX);382}383 384static int avs_dsp_do_send_msg(struct avs_dev *adev, struct avs_ipc_msg *request,385 struct avs_ipc_msg *reply, int timeout, const char *name)386{387 struct avs_ipc *ipc = adev->ipc;388 int ret;389 390 if (!ipc->ready)391 return -EPERM;392 393 mutex_lock(&ipc->msg_mutex);394 395 spin_lock(&ipc->rx_lock);396 avs_ipc_msg_init(ipc, reply);397 avs_dsp_send_tx(adev, request, true);398 spin_unlock(&ipc->rx_lock);399 400 ret = avs_ipc_wait_busy_completion(ipc, timeout);401 if (ret) {402 if (ret == -ETIMEDOUT) {403 union avs_notify_msg msg = AVS_NOTIFICATION(EXCEPTION_CAUGHT);404 405 /* Same treatment as on exception, just stack_dump=0. */406 avs_dsp_exception_caught(adev, &msg);407 }408 goto exit;409 }410 411 ret = ipc->rx.rsp.status;412 /*413 * If IPC channel is blocked e.g.: due to ongoing recovery,414 * -EPERM error code is expected and thus it's not an actual error.415 *416 * Unsupported IPCs are of no harm either.417 */418 if (ret == -EPERM || ret == AVS_IPC_NOT_SUPPORTED)419 dev_dbg(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n",420 name, request->glb.primary, request->glb.ext.val, ret);421 else if (ret)422 dev_err(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n",423 name, request->glb.primary, request->glb.ext.val, ret);424 425 if (reply) {426 reply->header = ipc->rx.header;427 reply->size = ipc->rx.size;428 if (reply->data && ipc->rx.size)429 memcpy(reply->data, ipc->rx.data, reply->size);430 }431 432exit:433 mutex_unlock(&ipc->msg_mutex);434 return ret;435}436 437static int avs_dsp_send_msg_sequence(struct avs_dev *adev, struct avs_ipc_msg *request,438 struct avs_ipc_msg *reply, int timeout, bool wake_d0i0,439 bool schedule_d0ix, const char *name)440{441 int ret;442 443 trace_avs_d0ix("wake", wake_d0i0, request->header);444 if (wake_d0i0) {445 ret = avs_dsp_wake_d0i0(adev, request);446 if (ret)447 return ret;448 }449 450 ret = avs_dsp_do_send_msg(adev, request, reply, timeout, name);451 if (ret)452 return ret;453 454 trace_avs_d0ix("schedule", schedule_d0ix, request->header);455 if (schedule_d0ix)456 avs_dsp_schedule_d0ix(adev, request);457 458 return 0;459}460 461int avs_dsp_send_msg_timeout(struct avs_dev *adev, struct avs_ipc_msg *request,462 struct avs_ipc_msg *reply, int timeout, const char *name)463{464 bool wake_d0i0 = avs_dsp_op(adev, d0ix_toggle, request, true);465 bool schedule_d0ix = avs_dsp_op(adev, d0ix_toggle, request, false);466 467 return avs_dsp_send_msg_sequence(adev, request, reply, timeout, wake_d0i0, schedule_d0ix,468 name);469}470 471int avs_dsp_send_msg(struct avs_dev *adev, struct avs_ipc_msg *request,472 struct avs_ipc_msg *reply, const char *name)473{474 return avs_dsp_send_msg_timeout(adev, request, reply, adev->ipc->default_timeout_ms, name);475}476 477int avs_dsp_send_pm_msg_timeout(struct avs_dev *adev, struct avs_ipc_msg *request,478 struct avs_ipc_msg *reply, int timeout, bool wake_d0i0,479 const char *name)480{481 return avs_dsp_send_msg_sequence(adev, request, reply, timeout, wake_d0i0, false, name);482}483 484int avs_dsp_send_pm_msg(struct avs_dev *adev, struct avs_ipc_msg *request,485 struct avs_ipc_msg *reply, bool wake_d0i0, const char *name)486{487 return avs_dsp_send_pm_msg_timeout(adev, request, reply, adev->ipc->default_timeout_ms,488 wake_d0i0, name);489}490 491static int avs_dsp_do_send_rom_msg(struct avs_dev *adev, struct avs_ipc_msg *request, int timeout,492 const char *name)493{494 struct avs_ipc *ipc = adev->ipc;495 int ret;496 497 mutex_lock(&ipc->msg_mutex);498 499 spin_lock(&ipc->rx_lock);500 avs_ipc_msg_init(ipc, NULL);501 /*502 * with hw still stalled, memory windows may not be503 * configured properly so avoid accessing SRAM504 */505 avs_dsp_send_tx(adev, request, false);506 spin_unlock(&ipc->rx_lock);507 508 /* ROM messages must be sent before main core is unstalled */509 ret = avs_dsp_op(adev, stall, AVS_MAIN_CORE_MASK, false);510 if (!ret) {511 ret = wait_for_completion_timeout(&ipc->done_completion, msecs_to_jiffies(timeout));512 ret = ret ? 0 : -ETIMEDOUT;513 }514 if (ret)515 dev_err(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n",516 name, request->glb.primary, request->glb.ext.val, ret);517 518 mutex_unlock(&ipc->msg_mutex);519 520 return ret;521}522 523int avs_dsp_send_rom_msg_timeout(struct avs_dev *adev, struct avs_ipc_msg *request, int timeout,524 const char *name)525{526 return avs_dsp_do_send_rom_msg(adev, request, timeout, name);527}528 529int avs_dsp_send_rom_msg(struct avs_dev *adev, struct avs_ipc_msg *request, const char *name)530{531 return avs_dsp_send_rom_msg_timeout(adev, request, adev->ipc->default_timeout_ms, name);532}533 534void avs_dsp_interrupt_control(struct avs_dev *adev, bool enable)535{536 const struct avs_spec *const spec = adev->spec;537 u32 value, mask;538 539 /*540 * No particular bit setting order. All of these are required541 * to have a functional SW <-> FW communication.542 */543 value = enable ? AVS_ADSP_ADSPIC_IPC : 0;544 snd_hdac_adsp_updatel(adev, AVS_ADSP_REG_ADSPIC, AVS_ADSP_ADSPIC_IPC, value);545 546 mask = AVS_ADSP_HIPCCTL_DONE | AVS_ADSP_HIPCCTL_BUSY;547 value = enable ? mask : 0;548 snd_hdac_adsp_updatel(adev, spec->hipc->ctl_offset, mask, value);549}550 551int avs_ipc_init(struct avs_ipc *ipc, struct device *dev)552{553 ipc->rx.data = devm_kzalloc(dev, AVS_MAILBOX_SIZE, GFP_KERNEL);554 if (!ipc->rx.data)555 return -ENOMEM;556 557 ipc->dev = dev;558 ipc->ready = false;559 ipc->default_timeout_ms = AVS_IPC_TIMEOUT_MS;560 INIT_WORK(&ipc->recovery_work, avs_dsp_recovery_work);561 INIT_DELAYED_WORK(&ipc->d0ix_work, avs_dsp_d0ix_work);562 init_completion(&ipc->done_completion);563 init_completion(&ipc->busy_completion);564 spin_lock_init(&ipc->rx_lock);565 mutex_init(&ipc->msg_mutex);566 567 return 0;568}569 570void avs_ipc_block(struct avs_ipc *ipc)571{572 ipc->ready = false;573 cancel_work_sync(&ipc->recovery_work);574 cancel_delayed_work_sync(&ipc->d0ix_work);575 ipc->in_d0ix = false;576}577