457 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (C) 2020-2024 Intel Corporation4 */5 6#include <linux/highmem.h>7#include <linux/moduleparam.h>8#include <linux/pci.h>9#include <linux/pm_runtime.h>10#include <linux/reboot.h>11 12#include "vpu_boot_api.h"13#include "ivpu_drv.h"14#include "ivpu_hw.h"15#include "ivpu_fw.h"16#include "ivpu_fw_log.h"17#include "ivpu_ipc.h"18#include "ivpu_job.h"19#include "ivpu_jsm_msg.h"20#include "ivpu_mmu.h"21#include "ivpu_ms.h"22#include "ivpu_pm.h"23 24static bool ivpu_disable_recovery;25module_param_named_unsafe(disable_recovery, ivpu_disable_recovery, bool, 0644);26MODULE_PARM_DESC(disable_recovery, "Disables recovery when NPU hang is detected");27 28static unsigned long ivpu_tdr_timeout_ms;29module_param_named(tdr_timeout_ms, ivpu_tdr_timeout_ms, ulong, 0644);30MODULE_PARM_DESC(tdr_timeout_ms, "Timeout for device hang detection, in milliseconds, 0 - default");31 32#define PM_RESCHEDULE_LIMIT 533 34static void ivpu_pm_prepare_cold_boot(struct ivpu_device *vdev)35{36 struct ivpu_fw_info *fw = vdev->fw;37 38 ivpu_cmdq_reset_all_contexts(vdev);39 ivpu_ipc_reset(vdev);40 ivpu_fw_load(vdev);41 fw->entry_point = fw->cold_boot_entry_point;42}43 44static void ivpu_pm_prepare_warm_boot(struct ivpu_device *vdev)45{46 struct ivpu_fw_info *fw = vdev->fw;47 struct vpu_boot_params *bp = ivpu_bo_vaddr(fw->mem);48 49 if (!bp->save_restore_ret_address) {50 ivpu_pm_prepare_cold_boot(vdev);51 return;52 }53 54 ivpu_dbg(vdev, FW_BOOT, "Save/restore entry point %llx", bp->save_restore_ret_address);55 fw->entry_point = bp->save_restore_ret_address;56}57 58static int ivpu_suspend(struct ivpu_device *vdev)59{60 int ret;61 62 ivpu_prepare_for_reset(vdev);63 64 ret = ivpu_shutdown(vdev);65 if (ret)66 ivpu_err(vdev, "Failed to shutdown NPU: %d\n", ret);67 68 return ret;69}70 71static int ivpu_resume(struct ivpu_device *vdev)72{73 int ret;74 75retry:76 pci_restore_state(to_pci_dev(vdev->drm.dev));77 pci_set_power_state(to_pci_dev(vdev->drm.dev), PCI_D0);78 79 ret = ivpu_hw_power_up(vdev);80 if (ret) {81 ivpu_err(vdev, "Failed to power up HW: %d\n", ret);82 goto err_power_down;83 }84 85 ret = ivpu_mmu_enable(vdev);86 if (ret) {87 ivpu_err(vdev, "Failed to resume MMU: %d\n", ret);88 goto err_power_down;89 }90 91 ret = ivpu_boot(vdev);92 if (ret)93 goto err_mmu_disable;94 95 return 0;96 97err_mmu_disable:98 ivpu_mmu_disable(vdev);99err_power_down:100 ivpu_hw_power_down(vdev);101 pci_set_power_state(to_pci_dev(vdev->drm.dev), PCI_D3hot);102 103 if (!ivpu_fw_is_cold_boot(vdev)) {104 ivpu_pm_prepare_cold_boot(vdev);105 goto retry;106 } else {107 ivpu_err(vdev, "Failed to resume the FW: %d\n", ret);108 }109 110 return ret;111}112 113static void ivpu_pm_recovery_work(struct work_struct *work)114{115 struct ivpu_pm_info *pm = container_of(work, struct ivpu_pm_info, recovery_work);116 struct ivpu_device *vdev = pm->vdev;117 char *evt[2] = {"IVPU_PM_EVENT=IVPU_RECOVER", NULL};118 int ret;119 120 ivpu_err(vdev, "Recovering the NPU (reset #%d)\n", atomic_read(&vdev->pm->reset_counter));121 122 ret = pm_runtime_resume_and_get(vdev->drm.dev);123 if (ret)124 ivpu_err(vdev, "Failed to resume NPU: %d\n", ret);125 126 ivpu_fw_log_dump(vdev);127 128 atomic_inc(&vdev->pm->reset_counter);129 atomic_set(&vdev->pm->reset_pending, 1);130 down_write(&vdev->pm->reset_lock);131 132 ivpu_suspend(vdev);133 ivpu_pm_prepare_cold_boot(vdev);134 ivpu_jobs_abort_all(vdev);135 ivpu_ms_cleanup_all(vdev);136 137 ret = ivpu_resume(vdev);138 if (ret)139 ivpu_err(vdev, "Failed to resume NPU: %d\n", ret);140 141 up_write(&vdev->pm->reset_lock);142 atomic_set(&vdev->pm->reset_pending, 0);143 144 kobject_uevent_env(&vdev->drm.dev->kobj, KOBJ_CHANGE, evt);145 pm_runtime_mark_last_busy(vdev->drm.dev);146 pm_runtime_put_autosuspend(vdev->drm.dev);147}148 149void ivpu_pm_trigger_recovery(struct ivpu_device *vdev, const char *reason)150{151 ivpu_err(vdev, "Recovery triggered by %s\n", reason);152 153 if (ivpu_disable_recovery) {154 ivpu_err(vdev, "Recovery not available when disable_recovery param is set\n");155 return;156 }157 158 if (ivpu_is_fpga(vdev)) {159 ivpu_err(vdev, "Recovery not available on FPGA\n");160 return;161 }162 163 /* Trigger recovery if it's not in progress */164 if (atomic_cmpxchg(&vdev->pm->reset_pending, 0, 1) == 0) {165 ivpu_hw_diagnose_failure(vdev);166 ivpu_hw_irq_disable(vdev); /* Disable IRQ early to protect from IRQ storm */167 queue_work(system_long_wq, &vdev->pm->recovery_work);168 }169}170 171static void ivpu_job_timeout_work(struct work_struct *work)172{173 struct ivpu_pm_info *pm = container_of(work, struct ivpu_pm_info, job_timeout_work.work);174 struct ivpu_device *vdev = pm->vdev;175 176 ivpu_pm_trigger_recovery(vdev, "TDR");177}178 179void ivpu_start_job_timeout_detection(struct ivpu_device *vdev)180{181 unsigned long timeout_ms = ivpu_tdr_timeout_ms ? ivpu_tdr_timeout_ms : vdev->timeout.tdr;182 183 /* No-op if already queued */184 queue_delayed_work(system_wq, &vdev->pm->job_timeout_work, msecs_to_jiffies(timeout_ms));185}186 187void ivpu_stop_job_timeout_detection(struct ivpu_device *vdev)188{189 cancel_delayed_work_sync(&vdev->pm->job_timeout_work);190}191 192int ivpu_pm_suspend_cb(struct device *dev)193{194 struct drm_device *drm = dev_get_drvdata(dev);195 struct ivpu_device *vdev = to_ivpu_device(drm);196 unsigned long timeout;197 198 ivpu_dbg(vdev, PM, "Suspend..\n");199 200 timeout = jiffies + msecs_to_jiffies(vdev->timeout.tdr);201 while (!ivpu_hw_is_idle(vdev)) {202 cond_resched();203 if (time_after_eq(jiffies, timeout)) {204 ivpu_err(vdev, "Failed to enter idle on system suspend\n");205 return -EBUSY;206 }207 }208 209 ivpu_jsm_pwr_d0i3_enter(vdev);210 211 ivpu_suspend(vdev);212 ivpu_pm_prepare_warm_boot(vdev);213 214 ivpu_dbg(vdev, PM, "Suspend done.\n");215 216 return 0;217}218 219int ivpu_pm_resume_cb(struct device *dev)220{221 struct drm_device *drm = dev_get_drvdata(dev);222 struct ivpu_device *vdev = to_ivpu_device(drm);223 int ret;224 225 ivpu_dbg(vdev, PM, "Resume..\n");226 227 ret = ivpu_resume(vdev);228 if (ret)229 ivpu_err(vdev, "Failed to resume: %d\n", ret);230 231 ivpu_dbg(vdev, PM, "Resume done.\n");232 233 return ret;234}235 236int ivpu_pm_runtime_suspend_cb(struct device *dev)237{238 struct drm_device *drm = dev_get_drvdata(dev);239 struct ivpu_device *vdev = to_ivpu_device(drm);240 int ret, ret_d0i3;241 bool is_idle;242 243 drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->submitted_jobs_xa));244 drm_WARN_ON(&vdev->drm, work_pending(&vdev->pm->recovery_work));245 246 ivpu_dbg(vdev, PM, "Runtime suspend..\n");247 248 ivpu_mmu_disable(vdev);249 250 is_idle = ivpu_hw_is_idle(vdev) || vdev->pm->dct_active_percent;251 if (!is_idle)252 ivpu_err(vdev, "NPU is not idle before autosuspend\n");253 254 ret_d0i3 = ivpu_jsm_pwr_d0i3_enter(vdev);255 if (ret_d0i3)256 ivpu_err(vdev, "Failed to prepare for d0i3: %d\n", ret_d0i3);257 258 ret = ivpu_suspend(vdev);259 if (ret)260 ivpu_err(vdev, "Failed to suspend NPU: %d\n", ret);261 262 if (!is_idle || ret_d0i3) {263 ivpu_err(vdev, "Forcing cold boot due to previous errors\n");264 atomic_inc(&vdev->pm->reset_counter);265 ivpu_fw_log_dump(vdev);266 ivpu_pm_prepare_cold_boot(vdev);267 } else {268 ivpu_pm_prepare_warm_boot(vdev);269 }270 271 ivpu_dbg(vdev, PM, "Runtime suspend done.\n");272 273 return 0;274}275 276int ivpu_pm_runtime_resume_cb(struct device *dev)277{278 struct drm_device *drm = dev_get_drvdata(dev);279 struct ivpu_device *vdev = to_ivpu_device(drm);280 int ret;281 282 ivpu_dbg(vdev, PM, "Runtime resume..\n");283 284 ret = ivpu_resume(vdev);285 if (ret)286 ivpu_err(vdev, "Failed to set RESUME state: %d\n", ret);287 288 ivpu_dbg(vdev, PM, "Runtime resume done.\n");289 290 return ret;291}292 293int ivpu_rpm_get(struct ivpu_device *vdev)294{295 int ret;296 297 ret = pm_runtime_resume_and_get(vdev->drm.dev);298 drm_WARN_ON(&vdev->drm, ret < 0);299 300 return ret;301}302 303void ivpu_rpm_put(struct ivpu_device *vdev)304{305 pm_runtime_mark_last_busy(vdev->drm.dev);306 pm_runtime_put_autosuspend(vdev->drm.dev);307}308 309void ivpu_pm_reset_prepare_cb(struct pci_dev *pdev)310{311 struct ivpu_device *vdev = pci_get_drvdata(pdev);312 313 ivpu_dbg(vdev, PM, "Pre-reset..\n");314 atomic_inc(&vdev->pm->reset_counter);315 atomic_set(&vdev->pm->reset_pending, 1);316 317 pm_runtime_get_sync(vdev->drm.dev);318 down_write(&vdev->pm->reset_lock);319 ivpu_prepare_for_reset(vdev);320 ivpu_hw_reset(vdev);321 ivpu_pm_prepare_cold_boot(vdev);322 ivpu_jobs_abort_all(vdev);323 ivpu_ms_cleanup_all(vdev);324 325 ivpu_dbg(vdev, PM, "Pre-reset done.\n");326}327 328void ivpu_pm_reset_done_cb(struct pci_dev *pdev)329{330 struct ivpu_device *vdev = pci_get_drvdata(pdev);331 int ret;332 333 ivpu_dbg(vdev, PM, "Post-reset..\n");334 ret = ivpu_resume(vdev);335 if (ret)336 ivpu_err(vdev, "Failed to set RESUME state: %d\n", ret);337 up_write(&vdev->pm->reset_lock);338 atomic_set(&vdev->pm->reset_pending, 0);339 ivpu_dbg(vdev, PM, "Post-reset done.\n");340 341 pm_runtime_mark_last_busy(vdev->drm.dev);342 pm_runtime_put_autosuspend(vdev->drm.dev);343}344 345void ivpu_pm_init(struct ivpu_device *vdev)346{347 struct device *dev = vdev->drm.dev;348 struct ivpu_pm_info *pm = vdev->pm;349 int delay;350 351 pm->vdev = vdev;352 353 init_rwsem(&pm->reset_lock);354 atomic_set(&pm->reset_pending, 0);355 atomic_set(&pm->reset_counter, 0);356 357 INIT_WORK(&pm->recovery_work, ivpu_pm_recovery_work);358 INIT_DELAYED_WORK(&pm->job_timeout_work, ivpu_job_timeout_work);359 360 if (ivpu_disable_recovery)361 delay = -1;362 else363 delay = vdev->timeout.autosuspend;364 365 pm_runtime_use_autosuspend(dev);366 pm_runtime_set_autosuspend_delay(dev, delay);367 368 ivpu_dbg(vdev, PM, "Autosuspend delay = %d\n", delay);369}370 371void ivpu_pm_cancel_recovery(struct ivpu_device *vdev)372{373 drm_WARN_ON(&vdev->drm, delayed_work_pending(&vdev->pm->job_timeout_work));374 cancel_work_sync(&vdev->pm->recovery_work);375}376 377void ivpu_pm_enable(struct ivpu_device *vdev)378{379 struct device *dev = vdev->drm.dev;380 381 pm_runtime_set_active(dev);382 pm_runtime_allow(dev);383 pm_runtime_mark_last_busy(dev);384 pm_runtime_put_autosuspend(dev);385}386 387void ivpu_pm_disable(struct ivpu_device *vdev)388{389 pm_runtime_get_noresume(vdev->drm.dev);390 pm_runtime_forbid(vdev->drm.dev);391}392 393int ivpu_pm_dct_init(struct ivpu_device *vdev)394{395 if (vdev->pm->dct_active_percent)396 return ivpu_pm_dct_enable(vdev, vdev->pm->dct_active_percent);397 398 return 0;399}400 401int ivpu_pm_dct_enable(struct ivpu_device *vdev, u8 active_percent)402{403 u32 active_us, inactive_us;404 int ret;405 406 if (active_percent == 0 || active_percent > 100)407 return -EINVAL;408 409 active_us = (DCT_PERIOD_US * active_percent) / 100;410 inactive_us = DCT_PERIOD_US - active_us;411 412 ret = ivpu_jsm_dct_enable(vdev, active_us, inactive_us);413 if (ret) {414 ivpu_err_ratelimited(vdev, "Filed to enable DCT: %d\n", ret);415 return ret;416 }417 418 vdev->pm->dct_active_percent = active_percent;419 420 ivpu_dbg(vdev, PM, "DCT set to %u%% (D0: %uus, D0i2: %uus)\n",421 active_percent, active_us, inactive_us);422 return 0;423}424 425int ivpu_pm_dct_disable(struct ivpu_device *vdev)426{427 int ret;428 429 ret = ivpu_jsm_dct_disable(vdev);430 if (ret) {431 ivpu_err_ratelimited(vdev, "Filed to disable DCT: %d\n", ret);432 return ret;433 }434 435 vdev->pm->dct_active_percent = 0;436 437 ivpu_dbg(vdev, PM, "DCT disabled\n");438 return 0;439}440 441void ivpu_pm_dct_irq_thread_handler(struct ivpu_device *vdev)442{443 bool enable;444 int ret;445 446 if (ivpu_hw_btrs_dct_get_request(vdev, &enable))447 return;448 449 if (vdev->pm->dct_active_percent)450 ret = ivpu_pm_dct_enable(vdev, DCT_DEFAULT_ACTIVE_PERCENT);451 else452 ret = ivpu_pm_dct_disable(vdev);453 454 if (!ret)455 ivpu_hw_btrs_dct_set_status(vdev, enable, vdev->pm->dct_active_percent);456}457