2137 lines · c
1/*2 * Copyright 2008 Advanced Micro Devices, Inc.3 * Copyright 2008 Red Hat Inc.4 * Copyright 2009 Jerome Glisse.5 *6 * Permission is hereby granted, free of charge, to any person obtaining a7 * copy of this software and associated documentation files (the "Software"),8 * to deal in the Software without restriction, including without limitation9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,10 * and/or sell copies of the Software, and to permit persons to whom the11 * Software is furnished to do so, subject to the following conditions:12 *13 * The above copyright notice and this permission notice shall be included in14 * all copies or substantial portions of the Software.15 *16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR22 * OTHER DEALINGS IN THE SOFTWARE.23 *24 */25 26#include <linux/kthread.h>27#include <linux/pci.h>28#include <linux/uaccess.h>29#include <linux/pm_runtime.h>30 31#include "amdgpu.h"32#include "amdgpu_pm.h"33#include "amdgpu_dm_debugfs.h"34#include "amdgpu_ras.h"35#include "amdgpu_rap.h"36#include "amdgpu_securedisplay.h"37#include "amdgpu_fw_attestation.h"38#include "amdgpu_umr.h"39 40#include "amdgpu_reset.h"41#include "amdgpu_psp_ta.h"42 43#if defined(CONFIG_DEBUG_FS)44 45/**46 * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes47 *48 * @read: True if reading49 * @f: open file handle50 * @buf: User buffer to write/read to51 * @size: Number of bytes to write/read52 * @pos: Offset to seek to53 *54 * This debugfs entry has special meaning on the offset being sought.55 * Various bits have different meanings:56 *57 * Bit 62: Indicates a GRBM bank switch is needed58 * Bit 61: Indicates a SRBM bank switch is needed (implies bit 62 is59 * zero)60 * Bits 24..33: The SE or ME selector if needed61 * Bits 34..43: The SH (or SA) or PIPE selector if needed62 * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed63 *64 * Bit 23: Indicates that the PM power gating lock should be held65 * This is necessary to read registers that might be66 * unreliable during a power gating transistion.67 *68 * The lower bits are the BYTE offset of the register to read. This69 * allows reading multiple registers in a single call and having70 * the returned size reflect that.71 */72static int amdgpu_debugfs_process_reg_op(bool read, struct file *f,73 char __user *buf, size_t size, loff_t *pos)74{75 struct amdgpu_device *adev = file_inode(f)->i_private;76 ssize_t result = 0;77 int r;78 bool pm_pg_lock, use_bank, use_ring;79 unsigned int instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;80 81 pm_pg_lock = use_bank = use_ring = false;82 instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;83 84 if (size & 0x3 || *pos & 0x3 ||85 ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))86 return -EINVAL;87 88 /* are we reading registers for which a PG lock is necessary? */89 pm_pg_lock = (*pos >> 23) & 1;90 91 if (*pos & (1ULL << 62)) {92 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;93 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;94 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;95 96 if (se_bank == 0x3FF)97 se_bank = 0xFFFFFFFF;98 if (sh_bank == 0x3FF)99 sh_bank = 0xFFFFFFFF;100 if (instance_bank == 0x3FF)101 instance_bank = 0xFFFFFFFF;102 use_bank = true;103 } else if (*pos & (1ULL << 61)) {104 105 me = (*pos & GENMASK_ULL(33, 24)) >> 24;106 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;107 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;108 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;109 110 use_ring = true;111 } else {112 use_bank = use_ring = false;113 }114 115 *pos &= (1UL << 22) - 1;116 117 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);118 if (r < 0) {119 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);120 return r;121 }122 123 r = amdgpu_virt_enable_access_debugfs(adev);124 if (r < 0) {125 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);126 return r;127 }128 129 if (use_bank) {130 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||131 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {132 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);133 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);134 amdgpu_virt_disable_access_debugfs(adev);135 return -EINVAL;136 }137 mutex_lock(&adev->grbm_idx_mutex);138 amdgpu_gfx_select_se_sh(adev, se_bank,139 sh_bank, instance_bank, 0);140 } else if (use_ring) {141 mutex_lock(&adev->srbm_mutex);142 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid, 0);143 }144 145 if (pm_pg_lock)146 mutex_lock(&adev->pm.mutex);147 148 while (size) {149 uint32_t value;150 151 if (read) {152 value = RREG32(*pos >> 2);153 r = put_user(value, (uint32_t *)buf);154 } else {155 r = get_user(value, (uint32_t *)buf);156 if (!r)157 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value, 0);158 }159 if (r) {160 result = r;161 goto end;162 }163 164 result += 4;165 buf += 4;166 *pos += 4;167 size -= 4;168 }169 170end:171 if (use_bank) {172 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);173 mutex_unlock(&adev->grbm_idx_mutex);174 } else if (use_ring) {175 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);176 mutex_unlock(&adev->srbm_mutex);177 }178 179 if (pm_pg_lock)180 mutex_unlock(&adev->pm.mutex);181 182 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);183 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);184 185 amdgpu_virt_disable_access_debugfs(adev);186 return result;187}188 189/*190 * amdgpu_debugfs_regs_read - Callback for reading MMIO registers191 */192static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,193 size_t size, loff_t *pos)194{195 return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);196}197 198/*199 * amdgpu_debugfs_regs_write - Callback for writing MMIO registers200 */201static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,202 size_t size, loff_t *pos)203{204 return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);205}206 207static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)208{209 struct amdgpu_debugfs_regs2_data *rd;210 211 rd = kzalloc(sizeof(*rd), GFP_KERNEL);212 if (!rd)213 return -ENOMEM;214 rd->adev = file_inode(file)->i_private;215 file->private_data = rd;216 mutex_init(&rd->lock);217 218 return 0;219}220 221static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)222{223 struct amdgpu_debugfs_regs2_data *rd = file->private_data;224 225 mutex_destroy(&rd->lock);226 kfree(file->private_data);227 return 0;228}229 230static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 offset, size_t size, int write_en)231{232 struct amdgpu_debugfs_regs2_data *rd = f->private_data;233 struct amdgpu_device *adev = rd->adev;234 ssize_t result = 0;235 int r;236 uint32_t value;237 238 if (size & 0x3 || offset & 0x3)239 return -EINVAL;240 241 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);242 if (r < 0) {243 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);244 return r;245 }246 247 r = amdgpu_virt_enable_access_debugfs(adev);248 if (r < 0) {249 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);250 return r;251 }252 253 mutex_lock(&rd->lock);254 255 if (rd->id.use_grbm) {256 if ((rd->id.grbm.sh != 0xFFFFFFFF && rd->id.grbm.sh >= adev->gfx.config.max_sh_per_se) ||257 (rd->id.grbm.se != 0xFFFFFFFF && rd->id.grbm.se >= adev->gfx.config.max_shader_engines)) {258 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);259 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);260 amdgpu_virt_disable_access_debugfs(adev);261 mutex_unlock(&rd->lock);262 return -EINVAL;263 }264 mutex_lock(&adev->grbm_idx_mutex);265 amdgpu_gfx_select_se_sh(adev, rd->id.grbm.se,266 rd->id.grbm.sh,267 rd->id.grbm.instance, rd->id.xcc_id);268 }269 270 if (rd->id.use_srbm) {271 mutex_lock(&adev->srbm_mutex);272 amdgpu_gfx_select_me_pipe_q(adev, rd->id.srbm.me, rd->id.srbm.pipe,273 rd->id.srbm.queue, rd->id.srbm.vmid, rd->id.xcc_id);274 }275 276 if (rd->id.pg_lock)277 mutex_lock(&adev->pm.mutex);278 279 while (size) {280 if (!write_en) {281 value = RREG32(offset >> 2);282 r = put_user(value, (uint32_t *)buf);283 } else {284 r = get_user(value, (uint32_t *)buf);285 if (!r)286 amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, value, rd->id.xcc_id);287 }288 if (r) {289 result = r;290 goto end;291 }292 offset += 4;293 size -= 4;294 result += 4;295 buf += 4;296 }297end:298 if (rd->id.use_grbm) {299 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, rd->id.xcc_id);300 mutex_unlock(&adev->grbm_idx_mutex);301 }302 303 if (rd->id.use_srbm) {304 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, rd->id.xcc_id);305 mutex_unlock(&adev->srbm_mutex);306 }307 308 if (rd->id.pg_lock)309 mutex_unlock(&adev->pm.mutex);310 311 mutex_unlock(&rd->lock);312 313 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);314 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);315 316 amdgpu_virt_disable_access_debugfs(adev);317 return result;318}319 320static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)321{322 struct amdgpu_debugfs_regs2_data *rd = f->private_data;323 struct amdgpu_debugfs_regs2_iocdata v1_data;324 int r;325 326 mutex_lock(&rd->lock);327 328 switch (cmd) {329 case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE_V2:330 r = copy_from_user(&rd->id, (struct amdgpu_debugfs_regs2_iocdata_v2 *)data,331 sizeof(rd->id));332 if (r)333 r = -EINVAL;334 goto done;335 case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:336 r = copy_from_user(&v1_data, (struct amdgpu_debugfs_regs2_iocdata *)data,337 sizeof(v1_data));338 if (r) {339 r = -EINVAL;340 goto done;341 }342 goto v1_copy;343 default:344 r = -EINVAL;345 goto done;346 }347 348v1_copy:349 rd->id.use_srbm = v1_data.use_srbm;350 rd->id.use_grbm = v1_data.use_grbm;351 rd->id.pg_lock = v1_data.pg_lock;352 rd->id.grbm.se = v1_data.grbm.se;353 rd->id.grbm.sh = v1_data.grbm.sh;354 rd->id.grbm.instance = v1_data.grbm.instance;355 rd->id.srbm.me = v1_data.srbm.me;356 rd->id.srbm.pipe = v1_data.srbm.pipe;357 rd->id.srbm.queue = v1_data.srbm.queue;358 rd->id.xcc_id = 0;359done:360 mutex_unlock(&rd->lock);361 return r;362}363 364static ssize_t amdgpu_debugfs_regs2_read(struct file *f, char __user *buf, size_t size, loff_t *pos)365{366 return amdgpu_debugfs_regs2_op(f, buf, *pos, size, 0);367}368 369static ssize_t amdgpu_debugfs_regs2_write(struct file *f, const char __user *buf, size_t size, loff_t *pos)370{371 return amdgpu_debugfs_regs2_op(f, (char __user *)buf, *pos, size, 1);372}373 374static int amdgpu_debugfs_gprwave_open(struct inode *inode, struct file *file)375{376 struct amdgpu_debugfs_gprwave_data *rd;377 378 rd = kzalloc(sizeof(*rd), GFP_KERNEL);379 if (!rd)380 return -ENOMEM;381 rd->adev = file_inode(file)->i_private;382 file->private_data = rd;383 mutex_init(&rd->lock);384 385 return 0;386}387 388static int amdgpu_debugfs_gprwave_release(struct inode *inode, struct file *file)389{390 struct amdgpu_debugfs_gprwave_data *rd = file->private_data;391 392 mutex_destroy(&rd->lock);393 kfree(file->private_data);394 return 0;395}396 397static ssize_t amdgpu_debugfs_gprwave_read(struct file *f, char __user *buf, size_t size, loff_t *pos)398{399 struct amdgpu_debugfs_gprwave_data *rd = f->private_data;400 struct amdgpu_device *adev = rd->adev;401 ssize_t result = 0;402 int r;403 uint32_t *data, x;404 405 if (size > 4096 || size & 0x3 || *pos & 0x3)406 return -EINVAL;407 408 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);409 if (r < 0) {410 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);411 return r;412 }413 414 r = amdgpu_virt_enable_access_debugfs(adev);415 if (r < 0) {416 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);417 return r;418 }419 420 data = kcalloc(1024, sizeof(*data), GFP_KERNEL);421 if (!data) {422 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);423 amdgpu_virt_disable_access_debugfs(adev);424 return -ENOMEM;425 }426 427 /* switch to the specific se/sh/cu */428 mutex_lock(&adev->grbm_idx_mutex);429 amdgpu_gfx_select_se_sh(adev, rd->id.se, rd->id.sh, rd->id.cu, rd->id.xcc_id);430 431 if (!rd->id.gpr_or_wave) {432 x = 0;433 if (adev->gfx.funcs->read_wave_data)434 adev->gfx.funcs->read_wave_data(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, data, &x);435 } else {436 x = size >> 2;437 if (rd->id.gpr.vpgr_or_sgpr) {438 if (adev->gfx.funcs->read_wave_vgprs)439 adev->gfx.funcs->read_wave_vgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, rd->id.gpr.thread, *pos, size>>2, data);440 } else {441 if (adev->gfx.funcs->read_wave_sgprs)442 adev->gfx.funcs->read_wave_sgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, *pos, size>>2, data);443 }444 }445 446 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, rd->id.xcc_id);447 mutex_unlock(&adev->grbm_idx_mutex);448 449 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);450 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);451 452 if (!x) {453 result = -EINVAL;454 goto done;455 }456 457 while (size && (*pos < x * 4)) {458 uint32_t value;459 460 value = data[*pos >> 2];461 r = put_user(value, (uint32_t *)buf);462 if (r) {463 result = r;464 goto done;465 }466 467 result += 4;468 buf += 4;469 *pos += 4;470 size -= 4;471 }472 473done:474 amdgpu_virt_disable_access_debugfs(adev);475 kfree(data);476 return result;477}478 479static long amdgpu_debugfs_gprwave_ioctl(struct file *f, unsigned int cmd, unsigned long data)480{481 struct amdgpu_debugfs_gprwave_data *rd = f->private_data;482 int r = 0;483 484 mutex_lock(&rd->lock);485 486 switch (cmd) {487 case AMDGPU_DEBUGFS_GPRWAVE_IOC_SET_STATE:488 if (copy_from_user(&rd->id,489 (struct amdgpu_debugfs_gprwave_iocdata *)data,490 sizeof(rd->id)))491 r = -EFAULT;492 goto done;493 default:494 r = -EINVAL;495 goto done;496 }497 498done:499 mutex_unlock(&rd->lock);500 return r;501}502 503 504 505 506/**507 * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register508 *509 * @f: open file handle510 * @buf: User buffer to store read data in511 * @size: Number of bytes to read512 * @pos: Offset to seek to513 *514 * The lower bits are the BYTE offset of the register to read. This515 * allows reading multiple registers in a single call and having516 * the returned size reflect that.517 */518static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,519 size_t size, loff_t *pos)520{521 struct amdgpu_device *adev = file_inode(f)->i_private;522 ssize_t result = 0;523 int r;524 525 if (size & 0x3 || *pos & 0x3)526 return -EINVAL;527 528 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);529 if (r < 0) {530 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);531 return r;532 }533 534 r = amdgpu_virt_enable_access_debugfs(adev);535 if (r < 0) {536 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);537 return r;538 }539 540 while (size) {541 uint32_t value;542 543 if (upper_32_bits(*pos))544 value = RREG32_PCIE_EXT(*pos);545 else546 value = RREG32_PCIE(*pos);547 548 r = put_user(value, (uint32_t *)buf);549 if (r)550 goto out;551 552 result += 4;553 buf += 4;554 *pos += 4;555 size -= 4;556 }557 558 r = result;559out:560 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);561 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);562 amdgpu_virt_disable_access_debugfs(adev);563 return r;564}565 566/**567 * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register568 *569 * @f: open file handle570 * @buf: User buffer to write data from571 * @size: Number of bytes to write572 * @pos: Offset to seek to573 *574 * The lower bits are the BYTE offset of the register to write. This575 * allows writing multiple registers in a single call and having576 * the returned size reflect that.577 */578static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,579 size_t size, loff_t *pos)580{581 struct amdgpu_device *adev = file_inode(f)->i_private;582 ssize_t result = 0;583 int r;584 585 if (size & 0x3 || *pos & 0x3)586 return -EINVAL;587 588 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);589 if (r < 0) {590 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);591 return r;592 }593 594 r = amdgpu_virt_enable_access_debugfs(adev);595 if (r < 0) {596 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);597 return r;598 }599 600 while (size) {601 uint32_t value;602 603 r = get_user(value, (uint32_t *)buf);604 if (r)605 goto out;606 607 if (upper_32_bits(*pos))608 WREG32_PCIE_EXT(*pos, value);609 else610 WREG32_PCIE(*pos, value);611 612 result += 4;613 buf += 4;614 *pos += 4;615 size -= 4;616 }617 618 r = result;619out:620 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);621 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);622 amdgpu_virt_disable_access_debugfs(adev);623 return r;624}625 626/**627 * amdgpu_debugfs_regs_didt_read - Read from a DIDT register628 *629 * @f: open file handle630 * @buf: User buffer to store read data in631 * @size: Number of bytes to read632 * @pos: Offset to seek to633 *634 * The lower bits are the BYTE offset of the register to read. This635 * allows reading multiple registers in a single call and having636 * the returned size reflect that.637 */638static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,639 size_t size, loff_t *pos)640{641 struct amdgpu_device *adev = file_inode(f)->i_private;642 ssize_t result = 0;643 int r;644 645 if (size & 0x3 || *pos & 0x3)646 return -EINVAL;647 648 if (!adev->didt_rreg)649 return -EOPNOTSUPP;650 651 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);652 if (r < 0) {653 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);654 return r;655 }656 657 r = amdgpu_virt_enable_access_debugfs(adev);658 if (r < 0) {659 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);660 return r;661 }662 663 while (size) {664 uint32_t value;665 666 value = RREG32_DIDT(*pos >> 2);667 r = put_user(value, (uint32_t *)buf);668 if (r)669 goto out;670 671 result += 4;672 buf += 4;673 *pos += 4;674 size -= 4;675 }676 677 r = result;678out:679 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);680 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);681 amdgpu_virt_disable_access_debugfs(adev);682 return r;683}684 685/**686 * amdgpu_debugfs_regs_didt_write - Write to a DIDT register687 *688 * @f: open file handle689 * @buf: User buffer to write data from690 * @size: Number of bytes to write691 * @pos: Offset to seek to692 *693 * The lower bits are the BYTE offset of the register to write. This694 * allows writing multiple registers in a single call and having695 * the returned size reflect that.696 */697static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,698 size_t size, loff_t *pos)699{700 struct amdgpu_device *adev = file_inode(f)->i_private;701 ssize_t result = 0;702 int r;703 704 if (size & 0x3 || *pos & 0x3)705 return -EINVAL;706 707 if (!adev->didt_wreg)708 return -EOPNOTSUPP;709 710 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);711 if (r < 0) {712 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);713 return r;714 }715 716 r = amdgpu_virt_enable_access_debugfs(adev);717 if (r < 0) {718 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);719 return r;720 }721 722 while (size) {723 uint32_t value;724 725 r = get_user(value, (uint32_t *)buf);726 if (r)727 goto out;728 729 WREG32_DIDT(*pos >> 2, value);730 731 result += 4;732 buf += 4;733 *pos += 4;734 size -= 4;735 }736 737 r = result;738out:739 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);740 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);741 amdgpu_virt_disable_access_debugfs(adev);742 return r;743}744 745/**746 * amdgpu_debugfs_regs_smc_read - Read from a SMC register747 *748 * @f: open file handle749 * @buf: User buffer to store read data in750 * @size: Number of bytes to read751 * @pos: Offset to seek to752 *753 * The lower bits are the BYTE offset of the register to read. This754 * allows reading multiple registers in a single call and having755 * the returned size reflect that.756 */757static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,758 size_t size, loff_t *pos)759{760 struct amdgpu_device *adev = file_inode(f)->i_private;761 ssize_t result = 0;762 int r;763 764 if (!adev->smc_rreg)765 return -EOPNOTSUPP;766 767 if (size & 0x3 || *pos & 0x3)768 return -EINVAL;769 770 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);771 if (r < 0) {772 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);773 return r;774 }775 776 r = amdgpu_virt_enable_access_debugfs(adev);777 if (r < 0) {778 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);779 return r;780 }781 782 while (size) {783 uint32_t value;784 785 value = RREG32_SMC(*pos);786 r = put_user(value, (uint32_t *)buf);787 if (r)788 goto out;789 790 result += 4;791 buf += 4;792 *pos += 4;793 size -= 4;794 }795 796 r = result;797out:798 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);799 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);800 amdgpu_virt_disable_access_debugfs(adev);801 return r;802}803 804/**805 * amdgpu_debugfs_regs_smc_write - Write to a SMC register806 *807 * @f: open file handle808 * @buf: User buffer to write data from809 * @size: Number of bytes to write810 * @pos: Offset to seek to811 *812 * The lower bits are the BYTE offset of the register to write. This813 * allows writing multiple registers in a single call and having814 * the returned size reflect that.815 */816static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,817 size_t size, loff_t *pos)818{819 struct amdgpu_device *adev = file_inode(f)->i_private;820 ssize_t result = 0;821 int r;822 823 if (!adev->smc_wreg)824 return -EOPNOTSUPP;825 826 if (size & 0x3 || *pos & 0x3)827 return -EINVAL;828 829 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);830 if (r < 0) {831 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);832 return r;833 }834 835 r = amdgpu_virt_enable_access_debugfs(adev);836 if (r < 0) {837 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);838 return r;839 }840 841 while (size) {842 uint32_t value;843 844 r = get_user(value, (uint32_t *)buf);845 if (r)846 goto out;847 848 WREG32_SMC(*pos, value);849 850 result += 4;851 buf += 4;852 *pos += 4;853 size -= 4;854 }855 856 r = result;857out:858 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);859 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);860 amdgpu_virt_disable_access_debugfs(adev);861 return r;862}863 864/**865 * amdgpu_debugfs_gca_config_read - Read from gfx config data866 *867 * @f: open file handle868 * @buf: User buffer to store read data in869 * @size: Number of bytes to read870 * @pos: Offset to seek to871 *872 * This file is used to access configuration data in a somewhat873 * stable fashion. The format is a series of DWORDs with the first874 * indicating which revision it is. New content is appended to the875 * end so that older software can still read the data.876 */877 878static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,879 size_t size, loff_t *pos)880{881 struct amdgpu_device *adev = file_inode(f)->i_private;882 ssize_t result = 0;883 int r;884 uint32_t *config, no_regs = 0;885 886 if (size & 0x3 || *pos & 0x3)887 return -EINVAL;888 889 config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);890 if (!config)891 return -ENOMEM;892 893 /* version, increment each time something is added */894 config[no_regs++] = 5;895 config[no_regs++] = adev->gfx.config.max_shader_engines;896 config[no_regs++] = adev->gfx.config.max_tile_pipes;897 config[no_regs++] = adev->gfx.config.max_cu_per_sh;898 config[no_regs++] = adev->gfx.config.max_sh_per_se;899 config[no_regs++] = adev->gfx.config.max_backends_per_se;900 config[no_regs++] = adev->gfx.config.max_texture_channel_caches;901 config[no_regs++] = adev->gfx.config.max_gprs;902 config[no_regs++] = adev->gfx.config.max_gs_threads;903 config[no_regs++] = adev->gfx.config.max_hw_contexts;904 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;905 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;906 config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;907 config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;908 config[no_regs++] = adev->gfx.config.num_tile_pipes;909 config[no_regs++] = adev->gfx.config.backend_enable_mask;910 config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;911 config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;912 config[no_regs++] = adev->gfx.config.shader_engine_tile_size;913 config[no_regs++] = adev->gfx.config.num_gpus;914 config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;915 config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;916 config[no_regs++] = adev->gfx.config.gb_addr_config;917 config[no_regs++] = adev->gfx.config.num_rbs;918 919 /* rev==1 */920 config[no_regs++] = adev->rev_id;921 config[no_regs++] = adev->pg_flags;922 config[no_regs++] = lower_32_bits(adev->cg_flags);923 924 /* rev==2 */925 config[no_regs++] = adev->family;926 config[no_regs++] = adev->external_rev_id;927 928 /* rev==3 */929 config[no_regs++] = adev->pdev->device;930 config[no_regs++] = adev->pdev->revision;931 config[no_regs++] = adev->pdev->subsystem_device;932 config[no_regs++] = adev->pdev->subsystem_vendor;933 934 /* rev==4 APU flag */935 config[no_regs++] = adev->flags & AMD_IS_APU ? 1 : 0;936 937 /* rev==5 PG/CG flag upper 32bit */938 config[no_regs++] = 0;939 config[no_regs++] = upper_32_bits(adev->cg_flags);940 941 while (size && (*pos < no_regs * 4)) {942 uint32_t value;943 944 value = config[*pos >> 2];945 r = put_user(value, (uint32_t *)buf);946 if (r) {947 kfree(config);948 return r;949 }950 951 result += 4;952 buf += 4;953 *pos += 4;954 size -= 4;955 }956 957 kfree(config);958 return result;959}960 961/**962 * amdgpu_debugfs_sensor_read - Read from the powerplay sensors963 *964 * @f: open file handle965 * @buf: User buffer to store read data in966 * @size: Number of bytes to read967 * @pos: Offset to seek to968 *969 * The offset is treated as the BYTE address of one of the sensors970 * enumerated in amd/include/kgd_pp_interface.h under the971 * 'amd_pp_sensors' enumeration. For instance to read the UVD VCLK972 * you would use the offset 3 * 4 = 12.973 */974static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,975 size_t size, loff_t *pos)976{977 struct amdgpu_device *adev = file_inode(f)->i_private;978 int idx, x, outsize, r, valuesize;979 uint32_t values[16];980 981 if (size & 3 || *pos & 0x3)982 return -EINVAL;983 984 if (!adev->pm.dpm_enabled)985 return -EINVAL;986 987 /* convert offset to sensor number */988 idx = *pos >> 2;989 990 valuesize = sizeof(values);991 992 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);993 if (r < 0) {994 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);995 return r;996 }997 998 r = amdgpu_virt_enable_access_debugfs(adev);999 if (r < 0) {1000 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1001 return r;1002 }1003 1004 r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);1005 1006 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);1007 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1008 1009 if (r) {1010 amdgpu_virt_disable_access_debugfs(adev);1011 return r;1012 }1013 1014 if (size > valuesize) {1015 amdgpu_virt_disable_access_debugfs(adev);1016 return -EINVAL;1017 }1018 1019 outsize = 0;1020 x = 0;1021 if (!r) {1022 while (size) {1023 r = put_user(values[x++], (int32_t *)buf);1024 buf += 4;1025 size -= 4;1026 outsize += 4;1027 }1028 }1029 1030 amdgpu_virt_disable_access_debugfs(adev);1031 return !r ? outsize : r;1032}1033 1034/** amdgpu_debugfs_wave_read - Read WAVE STATUS data1035 *1036 * @f: open file handle1037 * @buf: User buffer to store read data in1038 * @size: Number of bytes to read1039 * @pos: Offset to seek to1040 *1041 * The offset being sought changes which wave that the status data1042 * will be returned for. The bits are used as follows:1043 *1044 * Bits 0..6: Byte offset into data1045 * Bits 7..14: SE selector1046 * Bits 15..22: SH/SA selector1047 * Bits 23..30: CU/{WGP+SIMD} selector1048 * Bits 31..36: WAVE ID selector1049 * Bits 37..44: SIMD ID selector1050 *1051 * The returned data begins with one DWORD of version information1052 * Followed by WAVE STATUS registers relevant to the GFX IP version1053 * being used. See gfx_v8_0_read_wave_data() for an example output.1054 */1055static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,1056 size_t size, loff_t *pos)1057{1058 struct amdgpu_device *adev = f->f_inode->i_private;1059 int r, x;1060 ssize_t result = 0;1061 uint32_t offset, se, sh, cu, wave, simd, data[32];1062 1063 if (size & 3 || *pos & 3)1064 return -EINVAL;1065 1066 /* decode offset */1067 offset = (*pos & GENMASK_ULL(6, 0));1068 se = (*pos & GENMASK_ULL(14, 7)) >> 7;1069 sh = (*pos & GENMASK_ULL(22, 15)) >> 15;1070 cu = (*pos & GENMASK_ULL(30, 23)) >> 23;1071 wave = (*pos & GENMASK_ULL(36, 31)) >> 31;1072 simd = (*pos & GENMASK_ULL(44, 37)) >> 37;1073 1074 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);1075 if (r < 0) {1076 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1077 return r;1078 }1079 1080 r = amdgpu_virt_enable_access_debugfs(adev);1081 if (r < 0) {1082 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1083 return r;1084 }1085 1086 /* switch to the specific se/sh/cu */1087 mutex_lock(&adev->grbm_idx_mutex);1088 amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);1089 1090 x = 0;1091 if (adev->gfx.funcs->read_wave_data)1092 adev->gfx.funcs->read_wave_data(adev, 0, simd, wave, data, &x);1093 1094 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);1095 mutex_unlock(&adev->grbm_idx_mutex);1096 1097 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);1098 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1099 1100 if (!x) {1101 amdgpu_virt_disable_access_debugfs(adev);1102 return -EINVAL;1103 }1104 1105 while (size && (offset < x * 4)) {1106 uint32_t value;1107 1108 value = data[offset >> 2];1109 r = put_user(value, (uint32_t *)buf);1110 if (r) {1111 amdgpu_virt_disable_access_debugfs(adev);1112 return r;1113 }1114 1115 result += 4;1116 buf += 4;1117 offset += 4;1118 size -= 4;1119 }1120 1121 amdgpu_virt_disable_access_debugfs(adev);1122 return result;1123}1124 1125/** amdgpu_debugfs_gpr_read - Read wave gprs1126 *1127 * @f: open file handle1128 * @buf: User buffer to store read data in1129 * @size: Number of bytes to read1130 * @pos: Offset to seek to1131 *1132 * The offset being sought changes which wave that the status data1133 * will be returned for. The bits are used as follows:1134 *1135 * Bits 0..11: Byte offset into data1136 * Bits 12..19: SE selector1137 * Bits 20..27: SH/SA selector1138 * Bits 28..35: CU/{WGP+SIMD} selector1139 * Bits 36..43: WAVE ID selector1140 * Bits 37..44: SIMD ID selector1141 * Bits 52..59: Thread selector1142 * Bits 60..61: Bank selector (VGPR=0,SGPR=1)1143 *1144 * The return data comes from the SGPR or VGPR register bank for1145 * the selected operational unit.1146 */1147static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,1148 size_t size, loff_t *pos)1149{1150 struct amdgpu_device *adev = f->f_inode->i_private;1151 int r;1152 ssize_t result = 0;1153 uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;1154 1155 if (size > 4096 || size & 3 || *pos & 3)1156 return -EINVAL;1157 1158 /* decode offset */1159 offset = (*pos & GENMASK_ULL(11, 0)) >> 2;1160 se = (*pos & GENMASK_ULL(19, 12)) >> 12;1161 sh = (*pos & GENMASK_ULL(27, 20)) >> 20;1162 cu = (*pos & GENMASK_ULL(35, 28)) >> 28;1163 wave = (*pos & GENMASK_ULL(43, 36)) >> 36;1164 simd = (*pos & GENMASK_ULL(51, 44)) >> 44;1165 thread = (*pos & GENMASK_ULL(59, 52)) >> 52;1166 bank = (*pos & GENMASK_ULL(61, 60)) >> 60;1167 1168 data = kcalloc(1024, sizeof(*data), GFP_KERNEL);1169 if (!data)1170 return -ENOMEM;1171 1172 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);1173 if (r < 0)1174 goto err;1175 1176 r = amdgpu_virt_enable_access_debugfs(adev);1177 if (r < 0)1178 goto err;1179 1180 /* switch to the specific se/sh/cu */1181 mutex_lock(&adev->grbm_idx_mutex);1182 amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);1183 1184 if (bank == 0) {1185 if (adev->gfx.funcs->read_wave_vgprs)1186 adev->gfx.funcs->read_wave_vgprs(adev, 0, simd, wave, thread, offset, size>>2, data);1187 } else {1188 if (adev->gfx.funcs->read_wave_sgprs)1189 adev->gfx.funcs->read_wave_sgprs(adev, 0, simd, wave, offset, size>>2, data);1190 }1191 1192 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);1193 mutex_unlock(&adev->grbm_idx_mutex);1194 1195 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);1196 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1197 1198 while (size) {1199 uint32_t value;1200 1201 value = data[result >> 2];1202 r = put_user(value, (uint32_t *)buf);1203 if (r) {1204 amdgpu_virt_disable_access_debugfs(adev);1205 goto err;1206 }1207 1208 result += 4;1209 buf += 4;1210 size -= 4;1211 }1212 1213 kfree(data);1214 amdgpu_virt_disable_access_debugfs(adev);1215 return result;1216 1217err:1218 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1219 kfree(data);1220 return r;1221}1222 1223/**1224 * amdgpu_debugfs_gfxoff_residency_read - Read GFXOFF residency1225 *1226 * @f: open file handle1227 * @buf: User buffer to store read data in1228 * @size: Number of bytes to read1229 * @pos: Offset to seek to1230 *1231 * Read the last residency value logged. It doesn't auto update, one needs to1232 * stop logging before getting the current value.1233 */1234static ssize_t amdgpu_debugfs_gfxoff_residency_read(struct file *f, char __user *buf,1235 size_t size, loff_t *pos)1236{1237 struct amdgpu_device *adev = file_inode(f)->i_private;1238 ssize_t result = 0;1239 int r;1240 1241 if (size & 0x3 || *pos & 0x3)1242 return -EINVAL;1243 1244 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);1245 if (r < 0) {1246 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1247 return r;1248 }1249 1250 while (size) {1251 uint32_t value;1252 1253 r = amdgpu_get_gfx_off_residency(adev, &value);1254 if (r)1255 goto out;1256 1257 r = put_user(value, (uint32_t *)buf);1258 if (r)1259 goto out;1260 1261 result += 4;1262 buf += 4;1263 *pos += 4;1264 size -= 4;1265 }1266 1267 r = result;1268out:1269 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);1270 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1271 1272 return r;1273}1274 1275/**1276 * amdgpu_debugfs_gfxoff_residency_write - Log GFXOFF Residency1277 *1278 * @f: open file handle1279 * @buf: User buffer to write data from1280 * @size: Number of bytes to write1281 * @pos: Offset to seek to1282 *1283 * Write a 32-bit non-zero to start logging; write a 32-bit zero to stop1284 */1285static ssize_t amdgpu_debugfs_gfxoff_residency_write(struct file *f, const char __user *buf,1286 size_t size, loff_t *pos)1287{1288 struct amdgpu_device *adev = file_inode(f)->i_private;1289 ssize_t result = 0;1290 int r;1291 1292 if (size & 0x3 || *pos & 0x3)1293 return -EINVAL;1294 1295 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);1296 if (r < 0) {1297 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1298 return r;1299 }1300 1301 while (size) {1302 u32 value;1303 1304 r = get_user(value, (uint32_t *)buf);1305 if (r)1306 goto out;1307 1308 amdgpu_set_gfx_off_residency(adev, value ? true : false);1309 1310 result += 4;1311 buf += 4;1312 *pos += 4;1313 size -= 4;1314 }1315 1316 r = result;1317out:1318 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);1319 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1320 1321 return r;1322}1323 1324 1325/**1326 * amdgpu_debugfs_gfxoff_count_read - Read GFXOFF entry count1327 *1328 * @f: open file handle1329 * @buf: User buffer to store read data in1330 * @size: Number of bytes to read1331 * @pos: Offset to seek to1332 */1333static ssize_t amdgpu_debugfs_gfxoff_count_read(struct file *f, char __user *buf,1334 size_t size, loff_t *pos)1335{1336 struct amdgpu_device *adev = file_inode(f)->i_private;1337 ssize_t result = 0;1338 int r;1339 1340 if (size & 0x3 || *pos & 0x3)1341 return -EINVAL;1342 1343 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);1344 if (r < 0) {1345 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1346 return r;1347 }1348 1349 while (size) {1350 u64 value = 0;1351 1352 r = amdgpu_get_gfx_off_entrycount(adev, &value);1353 if (r)1354 goto out;1355 1356 r = put_user(value, (u64 *)buf);1357 if (r)1358 goto out;1359 1360 result += 4;1361 buf += 4;1362 *pos += 4;1363 size -= 4;1364 }1365 1366 r = result;1367out:1368 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);1369 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1370 1371 return r;1372}1373 1374/**1375 * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF1376 *1377 * @f: open file handle1378 * @buf: User buffer to write data from1379 * @size: Number of bytes to write1380 * @pos: Offset to seek to1381 *1382 * Write a 32-bit zero to disable or a 32-bit non-zero to enable1383 */1384static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,1385 size_t size, loff_t *pos)1386{1387 struct amdgpu_device *adev = file_inode(f)->i_private;1388 ssize_t result = 0;1389 int r;1390 1391 if (size & 0x3 || *pos & 0x3)1392 return -EINVAL;1393 1394 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);1395 if (r < 0) {1396 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1397 return r;1398 }1399 1400 while (size) {1401 uint32_t value;1402 1403 r = get_user(value, (uint32_t *)buf);1404 if (r)1405 goto out;1406 1407 amdgpu_gfx_off_ctrl(adev, value ? true : false);1408 1409 result += 4;1410 buf += 4;1411 *pos += 4;1412 size -= 4;1413 }1414 1415 r = result;1416out:1417 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);1418 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1419 1420 return r;1421}1422 1423 1424/**1425 * amdgpu_debugfs_gfxoff_read - read gfxoff status1426 *1427 * @f: open file handle1428 * @buf: User buffer to store read data in1429 * @size: Number of bytes to read1430 * @pos: Offset to seek to1431 */1432static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,1433 size_t size, loff_t *pos)1434{1435 struct amdgpu_device *adev = file_inode(f)->i_private;1436 ssize_t result = 0;1437 int r;1438 1439 if (size & 0x3 || *pos & 0x3)1440 return -EINVAL;1441 1442 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);1443 if (r < 0) {1444 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1445 return r;1446 }1447 1448 while (size) {1449 u32 value = adev->gfx.gfx_off_state;1450 1451 r = put_user(value, (u32 *)buf);1452 if (r)1453 goto out;1454 1455 result += 4;1456 buf += 4;1457 *pos += 4;1458 size -= 4;1459 }1460 1461 r = result;1462out:1463 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);1464 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1465 1466 return r;1467}1468 1469static ssize_t amdgpu_debugfs_gfxoff_status_read(struct file *f, char __user *buf,1470 size_t size, loff_t *pos)1471{1472 struct amdgpu_device *adev = file_inode(f)->i_private;1473 ssize_t result = 0;1474 int r;1475 1476 if (size & 0x3 || *pos & 0x3)1477 return -EINVAL;1478 1479 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);1480 if (r < 0) {1481 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1482 return r;1483 }1484 1485 while (size) {1486 u32 value;1487 1488 r = amdgpu_get_gfx_off_status(adev, &value);1489 if (r)1490 goto out;1491 1492 r = put_user(value, (u32 *)buf);1493 if (r)1494 goto out;1495 1496 result += 4;1497 buf += 4;1498 *pos += 4;1499 size -= 4;1500 }1501 1502 r = result;1503out:1504 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);1505 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1506 1507 return r;1508}1509 1510static const struct file_operations amdgpu_debugfs_regs2_fops = {1511 .owner = THIS_MODULE,1512 .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,1513 .read = amdgpu_debugfs_regs2_read,1514 .write = amdgpu_debugfs_regs2_write,1515 .open = amdgpu_debugfs_regs2_open,1516 .release = amdgpu_debugfs_regs2_release,1517 .llseek = default_llseek1518};1519 1520static const struct file_operations amdgpu_debugfs_gprwave_fops = {1521 .owner = THIS_MODULE,1522 .unlocked_ioctl = amdgpu_debugfs_gprwave_ioctl,1523 .read = amdgpu_debugfs_gprwave_read,1524 .open = amdgpu_debugfs_gprwave_open,1525 .release = amdgpu_debugfs_gprwave_release,1526 .llseek = default_llseek1527};1528 1529static const struct file_operations amdgpu_debugfs_regs_fops = {1530 .owner = THIS_MODULE,1531 .read = amdgpu_debugfs_regs_read,1532 .write = amdgpu_debugfs_regs_write,1533 .llseek = default_llseek1534};1535static const struct file_operations amdgpu_debugfs_regs_didt_fops = {1536 .owner = THIS_MODULE,1537 .read = amdgpu_debugfs_regs_didt_read,1538 .write = amdgpu_debugfs_regs_didt_write,1539 .llseek = default_llseek1540};1541static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {1542 .owner = THIS_MODULE,1543 .read = amdgpu_debugfs_regs_pcie_read,1544 .write = amdgpu_debugfs_regs_pcie_write,1545 .llseek = default_llseek1546};1547static const struct file_operations amdgpu_debugfs_regs_smc_fops = {1548 .owner = THIS_MODULE,1549 .read = amdgpu_debugfs_regs_smc_read,1550 .write = amdgpu_debugfs_regs_smc_write,1551 .llseek = default_llseek1552};1553 1554static const struct file_operations amdgpu_debugfs_gca_config_fops = {1555 .owner = THIS_MODULE,1556 .read = amdgpu_debugfs_gca_config_read,1557 .llseek = default_llseek1558};1559 1560static const struct file_operations amdgpu_debugfs_sensors_fops = {1561 .owner = THIS_MODULE,1562 .read = amdgpu_debugfs_sensor_read,1563 .llseek = default_llseek1564};1565 1566static const struct file_operations amdgpu_debugfs_wave_fops = {1567 .owner = THIS_MODULE,1568 .read = amdgpu_debugfs_wave_read,1569 .llseek = default_llseek1570};1571static const struct file_operations amdgpu_debugfs_gpr_fops = {1572 .owner = THIS_MODULE,1573 .read = amdgpu_debugfs_gpr_read,1574 .llseek = default_llseek1575};1576 1577static const struct file_operations amdgpu_debugfs_gfxoff_fops = {1578 .owner = THIS_MODULE,1579 .read = amdgpu_debugfs_gfxoff_read,1580 .write = amdgpu_debugfs_gfxoff_write,1581 .llseek = default_llseek1582};1583 1584static const struct file_operations amdgpu_debugfs_gfxoff_status_fops = {1585 .owner = THIS_MODULE,1586 .read = amdgpu_debugfs_gfxoff_status_read,1587 .llseek = default_llseek1588};1589 1590static const struct file_operations amdgpu_debugfs_gfxoff_count_fops = {1591 .owner = THIS_MODULE,1592 .read = amdgpu_debugfs_gfxoff_count_read,1593 .llseek = default_llseek1594};1595 1596static const struct file_operations amdgpu_debugfs_gfxoff_residency_fops = {1597 .owner = THIS_MODULE,1598 .read = amdgpu_debugfs_gfxoff_residency_read,1599 .write = amdgpu_debugfs_gfxoff_residency_write,1600 .llseek = default_llseek1601};1602 1603static const struct file_operations *debugfs_regs[] = {1604 &amdgpu_debugfs_regs_fops,1605 &amdgpu_debugfs_regs2_fops,1606 &amdgpu_debugfs_gprwave_fops,1607 &amdgpu_debugfs_regs_didt_fops,1608 &amdgpu_debugfs_regs_pcie_fops,1609 &amdgpu_debugfs_regs_smc_fops,1610 &amdgpu_debugfs_gca_config_fops,1611 &amdgpu_debugfs_sensors_fops,1612 &amdgpu_debugfs_wave_fops,1613 &amdgpu_debugfs_gpr_fops,1614 &amdgpu_debugfs_gfxoff_fops,1615 &amdgpu_debugfs_gfxoff_status_fops,1616 &amdgpu_debugfs_gfxoff_count_fops,1617 &amdgpu_debugfs_gfxoff_residency_fops,1618};1619 1620static const char * const debugfs_regs_names[] = {1621 "amdgpu_regs",1622 "amdgpu_regs2",1623 "amdgpu_gprwave",1624 "amdgpu_regs_didt",1625 "amdgpu_regs_pcie",1626 "amdgpu_regs_smc",1627 "amdgpu_gca_config",1628 "amdgpu_sensors",1629 "amdgpu_wave",1630 "amdgpu_gpr",1631 "amdgpu_gfxoff",1632 "amdgpu_gfxoff_status",1633 "amdgpu_gfxoff_count",1634 "amdgpu_gfxoff_residency",1635};1636 1637/**1638 * amdgpu_debugfs_regs_init - Initialize debugfs entries that provide1639 * register access.1640 *1641 * @adev: The device to attach the debugfs entries to1642 */1643int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)1644{1645 struct drm_minor *minor = adev_to_drm(adev)->primary;1646 struct dentry *ent, *root = minor->debugfs_root;1647 unsigned int i;1648 1649 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {1650 ent = debugfs_create_file(debugfs_regs_names[i],1651 S_IFREG | 0400, root,1652 adev, debugfs_regs[i]);1653 if (!i && !IS_ERR_OR_NULL(ent))1654 i_size_write(ent->d_inode, adev->rmmio_size);1655 }1656 1657 return 0;1658}1659 1660static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)1661{1662 struct amdgpu_device *adev = m->private;1663 struct drm_device *dev = adev_to_drm(adev);1664 int r = 0, i;1665 1666 r = pm_runtime_get_sync(dev->dev);1667 if (r < 0) {1668 pm_runtime_put_autosuspend(dev->dev);1669 return r;1670 }1671 1672 /* Avoid accidently unparking the sched thread during GPU reset */1673 r = down_write_killable(&adev->reset_domain->sem);1674 if (r)1675 return r;1676 1677 /* hold on the scheduler */1678 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {1679 struct amdgpu_ring *ring = adev->rings[i];1680 1681 if (!amdgpu_ring_sched_ready(ring))1682 continue;1683 drm_sched_wqueue_stop(&ring->sched);1684 }1685 1686 seq_puts(m, "run ib test:\n");1687 r = amdgpu_ib_ring_tests(adev);1688 if (r)1689 seq_printf(m, "ib ring tests failed (%d).\n", r);1690 else1691 seq_puts(m, "ib ring tests passed.\n");1692 1693 /* go on the scheduler */1694 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {1695 struct amdgpu_ring *ring = adev->rings[i];1696 1697 if (!amdgpu_ring_sched_ready(ring))1698 continue;1699 drm_sched_wqueue_start(&ring->sched);1700 }1701 1702 up_write(&adev->reset_domain->sem);1703 1704 pm_runtime_mark_last_busy(dev->dev);1705 pm_runtime_put_autosuspend(dev->dev);1706 1707 return 0;1708}1709 1710static int amdgpu_debugfs_evict_vram(void *data, u64 *val)1711{1712 struct amdgpu_device *adev = (struct amdgpu_device *)data;1713 struct drm_device *dev = adev_to_drm(adev);1714 int r;1715 1716 r = pm_runtime_get_sync(dev->dev);1717 if (r < 0) {1718 pm_runtime_put_autosuspend(dev->dev);1719 return r;1720 }1721 1722 *val = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);1723 1724 pm_runtime_mark_last_busy(dev->dev);1725 pm_runtime_put_autosuspend(dev->dev);1726 1727 return 0;1728}1729 1730 1731static int amdgpu_debugfs_evict_gtt(void *data, u64 *val)1732{1733 struct amdgpu_device *adev = (struct amdgpu_device *)data;1734 struct drm_device *dev = adev_to_drm(adev);1735 int r;1736 1737 r = pm_runtime_get_sync(dev->dev);1738 if (r < 0) {1739 pm_runtime_put_autosuspend(dev->dev);1740 return r;1741 }1742 1743 *val = amdgpu_ttm_evict_resources(adev, TTM_PL_TT);1744 1745 pm_runtime_mark_last_busy(dev->dev);1746 pm_runtime_put_autosuspend(dev->dev);1747 1748 return 0;1749}1750 1751static int amdgpu_debugfs_benchmark(void *data, u64 val)1752{1753 struct amdgpu_device *adev = (struct amdgpu_device *)data;1754 struct drm_device *dev = adev_to_drm(adev);1755 int r;1756 1757 r = pm_runtime_get_sync(dev->dev);1758 if (r < 0) {1759 pm_runtime_put_autosuspend(dev->dev);1760 return r;1761 }1762 1763 r = amdgpu_benchmark(adev, val);1764 1765 pm_runtime_mark_last_busy(dev->dev);1766 pm_runtime_put_autosuspend(dev->dev);1767 1768 return r;1769}1770 1771static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused)1772{1773 struct amdgpu_device *adev = m->private;1774 struct drm_device *dev = adev_to_drm(adev);1775 struct drm_file *file;1776 int r;1777 1778 r = mutex_lock_interruptible(&dev->filelist_mutex);1779 if (r)1780 return r;1781 1782 list_for_each_entry(file, &dev->filelist, lhead) {1783 struct amdgpu_fpriv *fpriv = file->driver_priv;1784 struct amdgpu_vm *vm = &fpriv->vm;1785 struct amdgpu_task_info *ti;1786 1787 ti = amdgpu_vm_get_task_info_vm(vm);1788 if (ti) {1789 seq_printf(m, "pid:%d\tProcess:%s ----------\n", ti->pid, ti->process_name);1790 amdgpu_vm_put_task_info(ti);1791 }1792 1793 r = amdgpu_bo_reserve(vm->root.bo, true);1794 if (r)1795 break;1796 amdgpu_debugfs_vm_bo_info(vm, m);1797 amdgpu_bo_unreserve(vm->root.bo);1798 }1799 1800 mutex_unlock(&dev->filelist_mutex);1801 1802 return r;1803}1804 1805DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib);1806DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info);1807DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram,1808 NULL, "%lld\n");1809DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt,1810 NULL, "%lld\n");1811DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_benchmark_fops, NULL, amdgpu_debugfs_benchmark,1812 "%lld\n");1813 1814static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,1815 struct dma_fence **fences)1816{1817 struct amdgpu_fence_driver *drv = &ring->fence_drv;1818 uint32_t sync_seq, last_seq;1819 1820 last_seq = atomic_read(&ring->fence_drv.last_seq);1821 sync_seq = ring->fence_drv.sync_seq;1822 1823 last_seq &= drv->num_fences_mask;1824 sync_seq &= drv->num_fences_mask;1825 1826 do {1827 struct dma_fence *fence, **ptr;1828 1829 ++last_seq;1830 last_seq &= drv->num_fences_mask;1831 ptr = &drv->fences[last_seq];1832 1833 fence = rcu_dereference_protected(*ptr, 1);1834 RCU_INIT_POINTER(*ptr, NULL);1835 1836 if (!fence)1837 continue;1838 1839 fences[last_seq] = fence;1840 1841 } while (last_seq != sync_seq);1842}1843 1844static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,1845 int length)1846{1847 int i;1848 struct dma_fence *fence;1849 1850 for (i = 0; i < length; i++) {1851 fence = fences[i];1852 if (!fence)1853 continue;1854 dma_fence_signal(fence);1855 dma_fence_put(fence);1856 }1857}1858 1859static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)1860{1861 struct drm_sched_job *s_job;1862 struct dma_fence *fence;1863 1864 spin_lock(&sched->job_list_lock);1865 list_for_each_entry(s_job, &sched->pending_list, list) {1866 fence = sched->ops->run_job(s_job);1867 dma_fence_put(fence);1868 }1869 spin_unlock(&sched->job_list_lock);1870}1871 1872static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)1873{1874 struct amdgpu_job *job;1875 struct drm_sched_job *s_job, *tmp;1876 uint32_t preempt_seq;1877 struct dma_fence *fence, **ptr;1878 struct amdgpu_fence_driver *drv = &ring->fence_drv;1879 struct drm_gpu_scheduler *sched = &ring->sched;1880 bool preempted = true;1881 1882 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)1883 return;1884 1885 preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));1886 if (preempt_seq <= atomic_read(&drv->last_seq)) {1887 preempted = false;1888 goto no_preempt;1889 }1890 1891 preempt_seq &= drv->num_fences_mask;1892 ptr = &drv->fences[preempt_seq];1893 fence = rcu_dereference_protected(*ptr, 1);1894 1895no_preempt:1896 spin_lock(&sched->job_list_lock);1897 list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {1898 if (dma_fence_is_signaled(&s_job->s_fence->finished)) {1899 /* remove job from ring_mirror_list */1900 list_del_init(&s_job->list);1901 sched->ops->free_job(s_job);1902 continue;1903 }1904 job = to_amdgpu_job(s_job);1905 if (preempted && (&job->hw_fence) == fence)1906 /* mark the job as preempted */1907 job->preemption_status |= AMDGPU_IB_PREEMPTED;1908 }1909 spin_unlock(&sched->job_list_lock);1910}1911 1912static int amdgpu_debugfs_ib_preempt(void *data, u64 val)1913{1914 int r, length;1915 struct amdgpu_ring *ring;1916 struct dma_fence **fences = NULL;1917 struct amdgpu_device *adev = (struct amdgpu_device *)data;1918 1919 if (val >= AMDGPU_MAX_RINGS)1920 return -EINVAL;1921 1922 ring = adev->rings[val];1923 1924 if (!amdgpu_ring_sched_ready(ring) ||1925 !ring->funcs->preempt_ib)1926 return -EINVAL;1927 1928 /* the last preemption failed */1929 if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))1930 return -EBUSY;1931 1932 length = ring->fence_drv.num_fences_mask + 1;1933 fences = kcalloc(length, sizeof(void *), GFP_KERNEL);1934 if (!fences)1935 return -ENOMEM;1936 1937 /* Avoid accidently unparking the sched thread during GPU reset */1938 r = down_read_killable(&adev->reset_domain->sem);1939 if (r)1940 goto pro_end;1941 1942 /* stop the scheduler */1943 drm_sched_wqueue_stop(&ring->sched);1944 1945 /* preempt the IB */1946 r = amdgpu_ring_preempt_ib(ring);1947 if (r) {1948 DRM_WARN("failed to preempt ring %d\n", ring->idx);1949 goto failure;1950 }1951 1952 amdgpu_fence_process(ring);1953 1954 if (atomic_read(&ring->fence_drv.last_seq) !=1955 ring->fence_drv.sync_seq) {1956 DRM_INFO("ring %d was preempted\n", ring->idx);1957 1958 amdgpu_ib_preempt_mark_partial_job(ring);1959 1960 /* swap out the old fences */1961 amdgpu_ib_preempt_fences_swap(ring, fences);1962 1963 amdgpu_fence_driver_force_completion(ring);1964 1965 /* resubmit unfinished jobs */1966 amdgpu_ib_preempt_job_recovery(&ring->sched);1967 1968 /* wait for jobs finished */1969 amdgpu_fence_wait_empty(ring);1970 1971 /* signal the old fences */1972 amdgpu_ib_preempt_signal_fences(fences, length);1973 }1974 1975failure:1976 /* restart the scheduler */1977 drm_sched_wqueue_start(&ring->sched);1978 1979 up_read(&adev->reset_domain->sem);1980 1981pro_end:1982 kfree(fences);1983 1984 return r;1985}1986 1987static int amdgpu_debugfs_sclk_set(void *data, u64 val)1988{1989 int ret = 0;1990 uint32_t max_freq, min_freq;1991 struct amdgpu_device *adev = (struct amdgpu_device *)data;1992 1993 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))1994 return -EINVAL;1995 1996 ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);1997 if (ret < 0) {1998 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);1999 return ret;2000 }2001 2002 ret = amdgpu_dpm_get_dpm_freq_range(adev, PP_SCLK, &min_freq, &max_freq);2003 if (ret == -EOPNOTSUPP) {2004 ret = 0;2005 goto out;2006 }2007 if (ret || val > max_freq || val < min_freq) {2008 ret = -EINVAL;2009 goto out;2010 }2011 2012 ret = amdgpu_dpm_set_soft_freq_range(adev, PP_SCLK, (uint32_t)val, (uint32_t)val);2013 if (ret)2014 ret = -EINVAL;2015 2016out:2017 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);2018 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);2019 2020 return ret;2021}2022 2023DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL,2024 amdgpu_debugfs_ib_preempt, "%llu\n");2025 2026DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL,2027 amdgpu_debugfs_sclk_set, "%llu\n");2028 2029int amdgpu_debugfs_init(struct amdgpu_device *adev)2030{2031 struct dentry *root = adev_to_drm(adev)->primary->debugfs_root;2032 struct dentry *ent;2033 int r, i;2034 2035 if (!debugfs_initialized())2036 return 0;2037 2038 debugfs_create_x32("amdgpu_smu_debug", 0600, root,2039 &adev->pm.smu_debug_mask);2040 2041 ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev,2042 &fops_ib_preempt);2043 if (IS_ERR(ent)) {2044 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");2045 return PTR_ERR(ent);2046 }2047 2048 ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev,2049 &fops_sclk_set);2050 if (IS_ERR(ent)) {2051 DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");2052 return PTR_ERR(ent);2053 }2054 2055 /* Register debugfs entries for amdgpu_ttm */2056 amdgpu_ttm_debugfs_init(adev);2057 amdgpu_debugfs_pm_init(adev);2058 amdgpu_debugfs_sa_init(adev);2059 amdgpu_debugfs_fence_init(adev);2060 amdgpu_debugfs_gem_init(adev);2061 2062 r = amdgpu_debugfs_regs_init(adev);2063 if (r)2064 DRM_ERROR("registering register debugfs failed (%d).\n", r);2065 2066 amdgpu_debugfs_firmware_init(adev);2067 amdgpu_ta_if_debugfs_init(adev);2068 2069 amdgpu_debugfs_mes_event_log_init(adev);2070 2071#if defined(CONFIG_DRM_AMD_DC)2072 if (adev->dc_enabled)2073 dtn_debugfs_init(adev);2074#endif2075 2076 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {2077 struct amdgpu_ring *ring = adev->rings[i];2078 2079 if (!ring)2080 continue;2081 2082 amdgpu_debugfs_ring_init(adev, ring);2083 }2084 2085 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {2086 if (!amdgpu_vcnfw_log)2087 break;2088 2089 if (adev->vcn.harvest_config & (1 << i))2090 continue;2091 2092 amdgpu_debugfs_vcn_fwlog_init(adev, i, &adev->vcn.inst[i]);2093 }2094 2095 if (amdgpu_umsch_mm & amdgpu_umsch_mm_fwlog)2096 amdgpu_debugfs_umsch_fwlog_init(adev, &adev->umsch_mm);2097 2098 amdgpu_ras_debugfs_create_all(adev);2099 amdgpu_rap_debugfs_init(adev);2100 amdgpu_securedisplay_debugfs_init(adev);2101 amdgpu_fw_attestation_debugfs_init(adev);2102 2103 debugfs_create_file("amdgpu_evict_vram", 0400, root, adev,2104 &amdgpu_evict_vram_fops);2105 debugfs_create_file("amdgpu_evict_gtt", 0400, root, adev,2106 &amdgpu_evict_gtt_fops);2107 debugfs_create_file("amdgpu_test_ib", 0400, root, adev,2108 &amdgpu_debugfs_test_ib_fops);2109 debugfs_create_file("amdgpu_vm_info", 0444, root, adev,2110 &amdgpu_debugfs_vm_info_fops);2111 debugfs_create_file("amdgpu_benchmark", 0200, root, adev,2112 &amdgpu_benchmark_fops);2113 2114 adev->debugfs_vbios_blob.data = adev->bios;2115 adev->debugfs_vbios_blob.size = adev->bios_size;2116 debugfs_create_blob("amdgpu_vbios", 0444, root,2117 &adev->debugfs_vbios_blob);2118 2119 adev->debugfs_discovery_blob.data = adev->mman.discovery_bin;2120 adev->debugfs_discovery_blob.size = adev->mman.discovery_tmr_size;2121 debugfs_create_blob("amdgpu_discovery", 0444, root,2122 &adev->debugfs_discovery_blob);2123 2124 return 0;2125}2126 2127#else2128int amdgpu_debugfs_init(struct amdgpu_device *adev)2129{2130 return 0;2131}2132int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)2133{2134 return 0;2135}2136#endif2137