1503 lines · c
1/*2 * Copyright 2018 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 *23 */24#include <linux/list.h>25#include "amdgpu.h"26#include "amdgpu_xgmi.h"27#include "amdgpu_ras.h"28#include "soc15.h"29#include "df/df_3_6_offset.h"30#include "xgmi/xgmi_4_0_0_smn.h"31#include "xgmi/xgmi_4_0_0_sh_mask.h"32#include "xgmi/xgmi_6_1_0_sh_mask.h"33#include "wafl/wafl2_4_0_0_smn.h"34#include "wafl/wafl2_4_0_0_sh_mask.h"35 36#include "amdgpu_reset.h"37 38#define smnPCS_XGMI3X16_PCS_ERROR_STATUS 0x11a0020c39#define smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK 0x11a0021840#define smnPCS_GOPX1_PCS_ERROR_STATUS 0x1220021041#define smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK 0x1220021842 43static DEFINE_MUTEX(xgmi_mutex);44 45#define AMDGPU_MAX_XGMI_DEVICE_PER_HIVE 446 47static LIST_HEAD(xgmi_hive_list);48 49static const int xgmi_pcs_err_status_reg_vg20[] = {50 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS,51 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x100000,52};53 54static const int wafl_pcs_err_status_reg_vg20[] = {55 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS,56 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS + 0x100000,57};58 59static const int xgmi_pcs_err_status_reg_arct[] = {60 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS,61 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x100000,62 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x500000,63 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x600000,64 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x700000,65 smnXGMI0_PCS_GOPX16_PCS_ERROR_STATUS + 0x800000,66};67 68/* same as vg20*/69static const int wafl_pcs_err_status_reg_arct[] = {70 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS,71 smnPCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS + 0x100000,72};73 74static const int xgmi3x16_pcs_err_status_reg_aldebaran[] = {75 smnPCS_XGMI3X16_PCS_ERROR_STATUS,76 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x100000,77 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x200000,78 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x300000,79 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x400000,80 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x500000,81 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x600000,82 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x70000083};84 85static const int xgmi3x16_pcs_err_noncorrectable_mask_reg_aldebaran[] = {86 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK,87 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000,88 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x200000,89 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x300000,90 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x400000,91 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x500000,92 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x600000,93 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x70000094};95 96static const int walf_pcs_err_status_reg_aldebaran[] = {97 smnPCS_GOPX1_PCS_ERROR_STATUS,98 smnPCS_GOPX1_PCS_ERROR_STATUS + 0x10000099};100 101static const int walf_pcs_err_noncorrectable_mask_reg_aldebaran[] = {102 smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK,103 smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000104};105 106static const int xgmi3x16_pcs_err_status_reg_v6_4[] = {107 smnPCS_XGMI3X16_PCS_ERROR_STATUS,108 smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x100000109};110 111static const int xgmi3x16_pcs_err_noncorrectable_mask_reg_v6_4[] = {112 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK,113 smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000114};115 116static const u64 xgmi_v6_4_0_mca_base_array[] = {117 0x11a09200,118 0x11b09200,119};120 121static const char *xgmi_v6_4_0_ras_error_code_ext[32] = {122 [0x00] = "XGMI PCS DataLossErr",123 [0x01] = "XGMI PCS TrainingErr",124 [0x02] = "XGMI PCS FlowCtrlAckErr",125 [0x03] = "XGMI PCS RxFifoUnderflowErr",126 [0x04] = "XGMI PCS RxFifoOverflowErr",127 [0x05] = "XGMI PCS CRCErr",128 [0x06] = "XGMI PCS BERExceededErr",129 [0x07] = "XGMI PCS TxMetaDataErr",130 [0x08] = "XGMI PCS ReplayBufParityErr",131 [0x09] = "XGMI PCS DataParityErr",132 [0x0a] = "XGMI PCS ReplayFifoOverflowErr",133 [0x0b] = "XGMI PCS ReplayFifoUnderflowErr",134 [0x0c] = "XGMI PCS ElasticFifoOverflowErr",135 [0x0d] = "XGMI PCS DeskewErr",136 [0x0e] = "XGMI PCS FlowCtrlCRCErr",137 [0x0f] = "XGMI PCS DataStartupLimitErr",138 [0x10] = "XGMI PCS FCInitTimeoutErr",139 [0x11] = "XGMI PCS RecoveryTimeoutErr",140 [0x12] = "XGMI PCS ReadySerialTimeoutErr",141 [0x13] = "XGMI PCS ReadySerialAttemptErr",142 [0x14] = "XGMI PCS RecoveryAttemptErr",143 [0x15] = "XGMI PCS RecoveryRelockAttemptErr",144 [0x16] = "XGMI PCS ReplayAttemptErr",145 [0x17] = "XGMI PCS SyncHdrErr",146 [0x18] = "XGMI PCS TxReplayTimeoutErr",147 [0x19] = "XGMI PCS RxReplayTimeoutErr",148 [0x1a] = "XGMI PCS LinkSubTxTimeoutErr",149 [0x1b] = "XGMI PCS LinkSubRxTimeoutErr",150 [0x1c] = "XGMI PCS RxCMDPktErr",151};152 153static const struct amdgpu_pcs_ras_field xgmi_pcs_ras_fields[] = {154 {"XGMI PCS DataLossErr",155 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataLossErr)},156 {"XGMI PCS TrainingErr",157 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, TrainingErr)},158 {"XGMI PCS CRCErr",159 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, CRCErr)},160 {"XGMI PCS BERExceededErr",161 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, BERExceededErr)},162 {"XGMI PCS TxMetaDataErr",163 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, TxMetaDataErr)},164 {"XGMI PCS ReplayBufParityErr",165 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayBufParityErr)},166 {"XGMI PCS DataParityErr",167 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataParityErr)},168 {"XGMI PCS ReplayFifoOverflowErr",169 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},170 {"XGMI PCS ReplayFifoUnderflowErr",171 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},172 {"XGMI PCS ElasticFifoOverflowErr",173 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},174 {"XGMI PCS DeskewErr",175 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DeskewErr)},176 {"XGMI PCS DataStartupLimitErr",177 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataStartupLimitErr)},178 {"XGMI PCS FCInitTimeoutErr",179 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, FCInitTimeoutErr)},180 {"XGMI PCS RecoveryTimeoutErr",181 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryTimeoutErr)},182 {"XGMI PCS ReadySerialTimeoutErr",183 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},184 {"XGMI PCS ReadySerialAttemptErr",185 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, ReadySerialAttemptErr)},186 {"XGMI PCS RecoveryAttemptErr",187 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryAttemptErr)},188 {"XGMI PCS RecoveryRelockAttemptErr",189 SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},190};191 192static const struct amdgpu_pcs_ras_field wafl_pcs_ras_fields[] = {193 {"WAFL PCS DataLossErr",194 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataLossErr)},195 {"WAFL PCS TrainingErr",196 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, TrainingErr)},197 {"WAFL PCS CRCErr",198 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, CRCErr)},199 {"WAFL PCS BERExceededErr",200 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, BERExceededErr)},201 {"WAFL PCS TxMetaDataErr",202 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, TxMetaDataErr)},203 {"WAFL PCS ReplayBufParityErr",204 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayBufParityErr)},205 {"WAFL PCS DataParityErr",206 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataParityErr)},207 {"WAFL PCS ReplayFifoOverflowErr",208 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},209 {"WAFL PCS ReplayFifoUnderflowErr",210 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},211 {"WAFL PCS ElasticFifoOverflowErr",212 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},213 {"WAFL PCS DeskewErr",214 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DeskewErr)},215 {"WAFL PCS DataStartupLimitErr",216 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, DataStartupLimitErr)},217 {"WAFL PCS FCInitTimeoutErr",218 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, FCInitTimeoutErr)},219 {"WAFL PCS RecoveryTimeoutErr",220 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryTimeoutErr)},221 {"WAFL PCS ReadySerialTimeoutErr",222 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},223 {"WAFL PCS ReadySerialAttemptErr",224 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, ReadySerialAttemptErr)},225 {"WAFL PCS RecoveryAttemptErr",226 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryAttemptErr)},227 {"WAFL PCS RecoveryRelockAttemptErr",228 SOC15_REG_FIELD(PCS_GOPX1_0_PCS_GOPX1_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},229};230 231static const struct amdgpu_pcs_ras_field xgmi3x16_pcs_ras_fields[] = {232 {"XGMI3X16 PCS DataLossErr",233 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataLossErr)},234 {"XGMI3X16 PCS TrainingErr",235 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TrainingErr)},236 {"XGMI3X16 PCS FlowCtrlAckErr",237 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FlowCtrlAckErr)},238 {"XGMI3X16 PCS RxFifoUnderflowErr",239 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxFifoUnderflowErr)},240 {"XGMI3X16 PCS RxFifoOverflowErr",241 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxFifoOverflowErr)},242 {"XGMI3X16 PCS CRCErr",243 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, CRCErr)},244 {"XGMI3X16 PCS BERExceededErr",245 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, BERExceededErr)},246 {"XGMI3X16 PCS TxVcidDataErr",247 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TxVcidDataErr)},248 {"XGMI3X16 PCS ReplayBufParityErr",249 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayBufParityErr)},250 {"XGMI3X16 PCS DataParityErr",251 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataParityErr)},252 {"XGMI3X16 PCS ReplayFifoOverflowErr",253 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayFifoOverflowErr)},254 {"XGMI3X16 PCS ReplayFifoUnderflowErr",255 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayFifoUnderflowErr)},256 {"XGMI3X16 PCS ElasticFifoOverflowErr",257 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ElasticFifoOverflowErr)},258 {"XGMI3X16 PCS DeskewErr",259 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DeskewErr)},260 {"XGMI3X16 PCS FlowCtrlCRCErr",261 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FlowCtrlCRCErr)},262 {"XGMI3X16 PCS DataStartupLimitErr",263 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, DataStartupLimitErr)},264 {"XGMI3X16 PCS FCInitTimeoutErr",265 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, FCInitTimeoutErr)},266 {"XGMI3X16 PCS RecoveryTimeoutErr",267 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryTimeoutErr)},268 {"XGMI3X16 PCS ReadySerialTimeoutErr",269 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReadySerialTimeoutErr)},270 {"XGMI3X16 PCS ReadySerialAttemptErr",271 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReadySerialAttemptErr)},272 {"XGMI3X16 PCS RecoveryAttemptErr",273 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryAttemptErr)},274 {"XGMI3X16 PCS RecoveryRelockAttemptErr",275 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RecoveryRelockAttemptErr)},276 {"XGMI3X16 PCS ReplayAttemptErr",277 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, ReplayAttemptErr)},278 {"XGMI3X16 PCS SyncHdrErr",279 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, SyncHdrErr)},280 {"XGMI3X16 PCS TxReplayTimeoutErr",281 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, TxReplayTimeoutErr)},282 {"XGMI3X16 PCS RxReplayTimeoutErr",283 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxReplayTimeoutErr)},284 {"XGMI3X16 PCS LinkSubTxTimeoutErr",285 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, LinkSubTxTimeoutErr)},286 {"XGMI3X16 PCS LinkSubRxTimeoutErr",287 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, LinkSubRxTimeoutErr)},288 {"XGMI3X16 PCS RxCMDPktErr",289 SOC15_REG_FIELD(PCS_XGMI3X16_PCS_ERROR_STATUS, RxCMDPktErr)},290};291 292/**293 * DOC: AMDGPU XGMI Support294 *295 * XGMI is a high speed interconnect that joins multiple GPU cards296 * into a homogeneous memory space that is organized by a collective297 * hive ID and individual node IDs, both of which are 64-bit numbers.298 *299 * The file xgmi_device_id contains the unique per GPU device ID and300 * is stored in the /sys/class/drm/card${cardno}/device/ directory.301 *302 * Inside the device directory a sub-directory 'xgmi_hive_info' is303 * created which contains the hive ID and the list of nodes.304 *305 * The hive ID is stored in:306 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/xgmi_hive_id307 *308 * The node information is stored in numbered directories:309 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/node${nodeno}/xgmi_device_id310 *311 * Each device has their own xgmi_hive_info direction with a mirror312 * set of node sub-directories.313 *314 * The XGMI memory space is built by contiguously adding the power of315 * two padded VRAM space from each node to each other.316 *317 */318 319static struct attribute amdgpu_xgmi_hive_id = {320 .name = "xgmi_hive_id",321 .mode = S_IRUGO322};323 324static struct attribute *amdgpu_xgmi_hive_attrs[] = {325 &amdgpu_xgmi_hive_id,326 NULL327};328ATTRIBUTE_GROUPS(amdgpu_xgmi_hive);329 330static ssize_t amdgpu_xgmi_show_attrs(struct kobject *kobj,331 struct attribute *attr, char *buf)332{333 struct amdgpu_hive_info *hive = container_of(334 kobj, struct amdgpu_hive_info, kobj);335 336 if (attr == &amdgpu_xgmi_hive_id)337 return snprintf(buf, PAGE_SIZE, "%llu\n", hive->hive_id);338 339 return 0;340}341 342static void amdgpu_xgmi_hive_release(struct kobject *kobj)343{344 struct amdgpu_hive_info *hive = container_of(345 kobj, struct amdgpu_hive_info, kobj);346 347 amdgpu_reset_put_reset_domain(hive->reset_domain);348 hive->reset_domain = NULL;349 350 mutex_destroy(&hive->hive_lock);351 kfree(hive);352}353 354static const struct sysfs_ops amdgpu_xgmi_hive_ops = {355 .show = amdgpu_xgmi_show_attrs,356};357 358static const struct kobj_type amdgpu_xgmi_hive_type = {359 .release = amdgpu_xgmi_hive_release,360 .sysfs_ops = &amdgpu_xgmi_hive_ops,361 .default_groups = amdgpu_xgmi_hive_groups,362};363 364static ssize_t amdgpu_xgmi_show_device_id(struct device *dev,365 struct device_attribute *attr,366 char *buf)367{368 struct drm_device *ddev = dev_get_drvdata(dev);369 struct amdgpu_device *adev = drm_to_adev(ddev);370 371 return sysfs_emit(buf, "%llu\n", adev->gmc.xgmi.node_id);372 373}374 375static ssize_t amdgpu_xgmi_show_physical_id(struct device *dev,376 struct device_attribute *attr,377 char *buf)378{379 struct drm_device *ddev = dev_get_drvdata(dev);380 struct amdgpu_device *adev = drm_to_adev(ddev);381 382 return sysfs_emit(buf, "%u\n", adev->gmc.xgmi.physical_node_id);383 384}385 386static ssize_t amdgpu_xgmi_show_num_hops(struct device *dev,387 struct device_attribute *attr,388 char *buf)389{390 struct drm_device *ddev = dev_get_drvdata(dev);391 struct amdgpu_device *adev = drm_to_adev(ddev);392 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;393 int i;394 395 for (i = 0; i < top->num_nodes; i++)396 sprintf(buf + 3 * i, "%02x ", top->nodes[i].num_hops);397 398 return sysfs_emit(buf, "%s\n", buf);399}400 401static ssize_t amdgpu_xgmi_show_num_links(struct device *dev,402 struct device_attribute *attr,403 char *buf)404{405 struct drm_device *ddev = dev_get_drvdata(dev);406 struct amdgpu_device *adev = drm_to_adev(ddev);407 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;408 int i;409 410 for (i = 0; i < top->num_nodes; i++)411 sprintf(buf + 3 * i, "%02x ", top->nodes[i].num_links);412 413 return sysfs_emit(buf, "%s\n", buf);414}415 416static ssize_t amdgpu_xgmi_show_connected_port_num(struct device *dev,417 struct device_attribute *attr,418 char *buf)419{420 struct drm_device *ddev = dev_get_drvdata(dev);421 struct amdgpu_device *adev = drm_to_adev(ddev);422 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;423 int i, j, size = 0;424 int current_node;425 /*426 * get the node id in the sysfs for the current socket and show427 * it in the port num info output in the sysfs for easy reading.428 * it is NOT the one retrieved from xgmi ta.429 */430 for (i = 0; i < top->num_nodes; i++) {431 if (top->nodes[i].node_id == adev->gmc.xgmi.node_id) {432 current_node = i;433 break;434 }435 }436 437 if (i == top->num_nodes)438 return -EINVAL;439 440 for (i = 0; i < top->num_nodes; i++) {441 for (j = 0; j < top->nodes[i].num_links; j++)442 /* node id in sysfs starts from 1 rather than 0 so +1 here */443 size += sysfs_emit_at(buf, size, "%02x:%02x -> %02x:%02x\n", current_node + 1,444 top->nodes[i].port_num[j].src_xgmi_port_num, i + 1,445 top->nodes[i].port_num[j].dst_xgmi_port_num);446 }447 448 return size;449}450 451#define AMDGPU_XGMI_SET_FICAA(o) ((o) | 0x456801)452static ssize_t amdgpu_xgmi_show_error(struct device *dev,453 struct device_attribute *attr,454 char *buf)455{456 struct drm_device *ddev = dev_get_drvdata(dev);457 struct amdgpu_device *adev = drm_to_adev(ddev);458 uint32_t ficaa_pie_ctl_in, ficaa_pie_status_in;459 uint64_t fica_out;460 unsigned int error_count = 0;461 462 ficaa_pie_ctl_in = AMDGPU_XGMI_SET_FICAA(0x200);463 ficaa_pie_status_in = AMDGPU_XGMI_SET_FICAA(0x208);464 465 if ((!adev->df.funcs) ||466 (!adev->df.funcs->get_fica) ||467 (!adev->df.funcs->set_fica))468 return -EINVAL;469 470 fica_out = adev->df.funcs->get_fica(adev, ficaa_pie_ctl_in);471 if (fica_out != 0x1f)472 pr_err("xGMI error counters not enabled!\n");473 474 fica_out = adev->df.funcs->get_fica(adev, ficaa_pie_status_in);475 476 if ((fica_out & 0xffff) == 2)477 error_count = ((fica_out >> 62) & 0x1) + (fica_out >> 63);478 479 adev->df.funcs->set_fica(adev, ficaa_pie_status_in, 0, 0);480 481 return sysfs_emit(buf, "%u\n", error_count);482}483 484 485static DEVICE_ATTR(xgmi_device_id, S_IRUGO, amdgpu_xgmi_show_device_id, NULL);486static DEVICE_ATTR(xgmi_physical_id, 0444, amdgpu_xgmi_show_physical_id, NULL);487static DEVICE_ATTR(xgmi_error, S_IRUGO, amdgpu_xgmi_show_error, NULL);488static DEVICE_ATTR(xgmi_num_hops, S_IRUGO, amdgpu_xgmi_show_num_hops, NULL);489static DEVICE_ATTR(xgmi_num_links, S_IRUGO, amdgpu_xgmi_show_num_links, NULL);490static DEVICE_ATTR(xgmi_port_num, S_IRUGO, amdgpu_xgmi_show_connected_port_num, NULL);491 492static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev,493 struct amdgpu_hive_info *hive)494{495 int ret = 0;496 char node[10] = { 0 };497 498 /* Create xgmi device id file */499 ret = device_create_file(adev->dev, &dev_attr_xgmi_device_id);500 if (ret) {501 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_device_id\n");502 return ret;503 }504 505 ret = device_create_file(adev->dev, &dev_attr_xgmi_physical_id);506 if (ret) {507 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_physical_id\n");508 return ret;509 }510 511 /* Create xgmi error file */512 ret = device_create_file(adev->dev, &dev_attr_xgmi_error);513 if (ret)514 pr_err("failed to create xgmi_error\n");515 516 /* Create xgmi num hops file */517 ret = device_create_file(adev->dev, &dev_attr_xgmi_num_hops);518 if (ret)519 pr_err("failed to create xgmi_num_hops\n");520 521 /* Create xgmi num links file */522 ret = device_create_file(adev->dev, &dev_attr_xgmi_num_links);523 if (ret)524 pr_err("failed to create xgmi_num_links\n");525 526 /* Create xgmi port num file if supported */527 if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG) {528 ret = device_create_file(adev->dev, &dev_attr_xgmi_port_num);529 if (ret)530 dev_err(adev->dev, "failed to create xgmi_port_num\n");531 }532 533 /* Create sysfs link to hive info folder on the first device */534 if (hive->kobj.parent != (&adev->dev->kobj)) {535 ret = sysfs_create_link(&adev->dev->kobj, &hive->kobj,536 "xgmi_hive_info");537 if (ret) {538 dev_err(adev->dev, "XGMI: Failed to create link to hive info");539 goto remove_file;540 }541 }542 543 sprintf(node, "node%d", atomic_read(&hive->number_devices));544 /* Create sysfs link form the hive folder to yourself */545 ret = sysfs_create_link(&hive->kobj, &adev->dev->kobj, node);546 if (ret) {547 dev_err(adev->dev, "XGMI: Failed to create link from hive info");548 goto remove_link;549 }550 551 goto success;552 553 554remove_link:555 sysfs_remove_link(&adev->dev->kobj, adev_to_drm(adev)->unique);556 557remove_file:558 device_remove_file(adev->dev, &dev_attr_xgmi_device_id);559 device_remove_file(adev->dev, &dev_attr_xgmi_physical_id);560 device_remove_file(adev->dev, &dev_attr_xgmi_error);561 device_remove_file(adev->dev, &dev_attr_xgmi_num_hops);562 device_remove_file(adev->dev, &dev_attr_xgmi_num_links);563 if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG)564 device_remove_file(adev->dev, &dev_attr_xgmi_port_num);565 566success:567 return ret;568}569 570static void amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device *adev,571 struct amdgpu_hive_info *hive)572{573 char node[10];574 memset(node, 0, sizeof(node));575 576 device_remove_file(adev->dev, &dev_attr_xgmi_device_id);577 device_remove_file(adev->dev, &dev_attr_xgmi_physical_id);578 device_remove_file(adev->dev, &dev_attr_xgmi_error);579 device_remove_file(adev->dev, &dev_attr_xgmi_num_hops);580 device_remove_file(adev->dev, &dev_attr_xgmi_num_links);581 if (adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG)582 device_remove_file(adev->dev, &dev_attr_xgmi_port_num);583 584 if (hive->kobj.parent != (&adev->dev->kobj))585 sysfs_remove_link(&adev->dev->kobj,"xgmi_hive_info");586 587 sprintf(node, "node%d", atomic_read(&hive->number_devices));588 sysfs_remove_link(&hive->kobj, node);589 590}591 592 593 594struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev)595{596 struct amdgpu_hive_info *hive = NULL;597 int ret;598 599 if (!adev->gmc.xgmi.hive_id)600 return NULL;601 602 if (adev->hive) {603 kobject_get(&adev->hive->kobj);604 return adev->hive;605 }606 607 mutex_lock(&xgmi_mutex);608 609 list_for_each_entry(hive, &xgmi_hive_list, node) {610 if (hive->hive_id == adev->gmc.xgmi.hive_id)611 goto pro_end;612 }613 614 hive = kzalloc(sizeof(*hive), GFP_KERNEL);615 if (!hive) {616 dev_err(adev->dev, "XGMI: allocation failed\n");617 ret = -ENOMEM;618 hive = NULL;619 goto pro_end;620 }621 622 /* initialize new hive if not exist */623 ret = kobject_init_and_add(&hive->kobj,624 &amdgpu_xgmi_hive_type,625 &adev->dev->kobj,626 "%s", "xgmi_hive_info");627 if (ret) {628 dev_err(adev->dev, "XGMI: failed initializing kobject for xgmi hive\n");629 kobject_put(&hive->kobj);630 hive = NULL;631 goto pro_end;632 }633 634 /**635 * Only init hive->reset_domain for none SRIOV configuration. For SRIOV,636 * Host driver decide how to reset the GPU either through FLR or chain reset.637 * Guest side will get individual notifications from the host for the FLR638 * if necessary.639 */640 if (!amdgpu_sriov_vf(adev)) {641 /**642 * Avoid recreating reset domain when hive is reconstructed for the case643 * of reset the devices in the XGMI hive during probe for passthrough GPU644 * See https://www.spinics.net/lists/amd-gfx/msg58836.html645 */646 if (adev->reset_domain->type != XGMI_HIVE) {647 hive->reset_domain =648 amdgpu_reset_create_reset_domain(XGMI_HIVE, "amdgpu-reset-hive");649 if (!hive->reset_domain) {650 dev_err(adev->dev, "XGMI: failed initializing reset domain for xgmi hive\n");651 ret = -ENOMEM;652 kobject_put(&hive->kobj);653 hive = NULL;654 goto pro_end;655 }656 } else {657 amdgpu_reset_get_reset_domain(adev->reset_domain);658 hive->reset_domain = adev->reset_domain;659 }660 }661 662 hive->hive_id = adev->gmc.xgmi.hive_id;663 INIT_LIST_HEAD(&hive->device_list);664 INIT_LIST_HEAD(&hive->node);665 mutex_init(&hive->hive_lock);666 atomic_set(&hive->number_devices, 0);667 task_barrier_init(&hive->tb);668 hive->pstate = AMDGPU_XGMI_PSTATE_UNKNOWN;669 hive->hi_req_gpu = NULL;670 671 /*672 * hive pstate on boot is high in vega20 so we have to go to low673 * pstate on after boot.674 */675 hive->hi_req_count = AMDGPU_MAX_XGMI_DEVICE_PER_HIVE;676 list_add_tail(&hive->node, &xgmi_hive_list);677 678pro_end:679 if (hive)680 kobject_get(&hive->kobj);681 mutex_unlock(&xgmi_mutex);682 return hive;683}684 685void amdgpu_put_xgmi_hive(struct amdgpu_hive_info *hive)686{687 if (hive)688 kobject_put(&hive->kobj);689}690 691int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate)692{693 int ret = 0;694 struct amdgpu_hive_info *hive;695 struct amdgpu_device *request_adev;696 bool is_hi_req = pstate == AMDGPU_XGMI_PSTATE_MAX_VEGA20;697 bool init_low;698 699 hive = amdgpu_get_xgmi_hive(adev);700 if (!hive)701 return 0;702 703 request_adev = hive->hi_req_gpu ? hive->hi_req_gpu : adev;704 init_low = hive->pstate == AMDGPU_XGMI_PSTATE_UNKNOWN;705 amdgpu_put_xgmi_hive(hive);706 /* fw bug so temporarily disable pstate switching */707 return 0;708 709 if (!hive || adev->asic_type != CHIP_VEGA20)710 return 0;711 712 mutex_lock(&hive->hive_lock);713 714 if (is_hi_req)715 hive->hi_req_count++;716 else717 hive->hi_req_count--;718 719 /*720 * Vega20 only needs single peer to request pstate high for the hive to721 * go high but all peers must request pstate low for the hive to go low722 */723 if (hive->pstate == pstate ||724 (!is_hi_req && hive->hi_req_count && !init_low))725 goto out;726 727 dev_dbg(request_adev->dev, "Set xgmi pstate %d.\n", pstate);728 729 ret = amdgpu_dpm_set_xgmi_pstate(request_adev, pstate);730 if (ret) {731 dev_err(request_adev->dev,732 "XGMI: Set pstate failure on device %llx, hive %llx, ret %d",733 request_adev->gmc.xgmi.node_id,734 request_adev->gmc.xgmi.hive_id, ret);735 goto out;736 }737 738 if (init_low)739 hive->pstate = hive->hi_req_count ?740 hive->pstate : AMDGPU_XGMI_PSTATE_MIN;741 else {742 hive->pstate = pstate;743 hive->hi_req_gpu = pstate != AMDGPU_XGMI_PSTATE_MIN ?744 adev : NULL;745 }746out:747 mutex_unlock(&hive->hive_lock);748 return ret;749}750 751int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_device *adev)752{753 int ret;754 755 if (amdgpu_sriov_vf(adev))756 return 0;757 758 /* Each psp need to set the latest topology */759 ret = psp_xgmi_set_topology_info(&adev->psp,760 atomic_read(&hive->number_devices),761 &adev->psp.xgmi_context.top_info);762 if (ret)763 dev_err(adev->dev,764 "XGMI: Set topology failure on device %llx, hive %llx, ret %d",765 adev->gmc.xgmi.node_id,766 adev->gmc.xgmi.hive_id, ret);767 768 return ret;769}770 771 772/*773 * NOTE psp_xgmi_node_info.num_hops layout is as follows:774 * num_hops[7:6] = link type (0 = xGMI2, 1 = xGMI3, 2/3 = reserved)775 * num_hops[5:3] = reserved776 * num_hops[2:0] = number of hops777 */778int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,779 struct amdgpu_device *peer_adev)780{781 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;782 uint8_t num_hops_mask = 0x7;783 int i;784 785 for (i = 0 ; i < top->num_nodes; ++i)786 if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)787 return top->nodes[i].num_hops & num_hops_mask;788 return -EINVAL;789}790 791int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,792 struct amdgpu_device *peer_adev)793{794 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;795 int i;796 797 for (i = 0 ; i < top->num_nodes; ++i)798 if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)799 return top->nodes[i].num_links;800 return -EINVAL;801}802 803/*804 * Devices that support extended data require the entire hive to initialize with805 * the shared memory buffer flag set.806 *807 * Hive locks and conditions apply - see amdgpu_xgmi_add_device808 */809static int amdgpu_xgmi_initialize_hive_get_data_partition(struct amdgpu_hive_info *hive,810 bool set_extended_data)811{812 struct amdgpu_device *tmp_adev;813 int ret;814 815 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {816 ret = psp_xgmi_initialize(&tmp_adev->psp, set_extended_data, false);817 if (ret) {818 dev_err(tmp_adev->dev,819 "XGMI: Failed to initialize xgmi session for data partition %i\n",820 set_extended_data);821 return ret;822 }823 824 }825 826 return 0;827}828 829static void amdgpu_xgmi_fill_topology_info(struct amdgpu_device *adev,830 struct amdgpu_device *peer_adev)831{832 struct psp_xgmi_topology_info *top_info = &adev->psp.xgmi_context.top_info;833 struct psp_xgmi_topology_info *peer_info = &peer_adev->psp.xgmi_context.top_info;834 835 for (int i = 0; i < peer_info->num_nodes; i++) {836 if (peer_info->nodes[i].node_id == adev->gmc.xgmi.node_id) {837 for (int j = 0; j < top_info->num_nodes; j++) {838 if (top_info->nodes[j].node_id == peer_adev->gmc.xgmi.node_id) {839 peer_info->nodes[i].num_hops = top_info->nodes[j].num_hops;840 peer_info->nodes[i].is_sharing_enabled =841 top_info->nodes[j].is_sharing_enabled;842 peer_info->nodes[i].num_links =843 top_info->nodes[j].num_links;844 return;845 }846 }847 }848 }849}850 851int amdgpu_xgmi_add_device(struct amdgpu_device *adev)852{853 struct psp_xgmi_topology_info *top_info;854 struct amdgpu_hive_info *hive;855 struct amdgpu_xgmi *entry;856 struct amdgpu_device *tmp_adev = NULL;857 858 int count = 0, ret = 0;859 860 if (!adev->gmc.xgmi.supported)861 return 0;862 863 if (!adev->gmc.xgmi.pending_reset &&864 amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) {865 ret = psp_xgmi_initialize(&adev->psp, false, true);866 if (ret) {867 dev_err(adev->dev,868 "XGMI: Failed to initialize xgmi session\n");869 return ret;870 }871 872 ret = psp_xgmi_get_hive_id(&adev->psp, &adev->gmc.xgmi.hive_id);873 if (ret) {874 dev_err(adev->dev,875 "XGMI: Failed to get hive id\n");876 return ret;877 }878 879 ret = psp_xgmi_get_node_id(&adev->psp, &adev->gmc.xgmi.node_id);880 if (ret) {881 dev_err(adev->dev,882 "XGMI: Failed to get node id\n");883 return ret;884 }885 } else {886 adev->gmc.xgmi.hive_id = 16;887 adev->gmc.xgmi.node_id = adev->gmc.xgmi.physical_node_id + 16;888 }889 890 hive = amdgpu_get_xgmi_hive(adev);891 if (!hive) {892 ret = -EINVAL;893 dev_err(adev->dev,894 "XGMI: node 0x%llx, can not match hive 0x%llx in the hive list.\n",895 adev->gmc.xgmi.node_id, adev->gmc.xgmi.hive_id);896 goto exit;897 }898 mutex_lock(&hive->hive_lock);899 900 top_info = &adev->psp.xgmi_context.top_info;901 902 list_add_tail(&adev->gmc.xgmi.head, &hive->device_list);903 list_for_each_entry(entry, &hive->device_list, head)904 top_info->nodes[count++].node_id = entry->node_id;905 top_info->num_nodes = count;906 atomic_set(&hive->number_devices, count);907 908 task_barrier_add_task(&hive->tb);909 910 if (!adev->gmc.xgmi.pending_reset &&911 amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) {912 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {913 /* update node list for other device in the hive */914 if (tmp_adev != adev) {915 top_info = &tmp_adev->psp.xgmi_context.top_info;916 top_info->nodes[count - 1].node_id =917 adev->gmc.xgmi.node_id;918 top_info->num_nodes = count;919 }920 ret = amdgpu_xgmi_update_topology(hive, tmp_adev);921 if (ret)922 goto exit_unlock;923 }924 925 if (amdgpu_sriov_vf(adev) &&926 adev->psp.xgmi_context.xgmi_ta_caps & EXTEND_PEER_LINK_INFO_CMD_FLAG) {927 /* only get topology for VF being init if it can support full duplex */928 ret = psp_xgmi_get_topology_info(&adev->psp, count,929 &adev->psp.xgmi_context.top_info, false);930 if (ret) {931 dev_err(adev->dev,932 "XGMI: Get topology failure on device %llx, hive %llx, ret %d",933 adev->gmc.xgmi.node_id,934 adev->gmc.xgmi.hive_id, ret);935 /* To do: continue with some node failed or disable the whole hive*/936 goto exit_unlock;937 }938 939 /* fill the topology info for peers instead of getting from PSP */940 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {941 amdgpu_xgmi_fill_topology_info(adev, tmp_adev);942 }943 } else {944 /* get latest topology info for each device from psp */945 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {946 ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count,947 &tmp_adev->psp.xgmi_context.top_info, false);948 if (ret) {949 dev_err(tmp_adev->dev,950 "XGMI: Get topology failure on device %llx, hive %llx, ret %d",951 tmp_adev->gmc.xgmi.node_id,952 tmp_adev->gmc.xgmi.hive_id, ret);953 /* To do : continue with some node failed or disable the whole hive */954 goto exit_unlock;955 }956 }957 }958 959 /* get topology again for hives that support extended data */960 if (adev->psp.xgmi_context.supports_extended_data) {961 962 /* initialize the hive to get extended data. */963 ret = amdgpu_xgmi_initialize_hive_get_data_partition(hive, true);964 if (ret)965 goto exit_unlock;966 967 /* get the extended data. */968 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {969 ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count,970 &tmp_adev->psp.xgmi_context.top_info, true);971 if (ret) {972 dev_err(tmp_adev->dev,973 "XGMI: Get topology for extended data failure on device %llx, hive %llx, ret %d",974 tmp_adev->gmc.xgmi.node_id,975 tmp_adev->gmc.xgmi.hive_id, ret);976 goto exit_unlock;977 }978 }979 980 /* initialize the hive to get non-extended data for the next round. */981 ret = amdgpu_xgmi_initialize_hive_get_data_partition(hive, false);982 if (ret)983 goto exit_unlock;984 985 }986 }987 988 if (!ret && !adev->gmc.xgmi.pending_reset)989 ret = amdgpu_xgmi_sysfs_add_dev_info(adev, hive);990 991exit_unlock:992 mutex_unlock(&hive->hive_lock);993exit:994 if (!ret) {995 adev->hive = hive;996 dev_info(adev->dev, "XGMI: Add node %d, hive 0x%llx.\n",997 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id);998 } else {999 amdgpu_put_xgmi_hive(hive);1000 dev_err(adev->dev, "XGMI: Failed to add node %d, hive 0x%llx ret: %d\n",1001 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id,1002 ret);1003 }1004 1005 return ret;1006}1007 1008int amdgpu_xgmi_remove_device(struct amdgpu_device *adev)1009{1010 struct amdgpu_hive_info *hive = adev->hive;1011 1012 if (!adev->gmc.xgmi.supported)1013 return -EINVAL;1014 1015 if (!hive)1016 return -EINVAL;1017 1018 mutex_lock(&hive->hive_lock);1019 task_barrier_rem_task(&hive->tb);1020 amdgpu_xgmi_sysfs_rem_dev_info(adev, hive);1021 if (hive->hi_req_gpu == adev)1022 hive->hi_req_gpu = NULL;1023 list_del(&adev->gmc.xgmi.head);1024 mutex_unlock(&hive->hive_lock);1025 1026 amdgpu_put_xgmi_hive(hive);1027 adev->hive = NULL;1028 1029 if (atomic_dec_return(&hive->number_devices) == 0) {1030 /* Remove the hive from global hive list */1031 mutex_lock(&xgmi_mutex);1032 list_del(&hive->node);1033 mutex_unlock(&xgmi_mutex);1034 1035 amdgpu_put_xgmi_hive(hive);1036 }1037 1038 return 0;1039}1040 1041static int xgmi_v6_4_0_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank,1042 enum aca_smu_type type, void *data)1043{1044 struct amdgpu_device *adev = handle->adev;1045 struct aca_bank_info info;1046 const char *error_str;1047 u64 status, count;1048 int ret, ext_error_code;1049 1050 ret = aca_bank_info_decode(bank, &info);1051 if (ret)1052 return ret;1053 1054 status = bank->regs[ACA_REG_IDX_STATUS];1055 ext_error_code = ACA_REG__STATUS__ERRORCODEEXT(status);1056 1057 error_str = ext_error_code < ARRAY_SIZE(xgmi_v6_4_0_ras_error_code_ext) ?1058 xgmi_v6_4_0_ras_error_code_ext[ext_error_code] : NULL;1059 if (error_str)1060 dev_info(adev->dev, "%s detected\n", error_str);1061 1062 count = ACA_REG__MISC0__ERRCNT(bank->regs[ACA_REG_IDX_MISC0]);1063 1064 switch (type) {1065 case ACA_SMU_TYPE_UE:1066 if (ext_error_code != 0 && ext_error_code != 9)1067 count = 0ULL;1068 1069 ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, count);1070 break;1071 case ACA_SMU_TYPE_CE:1072 count = ext_error_code == 6 ? count : 0ULL;1073 ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_CE, count);1074 break;1075 default:1076 return -EINVAL;1077 }1078 1079 return ret;1080}1081 1082static const struct aca_bank_ops xgmi_v6_4_0_aca_bank_ops = {1083 .aca_bank_parser = xgmi_v6_4_0_aca_bank_parser,1084};1085 1086static const struct aca_info xgmi_v6_4_0_aca_info = {1087 .hwip = ACA_HWIP_TYPE_PCS_XGMI,1088 .mask = ACA_ERROR_UE_MASK | ACA_ERROR_CE_MASK,1089 .bank_ops = &xgmi_v6_4_0_aca_bank_ops,1090};1091 1092static int amdgpu_xgmi_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block)1093{1094 int r;1095 1096 if (!adev->gmc.xgmi.supported ||1097 adev->gmc.xgmi.num_physical_nodes == 0)1098 return 0;1099 1100 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL);1101 1102 r = amdgpu_ras_block_late_init(adev, ras_block);1103 if (r)1104 return r;1105 1106 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {1107 case IP_VERSION(6, 4, 0):1108 r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL,1109 &xgmi_v6_4_0_aca_info, NULL);1110 if (r)1111 goto late_fini;1112 break;1113 default:1114 break;1115 }1116 1117 return 0;1118 1119late_fini:1120 amdgpu_ras_block_late_fini(adev, ras_block);1121 1122 return r;1123}1124 1125uint64_t amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device *adev,1126 uint64_t addr)1127{1128 struct amdgpu_xgmi *xgmi = &adev->gmc.xgmi;1129 return (addr + xgmi->physical_node_id * xgmi->node_segment_size);1130}1131 1132static void pcs_clear_status(struct amdgpu_device *adev, uint32_t pcs_status_reg)1133{1134 WREG32_PCIE(pcs_status_reg, 0xFFFFFFFF);1135 WREG32_PCIE(pcs_status_reg, 0);1136}1137 1138static void amdgpu_xgmi_legacy_reset_ras_error_count(struct amdgpu_device *adev)1139{1140 uint32_t i;1141 1142 switch (adev->asic_type) {1143 case CHIP_ARCTURUS:1144 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_arct); i++)1145 pcs_clear_status(adev,1146 xgmi_pcs_err_status_reg_arct[i]);1147 break;1148 case CHIP_VEGA20:1149 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_vg20); i++)1150 pcs_clear_status(adev,1151 xgmi_pcs_err_status_reg_vg20[i]);1152 break;1153 case CHIP_ALDEBARAN:1154 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_aldebaran); i++)1155 pcs_clear_status(adev,1156 xgmi3x16_pcs_err_status_reg_aldebaran[i]);1157 for (i = 0; i < ARRAY_SIZE(walf_pcs_err_status_reg_aldebaran); i++)1158 pcs_clear_status(adev,1159 walf_pcs_err_status_reg_aldebaran[i]);1160 break;1161 default:1162 break;1163 }1164 1165 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {1166 case IP_VERSION(6, 4, 0):1167 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_v6_4); i++)1168 pcs_clear_status(adev,1169 xgmi3x16_pcs_err_status_reg_v6_4[i]);1170 break;1171 default:1172 break;1173 }1174}1175 1176static void __xgmi_v6_4_0_reset_error_count(struct amdgpu_device *adev, int xgmi_inst, u64 mca_base)1177{1178 WREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS, 0ULL);1179}1180 1181static void xgmi_v6_4_0_reset_error_count(struct amdgpu_device *adev, int xgmi_inst)1182{1183 int i;1184 1185 for (i = 0; i < ARRAY_SIZE(xgmi_v6_4_0_mca_base_array); i++)1186 __xgmi_v6_4_0_reset_error_count(adev, xgmi_inst, xgmi_v6_4_0_mca_base_array[i]);1187}1188 1189static void xgmi_v6_4_0_reset_ras_error_count(struct amdgpu_device *adev)1190{1191 int i;1192 1193 for_each_inst(i, adev->aid_mask)1194 xgmi_v6_4_0_reset_error_count(adev, i);1195}1196 1197static void amdgpu_xgmi_reset_ras_error_count(struct amdgpu_device *adev)1198{1199 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {1200 case IP_VERSION(6, 4, 0):1201 xgmi_v6_4_0_reset_ras_error_count(adev);1202 break;1203 default:1204 amdgpu_xgmi_legacy_reset_ras_error_count(adev);1205 break;1206 }1207}1208 1209static int amdgpu_xgmi_query_pcs_error_status(struct amdgpu_device *adev,1210 uint32_t value,1211 uint32_t mask_value,1212 uint32_t *ue_count,1213 uint32_t *ce_count,1214 bool is_xgmi_pcs,1215 bool check_mask)1216{1217 int i;1218 int ue_cnt = 0;1219 const struct amdgpu_pcs_ras_field *pcs_ras_fields = NULL;1220 uint32_t field_array_size = 0;1221 1222 if (is_xgmi_pcs) {1223 if (amdgpu_ip_version(adev, XGMI_HWIP, 0) ==1224 IP_VERSION(6, 1, 0) ||1225 amdgpu_ip_version(adev, XGMI_HWIP, 0) ==1226 IP_VERSION(6, 4, 0)) {1227 pcs_ras_fields = &xgmi3x16_pcs_ras_fields[0];1228 field_array_size = ARRAY_SIZE(xgmi3x16_pcs_ras_fields);1229 } else {1230 pcs_ras_fields = &xgmi_pcs_ras_fields[0];1231 field_array_size = ARRAY_SIZE(xgmi_pcs_ras_fields);1232 }1233 } else {1234 pcs_ras_fields = &wafl_pcs_ras_fields[0];1235 field_array_size = ARRAY_SIZE(wafl_pcs_ras_fields);1236 }1237 1238 if (check_mask)1239 value = value & ~mask_value;1240 1241 /* query xgmi/walf pcs error status,1242 * only ue is supported */1243 for (i = 0; value && i < field_array_size; i++) {1244 ue_cnt = (value &1245 pcs_ras_fields[i].pcs_err_mask) >>1246 pcs_ras_fields[i].pcs_err_shift;1247 if (ue_cnt) {1248 dev_info(adev->dev, "%s detected\n",1249 pcs_ras_fields[i].err_name);1250 *ue_count += ue_cnt;1251 }1252 1253 /* reset bit value if the bit is checked */1254 value &= ~(pcs_ras_fields[i].pcs_err_mask);1255 }1256 1257 return 0;1258}1259 1260static void amdgpu_xgmi_legacy_query_ras_error_count(struct amdgpu_device *adev,1261 void *ras_error_status)1262{1263 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;1264 int i, supported = 1;1265 uint32_t data, mask_data = 0;1266 uint32_t ue_cnt = 0, ce_cnt = 0;1267 1268 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL))1269 return ;1270 1271 err_data->ue_count = 0;1272 err_data->ce_count = 0;1273 1274 switch (adev->asic_type) {1275 case CHIP_ARCTURUS:1276 /* check xgmi pcs error */1277 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_arct); i++) {1278 data = RREG32_PCIE(xgmi_pcs_err_status_reg_arct[i]);1279 if (data)1280 amdgpu_xgmi_query_pcs_error_status(adev, data,1281 mask_data, &ue_cnt, &ce_cnt, true, false);1282 }1283 /* check wafl pcs error */1284 for (i = 0; i < ARRAY_SIZE(wafl_pcs_err_status_reg_arct); i++) {1285 data = RREG32_PCIE(wafl_pcs_err_status_reg_arct[i]);1286 if (data)1287 amdgpu_xgmi_query_pcs_error_status(adev, data,1288 mask_data, &ue_cnt, &ce_cnt, false, false);1289 }1290 break;1291 case CHIP_VEGA20:1292 /* check xgmi pcs error */1293 for (i = 0; i < ARRAY_SIZE(xgmi_pcs_err_status_reg_vg20); i++) {1294 data = RREG32_PCIE(xgmi_pcs_err_status_reg_vg20[i]);1295 if (data)1296 amdgpu_xgmi_query_pcs_error_status(adev, data,1297 mask_data, &ue_cnt, &ce_cnt, true, false);1298 }1299 /* check wafl pcs error */1300 for (i = 0; i < ARRAY_SIZE(wafl_pcs_err_status_reg_vg20); i++) {1301 data = RREG32_PCIE(wafl_pcs_err_status_reg_vg20[i]);1302 if (data)1303 amdgpu_xgmi_query_pcs_error_status(adev, data,1304 mask_data, &ue_cnt, &ce_cnt, false, false);1305 }1306 break;1307 case CHIP_ALDEBARAN:1308 /* check xgmi3x16 pcs error */1309 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_aldebaran); i++) {1310 data = RREG32_PCIE(xgmi3x16_pcs_err_status_reg_aldebaran[i]);1311 mask_data =1312 RREG32_PCIE(xgmi3x16_pcs_err_noncorrectable_mask_reg_aldebaran[i]);1313 if (data)1314 amdgpu_xgmi_query_pcs_error_status(adev, data,1315 mask_data, &ue_cnt, &ce_cnt, true, true);1316 }1317 /* check wafl pcs error */1318 for (i = 0; i < ARRAY_SIZE(walf_pcs_err_status_reg_aldebaran); i++) {1319 data = RREG32_PCIE(walf_pcs_err_status_reg_aldebaran[i]);1320 mask_data =1321 RREG32_PCIE(walf_pcs_err_noncorrectable_mask_reg_aldebaran[i]);1322 if (data)1323 amdgpu_xgmi_query_pcs_error_status(adev, data,1324 mask_data, &ue_cnt, &ce_cnt, false, true);1325 }1326 break;1327 default:1328 supported = 0;1329 break;1330 }1331 1332 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {1333 case IP_VERSION(6, 4, 0):1334 /* check xgmi3x16 pcs error */1335 for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_v6_4); i++) {1336 data = RREG32_PCIE(xgmi3x16_pcs_err_status_reg_v6_4[i]);1337 mask_data =1338 RREG32_PCIE(xgmi3x16_pcs_err_noncorrectable_mask_reg_v6_4[i]);1339 if (data)1340 amdgpu_xgmi_query_pcs_error_status(adev, data,1341 mask_data, &ue_cnt, &ce_cnt, true, true);1342 }1343 break;1344 default:1345 if (!supported)1346 dev_warn(adev->dev, "XGMI RAS error query not supported");1347 break;1348 }1349 1350 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL);1351 1352 err_data->ue_count += ue_cnt;1353 err_data->ce_count += ce_cnt;1354}1355 1356static enum aca_error_type xgmi_v6_4_0_pcs_mca_get_error_type(struct amdgpu_device *adev, u64 status)1357{1358 const char *error_str;1359 int ext_error_code;1360 1361 ext_error_code = ACA_REG__STATUS__ERRORCODEEXT(status);1362 1363 error_str = ext_error_code < ARRAY_SIZE(xgmi_v6_4_0_ras_error_code_ext) ?1364 xgmi_v6_4_0_ras_error_code_ext[ext_error_code] : NULL;1365 if (error_str)1366 dev_info(adev->dev, "%s detected\n", error_str);1367 1368 switch (ext_error_code) {1369 case 0:1370 return ACA_ERROR_TYPE_UE;1371 case 6:1372 return ACA_ERROR_TYPE_CE;1373 default:1374 return -EINVAL;1375 }1376 1377 return -EINVAL;1378}1379 1380static void __xgmi_v6_4_0_query_error_count(struct amdgpu_device *adev, struct amdgpu_smuio_mcm_config_info *mcm_info,1381 u64 mca_base, struct ras_err_data *err_data)1382{1383 int xgmi_inst = mcm_info->die_id;1384 u64 status = 0;1385 1386 status = RREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS);1387 if (!ACA_REG__STATUS__VAL(status))1388 return;1389 1390 switch (xgmi_v6_4_0_pcs_mca_get_error_type(adev, status)) {1391 case ACA_ERROR_TYPE_UE:1392 amdgpu_ras_error_statistic_ue_count(err_data, mcm_info, 1ULL);1393 break;1394 case ACA_ERROR_TYPE_CE:1395 amdgpu_ras_error_statistic_ce_count(err_data, mcm_info, 1ULL);1396 break;1397 default:1398 break;1399 }1400 1401 WREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS, 0ULL);1402}1403 1404static void xgmi_v6_4_0_query_error_count(struct amdgpu_device *adev, int xgmi_inst, struct ras_err_data *err_data)1405{1406 struct amdgpu_smuio_mcm_config_info mcm_info = {1407 .socket_id = adev->smuio.funcs->get_socket_id(adev),1408 .die_id = xgmi_inst,1409 };1410 int i;1411 1412 for (i = 0; i < ARRAY_SIZE(xgmi_v6_4_0_mca_base_array); i++)1413 __xgmi_v6_4_0_query_error_count(adev, &mcm_info, xgmi_v6_4_0_mca_base_array[i], err_data);1414}1415 1416static void xgmi_v6_4_0_query_ras_error_count(struct amdgpu_device *adev, void *ras_error_status)1417{1418 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;1419 int i;1420 1421 for_each_inst(i, adev->aid_mask)1422 xgmi_v6_4_0_query_error_count(adev, i, err_data);1423}1424 1425static void amdgpu_xgmi_query_ras_error_count(struct amdgpu_device *adev,1426 void *ras_error_status)1427{1428 switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) {1429 case IP_VERSION(6, 4, 0):1430 xgmi_v6_4_0_query_ras_error_count(adev, ras_error_status);1431 break;1432 default:1433 amdgpu_xgmi_legacy_query_ras_error_count(adev, ras_error_status);1434 break;1435 }1436}1437 1438/* Trigger XGMI/WAFL error */1439static int amdgpu_ras_error_inject_xgmi(struct amdgpu_device *adev,1440 void *inject_if, uint32_t instance_mask)1441{1442 int ret1, ret2;1443 struct ta_ras_trigger_error_input *block_info =1444 (struct ta_ras_trigger_error_input *)inject_if;1445 1446 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))1447 dev_warn(adev->dev, "Failed to disallow df cstate");1448 1449 ret1 = amdgpu_dpm_set_pm_policy(adev, PP_PM_POLICY_XGMI_PLPD, XGMI_PLPD_DISALLOW);1450 if (ret1 && ret1 != -EOPNOTSUPP)1451 dev_warn(adev->dev, "Failed to disallow XGMI power down");1452 1453 ret2 = psp_ras_trigger_error(&adev->psp, block_info, instance_mask);1454 1455 if (amdgpu_ras_intr_triggered())1456 return ret2;1457 1458 ret1 = amdgpu_dpm_set_pm_policy(adev, PP_PM_POLICY_XGMI_PLPD, XGMI_PLPD_DEFAULT);1459 if (ret1 && ret1 != -EOPNOTSUPP)1460 dev_warn(adev->dev, "Failed to allow XGMI power down");1461 1462 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_ALLOW))1463 dev_warn(adev->dev, "Failed to allow df cstate");1464 1465 return ret2;1466}1467 1468struct amdgpu_ras_block_hw_ops xgmi_ras_hw_ops = {1469 .query_ras_error_count = amdgpu_xgmi_query_ras_error_count,1470 .reset_ras_error_count = amdgpu_xgmi_reset_ras_error_count,1471 .ras_error_inject = amdgpu_ras_error_inject_xgmi,1472};1473 1474struct amdgpu_xgmi_ras xgmi_ras = {1475 .ras_block = {1476 .hw_ops = &xgmi_ras_hw_ops,1477 .ras_late_init = amdgpu_xgmi_ras_late_init,1478 },1479};1480 1481int amdgpu_xgmi_ras_sw_init(struct amdgpu_device *adev)1482{1483 int err;1484 struct amdgpu_xgmi_ras *ras;1485 1486 if (!adev->gmc.xgmi.ras)1487 return 0;1488 1489 ras = adev->gmc.xgmi.ras;1490 err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);1491 if (err) {1492 dev_err(adev->dev, "Failed to register xgmi_wafl_pcs ras block!\n");1493 return err;1494 }1495 1496 strcpy(ras->ras_block.ras_comm.name, "xgmi_wafl");1497 ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__XGMI_WAFL;1498 ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;1499 adev->gmc.xgmi.ras_if = &ras->ras_block.ras_comm;1500 1501 return 0;1502}1503