346 lines · c
1/*2 * Copyright 2018 Advanced Micro Devices, Inc.3 *4 * Permission is hereby granted, free of charge, to any person obtaining a5 * copy of this software and associated documentation files (the "Software"),6 * to deal in the Software without restriction, including without limitation7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,8 * and/or sell copies of the Software, and to permit persons to whom the9 * Software is furnished to do so, subject to the following conditions:10 *11 * The above copyright notice and this permission notice shall be included in12 * all copies or substantial portions of the Software.13 *14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR20 * OTHER DEALINGS IN THE SOFTWARE.21 *22 */23 24#include <linux/firmware.h>25#include "amdgpu.h"26#include "amdgpu_sdma.h"27#include "amdgpu_ras.h"28 29#define AMDGPU_CSA_SDMA_SIZE 6430/* SDMA CSA reside in the 3rd page of CSA */31#define AMDGPU_CSA_SDMA_OFFSET (4096 * 2)32 33/*34 * GPU SDMA IP block helpers function.35 */36 37struct amdgpu_sdma_instance *amdgpu_sdma_get_instance_from_ring(struct amdgpu_ring *ring)38{39 struct amdgpu_device *adev = ring->adev;40 int i;41 42 for (i = 0; i < adev->sdma.num_instances; i++)43 if (ring == &adev->sdma.instance[i].ring ||44 ring == &adev->sdma.instance[i].page)45 return &adev->sdma.instance[i];46 47 return NULL;48}49 50int amdgpu_sdma_get_index_from_ring(struct amdgpu_ring *ring, uint32_t *index)51{52 struct amdgpu_device *adev = ring->adev;53 int i;54 55 for (i = 0; i < adev->sdma.num_instances; i++) {56 if (ring == &adev->sdma.instance[i].ring ||57 ring == &adev->sdma.instance[i].page) {58 *index = i;59 return 0;60 }61 }62 63 return -EINVAL;64}65 66uint64_t amdgpu_sdma_get_csa_mc_addr(struct amdgpu_ring *ring,67 unsigned int vmid)68{69 struct amdgpu_device *adev = ring->adev;70 uint64_t csa_mc_addr;71 uint32_t index = 0;72 int r;73 74 /* don't enable OS preemption on SDMA under SRIOV */75 if (amdgpu_sriov_vf(adev) || vmid == 0 || !adev->gfx.mcbp)76 return 0;77 78 if (ring->is_mes_queue) {79 uint32_t offset = 0;80 81 offset = offsetof(struct amdgpu_mes_ctx_meta_data,82 sdma[ring->idx].sdma_meta_data);83 csa_mc_addr = amdgpu_mes_ctx_get_offs_gpu_addr(ring, offset);84 } else {85 r = amdgpu_sdma_get_index_from_ring(ring, &index);86 87 if (r || index > 31)88 csa_mc_addr = 0;89 else90 csa_mc_addr = amdgpu_csa_vaddr(adev) +91 AMDGPU_CSA_SDMA_OFFSET +92 index * AMDGPU_CSA_SDMA_SIZE;93 }94 95 return csa_mc_addr;96}97 98int amdgpu_sdma_ras_late_init(struct amdgpu_device *adev,99 struct ras_common_if *ras_block)100{101 int r, i;102 103 r = amdgpu_ras_block_late_init(adev, ras_block);104 if (r)105 return r;106 107 if (amdgpu_ras_is_supported(adev, ras_block->block)) {108 for (i = 0; i < adev->sdma.num_instances; i++) {109 r = amdgpu_irq_get(adev, &adev->sdma.ecc_irq,110 AMDGPU_SDMA_IRQ_INSTANCE0 + i);111 if (r)112 goto late_fini;113 }114 }115 116 return 0;117 118late_fini:119 amdgpu_ras_block_late_fini(adev, ras_block);120 return r;121}122 123int amdgpu_sdma_process_ras_data_cb(struct amdgpu_device *adev,124 void *err_data,125 struct amdgpu_iv_entry *entry)126{127 kgd2kfd_set_sram_ecc_flag(adev->kfd.dev);128 129 if (amdgpu_sriov_vf(adev))130 return AMDGPU_RAS_SUCCESS;131 132 amdgpu_ras_reset_gpu(adev);133 134 return AMDGPU_RAS_SUCCESS;135}136 137int amdgpu_sdma_process_ecc_irq(struct amdgpu_device *adev,138 struct amdgpu_irq_src *source,139 struct amdgpu_iv_entry *entry)140{141 struct ras_common_if *ras_if = adev->sdma.ras_if;142 struct ras_dispatch_if ih_data = {143 .entry = entry,144 };145 146 if (!ras_if)147 return 0;148 149 ih_data.head = *ras_if;150 151 amdgpu_ras_interrupt_dispatch(adev, &ih_data);152 return 0;153}154 155static int amdgpu_sdma_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst)156{157 uint16_t version_major;158 const struct common_firmware_header *header = NULL;159 const struct sdma_firmware_header_v1_0 *hdr;160 const struct sdma_firmware_header_v2_0 *hdr_v2;161 const struct sdma_firmware_header_v3_0 *hdr_v3;162 163 header = (const struct common_firmware_header *)164 sdma_inst->fw->data;165 version_major = le16_to_cpu(header->header_version_major);166 167 switch (version_major) {168 case 1:169 hdr = (const struct sdma_firmware_header_v1_0 *)sdma_inst->fw->data;170 sdma_inst->fw_version = le32_to_cpu(hdr->header.ucode_version);171 sdma_inst->feature_version = le32_to_cpu(hdr->ucode_feature_version);172 break;173 case 2:174 hdr_v2 = (const struct sdma_firmware_header_v2_0 *)sdma_inst->fw->data;175 sdma_inst->fw_version = le32_to_cpu(hdr_v2->header.ucode_version);176 sdma_inst->feature_version = le32_to_cpu(hdr_v2->ucode_feature_version);177 break;178 case 3:179 hdr_v3 = (const struct sdma_firmware_header_v3_0 *)sdma_inst->fw->data;180 sdma_inst->fw_version = le32_to_cpu(hdr_v3->header.ucode_version);181 sdma_inst->feature_version = le32_to_cpu(hdr_v3->ucode_feature_version);182 break;183 default:184 return -EINVAL;185 }186 187 if (sdma_inst->feature_version >= 20)188 sdma_inst->burst_nop = true;189 190 return 0;191}192 193void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,194 bool duplicate)195{196 int i;197 198 for (i = 0; i < adev->sdma.num_instances; i++) {199 amdgpu_ucode_release(&adev->sdma.instance[i].fw);200 if (duplicate)201 break;202 }203 204 memset((void *)adev->sdma.instance, 0,205 sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES);206}207 208int amdgpu_sdma_init_microcode(struct amdgpu_device *adev,209 u32 instance, bool duplicate)210{211 struct amdgpu_firmware_info *info = NULL;212 const struct common_firmware_header *header = NULL;213 int err, i;214 const struct sdma_firmware_header_v2_0 *sdma_hdr;215 const struct sdma_firmware_header_v3_0 *sdma_hv3;216 uint16_t version_major;217 char ucode_prefix[30];218 219 amdgpu_ucode_ip_version_decode(adev, SDMA0_HWIP, ucode_prefix, sizeof(ucode_prefix));220 if (instance == 0)221 err = amdgpu_ucode_request(adev, &adev->sdma.instance[instance].fw,222 "amdgpu/%s.bin", ucode_prefix);223 else224 err = amdgpu_ucode_request(adev, &adev->sdma.instance[instance].fw,225 "amdgpu/%s%d.bin", ucode_prefix, instance);226 if (err)227 goto out;228 229 header = (const struct common_firmware_header *)230 adev->sdma.instance[instance].fw->data;231 version_major = le16_to_cpu(header->header_version_major);232 233 if ((duplicate && instance) || (!duplicate && version_major > 1)) {234 err = -EINVAL;235 goto out;236 }237 238 err = amdgpu_sdma_init_inst_ctx(&adev->sdma.instance[instance]);239 if (err)240 goto out;241 242 if (duplicate) {243 for (i = 1; i < adev->sdma.num_instances; i++)244 memcpy((void *)&adev->sdma.instance[i],245 (void *)&adev->sdma.instance[0],246 sizeof(struct amdgpu_sdma_instance));247 }248 249 DRM_DEBUG("psp_load == '%s'\n",250 adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false");251 252 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {253 switch (version_major) {254 case 1:255 for (i = 0; i < adev->sdma.num_instances; i++) {256 if (!duplicate && (instance != i))257 continue;258 else {259 /* Use a single copy per SDMA firmware type. PSP uses the same instance for all260 * groups of SDMAs */261 if ((amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==262 IP_VERSION(4, 4, 2) ||263 amdgpu_ip_version(adev, SDMA0_HWIP, 0) ==264 IP_VERSION(4, 4, 5)) &&265 adev->firmware.load_type ==266 AMDGPU_FW_LOAD_PSP &&267 adev->sdma.num_inst_per_aid == i) {268 break;269 }270 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i];271 info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i;272 info->fw = adev->sdma.instance[i].fw;273 adev->firmware.fw_size +=274 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);275 }276 }277 break;278 case 2:279 sdma_hdr = (const struct sdma_firmware_header_v2_0 *)280 adev->sdma.instance[0].fw->data;281 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH0];282 info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH0;283 info->fw = adev->sdma.instance[0].fw;284 adev->firmware.fw_size +=285 ALIGN(le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes), PAGE_SIZE);286 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH1];287 info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH1;288 info->fw = adev->sdma.instance[0].fw;289 adev->firmware.fw_size +=290 ALIGN(le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes), PAGE_SIZE);291 break;292 case 3:293 sdma_hv3 = (const struct sdma_firmware_header_v3_0 *)294 adev->sdma.instance[0].fw->data;295 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_RS64];296 info->ucode_id = AMDGPU_UCODE_ID_SDMA_RS64;297 info->fw = adev->sdma.instance[0].fw;298 adev->firmware.fw_size +=299 ALIGN(le32_to_cpu(sdma_hv3->ucode_size_bytes), PAGE_SIZE);300 break;301 default:302 err = -EINVAL;303 }304 }305 306out:307 if (err)308 amdgpu_sdma_destroy_inst_ctx(adev, duplicate);309 return err;310}311 312int amdgpu_sdma_ras_sw_init(struct amdgpu_device *adev)313{314 int err = 0;315 struct amdgpu_sdma_ras *ras = NULL;316 317 /* adev->sdma.ras is NULL, which means sdma does not318 * support ras function, then do nothing here.319 */320 if (!adev->sdma.ras)321 return 0;322 323 ras = adev->sdma.ras;324 325 err = amdgpu_ras_register_ras_block(adev, &ras->ras_block);326 if (err) {327 dev_err(adev->dev, "Failed to register sdma ras block!\n");328 return err;329 }330 331 strcpy(ras->ras_block.ras_comm.name, "sdma");332 ras->ras_block.ras_comm.block = AMDGPU_RAS_BLOCK__SDMA;333 ras->ras_block.ras_comm.type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;334 adev->sdma.ras_if = &ras->ras_block.ras_comm;335 336 /* If not define special ras_late_init function, use default ras_late_init */337 if (!ras->ras_block.ras_late_init)338 ras->ras_block.ras_late_init = amdgpu_sdma_ras_late_init;339 340 /* If not defined special ras_cb function, use default ras_cb */341 if (!ras->ras_block.ras_cb)342 ras->ras_block.ras_cb = amdgpu_sdma_process_ras_data_cb;343 344 return 0;345}346