240 lines · c
1// SPDX-License-Identifier: MIT2/*3 * Copyright 2022 Advanced Micro Devices, Inc.4 *5 * Permission is hereby granted, free of charge, to any person obtaining a6 * copy of this software and associated documentation files (the "Software"),7 * to deal in the Software without restriction, including without limitation8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,9 * and/or sell copies of the Software, and to permit persons to whom the10 * Software is furnished to do so, subject to the following conditions:11 *12 * The above copyright notice and this permission notice shall be included in13 * all copies or substantial portions of the Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR21 * OTHER DEALINGS IN THE SOFTWARE.22 *23 */24 25#include "amdgpu.h"26 27/**28 * amdgpu_mm_rdoorbell - read a doorbell dword29 *30 * @adev: amdgpu_device pointer31 * @index: doorbell index32 *33 * Returns the value in the doorbell aperture at the34 * requested doorbell index (CIK).35 */36u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)37{38 if (amdgpu_device_skip_hw_access(adev))39 return 0;40 41 if (index < adev->doorbell.num_kernel_doorbells)42 return readl(adev->doorbell.cpu_addr + index);43 44 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);45 return 0;46}47 48/**49 * amdgpu_mm_wdoorbell - write a doorbell dword50 *51 * @adev: amdgpu_device pointer52 * @index: doorbell index53 * @v: value to write54 *55 * Writes @v to the doorbell aperture at the56 * requested doorbell index (CIK).57 */58void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)59{60 if (amdgpu_device_skip_hw_access(adev))61 return;62 63 if (index < adev->doorbell.num_kernel_doorbells)64 writel(v, adev->doorbell.cpu_addr + index);65 else66 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);67}68 69/**70 * amdgpu_mm_rdoorbell64 - read a doorbell Qword71 *72 * @adev: amdgpu_device pointer73 * @index: doorbell index74 *75 * Returns the value in the doorbell aperture at the76 * requested doorbell index (VEGA10+).77 */78u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)79{80 if (amdgpu_device_skip_hw_access(adev))81 return 0;82 83 if (index < adev->doorbell.num_kernel_doorbells)84 return atomic64_read((atomic64_t *)(adev->doorbell.cpu_addr + index));85 86 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);87 return 0;88}89 90/**91 * amdgpu_mm_wdoorbell64 - write a doorbell Qword92 *93 * @adev: amdgpu_device pointer94 * @index: doorbell index95 * @v: value to write96 *97 * Writes @v to the doorbell aperture at the98 * requested doorbell index (VEGA10+).99 */100void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)101{102 if (amdgpu_device_skip_hw_access(adev))103 return;104 105 if (index < adev->doorbell.num_kernel_doorbells)106 atomic64_set((atomic64_t *)(adev->doorbell.cpu_addr + index), v);107 else108 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);109}110 111/**112 * amdgpu_doorbell_index_on_bar - Find doorbell's absolute offset in BAR113 *114 * @adev: amdgpu_device pointer115 * @db_bo: doorbell object's bo116 * @doorbell_index: doorbell relative index in this doorbell object117 * @db_size: doorbell size is in byte118 *119 * returns doorbell's absolute index in BAR120 */121uint32_t amdgpu_doorbell_index_on_bar(struct amdgpu_device *adev,122 struct amdgpu_bo *db_bo,123 uint32_t doorbell_index,124 uint32_t db_size)125{126 int db_bo_offset;127 128 db_bo_offset = amdgpu_bo_gpu_offset_no_check(db_bo);129 130 /* doorbell index is 32 bit but doorbell's size can be 32 bit131 * or 64 bit, so *db_size(in byte)/4 for alignment.132 */133 return db_bo_offset / sizeof(u32) + doorbell_index *134 DIV_ROUND_UP(db_size, 4);135}136 137/**138 * amdgpu_doorbell_create_kernel_doorbells - Create kernel doorbells for graphics139 *140 * @adev: amdgpu_device pointer141 *142 * Creates doorbells for graphics driver usages.143 * returns 0 on success, error otherwise.144 */145int amdgpu_doorbell_create_kernel_doorbells(struct amdgpu_device *adev)146{147 int r;148 int size;149 150 /* SI HW does not have doorbells, skip allocation */151 if (adev->doorbell.num_kernel_doorbells == 0)152 return 0;153 154 /* Reserve first num_kernel_doorbells (page-aligned) for kernel ops */155 size = ALIGN(adev->doorbell.num_kernel_doorbells * sizeof(u32), PAGE_SIZE);156 157 /* Allocate an extra page for MES kernel usages (ring test) */158 adev->mes.db_start_dw_offset = size / sizeof(u32);159 size += PAGE_SIZE;160 161 r = amdgpu_bo_create_kernel(adev,162 size,163 PAGE_SIZE,164 AMDGPU_GEM_DOMAIN_DOORBELL,165 &adev->doorbell.kernel_doorbells,166 NULL,167 (void **)&adev->doorbell.cpu_addr);168 if (r) {169 DRM_ERROR("Failed to allocate kernel doorbells, err=%d\n", r);170 return r;171 }172 173 adev->doorbell.num_kernel_doorbells = size / sizeof(u32);174 return 0;175}176 177/*178 * GPU doorbell aperture helpers function.179 */180/**181 * amdgpu_doorbell_init - Init doorbell driver information.182 *183 * @adev: amdgpu_device pointer184 *185 * Init doorbell driver information (CIK)186 * Returns 0 on success, error on failure.187 */188int amdgpu_doorbell_init(struct amdgpu_device *adev)189{190 191 /* No doorbell on SI hardware generation */192 if (adev->asic_type < CHIP_BONAIRE) {193 adev->doorbell.base = 0;194 adev->doorbell.size = 0;195 adev->doorbell.num_kernel_doorbells = 0;196 return 0;197 }198 199 if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)200 return -EINVAL;201 202 amdgpu_asic_init_doorbell_index(adev);203 204 /* doorbell bar mapping */205 adev->doorbell.base = pci_resource_start(adev->pdev, 2);206 adev->doorbell.size = pci_resource_len(adev->pdev, 2);207 208 adev->doorbell.num_kernel_doorbells =209 min_t(u32, adev->doorbell.size / sizeof(u32),210 adev->doorbell_index.max_assignment + 1);211 if (adev->doorbell.num_kernel_doorbells == 0)212 return -EINVAL;213 214 /*215 * For Vega, reserve and map two pages on doorbell BAR since SDMA216 * paging queue doorbell use the second page. The217 * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the218 * doorbells are in the first page. So with paging queue enabled,219 * the max num_kernel_doorbells should + 1 page (0x400 in dword)220 */221 if (adev->asic_type >= CHIP_VEGA10)222 adev->doorbell.num_kernel_doorbells += 0x400;223 224 return 0;225}226 227/**228 * amdgpu_doorbell_fini - Tear down doorbell driver information.229 *230 * @adev: amdgpu_device pointer231 *232 * Tear down doorbell driver information (CIK)233 */234void amdgpu_doorbell_fini(struct amdgpu_device *adev)235{236 amdgpu_bo_free_kernel(&adev->doorbell.kernel_doorbells,237 NULL,238 (void **)&adev->doorbell.cpu_addr);239}240