434 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR MIT2/*3 * Copyright 2014-2022 Advanced Micro Devices, Inc.4 *5 * Permission is hereby granted, free of charge, to any person obtaining a6 * copy of this software and associated documentation files (the "Software"),7 * to deal in the Software without restriction, including without limitation8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,9 * and/or sell copies of the Software, and to permit persons to whom the10 * Software is furnished to do so, subject to the following conditions:11 *12 * The above copyright notice and this permission notice shall be included in13 * all copies or substantial portions of the Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR21 * OTHER DEALINGS IN THE SOFTWARE.22 *23 */24 25#include <linux/device.h>26#include <linux/export.h>27#include <linux/err.h>28#include <linux/fs.h>29#include <linux/sched.h>30#include <linux/slab.h>31#include <linux/uaccess.h>32#include <linux/compat.h>33#include <uapi/linux/kfd_ioctl.h>34#include <linux/time.h>35#include "kfd_priv.h"36#include <linux/mm.h>37#include <linux/mman.h>38#include <linux/processor.h>39#include "amdgpu_vm.h"40 41/*42 * The primary memory I/O features being added for revisions of gfxip43 * beyond 7.0 (Kaveri) are:44 *45 * Access to ATC/IOMMU mapped memory w/ associated extension of VA to 48b46 *47 * “Flat” shader memory access – These are new shader vector memory48 * operations that do not reference a T#/V# so a “pointer” is what is49 * sourced from the vector gprs for direct access to memory.50 * This pointer space has the Shared(LDS) and Private(Scratch) memory51 * mapped into this pointer space as apertures.52 * The hardware then determines how to direct the memory request53 * based on what apertures the request falls in.54 *55 * Unaligned support and alignment check56 *57 *58 * System Unified Address - SUA59 *60 * The standard usage for GPU virtual addresses are that they are mapped by61 * a set of page tables we call GPUVM and these page tables are managed by62 * a combination of vidMM/driver software components. The current virtual63 * address (VA) range for GPUVM is 40b.64 *65 * As of gfxip7.1 and beyond we’re adding the ability for compute memory66 * clients (CP/RLC, DMA, SHADER(ifetch, scalar, and vector ops)) to access67 * the same page tables used by host x86 processors and that are managed by68 * the operating system. This is via a technique and hardware called ATC/IOMMU.69 * The GPU has the capability of accessing both the GPUVM and ATC address70 * spaces for a given VMID (process) simultaneously and we call this feature71 * system unified address (SUA).72 *73 * There are three fundamental address modes of operation for a given VMID74 * (process) on the GPU:75 *76 * HSA64 – 64b pointers and the default address space is ATC77 * HSA32 – 32b pointers and the default address space is ATC78 * GPUVM – 64b pointers and the default address space is GPUVM (driver79 * model mode)80 *81 *82 * HSA64 - ATC/IOMMU 64b83 *84 * A 64b pointer in the AMD64/IA64 CPU architecture is not fully utilized85 * by the CPU so an AMD CPU can only access the high area86 * (VA[63:47] == 0x1FFFF) and low area (VA[63:47 == 0) of the address space87 * so the actual VA carried to translation is 48b. There is a “hole” in88 * the middle of the 64b VA space.89 *90 * The GPU not only has access to all of the CPU accessible address space via91 * ATC/IOMMU, but it also has access to the GPUVM address space. The “system92 * unified address” feature (SUA) is the mapping of GPUVM and ATC address93 * spaces into a unified pointer space. The method we take for 64b mode is94 * to map the full 40b GPUVM address space into the hole of the 64b address95 * space.96 97 * The GPUVM_Base/GPUVM_Limit defines the aperture in the 64b space where we98 * direct requests to be translated via GPUVM page tables instead of the99 * IOMMU path.100 *101 *102 * 64b to 49b Address conversion103 *104 * Note that there are still significant portions of unused regions (holes)105 * in the 64b address space even for the GPU. There are several places in106 * the pipeline (sw and hw), we wish to compress the 64b virtual address107 * to a 49b address. This 49b address is constituted of an “ATC” bit108 * plus a 48b virtual address. This 49b address is what is passed to the109 * translation hardware. ATC==0 means the 48b address is a GPUVM address110 * (max of 2^40 – 1) intended to be translated via GPUVM page tables.111 * ATC==1 means the 48b address is intended to be translated via IOMMU112 * page tables.113 *114 * A 64b pointer is compared to the apertures that are defined (Base/Limit), in115 * this case the GPUVM aperture (red) is defined and if a pointer falls in this116 * aperture, we subtract the GPUVM_Base address and set the ATC bit to zero117 * as part of the 64b to 49b conversion.118 *119 * Where this 64b to 49b conversion is done is a function of the usage.120 * Most GPU memory access is via memory objects where the driver builds121 * a descriptor which consists of a base address and a memory access by122 * the GPU usually consists of some kind of an offset or Cartesian coordinate123 * that references this memory descriptor. This is the case for shader124 * instructions that reference the T# or V# constants, or for specified125 * locations of assets (ex. the shader program location). In these cases126 * the driver is what handles the 64b to 49b conversion and the base127 * address in the descriptor (ex. V# or T# or shader program location)128 * is defined as a 48b address w/ an ATC bit. For this usage a given129 * memory object cannot straddle multiple apertures in the 64b address130 * space. For example a shader program cannot jump in/out between ATC131 * and GPUVM space.132 *133 * In some cases we wish to pass a 64b pointer to the GPU hardware and134 * the GPU hw does the 64b to 49b conversion before passing memory135 * requests to the cache/memory system. This is the case for the136 * S_LOAD and FLAT_* shader memory instructions where we have 64b pointers137 * in scalar and vector GPRs respectively.138 *139 * In all cases (no matter where the 64b -> 49b conversion is done), the gfxip140 * hardware sends a 48b address along w/ an ATC bit, to the memory controller141 * on the memory request interfaces.142 *143 * <client>_MC_rdreq_atc // read request ATC bit144 *145 * 0 : <client>_MC_rdreq_addr is a GPUVM VA146 *147 * 1 : <client>_MC_rdreq_addr is a ATC VA148 *149 *150 * “Spare” aperture (APE1)151 *152 * We use the GPUVM aperture to differentiate ATC vs. GPUVM, but we also use153 * apertures to set the Mtype field for S_LOAD/FLAT_* ops which is input to the154 * config tables for setting cache policies. The “spare” (APE1) aperture is155 * motivated by getting a different Mtype from the default.156 * The default aperture isn’t an actual base/limit aperture; it is just the157 * address space that doesn’t hit any defined base/limit apertures.158 * The following diagram is a complete picture of the gfxip7.x SUA apertures.159 * The APE1 can be placed either below or above160 * the hole (cannot be in the hole).161 *162 *163 * General Aperture definitions and rules164 *165 * An aperture register definition consists of a Base, Limit, Mtype, and166 * usually an ATC bit indicating which translation tables that aperture uses.167 * In all cases (for SUA and DUA apertures discussed later), aperture base168 * and limit definitions are 64KB aligned.169 *170 * <ape>_Base[63:0] = { <ape>_Base_register[63:16], 0x0000 }171 *172 * <ape>_Limit[63:0] = { <ape>_Limit_register[63:16], 0xFFFF }173 *174 * The base and limit are considered inclusive to an aperture so being175 * inside an aperture means (address >= Base) AND (address <= Limit).176 *177 * In no case is a payload that straddles multiple apertures expected to work.178 * For example a load_dword_x4 that starts in one aperture and ends in another,179 * does not work. For the vector FLAT_* ops we have detection capability in180 * the shader for reporting a “memory violation” back to the181 * SQ block for use in traps.182 * A memory violation results when an op falls into the hole,183 * or a payload straddles multiple apertures. The S_LOAD instruction184 * does not have this detection.185 *186 * Apertures cannot overlap.187 *188 *189 *190 * HSA32 - ATC/IOMMU 32b191 *192 * For HSA32 mode, the pointers are interpreted as 32 bits and use a single GPR193 * instead of two for the S_LOAD and FLAT_* ops. The entire GPUVM space of 40b194 * will not fit so there is only partial visibility to the GPUVM195 * space (defined by the aperture) for S_LOAD and FLAT_* ops.196 * There is no spare (APE1) aperture for HSA32 mode.197 *198 *199 * GPUVM 64b mode (driver model)200 *201 * This mode is related to HSA64 in that the difference really is that202 * the default aperture is GPUVM (ATC==0) and not ATC space.203 * We have gfxip7.x hardware that has FLAT_* and S_LOAD support for204 * SUA GPUVM mode, but does not support HSA32/HSA64.205 *206 *207 * Device Unified Address - DUA208 *209 * Device unified address (DUA) is the name of the feature that maps the210 * Shared(LDS) memory and Private(Scratch) memory into the overall address211 * space for use by the new FLAT_* vector memory ops. The Shared and212 * Private memories are mapped as apertures into the address space,213 * and the hardware detects when a FLAT_* memory request is to be redirected214 * to the LDS or Scratch memory when it falls into one of these apertures.215 * Like the SUA apertures, the Shared/Private apertures are 64KB aligned and216 * the base/limit is “in” the aperture. For both HSA64 and GPUVM SUA modes,217 * the Shared/Private apertures are always placed in a limited selection of218 * options in the hole of the 64b address space. For HSA32 mode, the219 * Shared/Private apertures can be placed anywhere in the 32b space220 * except at 0.221 *222 *223 * HSA64 Apertures for FLAT_* vector ops224 *225 * For HSA64 SUA mode, the Shared and Private apertures are always placed226 * in the hole w/ a limited selection of possible locations. The requests227 * that fall in the private aperture are expanded as a function of the228 * work-item id (tid) and redirected to the location of the229 * “hidden private memory”. The hidden private can be placed in either GPUVM230 * or ATC space. The addresses that fall in the shared aperture are231 * re-directed to the on-chip LDS memory hardware.232 *233 *234 * HSA32 Apertures for FLAT_* vector ops235 *236 * In HSA32 mode, the Private and Shared apertures can be placed anywhere237 * in the 32b space except at 0 (Private or Shared Base at zero disables238 * the apertures). If the base address of the apertures are non-zero239 * (ie apertures exists), the size is always 64KB.240 *241 *242 * GPUVM Apertures for FLAT_* vector ops243 *244 * In GPUVM mode, the Shared/Private apertures are specified identically245 * to HSA64 mode where they are always in the hole at a limited selection246 * of locations.247 *248 *249 * Aperture Definitions for SUA and DUA250 *251 * The interpretation of the aperture register definitions for a given252 * VMID is a function of the “SUA Mode” which is one of HSA64, HSA32, or253 * GPUVM64 discussed in previous sections. The mode is first decoded, and254 * then the remaining register decode is a function of the mode.255 *256 *257 * SUA Mode Decode258 *259 * For the S_LOAD and FLAT_* shader operations, the SUA mode is decoded from260 * the COMPUTE_DISPATCH_INITIATOR:DATA_ATC bit and261 * the SH_MEM_CONFIG:PTR32 bits.262 *263 * COMPUTE_DISPATCH_INITIATOR:DATA_ATC SH_MEM_CONFIG:PTR32 Mode264 *265 * 1 0 HSA64266 *267 * 1 1 HSA32268 *269 * 0 X GPUVM64270 *271 * In general the hardware will ignore the PTR32 bit and treat272 * as “0” whenever DATA_ATC = “0”, but sw should set PTR32=0273 * when DATA_ATC=0.274 *275 * The DATA_ATC bit is only set for compute dispatches.276 * All “Draw” dispatches are hardcoded to GPUVM64 mode277 * for FLAT_* / S_LOAD operations.278 */279 280#define MAKE_GPUVM_APP_BASE_VI(gpu_num) \281 (((uint64_t)(gpu_num) << 61) + 0x1000000000000L)282 283#define MAKE_GPUVM_APP_LIMIT(base, size) \284 (((uint64_t)(base) & 0xFFFFFF0000000000UL) + (size) - 1)285 286#define MAKE_SCRATCH_APP_BASE_VI() \287 (((uint64_t)(0x1UL) << 61) + 0x100000000L)288 289#define MAKE_SCRATCH_APP_LIMIT(base) \290 (((uint64_t)base & 0xFFFFFFFF00000000UL) | 0xFFFFFFFF)291 292#define MAKE_LDS_APP_BASE_VI() \293 (((uint64_t)(0x1UL) << 61) + 0x0)294#define MAKE_LDS_APP_LIMIT(base) \295 (((uint64_t)(base) & 0xFFFFFFFF00000000UL) | 0xFFFFFFFF)296 297/* On GFXv9 the LDS and scratch apertures are programmed independently298 * using the high 16 bits of the 64-bit virtual address. They must be299 * in the hole, which will be the case as long as the high 16 bits are300 * not 0.301 *302 * The aperture sizes are still 4GB implicitly.303 *304 * A GPUVM aperture is not applicable on GFXv9.305 */306#define MAKE_LDS_APP_BASE_V9() ((uint64_t)(0x1UL) << 48)307#define MAKE_SCRATCH_APP_BASE_V9() ((uint64_t)(0x2UL) << 48)308 309/* User mode manages most of the SVM aperture address space. The low310 * 16MB are reserved for kernel use (CWSR trap handler and kernel IB311 * for now).312 */313#define SVM_USER_BASE (u64)(KFD_CWSR_TBA_TMA_SIZE + 2*PAGE_SIZE)314#define SVM_CWSR_BASE (SVM_USER_BASE - KFD_CWSR_TBA_TMA_SIZE)315#define SVM_IB_BASE (SVM_CWSR_BASE - PAGE_SIZE)316 317static void kfd_init_apertures_vi(struct kfd_process_device *pdd, uint8_t id)318{319 /*320 * node id couldn't be 0 - the three MSB bits of321 * aperture shouldn't be 0322 */323 pdd->lds_base = MAKE_LDS_APP_BASE_VI();324 pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base);325 326 /* dGPUs: SVM aperture starting at 0327 * with small reserved space for kernel.328 * Set them to CANONICAL addresses.329 */330 pdd->gpuvm_base = max(SVM_USER_BASE, AMDGPU_VA_RESERVED_BOTTOM);331 pdd->gpuvm_limit =332 pdd->dev->kfd->shared_resources.gpuvm_size - 1;333 334 /* dGPUs: the reserved space for kernel335 * before SVM336 */337 pdd->qpd.cwsr_base = SVM_CWSR_BASE;338 pdd->qpd.ib_base = SVM_IB_BASE;339 340 pdd->scratch_base = MAKE_SCRATCH_APP_BASE_VI();341 pdd->scratch_limit = MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base);342}343 344static void kfd_init_apertures_v9(struct kfd_process_device *pdd, uint8_t id)345{346 pdd->lds_base = MAKE_LDS_APP_BASE_V9();347 pdd->lds_limit = MAKE_LDS_APP_LIMIT(pdd->lds_base);348 349 pdd->gpuvm_base = AMDGPU_VA_RESERVED_BOTTOM;350 pdd->gpuvm_limit =351 pdd->dev->kfd->shared_resources.gpuvm_size - 1;352 353 pdd->scratch_base = MAKE_SCRATCH_APP_BASE_V9();354 pdd->scratch_limit = MAKE_SCRATCH_APP_LIMIT(pdd->scratch_base);355 356 /*357 * Place TBA/TMA on opposite side of VM hole to prevent358 * stray faults from triggering SVM on these pages.359 */360 pdd->qpd.cwsr_base = AMDGPU_VA_RESERVED_TRAP_START(pdd->dev->adev);361}362 363int kfd_init_apertures(struct kfd_process *process)364{365 uint8_t id = 0;366 struct kfd_node *dev;367 struct kfd_process_device *pdd;368 369 /*Iterating over all devices*/370 while (kfd_topology_enum_kfd_devices(id, &dev) == 0) {371 if (!dev || kfd_devcgroup_check_permission(dev)) {372 /* Skip non GPU devices and devices to which the373 * current process have no access to. Access can be374 * limited by placing the process in a specific375 * cgroup hierarchy376 */377 id++;378 continue;379 }380 381 pdd = kfd_create_process_device_data(dev, process);382 if (!pdd) {383 dev_err(dev->adev->dev,384 "Failed to create process device data\n");385 return -ENOMEM;386 }387 /*388 * For 64 bit process apertures will be statically reserved in389 * the x86_64 non canonical process address space390 * amdkfd doesn't currently support apertures for 32 bit process391 */392 if (process->is_32bit_user_mode) {393 pdd->lds_base = pdd->lds_limit = 0;394 pdd->gpuvm_base = pdd->gpuvm_limit = 0;395 pdd->scratch_base = pdd->scratch_limit = 0;396 } else {397 switch (dev->adev->asic_type) {398 case CHIP_KAVERI:399 case CHIP_HAWAII:400 case CHIP_CARRIZO:401 case CHIP_TONGA:402 case CHIP_FIJI:403 case CHIP_POLARIS10:404 case CHIP_POLARIS11:405 case CHIP_POLARIS12:406 case CHIP_VEGAM:407 kfd_init_apertures_vi(pdd, id);408 break;409 default:410 if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 0, 1))411 kfd_init_apertures_v9(pdd, id);412 else {413 WARN(1, "Unexpected ASIC family %u",414 dev->adev->asic_type);415 return -EINVAL;416 }417 }418 }419 420 dev_dbg(kfd_device, "node id %u\n", id);421 dev_dbg(kfd_device, "gpu id %u\n", pdd->dev->id);422 dev_dbg(kfd_device, "lds_base %llX\n", pdd->lds_base);423 dev_dbg(kfd_device, "lds_limit %llX\n", pdd->lds_limit);424 dev_dbg(kfd_device, "gpuvm_base %llX\n", pdd->gpuvm_base);425 dev_dbg(kfd_device, "gpuvm_limit %llX\n", pdd->gpuvm_limit);426 dev_dbg(kfd_device, "scratch_base %llX\n", pdd->scratch_base);427 dev_dbg(kfd_device, "scratch_limit %llX\n", pdd->scratch_limit);428 429 id++;430 }431 432 return 0;433}434