458 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR MIT2/*3 * Copyright 2014-2022 Advanced Micro Devices, Inc.4 *5 * Permission is hereby granted, free of charge, to any person obtaining a6 * copy of this software and associated documentation files (the "Software"),7 * to deal in the Software without restriction, including without limitation8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,9 * and/or sell copies of the Software, and to permit persons to whom the10 * Software is furnished to do so, subject to the following conditions:11 *12 * The above copyright notice and this permission notice shall be included in13 * all copies or substantial portions of the Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR21 * OTHER DEALINGS IN THE SOFTWARE.22 *23 */24 25#include <linux/slab.h>26#include "kfd_priv.h"27#include "kfd_topology.h"28#include "kfd_svm.h"29 30void print_queue_properties(struct queue_properties *q)31{32 if (!q)33 return;34 35 pr_debug("Printing queue properties:\n");36 pr_debug("Queue Type: %u\n", q->type);37 pr_debug("Queue Size: %llu\n", q->queue_size);38 pr_debug("Queue percent: %u\n", q->queue_percent);39 pr_debug("Queue Address: 0x%llX\n", q->queue_address);40 pr_debug("Queue Id: %u\n", q->queue_id);41 pr_debug("Queue Process Vmid: %u\n", q->vmid);42 pr_debug("Queue Read Pointer: 0x%px\n", q->read_ptr);43 pr_debug("Queue Write Pointer: 0x%px\n", q->write_ptr);44 pr_debug("Queue Doorbell Pointer: 0x%p\n", q->doorbell_ptr);45 pr_debug("Queue Doorbell Offset: %u\n", q->doorbell_off);46}47 48void print_queue(struct queue *q)49{50 if (!q)51 return;52 pr_debug("Printing queue:\n");53 pr_debug("Queue Type: %u\n", q->properties.type);54 pr_debug("Queue Size: %llu\n", q->properties.queue_size);55 pr_debug("Queue percent: %u\n", q->properties.queue_percent);56 pr_debug("Queue Address: 0x%llX\n", q->properties.queue_address);57 pr_debug("Queue Id: %u\n", q->properties.queue_id);58 pr_debug("Queue Process Vmid: %u\n", q->properties.vmid);59 pr_debug("Queue Read Pointer: 0x%px\n", q->properties.read_ptr);60 pr_debug("Queue Write Pointer: 0x%px\n", q->properties.write_ptr);61 pr_debug("Queue Doorbell Pointer: 0x%p\n", q->properties.doorbell_ptr);62 pr_debug("Queue Doorbell Offset: %u\n", q->properties.doorbell_off);63 pr_debug("Queue MQD Address: 0x%p\n", q->mqd);64 pr_debug("Queue MQD Gart: 0x%llX\n", q->gart_mqd_addr);65 pr_debug("Queue Process Address: 0x%p\n", q->process);66 pr_debug("Queue Device Address: 0x%p\n", q->device);67}68 69int init_queue(struct queue **q, const struct queue_properties *properties)70{71 struct queue *tmp_q;72 73 tmp_q = kzalloc(sizeof(*tmp_q), GFP_KERNEL);74 if (!tmp_q)75 return -ENOMEM;76 77 memcpy(&tmp_q->properties, properties, sizeof(*properties));78 79 *q = tmp_q;80 return 0;81}82 83void uninit_queue(struct queue *q)84{85 kfree(q);86}87 88#if IS_ENABLED(CONFIG_HSA_AMD_SVM)89 90static int kfd_queue_buffer_svm_get(struct kfd_process_device *pdd, u64 addr, u64 size)91{92 struct kfd_process *p = pdd->process;93 struct list_head update_list;94 struct svm_range *prange;95 int ret = -EINVAL;96 97 INIT_LIST_HEAD(&update_list);98 addr >>= PAGE_SHIFT;99 size >>= PAGE_SHIFT;100 101 mutex_lock(&p->svms.lock);102 103 /*104 * range may split to multiple svm pranges aligned to granularity boundaery.105 */106 while (size) {107 uint32_t gpuid, gpuidx;108 int r;109 110 prange = svm_range_from_addr(&p->svms, addr, NULL);111 if (!prange)112 break;113 114 if (!prange->mapped_to_gpu)115 break;116 117 r = kfd_process_gpuid_from_node(p, pdd->dev, &gpuid, &gpuidx);118 if (r < 0)119 break;120 if (!test_bit(gpuidx, prange->bitmap_access) &&121 !test_bit(gpuidx, prange->bitmap_aip))122 break;123 124 if (!(prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED))125 break;126 127 list_add(&prange->update_list, &update_list);128 129 if (prange->last - prange->start + 1 >= size) {130 size = 0;131 break;132 }133 134 size -= prange->last - prange->start + 1;135 addr += prange->last - prange->start + 1;136 }137 if (size) {138 pr_debug("[0x%llx 0x%llx] not registered\n", addr, addr + size - 1);139 goto out_unlock;140 }141 142 list_for_each_entry(prange, &update_list, update_list)143 atomic_inc(&prange->queue_refcount);144 ret = 0;145 146out_unlock:147 mutex_unlock(&p->svms.lock);148 return ret;149}150 151static void kfd_queue_buffer_svm_put(struct kfd_process_device *pdd, u64 addr, u64 size)152{153 struct kfd_process *p = pdd->process;154 struct svm_range *prange, *pchild;155 struct interval_tree_node *node;156 unsigned long last;157 158 addr >>= PAGE_SHIFT;159 last = addr + (size >> PAGE_SHIFT) - 1;160 161 mutex_lock(&p->svms.lock);162 163 node = interval_tree_iter_first(&p->svms.objects, addr, last);164 while (node) {165 struct interval_tree_node *next_node;166 unsigned long next_start;167 168 prange = container_of(node, struct svm_range, it_node);169 next_node = interval_tree_iter_next(node, addr, last);170 next_start = min(node->last, last) + 1;171 172 if (atomic_add_unless(&prange->queue_refcount, -1, 0)) {173 list_for_each_entry(pchild, &prange->child_list, child_list)174 atomic_add_unless(&pchild->queue_refcount, -1, 0);175 }176 177 node = next_node;178 addr = next_start;179 }180 181 mutex_unlock(&p->svms.lock);182}183#else184 185static int kfd_queue_buffer_svm_get(struct kfd_process_device *pdd, u64 addr, u64 size)186{187 return -EINVAL;188}189 190static void kfd_queue_buffer_svm_put(struct kfd_process_device *pdd, u64 addr, u64 size)191{192}193 194#endif195 196int kfd_queue_buffer_get(struct amdgpu_vm *vm, void __user *addr, struct amdgpu_bo **pbo,197 u64 expected_size)198{199 struct amdgpu_bo_va_mapping *mapping;200 u64 user_addr;201 u64 size;202 203 user_addr = (u64)addr >> AMDGPU_GPU_PAGE_SHIFT;204 size = expected_size >> AMDGPU_GPU_PAGE_SHIFT;205 206 mapping = amdgpu_vm_bo_lookup_mapping(vm, user_addr);207 if (!mapping)208 goto out_err;209 210 if (user_addr != mapping->start ||211 (size != 0 && user_addr + size - 1 != mapping->last)) {212 pr_debug("expected size 0x%llx not equal to mapping addr 0x%llx size 0x%llx\n",213 expected_size, mapping->start << AMDGPU_GPU_PAGE_SHIFT,214 (mapping->last - mapping->start + 1) << AMDGPU_GPU_PAGE_SHIFT);215 goto out_err;216 }217 218 *pbo = amdgpu_bo_ref(mapping->bo_va->base.bo);219 mapping->bo_va->queue_refcount++;220 return 0;221 222out_err:223 *pbo = NULL;224 return -EINVAL;225}226 227/* FIXME: remove this function, just call amdgpu_bo_unref directly */228void kfd_queue_buffer_put(struct amdgpu_bo **bo)229{230 amdgpu_bo_unref(bo);231}232 233int kfd_queue_acquire_buffers(struct kfd_process_device *pdd, struct queue_properties *properties)234{235 struct kfd_topology_device *topo_dev;236 struct amdgpu_vm *vm;237 u32 total_cwsr_size;238 int err;239 240 topo_dev = kfd_topology_device_by_id(pdd->dev->id);241 if (!topo_dev)242 return -EINVAL;243 244 vm = drm_priv_to_vm(pdd->drm_priv);245 err = amdgpu_bo_reserve(vm->root.bo, false);246 if (err)247 return err;248 249 err = kfd_queue_buffer_get(vm, properties->write_ptr, &properties->wptr_bo, PAGE_SIZE);250 if (err)251 goto out_err_unreserve;252 253 err = kfd_queue_buffer_get(vm, properties->read_ptr, &properties->rptr_bo, PAGE_SIZE);254 if (err)255 goto out_err_unreserve;256 257 err = kfd_queue_buffer_get(vm, (void *)properties->queue_address,258 &properties->ring_bo, properties->queue_size);259 if (err)260 goto out_err_unreserve;261 262 /* only compute queue requires EOP buffer and CWSR area */263 if (properties->type != KFD_QUEUE_TYPE_COMPUTE)264 goto out_unreserve;265 266 /* EOP buffer is not required for all ASICs */267 if (properties->eop_ring_buffer_address) {268 if (properties->eop_ring_buffer_size != topo_dev->node_props.eop_buffer_size) {269 pr_debug("queue eop bo size 0x%lx not equal to node eop buf size 0x%x\n",270 properties->eop_buf_bo->tbo.base.size,271 topo_dev->node_props.eop_buffer_size);272 err = -EINVAL;273 goto out_err_unreserve;274 }275 err = kfd_queue_buffer_get(vm, (void *)properties->eop_ring_buffer_address,276 &properties->eop_buf_bo,277 properties->eop_ring_buffer_size);278 if (err)279 goto out_err_unreserve;280 }281 282 if (properties->ctl_stack_size != topo_dev->node_props.ctl_stack_size) {283 pr_debug("queue ctl stack size 0x%x not equal to node ctl stack size 0x%x\n",284 properties->ctl_stack_size,285 topo_dev->node_props.ctl_stack_size);286 err = -EINVAL;287 goto out_err_unreserve;288 }289 290 if (properties->ctx_save_restore_area_size != topo_dev->node_props.cwsr_size) {291 pr_debug("queue cwsr size 0x%x not equal to node cwsr size 0x%x\n",292 properties->ctx_save_restore_area_size,293 topo_dev->node_props.cwsr_size);294 err = -EINVAL;295 goto out_err_unreserve;296 }297 298 total_cwsr_size = (topo_dev->node_props.cwsr_size + topo_dev->node_props.debug_memory_size)299 * NUM_XCC(pdd->dev->xcc_mask);300 total_cwsr_size = ALIGN(total_cwsr_size, PAGE_SIZE);301 302 err = kfd_queue_buffer_get(vm, (void *)properties->ctx_save_restore_area_address,303 &properties->cwsr_bo, total_cwsr_size);304 if (!err)305 goto out_unreserve;306 307 amdgpu_bo_unreserve(vm->root.bo);308 309 err = kfd_queue_buffer_svm_get(pdd, properties->ctx_save_restore_area_address,310 total_cwsr_size);311 if (err)312 goto out_err_release;313 314 return 0;315 316out_unreserve:317 amdgpu_bo_unreserve(vm->root.bo);318 return 0;319 320out_err_unreserve:321 amdgpu_bo_unreserve(vm->root.bo);322out_err_release:323 /* FIXME: make a _locked version of this that can be called before324 * dropping the VM reservation.325 */326 kfd_queue_unref_bo_vas(pdd, properties);327 kfd_queue_release_buffers(pdd, properties);328 return err;329}330 331int kfd_queue_release_buffers(struct kfd_process_device *pdd, struct queue_properties *properties)332{333 struct kfd_topology_device *topo_dev;334 u32 total_cwsr_size;335 336 kfd_queue_buffer_put(&properties->wptr_bo);337 kfd_queue_buffer_put(&properties->rptr_bo);338 kfd_queue_buffer_put(&properties->ring_bo);339 kfd_queue_buffer_put(&properties->eop_buf_bo);340 kfd_queue_buffer_put(&properties->cwsr_bo);341 342 topo_dev = kfd_topology_device_by_id(pdd->dev->id);343 if (!topo_dev)344 return -EINVAL;345 total_cwsr_size = (topo_dev->node_props.cwsr_size + topo_dev->node_props.debug_memory_size)346 * NUM_XCC(pdd->dev->xcc_mask);347 total_cwsr_size = ALIGN(total_cwsr_size, PAGE_SIZE);348 349 kfd_queue_buffer_svm_put(pdd, properties->ctx_save_restore_area_address, total_cwsr_size);350 return 0;351}352 353void kfd_queue_unref_bo_va(struct amdgpu_vm *vm, struct amdgpu_bo **bo)354{355 if (*bo) {356 struct amdgpu_bo_va *bo_va;357 358 bo_va = amdgpu_vm_bo_find(vm, *bo);359 if (bo_va && bo_va->queue_refcount)360 bo_va->queue_refcount--;361 }362}363 364int kfd_queue_unref_bo_vas(struct kfd_process_device *pdd,365 struct queue_properties *properties)366{367 struct amdgpu_vm *vm;368 int err;369 370 vm = drm_priv_to_vm(pdd->drm_priv);371 err = amdgpu_bo_reserve(vm->root.bo, false);372 if (err)373 return err;374 375 kfd_queue_unref_bo_va(vm, &properties->wptr_bo);376 kfd_queue_unref_bo_va(vm, &properties->rptr_bo);377 kfd_queue_unref_bo_va(vm, &properties->ring_bo);378 kfd_queue_unref_bo_va(vm, &properties->eop_buf_bo);379 kfd_queue_unref_bo_va(vm, &properties->cwsr_bo);380 381 amdgpu_bo_unreserve(vm->root.bo);382 return 0;383}384 385#define SGPR_SIZE_PER_CU 0x4000386#define LDS_SIZE_PER_CU 0x10000387#define HWREG_SIZE_PER_CU 0x1000388#define DEBUGGER_BYTES_ALIGN 64389#define DEBUGGER_BYTES_PER_WAVE 32390 391static u32 kfd_get_vgpr_size_per_cu(u32 gfxv)392{393 u32 vgpr_size = 0x40000;394 395 if ((gfxv / 100 * 100) == 90400 || /* GFX_VERSION_AQUA_VANJARAM */396 gfxv == 90010 || /* GFX_VERSION_ALDEBARAN */397 gfxv == 90008) /* GFX_VERSION_ARCTURUS */398 vgpr_size = 0x80000;399 else if (gfxv == 110000 || /* GFX_VERSION_PLUM_BONITO */400 gfxv == 110001 || /* GFX_VERSION_WHEAT_NAS */401 gfxv == 120000 || /* GFX_VERSION_GFX1200 */402 gfxv == 120001) /* GFX_VERSION_GFX1201 */403 vgpr_size = 0x60000;404 405 return vgpr_size;406}407 408#define WG_CONTEXT_DATA_SIZE_PER_CU(gfxv) \409 (kfd_get_vgpr_size_per_cu(gfxv) + SGPR_SIZE_PER_CU +\410 LDS_SIZE_PER_CU + HWREG_SIZE_PER_CU)411 412#define CNTL_STACK_BYTES_PER_WAVE(gfxv) \413 ((gfxv) >= 100100 ? 12 : 8) /* GFX_VERSION_NAVI10*/414 415#define SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER 40416 417void kfd_queue_ctx_save_restore_size(struct kfd_topology_device *dev)418{419 struct kfd_node_properties *props = &dev->node_props;420 u32 gfxv = props->gfx_target_version;421 u32 ctl_stack_size;422 u32 wg_data_size;423 u32 wave_num;424 u32 cu_num;425 426 if (gfxv < 80001) /* GFX_VERSION_CARRIZO */427 return;428 429 cu_num = props->simd_count / props->simd_per_cu / NUM_XCC(dev->gpu->xcc_mask);430 wave_num = (gfxv < 100100) ? /* GFX_VERSION_NAVI10 */431 min(cu_num * 40, props->array_count / props->simd_arrays_per_engine * 512)432 : cu_num * 32;433 434 wg_data_size = ALIGN(cu_num * WG_CONTEXT_DATA_SIZE_PER_CU(gfxv), PAGE_SIZE);435 ctl_stack_size = wave_num * CNTL_STACK_BYTES_PER_WAVE(gfxv) + 8;436 ctl_stack_size = ALIGN(SIZEOF_HSA_USER_CONTEXT_SAVE_AREA_HEADER + ctl_stack_size,437 PAGE_SIZE);438 439 if ((gfxv / 10000 * 10000) == 100000) {440 /* HW design limits control stack size to 0x7000.441 * This is insufficient for theoretical PM4 cases442 * but sufficient for AQL, limited by SPI events.443 */444 ctl_stack_size = min(ctl_stack_size, 0x7000);445 }446 447 props->ctl_stack_size = ctl_stack_size;448 props->debug_memory_size = ALIGN(wave_num * DEBUGGER_BYTES_PER_WAVE, DEBUGGER_BYTES_ALIGN);449 props->cwsr_size = ctl_stack_size + wg_data_size;450 451 if (gfxv == 80002) /* GFX_VERSION_TONGA */452 props->eop_buffer_size = 0x8000;453 else if ((gfxv / 100 * 100) == 90400) /* GFX_VERSION_AQUA_VANJARAM */454 props->eop_buffer_size = 4096;455 else if (gfxv >= 80000)456 props->eop_buffer_size = 4096;457}458