1504 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR MIT2/*3 * Copyright 2014-2022 Advanced Micro Devices, Inc.4 *5 * Permission is hereby granted, free of charge, to any person obtaining a6 * copy of this software and associated documentation files (the "Software"),7 * to deal in the Software without restriction, including without limitation8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,9 * and/or sell copies of the Software, and to permit persons to whom the10 * Software is furnished to do so, subject to the following conditions:11 *12 * The above copyright notice and this permission notice shall be included in13 * all copies or substantial portions of the Software.14 *15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR21 * OTHER DEALINGS IN THE SOFTWARE.22 */23 24#include <linux/bsearch.h>25#include <linux/pci.h>26#include <linux/slab.h>27#include "kfd_priv.h"28#include "kfd_device_queue_manager.h"29#include "kfd_pm4_headers_vi.h"30#include "kfd_pm4_headers_aldebaran.h"31#include "cwsr_trap_handler.h"32#include "amdgpu_amdkfd.h"33#include "kfd_smi_events.h"34#include "kfd_svm.h"35#include "kfd_migrate.h"36#include "amdgpu.h"37#include "amdgpu_xcp.h"38 39#define MQD_SIZE_ALIGNED 76840 41/*42 * kfd_locked is used to lock the kfd driver during suspend or reset43 * once locked, kfd driver will stop any further GPU execution.44 * create process (open) will return -EAGAIN.45 */46static int kfd_locked;47 48#ifdef CONFIG_DRM_AMDGPU_CIK49extern const struct kfd2kgd_calls gfx_v7_kfd2kgd;50#endif51extern const struct kfd2kgd_calls gfx_v8_kfd2kgd;52extern const struct kfd2kgd_calls gfx_v9_kfd2kgd;53extern const struct kfd2kgd_calls arcturus_kfd2kgd;54extern const struct kfd2kgd_calls aldebaran_kfd2kgd;55extern const struct kfd2kgd_calls gc_9_4_3_kfd2kgd;56extern const struct kfd2kgd_calls gfx_v10_kfd2kgd;57extern const struct kfd2kgd_calls gfx_v10_3_kfd2kgd;58extern const struct kfd2kgd_calls gfx_v11_kfd2kgd;59extern const struct kfd2kgd_calls gfx_v12_kfd2kgd;60 61static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,62 unsigned int chunk_size);63static void kfd_gtt_sa_fini(struct kfd_dev *kfd);64 65static int kfd_resume(struct kfd_node *kfd);66 67static void kfd_device_info_set_sdma_info(struct kfd_dev *kfd)68{69 uint32_t sdma_version = amdgpu_ip_version(kfd->adev, SDMA0_HWIP, 0);70 71 switch (sdma_version) {72 case IP_VERSION(4, 0, 0):/* VEGA10 */73 case IP_VERSION(4, 0, 1):/* VEGA12 */74 case IP_VERSION(4, 1, 0):/* RAVEN */75 case IP_VERSION(4, 1, 1):/* RAVEN */76 case IP_VERSION(4, 1, 2):/* RENOIR */77 case IP_VERSION(5, 2, 1):/* VANGOGH */78 case IP_VERSION(5, 2, 3):/* YELLOW_CARP */79 case IP_VERSION(5, 2, 6):/* GC 10.3.6 */80 case IP_VERSION(5, 2, 7):/* GC 10.3.7 */81 kfd->device_info.num_sdma_queues_per_engine = 2;82 break;83 case IP_VERSION(4, 2, 0):/* VEGA20 */84 case IP_VERSION(4, 2, 2):/* ARCTURUS */85 case IP_VERSION(4, 4, 0):/* ALDEBARAN */86 case IP_VERSION(4, 4, 2):87 case IP_VERSION(4, 4, 5):88 case IP_VERSION(5, 0, 0):/* NAVI10 */89 case IP_VERSION(5, 0, 1):/* CYAN_SKILLFISH */90 case IP_VERSION(5, 0, 2):/* NAVI14 */91 case IP_VERSION(5, 0, 5):/* NAVI12 */92 case IP_VERSION(5, 2, 0):/* SIENNA_CICHLID */93 case IP_VERSION(5, 2, 2):/* NAVY_FLOUNDER */94 case IP_VERSION(5, 2, 4):/* DIMGREY_CAVEFISH */95 case IP_VERSION(5, 2, 5):/* BEIGE_GOBY */96 case IP_VERSION(6, 0, 0):97 case IP_VERSION(6, 0, 1):98 case IP_VERSION(6, 0, 2):99 case IP_VERSION(6, 0, 3):100 case IP_VERSION(6, 1, 0):101 case IP_VERSION(6, 1, 1):102 case IP_VERSION(6, 1, 2):103 case IP_VERSION(7, 0, 0):104 case IP_VERSION(7, 0, 1):105 kfd->device_info.num_sdma_queues_per_engine = 8;106 break;107 default:108 dev_warn(kfd_device,109 "Default sdma queue per engine(8) is set due to mismatch of sdma ip block(SDMA_HWIP:0x%x).\n",110 sdma_version);111 kfd->device_info.num_sdma_queues_per_engine = 8;112 }113 114 bitmap_zero(kfd->device_info.reserved_sdma_queues_bitmap, KFD_MAX_SDMA_QUEUES);115 116 switch (sdma_version) {117 case IP_VERSION(6, 0, 0):118 case IP_VERSION(6, 0, 1):119 case IP_VERSION(6, 0, 2):120 case IP_VERSION(6, 0, 3):121 case IP_VERSION(6, 1, 0):122 case IP_VERSION(6, 1, 1):123 case IP_VERSION(6, 1, 2):124 case IP_VERSION(7, 0, 0):125 case IP_VERSION(7, 0, 1):126 /* Reserve 1 for paging and 1 for gfx */127 kfd->device_info.num_reserved_sdma_queues_per_engine = 2;128 /* BIT(0)=engine-0 queue-0; BIT(1)=engine-1 queue-0; BIT(2)=engine-0 queue-1; ... */129 bitmap_set(kfd->device_info.reserved_sdma_queues_bitmap, 0,130 kfd->adev->sdma.num_instances *131 kfd->device_info.num_reserved_sdma_queues_per_engine);132 break;133 default:134 break;135 }136}137 138static void kfd_device_info_set_event_interrupt_class(struct kfd_dev *kfd)139{140 uint32_t gc_version = KFD_GC_VERSION(kfd);141 142 switch (gc_version) {143 case IP_VERSION(9, 0, 1): /* VEGA10 */144 case IP_VERSION(9, 1, 0): /* RAVEN */145 case IP_VERSION(9, 2, 1): /* VEGA12 */146 case IP_VERSION(9, 2, 2): /* RAVEN */147 case IP_VERSION(9, 3, 0): /* RENOIR */148 case IP_VERSION(9, 4, 0): /* VEGA20 */149 case IP_VERSION(9, 4, 1): /* ARCTURUS */150 case IP_VERSION(9, 4, 2): /* ALDEBARAN */151 kfd->device_info.event_interrupt_class = &event_interrupt_class_v9;152 break;153 case IP_VERSION(9, 4, 3): /* GC 9.4.3 */154 case IP_VERSION(9, 4, 4): /* GC 9.4.4 */155 kfd->device_info.event_interrupt_class =156 &event_interrupt_class_v9_4_3;157 break;158 case IP_VERSION(10, 3, 1): /* VANGOGH */159 case IP_VERSION(10, 3, 3): /* YELLOW_CARP */160 case IP_VERSION(10, 3, 6): /* GC 10.3.6 */161 case IP_VERSION(10, 3, 7): /* GC 10.3.7 */162 case IP_VERSION(10, 1, 3): /* CYAN_SKILLFISH */163 case IP_VERSION(10, 1, 4):164 case IP_VERSION(10, 1, 10): /* NAVI10 */165 case IP_VERSION(10, 1, 2): /* NAVI12 */166 case IP_VERSION(10, 1, 1): /* NAVI14 */167 case IP_VERSION(10, 3, 0): /* SIENNA_CICHLID */168 case IP_VERSION(10, 3, 2): /* NAVY_FLOUNDER */169 case IP_VERSION(10, 3, 4): /* DIMGREY_CAVEFISH */170 case IP_VERSION(10, 3, 5): /* BEIGE_GOBY */171 kfd->device_info.event_interrupt_class = &event_interrupt_class_v10;172 break;173 case IP_VERSION(11, 0, 0):174 case IP_VERSION(11, 0, 1):175 case IP_VERSION(11, 0, 2):176 case IP_VERSION(11, 0, 3):177 case IP_VERSION(11, 0, 4):178 case IP_VERSION(11, 5, 0):179 case IP_VERSION(11, 5, 1):180 case IP_VERSION(11, 5, 2):181 kfd->device_info.event_interrupt_class = &event_interrupt_class_v11;182 break;183 case IP_VERSION(12, 0, 0):184 case IP_VERSION(12, 0, 1):185 /* GFX12_TODO: Change to v12 version. */186 kfd->device_info.event_interrupt_class = &event_interrupt_class_v11;187 break;188 default:189 dev_warn(kfd_device, "v9 event interrupt handler is set due to "190 "mismatch of gc ip block(GC_HWIP:0x%x).\n", gc_version);191 kfd->device_info.event_interrupt_class = &event_interrupt_class_v9;192 }193}194 195static void kfd_device_info_init(struct kfd_dev *kfd,196 bool vf, uint32_t gfx_target_version)197{198 uint32_t gc_version = KFD_GC_VERSION(kfd);199 uint32_t asic_type = kfd->adev->asic_type;200 201 kfd->device_info.max_pasid_bits = 16;202 kfd->device_info.max_no_of_hqd = 24;203 kfd->device_info.num_of_watch_points = 4;204 kfd->device_info.mqd_size_aligned = MQD_SIZE_ALIGNED;205 kfd->device_info.gfx_target_version = gfx_target_version;206 207 if (KFD_IS_SOC15(kfd)) {208 kfd->device_info.doorbell_size = 8;209 kfd->device_info.ih_ring_entry_size = 8 * sizeof(uint32_t);210 kfd->device_info.supports_cwsr = true;211 212 kfd_device_info_set_sdma_info(kfd);213 214 kfd_device_info_set_event_interrupt_class(kfd);215 216 if (gc_version < IP_VERSION(11, 0, 0)) {217 /* Navi2x+, Navi1x+ */218 if (gc_version == IP_VERSION(10, 3, 6))219 kfd->device_info.no_atomic_fw_version = 14;220 else if (gc_version == IP_VERSION(10, 3, 7))221 kfd->device_info.no_atomic_fw_version = 3;222 else if (gc_version >= IP_VERSION(10, 3, 0))223 kfd->device_info.no_atomic_fw_version = 92;224 else if (gc_version >= IP_VERSION(10, 1, 1))225 kfd->device_info.no_atomic_fw_version = 145;226 227 /* Navi1x+ */228 if (gc_version >= IP_VERSION(10, 1, 1))229 kfd->device_info.needs_pci_atomics = true;230 } else if (gc_version < IP_VERSION(12, 0, 0)) {231 /*232 * PCIe atomics support acknowledgment in GFX11 RS64 CPFW requires233 * MEC version >= 509. Prior RS64 CPFW versions (and all F32) require234 * PCIe atomics support.235 */236 kfd->device_info.needs_pci_atomics = true;237 kfd->device_info.no_atomic_fw_version = kfd->adev->gfx.rs64_enable ? 509 : 0;238 } else {239 kfd->device_info.needs_pci_atomics = true;240 }241 } else {242 kfd->device_info.doorbell_size = 4;243 kfd->device_info.ih_ring_entry_size = 4 * sizeof(uint32_t);244 kfd->device_info.event_interrupt_class = &event_interrupt_class_cik;245 kfd->device_info.num_sdma_queues_per_engine = 2;246 247 if (asic_type != CHIP_KAVERI &&248 asic_type != CHIP_HAWAII &&249 asic_type != CHIP_TONGA)250 kfd->device_info.supports_cwsr = true;251 252 if (asic_type != CHIP_HAWAII && !vf)253 kfd->device_info.needs_pci_atomics = true;254 }255}256 257struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf)258{259 struct kfd_dev *kfd = NULL;260 const struct kfd2kgd_calls *f2g = NULL;261 uint32_t gfx_target_version = 0;262 263 switch (adev->asic_type) {264#ifdef CONFIG_DRM_AMDGPU_CIK265 case CHIP_KAVERI:266 gfx_target_version = 70000;267 if (!vf)268 f2g = &gfx_v7_kfd2kgd;269 break;270#endif271 case CHIP_CARRIZO:272 gfx_target_version = 80001;273 if (!vf)274 f2g = &gfx_v8_kfd2kgd;275 break;276#ifdef CONFIG_DRM_AMDGPU_CIK277 case CHIP_HAWAII:278 gfx_target_version = 70001;279 if (!amdgpu_exp_hw_support)280 pr_info(281 "KFD support on Hawaii is experimental. See modparam exp_hw_support\n"282 );283 else if (!vf)284 f2g = &gfx_v7_kfd2kgd;285 break;286#endif287 case CHIP_TONGA:288 gfx_target_version = 80002;289 if (!vf)290 f2g = &gfx_v8_kfd2kgd;291 break;292 case CHIP_FIJI:293 case CHIP_POLARIS10:294 gfx_target_version = 80003;295 f2g = &gfx_v8_kfd2kgd;296 break;297 case CHIP_POLARIS11:298 case CHIP_POLARIS12:299 case CHIP_VEGAM:300 gfx_target_version = 80003;301 if (!vf)302 f2g = &gfx_v8_kfd2kgd;303 break;304 default:305 switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {306 /* Vega 10 */307 case IP_VERSION(9, 0, 1):308 gfx_target_version = 90000;309 f2g = &gfx_v9_kfd2kgd;310 break;311 /* Raven */312 case IP_VERSION(9, 1, 0):313 case IP_VERSION(9, 2, 2):314 gfx_target_version = 90002;315 if (!vf)316 f2g = &gfx_v9_kfd2kgd;317 break;318 /* Vega12 */319 case IP_VERSION(9, 2, 1):320 gfx_target_version = 90004;321 if (!vf)322 f2g = &gfx_v9_kfd2kgd;323 break;324 /* Renoir */325 case IP_VERSION(9, 3, 0):326 gfx_target_version = 90012;327 if (!vf)328 f2g = &gfx_v9_kfd2kgd;329 break;330 /* Vega20 */331 case IP_VERSION(9, 4, 0):332 gfx_target_version = 90006;333 if (!vf)334 f2g = &gfx_v9_kfd2kgd;335 break;336 /* Arcturus */337 case IP_VERSION(9, 4, 1):338 gfx_target_version = 90008;339 f2g = &arcturus_kfd2kgd;340 break;341 /* Aldebaran */342 case IP_VERSION(9, 4, 2):343 gfx_target_version = 90010;344 f2g = &aldebaran_kfd2kgd;345 break;346 case IP_VERSION(9, 4, 3):347 gfx_target_version = adev->rev_id >= 1 ? 90402348 : adev->flags & AMD_IS_APU ? 90400349 : 90401;350 f2g = &gc_9_4_3_kfd2kgd;351 break;352 case IP_VERSION(9, 4, 4):353 gfx_target_version = 90402;354 f2g = &gc_9_4_3_kfd2kgd;355 break;356 /* Navi10 */357 case IP_VERSION(10, 1, 10):358 gfx_target_version = 100100;359 if (!vf)360 f2g = &gfx_v10_kfd2kgd;361 break;362 /* Navi12 */363 case IP_VERSION(10, 1, 2):364 gfx_target_version = 100101;365 f2g = &gfx_v10_kfd2kgd;366 break;367 /* Navi14 */368 case IP_VERSION(10, 1, 1):369 gfx_target_version = 100102;370 if (!vf)371 f2g = &gfx_v10_kfd2kgd;372 break;373 /* Cyan Skillfish */374 case IP_VERSION(10, 1, 3):375 case IP_VERSION(10, 1, 4):376 gfx_target_version = 100103;377 if (!vf)378 f2g = &gfx_v10_kfd2kgd;379 break;380 /* Sienna Cichlid */381 case IP_VERSION(10, 3, 0):382 gfx_target_version = 100300;383 f2g = &gfx_v10_3_kfd2kgd;384 break;385 /* Navy Flounder */386 case IP_VERSION(10, 3, 2):387 gfx_target_version = 100301;388 f2g = &gfx_v10_3_kfd2kgd;389 break;390 /* Van Gogh */391 case IP_VERSION(10, 3, 1):392 gfx_target_version = 100303;393 if (!vf)394 f2g = &gfx_v10_3_kfd2kgd;395 break;396 /* Dimgrey Cavefish */397 case IP_VERSION(10, 3, 4):398 gfx_target_version = 100302;399 f2g = &gfx_v10_3_kfd2kgd;400 break;401 /* Beige Goby */402 case IP_VERSION(10, 3, 5):403 gfx_target_version = 100304;404 f2g = &gfx_v10_3_kfd2kgd;405 break;406 /* Yellow Carp */407 case IP_VERSION(10, 3, 3):408 gfx_target_version = 100305;409 if (!vf)410 f2g = &gfx_v10_3_kfd2kgd;411 break;412 case IP_VERSION(10, 3, 6):413 case IP_VERSION(10, 3, 7):414 gfx_target_version = 100306;415 if (!vf)416 f2g = &gfx_v10_3_kfd2kgd;417 break;418 case IP_VERSION(11, 0, 0):419 gfx_target_version = 110000;420 f2g = &gfx_v11_kfd2kgd;421 break;422 case IP_VERSION(11, 0, 1):423 case IP_VERSION(11, 0, 4):424 gfx_target_version = 110003;425 f2g = &gfx_v11_kfd2kgd;426 break;427 case IP_VERSION(11, 0, 2):428 gfx_target_version = 110002;429 f2g = &gfx_v11_kfd2kgd;430 break;431 case IP_VERSION(11, 0, 3):432 /* Note: Compiler version is 11.0.1 while HW version is 11.0.3 */433 gfx_target_version = 110001;434 f2g = &gfx_v11_kfd2kgd;435 break;436 case IP_VERSION(11, 5, 0):437 gfx_target_version = 110500;438 f2g = &gfx_v11_kfd2kgd;439 break;440 case IP_VERSION(11, 5, 1):441 gfx_target_version = 110501;442 f2g = &gfx_v11_kfd2kgd;443 break;444 case IP_VERSION(11, 5, 2):445 gfx_target_version = 110502;446 f2g = &gfx_v11_kfd2kgd;447 break;448 case IP_VERSION(12, 0, 0):449 gfx_target_version = 120000;450 f2g = &gfx_v12_kfd2kgd;451 break;452 case IP_VERSION(12, 0, 1):453 gfx_target_version = 120001;454 f2g = &gfx_v12_kfd2kgd;455 break;456 default:457 break;458 }459 break;460 }461 462 if (!f2g) {463 if (amdgpu_ip_version(adev, GC_HWIP, 0))464 dev_info(kfd_device,465 "GC IP %06x %s not supported in kfd\n",466 amdgpu_ip_version(adev, GC_HWIP, 0),467 vf ? "VF" : "");468 else469 dev_info(kfd_device, "%s %s not supported in kfd\n",470 amdgpu_asic_name[adev->asic_type], vf ? "VF" : "");471 return NULL;472 }473 474 kfd = kzalloc(sizeof(*kfd), GFP_KERNEL);475 if (!kfd)476 return NULL;477 478 kfd->adev = adev;479 kfd_device_info_init(kfd, vf, gfx_target_version);480 kfd->init_complete = false;481 kfd->kfd2kgd = f2g;482 atomic_set(&kfd->compute_profile, 0);483 484 mutex_init(&kfd->doorbell_mutex);485 486 ida_init(&kfd->doorbell_ida);487 488 return kfd;489}490 491static void kfd_cwsr_init(struct kfd_dev *kfd)492{493 if (cwsr_enable && kfd->device_info.supports_cwsr) {494 if (KFD_GC_VERSION(kfd) < IP_VERSION(9, 0, 1)) {495 BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex)496 > KFD_CWSR_TMA_OFFSET);497 kfd->cwsr_isa = cwsr_trap_gfx8_hex;498 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx8_hex);499 } else if (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 1)) {500 BUILD_BUG_ON(sizeof(cwsr_trap_arcturus_hex)501 > KFD_CWSR_TMA_OFFSET);502 kfd->cwsr_isa = cwsr_trap_arcturus_hex;503 kfd->cwsr_isa_size = sizeof(cwsr_trap_arcturus_hex);504 } else if (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 2)) {505 BUILD_BUG_ON(sizeof(cwsr_trap_aldebaran_hex)506 > KFD_CWSR_TMA_OFFSET);507 kfd->cwsr_isa = cwsr_trap_aldebaran_hex;508 kfd->cwsr_isa_size = sizeof(cwsr_trap_aldebaran_hex);509 } else if (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 3) ||510 KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 4)) {511 BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_4_3_hex)512 > KFD_CWSR_TMA_OFFSET);513 kfd->cwsr_isa = cwsr_trap_gfx9_4_3_hex;514 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_4_3_hex);515 } else if (KFD_GC_VERSION(kfd) < IP_VERSION(10, 1, 1)) {516 BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex)517 > KFD_CWSR_TMA_OFFSET);518 kfd->cwsr_isa = cwsr_trap_gfx9_hex;519 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_hex);520 } else if (KFD_GC_VERSION(kfd) < IP_VERSION(10, 3, 0)) {521 BUILD_BUG_ON(sizeof(cwsr_trap_nv1x_hex)522 > KFD_CWSR_TMA_OFFSET);523 kfd->cwsr_isa = cwsr_trap_nv1x_hex;524 kfd->cwsr_isa_size = sizeof(cwsr_trap_nv1x_hex);525 } else if (KFD_GC_VERSION(kfd) < IP_VERSION(11, 0, 0)) {526 BUILD_BUG_ON(sizeof(cwsr_trap_gfx10_hex)527 > KFD_CWSR_TMA_OFFSET);528 kfd->cwsr_isa = cwsr_trap_gfx10_hex;529 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx10_hex);530 } else if (KFD_GC_VERSION(kfd) < IP_VERSION(12, 0, 0)) {531 /* The gfx11 cwsr trap handler must fit inside a single532 page. */533 BUILD_BUG_ON(sizeof(cwsr_trap_gfx11_hex) > PAGE_SIZE);534 kfd->cwsr_isa = cwsr_trap_gfx11_hex;535 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx11_hex);536 } else {537 BUILD_BUG_ON(sizeof(cwsr_trap_gfx12_hex) > PAGE_SIZE);538 kfd->cwsr_isa = cwsr_trap_gfx12_hex;539 kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx12_hex);540 }541 542 kfd->cwsr_enabled = true;543 }544}545 546static int kfd_gws_init(struct kfd_node *node)547{548 int ret = 0;549 struct kfd_dev *kfd = node->kfd;550 uint32_t mes_rev = node->adev->mes.sched_version & AMDGPU_MES_VERSION_MASK;551 552 if (node->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS)553 return 0;554 555 if (hws_gws_support || (KFD_IS_SOC15(node) &&556 ((KFD_GC_VERSION(node) == IP_VERSION(9, 0, 1)557 && kfd->mec2_fw_version >= 0x81b3) ||558 (KFD_GC_VERSION(node) <= IP_VERSION(9, 4, 0)559 && kfd->mec2_fw_version >= 0x1b3) ||560 (KFD_GC_VERSION(node) == IP_VERSION(9, 4, 1)561 && kfd->mec2_fw_version >= 0x30) ||562 (KFD_GC_VERSION(node) == IP_VERSION(9, 4, 2)563 && kfd->mec2_fw_version >= 0x28) ||564 (KFD_GC_VERSION(node) == IP_VERSION(9, 4, 3) ||565 KFD_GC_VERSION(node) == IP_VERSION(9, 4, 4)) ||566 (KFD_GC_VERSION(node) >= IP_VERSION(10, 3, 0)567 && KFD_GC_VERSION(node) < IP_VERSION(11, 0, 0)568 && kfd->mec2_fw_version >= 0x6b) ||569 (KFD_GC_VERSION(node) >= IP_VERSION(11, 0, 0)570 && KFD_GC_VERSION(node) < IP_VERSION(12, 0, 0)571 && mes_rev >= 68))))572 ret = amdgpu_amdkfd_alloc_gws(node->adev,573 node->adev->gds.gws_size, &node->gws);574 575 return ret;576}577 578static void kfd_smi_init(struct kfd_node *dev)579{580 INIT_LIST_HEAD(&dev->smi_clients);581 spin_lock_init(&dev->smi_lock);582}583 584static int kfd_init_node(struct kfd_node *node)585{586 int err = -1;587 588 if (kfd_interrupt_init(node)) {589 dev_err(kfd_device, "Error initializing interrupts\n");590 goto kfd_interrupt_error;591 }592 593 node->dqm = device_queue_manager_init(node);594 if (!node->dqm) {595 dev_err(kfd_device, "Error initializing queue manager\n");596 goto device_queue_manager_error;597 }598 599 if (kfd_gws_init(node)) {600 dev_err(kfd_device, "Could not allocate %d gws\n",601 node->adev->gds.gws_size);602 goto gws_error;603 }604 605 if (kfd_resume(node))606 goto kfd_resume_error;607 608 if (kfd_topology_add_device(node)) {609 dev_err(kfd_device, "Error adding device to topology\n");610 goto kfd_topology_add_device_error;611 }612 613 kfd_smi_init(node);614 615 return 0;616 617kfd_topology_add_device_error:618kfd_resume_error:619gws_error:620 device_queue_manager_uninit(node->dqm);621device_queue_manager_error:622 kfd_interrupt_exit(node);623kfd_interrupt_error:624 if (node->gws)625 amdgpu_amdkfd_free_gws(node->adev, node->gws);626 627 /* Cleanup the node memory here */628 kfree(node);629 return err;630}631 632static void kfd_cleanup_nodes(struct kfd_dev *kfd, unsigned int num_nodes)633{634 struct kfd_node *knode;635 unsigned int i;636 637 for (i = 0; i < num_nodes; i++) {638 knode = kfd->nodes[i];639 device_queue_manager_uninit(knode->dqm);640 kfd_interrupt_exit(knode);641 kfd_topology_remove_device(knode);642 if (knode->gws)643 amdgpu_amdkfd_free_gws(knode->adev, knode->gws);644 kfree(knode);645 kfd->nodes[i] = NULL;646 }647}648 649static void kfd_setup_interrupt_bitmap(struct kfd_node *node,650 unsigned int kfd_node_idx)651{652 struct amdgpu_device *adev = node->adev;653 uint32_t xcc_mask = node->xcc_mask;654 uint32_t xcc, mapped_xcc;655 /*656 * Interrupt bitmap is setup for processing interrupts from657 * different XCDs and AIDs.658 * Interrupt bitmap is defined as follows:659 * 1. Bits 0-15 - correspond to the NodeId field.660 * Each bit corresponds to NodeId number. For example, if661 * a KFD node has interrupt bitmap set to 0x7, then this662 * KFD node will process interrupts with NodeId = 0, 1 and 2663 * in the IH cookie.664 * 2. Bits 16-31 - unused.665 *666 * Please note that the kfd_node_idx argument passed to this667 * function is not related to NodeId field received in the668 * IH cookie.669 *670 * In CPX mode, a KFD node will process an interrupt if:671 * - the Node Id matches the corresponding bit set in672 * Bits 0-15.673 * - AND VMID reported in the interrupt lies within the674 * VMID range of the node.675 */676 for_each_inst(xcc, xcc_mask) {677 mapped_xcc = GET_INST(GC, xcc);678 node->interrupt_bitmap |= (mapped_xcc % 2 ? 5 : 3) << (4 * (mapped_xcc / 2));679 }680 dev_info(kfd_device, "Node: %d, interrupt_bitmap: %x\n", kfd_node_idx,681 node->interrupt_bitmap);682}683 684bool kgd2kfd_device_init(struct kfd_dev *kfd,685 const struct kgd2kfd_shared_resources *gpu_resources)686{687 unsigned int size, map_process_packet_size, i;688 struct kfd_node *node;689 uint32_t first_vmid_kfd, last_vmid_kfd, vmid_num_kfd;690 unsigned int max_proc_per_quantum;691 int partition_mode;692 int xcp_idx;693 694 kfd->mec_fw_version = amdgpu_amdkfd_get_fw_version(kfd->adev,695 KGD_ENGINE_MEC1);696 kfd->mec2_fw_version = amdgpu_amdkfd_get_fw_version(kfd->adev,697 KGD_ENGINE_MEC2);698 kfd->sdma_fw_version = amdgpu_amdkfd_get_fw_version(kfd->adev,699 KGD_ENGINE_SDMA1);700 kfd->shared_resources = *gpu_resources;701 702 kfd->num_nodes = amdgpu_xcp_get_num_xcp(kfd->adev->xcp_mgr);703 704 if (kfd->num_nodes == 0) {705 dev_err(kfd_device,706 "KFD num nodes cannot be 0, num_xcc_in_node: %d\n",707 kfd->adev->gfx.num_xcc_per_xcp);708 goto out;709 }710 711 /* Allow BIF to recode atomics to PCIe 3.0 AtomicOps.712 * 32 and 64-bit requests are possible and must be713 * supported.714 */715 kfd->pci_atomic_requested = amdgpu_amdkfd_have_atomics_support(kfd->adev);716 if (!kfd->pci_atomic_requested &&717 kfd->device_info.needs_pci_atomics &&718 (!kfd->device_info.no_atomic_fw_version ||719 kfd->mec_fw_version < kfd->device_info.no_atomic_fw_version)) {720 dev_info(kfd_device,721 "skipped device %x:%x, PCI rejects atomics %d<%d\n",722 kfd->adev->pdev->vendor, kfd->adev->pdev->device,723 kfd->mec_fw_version,724 kfd->device_info.no_atomic_fw_version);725 return false;726 }727 728 first_vmid_kfd = ffs(gpu_resources->compute_vmid_bitmap)-1;729 last_vmid_kfd = fls(gpu_resources->compute_vmid_bitmap)-1;730 vmid_num_kfd = last_vmid_kfd - first_vmid_kfd + 1;731 732 /* For GFX9.4.3, we need special handling for VMIDs depending on733 * partition mode.734 * In CPX mode, the VMID range needs to be shared between XCDs.735 * Additionally, there are 13 VMIDs (3-15) available for KFD. To736 * divide them equally, we change starting VMID to 4 and not use737 * VMID 3.738 * If the VMID range changes for GFX9.4.3, then this code MUST be739 * revisited.740 */741 if (kfd->adev->xcp_mgr) {742 partition_mode = amdgpu_xcp_query_partition_mode(kfd->adev->xcp_mgr,743 AMDGPU_XCP_FL_LOCKED);744 if (partition_mode == AMDGPU_CPX_PARTITION_MODE &&745 kfd->num_nodes != 1) {746 vmid_num_kfd /= 2;747 first_vmid_kfd = last_vmid_kfd + 1 - vmid_num_kfd*2;748 }749 }750 751 /* Verify module parameters regarding mapped process number*/752 if (hws_max_conc_proc >= 0)753 max_proc_per_quantum = min((u32)hws_max_conc_proc, vmid_num_kfd);754 else755 max_proc_per_quantum = vmid_num_kfd;756 757 /* calculate max size of mqds needed for queues */758 size = max_num_of_queues_per_device *759 kfd->device_info.mqd_size_aligned;760 761 /*762 * calculate max size of runlist packet.763 * There can be only 2 packets at once764 */765 map_process_packet_size = KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 2) ?766 sizeof(struct pm4_mes_map_process_aldebaran) :767 sizeof(struct pm4_mes_map_process);768 size += (KFD_MAX_NUM_OF_PROCESSES * map_process_packet_size +769 max_num_of_queues_per_device * sizeof(struct pm4_mes_map_queues)770 + sizeof(struct pm4_mes_runlist)) * 2;771 772 /* Add size of HIQ & DIQ */773 size += KFD_KERNEL_QUEUE_SIZE * 2;774 775 /* add another 512KB for all other allocations on gart (HPD, fences) */776 size += 512 * 1024;777 778 if (amdgpu_amdkfd_alloc_gtt_mem(779 kfd->adev, size, &kfd->gtt_mem,780 &kfd->gtt_start_gpu_addr, &kfd->gtt_start_cpu_ptr,781 false)) {782 dev_err(kfd_device, "Could not allocate %d bytes\n", size);783 goto alloc_gtt_mem_failure;784 }785 786 dev_info(kfd_device, "Allocated %d bytes on gart\n", size);787 788 /* Initialize GTT sa with 512 byte chunk size */789 if (kfd_gtt_sa_init(kfd, size, 512) != 0) {790 dev_err(kfd_device, "Error initializing gtt sub-allocator\n");791 goto kfd_gtt_sa_init_error;792 }793 794 if (kfd_doorbell_init(kfd)) {795 dev_err(kfd_device,796 "Error initializing doorbell aperture\n");797 goto kfd_doorbell_error;798 }799 800 if (amdgpu_use_xgmi_p2p)801 kfd->hive_id = kfd->adev->gmc.xgmi.hive_id;802 803 /*804 * For GFX9.4.3, the KFD abstracts all partitions within a socket as805 * xGMI connected in the topology so assign a unique hive id per806 * device based on the pci device location if device is in PCIe mode.807 */808 if (!kfd->hive_id &&809 (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 3) ||810 KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 4)) &&811 kfd->num_nodes > 1)812 kfd->hive_id = pci_dev_id(kfd->adev->pdev);813 814 kfd->noretry = kfd->adev->gmc.noretry;815 816 kfd_cwsr_init(kfd);817 818 dev_info(kfd_device, "Total number of KFD nodes to be created: %d\n",819 kfd->num_nodes);820 821 /* Allocate the KFD nodes */822 for (i = 0, xcp_idx = 0; i < kfd->num_nodes; i++) {823 node = kzalloc(sizeof(struct kfd_node), GFP_KERNEL);824 if (!node)825 goto node_alloc_error;826 827 node->node_id = i;828 node->adev = kfd->adev;829 node->kfd = kfd;830 node->kfd2kgd = kfd->kfd2kgd;831 node->vm_info.vmid_num_kfd = vmid_num_kfd;832 node->xcp = amdgpu_get_next_xcp(kfd->adev->xcp_mgr, &xcp_idx);833 /* TODO : Check if error handling is needed */834 if (node->xcp) {835 amdgpu_xcp_get_inst_details(node->xcp, AMDGPU_XCP_GFX,836 &node->xcc_mask);837 ++xcp_idx;838 } else {839 node->xcc_mask =840 (1U << NUM_XCC(kfd->adev->gfx.xcc_mask)) - 1;841 }842 843 if (node->xcp) {844 dev_info(kfd_device, "KFD node %d partition %d size %lldM\n",845 node->node_id, node->xcp->mem_id,846 KFD_XCP_MEMORY_SIZE(node->adev, node->node_id) >> 20);847 }848 849 if ((KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 3) ||850 KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 4)) &&851 partition_mode == AMDGPU_CPX_PARTITION_MODE &&852 kfd->num_nodes != 1) {853 /* For GFX9.4.3 and CPX mode, first XCD gets VMID range854 * 4-9 and second XCD gets VMID range 10-15.855 */856 857 node->vm_info.first_vmid_kfd = (i%2 == 0) ?858 first_vmid_kfd :859 first_vmid_kfd+vmid_num_kfd;860 node->vm_info.last_vmid_kfd = (i%2 == 0) ?861 last_vmid_kfd-vmid_num_kfd :862 last_vmid_kfd;863 node->compute_vmid_bitmap =864 ((0x1 << (node->vm_info.last_vmid_kfd + 1)) - 1) -865 ((0x1 << (node->vm_info.first_vmid_kfd)) - 1);866 } else {867 node->vm_info.first_vmid_kfd = first_vmid_kfd;868 node->vm_info.last_vmid_kfd = last_vmid_kfd;869 node->compute_vmid_bitmap =870 gpu_resources->compute_vmid_bitmap;871 }872 node->max_proc_per_quantum = max_proc_per_quantum;873 atomic_set(&node->sram_ecc_flag, 0);874 875 amdgpu_amdkfd_get_local_mem_info(kfd->adev,876 &node->local_mem_info, node->xcp);877 878 if (KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 3) ||879 KFD_GC_VERSION(kfd) == IP_VERSION(9, 4, 4))880 kfd_setup_interrupt_bitmap(node, i);881 882 /* Initialize the KFD node */883 if (kfd_init_node(node)) {884 dev_err(kfd_device, "Error initializing KFD node\n");885 goto node_init_error;886 }887 888 spin_lock_init(&node->watch_points_lock);889 890 kfd->nodes[i] = node;891 }892 893 svm_range_set_max_pages(kfd->adev);894 895 kfd->init_complete = true;896 dev_info(kfd_device, "added device %x:%x\n", kfd->adev->pdev->vendor,897 kfd->adev->pdev->device);898 899 pr_debug("Starting kfd with the following scheduling policy %d\n",900 node->dqm->sched_policy);901 902 goto out;903 904node_init_error:905node_alloc_error:906 kfd_cleanup_nodes(kfd, i);907 kfd_doorbell_fini(kfd);908kfd_doorbell_error:909 kfd_gtt_sa_fini(kfd);910kfd_gtt_sa_init_error:911 amdgpu_amdkfd_free_gtt_mem(kfd->adev, &kfd->gtt_mem);912alloc_gtt_mem_failure:913 dev_err(kfd_device,914 "device %x:%x NOT added due to errors\n",915 kfd->adev->pdev->vendor, kfd->adev->pdev->device);916out:917 return kfd->init_complete;918}919 920void kgd2kfd_device_exit(struct kfd_dev *kfd)921{922 if (kfd->init_complete) {923 /* Cleanup KFD nodes */924 kfd_cleanup_nodes(kfd, kfd->num_nodes);925 /* Cleanup common/shared resources */926 kfd_doorbell_fini(kfd);927 ida_destroy(&kfd->doorbell_ida);928 kfd_gtt_sa_fini(kfd);929 amdgpu_amdkfd_free_gtt_mem(kfd->adev, &kfd->gtt_mem);930 }931 932 kfree(kfd);933}934 935int kgd2kfd_pre_reset(struct kfd_dev *kfd,936 struct amdgpu_reset_context *reset_context)937{938 struct kfd_node *node;939 int i;940 941 if (!kfd->init_complete)942 return 0;943 944 for (i = 0; i < kfd->num_nodes; i++) {945 node = kfd->nodes[i];946 kfd_smi_event_update_gpu_reset(node, false, reset_context);947 }948 949 kgd2kfd_suspend(kfd, false);950 951 for (i = 0; i < kfd->num_nodes; i++)952 kfd_signal_reset_event(kfd->nodes[i]);953 954 return 0;955}956 957/*958 * Fix me. KFD won't be able to resume existing process for now.959 * We will keep all existing process in a evicted state and960 * wait the process to be terminated.961 */962 963int kgd2kfd_post_reset(struct kfd_dev *kfd)964{965 int ret;966 struct kfd_node *node;967 int i;968 969 if (!kfd->init_complete)970 return 0;971 972 for (i = 0; i < kfd->num_nodes; i++) {973 ret = kfd_resume(kfd->nodes[i]);974 if (ret)975 return ret;976 }977 978 mutex_lock(&kfd_processes_mutex);979 --kfd_locked;980 mutex_unlock(&kfd_processes_mutex);981 982 for (i = 0; i < kfd->num_nodes; i++) {983 node = kfd->nodes[i];984 atomic_set(&node->sram_ecc_flag, 0);985 kfd_smi_event_update_gpu_reset(node, true, NULL);986 }987 988 return 0;989}990 991bool kfd_is_locked(void)992{993 lockdep_assert_held(&kfd_processes_mutex);994 return (kfd_locked > 0);995}996 997void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm)998{999 struct kfd_node *node;1000 int i;1001 1002 if (!kfd->init_complete)1003 return;1004 1005 /* for runtime suspend, skip locking kfd */1006 if (!run_pm) {1007 mutex_lock(&kfd_processes_mutex);1008 /* For first KFD device suspend all the KFD processes */1009 if (++kfd_locked == 1)1010 kfd_suspend_all_processes();1011 mutex_unlock(&kfd_processes_mutex);1012 }1013 1014 for (i = 0; i < kfd->num_nodes; i++) {1015 node = kfd->nodes[i];1016 node->dqm->ops.stop(node->dqm);1017 }1018}1019 1020int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm)1021{1022 int ret, i;1023 1024 if (!kfd->init_complete)1025 return 0;1026 1027 for (i = 0; i < kfd->num_nodes; i++) {1028 ret = kfd_resume(kfd->nodes[i]);1029 if (ret)1030 return ret;1031 }1032 1033 /* for runtime resume, skip unlocking kfd */1034 if (!run_pm) {1035 mutex_lock(&kfd_processes_mutex);1036 if (--kfd_locked == 0)1037 ret = kfd_resume_all_processes();1038 WARN_ONCE(kfd_locked < 0, "KFD suspend / resume ref. error");1039 mutex_unlock(&kfd_processes_mutex);1040 }1041 1042 return ret;1043}1044 1045static int kfd_resume(struct kfd_node *node)1046{1047 int err = 0;1048 1049 err = node->dqm->ops.start(node->dqm);1050 if (err)1051 dev_err(kfd_device,1052 "Error starting queue manager for device %x:%x\n",1053 node->adev->pdev->vendor, node->adev->pdev->device);1054 1055 return err;1056}1057 1058static inline void kfd_queue_work(struct workqueue_struct *wq,1059 struct work_struct *work)1060{1061 int cpu, new_cpu;1062 1063 cpu = new_cpu = smp_processor_id();1064 do {1065 new_cpu = cpumask_next(new_cpu, cpu_online_mask) % nr_cpu_ids;1066 if (cpu_to_node(new_cpu) == numa_node_id())1067 break;1068 } while (cpu != new_cpu);1069 1070 queue_work_on(new_cpu, wq, work);1071}1072 1073/* This is called directly from KGD at ISR. */1074void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)1075{1076 uint32_t patched_ihre[KFD_MAX_RING_ENTRY_SIZE], i;1077 bool is_patched = false;1078 unsigned long flags;1079 struct kfd_node *node;1080 1081 if (!kfd->init_complete)1082 return;1083 1084 if (kfd->device_info.ih_ring_entry_size > sizeof(patched_ihre)) {1085 dev_err_once(kfd_device, "Ring entry too small\n");1086 return;1087 }1088 1089 for (i = 0; i < kfd->num_nodes; i++) {1090 node = kfd->nodes[i];1091 spin_lock_irqsave(&node->interrupt_lock, flags);1092 1093 if (node->interrupts_active1094 && interrupt_is_wanted(node, ih_ring_entry,1095 patched_ihre, &is_patched)1096 && enqueue_ih_ring_entry(node,1097 is_patched ? patched_ihre : ih_ring_entry)) {1098 kfd_queue_work(node->ih_wq, &node->interrupt_work);1099 spin_unlock_irqrestore(&node->interrupt_lock, flags);1100 return;1101 }1102 spin_unlock_irqrestore(&node->interrupt_lock, flags);1103 }1104 1105}1106 1107int kgd2kfd_quiesce_mm(struct mm_struct *mm, uint32_t trigger)1108{1109 struct kfd_process *p;1110 int r;1111 1112 /* Because we are called from arbitrary context (workqueue) as opposed1113 * to process context, kfd_process could attempt to exit while we are1114 * running so the lookup function increments the process ref count.1115 */1116 p = kfd_lookup_process_by_mm(mm);1117 if (!p)1118 return -ESRCH;1119 1120 WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid);1121 r = kfd_process_evict_queues(p, trigger);1122 1123 kfd_unref_process(p);1124 return r;1125}1126 1127int kgd2kfd_resume_mm(struct mm_struct *mm)1128{1129 struct kfd_process *p;1130 int r;1131 1132 /* Because we are called from arbitrary context (workqueue) as opposed1133 * to process context, kfd_process could attempt to exit while we are1134 * running so the lookup function increments the process ref count.1135 */1136 p = kfd_lookup_process_by_mm(mm);1137 if (!p)1138 return -ESRCH;1139 1140 r = kfd_process_restore_queues(p);1141 1142 kfd_unref_process(p);1143 return r;1144}1145 1146/** kgd2kfd_schedule_evict_and_restore_process - Schedules work queue that will1147 * prepare for safe eviction of KFD BOs that belong to the specified1148 * process.1149 *1150 * @mm: mm_struct that identifies the specified KFD process1151 * @fence: eviction fence attached to KFD process BOs1152 *1153 */1154int kgd2kfd_schedule_evict_and_restore_process(struct mm_struct *mm,1155 struct dma_fence *fence)1156{1157 struct kfd_process *p;1158 unsigned long active_time;1159 unsigned long delay_jiffies = msecs_to_jiffies(PROCESS_ACTIVE_TIME_MS);1160 1161 if (!fence)1162 return -EINVAL;1163 1164 if (dma_fence_is_signaled(fence))1165 return 0;1166 1167 p = kfd_lookup_process_by_mm(mm);1168 if (!p)1169 return -ENODEV;1170 1171 if (fence->seqno == p->last_eviction_seqno)1172 goto out;1173 1174 p->last_eviction_seqno = fence->seqno;1175 1176 /* Avoid KFD process starvation. Wait for at least1177 * PROCESS_ACTIVE_TIME_MS before evicting the process again1178 */1179 active_time = get_jiffies_64() - p->last_restore_timestamp;1180 if (delay_jiffies > active_time)1181 delay_jiffies -= active_time;1182 else1183 delay_jiffies = 0;1184 1185 /* During process initialization eviction_work.dwork is initialized1186 * to kfd_evict_bo_worker1187 */1188 WARN(debug_evictions, "Scheduling eviction of pid %d in %ld jiffies",1189 p->lead_thread->pid, delay_jiffies);1190 schedule_delayed_work(&p->eviction_work, delay_jiffies);1191out:1192 kfd_unref_process(p);1193 return 0;1194}1195 1196static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,1197 unsigned int chunk_size)1198{1199 if (WARN_ON(buf_size < chunk_size))1200 return -EINVAL;1201 if (WARN_ON(buf_size == 0))1202 return -EINVAL;1203 if (WARN_ON(chunk_size == 0))1204 return -EINVAL;1205 1206 kfd->gtt_sa_chunk_size = chunk_size;1207 kfd->gtt_sa_num_of_chunks = buf_size / chunk_size;1208 1209 kfd->gtt_sa_bitmap = bitmap_zalloc(kfd->gtt_sa_num_of_chunks,1210 GFP_KERNEL);1211 if (!kfd->gtt_sa_bitmap)1212 return -ENOMEM;1213 1214 pr_debug("gtt_sa_num_of_chunks = %d, gtt_sa_bitmap = %p\n",1215 kfd->gtt_sa_num_of_chunks, kfd->gtt_sa_bitmap);1216 1217 mutex_init(&kfd->gtt_sa_lock);1218 1219 return 0;1220}1221 1222static void kfd_gtt_sa_fini(struct kfd_dev *kfd)1223{1224 mutex_destroy(&kfd->gtt_sa_lock);1225 bitmap_free(kfd->gtt_sa_bitmap);1226}1227 1228static inline uint64_t kfd_gtt_sa_calc_gpu_addr(uint64_t start_addr,1229 unsigned int bit_num,1230 unsigned int chunk_size)1231{1232 return start_addr + bit_num * chunk_size;1233}1234 1235static inline uint32_t *kfd_gtt_sa_calc_cpu_addr(void *start_addr,1236 unsigned int bit_num,1237 unsigned int chunk_size)1238{1239 return (uint32_t *) ((uint64_t) start_addr + bit_num * chunk_size);1240}1241 1242int kfd_gtt_sa_allocate(struct kfd_node *node, unsigned int size,1243 struct kfd_mem_obj **mem_obj)1244{1245 unsigned int found, start_search, cur_size;1246 struct kfd_dev *kfd = node->kfd;1247 1248 if (size == 0)1249 return -EINVAL;1250 1251 if (size > kfd->gtt_sa_num_of_chunks * kfd->gtt_sa_chunk_size)1252 return -ENOMEM;1253 1254 *mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);1255 if (!(*mem_obj))1256 return -ENOMEM;1257 1258 pr_debug("Allocated mem_obj = %p for size = %d\n", *mem_obj, size);1259 1260 start_search = 0;1261 1262 mutex_lock(&kfd->gtt_sa_lock);1263 1264kfd_gtt_restart_search:1265 /* Find the first chunk that is free */1266 found = find_next_zero_bit(kfd->gtt_sa_bitmap,1267 kfd->gtt_sa_num_of_chunks,1268 start_search);1269 1270 pr_debug("Found = %d\n", found);1271 1272 /* If there wasn't any free chunk, bail out */1273 if (found == kfd->gtt_sa_num_of_chunks)1274 goto kfd_gtt_no_free_chunk;1275 1276 /* Update fields of mem_obj */1277 (*mem_obj)->range_start = found;1278 (*mem_obj)->range_end = found;1279 (*mem_obj)->gpu_addr = kfd_gtt_sa_calc_gpu_addr(1280 kfd->gtt_start_gpu_addr,1281 found,1282 kfd->gtt_sa_chunk_size);1283 (*mem_obj)->cpu_ptr = kfd_gtt_sa_calc_cpu_addr(1284 kfd->gtt_start_cpu_ptr,1285 found,1286 kfd->gtt_sa_chunk_size);1287 1288 pr_debug("gpu_addr = %p, cpu_addr = %p\n",1289 (uint64_t *) (*mem_obj)->gpu_addr, (*mem_obj)->cpu_ptr);1290 1291 /* If we need only one chunk, mark it as allocated and get out */1292 if (size <= kfd->gtt_sa_chunk_size) {1293 pr_debug("Single bit\n");1294 __set_bit(found, kfd->gtt_sa_bitmap);1295 goto kfd_gtt_out;1296 }1297 1298 /* Otherwise, try to see if we have enough contiguous chunks */1299 cur_size = size - kfd->gtt_sa_chunk_size;1300 do {1301 (*mem_obj)->range_end =1302 find_next_zero_bit(kfd->gtt_sa_bitmap,1303 kfd->gtt_sa_num_of_chunks, ++found);1304 /*1305 * If next free chunk is not contiguous than we need to1306 * restart our search from the last free chunk we found (which1307 * wasn't contiguous to the previous ones1308 */1309 if ((*mem_obj)->range_end != found) {1310 start_search = found;1311 goto kfd_gtt_restart_search;1312 }1313 1314 /*1315 * If we reached end of buffer, bail out with error1316 */1317 if (found == kfd->gtt_sa_num_of_chunks)1318 goto kfd_gtt_no_free_chunk;1319 1320 /* Check if we don't need another chunk */1321 if (cur_size <= kfd->gtt_sa_chunk_size)1322 cur_size = 0;1323 else1324 cur_size -= kfd->gtt_sa_chunk_size;1325 1326 } while (cur_size > 0);1327 1328 pr_debug("range_start = %d, range_end = %d\n",1329 (*mem_obj)->range_start, (*mem_obj)->range_end);1330 1331 /* Mark the chunks as allocated */1332 bitmap_set(kfd->gtt_sa_bitmap, (*mem_obj)->range_start,1333 (*mem_obj)->range_end - (*mem_obj)->range_start + 1);1334 1335kfd_gtt_out:1336 mutex_unlock(&kfd->gtt_sa_lock);1337 return 0;1338 1339kfd_gtt_no_free_chunk:1340 pr_debug("Allocation failed with mem_obj = %p\n", *mem_obj);1341 mutex_unlock(&kfd->gtt_sa_lock);1342 kfree(*mem_obj);1343 return -ENOMEM;1344}1345 1346int kfd_gtt_sa_free(struct kfd_node *node, struct kfd_mem_obj *mem_obj)1347{1348 struct kfd_dev *kfd = node->kfd;1349 1350 /* Act like kfree when trying to free a NULL object */1351 if (!mem_obj)1352 return 0;1353 1354 pr_debug("Free mem_obj = %p, range_start = %d, range_end = %d\n",1355 mem_obj, mem_obj->range_start, mem_obj->range_end);1356 1357 mutex_lock(&kfd->gtt_sa_lock);1358 1359 /* Mark the chunks as free */1360 bitmap_clear(kfd->gtt_sa_bitmap, mem_obj->range_start,1361 mem_obj->range_end - mem_obj->range_start + 1);1362 1363 mutex_unlock(&kfd->gtt_sa_lock);1364 1365 kfree(mem_obj);1366 return 0;1367}1368 1369void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd)1370{1371 /*1372 * TODO: Currently update SRAM ECC flag for first node.1373 * This needs to be updated later when we can1374 * identify SRAM ECC error on other nodes also.1375 */1376 if (kfd)1377 atomic_inc(&kfd->nodes[0]->sram_ecc_flag);1378}1379 1380void kfd_inc_compute_active(struct kfd_node *node)1381{1382 if (atomic_inc_return(&node->kfd->compute_profile) == 1)1383 amdgpu_amdkfd_set_compute_idle(node->adev, false);1384}1385 1386void kfd_dec_compute_active(struct kfd_node *node)1387{1388 int count = atomic_dec_return(&node->kfd->compute_profile);1389 1390 if (count == 0)1391 amdgpu_amdkfd_set_compute_idle(node->adev, true);1392 WARN_ONCE(count < 0, "Compute profile ref. count error");1393}1394 1395void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint64_t throttle_bitmask)1396{1397 /*1398 * TODO: For now, raise the throttling event only on first node.1399 * This will need to change after we are able to determine1400 * which node raised the throttling event.1401 */1402 if (kfd && kfd->init_complete)1403 kfd_smi_event_update_thermal_throttling(kfd->nodes[0],1404 throttle_bitmask);1405}1406 1407/* kfd_get_num_sdma_engines returns the number of PCIe optimized SDMA and1408 * kfd_get_num_xgmi_sdma_engines returns the number of XGMI SDMA.1409 * When the device has more than two engines, we reserve two for PCIe to enable1410 * full-duplex and the rest are used as XGMI.1411 */1412unsigned int kfd_get_num_sdma_engines(struct kfd_node *node)1413{1414 /* If XGMI is not supported, all SDMA engines are PCIe */1415 if (!node->adev->gmc.xgmi.supported)1416 return node->adev->sdma.num_instances/(int)node->kfd->num_nodes;1417 1418 return min(node->adev->sdma.num_instances/(int)node->kfd->num_nodes, 2);1419}1420 1421unsigned int kfd_get_num_xgmi_sdma_engines(struct kfd_node *node)1422{1423 /* After reserved for PCIe, the rest of engines are XGMI */1424 return node->adev->sdma.num_instances/(int)node->kfd->num_nodes -1425 kfd_get_num_sdma_engines(node);1426}1427 1428int kgd2kfd_check_and_lock_kfd(void)1429{1430 mutex_lock(&kfd_processes_mutex);1431 if (!hash_empty(kfd_processes_table) || kfd_is_locked()) {1432 mutex_unlock(&kfd_processes_mutex);1433 return -EBUSY;1434 }1435 1436 ++kfd_locked;1437 mutex_unlock(&kfd_processes_mutex);1438 1439 return 0;1440}1441 1442void kgd2kfd_unlock_kfd(void)1443{1444 mutex_lock(&kfd_processes_mutex);1445 --kfd_locked;1446 mutex_unlock(&kfd_processes_mutex);1447}1448 1449int kgd2kfd_start_sched(struct kfd_dev *kfd, uint32_t node_id)1450{1451 struct kfd_node *node;1452 int ret;1453 1454 if (!kfd->init_complete)1455 return 0;1456 1457 if (node_id >= kfd->num_nodes) {1458 dev_warn(kfd->adev->dev, "Invalid node ID: %u exceeds %u\n",1459 node_id, kfd->num_nodes - 1);1460 return -EINVAL;1461 }1462 node = kfd->nodes[node_id];1463 1464 ret = node->dqm->ops.unhalt(node->dqm);1465 if (ret)1466 dev_err(kfd_device, "Error in starting scheduler\n");1467 1468 return ret;1469}1470 1471int kgd2kfd_stop_sched(struct kfd_dev *kfd, uint32_t node_id)1472{1473 struct kfd_node *node;1474 1475 if (!kfd->init_complete)1476 return 0;1477 1478 if (node_id >= kfd->num_nodes) {1479 dev_warn(kfd->adev->dev, "Invalid node ID: %u exceeds %u\n",1480 node_id, kfd->num_nodes - 1);1481 return -EINVAL;1482 }1483 1484 node = kfd->nodes[node_id];1485 return node->dqm->ops.halt(node->dqm);1486}1487 1488#if defined(CONFIG_DEBUG_FS)1489 1490/* This function will send a package to HIQ to hang the HWS1491 * which will trigger a GPU reset and bring the HWS back to normal state1492 */1493int kfd_debugfs_hang_hws(struct kfd_node *dev)1494{1495 if (dev->dqm->sched_policy != KFD_SCHED_POLICY_HWS) {1496 pr_err("HWS is not enabled");1497 return -EINVAL;1498 }1499 1500 return dqm_debugfs_hang_hws(dev->dqm);1501}1502 1503#endif1504