551 lines · c
1// SPDX-License-Identifier: GPL-2.02 3/*4 * Copyright 2016-2019 HabanaLabs, Ltd.5 * All Rights Reserved.6 */7 8#include "../habanalabs.h"9#include "../../include/hw_ip/mmu/mmu_general.h"10 11#include <linux/slab.h>12 13#define MMU_V1_MAX_HOPS (MMU_HOP4 + 1)14 15static inline u64 get_hop_pte_addr(struct hl_ctx *ctx, struct hl_mmu_properties *mmu_prop,16 u64 *hop_addr_arr, u64 virt_addr, enum mmu_hop_num hop_idx)17{18 u64 mask, shift;19 20 mask = mmu_prop->hop_masks[hop_idx];21 shift = mmu_prop->hop_shifts[hop_idx];22 return hop_addr_arr[hop_idx] +23 ctx->hdev->asic_prop.mmu_pte_size * ((virt_addr & mask) >> shift);24}25 26static int dram_default_mapping_init(struct hl_ctx *ctx)27{28 struct hl_device *hdev = ctx->hdev;29 struct asic_fixed_properties *prop = &hdev->asic_prop;30 u64 num_of_hop3, total_hops, hop0_addr, hop1_addr, hop2_addr,31 hop2_pte_addr, hop3_pte_addr, pte_val;32 int rc, i, j, hop3_allocated = 0;33 34 if ((!prop->dram_supports_virtual_memory) ||35 (!hdev->dram_default_page_mapping) ||36 (ctx->asid == HL_KERNEL_ASID_ID))37 return 0;38 39 num_of_hop3 = prop->dram_size_for_default_page_mapping;40 do_div(num_of_hop3, prop->dram_page_size);41 do_div(num_of_hop3, HOP_PTE_ENTRIES_512);42 43 /* add hop1 and hop2 */44 total_hops = num_of_hop3 + 2;45 46 ctx->dram_default_hops = kcalloc(total_hops, HL_PTE_SIZE, GFP_KERNEL);47 if (!ctx->dram_default_hops)48 return -ENOMEM;49 50 hop0_addr = hl_mmu_dr_get_hop0_addr(ctx);51 52 hop1_addr = hl_mmu_dr_alloc_hop(ctx);53 if (hop1_addr == ULLONG_MAX) {54 dev_err(hdev->dev, "failed to alloc hop 1\n");55 rc = -ENOMEM;56 goto hop1_err;57 }58 59 ctx->dram_default_hops[total_hops - 1] = hop1_addr;60 61 hop2_addr = hl_mmu_dr_alloc_hop(ctx);62 if (hop2_addr == ULLONG_MAX) {63 dev_err(hdev->dev, "failed to alloc hop 2\n");64 rc = -ENOMEM;65 goto hop2_err;66 }67 68 ctx->dram_default_hops[total_hops - 2] = hop2_addr;69 70 for (i = 0 ; i < num_of_hop3 ; i++) {71 ctx->dram_default_hops[i] = hl_mmu_dr_alloc_hop(ctx);72 if (ctx->dram_default_hops[i] == ULLONG_MAX) {73 dev_err(hdev->dev, "failed to alloc hop 3, i: %d\n", i);74 rc = -ENOMEM;75 goto hop3_err;76 }77 hop3_allocated++;78 }79 80 /* need only pte 0 in hops 0 and 1 */81 pte_val = (hop1_addr & HOP_PHYS_ADDR_MASK) | PAGE_PRESENT_MASK;82 hl_mmu_dr_write_pte(ctx, hop0_addr, pte_val);83 84 pte_val = (hop2_addr & HOP_PHYS_ADDR_MASK) | PAGE_PRESENT_MASK;85 hl_mmu_dr_write_pte(ctx, hop1_addr, pte_val);86 hl_mmu_dr_get_pte(ctx, hop1_addr);87 88 hop2_pte_addr = hop2_addr;89 for (i = 0 ; i < num_of_hop3 ; i++) {90 pte_val = (ctx->dram_default_hops[i] & HOP_PHYS_ADDR_MASK) |91 PAGE_PRESENT_MASK;92 hl_mmu_dr_write_pte(ctx, hop2_pte_addr, pte_val);93 hl_mmu_dr_get_pte(ctx, hop2_addr);94 hop2_pte_addr += HL_PTE_SIZE;95 }96 97 pte_val = (prop->mmu_dram_default_page_addr & HOP_PHYS_ADDR_MASK) |98 LAST_MASK | PAGE_PRESENT_MASK;99 100 for (i = 0 ; i < num_of_hop3 ; i++) {101 hop3_pte_addr = ctx->dram_default_hops[i];102 for (j = 0 ; j < HOP_PTE_ENTRIES_512 ; j++) {103 hl_mmu_dr_write_final_pte(ctx, hop3_pte_addr, pte_val);104 hl_mmu_dr_get_pte(ctx, ctx->dram_default_hops[i]);105 hop3_pte_addr += HL_PTE_SIZE;106 }107 }108 109 hl_mmu_dr_flush(ctx);110 111 return 0;112 113hop3_err:114 for (i = 0 ; i < hop3_allocated ; i++)115 hl_mmu_dr_free_hop(ctx, ctx->dram_default_hops[i]);116 117 hl_mmu_dr_free_hop(ctx, hop2_addr);118hop2_err:119 hl_mmu_dr_free_hop(ctx, hop1_addr);120hop1_err:121 kfree(ctx->dram_default_hops);122 123 return rc;124}125 126static void dram_default_mapping_fini(struct hl_ctx *ctx)127{128 struct hl_device *hdev = ctx->hdev;129 struct asic_fixed_properties *prop = &hdev->asic_prop;130 u64 num_of_hop3, total_hops, hop0_addr, hop1_addr, hop2_addr,131 hop2_pte_addr, hop3_pte_addr;132 int i, j;133 134 if ((!prop->dram_supports_virtual_memory) ||135 (!hdev->dram_default_page_mapping) ||136 (ctx->asid == HL_KERNEL_ASID_ID))137 return;138 139 num_of_hop3 = prop->dram_size_for_default_page_mapping;140 do_div(num_of_hop3, prop->dram_page_size);141 do_div(num_of_hop3, HOP_PTE_ENTRIES_512);142 143 hop0_addr = hl_mmu_dr_get_hop0_addr(ctx);144 /* add hop1 and hop2 */145 total_hops = num_of_hop3 + 2;146 hop1_addr = ctx->dram_default_hops[total_hops - 1];147 hop2_addr = ctx->dram_default_hops[total_hops - 2];148 149 for (i = 0 ; i < num_of_hop3 ; i++) {150 hop3_pte_addr = ctx->dram_default_hops[i];151 for (j = 0 ; j < HOP_PTE_ENTRIES_512 ; j++) {152 hl_mmu_dr_clear_pte(ctx, hop3_pte_addr);153 hl_mmu_dr_put_pte(ctx, ctx->dram_default_hops[i]);154 hop3_pte_addr += HL_PTE_SIZE;155 }156 }157 158 hop2_pte_addr = hop2_addr;159 for (i = 0 ; i < num_of_hop3 ; i++) {160 hl_mmu_dr_clear_pte(ctx, hop2_pte_addr);161 hl_mmu_dr_put_pte(ctx, hop2_addr);162 hop2_pte_addr += HL_PTE_SIZE;163 }164 165 hl_mmu_dr_clear_pte(ctx, hop1_addr);166 hl_mmu_dr_put_pte(ctx, hop1_addr);167 hl_mmu_dr_clear_pte(ctx, hop0_addr);168 169 kfree(ctx->dram_default_hops);170 171 hl_mmu_dr_flush(ctx);172}173 174/**175 * hl_mmu_v1_ctx_init() - initialize a context for using the MMU module.176 * @ctx: pointer to the context structure to initialize.177 *178 * Initialize a mutex to protect the concurrent mapping flow, a hash to hold all179 * page tables hops related to this context.180 * Return: 0 on success, non-zero otherwise.181 */182static int hl_mmu_v1_ctx_init(struct hl_ctx *ctx)183{184 hash_init(ctx->mmu_shadow_hash);185 return dram_default_mapping_init(ctx);186}187 188/*189 * hl_mmu_ctx_fini - disable a ctx from using the mmu module190 *191 * @ctx: pointer to the context structure192 *193 * This function does the following:194 * - Free any pgts which were not freed yet195 * - Free the mutex196 * - Free DRAM default page mapping hops197 */198static void hl_mmu_v1_ctx_fini(struct hl_ctx *ctx)199{200 struct hl_device *hdev = ctx->hdev;201 struct pgt_info *pgt_info;202 struct hlist_node *tmp;203 int i;204 205 dram_default_mapping_fini(ctx);206 207 if (!hash_empty(ctx->mmu_shadow_hash))208 dev_err(hdev->dev, "ctx %d is freed while it has pgts in use\n",209 ctx->asid);210 211 hash_for_each_safe(ctx->mmu_shadow_hash, i, tmp, pgt_info, node) {212 dev_err_ratelimited(hdev->dev,213 "pgt_info of addr 0x%llx of asid %d was not destroyed, num_ptes: %d\n",214 pgt_info->phys_addr, ctx->asid, pgt_info->num_of_ptes);215 hl_mmu_dr_free_pgt_node(ctx, pgt_info);216 }217}218 219static int hl_mmu_v1_unmap(struct hl_ctx *ctx,220 u64 virt_addr, bool is_dram_addr)221{222 u64 hop_addr[MMU_V1_MAX_HOPS] = {0}, hop_pte_addr[MMU_V1_MAX_HOPS] = {0}, curr_pte = 0;223 struct hl_device *hdev = ctx->hdev;224 struct asic_fixed_properties *prop = &hdev->asic_prop;225 struct hl_mmu_properties *mmu_prop;226 bool is_huge, clear_hop3 = true;227 int hop_idx;228 229 /* shifts and masks are the same in PMMU and HPMMU, use one of them */230 mmu_prop = is_dram_addr ? &prop->dmmu : &prop->pmmu;231 232 for (hop_idx = MMU_HOP0; hop_idx < MMU_HOP4; hop_idx++) {233 if (hop_idx == MMU_HOP0) {234 hop_addr[hop_idx] = hl_mmu_dr_get_hop0_addr(ctx);235 } else {236 hop_addr[hop_idx] = hl_mmu_get_next_hop_addr(ctx, curr_pte);237 if (hop_addr[hop_idx] == ULLONG_MAX)238 goto not_mapped;239 }240 241 hop_pte_addr[hop_idx] =242 get_hop_pte_addr(ctx, mmu_prop, hop_addr, virt_addr, hop_idx);243 244 curr_pte = *(u64 *) (uintptr_t) hop_pte_addr[hop_idx];245 }246 247 is_huge = curr_pte & mmu_prop->last_mask;248 249 if (is_dram_addr && !is_huge) {250 dev_err(hdev->dev, "DRAM unmapping should use huge pages only\n");251 return -EFAULT;252 }253 254 if (!is_huge) {255 hop_idx = MMU_HOP4;256 hop_addr[hop_idx] = hl_mmu_get_next_hop_addr(ctx, curr_pte);257 if (hop_addr[hop_idx] == ULLONG_MAX)258 goto not_mapped;259 260 hop_pte_addr[hop_idx] =261 get_hop_pte_addr(ctx, mmu_prop, hop_addr, virt_addr, hop_idx);262 curr_pte = *(u64 *) (uintptr_t) hop_pte_addr[hop_idx];263 clear_hop3 = false;264 }265 266 if (hdev->dram_default_page_mapping && is_dram_addr) {267 u64 default_pte = (prop->mmu_dram_default_page_addr &268 HOP_PHYS_ADDR_MASK) | mmu_prop->last_mask |269 PAGE_PRESENT_MASK;270 if (curr_pte == default_pte) {271 dev_err(hdev->dev,272 "DRAM: hop3 PTE points to zero page, can't unmap, va: 0x%llx\n",273 virt_addr);274 goto not_mapped;275 }276 277 if (!(curr_pte & PAGE_PRESENT_MASK)) {278 dev_err(hdev->dev,279 "DRAM: hop3 PTE is cleared! can't unmap, va: 0x%llx\n",280 virt_addr);281 goto not_mapped;282 }283 284 hop_idx = MMU_HOP3;285 hl_mmu_dr_write_final_pte(ctx, hop_pte_addr[hop_idx], default_pte);286 hl_mmu_dr_put_pte(ctx, hop_addr[hop_idx]);287 } else {288 if (!(curr_pte & PAGE_PRESENT_MASK))289 goto not_mapped;290 291 if (hop_addr[MMU_HOP4])292 hl_mmu_dr_clear_pte(ctx, hop_pte_addr[MMU_HOP4]);293 else294 hl_mmu_dr_clear_pte(ctx, hop_pte_addr[MMU_HOP3]);295 296 if (hop_addr[MMU_HOP4] && !hl_mmu_dr_put_pte(ctx, hop_addr[MMU_HOP4]))297 clear_hop3 = true;298 299 if (!clear_hop3)300 goto mapped;301 302 for (hop_idx = MMU_HOP3; hop_idx >= 0; hop_idx--) {303 hl_mmu_dr_clear_pte(ctx, hop_pte_addr[hop_idx]);304 305 if (hop_idx == MMU_HOP0)306 break;307 308 if (hl_mmu_dr_put_pte(ctx, hop_addr[hop_idx]))309 goto mapped;310 }311 }312 313mapped:314 return 0;315 316not_mapped:317 dev_err(hdev->dev, "virt addr 0x%llx is not mapped to phys addr\n",318 virt_addr);319 320 return -EINVAL;321}322 323static int hl_mmu_v1_map(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr,324 u32 page_size, bool is_dram_addr)325{326 u64 hop_addr[MMU_V1_MAX_HOPS] = {0}, hop_pte_addr[MMU_V1_MAX_HOPS] = {0}, curr_pte = 0;327 struct hl_device *hdev = ctx->hdev;328 struct asic_fixed_properties *prop = &hdev->asic_prop;329 struct hl_mmu_properties *mmu_prop;330 bool is_huge, hop_new[MMU_V1_MAX_HOPS] = {false};331 int num_hops, hop_idx, prev_hop, rc = -ENOMEM;332 333 /*334 * This mapping function can map a page or a huge page. For huge page335 * there are only 3 hops rather than 4. Currently the DRAM allocation336 * uses huge pages only but user memory could have been allocated with337 * one of the two page sizes. Since this is a common code for all the338 * three cases, we need this hugs page check.339 */340 if (is_dram_addr) {341 mmu_prop = &prop->dmmu;342 is_huge = true;343 } else if (page_size == prop->pmmu_huge.page_size) {344 mmu_prop = &prop->pmmu_huge;345 is_huge = true;346 } else {347 mmu_prop = &prop->pmmu;348 is_huge = false;349 }350 351 num_hops = is_huge ? (MMU_V1_MAX_HOPS - 1) : MMU_V1_MAX_HOPS;352 353 for (hop_idx = MMU_HOP0; hop_idx < num_hops; hop_idx++) {354 if (hop_idx == MMU_HOP0) {355 hop_addr[hop_idx] = hl_mmu_dr_get_hop0_addr(ctx);356 } else {357 hop_addr[hop_idx] =358 hl_mmu_dr_get_alloc_next_hop_addr(ctx, curr_pte, &hop_new[hop_idx]);359 if (hop_addr[hop_idx] == ULLONG_MAX)360 goto err;361 }362 363 hop_pte_addr[hop_idx] =364 get_hop_pte_addr(ctx, mmu_prop, hop_addr, virt_addr, hop_idx);365 curr_pte = *(u64 *) (uintptr_t) hop_pte_addr[hop_idx];366 }367 368 if (hdev->dram_default_page_mapping && is_dram_addr) {369 u64 default_pte = (prop->mmu_dram_default_page_addr &370 HOP_PHYS_ADDR_MASK) | mmu_prop->last_mask |371 PAGE_PRESENT_MASK;372 373 if (curr_pte != default_pte) {374 dev_err(hdev->dev,375 "DRAM: mapping already exists for virt_addr 0x%llx\n",376 virt_addr);377 rc = -EINVAL;378 goto err;379 }380 381 for (hop_idx = MMU_HOP1; hop_idx < num_hops; hop_idx++) {382 if (hop_new[hop_idx]) {383 dev_err(hdev->dev, "DRAM mapping should not allocate more hops\n");384 rc = -EFAULT;385 goto err;386 }387 }388 } else if (curr_pte & PAGE_PRESENT_MASK) {389 dev_err(hdev->dev,390 "mapping already exists for virt_addr 0x%llx\n",391 virt_addr);392 393 for (hop_idx = MMU_HOP0; hop_idx < num_hops; hop_idx++)394 dev_dbg(hdev->dev, "hop%d pte: 0x%llx (0x%llx)\n", hop_idx,395 *(u64 *) (uintptr_t) hop_pte_addr[hop_idx],396 hop_pte_addr[hop_idx]);397 398 rc = -EINVAL;399 goto err;400 }401 402 curr_pte = (phys_addr & HOP_PHYS_ADDR_MASK) | mmu_prop->last_mask403 | PAGE_PRESENT_MASK;404 405 hl_mmu_dr_write_final_pte(ctx, hop_pte_addr[num_hops - 1], curr_pte);406 407 for (hop_idx = MMU_HOP1; hop_idx < num_hops; hop_idx++) {408 prev_hop = hop_idx - 1;409 410 if (hop_new[hop_idx]) {411 curr_pte = (hop_addr[hop_idx] & HOP_PHYS_ADDR_MASK) | PAGE_PRESENT_MASK;412 hl_mmu_dr_write_pte(ctx, hop_pte_addr[prev_hop], curr_pte);413 if (hop_idx != MMU_HOP1)414 hl_mmu_dr_get_pte(ctx, hop_addr[prev_hop]);415 }416 }417 418 hl_mmu_dr_get_pte(ctx, hop_addr[num_hops - 1]);419 420 return 0;421 422err:423 for (hop_idx = num_hops; hop_idx > MMU_HOP0; hop_idx--) {424 if (hop_new[hop_idx])425 hl_mmu_dr_free_hop(ctx, hop_addr[hop_idx]);426 }427 428 return rc;429}430 431/*432 * hl_mmu_v1_swap_out - marks all mapping of the given ctx as swapped out433 *434 * @ctx: pointer to the context structure435 *436 */437static void hl_mmu_v1_swap_out(struct hl_ctx *ctx)438{439 440}441 442/*443 * hl_mmu_v1_swap_in - marks all mapping of the given ctx as swapped in444 *445 * @ctx: pointer to the context structure446 *447 */448static void hl_mmu_v1_swap_in(struct hl_ctx *ctx)449{450 451}452 453static int hl_mmu_v1_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr,454 struct hl_mmu_hop_info *hops)455{456 struct hl_device *hdev = ctx->hdev;457 struct asic_fixed_properties *prop = &hdev->asic_prop;458 struct hl_mmu_properties *mmu_prop;459 bool is_dram_addr, is_pmmu_addr, is_pmmu_h_addr, is_huge;460 int i, used_hops;461 462 is_dram_addr = hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size,463 prop->dmmu.start_addr,464 prop->dmmu.end_addr);465 is_pmmu_addr = hl_mem_area_inside_range(virt_addr, prop->pmmu.page_size,466 prop->pmmu.start_addr,467 prop->pmmu.end_addr);468 is_pmmu_h_addr = hl_mem_area_inside_range(virt_addr,469 prop->pmmu_huge.page_size,470 prop->pmmu_huge.start_addr,471 prop->pmmu_huge.end_addr);472 if (is_dram_addr) {473 mmu_prop = &prop->dmmu;474 is_huge = true;475 } else if (is_pmmu_addr) {476 mmu_prop = &prop->pmmu;477 is_huge = false;478 } else if (is_pmmu_h_addr) {479 mmu_prop = &prop->pmmu_huge;480 is_huge = true;481 } else {482 return -EINVAL;483 }484 485 used_hops = mmu_prop->num_hops;486 487 /* huge pages use lesser hops */488 if (is_huge)489 used_hops--;490 491 hops->hop_info[0].hop_addr = hl_mmu_dr_get_phys_hop0_addr(ctx);492 hops->hop_info[0].hop_pte_addr =493 hl_mmu_get_hop_pte_phys_addr(ctx, mmu_prop, 0,494 hops->hop_info[0].hop_addr, virt_addr);495 hops->hop_info[0].hop_pte_val =496 hdev->asic_funcs->read_pte(hdev,497 hops->hop_info[0].hop_pte_addr);498 499 for (i = 1 ; i < used_hops ; i++) {500 hops->hop_info[i].hop_addr =501 hl_mmu_get_next_hop_addr(ctx,502 hops->hop_info[i - 1].hop_pte_val);503 if (hops->hop_info[i].hop_addr == ULLONG_MAX)504 return -EFAULT;505 506 hops->hop_info[i].hop_pte_addr =507 hl_mmu_get_hop_pte_phys_addr(ctx, mmu_prop, i,508 hops->hop_info[i].hop_addr,509 virt_addr);510 hops->hop_info[i].hop_pte_val =511 hdev->asic_funcs->read_pte(hdev,512 hops->hop_info[i].hop_pte_addr);513 514 if (!(hops->hop_info[i].hop_pte_val & PAGE_PRESENT_MASK))515 return -EFAULT;516 517 if (hops->hop_info[i].hop_pte_val & mmu_prop->last_mask)518 break;519 }520 521 /* if passed over all hops then no last hop was found */522 if (i == mmu_prop->num_hops)523 return -EFAULT;524 525 if (!(hops->hop_info[i].hop_pte_val & PAGE_PRESENT_MASK))526 return -EFAULT;527 528 hops->used_hops = i + 1;529 530 return 0;531}532 533/*534 * hl_mmu_v1_prepare - prepare mmu for working with mmu v1535 *536 * @hdev: pointer to the device structure537 */538void hl_mmu_v1_set_funcs(struct hl_device *hdev, struct hl_mmu_funcs *mmu)539{540 mmu->init = hl_mmu_dr_init;541 mmu->fini = hl_mmu_dr_fini;542 mmu->ctx_init = hl_mmu_v1_ctx_init;543 mmu->ctx_fini = hl_mmu_v1_ctx_fini;544 mmu->map = hl_mmu_v1_map;545 mmu->unmap = hl_mmu_v1_unmap;546 mmu->flush = hl_mmu_dr_flush;547 mmu->swap_out = hl_mmu_v1_swap_out;548 mmu->swap_in = hl_mmu_v1_swap_in;549 mmu->get_tlb_info = hl_mmu_v1_get_tlb_info;550}551