774 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/**************************************************************************3 * Copyright (c) 2007, Intel Corporation.4 *5 **************************************************************************/6 7#include <linux/highmem.h>8#include <linux/vmalloc.h>9 10#include "mmu.h"11#include "psb_drv.h"12#include "psb_reg.h"13 14/*15 * Code for the SGX MMU:16 */17 18/*19 * clflush on one processor only:20 * clflush should apparently flush the cache line on all processors in an21 * SMP system.22 */23 24/*25 * kmap atomic:26 * The usage of the slots must be completely encapsulated within a spinlock, and27 * no other functions that may be using the locks for other purposed may be28 * called from within the locked region.29 * Since the slots are per processor, this will guarantee that we are the only30 * user.31 */32 33/*34 * TODO: Inserting ptes from an interrupt handler:35 * This may be desirable for some SGX functionality where the GPU can fault in36 * needed pages. For that, we need to make an atomic insert_pages function, that37 * may fail.38 * If it fails, the caller need to insert the page using a workqueue function,39 * but on average it should be fast.40 */41 42static inline uint32_t psb_mmu_pt_index(uint32_t offset)43{44 return (offset >> PSB_PTE_SHIFT) & 0x3FF;45}46 47static inline uint32_t psb_mmu_pd_index(uint32_t offset)48{49 return offset >> PSB_PDE_SHIFT;50}51 52static inline void psb_clflush(void *addr)53{54 __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory");55}56 57static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)58{59 if (!driver->has_clflush)60 return;61 62 mb();63 psb_clflush(addr);64 mb();65}66 67static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force)68{69 struct drm_device *dev = driver->dev;70 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);71 72 if (atomic_read(&driver->needs_tlbflush) || force) {73 uint32_t val = PSB_RSGX32(PSB_CR_BIF_CTRL);74 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);75 76 /* Make sure data cache is turned off before enabling it */77 wmb();78 PSB_WSGX32(val & ~_PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);79 (void)PSB_RSGX32(PSB_CR_BIF_CTRL);80 if (driver->msvdx_mmu_invaldc)81 atomic_set(driver->msvdx_mmu_invaldc, 1);82 }83 atomic_set(&driver->needs_tlbflush, 0);84}85 86#if 087static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force)88{89 down_write(&driver->sem);90 psb_mmu_flush_pd_locked(driver, force);91 up_write(&driver->sem);92}93#endif94 95void psb_mmu_flush(struct psb_mmu_driver *driver)96{97 struct drm_device *dev = driver->dev;98 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);99 uint32_t val;100 101 down_write(&driver->sem);102 val = PSB_RSGX32(PSB_CR_BIF_CTRL);103 if (atomic_read(&driver->needs_tlbflush))104 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);105 else106 PSB_WSGX32(val | _PSB_CB_CTRL_FLUSH, PSB_CR_BIF_CTRL);107 108 /* Make sure data cache is turned off and MMU is flushed before109 restoring bank interface control register */110 wmb();111 PSB_WSGX32(val & ~(_PSB_CB_CTRL_FLUSH | _PSB_CB_CTRL_INVALDC),112 PSB_CR_BIF_CTRL);113 (void)PSB_RSGX32(PSB_CR_BIF_CTRL);114 115 atomic_set(&driver->needs_tlbflush, 0);116 if (driver->msvdx_mmu_invaldc)117 atomic_set(driver->msvdx_mmu_invaldc, 1);118 up_write(&driver->sem);119}120 121void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context)122{123 struct drm_device *dev = pd->driver->dev;124 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);125 uint32_t offset = (hw_context == 0) ? PSB_CR_BIF_DIR_LIST_BASE0 :126 PSB_CR_BIF_DIR_LIST_BASE1 + hw_context * 4;127 128 down_write(&pd->driver->sem);129 PSB_WSGX32(page_to_pfn(pd->p) << PAGE_SHIFT, offset);130 wmb();131 psb_mmu_flush_pd_locked(pd->driver, 1);132 pd->hw_context = hw_context;133 up_write(&pd->driver->sem);134 135}136 137static inline unsigned long psb_pd_addr_end(unsigned long addr,138 unsigned long end)139{140 addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK;141 return (addr < end) ? addr : end;142}143 144static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type)145{146 uint32_t mask = PSB_PTE_VALID;147 148 if (type & PSB_MMU_CACHED_MEMORY)149 mask |= PSB_PTE_CACHED;150 if (type & PSB_MMU_RO_MEMORY)151 mask |= PSB_PTE_RO;152 if (type & PSB_MMU_WO_MEMORY)153 mask |= PSB_PTE_WO;154 155 return (pfn << PAGE_SHIFT) | mask;156}157 158struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver,159 int trap_pagefaults, int invalid_type)160{161 struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL);162 uint32_t *v;163 int i;164 165 if (!pd)166 return NULL;167 168 pd->p = alloc_page(GFP_DMA32);169 if (!pd->p)170 goto out_err1;171 pd->dummy_pt = alloc_page(GFP_DMA32);172 if (!pd->dummy_pt)173 goto out_err2;174 pd->dummy_page = alloc_page(GFP_DMA32);175 if (!pd->dummy_page)176 goto out_err3;177 178 if (!trap_pagefaults) {179 pd->invalid_pde = psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt),180 invalid_type);181 pd->invalid_pte = psb_mmu_mask_pte(page_to_pfn(pd->dummy_page),182 invalid_type);183 } else {184 pd->invalid_pde = 0;185 pd->invalid_pte = 0;186 }187 188 v = kmap_local_page(pd->dummy_pt);189 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)190 v[i] = pd->invalid_pte;191 192 kunmap_local(v);193 194 v = kmap_local_page(pd->p);195 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)196 v[i] = pd->invalid_pde;197 198 kunmap_local(v);199 200 clear_page(kmap(pd->dummy_page));201 kunmap(pd->dummy_page);202 203 pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024);204 if (!pd->tables)205 goto out_err4;206 207 pd->hw_context = -1;208 pd->pd_mask = PSB_PTE_VALID;209 pd->driver = driver;210 211 return pd;212 213out_err4:214 __free_page(pd->dummy_page);215out_err3:216 __free_page(pd->dummy_pt);217out_err2:218 __free_page(pd->p);219out_err1:220 kfree(pd);221 return NULL;222}223 224static void psb_mmu_free_pt(struct psb_mmu_pt *pt)225{226 __free_page(pt->p);227 kfree(pt);228}229 230void psb_mmu_free_pagedir(struct psb_mmu_pd *pd)231{232 struct psb_mmu_driver *driver = pd->driver;233 struct drm_device *dev = driver->dev;234 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);235 struct psb_mmu_pt *pt;236 int i;237 238 down_write(&driver->sem);239 if (pd->hw_context != -1) {240 PSB_WSGX32(0, PSB_CR_BIF_DIR_LIST_BASE0 + pd->hw_context * 4);241 psb_mmu_flush_pd_locked(driver, 1);242 }243 244 /* Should take the spinlock here, but we don't need to do that245 since we have the semaphore in write mode. */246 247 for (i = 0; i < 1024; ++i) {248 pt = pd->tables[i];249 if (pt)250 psb_mmu_free_pt(pt);251 }252 253 vfree(pd->tables);254 __free_page(pd->dummy_page);255 __free_page(pd->dummy_pt);256 __free_page(pd->p);257 kfree(pd);258 up_write(&driver->sem);259}260 261static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd)262{263 struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL);264 void *v;265 uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT;266 uint32_t clflush_count = PAGE_SIZE / clflush_add;267 spinlock_t *lock = &pd->driver->lock;268 uint8_t *clf;269 uint32_t *ptes;270 int i;271 272 if (!pt)273 return NULL;274 275 pt->p = alloc_page(GFP_DMA32);276 if (!pt->p) {277 kfree(pt);278 return NULL;279 }280 281 spin_lock(lock);282 283 v = kmap_atomic(pt->p);284 clf = (uint8_t *) v;285 ptes = (uint32_t *) v;286 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)287 *ptes++ = pd->invalid_pte;288 289 if (pd->driver->has_clflush && pd->hw_context != -1) {290 mb();291 for (i = 0; i < clflush_count; ++i) {292 psb_clflush(clf);293 clf += clflush_add;294 }295 mb();296 }297 kunmap_atomic(v);298 spin_unlock(lock);299 300 pt->count = 0;301 pt->pd = pd;302 pt->index = 0;303 304 return pt;305}306 307static struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd,308 unsigned long addr)309{310 uint32_t index = psb_mmu_pd_index(addr);311 struct psb_mmu_pt *pt;312 uint32_t *v;313 spinlock_t *lock = &pd->driver->lock;314 315 spin_lock(lock);316 pt = pd->tables[index];317 while (!pt) {318 spin_unlock(lock);319 pt = psb_mmu_alloc_pt(pd);320 if (!pt)321 return NULL;322 spin_lock(lock);323 324 if (pd->tables[index]) {325 spin_unlock(lock);326 psb_mmu_free_pt(pt);327 spin_lock(lock);328 pt = pd->tables[index];329 continue;330 }331 332 v = kmap_atomic(pd->p);333 pd->tables[index] = pt;334 v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask;335 pt->index = index;336 kunmap_atomic((void *) v);337 338 if (pd->hw_context != -1) {339 psb_mmu_clflush(pd->driver, (void *)&v[index]);340 atomic_set(&pd->driver->needs_tlbflush, 1);341 }342 }343 pt->v = kmap_atomic(pt->p);344 return pt;345}346 347static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd,348 unsigned long addr)349{350 uint32_t index = psb_mmu_pd_index(addr);351 struct psb_mmu_pt *pt;352 spinlock_t *lock = &pd->driver->lock;353 354 spin_lock(lock);355 pt = pd->tables[index];356 if (!pt) {357 spin_unlock(lock);358 return NULL;359 }360 pt->v = kmap_atomic(pt->p);361 return pt;362}363 364static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt)365{366 struct psb_mmu_pd *pd = pt->pd;367 uint32_t *v;368 369 kunmap_atomic(pt->v);370 if (pt->count == 0) {371 v = kmap_atomic(pd->p);372 v[pt->index] = pd->invalid_pde;373 pd->tables[pt->index] = NULL;374 375 if (pd->hw_context != -1) {376 psb_mmu_clflush(pd->driver, (void *)&v[pt->index]);377 atomic_set(&pd->driver->needs_tlbflush, 1);378 }379 kunmap_atomic(v);380 spin_unlock(&pd->driver->lock);381 psb_mmu_free_pt(pt);382 return;383 }384 spin_unlock(&pd->driver->lock);385}386 387static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr,388 uint32_t pte)389{390 pt->v[psb_mmu_pt_index(addr)] = pte;391}392 393static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt,394 unsigned long addr)395{396 pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte;397}398 399struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)400{401 struct psb_mmu_pd *pd;402 403 down_read(&driver->sem);404 pd = driver->default_pd;405 up_read(&driver->sem);406 407 return pd;408}409 410void psb_mmu_driver_takedown(struct psb_mmu_driver *driver)411{412 struct drm_device *dev = driver->dev;413 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);414 415 PSB_WSGX32(driver->bif_ctrl, PSB_CR_BIF_CTRL);416 psb_mmu_free_pagedir(driver->default_pd);417 kfree(driver);418}419 420struct psb_mmu_driver *psb_mmu_driver_init(struct drm_device *dev,421 int trap_pagefaults,422 int invalid_type,423 atomic_t *msvdx_mmu_invaldc)424{425 struct psb_mmu_driver *driver;426 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);427 428 driver = kmalloc(sizeof(*driver), GFP_KERNEL);429 430 if (!driver)431 return NULL;432 433 driver->dev = dev;434 driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults,435 invalid_type);436 if (!driver->default_pd)437 goto out_err1;438 439 spin_lock_init(&driver->lock);440 init_rwsem(&driver->sem);441 down_write(&driver->sem);442 atomic_set(&driver->needs_tlbflush, 1);443 driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc;444 445 driver->bif_ctrl = PSB_RSGX32(PSB_CR_BIF_CTRL);446 PSB_WSGX32(driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT,447 PSB_CR_BIF_CTRL);448 PSB_WSGX32(driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT,449 PSB_CR_BIF_CTRL);450 451 driver->has_clflush = 0;452 453 if (boot_cpu_has(X86_FEATURE_CLFLUSH)) {454 uint32_t tfms, misc, cap0, cap4, clflush_size;455 456 /*457 * clflush size is determined at kernel setup for x86_64 but not458 * for i386. We have to do it here.459 */460 461 cpuid(0x00000001, &tfms, &misc, &cap0, &cap4);462 clflush_size = ((misc >> 8) & 0xff) * 8;463 driver->has_clflush = 1;464 driver->clflush_add =465 PAGE_SIZE * clflush_size / sizeof(uint32_t);466 driver->clflush_mask = driver->clflush_add - 1;467 driver->clflush_mask = ~driver->clflush_mask;468 }469 470 up_write(&driver->sem);471 return driver;472 473out_err1:474 kfree(driver);475 return NULL;476}477 478static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,479 uint32_t num_pages, uint32_t desired_tile_stride,480 uint32_t hw_tile_stride)481{482 struct psb_mmu_pt *pt;483 uint32_t rows = 1;484 uint32_t i;485 unsigned long addr;486 unsigned long end;487 unsigned long next;488 unsigned long add;489 unsigned long row_add;490 unsigned long clflush_add = pd->driver->clflush_add;491 unsigned long clflush_mask = pd->driver->clflush_mask;492 493 if (!pd->driver->has_clflush)494 return;495 496 if (hw_tile_stride)497 rows = num_pages / desired_tile_stride;498 else499 desired_tile_stride = num_pages;500 501 add = desired_tile_stride << PAGE_SHIFT;502 row_add = hw_tile_stride << PAGE_SHIFT;503 mb();504 for (i = 0; i < rows; ++i) {505 506 addr = address;507 end = addr + add;508 509 do {510 next = psb_pd_addr_end(addr, end);511 pt = psb_mmu_pt_map_lock(pd, addr);512 if (!pt)513 continue;514 do {515 psb_clflush(&pt->v[psb_mmu_pt_index(addr)]);516 } while (addr += clflush_add,517 (addr & clflush_mask) < next);518 519 psb_mmu_pt_unmap_unlock(pt);520 } while (addr = next, next != end);521 address += row_add;522 }523 mb();524}525 526void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd,527 unsigned long address, uint32_t num_pages)528{529 struct psb_mmu_pt *pt;530 unsigned long addr;531 unsigned long end;532 unsigned long next;533 unsigned long f_address = address;534 535 down_read(&pd->driver->sem);536 537 addr = address;538 end = addr + (num_pages << PAGE_SHIFT);539 540 do {541 next = psb_pd_addr_end(addr, end);542 pt = psb_mmu_pt_alloc_map_lock(pd, addr);543 if (!pt)544 goto out;545 do {546 psb_mmu_invalidate_pte(pt, addr);547 --pt->count;548 } while (addr += PAGE_SIZE, addr < next);549 psb_mmu_pt_unmap_unlock(pt);550 551 } while (addr = next, next != end);552 553out:554 if (pd->hw_context != -1)555 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);556 557 up_read(&pd->driver->sem);558 559 if (pd->hw_context != -1)560 psb_mmu_flush(pd->driver);561 562 return;563}564 565void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address,566 uint32_t num_pages, uint32_t desired_tile_stride,567 uint32_t hw_tile_stride)568{569 struct psb_mmu_pt *pt;570 uint32_t rows = 1;571 uint32_t i;572 unsigned long addr;573 unsigned long end;574 unsigned long next;575 unsigned long add;576 unsigned long row_add;577 unsigned long f_address = address;578 579 if (hw_tile_stride)580 rows = num_pages / desired_tile_stride;581 else582 desired_tile_stride = num_pages;583 584 add = desired_tile_stride << PAGE_SHIFT;585 row_add = hw_tile_stride << PAGE_SHIFT;586 587 down_read(&pd->driver->sem);588 589 /* Make sure we only need to flush this processor's cache */590 591 for (i = 0; i < rows; ++i) {592 593 addr = address;594 end = addr + add;595 596 do {597 next = psb_pd_addr_end(addr, end);598 pt = psb_mmu_pt_map_lock(pd, addr);599 if (!pt)600 continue;601 do {602 psb_mmu_invalidate_pte(pt, addr);603 --pt->count;604 605 } while (addr += PAGE_SIZE, addr < next);606 psb_mmu_pt_unmap_unlock(pt);607 608 } while (addr = next, next != end);609 address += row_add;610 }611 if (pd->hw_context != -1)612 psb_mmu_flush_ptes(pd, f_address, num_pages,613 desired_tile_stride, hw_tile_stride);614 615 up_read(&pd->driver->sem);616 617 if (pd->hw_context != -1)618 psb_mmu_flush(pd->driver);619}620 621int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn,622 unsigned long address, uint32_t num_pages,623 int type)624{625 struct psb_mmu_pt *pt;626 uint32_t pte;627 unsigned long addr;628 unsigned long end;629 unsigned long next;630 unsigned long f_address = address;631 int ret = -ENOMEM;632 633 down_read(&pd->driver->sem);634 635 addr = address;636 end = addr + (num_pages << PAGE_SHIFT);637 638 do {639 next = psb_pd_addr_end(addr, end);640 pt = psb_mmu_pt_alloc_map_lock(pd, addr);641 if (!pt) {642 ret = -ENOMEM;643 goto out;644 }645 do {646 pte = psb_mmu_mask_pte(start_pfn++, type);647 psb_mmu_set_pte(pt, addr, pte);648 pt->count++;649 } while (addr += PAGE_SIZE, addr < next);650 psb_mmu_pt_unmap_unlock(pt);651 652 } while (addr = next, next != end);653 ret = 0;654 655out:656 if (pd->hw_context != -1)657 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);658 659 up_read(&pd->driver->sem);660 661 if (pd->hw_context != -1)662 psb_mmu_flush(pd->driver);663 664 return ret;665}666 667int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,668 unsigned long address, uint32_t num_pages,669 uint32_t desired_tile_stride, uint32_t hw_tile_stride,670 int type)671{672 struct psb_mmu_pt *pt;673 uint32_t rows = 1;674 uint32_t i;675 uint32_t pte;676 unsigned long addr;677 unsigned long end;678 unsigned long next;679 unsigned long add;680 unsigned long row_add;681 unsigned long f_address = address;682 int ret = -ENOMEM;683 684 if (hw_tile_stride) {685 if (num_pages % desired_tile_stride != 0)686 return -EINVAL;687 rows = num_pages / desired_tile_stride;688 } else {689 desired_tile_stride = num_pages;690 }691 692 add = desired_tile_stride << PAGE_SHIFT;693 row_add = hw_tile_stride << PAGE_SHIFT;694 695 down_read(&pd->driver->sem);696 697 for (i = 0; i < rows; ++i) {698 699 addr = address;700 end = addr + add;701 702 do {703 next = psb_pd_addr_end(addr, end);704 pt = psb_mmu_pt_alloc_map_lock(pd, addr);705 if (!pt)706 goto out;707 do {708 pte = psb_mmu_mask_pte(page_to_pfn(*pages++),709 type);710 psb_mmu_set_pte(pt, addr, pte);711 pt->count++;712 } while (addr += PAGE_SIZE, addr < next);713 psb_mmu_pt_unmap_unlock(pt);714 715 } while (addr = next, next != end);716 717 address += row_add;718 }719 720 ret = 0;721out:722 if (pd->hw_context != -1)723 psb_mmu_flush_ptes(pd, f_address, num_pages,724 desired_tile_stride, hw_tile_stride);725 726 up_read(&pd->driver->sem);727 728 if (pd->hw_context != -1)729 psb_mmu_flush(pd->driver);730 731 return ret;732}733 734int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual,735 unsigned long *pfn)736{737 int ret;738 struct psb_mmu_pt *pt;739 uint32_t tmp;740 spinlock_t *lock = &pd->driver->lock;741 742 down_read(&pd->driver->sem);743 pt = psb_mmu_pt_map_lock(pd, virtual);744 if (!pt) {745 uint32_t *v;746 747 spin_lock(lock);748 v = kmap_atomic(pd->p);749 tmp = v[psb_mmu_pd_index(virtual)];750 kunmap_atomic(v);751 spin_unlock(lock);752 753 if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) ||754 !(pd->invalid_pte & PSB_PTE_VALID)) {755 ret = -EINVAL;756 goto out;757 }758 ret = 0;759 *pfn = pd->invalid_pte >> PAGE_SHIFT;760 goto out;761 }762 tmp = pt->v[psb_mmu_pt_index(virtual)];763 if (!(tmp & PSB_PTE_VALID)) {764 ret = -EINVAL;765 } else {766 ret = 0;767 *pfn = tmp >> PAGE_SHIFT;768 }769 psb_mmu_pt_unmap_unlock(pt);770out:771 up_read(&pd->driver->sem);772 return ret;773}774