1552 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * This is a module to test the HMM (Heterogeneous Memory Management)4 * mirror and zone device private memory migration APIs of the kernel.5 * Userspace programs can register with the driver to mirror their own address6 * space and can use the device to read/write any valid virtual address.7 */8#include <linux/init.h>9#include <linux/fs.h>10#include <linux/mm.h>11#include <linux/module.h>12#include <linux/kernel.h>13#include <linux/cdev.h>14#include <linux/device.h>15#include <linux/memremap.h>16#include <linux/mutex.h>17#include <linux/rwsem.h>18#include <linux/sched.h>19#include <linux/slab.h>20#include <linux/highmem.h>21#include <linux/delay.h>22#include <linux/pagemap.h>23#include <linux/hmm.h>24#include <linux/vmalloc.h>25#include <linux/swap.h>26#include <linux/swapops.h>27#include <linux/sched/mm.h>28#include <linux/platform_device.h>29#include <linux/rmap.h>30#include <linux/mmu_notifier.h>31#include <linux/migrate.h>32 33#include "test_hmm_uapi.h"34 35#define DMIRROR_NDEVICES 436#define DMIRROR_RANGE_FAULT_TIMEOUT 100037#define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U)38#define DEVMEM_CHUNKS_RESERVE 1639 40/*41 * For device_private pages, dpage is just a dummy struct page42 * representing a piece of device memory. dmirror_devmem_alloc_page43 * allocates a real system memory page as backing storage to fake a44 * real device. zone_device_data points to that backing page. But45 * for device_coherent memory, the struct page represents real46 * physical CPU-accessible memory that we can use directly.47 */48#define BACKING_PAGE(page) (is_device_private_page((page)) ? \49 (page)->zone_device_data : (page))50 51static unsigned long spm_addr_dev0;52module_param(spm_addr_dev0, long, 0644);53MODULE_PARM_DESC(spm_addr_dev0,54 "Specify start address for SPM (special purpose memory) used for device 0. By setting this Coherent device type will be used. Make sure spm_addr_dev1 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE.");55 56static unsigned long spm_addr_dev1;57module_param(spm_addr_dev1, long, 0644);58MODULE_PARM_DESC(spm_addr_dev1,59 "Specify start address for SPM (special purpose memory) used for device 1. By setting this Coherent device type will be used. Make sure spm_addr_dev0 is set too. Minimum SPM size should be DEVMEM_CHUNK_SIZE.");60 61static const struct dev_pagemap_ops dmirror_devmem_ops;62static const struct mmu_interval_notifier_ops dmirror_min_ops;63static dev_t dmirror_dev;64 65struct dmirror_device;66 67struct dmirror_bounce {68 void *ptr;69 unsigned long size;70 unsigned long addr;71 unsigned long cpages;72};73 74#define DPT_XA_TAG_ATOMIC 1UL75#define DPT_XA_TAG_WRITE 3UL76 77/*78 * Data structure to track address ranges and register for mmu interval79 * notifier updates.80 */81struct dmirror_interval {82 struct mmu_interval_notifier notifier;83 struct dmirror *dmirror;84};85 86/*87 * Data attached to the open device file.88 * Note that it might be shared after a fork().89 */90struct dmirror {91 struct dmirror_device *mdevice;92 struct xarray pt;93 struct mmu_interval_notifier notifier;94 struct mutex mutex;95};96 97/*98 * ZONE_DEVICE pages for migration and simulating device memory.99 */100struct dmirror_chunk {101 struct dev_pagemap pagemap;102 struct dmirror_device *mdevice;103 bool remove;104};105 106/*107 * Per device data.108 */109struct dmirror_device {110 struct cdev cdevice;111 unsigned int zone_device_type;112 struct device device;113 114 unsigned int devmem_capacity;115 unsigned int devmem_count;116 struct dmirror_chunk **devmem_chunks;117 struct mutex devmem_lock; /* protects the above */118 119 unsigned long calloc;120 unsigned long cfree;121 struct page *free_pages;122 spinlock_t lock; /* protects the above */123};124 125static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES];126 127static int dmirror_bounce_init(struct dmirror_bounce *bounce,128 unsigned long addr,129 unsigned long size)130{131 bounce->addr = addr;132 bounce->size = size;133 bounce->cpages = 0;134 bounce->ptr = vmalloc(size);135 if (!bounce->ptr)136 return -ENOMEM;137 return 0;138}139 140static bool dmirror_is_private_zone(struct dmirror_device *mdevice)141{142 return (mdevice->zone_device_type ==143 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ? true : false;144}145 146static enum migrate_vma_direction147dmirror_select_device(struct dmirror *dmirror)148{149 return (dmirror->mdevice->zone_device_type ==150 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE) ?151 MIGRATE_VMA_SELECT_DEVICE_PRIVATE :152 MIGRATE_VMA_SELECT_DEVICE_COHERENT;153}154 155static void dmirror_bounce_fini(struct dmirror_bounce *bounce)156{157 vfree(bounce->ptr);158}159 160static int dmirror_fops_open(struct inode *inode, struct file *filp)161{162 struct cdev *cdev = inode->i_cdev;163 struct dmirror *dmirror;164 int ret;165 166 /* Mirror this process address space */167 dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL);168 if (dmirror == NULL)169 return -ENOMEM;170 171 dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice);172 mutex_init(&dmirror->mutex);173 xa_init(&dmirror->pt);174 175 ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm,176 0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops);177 if (ret) {178 kfree(dmirror);179 return ret;180 }181 182 filp->private_data = dmirror;183 return 0;184}185 186static int dmirror_fops_release(struct inode *inode, struct file *filp)187{188 struct dmirror *dmirror = filp->private_data;189 190 mmu_interval_notifier_remove(&dmirror->notifier);191 xa_destroy(&dmirror->pt);192 kfree(dmirror);193 return 0;194}195 196static struct dmirror_chunk *dmirror_page_to_chunk(struct page *page)197{198 return container_of(page->pgmap, struct dmirror_chunk, pagemap);199}200 201static struct dmirror_device *dmirror_page_to_device(struct page *page)202 203{204 return dmirror_page_to_chunk(page)->mdevice;205}206 207static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range)208{209 unsigned long *pfns = range->hmm_pfns;210 unsigned long pfn;211 212 for (pfn = (range->start >> PAGE_SHIFT);213 pfn < (range->end >> PAGE_SHIFT);214 pfn++, pfns++) {215 struct page *page;216 void *entry;217 218 /*219 * Since we asked for hmm_range_fault() to populate pages,220 * it shouldn't return an error entry on success.221 */222 WARN_ON(*pfns & HMM_PFN_ERROR);223 WARN_ON(!(*pfns & HMM_PFN_VALID));224 225 page = hmm_pfn_to_page(*pfns);226 WARN_ON(!page);227 228 entry = page;229 if (*pfns & HMM_PFN_WRITE)230 entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);231 else if (WARN_ON(range->default_flags & HMM_PFN_WRITE))232 return -EFAULT;233 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);234 if (xa_is_err(entry))235 return xa_err(entry);236 }237 238 return 0;239}240 241static void dmirror_do_update(struct dmirror *dmirror, unsigned long start,242 unsigned long end)243{244 unsigned long pfn;245 void *entry;246 247 /*248 * The XArray doesn't hold references to pages since it relies on249 * the mmu notifier to clear page pointers when they become stale.250 * Therefore, it is OK to just clear the entry.251 */252 xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT,253 end >> PAGE_SHIFT)254 xa_erase(&dmirror->pt, pfn);255}256 257static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni,258 const struct mmu_notifier_range *range,259 unsigned long cur_seq)260{261 struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);262 263 /*264 * Ignore invalidation callbacks for device private pages since265 * the invalidation is handled as part of the migration process.266 */267 if (range->event == MMU_NOTIFY_MIGRATE &&268 range->owner == dmirror->mdevice)269 return true;270 271 if (mmu_notifier_range_blockable(range))272 mutex_lock(&dmirror->mutex);273 else if (!mutex_trylock(&dmirror->mutex))274 return false;275 276 mmu_interval_set_seq(mni, cur_seq);277 dmirror_do_update(dmirror, range->start, range->end);278 279 mutex_unlock(&dmirror->mutex);280 return true;281}282 283static const struct mmu_interval_notifier_ops dmirror_min_ops = {284 .invalidate = dmirror_interval_invalidate,285};286 287static int dmirror_range_fault(struct dmirror *dmirror,288 struct hmm_range *range)289{290 struct mm_struct *mm = dmirror->notifier.mm;291 unsigned long timeout =292 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);293 int ret;294 295 while (true) {296 if (time_after(jiffies, timeout)) {297 ret = -EBUSY;298 goto out;299 }300 301 range->notifier_seq = mmu_interval_read_begin(range->notifier);302 mmap_read_lock(mm);303 ret = hmm_range_fault(range);304 mmap_read_unlock(mm);305 if (ret) {306 if (ret == -EBUSY)307 continue;308 goto out;309 }310 311 mutex_lock(&dmirror->mutex);312 if (mmu_interval_read_retry(range->notifier,313 range->notifier_seq)) {314 mutex_unlock(&dmirror->mutex);315 continue;316 }317 break;318 }319 320 ret = dmirror_do_fault(dmirror, range);321 322 mutex_unlock(&dmirror->mutex);323out:324 return ret;325}326 327static int dmirror_fault(struct dmirror *dmirror, unsigned long start,328 unsigned long end, bool write)329{330 struct mm_struct *mm = dmirror->notifier.mm;331 unsigned long addr;332 unsigned long pfns[64];333 struct hmm_range range = {334 .notifier = &dmirror->notifier,335 .hmm_pfns = pfns,336 .pfn_flags_mask = 0,337 .default_flags =338 HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0),339 .dev_private_owner = dmirror->mdevice,340 };341 int ret = 0;342 343 /* Since the mm is for the mirrored process, get a reference first. */344 if (!mmget_not_zero(mm))345 return 0;346 347 for (addr = start; addr < end; addr = range.end) {348 range.start = addr;349 range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);350 351 ret = dmirror_range_fault(dmirror, &range);352 if (ret)353 break;354 }355 356 mmput(mm);357 return ret;358}359 360static int dmirror_do_read(struct dmirror *dmirror, unsigned long start,361 unsigned long end, struct dmirror_bounce *bounce)362{363 unsigned long pfn;364 void *ptr;365 366 ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);367 368 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {369 void *entry;370 struct page *page;371 372 entry = xa_load(&dmirror->pt, pfn);373 page = xa_untag_pointer(entry);374 if (!page)375 return -ENOENT;376 377 memcpy_from_page(ptr, page, 0, PAGE_SIZE);378 379 ptr += PAGE_SIZE;380 bounce->cpages++;381 }382 383 return 0;384}385 386static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)387{388 struct dmirror_bounce bounce;389 unsigned long start, end;390 unsigned long size = cmd->npages << PAGE_SHIFT;391 int ret;392 393 start = cmd->addr;394 end = start + size;395 if (end < start)396 return -EINVAL;397 398 ret = dmirror_bounce_init(&bounce, start, size);399 if (ret)400 return ret;401 402 while (1) {403 mutex_lock(&dmirror->mutex);404 ret = dmirror_do_read(dmirror, start, end, &bounce);405 mutex_unlock(&dmirror->mutex);406 if (ret != -ENOENT)407 break;408 409 start = cmd->addr + (bounce.cpages << PAGE_SHIFT);410 ret = dmirror_fault(dmirror, start, end, false);411 if (ret)412 break;413 cmd->faults++;414 }415 416 if (ret == 0) {417 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,418 bounce.size))419 ret = -EFAULT;420 }421 cmd->cpages = bounce.cpages;422 dmirror_bounce_fini(&bounce);423 return ret;424}425 426static int dmirror_do_write(struct dmirror *dmirror, unsigned long start,427 unsigned long end, struct dmirror_bounce *bounce)428{429 unsigned long pfn;430 void *ptr;431 432 ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);433 434 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {435 void *entry;436 struct page *page;437 438 entry = xa_load(&dmirror->pt, pfn);439 page = xa_untag_pointer(entry);440 if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE)441 return -ENOENT;442 443 memcpy_to_page(page, 0, ptr, PAGE_SIZE);444 445 ptr += PAGE_SIZE;446 bounce->cpages++;447 }448 449 return 0;450}451 452static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)453{454 struct dmirror_bounce bounce;455 unsigned long start, end;456 unsigned long size = cmd->npages << PAGE_SHIFT;457 int ret;458 459 start = cmd->addr;460 end = start + size;461 if (end < start)462 return -EINVAL;463 464 ret = dmirror_bounce_init(&bounce, start, size);465 if (ret)466 return ret;467 if (copy_from_user(bounce.ptr, u64_to_user_ptr(cmd->ptr),468 bounce.size)) {469 ret = -EFAULT;470 goto fini;471 }472 473 while (1) {474 mutex_lock(&dmirror->mutex);475 ret = dmirror_do_write(dmirror, start, end, &bounce);476 mutex_unlock(&dmirror->mutex);477 if (ret != -ENOENT)478 break;479 480 start = cmd->addr + (bounce.cpages << PAGE_SHIFT);481 ret = dmirror_fault(dmirror, start, end, true);482 if (ret)483 break;484 cmd->faults++;485 }486 487fini:488 cmd->cpages = bounce.cpages;489 dmirror_bounce_fini(&bounce);490 return ret;491}492 493static int dmirror_allocate_chunk(struct dmirror_device *mdevice,494 struct page **ppage)495{496 struct dmirror_chunk *devmem;497 struct resource *res = NULL;498 unsigned long pfn;499 unsigned long pfn_first;500 unsigned long pfn_last;501 void *ptr;502 int ret = -ENOMEM;503 504 devmem = kzalloc(sizeof(*devmem), GFP_KERNEL);505 if (!devmem)506 return ret;507 508 switch (mdevice->zone_device_type) {509 case HMM_DMIRROR_MEMORY_DEVICE_PRIVATE:510 res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE,511 "hmm_dmirror");512 if (IS_ERR_OR_NULL(res))513 goto err_devmem;514 devmem->pagemap.range.start = res->start;515 devmem->pagemap.range.end = res->end;516 devmem->pagemap.type = MEMORY_DEVICE_PRIVATE;517 break;518 case HMM_DMIRROR_MEMORY_DEVICE_COHERENT:519 devmem->pagemap.range.start = (MINOR(mdevice->cdevice.dev) - 2) ?520 spm_addr_dev0 :521 spm_addr_dev1;522 devmem->pagemap.range.end = devmem->pagemap.range.start +523 DEVMEM_CHUNK_SIZE - 1;524 devmem->pagemap.type = MEMORY_DEVICE_COHERENT;525 break;526 default:527 ret = -EINVAL;528 goto err_devmem;529 }530 531 devmem->pagemap.nr_range = 1;532 devmem->pagemap.ops = &dmirror_devmem_ops;533 devmem->pagemap.owner = mdevice;534 535 mutex_lock(&mdevice->devmem_lock);536 537 if (mdevice->devmem_count == mdevice->devmem_capacity) {538 struct dmirror_chunk **new_chunks;539 unsigned int new_capacity;540 541 new_capacity = mdevice->devmem_capacity +542 DEVMEM_CHUNKS_RESERVE;543 new_chunks = krealloc(mdevice->devmem_chunks,544 sizeof(new_chunks[0]) * new_capacity,545 GFP_KERNEL);546 if (!new_chunks)547 goto err_release;548 mdevice->devmem_capacity = new_capacity;549 mdevice->devmem_chunks = new_chunks;550 }551 ptr = memremap_pages(&devmem->pagemap, numa_node_id());552 if (IS_ERR_OR_NULL(ptr)) {553 if (ptr)554 ret = PTR_ERR(ptr);555 else556 ret = -EFAULT;557 goto err_release;558 }559 560 devmem->mdevice = mdevice;561 pfn_first = devmem->pagemap.range.start >> PAGE_SHIFT;562 pfn_last = pfn_first + (range_len(&devmem->pagemap.range) >> PAGE_SHIFT);563 mdevice->devmem_chunks[mdevice->devmem_count++] = devmem;564 565 mutex_unlock(&mdevice->devmem_lock);566 567 pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n",568 DEVMEM_CHUNK_SIZE / (1024 * 1024),569 mdevice->devmem_count,570 mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)),571 pfn_first, pfn_last);572 573 spin_lock(&mdevice->lock);574 for (pfn = pfn_first; pfn < pfn_last; pfn++) {575 struct page *page = pfn_to_page(pfn);576 577 page->zone_device_data = mdevice->free_pages;578 mdevice->free_pages = page;579 }580 if (ppage) {581 *ppage = mdevice->free_pages;582 mdevice->free_pages = (*ppage)->zone_device_data;583 mdevice->calloc++;584 }585 spin_unlock(&mdevice->lock);586 587 return 0;588 589err_release:590 mutex_unlock(&mdevice->devmem_lock);591 if (res && devmem->pagemap.type == MEMORY_DEVICE_PRIVATE)592 release_mem_region(devmem->pagemap.range.start,593 range_len(&devmem->pagemap.range));594err_devmem:595 kfree(devmem);596 597 return ret;598}599 600static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice)601{602 struct page *dpage = NULL;603 struct page *rpage = NULL;604 605 /*606 * For ZONE_DEVICE private type, this is a fake device so we allocate607 * real system memory to store our device memory.608 * For ZONE_DEVICE coherent type we use the actual dpage to store the609 * data and ignore rpage.610 */611 if (dmirror_is_private_zone(mdevice)) {612 rpage = alloc_page(GFP_HIGHUSER);613 if (!rpage)614 return NULL;615 }616 spin_lock(&mdevice->lock);617 618 if (mdevice->free_pages) {619 dpage = mdevice->free_pages;620 mdevice->free_pages = dpage->zone_device_data;621 mdevice->calloc++;622 spin_unlock(&mdevice->lock);623 } else {624 spin_unlock(&mdevice->lock);625 if (dmirror_allocate_chunk(mdevice, &dpage))626 goto error;627 }628 629 zone_device_page_init(dpage);630 dpage->zone_device_data = rpage;631 return dpage;632 633error:634 if (rpage)635 __free_page(rpage);636 return NULL;637}638 639static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args,640 struct dmirror *dmirror)641{642 struct dmirror_device *mdevice = dmirror->mdevice;643 const unsigned long *src = args->src;644 unsigned long *dst = args->dst;645 unsigned long addr;646 647 for (addr = args->start; addr < args->end; addr += PAGE_SIZE,648 src++, dst++) {649 struct page *spage;650 struct page *dpage;651 struct page *rpage;652 653 if (!(*src & MIGRATE_PFN_MIGRATE))654 continue;655 656 /*657 * Note that spage might be NULL which is OK since it is an658 * unallocated pte_none() or read-only zero page.659 */660 spage = migrate_pfn_to_page(*src);661 if (WARN(spage && is_zone_device_page(spage),662 "page already in device spage pfn: 0x%lx\n",663 page_to_pfn(spage)))664 continue;665 666 dpage = dmirror_devmem_alloc_page(mdevice);667 if (!dpage)668 continue;669 670 rpage = BACKING_PAGE(dpage);671 if (spage)672 copy_highpage(rpage, spage);673 else674 clear_highpage(rpage);675 676 /*677 * Normally, a device would use the page->zone_device_data to678 * point to the mirror but here we use it to hold the page for679 * the simulated device memory and that page holds the pointer680 * to the mirror.681 */682 rpage->zone_device_data = dmirror;683 684 pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n",685 page_to_pfn(spage), page_to_pfn(dpage));686 *dst = migrate_pfn(page_to_pfn(dpage));687 if ((*src & MIGRATE_PFN_WRITE) ||688 (!spage && args->vma->vm_flags & VM_WRITE))689 *dst |= MIGRATE_PFN_WRITE;690 }691}692 693static int dmirror_check_atomic(struct dmirror *dmirror, unsigned long start,694 unsigned long end)695{696 unsigned long pfn;697 698 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {699 void *entry;700 701 entry = xa_load(&dmirror->pt, pfn);702 if (xa_pointer_tag(entry) == DPT_XA_TAG_ATOMIC)703 return -EPERM;704 }705 706 return 0;707}708 709static int dmirror_atomic_map(unsigned long start, unsigned long end,710 struct page **pages, struct dmirror *dmirror)711{712 unsigned long pfn, mapped = 0;713 int i;714 715 /* Map the migrated pages into the device's page tables. */716 mutex_lock(&dmirror->mutex);717 718 for (i = 0, pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++, i++) {719 void *entry;720 721 if (!pages[i])722 continue;723 724 entry = pages[i];725 entry = xa_tag_pointer(entry, DPT_XA_TAG_ATOMIC);726 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);727 if (xa_is_err(entry)) {728 mutex_unlock(&dmirror->mutex);729 return xa_err(entry);730 }731 732 mapped++;733 }734 735 mutex_unlock(&dmirror->mutex);736 return mapped;737}738 739static int dmirror_migrate_finalize_and_map(struct migrate_vma *args,740 struct dmirror *dmirror)741{742 unsigned long start = args->start;743 unsigned long end = args->end;744 const unsigned long *src = args->src;745 const unsigned long *dst = args->dst;746 unsigned long pfn;747 748 /* Map the migrated pages into the device's page tables. */749 mutex_lock(&dmirror->mutex);750 751 for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++,752 src++, dst++) {753 struct page *dpage;754 void *entry;755 756 if (!(*src & MIGRATE_PFN_MIGRATE))757 continue;758 759 dpage = migrate_pfn_to_page(*dst);760 if (!dpage)761 continue;762 763 entry = BACKING_PAGE(dpage);764 if (*dst & MIGRATE_PFN_WRITE)765 entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);766 entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);767 if (xa_is_err(entry)) {768 mutex_unlock(&dmirror->mutex);769 return xa_err(entry);770 }771 }772 773 mutex_unlock(&dmirror->mutex);774 return 0;775}776 777static int dmirror_exclusive(struct dmirror *dmirror,778 struct hmm_dmirror_cmd *cmd)779{780 unsigned long start, end, addr;781 unsigned long size = cmd->npages << PAGE_SHIFT;782 struct mm_struct *mm = dmirror->notifier.mm;783 struct page *pages[64];784 struct dmirror_bounce bounce;785 unsigned long next;786 int ret;787 788 start = cmd->addr;789 end = start + size;790 if (end < start)791 return -EINVAL;792 793 /* Since the mm is for the mirrored process, get a reference first. */794 if (!mmget_not_zero(mm))795 return -EINVAL;796 797 mmap_read_lock(mm);798 for (addr = start; addr < end; addr = next) {799 unsigned long mapped = 0;800 int i;801 802 next = min(end, addr + (ARRAY_SIZE(pages) << PAGE_SHIFT));803 804 ret = make_device_exclusive_range(mm, addr, next, pages, NULL);805 /*806 * Do dmirror_atomic_map() iff all pages are marked for807 * exclusive access to avoid accessing uninitialized808 * fields of pages.809 */810 if (ret == (next - addr) >> PAGE_SHIFT)811 mapped = dmirror_atomic_map(addr, next, pages, dmirror);812 for (i = 0; i < ret; i++) {813 if (pages[i]) {814 unlock_page(pages[i]);815 put_page(pages[i]);816 }817 }818 819 if (addr + (mapped << PAGE_SHIFT) < next) {820 mmap_read_unlock(mm);821 mmput(mm);822 return -EBUSY;823 }824 }825 mmap_read_unlock(mm);826 mmput(mm);827 828 /* Return the migrated data for verification. */829 ret = dmirror_bounce_init(&bounce, start, size);830 if (ret)831 return ret;832 mutex_lock(&dmirror->mutex);833 ret = dmirror_do_read(dmirror, start, end, &bounce);834 mutex_unlock(&dmirror->mutex);835 if (ret == 0) {836 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,837 bounce.size))838 ret = -EFAULT;839 }840 841 cmd->cpages = bounce.cpages;842 dmirror_bounce_fini(&bounce);843 return ret;844}845 846static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args,847 struct dmirror *dmirror)848{849 const unsigned long *src = args->src;850 unsigned long *dst = args->dst;851 unsigned long start = args->start;852 unsigned long end = args->end;853 unsigned long addr;854 855 for (addr = start; addr < end; addr += PAGE_SIZE,856 src++, dst++) {857 struct page *dpage, *spage;858 859 spage = migrate_pfn_to_page(*src);860 if (!spage || !(*src & MIGRATE_PFN_MIGRATE))861 continue;862 863 if (WARN_ON(!is_device_private_page(spage) &&864 !is_device_coherent_page(spage)))865 continue;866 spage = BACKING_PAGE(spage);867 dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr);868 if (!dpage)869 continue;870 pr_debug("migrating from dev to sys pfn src: 0x%lx pfn dst: 0x%lx\n",871 page_to_pfn(spage), page_to_pfn(dpage));872 873 lock_page(dpage);874 xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);875 copy_highpage(dpage, spage);876 *dst = migrate_pfn(page_to_pfn(dpage));877 if (*src & MIGRATE_PFN_WRITE)878 *dst |= MIGRATE_PFN_WRITE;879 }880 return 0;881}882 883static unsigned long884dmirror_successful_migrated_pages(struct migrate_vma *migrate)885{886 unsigned long cpages = 0;887 unsigned long i;888 889 for (i = 0; i < migrate->npages; i++) {890 if (migrate->src[i] & MIGRATE_PFN_VALID &&891 migrate->src[i] & MIGRATE_PFN_MIGRATE)892 cpages++;893 }894 return cpages;895}896 897static int dmirror_migrate_to_system(struct dmirror *dmirror,898 struct hmm_dmirror_cmd *cmd)899{900 unsigned long start, end, addr;901 unsigned long size = cmd->npages << PAGE_SHIFT;902 struct mm_struct *mm = dmirror->notifier.mm;903 struct vm_area_struct *vma;904 unsigned long src_pfns[64] = { 0 };905 unsigned long dst_pfns[64] = { 0 };906 struct migrate_vma args = { 0 };907 unsigned long next;908 int ret;909 910 start = cmd->addr;911 end = start + size;912 if (end < start)913 return -EINVAL;914 915 /* Since the mm is for the mirrored process, get a reference first. */916 if (!mmget_not_zero(mm))917 return -EINVAL;918 919 cmd->cpages = 0;920 mmap_read_lock(mm);921 for (addr = start; addr < end; addr = next) {922 vma = vma_lookup(mm, addr);923 if (!vma || !(vma->vm_flags & VM_READ)) {924 ret = -EINVAL;925 goto out;926 }927 next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT));928 if (next > vma->vm_end)929 next = vma->vm_end;930 931 args.vma = vma;932 args.src = src_pfns;933 args.dst = dst_pfns;934 args.start = addr;935 args.end = next;936 args.pgmap_owner = dmirror->mdevice;937 args.flags = dmirror_select_device(dmirror);938 939 ret = migrate_vma_setup(&args);940 if (ret)941 goto out;942 943 pr_debug("Migrating from device mem to sys mem\n");944 dmirror_devmem_fault_alloc_and_copy(&args, dmirror);945 946 migrate_vma_pages(&args);947 cmd->cpages += dmirror_successful_migrated_pages(&args);948 migrate_vma_finalize(&args);949 }950out:951 mmap_read_unlock(mm);952 mmput(mm);953 954 return ret;955}956 957static int dmirror_migrate_to_device(struct dmirror *dmirror,958 struct hmm_dmirror_cmd *cmd)959{960 unsigned long start, end, addr;961 unsigned long size = cmd->npages << PAGE_SHIFT;962 struct mm_struct *mm = dmirror->notifier.mm;963 struct vm_area_struct *vma;964 unsigned long src_pfns[64] = { 0 };965 unsigned long dst_pfns[64] = { 0 };966 struct dmirror_bounce bounce;967 struct migrate_vma args = { 0 };968 unsigned long next;969 int ret;970 971 start = cmd->addr;972 end = start + size;973 if (end < start)974 return -EINVAL;975 976 /* Since the mm is for the mirrored process, get a reference first. */977 if (!mmget_not_zero(mm))978 return -EINVAL;979 980 mmap_read_lock(mm);981 for (addr = start; addr < end; addr = next) {982 vma = vma_lookup(mm, addr);983 if (!vma || !(vma->vm_flags & VM_READ)) {984 ret = -EINVAL;985 goto out;986 }987 next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT));988 if (next > vma->vm_end)989 next = vma->vm_end;990 991 args.vma = vma;992 args.src = src_pfns;993 args.dst = dst_pfns;994 args.start = addr;995 args.end = next;996 args.pgmap_owner = dmirror->mdevice;997 args.flags = MIGRATE_VMA_SELECT_SYSTEM;998 ret = migrate_vma_setup(&args);999 if (ret)1000 goto out;1001 1002 pr_debug("Migrating from sys mem to device mem\n");1003 dmirror_migrate_alloc_and_copy(&args, dmirror);1004 migrate_vma_pages(&args);1005 dmirror_migrate_finalize_and_map(&args, dmirror);1006 migrate_vma_finalize(&args);1007 }1008 mmap_read_unlock(mm);1009 mmput(mm);1010 1011 /*1012 * Return the migrated data for verification.1013 * Only for pages in device zone1014 */1015 ret = dmirror_bounce_init(&bounce, start, size);1016 if (ret)1017 return ret;1018 mutex_lock(&dmirror->mutex);1019 ret = dmirror_do_read(dmirror, start, end, &bounce);1020 mutex_unlock(&dmirror->mutex);1021 if (ret == 0) {1022 if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,1023 bounce.size))1024 ret = -EFAULT;1025 }1026 cmd->cpages = bounce.cpages;1027 dmirror_bounce_fini(&bounce);1028 return ret;1029 1030out:1031 mmap_read_unlock(mm);1032 mmput(mm);1033 return ret;1034}1035 1036static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range,1037 unsigned char *perm, unsigned long entry)1038{1039 struct page *page;1040 1041 if (entry & HMM_PFN_ERROR) {1042 *perm = HMM_DMIRROR_PROT_ERROR;1043 return;1044 }1045 if (!(entry & HMM_PFN_VALID)) {1046 *perm = HMM_DMIRROR_PROT_NONE;1047 return;1048 }1049 1050 page = hmm_pfn_to_page(entry);1051 if (is_device_private_page(page)) {1052 /* Is the page migrated to this device or some other? */1053 if (dmirror->mdevice == dmirror_page_to_device(page))1054 *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL;1055 else1056 *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE;1057 } else if (is_device_coherent_page(page)) {1058 /* Is the page migrated to this device or some other? */1059 if (dmirror->mdevice == dmirror_page_to_device(page))1060 *perm = HMM_DMIRROR_PROT_DEV_COHERENT_LOCAL;1061 else1062 *perm = HMM_DMIRROR_PROT_DEV_COHERENT_REMOTE;1063 } else if (is_zero_pfn(page_to_pfn(page)))1064 *perm = HMM_DMIRROR_PROT_ZERO;1065 else1066 *perm = HMM_DMIRROR_PROT_NONE;1067 if (entry & HMM_PFN_WRITE)1068 *perm |= HMM_DMIRROR_PROT_WRITE;1069 else1070 *perm |= HMM_DMIRROR_PROT_READ;1071 if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT)1072 *perm |= HMM_DMIRROR_PROT_PMD;1073 else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT)1074 *perm |= HMM_DMIRROR_PROT_PUD;1075}1076 1077static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni,1078 const struct mmu_notifier_range *range,1079 unsigned long cur_seq)1080{1081 struct dmirror_interval *dmi =1082 container_of(mni, struct dmirror_interval, notifier);1083 struct dmirror *dmirror = dmi->dmirror;1084 1085 if (mmu_notifier_range_blockable(range))1086 mutex_lock(&dmirror->mutex);1087 else if (!mutex_trylock(&dmirror->mutex))1088 return false;1089 1090 /*1091 * Snapshots only need to set the sequence number since any1092 * invalidation in the interval invalidates the whole snapshot.1093 */1094 mmu_interval_set_seq(mni, cur_seq);1095 1096 mutex_unlock(&dmirror->mutex);1097 return true;1098}1099 1100static const struct mmu_interval_notifier_ops dmirror_mrn_ops = {1101 .invalidate = dmirror_snapshot_invalidate,1102};1103 1104static int dmirror_range_snapshot(struct dmirror *dmirror,1105 struct hmm_range *range,1106 unsigned char *perm)1107{1108 struct mm_struct *mm = dmirror->notifier.mm;1109 struct dmirror_interval notifier;1110 unsigned long timeout =1111 jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);1112 unsigned long i;1113 unsigned long n;1114 int ret = 0;1115 1116 notifier.dmirror = dmirror;1117 range->notifier = ¬ifier.notifier;1118 1119 ret = mmu_interval_notifier_insert(range->notifier, mm,1120 range->start, range->end - range->start,1121 &dmirror_mrn_ops);1122 if (ret)1123 return ret;1124 1125 while (true) {1126 if (time_after(jiffies, timeout)) {1127 ret = -EBUSY;1128 goto out;1129 }1130 1131 range->notifier_seq = mmu_interval_read_begin(range->notifier);1132 1133 mmap_read_lock(mm);1134 ret = hmm_range_fault(range);1135 mmap_read_unlock(mm);1136 if (ret) {1137 if (ret == -EBUSY)1138 continue;1139 goto out;1140 }1141 1142 mutex_lock(&dmirror->mutex);1143 if (mmu_interval_read_retry(range->notifier,1144 range->notifier_seq)) {1145 mutex_unlock(&dmirror->mutex);1146 continue;1147 }1148 break;1149 }1150 1151 n = (range->end - range->start) >> PAGE_SHIFT;1152 for (i = 0; i < n; i++)1153 dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]);1154 1155 mutex_unlock(&dmirror->mutex);1156out:1157 mmu_interval_notifier_remove(range->notifier);1158 return ret;1159}1160 1161static int dmirror_snapshot(struct dmirror *dmirror,1162 struct hmm_dmirror_cmd *cmd)1163{1164 struct mm_struct *mm = dmirror->notifier.mm;1165 unsigned long start, end;1166 unsigned long size = cmd->npages << PAGE_SHIFT;1167 unsigned long addr;1168 unsigned long next;1169 unsigned long pfns[64];1170 unsigned char perm[64];1171 char __user *uptr;1172 struct hmm_range range = {1173 .hmm_pfns = pfns,1174 .dev_private_owner = dmirror->mdevice,1175 };1176 int ret = 0;1177 1178 start = cmd->addr;1179 end = start + size;1180 if (end < start)1181 return -EINVAL;1182 1183 /* Since the mm is for the mirrored process, get a reference first. */1184 if (!mmget_not_zero(mm))1185 return -EINVAL;1186 1187 /*1188 * Register a temporary notifier to detect invalidations even if it1189 * overlaps with other mmu_interval_notifiers.1190 */1191 uptr = u64_to_user_ptr(cmd->ptr);1192 for (addr = start; addr < end; addr = next) {1193 unsigned long n;1194 1195 next = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);1196 range.start = addr;1197 range.end = next;1198 1199 ret = dmirror_range_snapshot(dmirror, &range, perm);1200 if (ret)1201 break;1202 1203 n = (range.end - range.start) >> PAGE_SHIFT;1204 if (copy_to_user(uptr, perm, n)) {1205 ret = -EFAULT;1206 break;1207 }1208 1209 cmd->cpages += n;1210 uptr += n;1211 }1212 mmput(mm);1213 1214 return ret;1215}1216 1217static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk)1218{1219 unsigned long start_pfn = chunk->pagemap.range.start >> PAGE_SHIFT;1220 unsigned long end_pfn = chunk->pagemap.range.end >> PAGE_SHIFT;1221 unsigned long npages = end_pfn - start_pfn + 1;1222 unsigned long i;1223 unsigned long *src_pfns;1224 unsigned long *dst_pfns;1225 1226 src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL);1227 dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL);1228 1229 migrate_device_range(src_pfns, start_pfn, npages);1230 for (i = 0; i < npages; i++) {1231 struct page *dpage, *spage;1232 1233 spage = migrate_pfn_to_page(src_pfns[i]);1234 if (!spage || !(src_pfns[i] & MIGRATE_PFN_MIGRATE))1235 continue;1236 1237 if (WARN_ON(!is_device_private_page(spage) &&1238 !is_device_coherent_page(spage)))1239 continue;1240 spage = BACKING_PAGE(spage);1241 dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL);1242 lock_page(dpage);1243 copy_highpage(dpage, spage);1244 dst_pfns[i] = migrate_pfn(page_to_pfn(dpage));1245 if (src_pfns[i] & MIGRATE_PFN_WRITE)1246 dst_pfns[i] |= MIGRATE_PFN_WRITE;1247 }1248 migrate_device_pages(src_pfns, dst_pfns, npages);1249 migrate_device_finalize(src_pfns, dst_pfns, npages);1250 kvfree(src_pfns);1251 kvfree(dst_pfns);1252}1253 1254/* Removes free pages from the free list so they can't be re-allocated */1255static void dmirror_remove_free_pages(struct dmirror_chunk *devmem)1256{1257 struct dmirror_device *mdevice = devmem->mdevice;1258 struct page *page;1259 1260 for (page = mdevice->free_pages; page; page = page->zone_device_data)1261 if (dmirror_page_to_chunk(page) == devmem)1262 mdevice->free_pages = page->zone_device_data;1263}1264 1265static void dmirror_device_remove_chunks(struct dmirror_device *mdevice)1266{1267 unsigned int i;1268 1269 mutex_lock(&mdevice->devmem_lock);1270 if (mdevice->devmem_chunks) {1271 for (i = 0; i < mdevice->devmem_count; i++) {1272 struct dmirror_chunk *devmem =1273 mdevice->devmem_chunks[i];1274 1275 spin_lock(&mdevice->lock);1276 devmem->remove = true;1277 dmirror_remove_free_pages(devmem);1278 spin_unlock(&mdevice->lock);1279 1280 dmirror_device_evict_chunk(devmem);1281 memunmap_pages(&devmem->pagemap);1282 if (devmem->pagemap.type == MEMORY_DEVICE_PRIVATE)1283 release_mem_region(devmem->pagemap.range.start,1284 range_len(&devmem->pagemap.range));1285 kfree(devmem);1286 }1287 mdevice->devmem_count = 0;1288 mdevice->devmem_capacity = 0;1289 mdevice->free_pages = NULL;1290 kfree(mdevice->devmem_chunks);1291 mdevice->devmem_chunks = NULL;1292 }1293 mutex_unlock(&mdevice->devmem_lock);1294}1295 1296static long dmirror_fops_unlocked_ioctl(struct file *filp,1297 unsigned int command,1298 unsigned long arg)1299{1300 void __user *uarg = (void __user *)arg;1301 struct hmm_dmirror_cmd cmd;1302 struct dmirror *dmirror;1303 int ret;1304 1305 dmirror = filp->private_data;1306 if (!dmirror)1307 return -EINVAL;1308 1309 if (copy_from_user(&cmd, uarg, sizeof(cmd)))1310 return -EFAULT;1311 1312 if (cmd.addr & ~PAGE_MASK)1313 return -EINVAL;1314 if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT)))1315 return -EINVAL;1316 1317 cmd.cpages = 0;1318 cmd.faults = 0;1319 1320 switch (command) {1321 case HMM_DMIRROR_READ:1322 ret = dmirror_read(dmirror, &cmd);1323 break;1324 1325 case HMM_DMIRROR_WRITE:1326 ret = dmirror_write(dmirror, &cmd);1327 break;1328 1329 case HMM_DMIRROR_MIGRATE_TO_DEV:1330 ret = dmirror_migrate_to_device(dmirror, &cmd);1331 break;1332 1333 case HMM_DMIRROR_MIGRATE_TO_SYS:1334 ret = dmirror_migrate_to_system(dmirror, &cmd);1335 break;1336 1337 case HMM_DMIRROR_EXCLUSIVE:1338 ret = dmirror_exclusive(dmirror, &cmd);1339 break;1340 1341 case HMM_DMIRROR_CHECK_EXCLUSIVE:1342 ret = dmirror_check_atomic(dmirror, cmd.addr,1343 cmd.addr + (cmd.npages << PAGE_SHIFT));1344 break;1345 1346 case HMM_DMIRROR_SNAPSHOT:1347 ret = dmirror_snapshot(dmirror, &cmd);1348 break;1349 1350 case HMM_DMIRROR_RELEASE:1351 dmirror_device_remove_chunks(dmirror->mdevice);1352 ret = 0;1353 break;1354 1355 default:1356 return -EINVAL;1357 }1358 if (ret)1359 return ret;1360 1361 if (copy_to_user(uarg, &cmd, sizeof(cmd)))1362 return -EFAULT;1363 1364 return 0;1365}1366 1367static int dmirror_fops_mmap(struct file *file, struct vm_area_struct *vma)1368{1369 unsigned long addr;1370 1371 for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) {1372 struct page *page;1373 int ret;1374 1375 page = alloc_page(GFP_KERNEL | __GFP_ZERO);1376 if (!page)1377 return -ENOMEM;1378 1379 ret = vm_insert_page(vma, addr, page);1380 if (ret) {1381 __free_page(page);1382 return ret;1383 }1384 put_page(page);1385 }1386 1387 return 0;1388}1389 1390static const struct file_operations dmirror_fops = {1391 .open = dmirror_fops_open,1392 .release = dmirror_fops_release,1393 .mmap = dmirror_fops_mmap,1394 .unlocked_ioctl = dmirror_fops_unlocked_ioctl,1395 .llseek = default_llseek,1396 .owner = THIS_MODULE,1397};1398 1399static void dmirror_devmem_free(struct page *page)1400{1401 struct page *rpage = BACKING_PAGE(page);1402 struct dmirror_device *mdevice;1403 1404 if (rpage != page)1405 __free_page(rpage);1406 1407 mdevice = dmirror_page_to_device(page);1408 spin_lock(&mdevice->lock);1409 1410 /* Return page to our allocator if not freeing the chunk */1411 if (!dmirror_page_to_chunk(page)->remove) {1412 mdevice->cfree++;1413 page->zone_device_data = mdevice->free_pages;1414 mdevice->free_pages = page;1415 }1416 spin_unlock(&mdevice->lock);1417}1418 1419static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf)1420{1421 struct migrate_vma args = { 0 };1422 unsigned long src_pfns = 0;1423 unsigned long dst_pfns = 0;1424 struct page *rpage;1425 struct dmirror *dmirror;1426 vm_fault_t ret;1427 1428 /*1429 * Normally, a device would use the page->zone_device_data to point to1430 * the mirror but here we use it to hold the page for the simulated1431 * device memory and that page holds the pointer to the mirror.1432 */1433 rpage = vmf->page->zone_device_data;1434 dmirror = rpage->zone_device_data;1435 1436 /* FIXME demonstrate how we can adjust migrate range */1437 args.vma = vmf->vma;1438 args.start = vmf->address;1439 args.end = args.start + PAGE_SIZE;1440 args.src = &src_pfns;1441 args.dst = &dst_pfns;1442 args.pgmap_owner = dmirror->mdevice;1443 args.flags = dmirror_select_device(dmirror);1444 args.fault_page = vmf->page;1445 1446 if (migrate_vma_setup(&args))1447 return VM_FAULT_SIGBUS;1448 1449 ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror);1450 if (ret)1451 return ret;1452 migrate_vma_pages(&args);1453 /*1454 * No device finalize step is needed since1455 * dmirror_devmem_fault_alloc_and_copy() will have already1456 * invalidated the device page table.1457 */1458 migrate_vma_finalize(&args);1459 return 0;1460}1461 1462static const struct dev_pagemap_ops dmirror_devmem_ops = {1463 .page_free = dmirror_devmem_free,1464 .migrate_to_ram = dmirror_devmem_fault,1465};1466 1467static int dmirror_device_init(struct dmirror_device *mdevice, int id)1468{1469 dev_t dev;1470 int ret;1471 1472 dev = MKDEV(MAJOR(dmirror_dev), id);1473 mutex_init(&mdevice->devmem_lock);1474 spin_lock_init(&mdevice->lock);1475 1476 cdev_init(&mdevice->cdevice, &dmirror_fops);1477 mdevice->cdevice.owner = THIS_MODULE;1478 device_initialize(&mdevice->device);1479 mdevice->device.devt = dev;1480 1481 ret = dev_set_name(&mdevice->device, "hmm_dmirror%u", id);1482 if (ret)1483 return ret;1484 1485 ret = cdev_device_add(&mdevice->cdevice, &mdevice->device);1486 if (ret)1487 return ret;1488 1489 /* Build a list of free ZONE_DEVICE struct pages */1490 return dmirror_allocate_chunk(mdevice, NULL);1491}1492 1493static void dmirror_device_remove(struct dmirror_device *mdevice)1494{1495 dmirror_device_remove_chunks(mdevice);1496 cdev_device_del(&mdevice->cdevice, &mdevice->device);1497}1498 1499static int __init hmm_dmirror_init(void)1500{1501 int ret;1502 int id = 0;1503 int ndevices = 0;1504 1505 ret = alloc_chrdev_region(&dmirror_dev, 0, DMIRROR_NDEVICES,1506 "HMM_DMIRROR");1507 if (ret)1508 goto err_unreg;1509 1510 memset(dmirror_devices, 0, DMIRROR_NDEVICES * sizeof(dmirror_devices[0]));1511 dmirror_devices[ndevices++].zone_device_type =1512 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE;1513 dmirror_devices[ndevices++].zone_device_type =1514 HMM_DMIRROR_MEMORY_DEVICE_PRIVATE;1515 if (spm_addr_dev0 && spm_addr_dev1) {1516 dmirror_devices[ndevices++].zone_device_type =1517 HMM_DMIRROR_MEMORY_DEVICE_COHERENT;1518 dmirror_devices[ndevices++].zone_device_type =1519 HMM_DMIRROR_MEMORY_DEVICE_COHERENT;1520 }1521 for (id = 0; id < ndevices; id++) {1522 ret = dmirror_device_init(dmirror_devices + id, id);1523 if (ret)1524 goto err_chrdev;1525 }1526 1527 pr_info("HMM test module loaded. This is only for testing HMM.\n");1528 return 0;1529 1530err_chrdev:1531 while (--id >= 0)1532 dmirror_device_remove(dmirror_devices + id);1533 unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);1534err_unreg:1535 return ret;1536}1537 1538static void __exit hmm_dmirror_exit(void)1539{1540 int id;1541 1542 for (id = 0; id < DMIRROR_NDEVICES; id++)1543 if (dmirror_devices[id].zone_device_type)1544 dmirror_device_remove(dmirror_devices + id);1545 unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);1546}1547 1548module_init(hmm_dmirror_init);1549module_exit(hmm_dmirror_exit);1550MODULE_DESCRIPTION("HMM (Heterogeneous Memory Management) test module");1551MODULE_LICENSE("GPL");1552