703 lines · c
1// SPDX-License-Identifier: GPL-2.02#include <linux/backing-dev.h>3#include <linux/falloc.h>4#include <linux/kvm_host.h>5#include <linux/pagemap.h>6#include <linux/anon_inodes.h>7 8#include "kvm_mm.h"9 10struct kvm_gmem {11 struct kvm *kvm;12 struct xarray bindings;13 struct list_head entry;14};15 16/**17 * folio_file_pfn - like folio_file_page, but return a pfn.18 * @folio: The folio which contains this index.19 * @index: The index we want to look up.20 *21 * Return: The pfn for this index.22 */23static inline kvm_pfn_t folio_file_pfn(struct folio *folio, pgoff_t index)24{25 return folio_pfn(folio) + (index & (folio_nr_pages(folio) - 1));26}27 28static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot,29 pgoff_t index, struct folio *folio)30{31#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE32 kvm_pfn_t pfn = folio_file_pfn(folio, index);33 gfn_t gfn = slot->base_gfn + index - slot->gmem.pgoff;34 int rc = kvm_arch_gmem_prepare(kvm, gfn, pfn, folio_order(folio));35 if (rc) {36 pr_warn_ratelimited("gmem: Failed to prepare folio for index %lx GFN %llx PFN %llx error %d.\n",37 index, gfn, pfn, rc);38 return rc;39 }40#endif41 42 return 0;43}44 45static inline void kvm_gmem_mark_prepared(struct folio *folio)46{47 folio_mark_uptodate(folio);48}49 50/*51 * Process @folio, which contains @gfn, so that the guest can use it.52 * The folio must be locked and the gfn must be contained in @slot.53 * On successful return the guest sees a zero page so as to avoid54 * leaking host data and the up-to-date flag is set.55 */56static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot,57 gfn_t gfn, struct folio *folio)58{59 unsigned long nr_pages, i;60 pgoff_t index;61 int r;62 63 nr_pages = folio_nr_pages(folio);64 for (i = 0; i < nr_pages; i++)65 clear_highpage(folio_page(folio, i));66 67 /*68 * Preparing huge folios should always be safe, since it should69 * be possible to split them later if needed.70 *71 * Right now the folio order is always going to be zero, but the72 * code is ready for huge folios. The only assumption is that73 * the base pgoff of memslots is naturally aligned with the74 * requested page order, ensuring that huge folios can also use75 * huge page table entries for GPA->HPA mapping.76 *77 * The order will be passed when creating the guest_memfd, and78 * checked when creating memslots.79 */80 WARN_ON(!IS_ALIGNED(slot->gmem.pgoff, 1 << folio_order(folio)));81 index = gfn - slot->base_gfn + slot->gmem.pgoff;82 index = ALIGN_DOWN(index, 1 << folio_order(folio));83 r = __kvm_gmem_prepare_folio(kvm, slot, index, folio);84 if (!r)85 kvm_gmem_mark_prepared(folio);86 87 return r;88}89 90/*91 * Returns a locked folio on success. The caller is responsible for92 * setting the up-to-date flag before the memory is mapped into the guest.93 * There is no backing storage for the memory, so the folio will remain94 * up-to-date until it's removed.95 *96 * Ignore accessed, referenced, and dirty flags. The memory is97 * unevictable and there is no storage to write back to.98 */99static struct folio *kvm_gmem_get_folio(struct inode *inode, pgoff_t index)100{101 /* TODO: Support huge pages. */102 return filemap_grab_folio(inode->i_mapping, index);103}104 105static void kvm_gmem_invalidate_begin(struct kvm_gmem *gmem, pgoff_t start,106 pgoff_t end)107{108 bool flush = false, found_memslot = false;109 struct kvm_memory_slot *slot;110 struct kvm *kvm = gmem->kvm;111 unsigned long index;112 113 xa_for_each_range(&gmem->bindings, index, slot, start, end - 1) {114 pgoff_t pgoff = slot->gmem.pgoff;115 116 struct kvm_gfn_range gfn_range = {117 .start = slot->base_gfn + max(pgoff, start) - pgoff,118 .end = slot->base_gfn + min(pgoff + slot->npages, end) - pgoff,119 .slot = slot,120 .may_block = true,121 };122 123 if (!found_memslot) {124 found_memslot = true;125 126 KVM_MMU_LOCK(kvm);127 kvm_mmu_invalidate_begin(kvm);128 }129 130 flush |= kvm_mmu_unmap_gfn_range(kvm, &gfn_range);131 }132 133 if (flush)134 kvm_flush_remote_tlbs(kvm);135 136 if (found_memslot)137 KVM_MMU_UNLOCK(kvm);138}139 140static void kvm_gmem_invalidate_end(struct kvm_gmem *gmem, pgoff_t start,141 pgoff_t end)142{143 struct kvm *kvm = gmem->kvm;144 145 if (xa_find(&gmem->bindings, &start, end - 1, XA_PRESENT)) {146 KVM_MMU_LOCK(kvm);147 kvm_mmu_invalidate_end(kvm);148 KVM_MMU_UNLOCK(kvm);149 }150}151 152static long kvm_gmem_punch_hole(struct inode *inode, loff_t offset, loff_t len)153{154 struct list_head *gmem_list = &inode->i_mapping->i_private_list;155 pgoff_t start = offset >> PAGE_SHIFT;156 pgoff_t end = (offset + len) >> PAGE_SHIFT;157 struct kvm_gmem *gmem;158 159 /*160 * Bindings must be stable across invalidation to ensure the start+end161 * are balanced.162 */163 filemap_invalidate_lock(inode->i_mapping);164 165 list_for_each_entry(gmem, gmem_list, entry)166 kvm_gmem_invalidate_begin(gmem, start, end);167 168 truncate_inode_pages_range(inode->i_mapping, offset, offset + len - 1);169 170 list_for_each_entry(gmem, gmem_list, entry)171 kvm_gmem_invalidate_end(gmem, start, end);172 173 filemap_invalidate_unlock(inode->i_mapping);174 175 return 0;176}177 178static long kvm_gmem_allocate(struct inode *inode, loff_t offset, loff_t len)179{180 struct address_space *mapping = inode->i_mapping;181 pgoff_t start, index, end;182 int r;183 184 /* Dedicated guest is immutable by default. */185 if (offset + len > i_size_read(inode))186 return -EINVAL;187 188 filemap_invalidate_lock_shared(mapping);189 190 start = offset >> PAGE_SHIFT;191 end = (offset + len) >> PAGE_SHIFT;192 193 r = 0;194 for (index = start; index < end; ) {195 struct folio *folio;196 197 if (signal_pending(current)) {198 r = -EINTR;199 break;200 }201 202 folio = kvm_gmem_get_folio(inode, index);203 if (IS_ERR(folio)) {204 r = PTR_ERR(folio);205 break;206 }207 208 index = folio_next_index(folio);209 210 folio_unlock(folio);211 folio_put(folio);212 213 /* 64-bit only, wrapping the index should be impossible. */214 if (WARN_ON_ONCE(!index))215 break;216 217 cond_resched();218 }219 220 filemap_invalidate_unlock_shared(mapping);221 222 return r;223}224 225static long kvm_gmem_fallocate(struct file *file, int mode, loff_t offset,226 loff_t len)227{228 int ret;229 230 if (!(mode & FALLOC_FL_KEEP_SIZE))231 return -EOPNOTSUPP;232 233 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))234 return -EOPNOTSUPP;235 236 if (!PAGE_ALIGNED(offset) || !PAGE_ALIGNED(len))237 return -EINVAL;238 239 if (mode & FALLOC_FL_PUNCH_HOLE)240 ret = kvm_gmem_punch_hole(file_inode(file), offset, len);241 else242 ret = kvm_gmem_allocate(file_inode(file), offset, len);243 244 if (!ret)245 file_modified(file);246 return ret;247}248 249static int kvm_gmem_release(struct inode *inode, struct file *file)250{251 struct kvm_gmem *gmem = file->private_data;252 struct kvm_memory_slot *slot;253 struct kvm *kvm = gmem->kvm;254 unsigned long index;255 256 /*257 * Prevent concurrent attempts to *unbind* a memslot. This is the last258 * reference to the file and thus no new bindings can be created, but259 * dereferencing the slot for existing bindings needs to be protected260 * against memslot updates, specifically so that unbind doesn't race261 * and free the memslot (kvm_gmem_get_file() will return NULL).262 */263 mutex_lock(&kvm->slots_lock);264 265 filemap_invalidate_lock(inode->i_mapping);266 267 xa_for_each(&gmem->bindings, index, slot)268 rcu_assign_pointer(slot->gmem.file, NULL);269 270 synchronize_rcu();271 272 /*273 * All in-flight operations are gone and new bindings can be created.274 * Zap all SPTEs pointed at by this file. Do not free the backing275 * memory, as its lifetime is associated with the inode, not the file.276 */277 kvm_gmem_invalidate_begin(gmem, 0, -1ul);278 kvm_gmem_invalidate_end(gmem, 0, -1ul);279 280 list_del(&gmem->entry);281 282 filemap_invalidate_unlock(inode->i_mapping);283 284 mutex_unlock(&kvm->slots_lock);285 286 xa_destroy(&gmem->bindings);287 kfree(gmem);288 289 kvm_put_kvm(kvm);290 291 return 0;292}293 294static inline struct file *kvm_gmem_get_file(struct kvm_memory_slot *slot)295{296 /*297 * Do not return slot->gmem.file if it has already been closed;298 * there might be some time between the last fput() and when299 * kvm_gmem_release() clears slot->gmem.file, and you do not300 * want to spin in the meanwhile.301 */302 return get_file_active(&slot->gmem.file);303}304 305static struct file_operations kvm_gmem_fops = {306 .open = generic_file_open,307 .release = kvm_gmem_release,308 .fallocate = kvm_gmem_fallocate,309};310 311void kvm_gmem_init(struct module *module)312{313 kvm_gmem_fops.owner = module;314}315 316static int kvm_gmem_migrate_folio(struct address_space *mapping,317 struct folio *dst, struct folio *src,318 enum migrate_mode mode)319{320 WARN_ON_ONCE(1);321 return -EINVAL;322}323 324static int kvm_gmem_error_folio(struct address_space *mapping, struct folio *folio)325{326 struct list_head *gmem_list = &mapping->i_private_list;327 struct kvm_gmem *gmem;328 pgoff_t start, end;329 330 filemap_invalidate_lock_shared(mapping);331 332 start = folio->index;333 end = start + folio_nr_pages(folio);334 335 list_for_each_entry(gmem, gmem_list, entry)336 kvm_gmem_invalidate_begin(gmem, start, end);337 338 /*339 * Do not truncate the range, what action is taken in response to the340 * error is userspace's decision (assuming the architecture supports341 * gracefully handling memory errors). If/when the guest attempts to342 * access a poisoned page, kvm_gmem_get_pfn() will return -EHWPOISON,343 * at which point KVM can either terminate the VM or propagate the344 * error to userspace.345 */346 347 list_for_each_entry(gmem, gmem_list, entry)348 kvm_gmem_invalidate_end(gmem, start, end);349 350 filemap_invalidate_unlock_shared(mapping);351 352 return MF_DELAYED;353}354 355#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE356static void kvm_gmem_free_folio(struct folio *folio)357{358 struct page *page = folio_page(folio, 0);359 kvm_pfn_t pfn = page_to_pfn(page);360 int order = folio_order(folio);361 362 kvm_arch_gmem_invalidate(pfn, pfn + (1ul << order));363}364#endif365 366static const struct address_space_operations kvm_gmem_aops = {367 .dirty_folio = noop_dirty_folio,368 .migrate_folio = kvm_gmem_migrate_folio,369 .error_remove_folio = kvm_gmem_error_folio,370#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE371 .free_folio = kvm_gmem_free_folio,372#endif373};374 375static int kvm_gmem_getattr(struct mnt_idmap *idmap, const struct path *path,376 struct kstat *stat, u32 request_mask,377 unsigned int query_flags)378{379 struct inode *inode = path->dentry->d_inode;380 381 generic_fillattr(idmap, request_mask, inode, stat);382 return 0;383}384 385static int kvm_gmem_setattr(struct mnt_idmap *idmap, struct dentry *dentry,386 struct iattr *attr)387{388 return -EINVAL;389}390static const struct inode_operations kvm_gmem_iops = {391 .getattr = kvm_gmem_getattr,392 .setattr = kvm_gmem_setattr,393};394 395static int __kvm_gmem_create(struct kvm *kvm, loff_t size, u64 flags)396{397 const char *anon_name = "[kvm-gmem]";398 struct kvm_gmem *gmem;399 struct inode *inode;400 struct file *file;401 int fd, err;402 403 fd = get_unused_fd_flags(0);404 if (fd < 0)405 return fd;406 407 gmem = kzalloc(sizeof(*gmem), GFP_KERNEL);408 if (!gmem) {409 err = -ENOMEM;410 goto err_fd;411 }412 413 file = anon_inode_create_getfile(anon_name, &kvm_gmem_fops, gmem,414 O_RDWR, NULL);415 if (IS_ERR(file)) {416 err = PTR_ERR(file);417 goto err_gmem;418 }419 420 file->f_flags |= O_LARGEFILE;421 422 inode = file->f_inode;423 WARN_ON(file->f_mapping != inode->i_mapping);424 425 inode->i_private = (void *)(unsigned long)flags;426 inode->i_op = &kvm_gmem_iops;427 inode->i_mapping->a_ops = &kvm_gmem_aops;428 inode->i_mode |= S_IFREG;429 inode->i_size = size;430 mapping_set_gfp_mask(inode->i_mapping, GFP_HIGHUSER);431 mapping_set_inaccessible(inode->i_mapping);432 /* Unmovable mappings are supposed to be marked unevictable as well. */433 WARN_ON_ONCE(!mapping_unevictable(inode->i_mapping));434 435 kvm_get_kvm(kvm);436 gmem->kvm = kvm;437 xa_init(&gmem->bindings);438 list_add(&gmem->entry, &inode->i_mapping->i_private_list);439 440 fd_install(fd, file);441 return fd;442 443err_gmem:444 kfree(gmem);445err_fd:446 put_unused_fd(fd);447 return err;448}449 450int kvm_gmem_create(struct kvm *kvm, struct kvm_create_guest_memfd *args)451{452 loff_t size = args->size;453 u64 flags = args->flags;454 u64 valid_flags = 0;455 456 if (flags & ~valid_flags)457 return -EINVAL;458 459 if (size <= 0 || !PAGE_ALIGNED(size))460 return -EINVAL;461 462 return __kvm_gmem_create(kvm, size, flags);463}464 465int kvm_gmem_bind(struct kvm *kvm, struct kvm_memory_slot *slot,466 unsigned int fd, loff_t offset)467{468 loff_t size = slot->npages << PAGE_SHIFT;469 unsigned long start, end;470 struct kvm_gmem *gmem;471 struct inode *inode;472 struct file *file;473 int r = -EINVAL;474 475 BUILD_BUG_ON(sizeof(gfn_t) != sizeof(slot->gmem.pgoff));476 477 file = fget(fd);478 if (!file)479 return -EBADF;480 481 if (file->f_op != &kvm_gmem_fops)482 goto err;483 484 gmem = file->private_data;485 if (gmem->kvm != kvm)486 goto err;487 488 inode = file_inode(file);489 490 if (offset < 0 || !PAGE_ALIGNED(offset) ||491 offset + size > i_size_read(inode))492 goto err;493 494 filemap_invalidate_lock(inode->i_mapping);495 496 start = offset >> PAGE_SHIFT;497 end = start + slot->npages;498 499 if (!xa_empty(&gmem->bindings) &&500 xa_find(&gmem->bindings, &start, end - 1, XA_PRESENT)) {501 filemap_invalidate_unlock(inode->i_mapping);502 goto err;503 }504 505 /*506 * No synchronize_rcu() needed, any in-flight readers are guaranteed to507 * be see either a NULL file or this new file, no need for them to go508 * away.509 */510 rcu_assign_pointer(slot->gmem.file, file);511 slot->gmem.pgoff = start;512 513 xa_store_range(&gmem->bindings, start, end - 1, slot, GFP_KERNEL);514 filemap_invalidate_unlock(inode->i_mapping);515 516 /*517 * Drop the reference to the file, even on success. The file pins KVM,518 * not the other way 'round. Active bindings are invalidated if the519 * file is closed before memslots are destroyed.520 */521 r = 0;522err:523 fput(file);524 return r;525}526 527void kvm_gmem_unbind(struct kvm_memory_slot *slot)528{529 unsigned long start = slot->gmem.pgoff;530 unsigned long end = start + slot->npages;531 struct kvm_gmem *gmem;532 struct file *file;533 534 /*535 * Nothing to do if the underlying file was already closed (or is being536 * closed right now), kvm_gmem_release() invalidates all bindings.537 */538 file = kvm_gmem_get_file(slot);539 if (!file)540 return;541 542 gmem = file->private_data;543 544 filemap_invalidate_lock(file->f_mapping);545 xa_store_range(&gmem->bindings, start, end - 1, NULL, GFP_KERNEL);546 rcu_assign_pointer(slot->gmem.file, NULL);547 synchronize_rcu();548 filemap_invalidate_unlock(file->f_mapping);549 550 fput(file);551}552 553/* Returns a locked folio on success. */554static struct folio *555__kvm_gmem_get_pfn(struct file *file, struct kvm_memory_slot *slot,556 gfn_t gfn, kvm_pfn_t *pfn, bool *is_prepared,557 int *max_order)558{559 pgoff_t index = gfn - slot->base_gfn + slot->gmem.pgoff;560 struct kvm_gmem *gmem = file->private_data;561 struct folio *folio;562 563 if (file != slot->gmem.file) {564 WARN_ON_ONCE(slot->gmem.file);565 return ERR_PTR(-EFAULT);566 }567 568 gmem = file->private_data;569 if (xa_load(&gmem->bindings, index) != slot) {570 WARN_ON_ONCE(xa_load(&gmem->bindings, index));571 return ERR_PTR(-EIO);572 }573 574 folio = kvm_gmem_get_folio(file_inode(file), index);575 if (IS_ERR(folio))576 return folio;577 578 if (folio_test_hwpoison(folio)) {579 folio_unlock(folio);580 folio_put(folio);581 return ERR_PTR(-EHWPOISON);582 }583 584 *pfn = folio_file_pfn(folio, index);585 if (max_order)586 *max_order = 0;587 588 *is_prepared = folio_test_uptodate(folio);589 return folio;590}591 592int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot,593 gfn_t gfn, kvm_pfn_t *pfn, int *max_order)594{595 struct file *file = kvm_gmem_get_file(slot);596 struct folio *folio;597 bool is_prepared = false;598 int r = 0;599 600 if (!file)601 return -EFAULT;602 603 folio = __kvm_gmem_get_pfn(file, slot, gfn, pfn, &is_prepared, max_order);604 if (IS_ERR(folio)) {605 r = PTR_ERR(folio);606 goto out;607 }608 609 if (!is_prepared)610 r = kvm_gmem_prepare_folio(kvm, slot, gfn, folio);611 612 folio_unlock(folio);613 if (r < 0)614 folio_put(folio);615 616out:617 fput(file);618 return r;619}620EXPORT_SYMBOL_GPL(kvm_gmem_get_pfn);621 622#ifdef CONFIG_KVM_GENERIC_PRIVATE_MEM623long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, long npages,624 kvm_gmem_populate_cb post_populate, void *opaque)625{626 struct file *file;627 struct kvm_memory_slot *slot;628 void __user *p;629 630 int ret = 0, max_order;631 long i;632 633 lockdep_assert_held(&kvm->slots_lock);634 if (npages < 0)635 return -EINVAL;636 637 slot = gfn_to_memslot(kvm, start_gfn);638 if (!kvm_slot_can_be_private(slot))639 return -EINVAL;640 641 file = kvm_gmem_get_file(slot);642 if (!file)643 return -EFAULT;644 645 filemap_invalidate_lock(file->f_mapping);646 647 npages = min_t(ulong, slot->npages - (start_gfn - slot->base_gfn), npages);648 for (i = 0; i < npages; i += (1 << max_order)) {649 struct folio *folio;650 gfn_t gfn = start_gfn + i;651 bool is_prepared = false;652 kvm_pfn_t pfn;653 654 if (signal_pending(current)) {655 ret = -EINTR;656 break;657 }658 659 folio = __kvm_gmem_get_pfn(file, slot, gfn, &pfn, &is_prepared, &max_order);660 if (IS_ERR(folio)) {661 ret = PTR_ERR(folio);662 break;663 }664 665 if (is_prepared) {666 folio_unlock(folio);667 folio_put(folio);668 ret = -EEXIST;669 break;670 }671 672 folio_unlock(folio);673 WARN_ON(!IS_ALIGNED(gfn, 1 << max_order) ||674 (npages - i) < (1 << max_order));675 676 ret = -EINVAL;677 while (!kvm_range_has_memory_attributes(kvm, gfn, gfn + (1 << max_order),678 KVM_MEMORY_ATTRIBUTE_PRIVATE,679 KVM_MEMORY_ATTRIBUTE_PRIVATE)) {680 if (!max_order)681 goto put_folio_and_exit;682 max_order--;683 }684 685 p = src ? src + i * PAGE_SIZE : NULL;686 ret = post_populate(kvm, gfn, pfn, p, max_order, opaque);687 if (!ret)688 kvm_gmem_mark_prepared(folio);689 690put_folio_and_exit:691 folio_put(folio);692 if (ret)693 break;694 }695 696 filemap_invalidate_unlock(file->f_mapping);697 698 fput(file);699 return ret && !i ? ret : i;700}701EXPORT_SYMBOL_GPL(kvm_gmem_populate);702#endif703