1924 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * VMware Balloon driver.4 *5 * Copyright (C) 2000-2018, VMware, Inc. All Rights Reserved.6 *7 * This is VMware physical memory management driver for Linux. The driver8 * acts like a "balloon" that can be inflated to reclaim physical pages by9 * reserving them in the guest and invalidating them in the monitor,10 * freeing up the underlying machine pages so they can be allocated to11 * other guests. The balloon can also be deflated to allow the guest to12 * use more physical memory. Higher level policies can control the sizes13 * of balloons in VMs in order to manage physical memory resources.14 */15 16//#define DEBUG17#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt18 19#include <linux/types.h>20#include <linux/io.h>21#include <linux/kernel.h>22#include <linux/mm.h>23#include <linux/vmalloc.h>24#include <linux/sched.h>25#include <linux/module.h>26#include <linux/workqueue.h>27#include <linux/debugfs.h>28#include <linux/seq_file.h>29#include <linux/rwsem.h>30#include <linux/slab.h>31#include <linux/spinlock.h>32#include <linux/balloon_compaction.h>33#include <linux/vmw_vmci_defs.h>34#include <linux/vmw_vmci_api.h>35#include <asm/hypervisor.h>36 37MODULE_AUTHOR("VMware, Inc.");38MODULE_DESCRIPTION("VMware Memory Control (Balloon) Driver");39MODULE_ALIAS("dmi:*:svnVMware*:*");40MODULE_ALIAS("vmware_vmmemctl");41MODULE_LICENSE("GPL");42 43static bool __read_mostly vmwballoon_shrinker_enable;44module_param(vmwballoon_shrinker_enable, bool, 0444);45MODULE_PARM_DESC(vmwballoon_shrinker_enable,46 "Enable non-cooperative out-of-memory protection. Disabled by default as it may degrade performance.");47 48/* Delay in seconds after shrink before inflation. */49#define VMBALLOON_SHRINK_DELAY (5)50 51/* Maximum number of refused pages we accumulate during inflation cycle */52#define VMW_BALLOON_MAX_REFUSED 1653 54/* Magic number for the balloon mount-point */55#define BALLOON_VMW_MAGIC 0x0ba1100756 57/*58 * Hypervisor communication port definitions.59 */60#define VMW_BALLOON_HV_PORT 0x567061#define VMW_BALLOON_HV_MAGIC 0x456c6d6f62#define VMW_BALLOON_GUEST_ID 1 /* Linux */63 64enum vmwballoon_capabilities {65 /*66 * Bit 0 is reserved and not associated to any capability.67 */68 VMW_BALLOON_BASIC_CMDS = (1 << 1),69 VMW_BALLOON_BATCHED_CMDS = (1 << 2),70 VMW_BALLOON_BATCHED_2M_CMDS = (1 << 3),71 VMW_BALLOON_SIGNALLED_WAKEUP_CMD = (1 << 4),72 VMW_BALLOON_64_BIT_TARGET = (1 << 5)73};74 75#define VMW_BALLOON_CAPABILITIES_COMMON (VMW_BALLOON_BASIC_CMDS \76 | VMW_BALLOON_BATCHED_CMDS \77 | VMW_BALLOON_BATCHED_2M_CMDS \78 | VMW_BALLOON_SIGNALLED_WAKEUP_CMD)79 80#define VMW_BALLOON_2M_ORDER (PMD_SHIFT - PAGE_SHIFT)81 82/*83 * 64-bit targets are only supported in 64-bit84 */85#ifdef CONFIG_64BIT86#define VMW_BALLOON_CAPABILITIES (VMW_BALLOON_CAPABILITIES_COMMON \87 | VMW_BALLOON_64_BIT_TARGET)88#else89#define VMW_BALLOON_CAPABILITIES VMW_BALLOON_CAPABILITIES_COMMON90#endif91 92enum vmballoon_page_size_type {93 VMW_BALLOON_4K_PAGE,94 VMW_BALLOON_2M_PAGE,95 VMW_BALLOON_LAST_SIZE = VMW_BALLOON_2M_PAGE96};97 98#define VMW_BALLOON_NUM_PAGE_SIZES (VMW_BALLOON_LAST_SIZE + 1)99 100static const char * const vmballoon_page_size_names[] = {101 [VMW_BALLOON_4K_PAGE] = "4k",102 [VMW_BALLOON_2M_PAGE] = "2M"103};104 105enum vmballoon_op {106 VMW_BALLOON_INFLATE,107 VMW_BALLOON_DEFLATE108};109 110enum vmballoon_op_stat_type {111 VMW_BALLOON_OP_STAT,112 VMW_BALLOON_OP_FAIL_STAT113};114 115#define VMW_BALLOON_OP_STAT_TYPES (VMW_BALLOON_OP_FAIL_STAT + 1)116 117/**118 * enum vmballoon_cmd_type - backdoor commands.119 *120 * Availability of the commands is as followed:121 *122 * %VMW_BALLOON_CMD_START, %VMW_BALLOON_CMD_GET_TARGET and123 * %VMW_BALLOON_CMD_GUEST_ID are always available.124 *125 * If the host reports %VMW_BALLOON_BASIC_CMDS are supported then126 * %VMW_BALLOON_CMD_LOCK and %VMW_BALLOON_CMD_UNLOCK commands are available.127 *128 * If the host reports %VMW_BALLOON_BATCHED_CMDS are supported then129 * %VMW_BALLOON_CMD_BATCHED_LOCK and VMW_BALLOON_CMD_BATCHED_UNLOCK commands130 * are available.131 *132 * If the host reports %VMW_BALLOON_BATCHED_2M_CMDS are supported then133 * %VMW_BALLOON_CMD_BATCHED_2M_LOCK and %VMW_BALLOON_CMD_BATCHED_2M_UNLOCK134 * are supported.135 *136 * If the host reports VMW_BALLOON_SIGNALLED_WAKEUP_CMD is supported then137 * VMW_BALLOON_CMD_VMCI_DOORBELL_SET command is supported.138 *139 * @VMW_BALLOON_CMD_START: Communicating supported version with the hypervisor.140 * @VMW_BALLOON_CMD_GET_TARGET: Gets the balloon target size.141 * @VMW_BALLOON_CMD_LOCK: Informs the hypervisor about a ballooned page.142 * @VMW_BALLOON_CMD_UNLOCK: Informs the hypervisor about a page that is about143 * to be deflated from the balloon.144 * @VMW_BALLOON_CMD_GUEST_ID: Informs the hypervisor about the type of OS that145 * runs in the VM.146 * @VMW_BALLOON_CMD_BATCHED_LOCK: Inform the hypervisor about a batch of147 * ballooned pages (up to 512).148 * @VMW_BALLOON_CMD_BATCHED_UNLOCK: Inform the hypervisor about a batch of149 * pages that are about to be deflated from the150 * balloon (up to 512).151 * @VMW_BALLOON_CMD_BATCHED_2M_LOCK: Similar to @VMW_BALLOON_CMD_BATCHED_LOCK152 * for 2MB pages.153 * @VMW_BALLOON_CMD_BATCHED_2M_UNLOCK: Similar to154 * @VMW_BALLOON_CMD_BATCHED_UNLOCK for 2MB155 * pages.156 * @VMW_BALLOON_CMD_VMCI_DOORBELL_SET: A command to set doorbell notification157 * that would be invoked when the balloon158 * size changes.159 * @VMW_BALLOON_CMD_LAST: Value of the last command.160 */161enum vmballoon_cmd_type {162 VMW_BALLOON_CMD_START,163 VMW_BALLOON_CMD_GET_TARGET,164 VMW_BALLOON_CMD_LOCK,165 VMW_BALLOON_CMD_UNLOCK,166 VMW_BALLOON_CMD_GUEST_ID,167 /* No command 5 */168 VMW_BALLOON_CMD_BATCHED_LOCK = 6,169 VMW_BALLOON_CMD_BATCHED_UNLOCK,170 VMW_BALLOON_CMD_BATCHED_2M_LOCK,171 VMW_BALLOON_CMD_BATCHED_2M_UNLOCK,172 VMW_BALLOON_CMD_VMCI_DOORBELL_SET,173 VMW_BALLOON_CMD_LAST = VMW_BALLOON_CMD_VMCI_DOORBELL_SET,174};175 176#define VMW_BALLOON_CMD_NUM (VMW_BALLOON_CMD_LAST + 1)177 178enum vmballoon_error_codes {179 VMW_BALLOON_SUCCESS,180 VMW_BALLOON_ERROR_CMD_INVALID,181 VMW_BALLOON_ERROR_PPN_INVALID,182 VMW_BALLOON_ERROR_PPN_LOCKED,183 VMW_BALLOON_ERROR_PPN_UNLOCKED,184 VMW_BALLOON_ERROR_PPN_PINNED,185 VMW_BALLOON_ERROR_PPN_NOTNEEDED,186 VMW_BALLOON_ERROR_RESET,187 VMW_BALLOON_ERROR_BUSY188};189 190#define VMW_BALLOON_SUCCESS_WITH_CAPABILITIES (0x03000000)191 192#define VMW_BALLOON_CMD_WITH_TARGET_MASK \193 ((1UL << VMW_BALLOON_CMD_GET_TARGET) | \194 (1UL << VMW_BALLOON_CMD_LOCK) | \195 (1UL << VMW_BALLOON_CMD_UNLOCK) | \196 (1UL << VMW_BALLOON_CMD_BATCHED_LOCK) | \197 (1UL << VMW_BALLOON_CMD_BATCHED_UNLOCK) | \198 (1UL << VMW_BALLOON_CMD_BATCHED_2M_LOCK) | \199 (1UL << VMW_BALLOON_CMD_BATCHED_2M_UNLOCK))200 201static const char * const vmballoon_cmd_names[] = {202 [VMW_BALLOON_CMD_START] = "start",203 [VMW_BALLOON_CMD_GET_TARGET] = "target",204 [VMW_BALLOON_CMD_LOCK] = "lock",205 [VMW_BALLOON_CMD_UNLOCK] = "unlock",206 [VMW_BALLOON_CMD_GUEST_ID] = "guestType",207 [VMW_BALLOON_CMD_BATCHED_LOCK] = "batchLock",208 [VMW_BALLOON_CMD_BATCHED_UNLOCK] = "batchUnlock",209 [VMW_BALLOON_CMD_BATCHED_2M_LOCK] = "2m-lock",210 [VMW_BALLOON_CMD_BATCHED_2M_UNLOCK] = "2m-unlock",211 [VMW_BALLOON_CMD_VMCI_DOORBELL_SET] = "doorbellSet"212};213 214enum vmballoon_stat_page {215 VMW_BALLOON_PAGE_STAT_ALLOC,216 VMW_BALLOON_PAGE_STAT_ALLOC_FAIL,217 VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,218 VMW_BALLOON_PAGE_STAT_REFUSED_FREE,219 VMW_BALLOON_PAGE_STAT_FREE,220 VMW_BALLOON_PAGE_STAT_LAST = VMW_BALLOON_PAGE_STAT_FREE221};222 223#define VMW_BALLOON_PAGE_STAT_NUM (VMW_BALLOON_PAGE_STAT_LAST + 1)224 225enum vmballoon_stat_general {226 VMW_BALLOON_STAT_TIMER,227 VMW_BALLOON_STAT_DOORBELL,228 VMW_BALLOON_STAT_RESET,229 VMW_BALLOON_STAT_SHRINK,230 VMW_BALLOON_STAT_SHRINK_FREE,231 VMW_BALLOON_STAT_LAST = VMW_BALLOON_STAT_SHRINK_FREE232};233 234#define VMW_BALLOON_STAT_NUM (VMW_BALLOON_STAT_LAST + 1)235 236static DEFINE_STATIC_KEY_TRUE(vmw_balloon_batching);237static DEFINE_STATIC_KEY_FALSE(balloon_stat_enabled);238 239struct vmballoon_ctl {240 struct list_head pages;241 struct list_head refused_pages;242 struct list_head prealloc_pages;243 unsigned int n_refused_pages;244 unsigned int n_pages;245 enum vmballoon_page_size_type page_size;246 enum vmballoon_op op;247};248 249/**250 * struct vmballoon_batch_entry - a batch entry for lock or unlock.251 *252 * @status: the status of the operation, which is written by the hypervisor.253 * @reserved: reserved for future use. Must be set to zero.254 * @pfn: the physical frame number of the page to be locked or unlocked.255 */256struct vmballoon_batch_entry {257 u64 status : 5;258 u64 reserved : PAGE_SHIFT - 5;259 u64 pfn : 52;260} __packed;261 262struct vmballoon {263 /**264 * @max_page_size: maximum supported page size for ballooning.265 *266 * Protected by @conf_sem267 */268 enum vmballoon_page_size_type max_page_size;269 270 /**271 * @size: balloon actual size in basic page size (frames).272 *273 * While we currently do not support size which is bigger than 32-bit,274 * in preparation for future support, use 64-bits.275 */276 atomic64_t size;277 278 /**279 * @target: balloon target size in basic page size (frames).280 *281 * We do not protect the target under the assumption that setting the282 * value is always done through a single write. If this assumption ever283 * breaks, we would have to use X_ONCE for accesses, and suffer the less284 * optimized code. Although we may read stale target value if multiple285 * accesses happen at once, the performance impact should be minor.286 */287 unsigned long target;288 289 /**290 * @reset_required: reset flag291 *292 * Setting this flag may introduce races, but the code is expected to293 * handle them gracefully. In the worst case, another operation will294 * fail as reset did not take place. Clearing the flag is done while295 * holding @conf_sem for write.296 */297 bool reset_required;298 299 /**300 * @capabilities: hypervisor balloon capabilities.301 *302 * Protected by @conf_sem.303 */304 unsigned long capabilities;305 306 /**307 * @batch_page: pointer to communication batch page.308 *309 * When batching is used, batch_page points to a page, which holds up to310 * %VMW_BALLOON_BATCH_MAX_PAGES entries for locking or unlocking.311 */312 struct vmballoon_batch_entry *batch_page;313 314 /**315 * @batch_max_pages: maximum pages that can be locked/unlocked.316 *317 * Indicates the number of pages that the hypervisor can lock or unlock318 * at once, according to whether batching is enabled. If batching is319 * disabled, only a single page can be locked/unlock on each operation.320 *321 * Protected by @conf_sem.322 */323 unsigned int batch_max_pages;324 325 /**326 * @page: page to be locked/unlocked by the hypervisor327 *328 * @page is only used when batching is disabled and a single page is329 * reclaimed on each iteration.330 *331 * Protected by @comm_lock.332 */333 struct page *page;334 335 /**336 * @shrink_timeout: timeout until the next inflation.337 *338 * After an shrink event, indicates the time in jiffies after which339 * inflation is allowed again. Can be written concurrently with reads,340 * so must use READ_ONCE/WRITE_ONCE when accessing.341 */342 unsigned long shrink_timeout;343 344 /* statistics */345 struct vmballoon_stats *stats;346 347 /**348 * @b_dev_info: balloon device information descriptor.349 */350 struct balloon_dev_info b_dev_info;351 352 struct delayed_work dwork;353 354 /**355 * @huge_pages - list of the inflated 2MB pages.356 *357 * Protected by @b_dev_info.pages_lock .358 */359 struct list_head huge_pages;360 361 /**362 * @vmci_doorbell.363 *364 * Protected by @conf_sem.365 */366 struct vmci_handle vmci_doorbell;367 368 /**369 * @conf_sem: semaphore to protect the configuration and the statistics.370 */371 struct rw_semaphore conf_sem;372 373 /**374 * @comm_lock: lock to protect the communication with the host.375 *376 * Lock ordering: @conf_sem -> @comm_lock .377 */378 spinlock_t comm_lock;379 380 /**381 * @shrinker: shrinker interface that is used to avoid over-inflation.382 */383 struct shrinker *shrinker;384};385 386static struct vmballoon balloon;387 388struct vmballoon_stats {389 /* timer / doorbell operations */390 atomic64_t general_stat[VMW_BALLOON_STAT_NUM];391 392 /* allocation statistics for huge and small pages */393 atomic64_t394 page_stat[VMW_BALLOON_PAGE_STAT_NUM][VMW_BALLOON_NUM_PAGE_SIZES];395 396 /* Monitor operations: total operations, and failures */397 atomic64_t ops[VMW_BALLOON_CMD_NUM][VMW_BALLOON_OP_STAT_TYPES];398};399 400static inline bool is_vmballoon_stats_on(void)401{402 return IS_ENABLED(CONFIG_DEBUG_FS) &&403 static_branch_unlikely(&balloon_stat_enabled);404}405 406static inline void vmballoon_stats_op_inc(struct vmballoon *b, unsigned int op,407 enum vmballoon_op_stat_type type)408{409 if (is_vmballoon_stats_on())410 atomic64_inc(&b->stats->ops[op][type]);411}412 413static inline void vmballoon_stats_gen_inc(struct vmballoon *b,414 enum vmballoon_stat_general stat)415{416 if (is_vmballoon_stats_on())417 atomic64_inc(&b->stats->general_stat[stat]);418}419 420static inline void vmballoon_stats_gen_add(struct vmballoon *b,421 enum vmballoon_stat_general stat,422 unsigned int val)423{424 if (is_vmballoon_stats_on())425 atomic64_add(val, &b->stats->general_stat[stat]);426}427 428static inline void vmballoon_stats_page_inc(struct vmballoon *b,429 enum vmballoon_stat_page stat,430 enum vmballoon_page_size_type size)431{432 if (is_vmballoon_stats_on())433 atomic64_inc(&b->stats->page_stat[stat][size]);434}435 436static inline void vmballoon_stats_page_add(struct vmballoon *b,437 enum vmballoon_stat_page stat,438 enum vmballoon_page_size_type size,439 unsigned int val)440{441 if (is_vmballoon_stats_on())442 atomic64_add(val, &b->stats->page_stat[stat][size]);443}444 445static inline unsigned long446__vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,447 unsigned long arg2, unsigned long *result)448{449 unsigned long status, dummy1, dummy2, dummy3, local_result;450 451 vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_STAT);452 453 asm volatile ("inl %%dx" :454 "=a"(status),455 "=c"(dummy1),456 "=d"(dummy2),457 "=b"(local_result),458 "=S"(dummy3) :459 "0"(VMW_BALLOON_HV_MAGIC),460 "1"(cmd),461 "2"(VMW_BALLOON_HV_PORT),462 "3"(arg1),463 "4"(arg2) :464 "memory");465 466 /* update the result if needed */467 if (result)468 *result = (cmd == VMW_BALLOON_CMD_START) ? dummy1 :469 local_result;470 471 /* update target when applicable */472 if (status == VMW_BALLOON_SUCCESS &&473 ((1ul << cmd) & VMW_BALLOON_CMD_WITH_TARGET_MASK))474 WRITE_ONCE(b->target, local_result);475 476 if (status != VMW_BALLOON_SUCCESS &&477 status != VMW_BALLOON_SUCCESS_WITH_CAPABILITIES) {478 vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_FAIL_STAT);479 pr_debug("%s: %s [0x%lx,0x%lx) failed, returned %ld\n",480 __func__, vmballoon_cmd_names[cmd], arg1, arg2,481 status);482 }483 484 /* mark reset required accordingly */485 if (status == VMW_BALLOON_ERROR_RESET)486 b->reset_required = true;487 488 return status;489}490 491static __always_inline unsigned long492vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,493 unsigned long arg2)494{495 unsigned long dummy;496 497 return __vmballoon_cmd(b, cmd, arg1, arg2, &dummy);498}499 500/*501 * Send "start" command to the host, communicating supported version502 * of the protocol.503 */504static int vmballoon_send_start(struct vmballoon *b, unsigned long req_caps)505{506 unsigned long status, capabilities;507 508 status = __vmballoon_cmd(b, VMW_BALLOON_CMD_START, req_caps, 0,509 &capabilities);510 511 switch (status) {512 case VMW_BALLOON_SUCCESS_WITH_CAPABILITIES:513 b->capabilities = capabilities;514 break;515 case VMW_BALLOON_SUCCESS:516 b->capabilities = VMW_BALLOON_BASIC_CMDS;517 break;518 default:519 return -EIO;520 }521 522 /*523 * 2MB pages are only supported with batching. If batching is for some524 * reason disabled, do not use 2MB pages, since otherwise the legacy525 * mechanism is used with 2MB pages, causing a failure.526 */527 b->max_page_size = VMW_BALLOON_4K_PAGE;528 if ((b->capabilities & VMW_BALLOON_BATCHED_2M_CMDS) &&529 (b->capabilities & VMW_BALLOON_BATCHED_CMDS))530 b->max_page_size = VMW_BALLOON_2M_PAGE;531 532 533 return 0;534}535 536/**537 * vmballoon_send_guest_id - communicate guest type to the host.538 *539 * @b: pointer to the balloon.540 *541 * Communicate guest type to the host so that it can adjust ballooning542 * algorithm to the one most appropriate for the guest. This command543 * is normally issued after sending "start" command and is part of544 * standard reset sequence.545 *546 * Return: zero on success or appropriate error code.547 */548static int vmballoon_send_guest_id(struct vmballoon *b)549{550 unsigned long status;551 552 status = vmballoon_cmd(b, VMW_BALLOON_CMD_GUEST_ID,553 VMW_BALLOON_GUEST_ID, 0);554 555 return status == VMW_BALLOON_SUCCESS ? 0 : -EIO;556}557 558/**559 * vmballoon_page_order() - return the order of the page560 * @page_size: the size of the page.561 *562 * Return: the allocation order.563 */564static inline565unsigned int vmballoon_page_order(enum vmballoon_page_size_type page_size)566{567 return page_size == VMW_BALLOON_2M_PAGE ? VMW_BALLOON_2M_ORDER : 0;568}569 570/**571 * vmballoon_page_in_frames() - returns the number of frames in a page.572 * @page_size: the size of the page.573 *574 * Return: the number of 4k frames.575 */576static inline unsigned int577vmballoon_page_in_frames(enum vmballoon_page_size_type page_size)578{579 return 1 << vmballoon_page_order(page_size);580}581 582/**583 * vmballoon_mark_page_offline() - mark a page as offline584 * @page: pointer for the page.585 * @page_size: the size of the page.586 */587static void588vmballoon_mark_page_offline(struct page *page,589 enum vmballoon_page_size_type page_size)590{591 int i;592 593 for (i = 0; i < vmballoon_page_in_frames(page_size); i++)594 __SetPageOffline(page + i);595}596 597/**598 * vmballoon_mark_page_online() - mark a page as online599 * @page: pointer for the page.600 * @page_size: the size of the page.601 */602static void603vmballoon_mark_page_online(struct page *page,604 enum vmballoon_page_size_type page_size)605{606 int i;607 608 for (i = 0; i < vmballoon_page_in_frames(page_size); i++)609 __ClearPageOffline(page + i);610}611 612/**613 * vmballoon_send_get_target() - Retrieve desired balloon size from the host.614 *615 * @b: pointer to the balloon.616 *617 * Return: zero on success, EINVAL if limit does not fit in 32-bit, as required618 * by the host-guest protocol and EIO if an error occurred in communicating with619 * the host.620 */621static int vmballoon_send_get_target(struct vmballoon *b)622{623 unsigned long status;624 unsigned long limit;625 626 limit = totalram_pages();627 628 /* Ensure limit fits in 32-bits if 64-bit targets are not supported */629 if (!(b->capabilities & VMW_BALLOON_64_BIT_TARGET) &&630 limit != (u32)limit)631 return -EINVAL;632 633 status = vmballoon_cmd(b, VMW_BALLOON_CMD_GET_TARGET, limit, 0);634 635 return status == VMW_BALLOON_SUCCESS ? 0 : -EIO;636}637 638/**639 * vmballoon_alloc_page_list - allocates a list of pages.640 *641 * @b: pointer to the balloon.642 * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation.643 * @req_n_pages: the number of requested pages.644 *645 * Tries to allocate @req_n_pages. Add them to the list of balloon pages in646 * @ctl.pages and updates @ctl.n_pages to reflect the number of pages.647 *648 * Return: zero on success or error code otherwise.649 */650static int vmballoon_alloc_page_list(struct vmballoon *b,651 struct vmballoon_ctl *ctl,652 unsigned int req_n_pages)653{654 struct page *page;655 unsigned int i;656 657 for (i = 0; i < req_n_pages; i++) {658 /*659 * First check if we happen to have pages that were allocated660 * before. This happens when 2MB page rejected during inflation661 * by the hypervisor, and then split into 4KB pages.662 */663 if (!list_empty(&ctl->prealloc_pages)) {664 page = list_first_entry(&ctl->prealloc_pages,665 struct page, lru);666 list_del(&page->lru);667 } else {668 if (ctl->page_size == VMW_BALLOON_2M_PAGE)669 page = alloc_pages(__GFP_HIGHMEM|__GFP_NOWARN|670 __GFP_NOMEMALLOC, VMW_BALLOON_2M_ORDER);671 else672 page = balloon_page_alloc();673 674 vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC,675 ctl->page_size);676 }677 678 if (page) {679 /* Success. Add the page to the list and continue. */680 list_add(&page->lru, &ctl->pages);681 continue;682 }683 684 /* Allocation failed. Update statistics and stop. */685 vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC_FAIL,686 ctl->page_size);687 break;688 }689 690 ctl->n_pages = i;691 692 return req_n_pages == ctl->n_pages ? 0 : -ENOMEM;693}694 695/**696 * vmballoon_handle_one_result - Handle lock/unlock result for a single page.697 *698 * @b: pointer for %struct vmballoon.699 * @page: pointer for the page whose result should be handled.700 * @page_size: size of the page.701 * @status: status of the operation as provided by the hypervisor.702 */703static int vmballoon_handle_one_result(struct vmballoon *b, struct page *page,704 enum vmballoon_page_size_type page_size,705 unsigned long status)706{707 /* On success do nothing. The page is already on the balloon list. */708 if (likely(status == VMW_BALLOON_SUCCESS))709 return 0;710 711 pr_debug("%s: failed comm pfn %lx status %lu page_size %s\n", __func__,712 page_to_pfn(page), status,713 vmballoon_page_size_names[page_size]);714 715 /* Error occurred */716 vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,717 page_size);718 719 return -EIO;720}721 722/**723 * vmballoon_status_page - returns the status of (un)lock operation724 *725 * @b: pointer to the balloon.726 * @idx: index for the page for which the operation is performed.727 * @p: pointer to where the page struct is returned.728 *729 * Following a lock or unlock operation, returns the status of the operation for730 * an individual page. Provides the page that the operation was performed on on731 * the @page argument.732 *733 * Returns: The status of a lock or unlock operation for an individual page.734 */735static unsigned long vmballoon_status_page(struct vmballoon *b, int idx,736 struct page **p)737{738 if (static_branch_likely(&vmw_balloon_batching)) {739 /* batching mode */740 *p = pfn_to_page(b->batch_page[idx].pfn);741 return b->batch_page[idx].status;742 }743 744 /* non-batching mode */745 *p = b->page;746 747 /*748 * If a failure occurs, the indication will be provided in the status749 * of the entire operation, which is considered before the individual750 * page status. So for non-batching mode, the indication is always of751 * success.752 */753 return VMW_BALLOON_SUCCESS;754}755 756/**757 * vmballoon_lock_op - notifies the host about inflated/deflated pages.758 * @b: pointer to the balloon.759 * @num_pages: number of inflated/deflated pages.760 * @page_size: size of the page.761 * @op: the type of operation (lock or unlock).762 *763 * Notify the host about page(s) that were ballooned (or removed from the764 * balloon) so that host can use it without fear that guest will need it (or765 * stop using them since the VM does). Host may reject some pages, we need to766 * check the return value and maybe submit a different page. The pages that are767 * inflated/deflated are pointed by @b->page.768 *769 * Return: result as provided by the hypervisor.770 */771static unsigned long vmballoon_lock_op(struct vmballoon *b,772 unsigned int num_pages,773 enum vmballoon_page_size_type page_size,774 enum vmballoon_op op)775{776 unsigned long cmd, pfn;777 778 lockdep_assert_held(&b->comm_lock);779 780 if (static_branch_likely(&vmw_balloon_batching)) {781 if (op == VMW_BALLOON_INFLATE)782 cmd = page_size == VMW_BALLOON_2M_PAGE ?783 VMW_BALLOON_CMD_BATCHED_2M_LOCK :784 VMW_BALLOON_CMD_BATCHED_LOCK;785 else786 cmd = page_size == VMW_BALLOON_2M_PAGE ?787 VMW_BALLOON_CMD_BATCHED_2M_UNLOCK :788 VMW_BALLOON_CMD_BATCHED_UNLOCK;789 790 pfn = PHYS_PFN(virt_to_phys(b->batch_page));791 } else {792 cmd = op == VMW_BALLOON_INFLATE ? VMW_BALLOON_CMD_LOCK :793 VMW_BALLOON_CMD_UNLOCK;794 pfn = page_to_pfn(b->page);795 796 /* In non-batching mode, PFNs must fit in 32-bit */797 if (unlikely(pfn != (u32)pfn))798 return VMW_BALLOON_ERROR_PPN_INVALID;799 }800 801 return vmballoon_cmd(b, cmd, pfn, num_pages);802}803 804/**805 * vmballoon_add_page - adds a page towards lock/unlock operation.806 *807 * @b: pointer to the balloon.808 * @idx: index of the page to be ballooned in this batch.809 * @p: pointer to the page that is about to be ballooned.810 *811 * Adds the page to be ballooned. Must be called while holding @comm_lock.812 */813static void vmballoon_add_page(struct vmballoon *b, unsigned int idx,814 struct page *p)815{816 lockdep_assert_held(&b->comm_lock);817 818 if (static_branch_likely(&vmw_balloon_batching))819 b->batch_page[idx] = (struct vmballoon_batch_entry)820 { .pfn = page_to_pfn(p) };821 else822 b->page = p;823}824 825/**826 * vmballoon_lock - lock or unlock a batch of pages.827 *828 * @b: pointer to the balloon.829 * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation.830 *831 * Notifies the host of about ballooned pages (after inflation or deflation,832 * according to @ctl). If the host rejects the page put it on the833 * @ctl refuse list. These refused page are then released when moving to the834 * next size of pages.835 *836 * Note that we neither free any @page here nor put them back on the ballooned837 * pages list. Instead we queue it for later processing. We do that for several838 * reasons. First, we do not want to free the page under the lock. Second, it839 * allows us to unify the handling of lock and unlock. In the inflate case, the840 * caller will check if there are too many refused pages and release them.841 * Although it is not identical to the past behavior, it should not affect842 * performance.843 */844static int vmballoon_lock(struct vmballoon *b, struct vmballoon_ctl *ctl)845{846 unsigned long batch_status;847 struct page *page;848 unsigned int i, num_pages;849 850 num_pages = ctl->n_pages;851 if (num_pages == 0)852 return 0;853 854 /* communication with the host is done under the communication lock */855 spin_lock(&b->comm_lock);856 857 i = 0;858 list_for_each_entry(page, &ctl->pages, lru)859 vmballoon_add_page(b, i++, page);860 861 batch_status = vmballoon_lock_op(b, ctl->n_pages, ctl->page_size,862 ctl->op);863 864 /*865 * Iterate over the pages in the provided list. Since we are changing866 * @ctl->n_pages we are saving the original value in @num_pages and867 * use this value to bound the loop.868 */869 for (i = 0; i < num_pages; i++) {870 unsigned long status;871 872 status = vmballoon_status_page(b, i, &page);873 874 /*875 * Failure of the whole batch overrides a single operation876 * results.877 */878 if (batch_status != VMW_BALLOON_SUCCESS)879 status = batch_status;880 881 /* Continue if no error happened */882 if (!vmballoon_handle_one_result(b, page, ctl->page_size,883 status))884 continue;885 886 /*887 * Error happened. Move the pages to the refused list and update888 * the pages number.889 */890 list_move(&page->lru, &ctl->refused_pages);891 ctl->n_pages--;892 ctl->n_refused_pages++;893 }894 895 spin_unlock(&b->comm_lock);896 897 return batch_status == VMW_BALLOON_SUCCESS ? 0 : -EIO;898}899 900/**901 * vmballoon_release_page_list() - Releases a page list902 *903 * @page_list: list of pages to release.904 * @n_pages: pointer to the number of pages.905 * @page_size: whether the pages in the list are 2MB (or else 4KB).906 *907 * Releases the list of pages and zeros the number of pages.908 */909static void vmballoon_release_page_list(struct list_head *page_list,910 int *n_pages,911 enum vmballoon_page_size_type page_size)912{913 struct page *page, *tmp;914 915 list_for_each_entry_safe(page, tmp, page_list, lru) {916 list_del(&page->lru);917 __free_pages(page, vmballoon_page_order(page_size));918 }919 920 if (n_pages)921 *n_pages = 0;922}923 924 925/*926 * Release pages that were allocated while attempting to inflate the927 * balloon but were refused by the host for one reason or another.928 */929static void vmballoon_release_refused_pages(struct vmballoon *b,930 struct vmballoon_ctl *ctl)931{932 vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_FREE,933 ctl->page_size);934 935 vmballoon_release_page_list(&ctl->refused_pages, &ctl->n_refused_pages,936 ctl->page_size);937}938 939/**940 * vmballoon_change - retrieve the required balloon change941 *942 * @b: pointer for the balloon.943 *944 * Return: the required change for the balloon size. A positive number945 * indicates inflation, a negative number indicates a deflation.946 */947static int64_t vmballoon_change(struct vmballoon *b)948{949 int64_t size, target;950 951 size = atomic64_read(&b->size);952 target = READ_ONCE(b->target);953 954 /*955 * We must cast first because of int sizes956 * Otherwise we might get huge positives instead of negatives957 */958 959 if (b->reset_required)960 return 0;961 962 /* consider a 2MB slack on deflate, unless the balloon is emptied */963 if (target < size && target != 0 &&964 size - target < vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE))965 return 0;966 967 /* If an out-of-memory recently occurred, inflation is disallowed. */968 if (target > size && time_before(jiffies, READ_ONCE(b->shrink_timeout)))969 return 0;970 971 return target - size;972}973 974/**975 * vmballoon_enqueue_page_list() - Enqueues list of pages after inflation.976 *977 * @b: pointer to balloon.978 * @pages: list of pages to enqueue.979 * @n_pages: pointer to number of pages in list. The value is zeroed.980 * @page_size: whether the pages are 2MB or 4KB pages.981 *982 * Enqueues the provides list of pages in the ballooned page list, clears the983 * list and zeroes the number of pages that was provided.984 */985static void vmballoon_enqueue_page_list(struct vmballoon *b,986 struct list_head *pages,987 unsigned int *n_pages,988 enum vmballoon_page_size_type page_size)989{990 unsigned long flags;991 struct page *page;992 993 if (page_size == VMW_BALLOON_4K_PAGE) {994 balloon_page_list_enqueue(&b->b_dev_info, pages);995 } else {996 /*997 * Keep the huge pages in a local list which is not available998 * for the balloon compaction mechanism.999 */1000 spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);1001 1002 list_for_each_entry(page, pages, lru) {1003 vmballoon_mark_page_offline(page, VMW_BALLOON_2M_PAGE);1004 }1005 1006 list_splice_init(pages, &b->huge_pages);1007 __count_vm_events(BALLOON_INFLATE, *n_pages *1008 vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE));1009 spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);1010 }1011 1012 *n_pages = 0;1013}1014 1015/**1016 * vmballoon_dequeue_page_list() - Dequeues page lists for deflation.1017 *1018 * @b: pointer to balloon.1019 * @pages: list of pages to enqueue.1020 * @n_pages: pointer to number of pages in list. The value is zeroed.1021 * @page_size: whether the pages are 2MB or 4KB pages.1022 * @n_req_pages: the number of requested pages.1023 *1024 * Dequeues the number of requested pages from the balloon for deflation. The1025 * number of dequeued pages may be lower, if not enough pages in the requested1026 * size are available.1027 */1028static void vmballoon_dequeue_page_list(struct vmballoon *b,1029 struct list_head *pages,1030 unsigned int *n_pages,1031 enum vmballoon_page_size_type page_size,1032 unsigned int n_req_pages)1033{1034 struct page *page, *tmp;1035 unsigned int i = 0;1036 unsigned long flags;1037 1038 /* In the case of 4k pages, use the compaction infrastructure */1039 if (page_size == VMW_BALLOON_4K_PAGE) {1040 *n_pages = balloon_page_list_dequeue(&b->b_dev_info, pages,1041 n_req_pages);1042 return;1043 }1044 1045 /* 2MB pages */1046 spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);1047 list_for_each_entry_safe(page, tmp, &b->huge_pages, lru) {1048 vmballoon_mark_page_online(page, VMW_BALLOON_2M_PAGE);1049 1050 list_move(&page->lru, pages);1051 if (++i == n_req_pages)1052 break;1053 }1054 1055 __count_vm_events(BALLOON_DEFLATE,1056 i * vmballoon_page_in_frames(VMW_BALLOON_2M_PAGE));1057 spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);1058 *n_pages = i;1059}1060 1061/**1062 * vmballoon_split_refused_pages() - Split the 2MB refused pages to 4k.1063 *1064 * If inflation of 2MB pages was denied by the hypervisor, it is likely to be1065 * due to one or few 4KB pages. These 2MB pages may keep being allocated and1066 * then being refused. To prevent this case, this function splits the refused1067 * pages into 4KB pages and adds them into @prealloc_pages list.1068 *1069 * @ctl: pointer for the %struct vmballoon_ctl, which defines the operation.1070 */1071static void vmballoon_split_refused_pages(struct vmballoon_ctl *ctl)1072{1073 struct page *page, *tmp;1074 unsigned int i, order;1075 1076 order = vmballoon_page_order(ctl->page_size);1077 1078 list_for_each_entry_safe(page, tmp, &ctl->refused_pages, lru) {1079 list_del(&page->lru);1080 split_page(page, order);1081 for (i = 0; i < (1 << order); i++)1082 list_add(&page[i].lru, &ctl->prealloc_pages);1083 }1084 ctl->n_refused_pages = 0;1085}1086 1087/**1088 * vmballoon_inflate() - Inflate the balloon towards its target size.1089 *1090 * @b: pointer to the balloon.1091 */1092static void vmballoon_inflate(struct vmballoon *b)1093{1094 int64_t to_inflate_frames;1095 struct vmballoon_ctl ctl = {1096 .pages = LIST_HEAD_INIT(ctl.pages),1097 .refused_pages = LIST_HEAD_INIT(ctl.refused_pages),1098 .prealloc_pages = LIST_HEAD_INIT(ctl.prealloc_pages),1099 .page_size = b->max_page_size,1100 .op = VMW_BALLOON_INFLATE1101 };1102 1103 while ((to_inflate_frames = vmballoon_change(b)) > 0) {1104 unsigned int to_inflate_pages, page_in_frames;1105 int alloc_error, lock_error = 0;1106 1107 VM_BUG_ON(!list_empty(&ctl.pages));1108 VM_BUG_ON(ctl.n_pages != 0);1109 1110 page_in_frames = vmballoon_page_in_frames(ctl.page_size);1111 1112 to_inflate_pages = min_t(unsigned long, b->batch_max_pages,1113 DIV_ROUND_UP_ULL(to_inflate_frames,1114 page_in_frames));1115 1116 /* Start by allocating */1117 alloc_error = vmballoon_alloc_page_list(b, &ctl,1118 to_inflate_pages);1119 1120 /* Actually lock the pages by telling the hypervisor */1121 lock_error = vmballoon_lock(b, &ctl);1122 1123 /*1124 * If an error indicates that something serious went wrong,1125 * stop the inflation.1126 */1127 if (lock_error)1128 break;1129 1130 /* Update the balloon size */1131 atomic64_add(ctl.n_pages * page_in_frames, &b->size);1132 1133 vmballoon_enqueue_page_list(b, &ctl.pages, &ctl.n_pages,1134 ctl.page_size);1135 1136 /*1137 * If allocation failed or the number of refused pages exceeds1138 * the maximum allowed, move to the next page size.1139 */1140 if (alloc_error ||1141 ctl.n_refused_pages >= VMW_BALLOON_MAX_REFUSED) {1142 if (ctl.page_size == VMW_BALLOON_4K_PAGE)1143 break;1144 1145 /*1146 * Split the refused pages to 4k. This will also empty1147 * the refused pages list.1148 */1149 vmballoon_split_refused_pages(&ctl);1150 ctl.page_size--;1151 }1152 1153 cond_resched();1154 }1155 1156 /*1157 * Release pages that were allocated while attempting to inflate the1158 * balloon but were refused by the host for one reason or another,1159 * and update the statistics.1160 */1161 if (ctl.n_refused_pages != 0)1162 vmballoon_release_refused_pages(b, &ctl);1163 1164 vmballoon_release_page_list(&ctl.prealloc_pages, NULL, ctl.page_size);1165}1166 1167/**1168 * vmballoon_deflate() - Decrease the size of the balloon.1169 *1170 * @b: pointer to the balloon1171 * @n_frames: the number of frames to deflate. If zero, automatically1172 * calculated according to the target size.1173 * @coordinated: whether to coordinate with the host1174 *1175 * Decrease the size of the balloon allowing guest to use more memory.1176 *1177 * Return: The number of deflated frames (i.e., basic page size units)1178 */1179static unsigned long vmballoon_deflate(struct vmballoon *b, uint64_t n_frames,1180 bool coordinated)1181{1182 unsigned long deflated_frames = 0;1183 unsigned long tried_frames = 0;1184 struct vmballoon_ctl ctl = {1185 .pages = LIST_HEAD_INIT(ctl.pages),1186 .refused_pages = LIST_HEAD_INIT(ctl.refused_pages),1187 .page_size = VMW_BALLOON_4K_PAGE,1188 .op = VMW_BALLOON_DEFLATE1189 };1190 1191 /* free pages to reach target */1192 while (true) {1193 unsigned int to_deflate_pages, n_unlocked_frames;1194 unsigned int page_in_frames;1195 int64_t to_deflate_frames;1196 bool deflated_all;1197 1198 page_in_frames = vmballoon_page_in_frames(ctl.page_size);1199 1200 VM_BUG_ON(!list_empty(&ctl.pages));1201 VM_BUG_ON(ctl.n_pages);1202 VM_BUG_ON(!list_empty(&ctl.refused_pages));1203 VM_BUG_ON(ctl.n_refused_pages);1204 1205 /*1206 * If we were requested a specific number of frames, we try to1207 * deflate this number of frames. Otherwise, deflation is1208 * performed according to the target and balloon size.1209 */1210 to_deflate_frames = n_frames ? n_frames - tried_frames :1211 -vmballoon_change(b);1212 1213 /* break if no work to do */1214 if (to_deflate_frames <= 0)1215 break;1216 1217 /*1218 * Calculate the number of frames based on current page size,1219 * but limit the deflated frames to a single chunk1220 */1221 to_deflate_pages = min_t(unsigned long, b->batch_max_pages,1222 DIV_ROUND_UP_ULL(to_deflate_frames,1223 page_in_frames));1224 1225 /* First take the pages from the balloon pages. */1226 vmballoon_dequeue_page_list(b, &ctl.pages, &ctl.n_pages,1227 ctl.page_size, to_deflate_pages);1228 1229 /*1230 * Before pages are moving to the refused list, count their1231 * frames as frames that we tried to deflate.1232 */1233 tried_frames += ctl.n_pages * page_in_frames;1234 1235 /*1236 * Unlock the pages by communicating with the hypervisor if the1237 * communication is coordinated (i.e., not pop). We ignore the1238 * return code. Instead we check if all the pages we manage to1239 * unlock all the pages. If we failed, we will move to the next1240 * page size, and would eventually try again later.1241 */1242 if (coordinated)1243 vmballoon_lock(b, &ctl);1244 1245 /*1246 * Check if we deflated enough. We will move to the next page1247 * size if we did not manage to do so. This calculation takes1248 * place now, as once the pages are released, the number of1249 * pages is zeroed.1250 */1251 deflated_all = (ctl.n_pages == to_deflate_pages);1252 1253 /* Update local and global counters */1254 n_unlocked_frames = ctl.n_pages * page_in_frames;1255 atomic64_sub(n_unlocked_frames, &b->size);1256 deflated_frames += n_unlocked_frames;1257 1258 vmballoon_stats_page_add(b, VMW_BALLOON_PAGE_STAT_FREE,1259 ctl.page_size, ctl.n_pages);1260 1261 /* free the ballooned pages */1262 vmballoon_release_page_list(&ctl.pages, &ctl.n_pages,1263 ctl.page_size);1264 1265 /* Return the refused pages to the ballooned list. */1266 vmballoon_enqueue_page_list(b, &ctl.refused_pages,1267 &ctl.n_refused_pages,1268 ctl.page_size);1269 1270 /* If we failed to unlock all the pages, move to next size. */1271 if (!deflated_all) {1272 if (ctl.page_size == b->max_page_size)1273 break;1274 ctl.page_size++;1275 }1276 1277 cond_resched();1278 }1279 1280 return deflated_frames;1281}1282 1283/**1284 * vmballoon_deinit_batching - disables batching mode.1285 *1286 * @b: pointer to &struct vmballoon.1287 *1288 * Disables batching, by deallocating the page for communication with the1289 * hypervisor and disabling the static key to indicate that batching is off.1290 */1291static void vmballoon_deinit_batching(struct vmballoon *b)1292{1293 free_page((unsigned long)b->batch_page);1294 b->batch_page = NULL;1295 static_branch_disable(&vmw_balloon_batching);1296 b->batch_max_pages = 1;1297}1298 1299/**1300 * vmballoon_init_batching - enable batching mode.1301 *1302 * @b: pointer to &struct vmballoon.1303 *1304 * Enables batching, by allocating a page for communication with the hypervisor1305 * and enabling the static_key to use batching.1306 *1307 * Return: zero on success or an appropriate error-code.1308 */1309static int vmballoon_init_batching(struct vmballoon *b)1310{1311 struct page *page;1312 1313 page = alloc_page(GFP_KERNEL | __GFP_ZERO);1314 if (!page)1315 return -ENOMEM;1316 1317 b->batch_page = page_address(page);1318 b->batch_max_pages = PAGE_SIZE / sizeof(struct vmballoon_batch_entry);1319 1320 static_branch_enable(&vmw_balloon_batching);1321 1322 return 0;1323}1324 1325/*1326 * Receive notification and resize balloon1327 */1328static void vmballoon_doorbell(void *client_data)1329{1330 struct vmballoon *b = client_data;1331 1332 vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_DOORBELL);1333 1334 mod_delayed_work(system_freezable_wq, &b->dwork, 0);1335}1336 1337/*1338 * Clean up vmci doorbell1339 */1340static void vmballoon_vmci_cleanup(struct vmballoon *b)1341{1342 vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET,1343 VMCI_INVALID_ID, VMCI_INVALID_ID);1344 1345 if (!vmci_handle_is_invalid(b->vmci_doorbell)) {1346 vmci_doorbell_destroy(b->vmci_doorbell);1347 b->vmci_doorbell = VMCI_INVALID_HANDLE;1348 }1349}1350 1351/**1352 * vmballoon_vmci_init - Initialize vmci doorbell.1353 *1354 * @b: pointer to the balloon.1355 *1356 * Return: zero on success or when wakeup command not supported. Error-code1357 * otherwise.1358 *1359 * Initialize vmci doorbell, to get notified as soon as balloon changes.1360 */1361static int vmballoon_vmci_init(struct vmballoon *b)1362{1363 unsigned long error;1364 1365 if ((b->capabilities & VMW_BALLOON_SIGNALLED_WAKEUP_CMD) == 0)1366 return 0;1367 1368 error = vmci_doorbell_create(&b->vmci_doorbell, VMCI_FLAG_DELAYED_CB,1369 VMCI_PRIVILEGE_FLAG_RESTRICTED,1370 vmballoon_doorbell, b);1371 1372 if (error != VMCI_SUCCESS)1373 goto fail;1374 1375 error = __vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET,1376 b->vmci_doorbell.context,1377 b->vmci_doorbell.resource, NULL);1378 1379 if (error != VMW_BALLOON_SUCCESS)1380 goto fail;1381 1382 return 0;1383fail:1384 vmballoon_vmci_cleanup(b);1385 return -EIO;1386}1387 1388/**1389 * vmballoon_pop - Quickly release all pages allocate for the balloon.1390 *1391 * @b: pointer to the balloon.1392 *1393 * This function is called when host decides to "reset" balloon for one reason1394 * or another. Unlike normal "deflate" we do not (shall not) notify host of the1395 * pages being released.1396 */1397static void vmballoon_pop(struct vmballoon *b)1398{1399 unsigned long size;1400 1401 while ((size = atomic64_read(&b->size)))1402 vmballoon_deflate(b, size, false);1403}1404 1405/*1406 * Perform standard reset sequence by popping the balloon (in case it1407 * is not empty) and then restarting protocol. This operation normally1408 * happens when host responds with VMW_BALLOON_ERROR_RESET to a command.1409 */1410static void vmballoon_reset(struct vmballoon *b)1411{1412 int error;1413 1414 down_write(&b->conf_sem);1415 1416 vmballoon_vmci_cleanup(b);1417 1418 /* free all pages, skipping monitor unlock */1419 vmballoon_pop(b);1420 1421 if (vmballoon_send_start(b, VMW_BALLOON_CAPABILITIES))1422 goto unlock;1423 1424 if ((b->capabilities & VMW_BALLOON_BATCHED_CMDS) != 0) {1425 if (vmballoon_init_batching(b)) {1426 /*1427 * We failed to initialize batching, inform the monitor1428 * about it by sending a null capability.1429 *1430 * The guest will retry in one second.1431 */1432 vmballoon_send_start(b, 0);1433 goto unlock;1434 }1435 } else if ((b->capabilities & VMW_BALLOON_BASIC_CMDS) != 0) {1436 vmballoon_deinit_batching(b);1437 }1438 1439 vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_RESET);1440 b->reset_required = false;1441 1442 error = vmballoon_vmci_init(b);1443 if (error)1444 pr_err_once("failed to initialize vmci doorbell\n");1445 1446 if (vmballoon_send_guest_id(b))1447 pr_err_once("failed to send guest ID to the host\n");1448 1449unlock:1450 up_write(&b->conf_sem);1451}1452 1453/**1454 * vmballoon_work - periodic balloon worker for reset, inflation and deflation.1455 *1456 * @work: pointer to the &work_struct which is provided by the workqueue.1457 *1458 * Resets the protocol if needed, gets the new size and adjusts balloon as1459 * needed. Repeat in 1 sec.1460 */1461static void vmballoon_work(struct work_struct *work)1462{1463 struct delayed_work *dwork = to_delayed_work(work);1464 struct vmballoon *b = container_of(dwork, struct vmballoon, dwork);1465 int64_t change = 0;1466 1467 if (b->reset_required)1468 vmballoon_reset(b);1469 1470 down_read(&b->conf_sem);1471 1472 /*1473 * Update the stats while holding the semaphore to ensure that1474 * @stats_enabled is consistent with whether the stats are actually1475 * enabled1476 */1477 vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_TIMER);1478 1479 if (!vmballoon_send_get_target(b))1480 change = vmballoon_change(b);1481 1482 if (change != 0) {1483 pr_debug("%s - size: %llu, target %lu\n", __func__,1484 atomic64_read(&b->size), READ_ONCE(b->target));1485 1486 if (change > 0)1487 vmballoon_inflate(b);1488 else /* (change < 0) */1489 vmballoon_deflate(b, 0, true);1490 }1491 1492 up_read(&b->conf_sem);1493 1494 /*1495 * We are using a freezable workqueue so that balloon operations are1496 * stopped while the system transitions to/from sleep/hibernation.1497 */1498 queue_delayed_work(system_freezable_wq,1499 dwork, round_jiffies_relative(HZ));1500 1501}1502 1503/**1504 * vmballoon_shrinker_scan() - deflate the balloon due to memory pressure.1505 * @shrinker: pointer to the balloon shrinker.1506 * @sc: page reclaim information.1507 *1508 * Returns: number of pages that were freed during deflation.1509 */1510static unsigned long vmballoon_shrinker_scan(struct shrinker *shrinker,1511 struct shrink_control *sc)1512{1513 struct vmballoon *b = &balloon;1514 unsigned long deflated_frames;1515 1516 pr_debug("%s - size: %llu", __func__, atomic64_read(&b->size));1517 1518 vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_SHRINK);1519 1520 /*1521 * If the lock is also contended for read, we cannot easily reclaim and1522 * we bail out.1523 */1524 if (!down_read_trylock(&b->conf_sem))1525 return 0;1526 1527 deflated_frames = vmballoon_deflate(b, sc->nr_to_scan, true);1528 1529 vmballoon_stats_gen_add(b, VMW_BALLOON_STAT_SHRINK_FREE,1530 deflated_frames);1531 1532 /*1533 * Delay future inflation for some time to mitigate the situations in1534 * which balloon continuously grows and shrinks. Use WRITE_ONCE() since1535 * the access is asynchronous.1536 */1537 WRITE_ONCE(b->shrink_timeout, jiffies + HZ * VMBALLOON_SHRINK_DELAY);1538 1539 up_read(&b->conf_sem);1540 1541 return deflated_frames;1542}1543 1544/**1545 * vmballoon_shrinker_count() - return the number of ballooned pages.1546 * @shrinker: pointer to the balloon shrinker.1547 * @sc: page reclaim information.1548 *1549 * Returns: number of 4k pages that are allocated for the balloon and can1550 * therefore be reclaimed under pressure.1551 */1552static unsigned long vmballoon_shrinker_count(struct shrinker *shrinker,1553 struct shrink_control *sc)1554{1555 struct vmballoon *b = &balloon;1556 1557 return atomic64_read(&b->size);1558}1559 1560static void vmballoon_unregister_shrinker(struct vmballoon *b)1561{1562 shrinker_free(b->shrinker);1563 b->shrinker = NULL;1564}1565 1566static int vmballoon_register_shrinker(struct vmballoon *b)1567{1568 /* Do nothing if the shrinker is not enabled */1569 if (!vmwballoon_shrinker_enable)1570 return 0;1571 1572 b->shrinker = shrinker_alloc(0, "vmw-balloon");1573 if (!b->shrinker)1574 return -ENOMEM;1575 1576 b->shrinker->scan_objects = vmballoon_shrinker_scan;1577 b->shrinker->count_objects = vmballoon_shrinker_count;1578 b->shrinker->private_data = b;1579 1580 shrinker_register(b->shrinker);1581 1582 return 0;1583}1584 1585/*1586 * DEBUGFS Interface1587 */1588#ifdef CONFIG_DEBUG_FS1589 1590static const char * const vmballoon_stat_page_names[] = {1591 [VMW_BALLOON_PAGE_STAT_ALLOC] = "alloc",1592 [VMW_BALLOON_PAGE_STAT_ALLOC_FAIL] = "allocFail",1593 [VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC] = "errAlloc",1594 [VMW_BALLOON_PAGE_STAT_REFUSED_FREE] = "errFree",1595 [VMW_BALLOON_PAGE_STAT_FREE] = "free"1596};1597 1598static const char * const vmballoon_stat_names[] = {1599 [VMW_BALLOON_STAT_TIMER] = "timer",1600 [VMW_BALLOON_STAT_DOORBELL] = "doorbell",1601 [VMW_BALLOON_STAT_RESET] = "reset",1602 [VMW_BALLOON_STAT_SHRINK] = "shrink",1603 [VMW_BALLOON_STAT_SHRINK_FREE] = "shrinkFree"1604};1605 1606static int vmballoon_enable_stats(struct vmballoon *b)1607{1608 int r = 0;1609 1610 down_write(&b->conf_sem);1611 1612 /* did we somehow race with another reader which enabled stats? */1613 if (b->stats)1614 goto out;1615 1616 b->stats = kzalloc(sizeof(*b->stats), GFP_KERNEL);1617 1618 if (!b->stats) {1619 /* allocation failed */1620 r = -ENOMEM;1621 goto out;1622 }1623 static_key_enable(&balloon_stat_enabled.key);1624out:1625 up_write(&b->conf_sem);1626 return r;1627}1628 1629/**1630 * vmballoon_debug_show - shows statistics of balloon operations.1631 * @f: pointer to the &struct seq_file.1632 * @offset: ignored.1633 *1634 * Provides the statistics that can be accessed in vmmemctl in the debugfs.1635 * To avoid the overhead - mainly that of memory - of collecting the statistics,1636 * we only collect statistics after the first time the counters are read.1637 *1638 * Return: zero on success or an error code.1639 */1640static int vmballoon_debug_show(struct seq_file *f, void *offset)1641{1642 struct vmballoon *b = f->private;1643 int i, j;1644 1645 /* enables stats if they are disabled */1646 if (!b->stats) {1647 int r = vmballoon_enable_stats(b);1648 1649 if (r)1650 return r;1651 }1652 1653 /* format capabilities info */1654 seq_printf(f, "%-22s: %#16x\n", "balloon capabilities",1655 VMW_BALLOON_CAPABILITIES);1656 seq_printf(f, "%-22s: %#16lx\n", "used capabilities", b->capabilities);1657 seq_printf(f, "%-22s: %16s\n", "is resetting",1658 b->reset_required ? "y" : "n");1659 1660 /* format size info */1661 seq_printf(f, "%-22s: %16lu\n", "target", READ_ONCE(b->target));1662 seq_printf(f, "%-22s: %16llu\n", "current", atomic64_read(&b->size));1663 1664 for (i = 0; i < VMW_BALLOON_CMD_NUM; i++) {1665 if (vmballoon_cmd_names[i] == NULL)1666 continue;1667 1668 seq_printf(f, "%-22s: %16llu (%llu failed)\n",1669 vmballoon_cmd_names[i],1670 atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_STAT]),1671 atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_FAIL_STAT]));1672 }1673 1674 for (i = 0; i < VMW_BALLOON_STAT_NUM; i++)1675 seq_printf(f, "%-22s: %16llu\n",1676 vmballoon_stat_names[i],1677 atomic64_read(&b->stats->general_stat[i]));1678 1679 for (i = 0; i < VMW_BALLOON_PAGE_STAT_NUM; i++) {1680 for (j = 0; j < VMW_BALLOON_NUM_PAGE_SIZES; j++)1681 seq_printf(f, "%-18s(%s): %16llu\n",1682 vmballoon_stat_page_names[i],1683 vmballoon_page_size_names[j],1684 atomic64_read(&b->stats->page_stat[i][j]));1685 }1686 1687 return 0;1688}1689 1690DEFINE_SHOW_ATTRIBUTE(vmballoon_debug);1691 1692static void __init vmballoon_debugfs_init(struct vmballoon *b)1693{1694 debugfs_create_file("vmmemctl", S_IRUGO, NULL, b,1695 &vmballoon_debug_fops);1696}1697 1698static void __exit vmballoon_debugfs_exit(struct vmballoon *b)1699{1700 static_key_disable(&balloon_stat_enabled.key);1701 debugfs_lookup_and_remove("vmmemctl", NULL);1702 kfree(b->stats);1703 b->stats = NULL;1704}1705 1706#else1707 1708static inline void vmballoon_debugfs_init(struct vmballoon *b)1709{1710}1711 1712static inline void vmballoon_debugfs_exit(struct vmballoon *b)1713{1714}1715 1716#endif /* CONFIG_DEBUG_FS */1717 1718 1719#ifdef CONFIG_BALLOON_COMPACTION1720/**1721 * vmballoon_migratepage() - migrates a balloon page.1722 * @b_dev_info: balloon device information descriptor.1723 * @newpage: the page to which @page should be migrated.1724 * @page: a ballooned page that should be migrated.1725 * @mode: migration mode, ignored.1726 *1727 * This function is really open-coded, but that is according to the interface1728 * that balloon_compaction provides.1729 *1730 * Return: zero on success, -EAGAIN when migration cannot be performed1731 * momentarily, and -EBUSY if migration failed and should be retried1732 * with that specific page.1733 */1734static int vmballoon_migratepage(struct balloon_dev_info *b_dev_info,1735 struct page *newpage, struct page *page,1736 enum migrate_mode mode)1737{1738 unsigned long status, flags;1739 struct vmballoon *b;1740 int ret;1741 1742 b = container_of(b_dev_info, struct vmballoon, b_dev_info);1743 1744 /*1745 * If the semaphore is taken, there is ongoing configuration change1746 * (i.e., balloon reset), so try again.1747 */1748 if (!down_read_trylock(&b->conf_sem))1749 return -EAGAIN;1750 1751 spin_lock(&b->comm_lock);1752 /*1753 * We must start by deflating and not inflating, as otherwise the1754 * hypervisor may tell us that it has enough memory and the new page is1755 * not needed. Since the old page is isolated, we cannot use the list1756 * interface to unlock it, as the LRU field is used for isolation.1757 * Instead, we use the native interface directly.1758 */1759 vmballoon_add_page(b, 0, page);1760 status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE,1761 VMW_BALLOON_DEFLATE);1762 1763 if (status == VMW_BALLOON_SUCCESS)1764 status = vmballoon_status_page(b, 0, &page);1765 1766 /*1767 * If a failure happened, let the migration mechanism know that it1768 * should not retry.1769 */1770 if (status != VMW_BALLOON_SUCCESS) {1771 spin_unlock(&b->comm_lock);1772 ret = -EBUSY;1773 goto out_unlock;1774 }1775 1776 /*1777 * The page is isolated, so it is safe to delete it without holding1778 * @pages_lock . We keep holding @comm_lock since we will need it in a1779 * second.1780 */1781 balloon_page_delete(page);1782 1783 put_page(page);1784 1785 /* Inflate */1786 vmballoon_add_page(b, 0, newpage);1787 status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE,1788 VMW_BALLOON_INFLATE);1789 1790 if (status == VMW_BALLOON_SUCCESS)1791 status = vmballoon_status_page(b, 0, &newpage);1792 1793 spin_unlock(&b->comm_lock);1794 1795 if (status != VMW_BALLOON_SUCCESS) {1796 /*1797 * A failure happened. While we can deflate the page we just1798 * inflated, this deflation can also encounter an error. Instead1799 * we will decrease the size of the balloon to reflect the1800 * change and report failure.1801 */1802 atomic64_dec(&b->size);1803 ret = -EBUSY;1804 } else {1805 /*1806 * Success. Take a reference for the page, and we will add it to1807 * the list after acquiring the lock.1808 */1809 get_page(newpage);1810 ret = MIGRATEPAGE_SUCCESS;1811 }1812 1813 /* Update the balloon list under the @pages_lock */1814 spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);1815 1816 /*1817 * On inflation success, we already took a reference for the @newpage.1818 * If we succeed just insert it to the list and update the statistics1819 * under the lock.1820 */1821 if (ret == MIGRATEPAGE_SUCCESS) {1822 balloon_page_insert(&b->b_dev_info, newpage);1823 __count_vm_event(BALLOON_MIGRATE);1824 }1825 1826 /*1827 * We deflated successfully, so regardless to the inflation success, we1828 * need to reduce the number of isolated_pages.1829 */1830 b->b_dev_info.isolated_pages--;1831 spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);1832 1833out_unlock:1834 up_read(&b->conf_sem);1835 return ret;1836}1837 1838/**1839 * vmballoon_compaction_init() - initialized compaction for the balloon.1840 *1841 * @b: pointer to the balloon.1842 *1843 * If during the initialization a failure occurred, this function does not1844 * perform cleanup. The caller must call vmballoon_compaction_deinit() in this1845 * case.1846 *1847 * Return: zero on success or error code on failure.1848 */1849static __init void vmballoon_compaction_init(struct vmballoon *b)1850{1851 b->b_dev_info.migratepage = vmballoon_migratepage;1852}1853 1854#else /* CONFIG_BALLOON_COMPACTION */1855static inline void vmballoon_compaction_init(struct vmballoon *b)1856{1857}1858#endif /* CONFIG_BALLOON_COMPACTION */1859 1860static int __init vmballoon_init(void)1861{1862 int error;1863 1864 /*1865 * Check if we are running on VMware's hypervisor and bail out1866 * if we are not.1867 */1868 if (x86_hyper_type != X86_HYPER_VMWARE)1869 return -ENODEV;1870 1871 INIT_DELAYED_WORK(&balloon.dwork, vmballoon_work);1872 1873 error = vmballoon_register_shrinker(&balloon);1874 if (error)1875 return error;1876 1877 /*1878 * Initialization of compaction must be done after the call to1879 * balloon_devinfo_init() .1880 */1881 balloon_devinfo_init(&balloon.b_dev_info);1882 vmballoon_compaction_init(&balloon);1883 1884 INIT_LIST_HEAD(&balloon.huge_pages);1885 spin_lock_init(&balloon.comm_lock);1886 init_rwsem(&balloon.conf_sem);1887 balloon.vmci_doorbell = VMCI_INVALID_HANDLE;1888 balloon.batch_page = NULL;1889 balloon.page = NULL;1890 balloon.reset_required = true;1891 1892 queue_delayed_work(system_freezable_wq, &balloon.dwork, 0);1893 1894 vmballoon_debugfs_init(&balloon);1895 1896 return 0;1897}1898 1899/*1900 * Using late_initcall() instead of module_init() allows the balloon to use the1901 * VMCI doorbell even when the balloon is built into the kernel. Otherwise the1902 * VMCI is probed only after the balloon is initialized. If the balloon is used1903 * as a module, late_initcall() is equivalent to module_init().1904 */1905late_initcall(vmballoon_init);1906 1907static void __exit vmballoon_exit(void)1908{1909 vmballoon_unregister_shrinker(&balloon);1910 vmballoon_vmci_cleanup(&balloon);1911 cancel_delayed_work_sync(&balloon.dwork);1912 1913 vmballoon_debugfs_exit(&balloon);1914 1915 /*1916 * Deallocate all reserved memory, and reset connection with monitor.1917 * Reset connection before deallocating memory to avoid potential for1918 * additional spurious resets from guest touching deallocated pages.1919 */1920 vmballoon_send_start(&balloon, 0);1921 vmballoon_pop(&balloon);1922}1923module_exit(vmballoon_exit);1924