1700 lines · c
1/******************************************************************************2 * grant_table.c3 *4 * Granting foreign access to our memory reservation.5 *6 * Copyright (c) 2005-2006, Christopher Clark7 * Copyright (c) 2004-2005, K A Fraser8 *9 * This program is free software; you can redistribute it and/or10 * modify it under the terms of the GNU General Public License version 211 * as published by the Free Software Foundation; or, when distributed12 * separately from the Linux kernel or incorporated into other13 * software packages, subject to the following license:14 *15 * Permission is hereby granted, free of charge, to any person obtaining a copy16 * of this source file (the "Software"), to deal in the Software without17 * restriction, including without limitation the rights to use, copy, modify,18 * merge, publish, distribute, sublicense, and/or sell copies of the Software,19 * and to permit persons to whom the Software is furnished to do so, subject to20 * the following conditions:21 *22 * The above copyright notice and this permission notice shall be included in23 * all copies or substantial portions of the Software.24 *25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS31 * IN THE SOFTWARE.32 */33 34#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt35 36#include <linux/bitmap.h>37#include <linux/memblock.h>38#include <linux/sched.h>39#include <linux/mm.h>40#include <linux/slab.h>41#include <linux/vmalloc.h>42#include <linux/uaccess.h>43#include <linux/io.h>44#include <linux/delay.h>45#include <linux/hardirq.h>46#include <linux/workqueue.h>47#include <linux/ratelimit.h>48#include <linux/moduleparam.h>49#ifdef CONFIG_XEN_GRANT_DMA_ALLOC50#include <linux/dma-mapping.h>51#endif52 53#include <xen/xen.h>54#include <xen/interface/xen.h>55#include <xen/page.h>56#include <xen/grant_table.h>57#include <xen/interface/memory.h>58#include <xen/hvc-console.h>59#include <xen/swiotlb-xen.h>60#include <xen/balloon.h>61#ifdef CONFIG_X8662#include <asm/xen/cpuid.h>63#endif64#include <xen/mem-reservation.h>65#include <asm/xen/hypercall.h>66#include <asm/xen/interface.h>67 68#include <asm/sync_bitops.h>69 70#define GNTTAB_LIST_END 0xffffffff71 72static grant_ref_t **gnttab_list;73static unsigned int nr_grant_frames;74 75/*76 * Handling of free grants:77 *78 * Free grants are in a simple list anchored in gnttab_free_head. They are79 * linked by grant ref, the last element contains GNTTAB_LIST_END. The number80 * of free entries is stored in gnttab_free_count.81 * Additionally there is a bitmap of free entries anchored in82 * gnttab_free_bitmap. This is being used for simplifying allocation of83 * multiple consecutive grants, which is needed e.g. for support of virtio.84 * gnttab_last_free is used to add free entries of new frames at the end of85 * the free list.86 * gnttab_free_tail_ptr specifies the variable which references the start87 * of consecutive free grants ending with gnttab_last_free. This pointer is88 * updated in a rather defensive way, in order to avoid performance hits in89 * hot paths.90 * All those variables are protected by gnttab_list_lock.91 */92static int gnttab_free_count;93static unsigned int gnttab_size;94static grant_ref_t gnttab_free_head = GNTTAB_LIST_END;95static grant_ref_t gnttab_last_free = GNTTAB_LIST_END;96static grant_ref_t *gnttab_free_tail_ptr;97static unsigned long *gnttab_free_bitmap;98static DEFINE_SPINLOCK(gnttab_list_lock);99 100struct grant_frames xen_auto_xlat_grant_frames;101static unsigned int xen_gnttab_version;102module_param_named(version, xen_gnttab_version, uint, 0);103 104static union {105 struct grant_entry_v1 *v1;106 union grant_entry_v2 *v2;107 void *addr;108} gnttab_shared;109 110/*This is a structure of function pointers for grant table*/111struct gnttab_ops {112 /*113 * Version of the grant interface.114 */115 unsigned int version;116 /*117 * Grant refs per grant frame.118 */119 unsigned int grefs_per_grant_frame;120 /*121 * Mapping a list of frames for storing grant entries. Frames parameter122 * is used to store grant table address when grant table being setup,123 * nr_gframes is the number of frames to map grant table. Returning124 * GNTST_okay means success and negative value means failure.125 */126 int (*map_frames)(xen_pfn_t *frames, unsigned int nr_gframes);127 /*128 * Release a list of frames which are mapped in map_frames for grant129 * entry status.130 */131 void (*unmap_frames)(void);132 /*133 * Introducing a valid entry into the grant table, granting the frame of134 * this grant entry to domain for accessing. Ref135 * parameter is reference of this introduced grant entry, domid is id of136 * granted domain, frame is the page frame to be granted, and flags is137 * status of the grant entry to be updated.138 */139 void (*update_entry)(grant_ref_t ref, domid_t domid,140 unsigned long frame, unsigned flags);141 /*142 * Stop granting a grant entry to domain for accessing. Ref parameter is143 * reference of a grant entry whose grant access will be stopped.144 * If the grant entry is currently mapped for reading or writing, just145 * return failure(==0) directly and don't tear down the grant access.146 * Otherwise, stop grant access for this entry and return success(==1).147 */148 int (*end_foreign_access_ref)(grant_ref_t ref);149 /*150 * Read the frame number related to a given grant reference.151 */152 unsigned long (*read_frame)(grant_ref_t ref);153};154 155struct unmap_refs_callback_data {156 struct completion completion;157 int result;158};159 160static const struct gnttab_ops *gnttab_interface;161 162/* This reflects status of grant entries, so act as a global value. */163static grant_status_t *grstatus;164 165static struct gnttab_free_callback *gnttab_free_callback_list;166 167static int gnttab_expand(unsigned int req_entries);168 169#define RPP (PAGE_SIZE / sizeof(grant_ref_t))170#define SPP (PAGE_SIZE / sizeof(grant_status_t))171 172static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)173{174 return &gnttab_list[(entry) / RPP][(entry) % RPP];175}176/* This can be used as an l-value */177#define gnttab_entry(entry) (*__gnttab_entry(entry))178 179static int get_free_entries(unsigned count)180{181 unsigned long flags;182 int ref, rc = 0;183 grant_ref_t head;184 185 spin_lock_irqsave(&gnttab_list_lock, flags);186 187 if ((gnttab_free_count < count) &&188 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {189 spin_unlock_irqrestore(&gnttab_list_lock, flags);190 return rc;191 }192 193 ref = head = gnttab_free_head;194 gnttab_free_count -= count;195 while (count--) {196 bitmap_clear(gnttab_free_bitmap, head, 1);197 if (gnttab_free_tail_ptr == __gnttab_entry(head))198 gnttab_free_tail_ptr = &gnttab_free_head;199 if (count)200 head = gnttab_entry(head);201 }202 gnttab_free_head = gnttab_entry(head);203 gnttab_entry(head) = GNTTAB_LIST_END;204 205 if (!gnttab_free_count) {206 gnttab_last_free = GNTTAB_LIST_END;207 gnttab_free_tail_ptr = NULL;208 }209 210 spin_unlock_irqrestore(&gnttab_list_lock, flags);211 212 return ref;213}214 215static int get_seq_entry_count(void)216{217 if (gnttab_last_free == GNTTAB_LIST_END || !gnttab_free_tail_ptr ||218 *gnttab_free_tail_ptr == GNTTAB_LIST_END)219 return 0;220 221 return gnttab_last_free - *gnttab_free_tail_ptr + 1;222}223 224/* Rebuilds the free grant list and tries to find count consecutive entries. */225static int get_free_seq(unsigned int count)226{227 int ret = -ENOSPC;228 unsigned int from, to;229 grant_ref_t *last;230 231 gnttab_free_tail_ptr = &gnttab_free_head;232 last = &gnttab_free_head;233 234 for (from = find_first_bit(gnttab_free_bitmap, gnttab_size);235 from < gnttab_size;236 from = find_next_bit(gnttab_free_bitmap, gnttab_size, to + 1)) {237 to = find_next_zero_bit(gnttab_free_bitmap, gnttab_size,238 from + 1);239 if (ret < 0 && to - from >= count) {240 ret = from;241 bitmap_clear(gnttab_free_bitmap, ret, count);242 from += count;243 gnttab_free_count -= count;244 if (from == to)245 continue;246 }247 248 /*249 * Recreate the free list in order to have it properly sorted.250 * This is needed to make sure that the free tail has the maximum251 * possible size.252 */253 while (from < to) {254 *last = from;255 last = __gnttab_entry(from);256 gnttab_last_free = from;257 from++;258 }259 if (to < gnttab_size)260 gnttab_free_tail_ptr = __gnttab_entry(to - 1);261 }262 263 *last = GNTTAB_LIST_END;264 if (gnttab_last_free != gnttab_size - 1)265 gnttab_free_tail_ptr = NULL;266 267 return ret;268}269 270static int get_free_entries_seq(unsigned int count)271{272 unsigned long flags;273 int ret = 0;274 275 spin_lock_irqsave(&gnttab_list_lock, flags);276 277 if (gnttab_free_count < count) {278 ret = gnttab_expand(count - gnttab_free_count);279 if (ret < 0)280 goto out;281 }282 283 if (get_seq_entry_count() < count) {284 ret = get_free_seq(count);285 if (ret >= 0)286 goto out;287 ret = gnttab_expand(count - get_seq_entry_count());288 if (ret < 0)289 goto out;290 }291 292 ret = *gnttab_free_tail_ptr;293 *gnttab_free_tail_ptr = gnttab_entry(ret + count - 1);294 gnttab_free_count -= count;295 if (!gnttab_free_count)296 gnttab_free_tail_ptr = NULL;297 bitmap_clear(gnttab_free_bitmap, ret, count);298 299 out:300 spin_unlock_irqrestore(&gnttab_list_lock, flags);301 302 return ret;303}304 305static void do_free_callbacks(void)306{307 struct gnttab_free_callback *callback, *next;308 309 callback = gnttab_free_callback_list;310 gnttab_free_callback_list = NULL;311 312 while (callback != NULL) {313 next = callback->next;314 if (gnttab_free_count >= callback->count) {315 callback->next = NULL;316 callback->fn(callback->arg);317 } else {318 callback->next = gnttab_free_callback_list;319 gnttab_free_callback_list = callback;320 }321 callback = next;322 }323}324 325static inline void check_free_callbacks(void)326{327 if (unlikely(gnttab_free_callback_list))328 do_free_callbacks();329}330 331static void put_free_entry_locked(grant_ref_t ref)332{333 if (unlikely(ref < GNTTAB_NR_RESERVED_ENTRIES))334 return;335 336 gnttab_entry(ref) = gnttab_free_head;337 gnttab_free_head = ref;338 if (!gnttab_free_count)339 gnttab_last_free = ref;340 if (gnttab_free_tail_ptr == &gnttab_free_head)341 gnttab_free_tail_ptr = __gnttab_entry(ref);342 gnttab_free_count++;343 bitmap_set(gnttab_free_bitmap, ref, 1);344}345 346static void put_free_entry(grant_ref_t ref)347{348 unsigned long flags;349 350 spin_lock_irqsave(&gnttab_list_lock, flags);351 put_free_entry_locked(ref);352 check_free_callbacks();353 spin_unlock_irqrestore(&gnttab_list_lock, flags);354}355 356static void gnttab_set_free(unsigned int start, unsigned int n)357{358 unsigned int i;359 360 for (i = start; i < start + n - 1; i++)361 gnttab_entry(i) = i + 1;362 363 gnttab_entry(i) = GNTTAB_LIST_END;364 if (!gnttab_free_count) {365 gnttab_free_head = start;366 gnttab_free_tail_ptr = &gnttab_free_head;367 } else {368 gnttab_entry(gnttab_last_free) = start;369 }370 gnttab_free_count += n;371 gnttab_last_free = i;372 373 bitmap_set(gnttab_free_bitmap, start, n);374}375 376/*377 * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2.378 * Introducing a valid entry into the grant table:379 * 1. Write ent->domid.380 * 2. Write ent->frame: Frame to which access is permitted.381 * 3. Write memory barrier (WMB).382 * 4. Write ent->flags, inc. valid type.383 */384static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid,385 unsigned long frame, unsigned flags)386{387 gnttab_shared.v1[ref].domid = domid;388 gnttab_shared.v1[ref].frame = frame;389 wmb();390 gnttab_shared.v1[ref].flags = flags;391}392 393static void gnttab_update_entry_v2(grant_ref_t ref, domid_t domid,394 unsigned long frame, unsigned int flags)395{396 gnttab_shared.v2[ref].hdr.domid = domid;397 gnttab_shared.v2[ref].full_page.frame = frame;398 wmb(); /* Hypervisor concurrent accesses. */399 gnttab_shared.v2[ref].hdr.flags = GTF_permit_access | flags;400}401 402/*403 * Public grant-issuing interface functions404 */405void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,406 unsigned long frame, int readonly)407{408 gnttab_interface->update_entry(ref, domid, frame,409 GTF_permit_access | (readonly ? GTF_readonly : 0));410}411EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);412 413int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,414 int readonly)415{416 int ref;417 418 ref = get_free_entries(1);419 if (unlikely(ref < 0))420 return -ENOSPC;421 422 gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);423 424 return ref;425}426EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);427 428static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref)429{430 u16 *pflags = &gnttab_shared.v1[ref].flags;431 u16 flags;432 433 flags = *pflags;434 do {435 if (flags & (GTF_reading|GTF_writing))436 return 0;437 } while (!sync_try_cmpxchg(pflags, &flags, 0));438 439 return 1;440}441 442static int gnttab_end_foreign_access_ref_v2(grant_ref_t ref)443{444 gnttab_shared.v2[ref].hdr.flags = 0;445 mb(); /* Concurrent access by hypervisor. */446 if (grstatus[ref] & (GTF_reading|GTF_writing)) {447 return 0;448 } else {449 /*450 * The read of grstatus needs to have acquire semantics.451 * On x86, reads already have that, and we just need to452 * protect against compiler reorderings.453 * On other architectures we may need a full barrier.454 */455#ifdef CONFIG_X86456 barrier();457#else458 mb();459#endif460 }461 462 return 1;463}464 465static inline int _gnttab_end_foreign_access_ref(grant_ref_t ref)466{467 return gnttab_interface->end_foreign_access_ref(ref);468}469 470int gnttab_end_foreign_access_ref(grant_ref_t ref)471{472 if (_gnttab_end_foreign_access_ref(ref))473 return 1;474 pr_warn("WARNING: g.e. %#x still in use!\n", ref);475 return 0;476}477EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);478 479static unsigned long gnttab_read_frame_v1(grant_ref_t ref)480{481 return gnttab_shared.v1[ref].frame;482}483 484static unsigned long gnttab_read_frame_v2(grant_ref_t ref)485{486 return gnttab_shared.v2[ref].full_page.frame;487}488 489struct deferred_entry {490 struct list_head list;491 grant_ref_t ref;492 uint16_t warn_delay;493 struct page *page;494};495static LIST_HEAD(deferred_list);496static void gnttab_handle_deferred(struct timer_list *);497static DEFINE_TIMER(deferred_timer, gnttab_handle_deferred);498 499static atomic64_t deferred_count;500static atomic64_t leaked_count;501static unsigned int free_per_iteration = 10;502module_param(free_per_iteration, uint, 0600);503 504static void gnttab_handle_deferred(struct timer_list *unused)505{506 unsigned int nr = READ_ONCE(free_per_iteration);507 const bool ignore_limit = nr == 0;508 struct deferred_entry *first = NULL;509 unsigned long flags;510 size_t freed = 0;511 512 spin_lock_irqsave(&gnttab_list_lock, flags);513 while ((ignore_limit || nr--) && !list_empty(&deferred_list)) {514 struct deferred_entry *entry515 = list_first_entry(&deferred_list,516 struct deferred_entry, list);517 518 if (entry == first)519 break;520 list_del(&entry->list);521 spin_unlock_irqrestore(&gnttab_list_lock, flags);522 if (_gnttab_end_foreign_access_ref(entry->ref)) {523 uint64_t ret = atomic64_dec_return(&deferred_count);524 525 put_free_entry(entry->ref);526 pr_debug("freeing g.e. %#x (pfn %#lx), %llu remaining\n",527 entry->ref, page_to_pfn(entry->page),528 (unsigned long long)ret);529 put_page(entry->page);530 freed++;531 kfree(entry);532 entry = NULL;533 } else {534 if (!--entry->warn_delay)535 pr_info("g.e. %#x still pending\n", entry->ref);536 if (!first)537 first = entry;538 }539 spin_lock_irqsave(&gnttab_list_lock, flags);540 if (entry)541 list_add_tail(&entry->list, &deferred_list);542 }543 if (list_empty(&deferred_list))544 WARN_ON(atomic64_read(&deferred_count));545 else if (!timer_pending(&deferred_timer)) {546 deferred_timer.expires = jiffies + HZ;547 add_timer(&deferred_timer);548 }549 spin_unlock_irqrestore(&gnttab_list_lock, flags);550 pr_debug("Freed %zu references", freed);551}552 553static void gnttab_add_deferred(grant_ref_t ref, struct page *page)554{555 struct deferred_entry *entry;556 gfp_t gfp = (in_atomic() || irqs_disabled()) ? GFP_ATOMIC : GFP_KERNEL;557 uint64_t leaked, deferred;558 559 entry = kmalloc(sizeof(*entry), gfp);560 if (!page) {561 unsigned long gfn = gnttab_interface->read_frame(ref);562 563 page = pfn_to_page(gfn_to_pfn(gfn));564 get_page(page);565 }566 567 if (entry) {568 unsigned long flags;569 570 entry->ref = ref;571 entry->page = page;572 entry->warn_delay = 60;573 spin_lock_irqsave(&gnttab_list_lock, flags);574 list_add_tail(&entry->list, &deferred_list);575 if (!timer_pending(&deferred_timer)) {576 deferred_timer.expires = jiffies + HZ;577 add_timer(&deferred_timer);578 }579 spin_unlock_irqrestore(&gnttab_list_lock, flags);580 deferred = atomic64_inc_return(&deferred_count);581 leaked = atomic64_read(&leaked_count);582 pr_debug("deferring g.e. %#x (pfn %#lx) (total deferred %llu, total leaked %llu)\n",583 ref, page ? page_to_pfn(page) : -1, deferred, leaked);584 } else {585 deferred = atomic64_read(&deferred_count);586 leaked = atomic64_inc_return(&leaked_count);587 pr_warn("leaking g.e. %#x (pfn %#lx) (total deferred %llu, total leaked %llu)\n",588 ref, page ? page_to_pfn(page) : -1, deferred, leaked);589 }590}591 592int gnttab_try_end_foreign_access(grant_ref_t ref)593{594 int ret = _gnttab_end_foreign_access_ref(ref);595 596 if (ret)597 put_free_entry(ref);598 599 return ret;600}601EXPORT_SYMBOL_GPL(gnttab_try_end_foreign_access);602 603void gnttab_end_foreign_access(grant_ref_t ref, struct page *page)604{605 if (gnttab_try_end_foreign_access(ref)) {606 if (page)607 put_page(page);608 } else609 gnttab_add_deferred(ref, page);610}611EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);612 613void gnttab_free_grant_reference(grant_ref_t ref)614{615 put_free_entry(ref);616}617EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);618 619void gnttab_free_grant_references(grant_ref_t head)620{621 grant_ref_t ref;622 unsigned long flags;623 624 spin_lock_irqsave(&gnttab_list_lock, flags);625 while (head != GNTTAB_LIST_END) {626 ref = gnttab_entry(head);627 put_free_entry_locked(head);628 head = ref;629 }630 check_free_callbacks();631 spin_unlock_irqrestore(&gnttab_list_lock, flags);632}633EXPORT_SYMBOL_GPL(gnttab_free_grant_references);634 635void gnttab_free_grant_reference_seq(grant_ref_t head, unsigned int count)636{637 unsigned long flags;638 unsigned int i;639 640 spin_lock_irqsave(&gnttab_list_lock, flags);641 for (i = count; i > 0; i--)642 put_free_entry_locked(head + i - 1);643 check_free_callbacks();644 spin_unlock_irqrestore(&gnttab_list_lock, flags);645}646EXPORT_SYMBOL_GPL(gnttab_free_grant_reference_seq);647 648int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)649{650 int h = get_free_entries(count);651 652 if (h < 0)653 return -ENOSPC;654 655 *head = h;656 657 return 0;658}659EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);660 661int gnttab_alloc_grant_reference_seq(unsigned int count, grant_ref_t *first)662{663 int h;664 665 if (count == 1)666 h = get_free_entries(1);667 else668 h = get_free_entries_seq(count);669 670 if (h < 0)671 return -ENOSPC;672 673 *first = h;674 675 return 0;676}677EXPORT_SYMBOL_GPL(gnttab_alloc_grant_reference_seq);678 679int gnttab_empty_grant_references(const grant_ref_t *private_head)680{681 return (*private_head == GNTTAB_LIST_END);682}683EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);684 685int gnttab_claim_grant_reference(grant_ref_t *private_head)686{687 grant_ref_t g = *private_head;688 if (unlikely(g == GNTTAB_LIST_END))689 return -ENOSPC;690 *private_head = gnttab_entry(g);691 return g;692}693EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);694 695void gnttab_release_grant_reference(grant_ref_t *private_head,696 grant_ref_t release)697{698 gnttab_entry(release) = *private_head;699 *private_head = release;700}701EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);702 703void gnttab_request_free_callback(struct gnttab_free_callback *callback,704 void (*fn)(void *), void *arg, u16 count)705{706 unsigned long flags;707 struct gnttab_free_callback *cb;708 709 spin_lock_irqsave(&gnttab_list_lock, flags);710 711 /* Check if the callback is already on the list */712 cb = gnttab_free_callback_list;713 while (cb) {714 if (cb == callback)715 goto out;716 cb = cb->next;717 }718 719 callback->fn = fn;720 callback->arg = arg;721 callback->count = count;722 callback->next = gnttab_free_callback_list;723 gnttab_free_callback_list = callback;724 check_free_callbacks();725out:726 spin_unlock_irqrestore(&gnttab_list_lock, flags);727}728EXPORT_SYMBOL_GPL(gnttab_request_free_callback);729 730void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)731{732 struct gnttab_free_callback **pcb;733 unsigned long flags;734 735 spin_lock_irqsave(&gnttab_list_lock, flags);736 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {737 if (*pcb == callback) {738 *pcb = callback->next;739 break;740 }741 }742 spin_unlock_irqrestore(&gnttab_list_lock, flags);743}744EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);745 746static unsigned int gnttab_frames(unsigned int frames, unsigned int align)747{748 return (frames * gnttab_interface->grefs_per_grant_frame + align - 1) /749 align;750}751 752static int grow_gnttab_list(unsigned int more_frames)753{754 unsigned int new_nr_grant_frames, extra_entries, i;755 unsigned int nr_glist_frames, new_nr_glist_frames;756 unsigned int grefs_per_frame;757 758 grefs_per_frame = gnttab_interface->grefs_per_grant_frame;759 760 new_nr_grant_frames = nr_grant_frames + more_frames;761 extra_entries = more_frames * grefs_per_frame;762 763 nr_glist_frames = gnttab_frames(nr_grant_frames, RPP);764 new_nr_glist_frames = gnttab_frames(new_nr_grant_frames, RPP);765 for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {766 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);767 if (!gnttab_list[i])768 goto grow_nomem;769 }770 771 gnttab_set_free(gnttab_size, extra_entries);772 773 if (!gnttab_free_tail_ptr)774 gnttab_free_tail_ptr = __gnttab_entry(gnttab_size);775 776 nr_grant_frames = new_nr_grant_frames;777 gnttab_size += extra_entries;778 779 check_free_callbacks();780 781 return 0;782 783grow_nomem:784 while (i-- > nr_glist_frames)785 free_page((unsigned long) gnttab_list[i]);786 return -ENOMEM;787}788 789static unsigned int __max_nr_grant_frames(void)790{791 struct gnttab_query_size query;792 int rc;793 794 query.dom = DOMID_SELF;795 796 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);797 if ((rc < 0) || (query.status != GNTST_okay))798 return 4; /* Legacy max supported number of frames */799 800 return query.max_nr_frames;801}802 803unsigned int gnttab_max_grant_frames(void)804{805 unsigned int xen_max = __max_nr_grant_frames();806 static unsigned int boot_max_nr_grant_frames;807 808 /* First time, initialize it properly. */809 if (!boot_max_nr_grant_frames)810 boot_max_nr_grant_frames = __max_nr_grant_frames();811 812 if (xen_max > boot_max_nr_grant_frames)813 return boot_max_nr_grant_frames;814 return xen_max;815}816EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);817 818int gnttab_setup_auto_xlat_frames(phys_addr_t addr)819{820 xen_pfn_t *pfn;821 unsigned int max_nr_gframes = __max_nr_grant_frames();822 unsigned int i;823 void *vaddr;824 825 if (xen_auto_xlat_grant_frames.count)826 return -EINVAL;827 828 vaddr = memremap(addr, XEN_PAGE_SIZE * max_nr_gframes, MEMREMAP_WB);829 if (vaddr == NULL) {830 pr_warn("Failed to ioremap gnttab share frames (addr=%pa)!\n",831 &addr);832 return -ENOMEM;833 }834 pfn = kcalloc(max_nr_gframes, sizeof(pfn[0]), GFP_KERNEL);835 if (!pfn) {836 memunmap(vaddr);837 return -ENOMEM;838 }839 for (i = 0; i < max_nr_gframes; i++)840 pfn[i] = XEN_PFN_DOWN(addr) + i;841 842 xen_auto_xlat_grant_frames.vaddr = vaddr;843 xen_auto_xlat_grant_frames.pfn = pfn;844 xen_auto_xlat_grant_frames.count = max_nr_gframes;845 846 return 0;847}848EXPORT_SYMBOL_GPL(gnttab_setup_auto_xlat_frames);849 850void gnttab_free_auto_xlat_frames(void)851{852 if (!xen_auto_xlat_grant_frames.count)853 return;854 kfree(xen_auto_xlat_grant_frames.pfn);855 memunmap(xen_auto_xlat_grant_frames.vaddr);856 857 xen_auto_xlat_grant_frames.pfn = NULL;858 xen_auto_xlat_grant_frames.count = 0;859 xen_auto_xlat_grant_frames.vaddr = NULL;860}861EXPORT_SYMBOL_GPL(gnttab_free_auto_xlat_frames);862 863int gnttab_pages_set_private(int nr_pages, struct page **pages)864{865 int i;866 867 for (i = 0; i < nr_pages; i++) {868#if BITS_PER_LONG < 64869 struct xen_page_foreign *foreign;870 871 foreign = kzalloc(sizeof(*foreign), GFP_KERNEL);872 if (!foreign)873 return -ENOMEM;874 875 set_page_private(pages[i], (unsigned long)foreign);876#endif877 SetPagePrivate(pages[i]);878 }879 880 return 0;881}882EXPORT_SYMBOL_GPL(gnttab_pages_set_private);883 884/**885 * gnttab_alloc_pages - alloc pages suitable for grant mapping into886 * @nr_pages: number of pages to alloc887 * @pages: returns the pages888 */889int gnttab_alloc_pages(int nr_pages, struct page **pages)890{891 int ret;892 893 ret = xen_alloc_unpopulated_pages(nr_pages, pages);894 if (ret < 0)895 return ret;896 897 ret = gnttab_pages_set_private(nr_pages, pages);898 if (ret < 0)899 gnttab_free_pages(nr_pages, pages);900 901 return ret;902}903EXPORT_SYMBOL_GPL(gnttab_alloc_pages);904 905#ifdef CONFIG_XEN_UNPOPULATED_ALLOC906static inline void cache_init(struct gnttab_page_cache *cache)907{908 cache->pages = NULL;909}910 911static inline bool cache_empty(struct gnttab_page_cache *cache)912{913 return !cache->pages;914}915 916static inline struct page *cache_deq(struct gnttab_page_cache *cache)917{918 struct page *page;919 920 page = cache->pages;921 cache->pages = page->zone_device_data;922 923 return page;924}925 926static inline void cache_enq(struct gnttab_page_cache *cache, struct page *page)927{928 page->zone_device_data = cache->pages;929 cache->pages = page;930}931#else932static inline void cache_init(struct gnttab_page_cache *cache)933{934 INIT_LIST_HEAD(&cache->pages);935}936 937static inline bool cache_empty(struct gnttab_page_cache *cache)938{939 return list_empty(&cache->pages);940}941 942static inline struct page *cache_deq(struct gnttab_page_cache *cache)943{944 struct page *page;945 946 page = list_first_entry(&cache->pages, struct page, lru);947 list_del(&page->lru);948 949 return page;950}951 952static inline void cache_enq(struct gnttab_page_cache *cache, struct page *page)953{954 list_add(&page->lru, &cache->pages);955}956#endif957 958void gnttab_page_cache_init(struct gnttab_page_cache *cache)959{960 spin_lock_init(&cache->lock);961 cache_init(cache);962 cache->num_pages = 0;963}964EXPORT_SYMBOL_GPL(gnttab_page_cache_init);965 966int gnttab_page_cache_get(struct gnttab_page_cache *cache, struct page **page)967{968 unsigned long flags;969 970 spin_lock_irqsave(&cache->lock, flags);971 972 if (cache_empty(cache)) {973 spin_unlock_irqrestore(&cache->lock, flags);974 return gnttab_alloc_pages(1, page);975 }976 977 page[0] = cache_deq(cache);978 cache->num_pages--;979 980 spin_unlock_irqrestore(&cache->lock, flags);981 982 return 0;983}984EXPORT_SYMBOL_GPL(gnttab_page_cache_get);985 986void gnttab_page_cache_put(struct gnttab_page_cache *cache, struct page **page,987 unsigned int num)988{989 unsigned long flags;990 unsigned int i;991 992 spin_lock_irqsave(&cache->lock, flags);993 994 for (i = 0; i < num; i++)995 cache_enq(cache, page[i]);996 cache->num_pages += num;997 998 spin_unlock_irqrestore(&cache->lock, flags);999}1000EXPORT_SYMBOL_GPL(gnttab_page_cache_put);1001 1002void gnttab_page_cache_shrink(struct gnttab_page_cache *cache, unsigned int num)1003{1004 struct page *page[10];1005 unsigned int i = 0;1006 unsigned long flags;1007 1008 spin_lock_irqsave(&cache->lock, flags);1009 1010 while (cache->num_pages > num) {1011 page[i] = cache_deq(cache);1012 cache->num_pages--;1013 if (++i == ARRAY_SIZE(page)) {1014 spin_unlock_irqrestore(&cache->lock, flags);1015 gnttab_free_pages(i, page);1016 i = 0;1017 spin_lock_irqsave(&cache->lock, flags);1018 }1019 }1020 1021 spin_unlock_irqrestore(&cache->lock, flags);1022 1023 if (i != 0)1024 gnttab_free_pages(i, page);1025}1026EXPORT_SYMBOL_GPL(gnttab_page_cache_shrink);1027 1028void gnttab_pages_clear_private(int nr_pages, struct page **pages)1029{1030 int i;1031 1032 for (i = 0; i < nr_pages; i++) {1033 if (PagePrivate(pages[i])) {1034#if BITS_PER_LONG < 641035 kfree((void *)page_private(pages[i]));1036#endif1037 ClearPagePrivate(pages[i]);1038 }1039 }1040}1041EXPORT_SYMBOL_GPL(gnttab_pages_clear_private);1042 1043/**1044 * gnttab_free_pages - free pages allocated by gnttab_alloc_pages()1045 * @nr_pages: number of pages to free1046 * @pages: the pages1047 */1048void gnttab_free_pages(int nr_pages, struct page **pages)1049{1050 gnttab_pages_clear_private(nr_pages, pages);1051 xen_free_unpopulated_pages(nr_pages, pages);1052}1053EXPORT_SYMBOL_GPL(gnttab_free_pages);1054 1055#ifdef CONFIG_XEN_GRANT_DMA_ALLOC1056/**1057 * gnttab_dma_alloc_pages - alloc DMAable pages suitable for grant mapping into1058 * @args: arguments to the function1059 */1060int gnttab_dma_alloc_pages(struct gnttab_dma_alloc_args *args)1061{1062 unsigned long pfn, start_pfn;1063 size_t size;1064 int i, ret;1065 1066 if (args->nr_pages < 0 || args->nr_pages > (INT_MAX >> PAGE_SHIFT))1067 return -ENOMEM;1068 1069 size = args->nr_pages << PAGE_SHIFT;1070 if (args->coherent)1071 args->vaddr = dma_alloc_coherent(args->dev, size,1072 &args->dev_bus_addr,1073 GFP_KERNEL | __GFP_NOWARN);1074 else1075 args->vaddr = dma_alloc_wc(args->dev, size,1076 &args->dev_bus_addr,1077 GFP_KERNEL | __GFP_NOWARN);1078 if (!args->vaddr) {1079 pr_debug("Failed to allocate DMA buffer of size %zu\n", size);1080 return -ENOMEM;1081 }1082 1083 start_pfn = __phys_to_pfn(args->dev_bus_addr);1084 for (pfn = start_pfn, i = 0; pfn < start_pfn + args->nr_pages;1085 pfn++, i++) {1086 struct page *page = pfn_to_page(pfn);1087 1088 args->pages[i] = page;1089 args->frames[i] = xen_page_to_gfn(page);1090 xenmem_reservation_scrub_page(page);1091 }1092 1093 xenmem_reservation_va_mapping_reset(args->nr_pages, args->pages);1094 1095 ret = xenmem_reservation_decrease(args->nr_pages, args->frames);1096 if (ret != args->nr_pages) {1097 pr_debug("Failed to decrease reservation for DMA buffer\n");1098 ret = -EFAULT;1099 goto fail;1100 }1101 1102 ret = gnttab_pages_set_private(args->nr_pages, args->pages);1103 if (ret < 0)1104 goto fail;1105 1106 return 0;1107 1108fail:1109 gnttab_dma_free_pages(args);1110 return ret;1111}1112EXPORT_SYMBOL_GPL(gnttab_dma_alloc_pages);1113 1114/**1115 * gnttab_dma_free_pages - free DMAable pages1116 * @args: arguments to the function1117 */1118int gnttab_dma_free_pages(struct gnttab_dma_alloc_args *args)1119{1120 size_t size;1121 int i, ret;1122 1123 gnttab_pages_clear_private(args->nr_pages, args->pages);1124 1125 for (i = 0; i < args->nr_pages; i++)1126 args->frames[i] = page_to_xen_pfn(args->pages[i]);1127 1128 ret = xenmem_reservation_increase(args->nr_pages, args->frames);1129 if (ret != args->nr_pages) {1130 pr_debug("Failed to increase reservation for DMA buffer\n");1131 ret = -EFAULT;1132 } else {1133 ret = 0;1134 }1135 1136 xenmem_reservation_va_mapping_update(args->nr_pages, args->pages,1137 args->frames);1138 1139 size = args->nr_pages << PAGE_SHIFT;1140 if (args->coherent)1141 dma_free_coherent(args->dev, size,1142 args->vaddr, args->dev_bus_addr);1143 else1144 dma_free_wc(args->dev, size,1145 args->vaddr, args->dev_bus_addr);1146 return ret;1147}1148EXPORT_SYMBOL_GPL(gnttab_dma_free_pages);1149#endif1150 1151/* Handling of paged out grant targets (GNTST_eagain) */1152#define MAX_DELAY 2561153static inline void1154gnttab_retry_eagain_gop(unsigned int cmd, void *gop, int16_t *status,1155 const char *func)1156{1157 unsigned delay = 1;1158 1159 do {1160 BUG_ON(HYPERVISOR_grant_table_op(cmd, gop, 1));1161 if (*status == GNTST_eagain)1162 msleep(delay++);1163 } while ((*status == GNTST_eagain) && (delay < MAX_DELAY));1164 1165 if (delay >= MAX_DELAY) {1166 pr_err("%s: %s eagain grant\n", func, current->comm);1167 *status = GNTST_bad_page;1168 }1169}1170 1171void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count)1172{1173 struct gnttab_map_grant_ref *op;1174 1175 if (HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, batch, count))1176 BUG();1177 for (op = batch; op < batch + count; op++)1178 if (op->status == GNTST_eagain)1179 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, op,1180 &op->status, __func__);1181}1182EXPORT_SYMBOL_GPL(gnttab_batch_map);1183 1184void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count)1185{1186 struct gnttab_copy *op;1187 1188 if (HYPERVISOR_grant_table_op(GNTTABOP_copy, batch, count))1189 BUG();1190 for (op = batch; op < batch + count; op++)1191 if (op->status == GNTST_eagain)1192 gnttab_retry_eagain_gop(GNTTABOP_copy, op,1193 &op->status, __func__);1194}1195EXPORT_SYMBOL_GPL(gnttab_batch_copy);1196 1197void gnttab_foreach_grant_in_range(struct page *page,1198 unsigned int offset,1199 unsigned int len,1200 xen_grant_fn_t fn,1201 void *data)1202{1203 unsigned int goffset;1204 unsigned int glen;1205 unsigned long xen_pfn;1206 1207 len = min_t(unsigned int, PAGE_SIZE - offset, len);1208 goffset = xen_offset_in_page(offset);1209 1210 xen_pfn = page_to_xen_pfn(page) + XEN_PFN_DOWN(offset);1211 1212 while (len) {1213 glen = min_t(unsigned int, XEN_PAGE_SIZE - goffset, len);1214 fn(pfn_to_gfn(xen_pfn), goffset, glen, data);1215 1216 goffset = 0;1217 xen_pfn++;1218 len -= glen;1219 }1220}1221EXPORT_SYMBOL_GPL(gnttab_foreach_grant_in_range);1222 1223void gnttab_foreach_grant(struct page **pages,1224 unsigned int nr_grefs,1225 xen_grant_fn_t fn,1226 void *data)1227{1228 unsigned int goffset = 0;1229 unsigned long xen_pfn = 0;1230 unsigned int i;1231 1232 for (i = 0; i < nr_grefs; i++) {1233 if ((i % XEN_PFN_PER_PAGE) == 0) {1234 xen_pfn = page_to_xen_pfn(pages[i / XEN_PFN_PER_PAGE]);1235 goffset = 0;1236 }1237 1238 fn(pfn_to_gfn(xen_pfn), goffset, XEN_PAGE_SIZE, data);1239 1240 goffset += XEN_PAGE_SIZE;1241 xen_pfn++;1242 }1243}1244 1245int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,1246 struct gnttab_map_grant_ref *kmap_ops,1247 struct page **pages, unsigned int count)1248{1249 int i, ret;1250 1251 ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count);1252 if (ret)1253 return ret;1254 1255 for (i = 0; i < count; i++) {1256 switch (map_ops[i].status) {1257 case GNTST_okay:1258 {1259 struct xen_page_foreign *foreign;1260 1261 SetPageForeign(pages[i]);1262 foreign = xen_page_foreign(pages[i]);1263 foreign->domid = map_ops[i].dom;1264 foreign->gref = map_ops[i].ref;1265 break;1266 }1267 1268 case GNTST_no_device_space:1269 pr_warn_ratelimited("maptrack limit reached, can't map all guest pages\n");1270 break;1271 1272 case GNTST_eagain:1273 /* Retry eagain maps */1274 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref,1275 map_ops + i,1276 &map_ops[i].status, __func__);1277 /* Test status in next loop iteration. */1278 i--;1279 break;1280 1281 default:1282 break;1283 }1284 }1285 1286 return set_foreign_p2m_mapping(map_ops, kmap_ops, pages, count);1287}1288EXPORT_SYMBOL_GPL(gnttab_map_refs);1289 1290int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,1291 struct gnttab_unmap_grant_ref *kunmap_ops,1292 struct page **pages, unsigned int count)1293{1294 unsigned int i;1295 int ret;1296 1297 ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);1298 if (ret)1299 return ret;1300 1301 for (i = 0; i < count; i++)1302 ClearPageForeign(pages[i]);1303 1304 return clear_foreign_p2m_mapping(unmap_ops, kunmap_ops, pages, count);1305}1306EXPORT_SYMBOL_GPL(gnttab_unmap_refs);1307 1308#define GNTTAB_UNMAP_REFS_DELAY 51309 1310static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item);1311 1312static void gnttab_unmap_work(struct work_struct *work)1313{1314 struct gntab_unmap_queue_data1315 *unmap_data = container_of(work, 1316 struct gntab_unmap_queue_data,1317 gnttab_work.work);1318 if (unmap_data->age != UINT_MAX)1319 unmap_data->age++;1320 __gnttab_unmap_refs_async(unmap_data);1321}1322 1323static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item)1324{1325 int ret;1326 int pc;1327 1328 for (pc = 0; pc < item->count; pc++) {1329 if (page_count(item->pages[pc]) > 1) {1330 unsigned long delay = GNTTAB_UNMAP_REFS_DELAY * (item->age + 1);1331 schedule_delayed_work(&item->gnttab_work,1332 msecs_to_jiffies(delay));1333 return;1334 }1335 }1336 1337 ret = gnttab_unmap_refs(item->unmap_ops, item->kunmap_ops,1338 item->pages, item->count);1339 item->done(ret, item);1340}1341 1342void gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item)1343{1344 INIT_DELAYED_WORK(&item->gnttab_work, gnttab_unmap_work);1345 item->age = 0;1346 1347 __gnttab_unmap_refs_async(item);1348}1349EXPORT_SYMBOL_GPL(gnttab_unmap_refs_async);1350 1351static void unmap_refs_callback(int result,1352 struct gntab_unmap_queue_data *data)1353{1354 struct unmap_refs_callback_data *d = data->data;1355 1356 d->result = result;1357 complete(&d->completion);1358}1359 1360int gnttab_unmap_refs_sync(struct gntab_unmap_queue_data *item)1361{1362 struct unmap_refs_callback_data data;1363 1364 init_completion(&data.completion);1365 item->data = &data;1366 item->done = &unmap_refs_callback;1367 gnttab_unmap_refs_async(item);1368 wait_for_completion(&data.completion);1369 1370 return data.result;1371}1372EXPORT_SYMBOL_GPL(gnttab_unmap_refs_sync);1373 1374static unsigned int nr_status_frames(unsigned int nr_grant_frames)1375{1376 return gnttab_frames(nr_grant_frames, SPP);1377}1378 1379static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes)1380{1381 int rc;1382 1383 rc = arch_gnttab_map_shared(frames, nr_gframes,1384 gnttab_max_grant_frames(),1385 &gnttab_shared.addr);1386 BUG_ON(rc);1387 1388 return 0;1389}1390 1391static void gnttab_unmap_frames_v1(void)1392{1393 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);1394}1395 1396static int gnttab_map_frames_v2(xen_pfn_t *frames, unsigned int nr_gframes)1397{1398 uint64_t *sframes;1399 unsigned int nr_sframes;1400 struct gnttab_get_status_frames getframes;1401 int rc;1402 1403 nr_sframes = nr_status_frames(nr_gframes);1404 1405 /* No need for kzalloc as it is initialized in following hypercall1406 * GNTTABOP_get_status_frames.1407 */1408 sframes = kmalloc_array(nr_sframes, sizeof(uint64_t), GFP_ATOMIC);1409 if (!sframes)1410 return -ENOMEM;1411 1412 getframes.dom = DOMID_SELF;1413 getframes.nr_frames = nr_sframes;1414 set_xen_guest_handle(getframes.frame_list, sframes);1415 1416 rc = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,1417 &getframes, 1);1418 if (rc == -ENOSYS) {1419 kfree(sframes);1420 return -ENOSYS;1421 }1422 1423 BUG_ON(rc || getframes.status);1424 1425 rc = arch_gnttab_map_status(sframes, nr_sframes,1426 nr_status_frames(gnttab_max_grant_frames()),1427 &grstatus);1428 BUG_ON(rc);1429 kfree(sframes);1430 1431 rc = arch_gnttab_map_shared(frames, nr_gframes,1432 gnttab_max_grant_frames(),1433 &gnttab_shared.addr);1434 BUG_ON(rc);1435 1436 return 0;1437}1438 1439static void gnttab_unmap_frames_v2(void)1440{1441 arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);1442 arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));1443}1444 1445static int gnttab_map(unsigned int start_idx, unsigned int end_idx)1446{1447 struct gnttab_setup_table setup;1448 xen_pfn_t *frames;1449 unsigned int nr_gframes = end_idx + 1;1450 int rc;1451 1452 if (xen_feature(XENFEAT_auto_translated_physmap)) {1453 struct xen_add_to_physmap xatp;1454 unsigned int i = end_idx;1455 rc = 0;1456 BUG_ON(xen_auto_xlat_grant_frames.count < nr_gframes);1457 /*1458 * Loop backwards, so that the first hypercall has the largest1459 * index, ensuring that the table will grow only once.1460 */1461 do {1462 xatp.domid = DOMID_SELF;1463 xatp.idx = i;1464 xatp.space = XENMAPSPACE_grant_table;1465 xatp.gpfn = xen_auto_xlat_grant_frames.pfn[i];1466 rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);1467 if (rc != 0) {1468 pr_warn("grant table add_to_physmap failed, err=%d\n",1469 rc);1470 break;1471 }1472 } while (i-- > start_idx);1473 1474 return rc;1475 }1476 1477 /* No need for kzalloc as it is initialized in following hypercall1478 * GNTTABOP_setup_table.1479 */1480 frames = kmalloc_array(nr_gframes, sizeof(unsigned long), GFP_ATOMIC);1481 if (!frames)1482 return -ENOMEM;1483 1484 setup.dom = DOMID_SELF;1485 setup.nr_frames = nr_gframes;1486 set_xen_guest_handle(setup.frame_list, frames);1487 1488 rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);1489 if (rc == -ENOSYS) {1490 kfree(frames);1491 return -ENOSYS;1492 }1493 1494 BUG_ON(rc || setup.status);1495 1496 rc = gnttab_interface->map_frames(frames, nr_gframes);1497 1498 kfree(frames);1499 1500 return rc;1501}1502 1503static const struct gnttab_ops gnttab_v1_ops = {1504 .version = 1,1505 .grefs_per_grant_frame = XEN_PAGE_SIZE /1506 sizeof(struct grant_entry_v1),1507 .map_frames = gnttab_map_frames_v1,1508 .unmap_frames = gnttab_unmap_frames_v1,1509 .update_entry = gnttab_update_entry_v1,1510 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v1,1511 .read_frame = gnttab_read_frame_v1,1512};1513 1514static const struct gnttab_ops gnttab_v2_ops = {1515 .version = 2,1516 .grefs_per_grant_frame = XEN_PAGE_SIZE /1517 sizeof(union grant_entry_v2),1518 .map_frames = gnttab_map_frames_v2,1519 .unmap_frames = gnttab_unmap_frames_v2,1520 .update_entry = gnttab_update_entry_v2,1521 .end_foreign_access_ref = gnttab_end_foreign_access_ref_v2,1522 .read_frame = gnttab_read_frame_v2,1523};1524 1525static bool gnttab_need_v2(void)1526{1527#ifdef CONFIG_X861528 uint32_t base, width;1529 1530 if (xen_pv_domain()) {1531 base = xen_cpuid_base();1532 if (cpuid_eax(base) < 5)1533 return false; /* Information not available, use V1. */1534 width = cpuid_ebx(base + 5) &1535 XEN_CPUID_MACHINE_ADDRESS_WIDTH_MASK;1536 return width > 32 + PAGE_SHIFT;1537 }1538#endif1539 return !!(max_possible_pfn >> 32);1540}1541 1542static void gnttab_request_version(void)1543{1544 long rc;1545 struct gnttab_set_version gsv;1546 1547 if (gnttab_need_v2())1548 gsv.version = 2;1549 else1550 gsv.version = 1;1551 1552 /* Boot parameter overrides automatic selection. */1553 if (xen_gnttab_version >= 1 && xen_gnttab_version <= 2)1554 gsv.version = xen_gnttab_version;1555 1556 rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gsv, 1);1557 if (rc == 0 && gsv.version == 2)1558 gnttab_interface = &gnttab_v2_ops;1559 else1560 gnttab_interface = &gnttab_v1_ops;1561 pr_info("Grant tables using version %d layout\n",1562 gnttab_interface->version);1563}1564 1565static int gnttab_setup(void)1566{1567 unsigned int max_nr_gframes;1568 1569 max_nr_gframes = gnttab_max_grant_frames();1570 if (max_nr_gframes < nr_grant_frames)1571 return -ENOSYS;1572 1573 if (xen_feature(XENFEAT_auto_translated_physmap) && gnttab_shared.addr == NULL) {1574 gnttab_shared.addr = xen_auto_xlat_grant_frames.vaddr;1575 if (gnttab_shared.addr == NULL) {1576 pr_warn("gnttab share frames is not mapped!\n");1577 return -ENOMEM;1578 }1579 }1580 return gnttab_map(0, nr_grant_frames - 1);1581}1582 1583int gnttab_resume(void)1584{1585 gnttab_request_version();1586 return gnttab_setup();1587}1588 1589int gnttab_suspend(void)1590{1591 if (!xen_feature(XENFEAT_auto_translated_physmap))1592 gnttab_interface->unmap_frames();1593 return 0;1594}1595 1596static int gnttab_expand(unsigned int req_entries)1597{1598 int rc;1599 unsigned int cur, extra;1600 1601 cur = nr_grant_frames;1602 extra = ((req_entries + gnttab_interface->grefs_per_grant_frame - 1) /1603 gnttab_interface->grefs_per_grant_frame);1604 if (cur + extra > gnttab_max_grant_frames()) {1605 pr_warn_ratelimited("xen/grant-table: max_grant_frames reached"1606 " cur=%u extra=%u limit=%u"1607 " gnttab_free_count=%u req_entries=%u\n",1608 cur, extra, gnttab_max_grant_frames(),1609 gnttab_free_count, req_entries);1610 return -ENOSPC;1611 }1612 1613 rc = gnttab_map(cur, cur + extra - 1);1614 if (rc == 0)1615 rc = grow_gnttab_list(extra);1616 1617 return rc;1618}1619 1620int gnttab_init(void)1621{1622 int i;1623 unsigned long max_nr_grant_frames, max_nr_grefs;1624 unsigned int max_nr_glist_frames, nr_glist_frames;1625 int ret;1626 1627 gnttab_request_version();1628 max_nr_grant_frames = gnttab_max_grant_frames();1629 max_nr_grefs = max_nr_grant_frames *1630 gnttab_interface->grefs_per_grant_frame;1631 nr_grant_frames = 1;1632 1633 /* Determine the maximum number of frames required for the1634 * grant reference free list on the current hypervisor.1635 */1636 max_nr_glist_frames = max_nr_grefs / RPP;1637 1638 gnttab_list = kmalloc_array(max_nr_glist_frames,1639 sizeof(grant_ref_t *),1640 GFP_KERNEL);1641 if (gnttab_list == NULL)1642 return -ENOMEM;1643 1644 nr_glist_frames = gnttab_frames(nr_grant_frames, RPP);1645 for (i = 0; i < nr_glist_frames; i++) {1646 gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);1647 if (gnttab_list[i] == NULL) {1648 ret = -ENOMEM;1649 goto ini_nomem;1650 }1651 }1652 1653 gnttab_free_bitmap = bitmap_zalloc(max_nr_grefs, GFP_KERNEL);1654 if (!gnttab_free_bitmap) {1655 ret = -ENOMEM;1656 goto ini_nomem;1657 }1658 1659 ret = arch_gnttab_init(max_nr_grant_frames,1660 nr_status_frames(max_nr_grant_frames));1661 if (ret < 0)1662 goto ini_nomem;1663 1664 if (gnttab_setup() < 0) {1665 ret = -ENODEV;1666 goto ini_nomem;1667 }1668 1669 gnttab_size = nr_grant_frames * gnttab_interface->grefs_per_grant_frame;1670 1671 gnttab_set_free(GNTTAB_NR_RESERVED_ENTRIES,1672 gnttab_size - GNTTAB_NR_RESERVED_ENTRIES);1673 1674 printk("Grant table initialized\n");1675 return 0;1676 1677 ini_nomem:1678 for (i--; i >= 0; i--)1679 free_page((unsigned long)gnttab_list[i]);1680 kfree(gnttab_list);1681 bitmap_free(gnttab_free_bitmap);1682 return ret;1683}1684EXPORT_SYMBOL_GPL(gnttab_init);1685 1686static int __gnttab_init(void)1687{1688 if (!xen_domain())1689 return -ENODEV;1690 1691 /* Delay grant-table initialization in the PV on HVM case */1692 if (xen_hvm_domain() && !xen_pvh_domain())1693 return 0;1694 1695 return gnttab_init();1696}1697/* Starts after core_initcall so that xen_pvh_gnttab_setup can be called1698 * beforehand to initialize xen_auto_xlat_grant_frames. */1699core_initcall_sync(__gnttab_init);1700