230 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2 3#include <linux/efi.h>4#include <linux/memblock.h>5#include <linux/spinlock.h>6#include <linux/crash_dump.h>7#include <linux/nmi.h>8#include <asm/unaccepted_memory.h>9 10/* Protects unaccepted memory bitmap and accepting_list */11static DEFINE_SPINLOCK(unaccepted_memory_lock);12 13struct accept_range {14 struct list_head list;15 unsigned long start;16 unsigned long end;17};18 19static LIST_HEAD(accepting_list);20 21/*22 * accept_memory() -- Consult bitmap and accept the memory if needed.23 *24 * Only memory that is explicitly marked as unaccepted in the bitmap requires25 * an action. All the remaining memory is implicitly accepted and doesn't need26 * acceptance.27 *28 * No need to accept:29 * - anything if the system has no unaccepted table;30 * - memory that is below phys_base;31 * - memory that is above the memory that addressable by the bitmap;32 */33void accept_memory(phys_addr_t start, unsigned long size)34{35 struct efi_unaccepted_memory *unaccepted;36 unsigned long range_start, range_end;37 struct accept_range range, *entry;38 phys_addr_t end = start + size;39 unsigned long flags;40 u64 unit_size;41 42 unaccepted = efi_get_unaccepted_table();43 if (!unaccepted)44 return;45 46 unit_size = unaccepted->unit_size;47 48 /*49 * Only care for the part of the range that is represented50 * in the bitmap.51 */52 if (start < unaccepted->phys_base)53 start = unaccepted->phys_base;54 if (end < unaccepted->phys_base)55 return;56 57 /* Translate to offsets from the beginning of the bitmap */58 start -= unaccepted->phys_base;59 end -= unaccepted->phys_base;60 61 /*62 * load_unaligned_zeropad() can lead to unwanted loads across page63 * boundaries. The unwanted loads are typically harmless. But, they64 * might be made to totally unrelated or even unmapped memory.65 * load_unaligned_zeropad() relies on exception fixup (#PF, #GP and now66 * #VE) to recover from these unwanted loads.67 *68 * But, this approach does not work for unaccepted memory. For TDX, a69 * load from unaccepted memory will not lead to a recoverable exception70 * within the guest. The guest will exit to the VMM where the only71 * recourse is to terminate the guest.72 *73 * There are two parts to fix this issue and comprehensively avoid74 * access to unaccepted memory. Together these ensure that an extra75 * "guard" page is accepted in addition to the memory that needs to be76 * used:77 *78 * 1. Implicitly extend the range_contains_unaccepted_memory(start, size)79 * checks up to the next unit_size if 'start+size' is aligned on a80 * unit_size boundary.81 *82 * 2. Implicitly extend accept_memory(start, size) to the next unit_size83 * if 'size+end' is aligned on a unit_size boundary. (immediately84 * following this comment)85 */86 if (!(end % unit_size))87 end += unit_size;88 89 /* Make sure not to overrun the bitmap */90 if (end > unaccepted->size * unit_size * BITS_PER_BYTE)91 end = unaccepted->size * unit_size * BITS_PER_BYTE;92 93 range.start = start / unit_size;94 range.end = DIV_ROUND_UP(end, unit_size);95retry:96 spin_lock_irqsave(&unaccepted_memory_lock, flags);97 98 /*99 * Check if anybody works on accepting the same range of the memory.100 *101 * The check is done with unit_size granularity. It is crucial to catch102 * all accept requests to the same unit_size block, even if they don't103 * overlap on physical address level.104 */105 list_for_each_entry(entry, &accepting_list, list) {106 if (entry->end <= range.start)107 continue;108 if (entry->start >= range.end)109 continue;110 111 /*112 * Somebody else accepting the range. Or at least part of it.113 *114 * Drop the lock and retry until it is complete.115 */116 spin_unlock_irqrestore(&unaccepted_memory_lock, flags);117 goto retry;118 }119 120 /*121 * Register that the range is about to be accepted.122 * Make sure nobody else will accept it.123 */124 list_add(&range.list, &accepting_list);125 126 range_start = range.start;127 for_each_set_bitrange_from(range_start, range_end, unaccepted->bitmap,128 range.end) {129 unsigned long phys_start, phys_end;130 unsigned long len = range_end - range_start;131 132 phys_start = range_start * unit_size + unaccepted->phys_base;133 phys_end = range_end * unit_size + unaccepted->phys_base;134 135 /*136 * Keep interrupts disabled until the accept operation is137 * complete in order to prevent deadlocks.138 *139 * Enabling interrupts before calling arch_accept_memory()140 * creates an opportunity for an interrupt handler to request141 * acceptance for the same memory. The handler will continuously142 * spin with interrupts disabled, preventing other task from143 * making progress with the acceptance process.144 */145 spin_unlock(&unaccepted_memory_lock);146 147 arch_accept_memory(phys_start, phys_end);148 149 spin_lock(&unaccepted_memory_lock);150 bitmap_clear(unaccepted->bitmap, range_start, len);151 }152 153 list_del(&range.list);154 155 touch_softlockup_watchdog();156 157 spin_unlock_irqrestore(&unaccepted_memory_lock, flags);158}159 160bool range_contains_unaccepted_memory(phys_addr_t start, unsigned long size)161{162 struct efi_unaccepted_memory *unaccepted;163 phys_addr_t end = start + size;164 unsigned long flags;165 bool ret = false;166 u64 unit_size;167 168 unaccepted = efi_get_unaccepted_table();169 if (!unaccepted)170 return false;171 172 unit_size = unaccepted->unit_size;173 174 /*175 * Only care for the part of the range that is represented176 * in the bitmap.177 */178 if (start < unaccepted->phys_base)179 start = unaccepted->phys_base;180 if (end < unaccepted->phys_base)181 return false;182 183 /* Translate to offsets from the beginning of the bitmap */184 start -= unaccepted->phys_base;185 end -= unaccepted->phys_base;186 187 /*188 * Also consider the unaccepted state of the *next* page. See fix #1 in189 * the comment on load_unaligned_zeropad() in accept_memory().190 */191 if (!(end % unit_size))192 end += unit_size;193 194 /* Make sure not to overrun the bitmap */195 if (end > unaccepted->size * unit_size * BITS_PER_BYTE)196 end = unaccepted->size * unit_size * BITS_PER_BYTE;197 198 spin_lock_irqsave(&unaccepted_memory_lock, flags);199 while (start < end) {200 if (test_bit(start / unit_size, unaccepted->bitmap)) {201 ret = true;202 break;203 }204 205 start += unit_size;206 }207 spin_unlock_irqrestore(&unaccepted_memory_lock, flags);208 209 return ret;210}211 212#ifdef CONFIG_PROC_VMCORE213static bool unaccepted_memory_vmcore_pfn_is_ram(struct vmcore_cb *cb,214 unsigned long pfn)215{216 return !pfn_is_unaccepted_memory(pfn);217}218 219static struct vmcore_cb vmcore_cb = {220 .pfn_is_ram = unaccepted_memory_vmcore_pfn_is_ram,221};222 223static int __init unaccepted_memory_init_kdump(void)224{225 register_vmcore_cb(&vmcore_cb);226 return 0;227}228core_initcall(unaccepted_memory_init_kdump);229#endif /* CONFIG_PROC_VMCORE */230