502 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (c) 2009, Microsoft Corporation.4 *5 * Authors:6 * Haiyang Zhang <haiyangz@microsoft.com>7 * Hank Janssen <hjanssen@microsoft.com>8 */9#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt10 11#include <linux/io.h>12#include <linux/kernel.h>13#include <linux/mm.h>14#include <linux/slab.h>15#include <linux/vmalloc.h>16#include <linux/hyperv.h>17#include <linux/random.h>18#include <linux/clockchips.h>19#include <linux/delay.h>20#include <linux/interrupt.h>21#include <clocksource/hyperv_timer.h>22#include <asm/mshyperv.h>23#include <linux/set_memory.h>24#include "hyperv_vmbus.h"25 26/* The one and only */27struct hv_context hv_context;28 29/*30 * hv_init - Main initialization routine.31 *32 * This routine must be called before any other routines in here are called33 */34int hv_init(void)35{36 hv_context.cpu_context = alloc_percpu(struct hv_per_cpu_context);37 if (!hv_context.cpu_context)38 return -ENOMEM;39 return 0;40}41 42/*43 * hv_post_message - Post a message using the hypervisor message IPC.44 *45 * This involves a hypercall.46 */47int hv_post_message(union hv_connection_id connection_id,48 enum hv_message_type message_type,49 void *payload, size_t payload_size)50{51 struct hv_input_post_message *aligned_msg;52 unsigned long flags;53 u64 status;54 55 if (payload_size > HV_MESSAGE_PAYLOAD_BYTE_COUNT)56 return -EMSGSIZE;57 58 local_irq_save(flags);59 60 /*61 * A TDX VM with the paravisor must use the decrypted post_msg_page: see62 * the comment in struct hv_per_cpu_context. A SNP VM with the paravisor63 * can use the encrypted hyperv_pcpu_input_arg because it copies the64 * input into the GHCB page, which has been decrypted by the paravisor.65 */66 if (hv_isolation_type_tdx() && ms_hyperv.paravisor_present)67 aligned_msg = this_cpu_ptr(hv_context.cpu_context)->post_msg_page;68 else69 aligned_msg = *this_cpu_ptr(hyperv_pcpu_input_arg);70 71 aligned_msg->connectionid = connection_id;72 aligned_msg->reserved = 0;73 aligned_msg->message_type = message_type;74 aligned_msg->payload_size = payload_size;75 memcpy((void *)aligned_msg->payload, payload, payload_size);76 77 if (ms_hyperv.paravisor_present) {78 if (hv_isolation_type_tdx())79 status = hv_tdx_hypercall(HVCALL_POST_MESSAGE,80 virt_to_phys(aligned_msg), 0);81 else if (hv_isolation_type_snp())82 status = hv_ghcb_hypercall(HVCALL_POST_MESSAGE,83 aligned_msg, NULL,84 sizeof(*aligned_msg));85 else86 status = HV_STATUS_INVALID_PARAMETER;87 } else {88 status = hv_do_hypercall(HVCALL_POST_MESSAGE,89 aligned_msg, NULL);90 }91 92 local_irq_restore(flags);93 94 return hv_result(status);95}96 97int hv_synic_alloc(void)98{99 int cpu, ret = -ENOMEM;100 struct hv_per_cpu_context *hv_cpu;101 102 /*103 * First, zero all per-cpu memory areas so hv_synic_free() can104 * detect what memory has been allocated and cleanup properly105 * after any failures.106 */107 for_each_present_cpu(cpu) {108 hv_cpu = per_cpu_ptr(hv_context.cpu_context, cpu);109 memset(hv_cpu, 0, sizeof(*hv_cpu));110 }111 112 hv_context.hv_numa_map = kcalloc(nr_node_ids, sizeof(struct cpumask),113 GFP_KERNEL);114 if (!hv_context.hv_numa_map) {115 pr_err("Unable to allocate NUMA map\n");116 goto err;117 }118 119 for_each_present_cpu(cpu) {120 hv_cpu = per_cpu_ptr(hv_context.cpu_context, cpu);121 122 tasklet_init(&hv_cpu->msg_dpc,123 vmbus_on_msg_dpc, (unsigned long)hv_cpu);124 125 if (ms_hyperv.paravisor_present && hv_isolation_type_tdx()) {126 hv_cpu->post_msg_page = (void *)get_zeroed_page(GFP_ATOMIC);127 if (!hv_cpu->post_msg_page) {128 pr_err("Unable to allocate post msg page\n");129 goto err;130 }131 132 ret = set_memory_decrypted((unsigned long)hv_cpu->post_msg_page, 1);133 if (ret) {134 pr_err("Failed to decrypt post msg page: %d\n", ret);135 /* Just leak the page, as it's unsafe to free the page. */136 hv_cpu->post_msg_page = NULL;137 goto err;138 }139 140 memset(hv_cpu->post_msg_page, 0, PAGE_SIZE);141 }142 143 /*144 * Synic message and event pages are allocated by paravisor.145 * Skip these pages allocation here.146 */147 if (!ms_hyperv.paravisor_present && !hv_root_partition) {148 hv_cpu->synic_message_page =149 (void *)get_zeroed_page(GFP_ATOMIC);150 if (!hv_cpu->synic_message_page) {151 pr_err("Unable to allocate SYNIC message page\n");152 goto err;153 }154 155 hv_cpu->synic_event_page =156 (void *)get_zeroed_page(GFP_ATOMIC);157 if (!hv_cpu->synic_event_page) {158 pr_err("Unable to allocate SYNIC event page\n");159 160 free_page((unsigned long)hv_cpu->synic_message_page);161 hv_cpu->synic_message_page = NULL;162 goto err;163 }164 }165 166 if (!ms_hyperv.paravisor_present &&167 (hv_isolation_type_snp() || hv_isolation_type_tdx())) {168 ret = set_memory_decrypted((unsigned long)169 hv_cpu->synic_message_page, 1);170 if (ret) {171 pr_err("Failed to decrypt SYNIC msg page: %d\n", ret);172 hv_cpu->synic_message_page = NULL;173 174 /*175 * Free the event page here so that hv_synic_free()176 * won't later try to re-encrypt it.177 */178 free_page((unsigned long)hv_cpu->synic_event_page);179 hv_cpu->synic_event_page = NULL;180 goto err;181 }182 183 ret = set_memory_decrypted((unsigned long)184 hv_cpu->synic_event_page, 1);185 if (ret) {186 pr_err("Failed to decrypt SYNIC event page: %d\n", ret);187 hv_cpu->synic_event_page = NULL;188 goto err;189 }190 191 memset(hv_cpu->synic_message_page, 0, PAGE_SIZE);192 memset(hv_cpu->synic_event_page, 0, PAGE_SIZE);193 }194 }195 196 return 0;197 198err:199 /*200 * Any memory allocations that succeeded will be freed when201 * the caller cleans up by calling hv_synic_free()202 */203 return ret;204}205 206void hv_synic_free(void)207{208 int cpu, ret;209 210 for_each_present_cpu(cpu) {211 struct hv_per_cpu_context *hv_cpu =212 per_cpu_ptr(hv_context.cpu_context, cpu);213 214 /* It's better to leak the page if the encryption fails. */215 if (ms_hyperv.paravisor_present && hv_isolation_type_tdx()) {216 if (hv_cpu->post_msg_page) {217 ret = set_memory_encrypted((unsigned long)218 hv_cpu->post_msg_page, 1);219 if (ret) {220 pr_err("Failed to encrypt post msg page: %d\n", ret);221 hv_cpu->post_msg_page = NULL;222 }223 }224 }225 226 if (!ms_hyperv.paravisor_present &&227 (hv_isolation_type_snp() || hv_isolation_type_tdx())) {228 if (hv_cpu->synic_message_page) {229 ret = set_memory_encrypted((unsigned long)230 hv_cpu->synic_message_page, 1);231 if (ret) {232 pr_err("Failed to encrypt SYNIC msg page: %d\n", ret);233 hv_cpu->synic_message_page = NULL;234 }235 }236 237 if (hv_cpu->synic_event_page) {238 ret = set_memory_encrypted((unsigned long)239 hv_cpu->synic_event_page, 1);240 if (ret) {241 pr_err("Failed to encrypt SYNIC event page: %d\n", ret);242 hv_cpu->synic_event_page = NULL;243 }244 }245 }246 247 free_page((unsigned long)hv_cpu->post_msg_page);248 free_page((unsigned long)hv_cpu->synic_event_page);249 free_page((unsigned long)hv_cpu->synic_message_page);250 }251 252 kfree(hv_context.hv_numa_map);253}254 255/*256 * hv_synic_init - Initialize the Synthetic Interrupt Controller.257 *258 * If it is already initialized by another entity (ie x2v shim), we need to259 * retrieve the initialized message and event pages. Otherwise, we create and260 * initialize the message and event pages.261 */262void hv_synic_enable_regs(unsigned int cpu)263{264 struct hv_per_cpu_context *hv_cpu =265 per_cpu_ptr(hv_context.cpu_context, cpu);266 union hv_synic_simp simp;267 union hv_synic_siefp siefp;268 union hv_synic_sint shared_sint;269 union hv_synic_scontrol sctrl;270 271 /* Setup the Synic's message page */272 simp.as_uint64 = hv_get_msr(HV_MSR_SIMP);273 simp.simp_enabled = 1;274 275 if (ms_hyperv.paravisor_present || hv_root_partition) {276 /* Mask out vTOM bit. ioremap_cache() maps decrypted */277 u64 base = (simp.base_simp_gpa << HV_HYP_PAGE_SHIFT) &278 ~ms_hyperv.shared_gpa_boundary;279 hv_cpu->synic_message_page =280 (void *)ioremap_cache(base, HV_HYP_PAGE_SIZE);281 if (!hv_cpu->synic_message_page)282 pr_err("Fail to map synic message page.\n");283 } else {284 simp.base_simp_gpa = virt_to_phys(hv_cpu->synic_message_page)285 >> HV_HYP_PAGE_SHIFT;286 }287 288 hv_set_msr(HV_MSR_SIMP, simp.as_uint64);289 290 /* Setup the Synic's event page */291 siefp.as_uint64 = hv_get_msr(HV_MSR_SIEFP);292 siefp.siefp_enabled = 1;293 294 if (ms_hyperv.paravisor_present || hv_root_partition) {295 /* Mask out vTOM bit. ioremap_cache() maps decrypted */296 u64 base = (siefp.base_siefp_gpa << HV_HYP_PAGE_SHIFT) &297 ~ms_hyperv.shared_gpa_boundary;298 hv_cpu->synic_event_page =299 (void *)ioremap_cache(base, HV_HYP_PAGE_SIZE);300 if (!hv_cpu->synic_event_page)301 pr_err("Fail to map synic event page.\n");302 } else {303 siefp.base_siefp_gpa = virt_to_phys(hv_cpu->synic_event_page)304 >> HV_HYP_PAGE_SHIFT;305 }306 307 hv_set_msr(HV_MSR_SIEFP, siefp.as_uint64);308 309 /* Setup the shared SINT. */310 if (vmbus_irq != -1)311 enable_percpu_irq(vmbus_irq, 0);312 shared_sint.as_uint64 = hv_get_msr(HV_MSR_SINT0 + VMBUS_MESSAGE_SINT);313 314 shared_sint.vector = vmbus_interrupt;315 shared_sint.masked = false;316 317 /*318 * On architectures where Hyper-V doesn't support AEOI (e.g., ARM64),319 * it doesn't provide a recommendation flag and AEOI must be disabled.320 */321#ifdef HV_DEPRECATING_AEOI_RECOMMENDED322 shared_sint.auto_eoi =323 !(ms_hyperv.hints & HV_DEPRECATING_AEOI_RECOMMENDED);324#else325 shared_sint.auto_eoi = 0;326#endif327 hv_set_msr(HV_MSR_SINT0 + VMBUS_MESSAGE_SINT, shared_sint.as_uint64);328 329 /* Enable the global synic bit */330 sctrl.as_uint64 = hv_get_msr(HV_MSR_SCONTROL);331 sctrl.enable = 1;332 333 hv_set_msr(HV_MSR_SCONTROL, sctrl.as_uint64);334}335 336int hv_synic_init(unsigned int cpu)337{338 hv_synic_enable_regs(cpu);339 340 hv_stimer_legacy_init(cpu, VMBUS_MESSAGE_SINT);341 342 return 0;343}344 345void hv_synic_disable_regs(unsigned int cpu)346{347 struct hv_per_cpu_context *hv_cpu =348 per_cpu_ptr(hv_context.cpu_context, cpu);349 union hv_synic_sint shared_sint;350 union hv_synic_simp simp;351 union hv_synic_siefp siefp;352 union hv_synic_scontrol sctrl;353 354 shared_sint.as_uint64 = hv_get_msr(HV_MSR_SINT0 + VMBUS_MESSAGE_SINT);355 356 shared_sint.masked = 1;357 358 /* Need to correctly cleanup in the case of SMP!!! */359 /* Disable the interrupt */360 hv_set_msr(HV_MSR_SINT0 + VMBUS_MESSAGE_SINT, shared_sint.as_uint64);361 362 simp.as_uint64 = hv_get_msr(HV_MSR_SIMP);363 /*364 * In Isolation VM, sim and sief pages are allocated by365 * paravisor. These pages also will be used by kdump366 * kernel. So just reset enable bit here and keep page367 * addresses.368 */369 simp.simp_enabled = 0;370 if (ms_hyperv.paravisor_present || hv_root_partition) {371 iounmap(hv_cpu->synic_message_page);372 hv_cpu->synic_message_page = NULL;373 } else {374 simp.base_simp_gpa = 0;375 }376 377 hv_set_msr(HV_MSR_SIMP, simp.as_uint64);378 379 siefp.as_uint64 = hv_get_msr(HV_MSR_SIEFP);380 siefp.siefp_enabled = 0;381 382 if (ms_hyperv.paravisor_present || hv_root_partition) {383 iounmap(hv_cpu->synic_event_page);384 hv_cpu->synic_event_page = NULL;385 } else {386 siefp.base_siefp_gpa = 0;387 }388 389 hv_set_msr(HV_MSR_SIEFP, siefp.as_uint64);390 391 /* Disable the global synic bit */392 sctrl.as_uint64 = hv_get_msr(HV_MSR_SCONTROL);393 sctrl.enable = 0;394 hv_set_msr(HV_MSR_SCONTROL, sctrl.as_uint64);395 396 if (vmbus_irq != -1)397 disable_percpu_irq(vmbus_irq);398}399 400#define HV_MAX_TRIES 3401/*402 * Scan the event flags page of 'this' CPU looking for any bit that is set. If we find one403 * bit set, then wait for a few milliseconds. Repeat these steps for a maximum of 3 times.404 * Return 'true', if there is still any set bit after this operation; 'false', otherwise.405 *406 * If a bit is set, that means there is a pending channel interrupt. The expectation is407 * that the normal interrupt handling mechanism will find and process the channel interrupt408 * "very soon", and in the process clear the bit.409 */410static bool hv_synic_event_pending(void)411{412 struct hv_per_cpu_context *hv_cpu = this_cpu_ptr(hv_context.cpu_context);413 union hv_synic_event_flags *event =414 (union hv_synic_event_flags *)hv_cpu->synic_event_page + VMBUS_MESSAGE_SINT;415 unsigned long *recv_int_page = event->flags; /* assumes VMBus version >= VERSION_WIN8 */416 bool pending;417 u32 relid;418 int tries = 0;419 420retry:421 pending = false;422 for_each_set_bit(relid, recv_int_page, HV_EVENT_FLAGS_COUNT) {423 /* Special case - VMBus channel protocol messages */424 if (relid == 0)425 continue;426 pending = true;427 break;428 }429 if (pending && tries++ < HV_MAX_TRIES) {430 usleep_range(10000, 20000);431 goto retry;432 }433 return pending;434}435 436/*437 * hv_synic_cleanup - Cleanup routine for hv_synic_init().438 */439int hv_synic_cleanup(unsigned int cpu)440{441 struct vmbus_channel *channel, *sc;442 bool channel_found = false;443 444 if (vmbus_connection.conn_state != CONNECTED)445 goto always_cleanup;446 447 /*448 * Hyper-V does not provide a way to change the connect CPU once449 * it is set; we must prevent the connect CPU from going offline450 * while the VM is running normally. But in the panic or kexec()451 * path where the vmbus is already disconnected, the CPU must be452 * allowed to shut down.453 */454 if (cpu == VMBUS_CONNECT_CPU)455 return -EBUSY;456 457 /*458 * Search for channels which are bound to the CPU we're about to459 * cleanup. In case we find one and vmbus is still connected, we460 * fail; this will effectively prevent CPU offlining.461 *462 * TODO: Re-bind the channels to different CPUs.463 */464 mutex_lock(&vmbus_connection.channel_mutex);465 list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) {466 if (channel->target_cpu == cpu) {467 channel_found = true;468 break;469 }470 list_for_each_entry(sc, &channel->sc_list, sc_list) {471 if (sc->target_cpu == cpu) {472 channel_found = true;473 break;474 }475 }476 if (channel_found)477 break;478 }479 mutex_unlock(&vmbus_connection.channel_mutex);480 481 if (channel_found)482 return -EBUSY;483 484 /*485 * channel_found == false means that any channels that were previously486 * assigned to the CPU have been reassigned elsewhere with a call of487 * vmbus_send_modifychannel(). Scan the event flags page looking for488 * bits that are set and waiting with a timeout for vmbus_chan_sched()489 * to process such bits. If bits are still set after this operation490 * and VMBus is connected, fail the CPU offlining operation.491 */492 if (vmbus_proto_version >= VERSION_WIN10_V4_1 && hv_synic_event_pending())493 return -EBUSY;494 495always_cleanup:496 hv_stimer_legacy_cleanup(cpu);497 498 hv_synic_disable_regs(cpu);499 500 return 0;501}502