1629 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (c) 2009, Microsoft Corporation.4 *5 * Authors:6 * Haiyang Zhang <haiyangz@microsoft.com>7 * Hank Janssen <hjanssen@microsoft.com>8 */9#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt10 11#include <linux/kernel.h>12#include <linux/interrupt.h>13#include <linux/sched.h>14#include <linux/wait.h>15#include <linux/mm.h>16#include <linux/slab.h>17#include <linux/list.h>18#include <linux/module.h>19#include <linux/completion.h>20#include <linux/delay.h>21#include <linux/cpu.h>22#include <linux/hyperv.h>23#include <asm/mshyperv.h>24#include <linux/sched/isolation.h>25 26#include "hyperv_vmbus.h"27 28static void init_vp_index(struct vmbus_channel *channel);29 30const struct vmbus_device vmbus_devs[] = {31 /* IDE */32 { .dev_type = HV_IDE,33 HV_IDE_GUID,34 .perf_device = true,35 .allowed_in_isolated = false,36 },37 38 /* SCSI */39 { .dev_type = HV_SCSI,40 HV_SCSI_GUID,41 .perf_device = true,42 .allowed_in_isolated = true,43 },44 45 /* Fibre Channel */46 { .dev_type = HV_FC,47 HV_SYNTHFC_GUID,48 .perf_device = true,49 .allowed_in_isolated = false,50 },51 52 /* Synthetic NIC */53 { .dev_type = HV_NIC,54 HV_NIC_GUID,55 .perf_device = true,56 .allowed_in_isolated = true,57 },58 59 /* Network Direct */60 { .dev_type = HV_ND,61 HV_ND_GUID,62 .perf_device = true,63 .allowed_in_isolated = false,64 },65 66 /* PCIE */67 { .dev_type = HV_PCIE,68 HV_PCIE_GUID,69 .perf_device = false,70 .allowed_in_isolated = true,71 },72 73 /* Synthetic Frame Buffer */74 { .dev_type = HV_FB,75 HV_SYNTHVID_GUID,76 .perf_device = false,77 .allowed_in_isolated = false,78 },79 80 /* Synthetic Keyboard */81 { .dev_type = HV_KBD,82 HV_KBD_GUID,83 .perf_device = false,84 .allowed_in_isolated = false,85 },86 87 /* Synthetic MOUSE */88 { .dev_type = HV_MOUSE,89 HV_MOUSE_GUID,90 .perf_device = false,91 .allowed_in_isolated = false,92 },93 94 /* KVP */95 { .dev_type = HV_KVP,96 HV_KVP_GUID,97 .perf_device = false,98 .allowed_in_isolated = false,99 },100 101 /* Time Synch */102 { .dev_type = HV_TS,103 HV_TS_GUID,104 .perf_device = false,105 .allowed_in_isolated = true,106 },107 108 /* Heartbeat */109 { .dev_type = HV_HB,110 HV_HEART_BEAT_GUID,111 .perf_device = false,112 .allowed_in_isolated = true,113 },114 115 /* Shutdown */116 { .dev_type = HV_SHUTDOWN,117 HV_SHUTDOWN_GUID,118 .perf_device = false,119 .allowed_in_isolated = true,120 },121 122 /* File copy */123 /* fcopy always uses 16KB ring buffer size and is working well for last many years */124 { .pref_ring_size = 0x4000,125 .dev_type = HV_FCOPY,126 HV_FCOPY_GUID,127 .perf_device = false,128 .allowed_in_isolated = false,129 },130 131 /* Backup */132 { .dev_type = HV_BACKUP,133 HV_VSS_GUID,134 .perf_device = false,135 .allowed_in_isolated = false,136 },137 138 /* Dynamic Memory */139 { .dev_type = HV_DM,140 HV_DM_GUID,141 .perf_device = false,142 .allowed_in_isolated = false,143 },144 145 /*146 * Unknown GUID147 * 64 KB ring buffer + 4 KB header should be sufficient size for any Hyper-V device apart148 * from HV_NIC and HV_SCSI. This case avoid the fallback for unknown devices to allocate149 * much bigger (2 MB) of ring size.150 */151 { .pref_ring_size = 0x11000,152 .dev_type = HV_UNKNOWN,153 .perf_device = false,154 .allowed_in_isolated = false,155 },156};157EXPORT_SYMBOL_GPL(vmbus_devs);158 159static const struct {160 guid_t guid;161} vmbus_unsupported_devs[] = {162 { HV_AVMA1_GUID },163 { HV_AVMA2_GUID },164 { HV_RDV_GUID },165 { HV_IMC_GUID },166};167 168/*169 * The rescinded channel may be blocked waiting for a response from the host;170 * take care of that.171 */172static void vmbus_rescind_cleanup(struct vmbus_channel *channel)173{174 struct vmbus_channel_msginfo *msginfo;175 unsigned long flags;176 177 178 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);179 channel->rescind = true;180 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,181 msglistentry) {182 183 if (msginfo->waiting_channel == channel) {184 complete(&msginfo->waitevent);185 break;186 }187 }188 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);189}190 191static bool is_unsupported_vmbus_devs(const guid_t *guid)192{193 int i;194 195 for (i = 0; i < ARRAY_SIZE(vmbus_unsupported_devs); i++)196 if (guid_equal(guid, &vmbus_unsupported_devs[i].guid))197 return true;198 return false;199}200 201static u16 hv_get_dev_type(const struct vmbus_channel *channel)202{203 const guid_t *guid = &channel->offermsg.offer.if_type;204 u16 i;205 206 if (is_hvsock_channel(channel) || is_unsupported_vmbus_devs(guid))207 return HV_UNKNOWN;208 209 for (i = HV_IDE; i < HV_UNKNOWN; i++) {210 if (guid_equal(guid, &vmbus_devs[i].guid))211 return i;212 }213 pr_info("Unknown GUID: %pUl\n", guid);214 return i;215}216 217/**218 * vmbus_prep_negotiate_resp() - Create default response for Negotiate message219 * @icmsghdrp: Pointer to msg header structure220 * @buf: Raw buffer channel data221 * @buflen: Length of the raw buffer channel data.222 * @fw_version: The framework versions we can support.223 * @fw_vercnt: The size of @fw_version.224 * @srv_version: The service versions we can support.225 * @srv_vercnt: The size of @srv_version.226 * @nego_fw_version: The selected framework version.227 * @nego_srv_version: The selected service version.228 *229 * Note: Versions are given in decreasing order.230 *231 * Set up and fill in default negotiate response message.232 * Mainly used by Hyper-V drivers.233 */234bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp, u8 *buf,235 u32 buflen, const int *fw_version, int fw_vercnt,236 const int *srv_version, int srv_vercnt,237 int *nego_fw_version, int *nego_srv_version)238{239 int icframe_major, icframe_minor;240 int icmsg_major, icmsg_minor;241 int fw_major, fw_minor;242 int srv_major, srv_minor;243 int i, j;244 bool found_match = false;245 struct icmsg_negotiate *negop;246 247 /* Check that there's enough space for icframe_vercnt, icmsg_vercnt */248 if (buflen < ICMSG_HDR + offsetof(struct icmsg_negotiate, reserved)) {249 pr_err_ratelimited("Invalid icmsg negotiate\n");250 return false;251 }252 253 icmsghdrp->icmsgsize = 0x10;254 negop = (struct icmsg_negotiate *)&buf[ICMSG_HDR];255 256 icframe_major = negop->icframe_vercnt;257 icframe_minor = 0;258 259 icmsg_major = negop->icmsg_vercnt;260 icmsg_minor = 0;261 262 /* Validate negop packet */263 if (icframe_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT ||264 icmsg_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT ||265 ICMSG_NEGOTIATE_PKT_SIZE(icframe_major, icmsg_major) > buflen) {266 pr_err_ratelimited("Invalid icmsg negotiate - icframe_major: %u, icmsg_major: %u\n",267 icframe_major, icmsg_major);268 goto fw_error;269 }270 271 /*272 * Select the framework version number we will273 * support.274 */275 276 for (i = 0; i < fw_vercnt; i++) {277 fw_major = (fw_version[i] >> 16);278 fw_minor = (fw_version[i] & 0xFFFF);279 280 for (j = 0; j < negop->icframe_vercnt; j++) {281 if ((negop->icversion_data[j].major == fw_major) &&282 (negop->icversion_data[j].minor == fw_minor)) {283 icframe_major = negop->icversion_data[j].major;284 icframe_minor = negop->icversion_data[j].minor;285 found_match = true;286 break;287 }288 }289 290 if (found_match)291 break;292 }293 294 if (!found_match)295 goto fw_error;296 297 found_match = false;298 299 for (i = 0; i < srv_vercnt; i++) {300 srv_major = (srv_version[i] >> 16);301 srv_minor = (srv_version[i] & 0xFFFF);302 303 for (j = negop->icframe_vercnt;304 (j < negop->icframe_vercnt + negop->icmsg_vercnt);305 j++) {306 307 if ((negop->icversion_data[j].major == srv_major) &&308 (negop->icversion_data[j].minor == srv_minor)) {309 310 icmsg_major = negop->icversion_data[j].major;311 icmsg_minor = negop->icversion_data[j].minor;312 found_match = true;313 break;314 }315 }316 317 if (found_match)318 break;319 }320 321 /*322 * Respond with the framework and service323 * version numbers we can support.324 */325 326fw_error:327 if (!found_match) {328 negop->icframe_vercnt = 0;329 negop->icmsg_vercnt = 0;330 } else {331 negop->icframe_vercnt = 1;332 negop->icmsg_vercnt = 1;333 }334 335 if (nego_fw_version)336 *nego_fw_version = (icframe_major << 16) | icframe_minor;337 338 if (nego_srv_version)339 *nego_srv_version = (icmsg_major << 16) | icmsg_minor;340 341 negop->icversion_data[0].major = icframe_major;342 negop->icversion_data[0].minor = icframe_minor;343 negop->icversion_data[1].major = icmsg_major;344 negop->icversion_data[1].minor = icmsg_minor;345 return found_match;346}347EXPORT_SYMBOL_GPL(vmbus_prep_negotiate_resp);348 349/*350 * alloc_channel - Allocate and initialize a vmbus channel object351 */352static struct vmbus_channel *alloc_channel(void)353{354 struct vmbus_channel *channel;355 356 channel = kzalloc(sizeof(*channel), GFP_ATOMIC);357 if (!channel)358 return NULL;359 360 spin_lock_init(&channel->sched_lock);361 init_completion(&channel->rescind_event);362 363 INIT_LIST_HEAD(&channel->sc_list);364 365 tasklet_init(&channel->callback_event,366 vmbus_on_event, (unsigned long)channel);367 368 hv_ringbuffer_pre_init(channel);369 370 return channel;371}372 373/*374 * free_channel - Release the resources used by the vmbus channel object375 */376static void free_channel(struct vmbus_channel *channel)377{378 tasklet_kill(&channel->callback_event);379 vmbus_remove_channel_attr_group(channel);380 381 kobject_put(&channel->kobj);382}383 384void vmbus_channel_map_relid(struct vmbus_channel *channel)385{386 if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS))387 return;388 /*389 * The mapping of the channel's relid is visible from the CPUs that390 * execute vmbus_chan_sched() by the time that vmbus_chan_sched() will391 * execute:392 *393 * (a) In the "normal (i.e., not resuming from hibernation)" path,394 * the full barrier in virt_store_mb() guarantees that the store395 * is propagated to all CPUs before the add_channel_work work396 * is queued. In turn, add_channel_work is queued before the397 * channel's ring buffer is allocated/initialized and the398 * OPENCHANNEL message for the channel is sent in vmbus_open().399 * Hyper-V won't start sending the interrupts for the channel400 * before the OPENCHANNEL message is acked. The memory barrier401 * in vmbus_chan_sched() -> sync_test_and_clear_bit() ensures402 * that vmbus_chan_sched() must find the channel's relid in403 * recv_int_page before retrieving the channel pointer from the404 * array of channels.405 *406 * (b) In the "resuming from hibernation" path, the virt_store_mb()407 * guarantees that the store is propagated to all CPUs before408 * the VMBus connection is marked as ready for the resume event409 * (cf. check_ready_for_resume_event()). The interrupt handler410 * of the VMBus driver and vmbus_chan_sched() can not run before411 * vmbus_bus_resume() has completed execution (cf. resume_noirq).412 */413 virt_store_mb(414 vmbus_connection.channels[channel->offermsg.child_relid],415 channel);416}417 418void vmbus_channel_unmap_relid(struct vmbus_channel *channel)419{420 if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS))421 return;422 WRITE_ONCE(423 vmbus_connection.channels[channel->offermsg.child_relid],424 NULL);425}426 427static void vmbus_release_relid(u32 relid)428{429 struct vmbus_channel_relid_released msg;430 int ret;431 432 memset(&msg, 0, sizeof(struct vmbus_channel_relid_released));433 msg.child_relid = relid;434 msg.header.msgtype = CHANNELMSG_RELID_RELEASED;435 ret = vmbus_post_msg(&msg, sizeof(struct vmbus_channel_relid_released),436 true);437 438 trace_vmbus_release_relid(&msg, ret);439}440 441void hv_process_channel_removal(struct vmbus_channel *channel)442{443 lockdep_assert_held(&vmbus_connection.channel_mutex);444 BUG_ON(!channel->rescind);445 446 /*447 * hv_process_channel_removal() could find INVALID_RELID only for448 * hv_sock channels. See the inline comments in vmbus_onoffer().449 */450 WARN_ON(channel->offermsg.child_relid == INVALID_RELID &&451 !is_hvsock_channel(channel));452 453 /*454 * Upon suspend, an in-use hv_sock channel is removed from the array of455 * channels and the relid is invalidated. After hibernation, when the456 * user-space application destroys the channel, it's unnecessary and457 * unsafe to remove the channel from the array of channels. See also458 * the inline comments before the call of vmbus_release_relid() below.459 */460 if (channel->offermsg.child_relid != INVALID_RELID)461 vmbus_channel_unmap_relid(channel);462 463 if (channel->primary_channel == NULL)464 list_del(&channel->listentry);465 else466 list_del(&channel->sc_list);467 468 /*469 * If this is a "perf" channel, updates the hv_numa_map[] masks so that470 * init_vp_index() can (re-)use the CPU.471 */472 if (hv_is_perf_channel(channel))473 hv_clear_allocated_cpu(channel->target_cpu);474 475 /*476 * Upon suspend, an in-use hv_sock channel is marked as "rescinded" and477 * the relid is invalidated; after hibernation, when the user-space app478 * destroys the channel, the relid is INVALID_RELID, and in this case479 * it's unnecessary and unsafe to release the old relid, since the same480 * relid can refer to a completely different channel now.481 */482 if (channel->offermsg.child_relid != INVALID_RELID)483 vmbus_release_relid(channel->offermsg.child_relid);484 485 free_channel(channel);486}487 488void vmbus_free_channels(void)489{490 struct vmbus_channel *channel, *tmp;491 492 list_for_each_entry_safe(channel, tmp, &vmbus_connection.chn_list,493 listentry) {494 /* hv_process_channel_removal() needs this */495 channel->rescind = true;496 497 vmbus_device_unregister(channel->device_obj);498 }499}500 501/* Note: the function can run concurrently for primary/sub channels. */502static void vmbus_add_channel_work(struct work_struct *work)503{504 struct vmbus_channel *newchannel =505 container_of(work, struct vmbus_channel, add_channel_work);506 struct vmbus_channel *primary_channel = newchannel->primary_channel;507 int ret;508 509 /*510 * This state is used to indicate a successful open511 * so that when we do close the channel normally, we512 * can cleanup properly.513 */514 newchannel->state = CHANNEL_OPEN_STATE;515 516 if (primary_channel != NULL) {517 /* newchannel is a sub-channel. */518 struct hv_device *dev = primary_channel->device_obj;519 520 if (vmbus_add_channel_kobj(dev, newchannel))521 goto err_deq_chan;522 523 if (primary_channel->sc_creation_callback != NULL)524 primary_channel->sc_creation_callback(newchannel);525 526 newchannel->probe_done = true;527 return;528 }529 530 /*531 * Start the process of binding the primary channel to the driver532 */533 newchannel->device_obj = vmbus_device_create(534 &newchannel->offermsg.offer.if_type,535 &newchannel->offermsg.offer.if_instance,536 newchannel);537 if (!newchannel->device_obj)538 goto err_deq_chan;539 540 newchannel->device_obj->device_id = newchannel->device_id;541 /*542 * Add the new device to the bus. This will kick off device-driver543 * binding which eventually invokes the device driver's AddDevice()544 * method.545 *546 * If vmbus_device_register() fails, the 'device_obj' is freed in547 * vmbus_device_release() as called by device_unregister() in the548 * error path of vmbus_device_register(). In the outside error549 * path, there's no need to free it.550 */551 ret = vmbus_device_register(newchannel->device_obj);552 553 if (ret != 0) {554 pr_err("unable to add child device object (relid %d)\n",555 newchannel->offermsg.child_relid);556 goto err_deq_chan;557 }558 559 newchannel->probe_done = true;560 return;561 562err_deq_chan:563 mutex_lock(&vmbus_connection.channel_mutex);564 565 /*566 * We need to set the flag, otherwise567 * vmbus_onoffer_rescind() can be blocked.568 */569 newchannel->probe_done = true;570 571 if (primary_channel == NULL)572 list_del(&newchannel->listentry);573 else574 list_del(&newchannel->sc_list);575 576 /* vmbus_process_offer() has mapped the channel. */577 vmbus_channel_unmap_relid(newchannel);578 579 mutex_unlock(&vmbus_connection.channel_mutex);580 581 vmbus_release_relid(newchannel->offermsg.child_relid);582 583 free_channel(newchannel);584}585 586/*587 * vmbus_process_offer - Process the offer by creating a channel/device588 * associated with this offer589 */590static void vmbus_process_offer(struct vmbus_channel *newchannel)591{592 struct vmbus_channel *channel;593 struct workqueue_struct *wq;594 bool fnew = true;595 596 /*597 * Synchronize vmbus_process_offer() and CPU hotplugging:598 *599 * CPU1 CPU2600 *601 * [vmbus_process_offer()] [Hot removal of the CPU]602 *603 * CPU_READ_LOCK CPUS_WRITE_LOCK604 * LOAD cpu_online_mask SEARCH chn_list605 * STORE target_cpu LOAD target_cpu606 * INSERT chn_list STORE cpu_online_mask607 * CPUS_READ_UNLOCK CPUS_WRITE_UNLOCK608 *609 * Forbids: CPU1's LOAD from *not* seing CPU2's STORE &&610 * CPU2's SEARCH from *not* seeing CPU1's INSERT611 *612 * Forbids: CPU2's SEARCH from seeing CPU1's INSERT &&613 * CPU2's LOAD from *not* seing CPU1's STORE614 */615 cpus_read_lock();616 617 /*618 * Serializes the modifications of the chn_list list as well as619 * the accesses to next_numa_node_id in init_vp_index().620 */621 mutex_lock(&vmbus_connection.channel_mutex);622 623 list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) {624 if (guid_equal(&channel->offermsg.offer.if_type,625 &newchannel->offermsg.offer.if_type) &&626 guid_equal(&channel->offermsg.offer.if_instance,627 &newchannel->offermsg.offer.if_instance)) {628 fnew = false;629 newchannel->primary_channel = channel;630 break;631 }632 }633 634 init_vp_index(newchannel);635 636 /* Remember the channels that should be cleaned up upon suspend. */637 if (is_hvsock_channel(newchannel) || is_sub_channel(newchannel))638 atomic_inc(&vmbus_connection.nr_chan_close_on_suspend);639 640 /*641 * Now that we have acquired the channel_mutex,642 * we can release the potentially racing rescind thread.643 */644 atomic_dec(&vmbus_connection.offer_in_progress);645 646 if (fnew) {647 list_add_tail(&newchannel->listentry,648 &vmbus_connection.chn_list);649 } else {650 /*651 * Check to see if this is a valid sub-channel.652 */653 if (newchannel->offermsg.offer.sub_channel_index == 0) {654 mutex_unlock(&vmbus_connection.channel_mutex);655 cpus_read_unlock();656 /*657 * Don't call free_channel(), because newchannel->kobj658 * is not initialized yet.659 */660 kfree(newchannel);661 WARN_ON_ONCE(1);662 return;663 }664 /*665 * Process the sub-channel.666 */667 list_add_tail(&newchannel->sc_list, &channel->sc_list);668 }669 670 vmbus_channel_map_relid(newchannel);671 672 mutex_unlock(&vmbus_connection.channel_mutex);673 cpus_read_unlock();674 675 /*676 * vmbus_process_offer() mustn't call channel->sc_creation_callback()677 * directly for sub-channels, because sc_creation_callback() ->678 * vmbus_open() may never get the host's response to the679 * OPEN_CHANNEL message (the host may rescind a channel at any time,680 * e.g. in the case of hot removing a NIC), and vmbus_onoffer_rescind()681 * may not wake up the vmbus_open() as it's blocked due to a non-zero682 * vmbus_connection.offer_in_progress, and finally we have a deadlock.683 *684 * The above is also true for primary channels, if the related device685 * drivers use sync probing mode by default.686 *687 * And, usually the handling of primary channels and sub-channels can688 * depend on each other, so we should offload them to different689 * workqueues to avoid possible deadlock, e.g. in sync-probing mode,690 * NIC1's netvsc_subchan_work() can race with NIC2's netvsc_probe() ->691 * rtnl_lock(), and causes deadlock: the former gets the rtnl_lock692 * and waits for all the sub-channels to appear, but the latter693 * can't get the rtnl_lock and this blocks the handling of694 * sub-channels.695 */696 INIT_WORK(&newchannel->add_channel_work, vmbus_add_channel_work);697 wq = fnew ? vmbus_connection.handle_primary_chan_wq :698 vmbus_connection.handle_sub_chan_wq;699 queue_work(wq, &newchannel->add_channel_work);700}701 702/*703 * Check if CPUs used by other channels of the same device.704 * It should only be called by init_vp_index().705 */706static bool hv_cpuself_used(u32 cpu, struct vmbus_channel *chn)707{708 struct vmbus_channel *primary = chn->primary_channel;709 struct vmbus_channel *sc;710 711 lockdep_assert_held(&vmbus_connection.channel_mutex);712 713 if (!primary)714 return false;715 716 if (primary->target_cpu == cpu)717 return true;718 719 list_for_each_entry(sc, &primary->sc_list, sc_list)720 if (sc != chn && sc->target_cpu == cpu)721 return true;722 723 return false;724}725 726/*727 * We use this state to statically distribute the channel interrupt load.728 */729static int next_numa_node_id;730 731/*732 * We can statically distribute the incoming channel interrupt load733 * by binding a channel to VCPU.734 *735 * For non-performance critical channels we assign the VMBUS_CONNECT_CPU.736 * Performance critical channels will be distributed evenly among all737 * the available NUMA nodes. Once the node is assigned, we will assign738 * the CPU based on a simple round robin scheme.739 */740static void init_vp_index(struct vmbus_channel *channel)741{742 bool perf_chn = hv_is_perf_channel(channel);743 u32 i, ncpu = num_online_cpus();744 cpumask_var_t available_mask;745 struct cpumask *allocated_mask;746 const struct cpumask *hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ);747 u32 target_cpu;748 int numa_node;749 750 if (!perf_chn ||751 !alloc_cpumask_var(&available_mask, GFP_KERNEL) ||752 cpumask_empty(hk_mask)) {753 /*754 * If the channel is not a performance critical755 * channel, bind it to VMBUS_CONNECT_CPU.756 * In case alloc_cpumask_var() fails, bind it to757 * VMBUS_CONNECT_CPU.758 * If all the cpus are isolated, bind it to759 * VMBUS_CONNECT_CPU.760 */761 channel->target_cpu = VMBUS_CONNECT_CPU;762 if (perf_chn)763 hv_set_allocated_cpu(VMBUS_CONNECT_CPU);764 return;765 }766 767 for (i = 1; i <= ncpu + 1; i++) {768 while (true) {769 numa_node = next_numa_node_id++;770 if (numa_node == nr_node_ids) {771 next_numa_node_id = 0;772 continue;773 }774 if (cpumask_empty(cpumask_of_node(numa_node)))775 continue;776 break;777 }778 allocated_mask = &hv_context.hv_numa_map[numa_node];779 780retry:781 cpumask_xor(available_mask, allocated_mask, cpumask_of_node(numa_node));782 cpumask_and(available_mask, available_mask, hk_mask);783 784 if (cpumask_empty(available_mask)) {785 /*786 * We have cycled through all the CPUs in the node;787 * reset the allocated map.788 */789 cpumask_clear(allocated_mask);790 goto retry;791 }792 793 target_cpu = cpumask_first(available_mask);794 cpumask_set_cpu(target_cpu, allocated_mask);795 796 if (channel->offermsg.offer.sub_channel_index >= ncpu ||797 i > ncpu || !hv_cpuself_used(target_cpu, channel))798 break;799 }800 801 channel->target_cpu = target_cpu;802 803 free_cpumask_var(available_mask);804}805 806#define UNLOAD_DELAY_UNIT_MS 10 /* 10 milliseconds */807#define UNLOAD_WAIT_MS (100*1000) /* 100 seconds */808#define UNLOAD_WAIT_LOOPS (UNLOAD_WAIT_MS/UNLOAD_DELAY_UNIT_MS)809#define UNLOAD_MSG_MS (5*1000) /* Every 5 seconds */810#define UNLOAD_MSG_LOOPS (UNLOAD_MSG_MS/UNLOAD_DELAY_UNIT_MS)811 812static void vmbus_wait_for_unload(void)813{814 int cpu;815 void *page_addr;816 struct hv_message *msg;817 struct vmbus_channel_message_header *hdr;818 u32 message_type, i;819 820 /*821 * CHANNELMSG_UNLOAD_RESPONSE is always delivered to the CPU which was822 * used for initial contact or to CPU0 depending on host version. When823 * we're crashing on a different CPU let's hope that IRQ handler on824 * the cpu which receives CHANNELMSG_UNLOAD_RESPONSE is still825 * functional and vmbus_unload_response() will complete826 * vmbus_connection.unload_event. If not, the last thing we can do is827 * read message pages for all CPUs directly.828 *829 * Wait up to 100 seconds since an Azure host must writeback any dirty830 * data in its disk cache before the VMbus UNLOAD request will831 * complete. This flushing has been empirically observed to take up832 * to 50 seconds in cases with a lot of dirty data, so allow additional833 * leeway and for inaccuracies in mdelay(). But eventually time out so834 * that the panic path can't get hung forever in case the response835 * message isn't seen.836 */837 for (i = 1; i <= UNLOAD_WAIT_LOOPS; i++) {838 if (completion_done(&vmbus_connection.unload_event))839 goto completed;840 841 for_each_present_cpu(cpu) {842 struct hv_per_cpu_context *hv_cpu843 = per_cpu_ptr(hv_context.cpu_context, cpu);844 845 /*846 * In a CoCo VM the synic_message_page is not allocated847 * in hv_synic_alloc(). Instead it is set/cleared in848 * hv_synic_enable_regs() and hv_synic_disable_regs()849 * such that it is set only when the CPU is online. If850 * not all present CPUs are online, the message page851 * might be NULL, so skip such CPUs.852 */853 page_addr = hv_cpu->synic_message_page;854 if (!page_addr)855 continue;856 857 msg = (struct hv_message *)page_addr858 + VMBUS_MESSAGE_SINT;859 860 message_type = READ_ONCE(msg->header.message_type);861 if (message_type == HVMSG_NONE)862 continue;863 864 hdr = (struct vmbus_channel_message_header *)865 msg->u.payload;866 867 if (hdr->msgtype == CHANNELMSG_UNLOAD_RESPONSE)868 complete(&vmbus_connection.unload_event);869 870 vmbus_signal_eom(msg, message_type);871 }872 873 /*874 * Give a notice periodically so someone watching the875 * serial output won't think it is completely hung.876 */877 if (!(i % UNLOAD_MSG_LOOPS))878 pr_notice("Waiting for VMBus UNLOAD to complete\n");879 880 mdelay(UNLOAD_DELAY_UNIT_MS);881 }882 pr_err("Continuing even though VMBus UNLOAD did not complete\n");883 884completed:885 /*886 * We're crashing and already got the UNLOAD_RESPONSE, cleanup all887 * maybe-pending messages on all CPUs to be able to receive new888 * messages after we reconnect.889 */890 for_each_present_cpu(cpu) {891 struct hv_per_cpu_context *hv_cpu892 = per_cpu_ptr(hv_context.cpu_context, cpu);893 894 page_addr = hv_cpu->synic_message_page;895 if (!page_addr)896 continue;897 898 msg = (struct hv_message *)page_addr + VMBUS_MESSAGE_SINT;899 msg->header.message_type = HVMSG_NONE;900 }901}902 903/*904 * vmbus_unload_response - Handler for the unload response.905 */906static void vmbus_unload_response(struct vmbus_channel_message_header *hdr)907{908 /*909 * This is a global event; just wakeup the waiting thread.910 * Once we successfully unload, we can cleanup the monitor state.911 *912 * NB. A malicious or compromised Hyper-V could send a spurious913 * message of type CHANNELMSG_UNLOAD_RESPONSE, and trigger a call914 * of the complete() below. Make sure that unload_event has been915 * initialized by the time this complete() is executed.916 */917 complete(&vmbus_connection.unload_event);918}919 920void vmbus_initiate_unload(bool crash)921{922 struct vmbus_channel_message_header hdr;923 924 if (xchg(&vmbus_connection.conn_state, DISCONNECTED) == DISCONNECTED)925 return;926 927 /* Pre-Win2012R2 hosts don't support reconnect */928 if (vmbus_proto_version < VERSION_WIN8_1)929 return;930 931 reinit_completion(&vmbus_connection.unload_event);932 memset(&hdr, 0, sizeof(struct vmbus_channel_message_header));933 hdr.msgtype = CHANNELMSG_UNLOAD;934 vmbus_post_msg(&hdr, sizeof(struct vmbus_channel_message_header),935 !crash);936 937 /*938 * vmbus_initiate_unload() is also called on crash and the crash can be939 * happening in an interrupt context, where scheduling is impossible.940 */941 if (!crash)942 wait_for_completion(&vmbus_connection.unload_event);943 else944 vmbus_wait_for_unload();945}946 947static void check_ready_for_resume_event(void)948{949 /*950 * If all the old primary channels have been fixed up, then it's safe951 * to resume.952 */953 if (atomic_dec_and_test(&vmbus_connection.nr_chan_fixup_on_resume))954 complete(&vmbus_connection.ready_for_resume_event);955}956 957static void vmbus_setup_channel_state(struct vmbus_channel *channel,958 struct vmbus_channel_offer_channel *offer)959{960 /*961 * Setup state for signalling the host.962 */963 channel->sig_event = VMBUS_EVENT_CONNECTION_ID;964 965 channel->is_dedicated_interrupt =966 (offer->is_dedicated_interrupt != 0);967 channel->sig_event = offer->connection_id;968 969 memcpy(&channel->offermsg, offer,970 sizeof(struct vmbus_channel_offer_channel));971 channel->monitor_grp = (u8)offer->monitorid / 32;972 channel->monitor_bit = (u8)offer->monitorid % 32;973 channel->device_id = hv_get_dev_type(channel);974}975 976/*977 * find_primary_channel_by_offer - Get the channel object given the new offer.978 * This is only used in the resume path of hibernation.979 */980static struct vmbus_channel *981find_primary_channel_by_offer(const struct vmbus_channel_offer_channel *offer)982{983 struct vmbus_channel *channel = NULL, *iter;984 const guid_t *inst1, *inst2;985 986 /* Ignore sub-channel offers. */987 if (offer->offer.sub_channel_index != 0)988 return NULL;989 990 mutex_lock(&vmbus_connection.channel_mutex);991 992 list_for_each_entry(iter, &vmbus_connection.chn_list, listentry) {993 inst1 = &iter->offermsg.offer.if_instance;994 inst2 = &offer->offer.if_instance;995 996 if (guid_equal(inst1, inst2)) {997 channel = iter;998 break;999 }1000 }1001 1002 mutex_unlock(&vmbus_connection.channel_mutex);1003 1004 return channel;1005}1006 1007static bool vmbus_is_valid_offer(const struct vmbus_channel_offer_channel *offer)1008{1009 const guid_t *guid = &offer->offer.if_type;1010 u16 i;1011 1012 if (!hv_is_isolation_supported())1013 return true;1014 1015 if (is_hvsock_offer(offer))1016 return true;1017 1018 for (i = 0; i < ARRAY_SIZE(vmbus_devs); i++) {1019 if (guid_equal(guid, &vmbus_devs[i].guid))1020 return vmbus_devs[i].allowed_in_isolated;1021 }1022 return false;1023}1024 1025/*1026 * vmbus_onoffer - Handler for channel offers from vmbus in parent partition.1027 *1028 */1029static void vmbus_onoffer(struct vmbus_channel_message_header *hdr)1030{1031 struct vmbus_channel_offer_channel *offer;1032 struct vmbus_channel *oldchannel, *newchannel;1033 size_t offer_sz;1034 1035 offer = (struct vmbus_channel_offer_channel *)hdr;1036 1037 trace_vmbus_onoffer(offer);1038 1039 if (!vmbus_is_valid_offer(offer)) {1040 pr_err_ratelimited("Invalid offer %d from the host supporting isolation\n",1041 offer->child_relid);1042 atomic_dec(&vmbus_connection.offer_in_progress);1043 return;1044 }1045 1046 oldchannel = find_primary_channel_by_offer(offer);1047 1048 if (oldchannel != NULL) {1049 /*1050 * We're resuming from hibernation: all the sub-channel and1051 * hv_sock channels we had before the hibernation should have1052 * been cleaned up, and now we must be seeing a re-offered1053 * primary channel that we had before the hibernation.1054 */1055 1056 /*1057 * { Initially: channel relid = INVALID_RELID,1058 * channels[valid_relid] = NULL }1059 *1060 * CPU1 CPU21061 *1062 * [vmbus_onoffer()] [vmbus_device_release()]1063 *1064 * LOCK channel_mutex LOCK channel_mutex1065 * STORE channel relid = valid_relid LOAD r1 = channel relid1066 * MAP_RELID channel if (r1 != INVALID_RELID)1067 * UNLOCK channel_mutex UNMAP_RELID channel1068 * UNLOCK channel_mutex1069 *1070 * Forbids: r1 == valid_relid &&1071 * channels[valid_relid] == channel1072 *1073 * Note. r1 can be INVALID_RELID only for an hv_sock channel.1074 * None of the hv_sock channels which were present before the1075 * suspend are re-offered upon the resume. See the WARN_ON()1076 * in hv_process_channel_removal().1077 */1078 mutex_lock(&vmbus_connection.channel_mutex);1079 1080 atomic_dec(&vmbus_connection.offer_in_progress);1081 1082 WARN_ON(oldchannel->offermsg.child_relid != INVALID_RELID);1083 /* Fix up the relid. */1084 oldchannel->offermsg.child_relid = offer->child_relid;1085 1086 offer_sz = sizeof(*offer);1087 if (memcmp(offer, &oldchannel->offermsg, offer_sz) != 0) {1088 /*1089 * This is not an error, since the host can also change1090 * the other field(s) of the offer, e.g. on WS RS51091 * (Build 17763), the offer->connection_id of the1092 * Mellanox VF vmbus device can change when the host1093 * reoffers the device upon resume.1094 */1095 pr_debug("vmbus offer changed: relid=%d\n",1096 offer->child_relid);1097 1098 print_hex_dump_debug("Old vmbus offer: ",1099 DUMP_PREFIX_OFFSET, 16, 4,1100 &oldchannel->offermsg, offer_sz,1101 false);1102 print_hex_dump_debug("New vmbus offer: ",1103 DUMP_PREFIX_OFFSET, 16, 4,1104 offer, offer_sz, false);1105 1106 /* Fix up the old channel. */1107 vmbus_setup_channel_state(oldchannel, offer);1108 }1109 1110 /* Add the channel back to the array of channels. */1111 vmbus_channel_map_relid(oldchannel);1112 check_ready_for_resume_event();1113 1114 mutex_unlock(&vmbus_connection.channel_mutex);1115 return;1116 }1117 1118 /* Allocate the channel object and save this offer. */1119 newchannel = alloc_channel();1120 if (!newchannel) {1121 vmbus_release_relid(offer->child_relid);1122 atomic_dec(&vmbus_connection.offer_in_progress);1123 pr_err("Unable to allocate channel object\n");1124 return;1125 }1126 1127 vmbus_setup_channel_state(newchannel, offer);1128 1129 vmbus_process_offer(newchannel);1130}1131 1132static void check_ready_for_suspend_event(void)1133{1134 /*1135 * If all the sub-channels or hv_sock channels have been cleaned up,1136 * then it's safe to suspend.1137 */1138 if (atomic_dec_and_test(&vmbus_connection.nr_chan_close_on_suspend))1139 complete(&vmbus_connection.ready_for_suspend_event);1140}1141 1142/*1143 * vmbus_onoffer_rescind - Rescind offer handler.1144 *1145 * We queue a work item to process this offer synchronously1146 */1147static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr)1148{1149 struct vmbus_channel_rescind_offer *rescind;1150 struct vmbus_channel *channel;1151 struct device *dev;1152 bool clean_up_chan_for_suspend;1153 1154 rescind = (struct vmbus_channel_rescind_offer *)hdr;1155 1156 trace_vmbus_onoffer_rescind(rescind);1157 1158 /*1159 * The offer msg and the corresponding rescind msg1160 * from the host are guranteed to be ordered -1161 * offer comes in first and then the rescind.1162 * Since we process these events in work elements,1163 * and with preemption, we may end up processing1164 * the events out of order. We rely on the synchronization1165 * provided by offer_in_progress and by channel_mutex for1166 * ordering these events:1167 *1168 * { Initially: offer_in_progress = 1 }1169 *1170 * CPU1 CPU21171 *1172 * [vmbus_onoffer()] [vmbus_onoffer_rescind()]1173 *1174 * LOCK channel_mutex WAIT_ON offer_in_progress == 01175 * DECREMENT offer_in_progress LOCK channel_mutex1176 * STORE channels[] LOAD channels[]1177 * UNLOCK channel_mutex UNLOCK channel_mutex1178 *1179 * Forbids: CPU2's LOAD from *not* seeing CPU1's STORE1180 */1181 1182 while (atomic_read(&vmbus_connection.offer_in_progress) != 0) {1183 /*1184 * We wait here until any channel offer is currently1185 * being processed.1186 */1187 msleep(1);1188 }1189 1190 mutex_lock(&vmbus_connection.channel_mutex);1191 channel = relid2channel(rescind->child_relid);1192 if (channel != NULL) {1193 /*1194 * Guarantee that no other instance of vmbus_onoffer_rescind()1195 * has got a reference to the channel object. Synchronize on1196 * &vmbus_connection.channel_mutex.1197 */1198 if (channel->rescind_ref) {1199 mutex_unlock(&vmbus_connection.channel_mutex);1200 return;1201 }1202 channel->rescind_ref = true;1203 }1204 mutex_unlock(&vmbus_connection.channel_mutex);1205 1206 if (channel == NULL) {1207 /*1208 * We failed in processing the offer message;1209 * we would have cleaned up the relid in that1210 * failure path.1211 */1212 return;1213 }1214 1215 clean_up_chan_for_suspend = is_hvsock_channel(channel) ||1216 is_sub_channel(channel);1217 /*1218 * Before setting channel->rescind in vmbus_rescind_cleanup(), we1219 * should make sure the channel callback is not running any more.1220 */1221 vmbus_reset_channel_cb(channel);1222 1223 /*1224 * Now wait for offer handling to complete.1225 */1226 vmbus_rescind_cleanup(channel);1227 while (READ_ONCE(channel->probe_done) == false) {1228 /*1229 * We wait here until any channel offer is currently1230 * being processed.1231 */1232 msleep(1);1233 }1234 1235 /*1236 * At this point, the rescind handling can proceed safely.1237 */1238 1239 if (channel->device_obj) {1240 if (channel->chn_rescind_callback) {1241 channel->chn_rescind_callback(channel);1242 1243 if (clean_up_chan_for_suspend)1244 check_ready_for_suspend_event();1245 1246 return;1247 }1248 /*1249 * We will have to unregister this device from the1250 * driver core.1251 */1252 dev = get_device(&channel->device_obj->device);1253 if (dev) {1254 vmbus_device_unregister(channel->device_obj);1255 put_device(dev);1256 }1257 } else if (channel->primary_channel != NULL) {1258 /*1259 * Sub-channel is being rescinded. Following is the channel1260 * close sequence when initiated from the driveri (refer to1261 * vmbus_close() for details):1262 * 1. Close all sub-channels first1263 * 2. Then close the primary channel.1264 */1265 mutex_lock(&vmbus_connection.channel_mutex);1266 if (channel->state == CHANNEL_OPEN_STATE) {1267 /*1268 * The channel is currently not open;1269 * it is safe for us to cleanup the channel.1270 */1271 hv_process_channel_removal(channel);1272 } else {1273 complete(&channel->rescind_event);1274 }1275 mutex_unlock(&vmbus_connection.channel_mutex);1276 }1277 1278 /* The "channel" may have been freed. Do not access it any longer. */1279 1280 if (clean_up_chan_for_suspend)1281 check_ready_for_suspend_event();1282}1283 1284void vmbus_hvsock_device_unregister(struct vmbus_channel *channel)1285{1286 BUG_ON(!is_hvsock_channel(channel));1287 1288 /* We always get a rescind msg when a connection is closed. */1289 while (!READ_ONCE(channel->probe_done) || !READ_ONCE(channel->rescind))1290 msleep(1);1291 1292 vmbus_device_unregister(channel->device_obj);1293}1294EXPORT_SYMBOL_GPL(vmbus_hvsock_device_unregister);1295 1296 1297/*1298 * vmbus_onoffers_delivered -1299 * This is invoked when all offers have been delivered.1300 *1301 * Nothing to do here.1302 */1303static void vmbus_onoffers_delivered(1304 struct vmbus_channel_message_header *hdr)1305{1306}1307 1308/*1309 * vmbus_onopen_result - Open result handler.1310 *1311 * This is invoked when we received a response to our channel open request.1312 * Find the matching request, copy the response and signal the requesting1313 * thread.1314 */1315static void vmbus_onopen_result(struct vmbus_channel_message_header *hdr)1316{1317 struct vmbus_channel_open_result *result;1318 struct vmbus_channel_msginfo *msginfo;1319 struct vmbus_channel_message_header *requestheader;1320 struct vmbus_channel_open_channel *openmsg;1321 unsigned long flags;1322 1323 result = (struct vmbus_channel_open_result *)hdr;1324 1325 trace_vmbus_onopen_result(result);1326 1327 /*1328 * Find the open msg, copy the result and signal/unblock the wait event1329 */1330 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);1331 1332 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,1333 msglistentry) {1334 requestheader =1335 (struct vmbus_channel_message_header *)msginfo->msg;1336 1337 if (requestheader->msgtype == CHANNELMSG_OPENCHANNEL) {1338 openmsg =1339 (struct vmbus_channel_open_channel *)msginfo->msg;1340 if (openmsg->child_relid == result->child_relid &&1341 openmsg->openid == result->openid) {1342 memcpy(&msginfo->response.open_result,1343 result,1344 sizeof(1345 struct vmbus_channel_open_result));1346 complete(&msginfo->waitevent);1347 break;1348 }1349 }1350 }1351 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);1352}1353 1354/*1355 * vmbus_ongpadl_created - GPADL created handler.1356 *1357 * This is invoked when we received a response to our gpadl create request.1358 * Find the matching request, copy the response and signal the requesting1359 * thread.1360 */1361static void vmbus_ongpadl_created(struct vmbus_channel_message_header *hdr)1362{1363 struct vmbus_channel_gpadl_created *gpadlcreated;1364 struct vmbus_channel_msginfo *msginfo;1365 struct vmbus_channel_message_header *requestheader;1366 struct vmbus_channel_gpadl_header *gpadlheader;1367 unsigned long flags;1368 1369 gpadlcreated = (struct vmbus_channel_gpadl_created *)hdr;1370 1371 trace_vmbus_ongpadl_created(gpadlcreated);1372 1373 /*1374 * Find the establish msg, copy the result and signal/unblock the wait1375 * event1376 */1377 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);1378 1379 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,1380 msglistentry) {1381 requestheader =1382 (struct vmbus_channel_message_header *)msginfo->msg;1383 1384 if (requestheader->msgtype == CHANNELMSG_GPADL_HEADER) {1385 gpadlheader =1386 (struct vmbus_channel_gpadl_header *)requestheader;1387 1388 if ((gpadlcreated->child_relid ==1389 gpadlheader->child_relid) &&1390 (gpadlcreated->gpadl == gpadlheader->gpadl)) {1391 memcpy(&msginfo->response.gpadl_created,1392 gpadlcreated,1393 sizeof(1394 struct vmbus_channel_gpadl_created));1395 complete(&msginfo->waitevent);1396 break;1397 }1398 }1399 }1400 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);1401}1402 1403/*1404 * vmbus_onmodifychannel_response - Modify Channel response handler.1405 *1406 * This is invoked when we received a response to our channel modify request.1407 * Find the matching request, copy the response and signal the requesting thread.1408 */1409static void vmbus_onmodifychannel_response(struct vmbus_channel_message_header *hdr)1410{1411 struct vmbus_channel_modifychannel_response *response;1412 struct vmbus_channel_msginfo *msginfo;1413 unsigned long flags;1414 1415 response = (struct vmbus_channel_modifychannel_response *)hdr;1416 1417 trace_vmbus_onmodifychannel_response(response);1418 1419 /*1420 * Find the modify msg, copy the response and signal/unblock the wait event.1421 */1422 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);1423 1424 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, msglistentry) {1425 struct vmbus_channel_message_header *responseheader =1426 (struct vmbus_channel_message_header *)msginfo->msg;1427 1428 if (responseheader->msgtype == CHANNELMSG_MODIFYCHANNEL) {1429 struct vmbus_channel_modifychannel *modifymsg;1430 1431 modifymsg = (struct vmbus_channel_modifychannel *)msginfo->msg;1432 if (modifymsg->child_relid == response->child_relid) {1433 memcpy(&msginfo->response.modify_response, response,1434 sizeof(*response));1435 complete(&msginfo->waitevent);1436 break;1437 }1438 }1439 }1440 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);1441}1442 1443/*1444 * vmbus_ongpadl_torndown - GPADL torndown handler.1445 *1446 * This is invoked when we received a response to our gpadl teardown request.1447 * Find the matching request, copy the response and signal the requesting1448 * thread.1449 */1450static void vmbus_ongpadl_torndown(1451 struct vmbus_channel_message_header *hdr)1452{1453 struct vmbus_channel_gpadl_torndown *gpadl_torndown;1454 struct vmbus_channel_msginfo *msginfo;1455 struct vmbus_channel_message_header *requestheader;1456 struct vmbus_channel_gpadl_teardown *gpadl_teardown;1457 unsigned long flags;1458 1459 gpadl_torndown = (struct vmbus_channel_gpadl_torndown *)hdr;1460 1461 trace_vmbus_ongpadl_torndown(gpadl_torndown);1462 1463 /*1464 * Find the open msg, copy the result and signal/unblock the wait event1465 */1466 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);1467 1468 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,1469 msglistentry) {1470 requestheader =1471 (struct vmbus_channel_message_header *)msginfo->msg;1472 1473 if (requestheader->msgtype == CHANNELMSG_GPADL_TEARDOWN) {1474 gpadl_teardown =1475 (struct vmbus_channel_gpadl_teardown *)requestheader;1476 1477 if (gpadl_torndown->gpadl == gpadl_teardown->gpadl) {1478 memcpy(&msginfo->response.gpadl_torndown,1479 gpadl_torndown,1480 sizeof(1481 struct vmbus_channel_gpadl_torndown));1482 complete(&msginfo->waitevent);1483 break;1484 }1485 }1486 }1487 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);1488}1489 1490/*1491 * vmbus_onversion_response - Version response handler1492 *1493 * This is invoked when we received a response to our initiate contact request.1494 * Find the matching request, copy the response and signal the requesting1495 * thread.1496 */1497static void vmbus_onversion_response(1498 struct vmbus_channel_message_header *hdr)1499{1500 struct vmbus_channel_msginfo *msginfo;1501 struct vmbus_channel_message_header *requestheader;1502 struct vmbus_channel_version_response *version_response;1503 unsigned long flags;1504 1505 version_response = (struct vmbus_channel_version_response *)hdr;1506 1507 trace_vmbus_onversion_response(version_response);1508 1509 spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags);1510 1511 list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list,1512 msglistentry) {1513 requestheader =1514 (struct vmbus_channel_message_header *)msginfo->msg;1515 1516 if (requestheader->msgtype ==1517 CHANNELMSG_INITIATE_CONTACT) {1518 memcpy(&msginfo->response.version_response,1519 version_response,1520 sizeof(struct vmbus_channel_version_response));1521 complete(&msginfo->waitevent);1522 }1523 }1524 spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags);1525}1526 1527/* Channel message dispatch table */1528const struct vmbus_channel_message_table_entry1529channel_message_table[CHANNELMSG_COUNT] = {1530 { CHANNELMSG_INVALID, 0, NULL, 0},1531 { CHANNELMSG_OFFERCHANNEL, 0, vmbus_onoffer,1532 sizeof(struct vmbus_channel_offer_channel)},1533 { CHANNELMSG_RESCIND_CHANNELOFFER, 0, vmbus_onoffer_rescind,1534 sizeof(struct vmbus_channel_rescind_offer) },1535 { CHANNELMSG_REQUESTOFFERS, 0, NULL, 0},1536 { CHANNELMSG_ALLOFFERS_DELIVERED, 1, vmbus_onoffers_delivered, 0},1537 { CHANNELMSG_OPENCHANNEL, 0, NULL, 0},1538 { CHANNELMSG_OPENCHANNEL_RESULT, 1, vmbus_onopen_result,1539 sizeof(struct vmbus_channel_open_result)},1540 { CHANNELMSG_CLOSECHANNEL, 0, NULL, 0},1541 { CHANNELMSG_GPADL_HEADER, 0, NULL, 0},1542 { CHANNELMSG_GPADL_BODY, 0, NULL, 0},1543 { CHANNELMSG_GPADL_CREATED, 1, vmbus_ongpadl_created,1544 sizeof(struct vmbus_channel_gpadl_created)},1545 { CHANNELMSG_GPADL_TEARDOWN, 0, NULL, 0},1546 { CHANNELMSG_GPADL_TORNDOWN, 1, vmbus_ongpadl_torndown,1547 sizeof(struct vmbus_channel_gpadl_torndown) },1548 { CHANNELMSG_RELID_RELEASED, 0, NULL, 0},1549 { CHANNELMSG_INITIATE_CONTACT, 0, NULL, 0},1550 { CHANNELMSG_VERSION_RESPONSE, 1, vmbus_onversion_response,1551 sizeof(struct vmbus_channel_version_response)},1552 { CHANNELMSG_UNLOAD, 0, NULL, 0},1553 { CHANNELMSG_UNLOAD_RESPONSE, 1, vmbus_unload_response, 0},1554 { CHANNELMSG_18, 0, NULL, 0},1555 { CHANNELMSG_19, 0, NULL, 0},1556 { CHANNELMSG_20, 0, NULL, 0},1557 { CHANNELMSG_TL_CONNECT_REQUEST, 0, NULL, 0},1558 { CHANNELMSG_MODIFYCHANNEL, 0, NULL, 0},1559 { CHANNELMSG_TL_CONNECT_RESULT, 0, NULL, 0},1560 { CHANNELMSG_MODIFYCHANNEL_RESPONSE, 1, vmbus_onmodifychannel_response,1561 sizeof(struct vmbus_channel_modifychannel_response)},1562};1563 1564/*1565 * vmbus_onmessage - Handler for channel protocol messages.1566 *1567 * This is invoked in the vmbus worker thread context.1568 */1569void vmbus_onmessage(struct vmbus_channel_message_header *hdr)1570{1571 trace_vmbus_on_message(hdr);1572 1573 /*1574 * vmbus_on_msg_dpc() makes sure the hdr->msgtype here can not go1575 * out of bound and the message_handler pointer can not be NULL.1576 */1577 channel_message_table[hdr->msgtype].message_handler(hdr);1578}1579 1580/*1581 * vmbus_request_offers - Send a request to get all our pending offers.1582 */1583int vmbus_request_offers(void)1584{1585 struct vmbus_channel_message_header *msg;1586 struct vmbus_channel_msginfo *msginfo;1587 int ret;1588 1589 msginfo = kzalloc(sizeof(*msginfo) +1590 sizeof(struct vmbus_channel_message_header),1591 GFP_KERNEL);1592 if (!msginfo)1593 return -ENOMEM;1594 1595 msg = (struct vmbus_channel_message_header *)msginfo->msg;1596 1597 msg->msgtype = CHANNELMSG_REQUESTOFFERS;1598 1599 ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_message_header),1600 true);1601 1602 trace_vmbus_request_offers(ret);1603 1604 if (ret != 0) {1605 pr_err("Unable to request offers - %d\n", ret);1606 1607 goto cleanup;1608 }1609 1610cleanup:1611 kfree(msginfo);1612 1613 return ret;1614}1615 1616void vmbus_set_sc_create_callback(struct vmbus_channel *primary_channel,1617 void (*sc_cr_cb)(struct vmbus_channel *new_sc))1618{1619 primary_channel->sc_creation_callback = sc_cr_cb;1620}1621EXPORT_SYMBOL_GPL(vmbus_set_sc_create_callback);1622 1623void vmbus_set_chn_rescind_callback(struct vmbus_channel *channel,1624 void (*chn_rescind_cb)(struct vmbus_channel *))1625{1626 channel->chn_rescind_callback = chn_rescind_cb;1627}1628EXPORT_SYMBOL_GPL(vmbus_set_chn_rescind_callback);1629