605 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * VMware VMCI Driver4 *5 * Copyright (C) 2012 VMware, Inc. All rights reserved.6 */7 8#include <linux/vmw_vmci_defs.h>9#include <linux/vmw_vmci_api.h>10#include <linux/completion.h>11#include <linux/hash.h>12#include <linux/kernel.h>13#include <linux/list.h>14#include <linux/module.h>15#include <linux/sched.h>16#include <linux/slab.h>17 18#include "vmci_datagram.h"19#include "vmci_doorbell.h"20#include "vmci_resource.h"21#include "vmci_driver.h"22#include "vmci_route.h"23 24 25#define VMCI_DOORBELL_INDEX_BITS 626#define VMCI_DOORBELL_INDEX_TABLE_SIZE (1 << VMCI_DOORBELL_INDEX_BITS)27#define VMCI_DOORBELL_HASH(_idx) hash_32(_idx, VMCI_DOORBELL_INDEX_BITS)28 29/*30 * DoorbellEntry describes the a doorbell notification handle allocated by the31 * host.32 */33struct dbell_entry {34 struct vmci_resource resource;35 struct hlist_node node;36 struct work_struct work;37 vmci_callback notify_cb;38 void *client_data;39 u32 idx;40 u32 priv_flags;41 bool run_delayed;42 atomic_t active; /* Only used by guest personality */43};44 45/* The VMCI index table keeps track of currently registered doorbells. */46struct dbell_index_table {47 spinlock_t lock; /* Index table lock */48 struct hlist_head entries[VMCI_DOORBELL_INDEX_TABLE_SIZE];49};50 51static struct dbell_index_table vmci_doorbell_it = {52 .lock = __SPIN_LOCK_UNLOCKED(vmci_doorbell_it.lock),53};54 55/*56 * The max_notify_idx is one larger than the currently known bitmap index in57 * use, and is used to determine how much of the bitmap needs to be scanned.58 */59static u32 max_notify_idx;60 61/*62 * The notify_idx_count is used for determining whether there are free entries63 * within the bitmap (if notify_idx_count + 1 < max_notify_idx).64 */65static u32 notify_idx_count;66 67/*68 * The last_notify_idx_reserved is used to track the last index handed out - in69 * the case where multiple handles share a notification index, we hand out70 * indexes round robin based on last_notify_idx_reserved.71 */72static u32 last_notify_idx_reserved;73 74/* This is a one entry cache used to by the index allocation. */75static u32 last_notify_idx_released = PAGE_SIZE;76 77 78/*79 * Utility function that retrieves the privilege flags associated80 * with a given doorbell handle. For guest endpoints, the81 * privileges are determined by the context ID, but for host82 * endpoints privileges are associated with the complete83 * handle. Hypervisor endpoints are not yet supported.84 */85int vmci_dbell_get_priv_flags(struct vmci_handle handle, u32 *priv_flags)86{87 if (priv_flags == NULL || handle.context == VMCI_INVALID_ID)88 return VMCI_ERROR_INVALID_ARGS;89 90 if (handle.context == VMCI_HOST_CONTEXT_ID) {91 struct dbell_entry *entry;92 struct vmci_resource *resource;93 94 resource = vmci_resource_by_handle(handle,95 VMCI_RESOURCE_TYPE_DOORBELL);96 if (!resource)97 return VMCI_ERROR_NOT_FOUND;98 99 entry = container_of(resource, struct dbell_entry, resource);100 *priv_flags = entry->priv_flags;101 vmci_resource_put(resource);102 } else if (handle.context == VMCI_HYPERVISOR_CONTEXT_ID) {103 /*104 * Hypervisor endpoints for notifications are not105 * supported (yet).106 */107 return VMCI_ERROR_INVALID_ARGS;108 } else {109 *priv_flags = vmci_context_get_priv_flags(handle.context);110 }111 112 return VMCI_SUCCESS;113}114 115/*116 * Find doorbell entry by bitmap index.117 */118static struct dbell_entry *dbell_index_table_find(u32 idx)119{120 u32 bucket = VMCI_DOORBELL_HASH(idx);121 struct dbell_entry *dbell;122 123 hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket],124 node) {125 if (idx == dbell->idx)126 return dbell;127 }128 129 return NULL;130}131 132/*133 * Add the given entry to the index table. This willi take a reference to the134 * entry's resource so that the entry is not deleted before it is removed from135 * the * table.136 */137static void dbell_index_table_add(struct dbell_entry *entry)138{139 u32 bucket;140 u32 new_notify_idx;141 142 vmci_resource_get(&entry->resource);143 144 spin_lock_bh(&vmci_doorbell_it.lock);145 146 /*147 * Below we try to allocate an index in the notification148 * bitmap with "not too much" sharing between resources. If we149 * use less that the full bitmap, we either add to the end if150 * there are no unused flags within the currently used area,151 * or we search for unused ones. If we use the full bitmap, we152 * allocate the index round robin.153 */154 if (max_notify_idx < PAGE_SIZE || notify_idx_count < PAGE_SIZE) {155 if (last_notify_idx_released < max_notify_idx &&156 !dbell_index_table_find(last_notify_idx_released)) {157 new_notify_idx = last_notify_idx_released;158 last_notify_idx_released = PAGE_SIZE;159 } else {160 bool reused = false;161 new_notify_idx = last_notify_idx_reserved;162 if (notify_idx_count + 1 < max_notify_idx) {163 do {164 if (!dbell_index_table_find165 (new_notify_idx)) {166 reused = true;167 break;168 }169 new_notify_idx = (new_notify_idx + 1) %170 max_notify_idx;171 } while (new_notify_idx !=172 last_notify_idx_released);173 }174 if (!reused) {175 new_notify_idx = max_notify_idx;176 max_notify_idx++;177 }178 }179 } else {180 new_notify_idx = (last_notify_idx_reserved + 1) % PAGE_SIZE;181 }182 183 last_notify_idx_reserved = new_notify_idx;184 notify_idx_count++;185 186 entry->idx = new_notify_idx;187 bucket = VMCI_DOORBELL_HASH(entry->idx);188 hlist_add_head(&entry->node, &vmci_doorbell_it.entries[bucket]);189 190 spin_unlock_bh(&vmci_doorbell_it.lock);191}192 193/*194 * Remove the given entry from the index table. This will release() the195 * entry's resource.196 */197static void dbell_index_table_remove(struct dbell_entry *entry)198{199 spin_lock_bh(&vmci_doorbell_it.lock);200 201 hlist_del_init(&entry->node);202 203 notify_idx_count--;204 if (entry->idx == max_notify_idx - 1) {205 /*206 * If we delete an entry with the maximum known207 * notification index, we take the opportunity to208 * prune the current max. As there might be other209 * unused indices immediately below, we lower the210 * maximum until we hit an index in use.211 */212 while (max_notify_idx > 0 &&213 !dbell_index_table_find(max_notify_idx - 1))214 max_notify_idx--;215 }216 217 last_notify_idx_released = entry->idx;218 219 spin_unlock_bh(&vmci_doorbell_it.lock);220 221 vmci_resource_put(&entry->resource);222}223 224/*225 * Creates a link between the given doorbell handle and the given226 * index in the bitmap in the device backend. A notification state227 * is created in hypervisor.228 */229static int dbell_link(struct vmci_handle handle, u32 notify_idx)230{231 struct vmci_doorbell_link_msg link_msg;232 233 link_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,234 VMCI_DOORBELL_LINK);235 link_msg.hdr.src = VMCI_ANON_SRC_HANDLE;236 link_msg.hdr.payload_size = sizeof(link_msg) - VMCI_DG_HEADERSIZE;237 link_msg.handle = handle;238 link_msg.notify_idx = notify_idx;239 240 return vmci_send_datagram(&link_msg.hdr);241}242 243/*244 * Unlinks the given doorbell handle from an index in the bitmap in245 * the device backend. The notification state is destroyed in hypervisor.246 */247static int dbell_unlink(struct vmci_handle handle)248{249 struct vmci_doorbell_unlink_msg unlink_msg;250 251 unlink_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,252 VMCI_DOORBELL_UNLINK);253 unlink_msg.hdr.src = VMCI_ANON_SRC_HANDLE;254 unlink_msg.hdr.payload_size = sizeof(unlink_msg) - VMCI_DG_HEADERSIZE;255 unlink_msg.handle = handle;256 257 return vmci_send_datagram(&unlink_msg.hdr);258}259 260/*261 * Notify another guest or the host. We send a datagram down to the262 * host via the hypervisor with the notification info.263 */264static int dbell_notify_as_guest(struct vmci_handle handle, u32 priv_flags)265{266 struct vmci_doorbell_notify_msg notify_msg;267 268 notify_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,269 VMCI_DOORBELL_NOTIFY);270 notify_msg.hdr.src = VMCI_ANON_SRC_HANDLE;271 notify_msg.hdr.payload_size = sizeof(notify_msg) - VMCI_DG_HEADERSIZE;272 notify_msg.handle = handle;273 274 return vmci_send_datagram(¬ify_msg.hdr);275}276 277/*278 * Calls the specified callback in a delayed context.279 */280static void dbell_delayed_dispatch(struct work_struct *work)281{282 struct dbell_entry *entry = container_of(work,283 struct dbell_entry, work);284 285 entry->notify_cb(entry->client_data);286 vmci_resource_put(&entry->resource);287}288 289/*290 * Dispatches a doorbell notification to the host context.291 */292int vmci_dbell_host_context_notify(u32 src_cid, struct vmci_handle handle)293{294 struct dbell_entry *entry;295 struct vmci_resource *resource;296 297 if (vmci_handle_is_invalid(handle)) {298 pr_devel("Notifying an invalid doorbell (handle=0x%x:0x%x)\n",299 handle.context, handle.resource);300 return VMCI_ERROR_INVALID_ARGS;301 }302 303 resource = vmci_resource_by_handle(handle,304 VMCI_RESOURCE_TYPE_DOORBELL);305 if (!resource) {306 pr_devel("Notifying an unknown doorbell (handle=0x%x:0x%x)\n",307 handle.context, handle.resource);308 return VMCI_ERROR_NOT_FOUND;309 }310 311 entry = container_of(resource, struct dbell_entry, resource);312 if (entry->run_delayed) {313 if (!schedule_work(&entry->work))314 vmci_resource_put(resource);315 } else {316 entry->notify_cb(entry->client_data);317 vmci_resource_put(resource);318 }319 320 return VMCI_SUCCESS;321}322 323/*324 * Register the notification bitmap with the host.325 */326bool vmci_dbell_register_notification_bitmap(u64 bitmap_ppn)327{328 int result;329 struct vmci_notify_bm_set_msg bitmap_set_msg = { };330 331 bitmap_set_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,332 VMCI_SET_NOTIFY_BITMAP);333 bitmap_set_msg.hdr.src = VMCI_ANON_SRC_HANDLE;334 bitmap_set_msg.hdr.payload_size = sizeof(bitmap_set_msg) -335 VMCI_DG_HEADERSIZE;336 if (vmci_use_ppn64())337 bitmap_set_msg.bitmap_ppn64 = bitmap_ppn;338 else339 bitmap_set_msg.bitmap_ppn32 = (u32) bitmap_ppn;340 341 result = vmci_send_datagram(&bitmap_set_msg.hdr);342 if (result != VMCI_SUCCESS) {343 pr_devel("Failed to register (PPN=%llu) as notification bitmap (error=%d)\n",344 bitmap_ppn, result);345 return false;346 }347 return true;348}349 350/*351 * Executes or schedules the handlers for a given notify index.352 */353static void dbell_fire_entries(u32 notify_idx)354{355 u32 bucket = VMCI_DOORBELL_HASH(notify_idx);356 struct dbell_entry *dbell;357 358 spin_lock_bh(&vmci_doorbell_it.lock);359 360 hlist_for_each_entry(dbell, &vmci_doorbell_it.entries[bucket], node) {361 if (dbell->idx == notify_idx &&362 atomic_read(&dbell->active) == 1) {363 if (dbell->run_delayed) {364 vmci_resource_get(&dbell->resource);365 if (!schedule_work(&dbell->work))366 vmci_resource_put(&dbell->resource);367 } else {368 dbell->notify_cb(dbell->client_data);369 }370 }371 }372 373 spin_unlock_bh(&vmci_doorbell_it.lock);374}375 376/*377 * Scans the notification bitmap, collects pending notifications,378 * resets the bitmap and invokes appropriate callbacks.379 */380void vmci_dbell_scan_notification_entries(u8 *bitmap)381{382 u32 idx;383 384 for (idx = 0; idx < max_notify_idx; idx++) {385 if (bitmap[idx] & 0x1) {386 bitmap[idx] &= ~1;387 dbell_fire_entries(idx);388 }389 }390}391 392/*393 * vmci_doorbell_create() - Creates a doorbell394 * @handle: A handle used to track the resource. Can be invalid.395 * @flags: Flag that determines context of callback.396 * @priv_flags: Privileges flags.397 * @notify_cb: The callback to be ivoked when the doorbell fires.398 * @client_data: A parameter to be passed to the callback.399 *400 * Creates a doorbell with the given callback. If the handle is401 * VMCI_INVALID_HANDLE, a free handle will be assigned, if402 * possible. The callback can be run immediately (potentially with403 * locks held - the default) or delayed (in a kernel thread) by404 * specifying the flag VMCI_FLAG_DELAYED_CB. If delayed execution405 * is selected, a given callback may not be run if the kernel is406 * unable to allocate memory for the delayed execution (highly407 * unlikely).408 */409int vmci_doorbell_create(struct vmci_handle *handle,410 u32 flags,411 u32 priv_flags,412 vmci_callback notify_cb, void *client_data)413{414 struct dbell_entry *entry;415 struct vmci_handle new_handle;416 int result;417 418 if (!handle || !notify_cb || flags & ~VMCI_FLAG_DELAYED_CB ||419 priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS)420 return VMCI_ERROR_INVALID_ARGS;421 422 entry = kmalloc(sizeof(*entry), GFP_KERNEL);423 if (entry == NULL) {424 pr_warn("Failed allocating memory for datagram entry\n");425 return VMCI_ERROR_NO_MEM;426 }427 428 if (vmci_handle_is_invalid(*handle)) {429 u32 context_id = vmci_get_context_id();430 431 if (context_id == VMCI_INVALID_ID) {432 pr_warn("Failed to get context ID\n");433 result = VMCI_ERROR_NO_RESOURCES;434 goto free_mem;435 }436 437 /* Let resource code allocate a free ID for us */438 new_handle = vmci_make_handle(context_id, VMCI_INVALID_ID);439 } else {440 bool valid_context = false;441 442 /*443 * Validate the handle. We must do both of the checks below444 * because we can be acting as both a host and a guest at the445 * same time. We always allow the host context ID, since the446 * host functionality is in practice always there with the447 * unified driver.448 */449 if (handle->context == VMCI_HOST_CONTEXT_ID ||450 (vmci_guest_code_active() &&451 vmci_get_context_id() == handle->context)) {452 valid_context = true;453 }454 455 if (!valid_context || handle->resource == VMCI_INVALID_ID) {456 pr_devel("Invalid argument (handle=0x%x:0x%x)\n",457 handle->context, handle->resource);458 result = VMCI_ERROR_INVALID_ARGS;459 goto free_mem;460 }461 462 new_handle = *handle;463 }464 465 entry->idx = 0;466 INIT_HLIST_NODE(&entry->node);467 entry->priv_flags = priv_flags;468 INIT_WORK(&entry->work, dbell_delayed_dispatch);469 entry->run_delayed = flags & VMCI_FLAG_DELAYED_CB;470 entry->notify_cb = notify_cb;471 entry->client_data = client_data;472 atomic_set(&entry->active, 0);473 474 result = vmci_resource_add(&entry->resource,475 VMCI_RESOURCE_TYPE_DOORBELL,476 new_handle);477 if (result != VMCI_SUCCESS) {478 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d\n",479 new_handle.context, new_handle.resource, result);480 goto free_mem;481 }482 483 new_handle = vmci_resource_handle(&entry->resource);484 if (vmci_guest_code_active()) {485 dbell_index_table_add(entry);486 result = dbell_link(new_handle, entry->idx);487 if (VMCI_SUCCESS != result)488 goto destroy_resource;489 490 atomic_set(&entry->active, 1);491 }492 493 *handle = new_handle;494 495 return result;496 497 destroy_resource:498 dbell_index_table_remove(entry);499 vmci_resource_remove(&entry->resource);500 free_mem:501 kfree(entry);502 return result;503}504EXPORT_SYMBOL_GPL(vmci_doorbell_create);505 506/*507 * vmci_doorbell_destroy() - Destroy a doorbell.508 * @handle: The handle tracking the resource.509 *510 * Destroys a doorbell previously created with vmcii_doorbell_create. This511 * operation may block waiting for a callback to finish.512 */513int vmci_doorbell_destroy(struct vmci_handle handle)514{515 struct dbell_entry *entry;516 struct vmci_resource *resource;517 518 if (vmci_handle_is_invalid(handle))519 return VMCI_ERROR_INVALID_ARGS;520 521 resource = vmci_resource_by_handle(handle,522 VMCI_RESOURCE_TYPE_DOORBELL);523 if (!resource) {524 pr_devel("Failed to destroy doorbell (handle=0x%x:0x%x)\n",525 handle.context, handle.resource);526 return VMCI_ERROR_NOT_FOUND;527 }528 529 entry = container_of(resource, struct dbell_entry, resource);530 531 if (!hlist_unhashed(&entry->node)) {532 int result;533 534 dbell_index_table_remove(entry);535 536 result = dbell_unlink(handle);537 if (VMCI_SUCCESS != result) {538 539 /*540 * The only reason this should fail would be541 * an inconsistency between guest and542 * hypervisor state, where the guest believes543 * it has an active registration whereas the544 * hypervisor doesn't. One case where this may545 * happen is if a doorbell is unregistered546 * following a hibernation at a time where the547 * doorbell state hasn't been restored on the548 * hypervisor side yet. Since the handle has549 * now been removed in the guest, we just550 * print a warning and return success.551 */552 pr_devel("Unlink of doorbell (handle=0x%x:0x%x) unknown by hypervisor (error=%d)\n",553 handle.context, handle.resource, result);554 }555 }556 557 /*558 * Now remove the resource from the table. It might still be in use559 * after this, in a callback or still on the delayed work queue.560 */561 vmci_resource_put(&entry->resource);562 vmci_resource_remove(&entry->resource);563 564 kfree(entry);565 566 return VMCI_SUCCESS;567}568EXPORT_SYMBOL_GPL(vmci_doorbell_destroy);569 570/*571 * vmci_doorbell_notify() - Ring the doorbell (and hide in the bushes).572 * @dst: The handlle identifying the doorbell resource573 * @priv_flags: Priviledge flags.574 *575 * Generates a notification on the doorbell identified by the576 * handle. For host side generation of notifications, the caller577 * can specify what the privilege of the calling side is.578 */579int vmci_doorbell_notify(struct vmci_handle dst, u32 priv_flags)580{581 int retval;582 enum vmci_route route;583 struct vmci_handle src;584 585 if (vmci_handle_is_invalid(dst) ||586 (priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS))587 return VMCI_ERROR_INVALID_ARGS;588 589 src = VMCI_INVALID_HANDLE;590 retval = vmci_route(&src, &dst, false, &route);591 if (retval < VMCI_SUCCESS)592 return retval;593 594 if (VMCI_ROUTE_AS_HOST == route)595 return vmci_ctx_notify_dbell(VMCI_HOST_CONTEXT_ID,596 dst, priv_flags);597 598 if (VMCI_ROUTE_AS_GUEST == route)599 return dbell_notify_as_guest(dst, priv_flags);600 601 pr_warn("Unknown route (%d) for doorbell\n", route);602 return VMCI_ERROR_DST_UNREACHABLE;603}604EXPORT_SYMBOL_GPL(vmci_doorbell_notify);605