653 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * ACRN_HSM: Handle I/O requests4 *5 * Copyright (C) 2020 Intel Corporation. All rights reserved.6 *7 * Authors:8 * Jason Chen CJ <jason.cj.chen@intel.com>9 * Fengwei Yin <fengwei.yin@intel.com>10 */11 12#include <linux/interrupt.h>13#include <linux/io.h>14#include <linux/kthread.h>15#include <linux/mm.h>16#include <linux/slab.h>17 18#include <asm/acrn.h>19 20#include "acrn_drv.h"21 22static void ioreq_pause(void);23static void ioreq_resume(void);24 25static void ioreq_dispatcher(struct work_struct *work);26static struct workqueue_struct *ioreq_wq;27static DECLARE_WORK(ioreq_work, ioreq_dispatcher);28 29static inline bool has_pending_request(struct acrn_ioreq_client *client)30{31 return !bitmap_empty(client->ioreqs_map, ACRN_IO_REQUEST_MAX);32}33 34static inline bool is_destroying(struct acrn_ioreq_client *client)35{36 return test_bit(ACRN_IOREQ_CLIENT_DESTROYING, &client->flags);37}38 39static int ioreq_complete_request(struct acrn_vm *vm, u16 vcpu,40 struct acrn_io_request *acrn_req)41{42 bool polling_mode;43 int ret = 0;44 45 polling_mode = acrn_req->completion_polling;46 /* Add barrier() to make sure the writes are done before completion */47 smp_store_release(&acrn_req->processed, ACRN_IOREQ_STATE_COMPLETE);48 49 /*50 * To fulfill the requirement of real-time in several industry51 * scenarios, like automotive, ACRN can run under the partition mode,52 * in which User VMs and Service VM are bound to dedicated CPU cores.53 * Polling mode of handling the I/O request is introduced to achieve a54 * faster I/O request handling. In polling mode, the hypervisor polls55 * I/O request's completion. Once an I/O request is marked as56 * ACRN_IOREQ_STATE_COMPLETE, hypervisor resumes from the polling point57 * to continue the I/O request flow. Thus, the completion notification58 * from HSM of I/O request is not needed. Please note,59 * completion_polling needs to be read before the I/O request being60 * marked as ACRN_IOREQ_STATE_COMPLETE to avoid racing with the61 * hypervisor.62 */63 if (!polling_mode) {64 ret = hcall_notify_req_finish(vm->vmid, vcpu);65 if (ret < 0)66 dev_err(acrn_dev.this_device,67 "Notify I/O request finished failed!\n");68 }69 70 return ret;71}72 73static int acrn_ioreq_complete_request(struct acrn_ioreq_client *client,74 u16 vcpu,75 struct acrn_io_request *acrn_req)76{77 int ret;78 79 if (vcpu >= client->vm->vcpu_num)80 return -EINVAL;81 82 clear_bit(vcpu, client->ioreqs_map);83 if (!acrn_req) {84 acrn_req = (struct acrn_io_request *)client->vm->ioreq_buf;85 acrn_req += vcpu;86 }87 88 ret = ioreq_complete_request(client->vm, vcpu, acrn_req);89 90 return ret;91}92 93int acrn_ioreq_request_default_complete(struct acrn_vm *vm, u16 vcpu)94{95 int ret = 0;96 97 spin_lock_bh(&vm->ioreq_clients_lock);98 if (vm->default_client)99 ret = acrn_ioreq_complete_request(vm->default_client,100 vcpu, NULL);101 spin_unlock_bh(&vm->ioreq_clients_lock);102 103 return ret;104}105 106/**107 * acrn_ioreq_range_add() - Add an iorange monitored by an ioreq client108 * @client: The ioreq client109 * @type: Type (ACRN_IOREQ_TYPE_MMIO or ACRN_IOREQ_TYPE_PORTIO)110 * @start: Start address of iorange111 * @end: End address of iorange112 *113 * Return: 0 on success, <0 on error114 */115int acrn_ioreq_range_add(struct acrn_ioreq_client *client,116 u32 type, u64 start, u64 end)117{118 struct acrn_ioreq_range *range;119 120 if (end < start) {121 dev_err(acrn_dev.this_device,122 "Invalid IO range [0x%llx,0x%llx]\n", start, end);123 return -EINVAL;124 }125 126 range = kzalloc(sizeof(*range), GFP_KERNEL);127 if (!range)128 return -ENOMEM;129 130 range->type = type;131 range->start = start;132 range->end = end;133 134 write_lock_bh(&client->range_lock);135 list_add(&range->list, &client->range_list);136 write_unlock_bh(&client->range_lock);137 138 return 0;139}140 141/**142 * acrn_ioreq_range_del() - Del an iorange monitored by an ioreq client143 * @client: The ioreq client144 * @type: Type (ACRN_IOREQ_TYPE_MMIO or ACRN_IOREQ_TYPE_PORTIO)145 * @start: Start address of iorange146 * @end: End address of iorange147 */148void acrn_ioreq_range_del(struct acrn_ioreq_client *client,149 u32 type, u64 start, u64 end)150{151 struct acrn_ioreq_range *range;152 153 write_lock_bh(&client->range_lock);154 list_for_each_entry(range, &client->range_list, list) {155 if (type == range->type &&156 start == range->start &&157 end == range->end) {158 list_del(&range->list);159 kfree(range);160 break;161 }162 }163 write_unlock_bh(&client->range_lock);164}165 166/*167 * ioreq_task() is the execution entity of handler thread of an I/O client.168 * The handler callback of the I/O client is called within the handler thread.169 */170static int ioreq_task(void *data)171{172 struct acrn_ioreq_client *client = data;173 struct acrn_io_request *req;174 unsigned long *ioreqs_map;175 int vcpu, ret;176 177 /*178 * Lockless access to ioreqs_map is safe, because179 * 1) set_bit() and clear_bit() are atomic operations.180 * 2) I/O requests arrives serialized. The access flow of ioreqs_map is:181 * set_bit() - in ioreq_work handler182 * Handler callback handles corresponding I/O request183 * clear_bit() - in handler thread (include ACRN userspace)184 * Mark corresponding I/O request completed185 * Loop again if a new I/O request occurs186 */187 ioreqs_map = client->ioreqs_map;188 while (!kthread_should_stop()) {189 acrn_ioreq_client_wait(client);190 while (has_pending_request(client)) {191 vcpu = find_first_bit(ioreqs_map, client->vm->vcpu_num);192 req = client->vm->ioreq_buf->req_slot + vcpu;193 ret = client->handler(client, req);194 if (ret < 0) {195 dev_err(acrn_dev.this_device,196 "IO handle failure: %d\n", ret);197 break;198 }199 acrn_ioreq_complete_request(client, vcpu, req);200 }201 }202 203 return 0;204}205 206/*207 * For the non-default I/O clients, give them chance to complete the current208 * I/O requests if there are any. For the default I/O client, it is safe to209 * clear all pending I/O requests because the clearing request is from ACRN210 * userspace.211 */212void acrn_ioreq_request_clear(struct acrn_vm *vm)213{214 struct acrn_ioreq_client *client;215 bool has_pending = false;216 unsigned long vcpu;217 int retry = 10;218 219 /*220 * IO requests of this VM will be completed directly in221 * acrn_ioreq_dispatch if ACRN_VM_FLAG_CLEARING_IOREQ flag is set.222 */223 set_bit(ACRN_VM_FLAG_CLEARING_IOREQ, &vm->flags);224 225 /*226 * acrn_ioreq_request_clear is only called in VM reset case. Simply227 * wait 100ms in total for the IO requests' completion.228 */229 do {230 spin_lock_bh(&vm->ioreq_clients_lock);231 list_for_each_entry(client, &vm->ioreq_clients, list) {232 has_pending = has_pending_request(client);233 if (has_pending)234 break;235 }236 spin_unlock_bh(&vm->ioreq_clients_lock);237 238 if (has_pending)239 schedule_timeout_interruptible(HZ / 100);240 } while (has_pending && --retry > 0);241 if (retry == 0)242 dev_warn(acrn_dev.this_device,243 "%s cannot flush pending request!\n", client->name);244 245 /* Clear all ioreqs belonging to the default client */246 spin_lock_bh(&vm->ioreq_clients_lock);247 client = vm->default_client;248 if (client) {249 for_each_set_bit(vcpu, client->ioreqs_map, ACRN_IO_REQUEST_MAX)250 acrn_ioreq_complete_request(client, vcpu, NULL);251 }252 spin_unlock_bh(&vm->ioreq_clients_lock);253 254 /* Clear ACRN_VM_FLAG_CLEARING_IOREQ flag after the clearing */255 clear_bit(ACRN_VM_FLAG_CLEARING_IOREQ, &vm->flags);256}257 258int acrn_ioreq_client_wait(struct acrn_ioreq_client *client)259{260 if (client->is_default) {261 /*262 * In the default client, a user space thread waits on the263 * waitqueue. The is_destroying() check is used to notify user264 * space the client is going to be destroyed.265 */266 wait_event_interruptible(client->wq,267 has_pending_request(client) ||268 is_destroying(client));269 if (is_destroying(client))270 return -ENODEV;271 } else {272 wait_event_interruptible(client->wq,273 has_pending_request(client) ||274 kthread_should_stop());275 }276 277 return 0;278}279 280static bool is_cfg_addr(struct acrn_io_request *req)281{282 return ((req->type == ACRN_IOREQ_TYPE_PORTIO) &&283 (req->reqs.pio_request.address == 0xcf8));284}285 286static bool is_cfg_data(struct acrn_io_request *req)287{288 return ((req->type == ACRN_IOREQ_TYPE_PORTIO) &&289 ((req->reqs.pio_request.address >= 0xcfc) &&290 (req->reqs.pio_request.address < (0xcfc + 4))));291}292 293/* The low 8-bit of supported pci_reg addr.*/294#define PCI_LOWREG_MASK 0xFC295/* The high 4-bit of supported pci_reg addr */296#define PCI_HIGHREG_MASK 0xF00297/* Max number of supported functions */298#define PCI_FUNCMAX 7299/* Max number of supported slots */300#define PCI_SLOTMAX 31301/* Max number of supported buses */302#define PCI_BUSMAX 255303#define CONF1_ENABLE 0x80000000UL304/*305 * A PCI configuration space access via PIO 0xCF8 and 0xCFC normally has two306 * following steps:307 * 1) writes address into 0xCF8 port308 * 2) accesses data in/from 0xCFC309 * This function combines such paired PCI configuration space I/O requests into310 * one ACRN_IOREQ_TYPE_PCICFG type I/O request and continues the processing.311 */312static bool handle_cf8cfc(struct acrn_vm *vm,313 struct acrn_io_request *req, u16 vcpu)314{315 int offset, pci_cfg_addr, pci_reg;316 bool is_handled = false;317 318 if (is_cfg_addr(req)) {319 WARN_ON(req->reqs.pio_request.size != 4);320 if (req->reqs.pio_request.direction == ACRN_IOREQ_DIR_WRITE)321 vm->pci_conf_addr = req->reqs.pio_request.value;322 else323 req->reqs.pio_request.value = vm->pci_conf_addr;324 is_handled = true;325 } else if (is_cfg_data(req)) {326 if (!(vm->pci_conf_addr & CONF1_ENABLE)) {327 if (req->reqs.pio_request.direction ==328 ACRN_IOREQ_DIR_READ)329 req->reqs.pio_request.value = 0xffffffff;330 is_handled = true;331 } else {332 offset = req->reqs.pio_request.address - 0xcfc;333 334 req->type = ACRN_IOREQ_TYPE_PCICFG;335 pci_cfg_addr = vm->pci_conf_addr;336 req->reqs.pci_request.bus =337 (pci_cfg_addr >> 16) & PCI_BUSMAX;338 req->reqs.pci_request.dev =339 (pci_cfg_addr >> 11) & PCI_SLOTMAX;340 req->reqs.pci_request.func =341 (pci_cfg_addr >> 8) & PCI_FUNCMAX;342 pci_reg = (pci_cfg_addr & PCI_LOWREG_MASK) +343 ((pci_cfg_addr >> 16) & PCI_HIGHREG_MASK);344 req->reqs.pci_request.reg = pci_reg + offset;345 }346 }347 348 if (is_handled)349 ioreq_complete_request(vm, vcpu, req);350 351 return is_handled;352}353 354static bool acrn_in_range(struct acrn_ioreq_range *range,355 struct acrn_io_request *req)356{357 bool ret = false;358 359 if (range->type == req->type) {360 switch (req->type) {361 case ACRN_IOREQ_TYPE_MMIO:362 if (req->reqs.mmio_request.address >= range->start &&363 (req->reqs.mmio_request.address +364 req->reqs.mmio_request.size - 1) <= range->end)365 ret = true;366 break;367 case ACRN_IOREQ_TYPE_PORTIO:368 if (req->reqs.pio_request.address >= range->start &&369 (req->reqs.pio_request.address +370 req->reqs.pio_request.size - 1) <= range->end)371 ret = true;372 break;373 default:374 break;375 }376 }377 378 return ret;379}380 381static struct acrn_ioreq_client *find_ioreq_client(struct acrn_vm *vm,382 struct acrn_io_request *req)383{384 struct acrn_ioreq_client *client, *found = NULL;385 struct acrn_ioreq_range *range;386 387 lockdep_assert_held(&vm->ioreq_clients_lock);388 389 list_for_each_entry(client, &vm->ioreq_clients, list) {390 read_lock_bh(&client->range_lock);391 list_for_each_entry(range, &client->range_list, list) {392 if (acrn_in_range(range, req)) {393 found = client;394 break;395 }396 }397 read_unlock_bh(&client->range_lock);398 if (found)399 break;400 }401 return found ? found : vm->default_client;402}403 404/**405 * acrn_ioreq_client_create() - Create an ioreq client406 * @vm: The VM that this client belongs to407 * @handler: The ioreq_handler of ioreq client acrn_hsm will create a kernel408 * thread and call the handler to handle I/O requests.409 * @priv: Private data for the handler410 * @is_default: If it is the default client411 * @name: The name of ioreq client412 *413 * Return: acrn_ioreq_client pointer on success, NULL on error414 */415struct acrn_ioreq_client *acrn_ioreq_client_create(struct acrn_vm *vm,416 ioreq_handler_t handler,417 void *priv, bool is_default,418 const char *name)419{420 struct acrn_ioreq_client *client;421 422 if (!handler && !is_default) {423 dev_dbg(acrn_dev.this_device,424 "Cannot create non-default client w/o handler!\n");425 return NULL;426 }427 client = kzalloc(sizeof(*client), GFP_KERNEL);428 if (!client)429 return NULL;430 431 client->handler = handler;432 client->vm = vm;433 client->priv = priv;434 client->is_default = is_default;435 if (name)436 strscpy(client->name, name);437 rwlock_init(&client->range_lock);438 INIT_LIST_HEAD(&client->range_list);439 init_waitqueue_head(&client->wq);440 441 if (client->handler) {442 client->thread = kthread_run(ioreq_task, client, "VM%u-%s",443 client->vm->vmid, client->name);444 if (IS_ERR(client->thread)) {445 kfree(client);446 return NULL;447 }448 }449 450 spin_lock_bh(&vm->ioreq_clients_lock);451 if (is_default)452 vm->default_client = client;453 else454 list_add(&client->list, &vm->ioreq_clients);455 spin_unlock_bh(&vm->ioreq_clients_lock);456 457 dev_dbg(acrn_dev.this_device, "Created ioreq client %s.\n", name);458 return client;459}460 461/**462 * acrn_ioreq_client_destroy() - Destroy an ioreq client463 * @client: The ioreq client464 */465void acrn_ioreq_client_destroy(struct acrn_ioreq_client *client)466{467 struct acrn_ioreq_range *range, *next;468 struct acrn_vm *vm = client->vm;469 470 dev_dbg(acrn_dev.this_device,471 "Destroy ioreq client %s.\n", client->name);472 ioreq_pause();473 set_bit(ACRN_IOREQ_CLIENT_DESTROYING, &client->flags);474 if (client->is_default)475 wake_up_interruptible(&client->wq);476 else477 kthread_stop(client->thread);478 479 spin_lock_bh(&vm->ioreq_clients_lock);480 if (client->is_default)481 vm->default_client = NULL;482 else483 list_del(&client->list);484 spin_unlock_bh(&vm->ioreq_clients_lock);485 486 write_lock_bh(&client->range_lock);487 list_for_each_entry_safe(range, next, &client->range_list, list) {488 list_del(&range->list);489 kfree(range);490 }491 write_unlock_bh(&client->range_lock);492 kfree(client);493 494 ioreq_resume();495}496 497static int acrn_ioreq_dispatch(struct acrn_vm *vm)498{499 struct acrn_ioreq_client *client;500 struct acrn_io_request *req;501 int i;502 503 for (i = 0; i < vm->vcpu_num; i++) {504 req = vm->ioreq_buf->req_slot + i;505 506 /* barrier the read of processed of acrn_io_request */507 if (smp_load_acquire(&req->processed) ==508 ACRN_IOREQ_STATE_PENDING) {509 /* Complete the IO request directly in clearing stage */510 if (test_bit(ACRN_VM_FLAG_CLEARING_IOREQ, &vm->flags)) {511 ioreq_complete_request(vm, i, req);512 continue;513 }514 if (handle_cf8cfc(vm, req, i))515 continue;516 517 spin_lock_bh(&vm->ioreq_clients_lock);518 client = find_ioreq_client(vm, req);519 if (!client) {520 dev_err(acrn_dev.this_device,521 "Failed to find ioreq client!\n");522 spin_unlock_bh(&vm->ioreq_clients_lock);523 return -EINVAL;524 }525 if (!client->is_default)526 req->kernel_handled = 1;527 else528 req->kernel_handled = 0;529 /*530 * Add barrier() to make sure the writes are done531 * before setting ACRN_IOREQ_STATE_PROCESSING532 */533 smp_store_release(&req->processed,534 ACRN_IOREQ_STATE_PROCESSING);535 set_bit(i, client->ioreqs_map);536 wake_up_interruptible(&client->wq);537 spin_unlock_bh(&vm->ioreq_clients_lock);538 }539 }540 541 return 0;542}543 544static void ioreq_dispatcher(struct work_struct *work)545{546 struct acrn_vm *vm;547 548 read_lock(&acrn_vm_list_lock);549 list_for_each_entry(vm, &acrn_vm_list, list) {550 if (!vm->ioreq_buf)551 break;552 acrn_ioreq_dispatch(vm);553 }554 read_unlock(&acrn_vm_list_lock);555}556 557static void ioreq_intr_handler(void)558{559 queue_work(ioreq_wq, &ioreq_work);560}561 562static void ioreq_pause(void)563{564 /* Flush and unarm the handler to ensure no I/O requests pending */565 acrn_remove_intr_handler();566 drain_workqueue(ioreq_wq);567}568 569static void ioreq_resume(void)570{571 /* Schedule after enabling in case other clients miss interrupt */572 acrn_setup_intr_handler(ioreq_intr_handler);573 queue_work(ioreq_wq, &ioreq_work);574}575 576int acrn_ioreq_intr_setup(void)577{578 acrn_setup_intr_handler(ioreq_intr_handler);579 ioreq_wq = alloc_ordered_workqueue("ioreq_wq",580 WQ_HIGHPRI | WQ_MEM_RECLAIM);581 if (!ioreq_wq) {582 dev_err(acrn_dev.this_device, "Failed to alloc workqueue!\n");583 acrn_remove_intr_handler();584 return -ENOMEM;585 }586 return 0;587}588 589void acrn_ioreq_intr_remove(void)590{591 if (ioreq_wq)592 destroy_workqueue(ioreq_wq);593 acrn_remove_intr_handler();594}595 596int acrn_ioreq_init(struct acrn_vm *vm, u64 buf_vma)597{598 struct acrn_ioreq_buffer *set_buffer;599 struct page *page;600 int ret;601 602 if (vm->ioreq_buf)603 return -EEXIST;604 605 set_buffer = kzalloc(sizeof(*set_buffer), GFP_KERNEL);606 if (!set_buffer)607 return -ENOMEM;608 609 ret = pin_user_pages_fast(buf_vma, 1,610 FOLL_WRITE | FOLL_LONGTERM, &page);611 if (unlikely(ret != 1) || !page) {612 dev_err(acrn_dev.this_device, "Failed to pin ioreq page!\n");613 ret = -EFAULT;614 goto free_buf;615 }616 617 vm->ioreq_buf = page_address(page);618 vm->ioreq_page = page;619 set_buffer->ioreq_buf = page_to_phys(page);620 ret = hcall_set_ioreq_buffer(vm->vmid, virt_to_phys(set_buffer));621 if (ret < 0) {622 dev_err(acrn_dev.this_device, "Failed to init ioreq buffer!\n");623 unpin_user_page(page);624 vm->ioreq_buf = NULL;625 goto free_buf;626 }627 628 dev_dbg(acrn_dev.this_device,629 "Init ioreq buffer %pK!\n", vm->ioreq_buf);630 ret = 0;631free_buf:632 kfree(set_buffer);633 return ret;634}635 636void acrn_ioreq_deinit(struct acrn_vm *vm)637{638 struct acrn_ioreq_client *client, *next;639 640 dev_dbg(acrn_dev.this_device,641 "Deinit ioreq buffer %pK!\n", vm->ioreq_buf);642 /* Destroy all clients belonging to this VM */643 list_for_each_entry_safe(client, next, &vm->ioreq_clients, list)644 acrn_ioreq_client_destroy(client);645 if (vm->default_client)646 acrn_ioreq_client_destroy(vm->default_client);647 648 if (vm->ioreq_buf && vm->ioreq_page) {649 unpin_user_page(vm->ioreq_page);650 vm->ioreq_buf = NULL;651 }652}653