1999 lines · c
1/*2 * KVMGT - the implementation of Intel mediated pass-through framework for KVM3 *4 * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.5 *6 * Permission is hereby granted, free of charge, to any person obtaining a7 * copy of this software and associated documentation files (the "Software"),8 * to deal in the Software without restriction, including without limitation9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,10 * and/or sell copies of the Software, and to permit persons to whom the11 * Software is furnished to do so, subject to the following conditions:12 *13 * The above copyright notice and this permission notice (including the next14 * paragraph) shall be included in all copies or substantial portions of the15 * Software.16 *17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE23 * SOFTWARE.24 *25 * Authors:26 * Kevin Tian <kevin.tian@intel.com>27 * Jike Song <jike.song@intel.com>28 * Xiaoguang Chen <xiaoguang.chen@intel.com>29 * Eddie Dong <eddie.dong@intel.com>30 *31 * Contributors:32 * Niu Bing <bing.niu@intel.com>33 * Zhi Wang <zhi.a.wang@intel.com>34 */35 36#include <linux/init.h>37#include <linux/mm.h>38#include <linux/kthread.h>39#include <linux/sched/mm.h>40#include <linux/types.h>41#include <linux/list.h>42#include <linux/rbtree.h>43#include <linux/spinlock.h>44#include <linux/eventfd.h>45#include <linux/mdev.h>46#include <linux/debugfs.h>47 48#include <linux/nospec.h>49 50#include <drm/drm_edid.h>51 52#include "i915_drv.h"53#include "intel_gvt.h"54#include "gvt.h"55 56MODULE_IMPORT_NS(DMA_BUF);57MODULE_IMPORT_NS(I915_GVT);58 59/* helper macros copied from vfio-pci */60#define VFIO_PCI_OFFSET_SHIFT 4061#define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT)62#define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT)63#define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)64 65#define EDID_BLOB_OFFSET (PAGE_SIZE/2)66 67#define OPREGION_SIGNATURE "IntelGraphicsMem"68 69struct vfio_region;70struct intel_vgpu_regops {71 size_t (*rw)(struct intel_vgpu *vgpu, char *buf,72 size_t count, loff_t *ppos, bool iswrite);73 void (*release)(struct intel_vgpu *vgpu,74 struct vfio_region *region);75};76 77struct vfio_region {78 u32 type;79 u32 subtype;80 size_t size;81 u32 flags;82 const struct intel_vgpu_regops *ops;83 void *data;84};85 86struct vfio_edid_region {87 struct vfio_region_gfx_edid vfio_edid_regs;88 void *edid_blob;89};90 91struct kvmgt_pgfn {92 gfn_t gfn;93 struct hlist_node hnode;94};95 96struct gvt_dma {97 struct intel_vgpu *vgpu;98 struct rb_node gfn_node;99 struct rb_node dma_addr_node;100 gfn_t gfn;101 dma_addr_t dma_addr;102 unsigned long size;103 struct kref ref;104};105 106#define vfio_dev_to_vgpu(vfio_dev) \107 container_of((vfio_dev), struct intel_vgpu, vfio_device)108 109static void kvmgt_page_track_write(gpa_t gpa, const u8 *val, int len,110 struct kvm_page_track_notifier_node *node);111static void kvmgt_page_track_remove_region(gfn_t gfn, unsigned long nr_pages,112 struct kvm_page_track_notifier_node *node);113 114static ssize_t intel_vgpu_show_description(struct mdev_type *mtype, char *buf)115{116 struct intel_vgpu_type *type =117 container_of(mtype, struct intel_vgpu_type, type);118 119 return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n"120 "fence: %d\nresolution: %s\n"121 "weight: %d\n",122 BYTES_TO_MB(type->conf->low_mm),123 BYTES_TO_MB(type->conf->high_mm),124 type->conf->fence, vgpu_edid_str(type->conf->edid),125 type->conf->weight);126}127 128static void gvt_unpin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,129 unsigned long size)130{131 vfio_unpin_pages(&vgpu->vfio_device, gfn << PAGE_SHIFT,132 DIV_ROUND_UP(size, PAGE_SIZE));133}134 135/* Pin a normal or compound guest page for dma. */136static int gvt_pin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,137 unsigned long size, struct page **page)138{139 int total_pages = DIV_ROUND_UP(size, PAGE_SIZE);140 struct page *base_page = NULL;141 int npage;142 int ret;143 144 /*145 * We pin the pages one-by-one to avoid allocating a big arrary146 * on stack to hold pfns.147 */148 for (npage = 0; npage < total_pages; npage++) {149 dma_addr_t cur_iova = (gfn + npage) << PAGE_SHIFT;150 struct page *cur_page;151 152 ret = vfio_pin_pages(&vgpu->vfio_device, cur_iova, 1,153 IOMMU_READ | IOMMU_WRITE, &cur_page);154 if (ret != 1) {155 gvt_vgpu_err("vfio_pin_pages failed for iova %pad, ret %d\n",156 &cur_iova, ret);157 goto err;158 }159 160 if (npage == 0)161 base_page = cur_page;162 else if (page_to_pfn(base_page) + npage != page_to_pfn(cur_page)) {163 ret = -EINVAL;164 npage++;165 goto err;166 }167 }168 169 *page = base_page;170 return 0;171err:172 if (npage)173 gvt_unpin_guest_page(vgpu, gfn, npage * PAGE_SIZE);174 return ret;175}176 177static int gvt_dma_map_page(struct intel_vgpu *vgpu, unsigned long gfn,178 dma_addr_t *dma_addr, unsigned long size)179{180 struct device *dev = vgpu->gvt->gt->i915->drm.dev;181 struct page *page = NULL;182 int ret;183 184 ret = gvt_pin_guest_page(vgpu, gfn, size, &page);185 if (ret)186 return ret;187 188 /* Setup DMA mapping. */189 *dma_addr = dma_map_page(dev, page, 0, size, DMA_BIDIRECTIONAL);190 if (dma_mapping_error(dev, *dma_addr)) {191 gvt_vgpu_err("DMA mapping failed for pfn 0x%lx, ret %d\n",192 page_to_pfn(page), ret);193 gvt_unpin_guest_page(vgpu, gfn, size);194 return -ENOMEM;195 }196 197 return 0;198}199 200static void gvt_dma_unmap_page(struct intel_vgpu *vgpu, unsigned long gfn,201 dma_addr_t dma_addr, unsigned long size)202{203 struct device *dev = vgpu->gvt->gt->i915->drm.dev;204 205 dma_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL);206 gvt_unpin_guest_page(vgpu, gfn, size);207}208 209static struct gvt_dma *__gvt_cache_find_dma_addr(struct intel_vgpu *vgpu,210 dma_addr_t dma_addr)211{212 struct rb_node *node = vgpu->dma_addr_cache.rb_node;213 struct gvt_dma *itr;214 215 while (node) {216 itr = rb_entry(node, struct gvt_dma, dma_addr_node);217 218 if (dma_addr < itr->dma_addr)219 node = node->rb_left;220 else if (dma_addr > itr->dma_addr)221 node = node->rb_right;222 else223 return itr;224 }225 return NULL;226}227 228static struct gvt_dma *__gvt_cache_find_gfn(struct intel_vgpu *vgpu, gfn_t gfn)229{230 struct rb_node *node = vgpu->gfn_cache.rb_node;231 struct gvt_dma *itr;232 233 while (node) {234 itr = rb_entry(node, struct gvt_dma, gfn_node);235 236 if (gfn < itr->gfn)237 node = node->rb_left;238 else if (gfn > itr->gfn)239 node = node->rb_right;240 else241 return itr;242 }243 return NULL;244}245 246static int __gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn,247 dma_addr_t dma_addr, unsigned long size)248{249 struct gvt_dma *new, *itr;250 struct rb_node **link, *parent = NULL;251 252 new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL);253 if (!new)254 return -ENOMEM;255 256 new->vgpu = vgpu;257 new->gfn = gfn;258 new->dma_addr = dma_addr;259 new->size = size;260 kref_init(&new->ref);261 262 /* gfn_cache maps gfn to struct gvt_dma. */263 link = &vgpu->gfn_cache.rb_node;264 while (*link) {265 parent = *link;266 itr = rb_entry(parent, struct gvt_dma, gfn_node);267 268 if (gfn < itr->gfn)269 link = &parent->rb_left;270 else271 link = &parent->rb_right;272 }273 rb_link_node(&new->gfn_node, parent, link);274 rb_insert_color(&new->gfn_node, &vgpu->gfn_cache);275 276 /* dma_addr_cache maps dma addr to struct gvt_dma. */277 parent = NULL;278 link = &vgpu->dma_addr_cache.rb_node;279 while (*link) {280 parent = *link;281 itr = rb_entry(parent, struct gvt_dma, dma_addr_node);282 283 if (dma_addr < itr->dma_addr)284 link = &parent->rb_left;285 else286 link = &parent->rb_right;287 }288 rb_link_node(&new->dma_addr_node, parent, link);289 rb_insert_color(&new->dma_addr_node, &vgpu->dma_addr_cache);290 291 vgpu->nr_cache_entries++;292 return 0;293}294 295static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu,296 struct gvt_dma *entry)297{298 rb_erase(&entry->gfn_node, &vgpu->gfn_cache);299 rb_erase(&entry->dma_addr_node, &vgpu->dma_addr_cache);300 kfree(entry);301 vgpu->nr_cache_entries--;302}303 304static void gvt_cache_destroy(struct intel_vgpu *vgpu)305{306 struct gvt_dma *dma;307 struct rb_node *node = NULL;308 309 for (;;) {310 mutex_lock(&vgpu->cache_lock);311 node = rb_first(&vgpu->gfn_cache);312 if (!node) {313 mutex_unlock(&vgpu->cache_lock);314 break;315 }316 dma = rb_entry(node, struct gvt_dma, gfn_node);317 gvt_dma_unmap_page(vgpu, dma->gfn, dma->dma_addr, dma->size);318 __gvt_cache_remove_entry(vgpu, dma);319 mutex_unlock(&vgpu->cache_lock);320 }321}322 323static void gvt_cache_init(struct intel_vgpu *vgpu)324{325 vgpu->gfn_cache = RB_ROOT;326 vgpu->dma_addr_cache = RB_ROOT;327 vgpu->nr_cache_entries = 0;328 mutex_init(&vgpu->cache_lock);329}330 331static void kvmgt_protect_table_init(struct intel_vgpu *info)332{333 hash_init(info->ptable);334}335 336static void kvmgt_protect_table_destroy(struct intel_vgpu *info)337{338 struct kvmgt_pgfn *p;339 struct hlist_node *tmp;340 int i;341 342 hash_for_each_safe(info->ptable, i, tmp, p, hnode) {343 hash_del(&p->hnode);344 kfree(p);345 }346}347 348static struct kvmgt_pgfn *349__kvmgt_protect_table_find(struct intel_vgpu *info, gfn_t gfn)350{351 struct kvmgt_pgfn *p, *res = NULL;352 353 lockdep_assert_held(&info->vgpu_lock);354 355 hash_for_each_possible(info->ptable, p, hnode, gfn) {356 if (gfn == p->gfn) {357 res = p;358 break;359 }360 }361 362 return res;363}364 365static bool kvmgt_gfn_is_write_protected(struct intel_vgpu *info, gfn_t gfn)366{367 struct kvmgt_pgfn *p;368 369 p = __kvmgt_protect_table_find(info, gfn);370 return !!p;371}372 373static void kvmgt_protect_table_add(struct intel_vgpu *info, gfn_t gfn)374{375 struct kvmgt_pgfn *p;376 377 if (kvmgt_gfn_is_write_protected(info, gfn))378 return;379 380 p = kzalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC);381 if (WARN(!p, "gfn: 0x%llx\n", gfn))382 return;383 384 p->gfn = gfn;385 hash_add(info->ptable, &p->hnode, gfn);386}387 388static void kvmgt_protect_table_del(struct intel_vgpu *info, gfn_t gfn)389{390 struct kvmgt_pgfn *p;391 392 p = __kvmgt_protect_table_find(info, gfn);393 if (p) {394 hash_del(&p->hnode);395 kfree(p);396 }397}398 399static size_t intel_vgpu_reg_rw_opregion(struct intel_vgpu *vgpu, char *buf,400 size_t count, loff_t *ppos, bool iswrite)401{402 unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) -403 VFIO_PCI_NUM_REGIONS;404 void *base = vgpu->region[i].data;405 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;406 407 408 if (pos >= vgpu->region[i].size || iswrite) {409 gvt_vgpu_err("invalid op or offset for Intel vgpu OpRegion\n");410 return -EINVAL;411 }412 count = min(count, (size_t)(vgpu->region[i].size - pos));413 memcpy(buf, base + pos, count);414 415 return count;416}417 418static void intel_vgpu_reg_release_opregion(struct intel_vgpu *vgpu,419 struct vfio_region *region)420{421}422 423static const struct intel_vgpu_regops intel_vgpu_regops_opregion = {424 .rw = intel_vgpu_reg_rw_opregion,425 .release = intel_vgpu_reg_release_opregion,426};427 428static bool edid_valid(const void *edid, size_t size)429{430 const struct drm_edid *drm_edid;431 bool is_valid;432 433 drm_edid = drm_edid_alloc(edid, size);434 is_valid = drm_edid_valid(drm_edid);435 drm_edid_free(drm_edid);436 437 return is_valid;438}439 440static int handle_edid_regs(struct intel_vgpu *vgpu,441 struct vfio_edid_region *region, char *buf,442 size_t count, u16 offset, bool is_write)443{444 struct vfio_region_gfx_edid *regs = ®ion->vfio_edid_regs;445 unsigned int data;446 447 if (offset + count > sizeof(*regs))448 return -EINVAL;449 450 if (count != 4)451 return -EINVAL;452 453 if (is_write) {454 data = *((unsigned int *)buf);455 switch (offset) {456 case offsetof(struct vfio_region_gfx_edid, link_state):457 if (data == VFIO_DEVICE_GFX_LINK_STATE_UP) {458 if (!edid_valid(region->edid_blob, EDID_SIZE)) {459 gvt_vgpu_err("invalid EDID blob\n");460 return -EINVAL;461 }462 intel_vgpu_emulate_hotplug(vgpu, true);463 } else if (data == VFIO_DEVICE_GFX_LINK_STATE_DOWN)464 intel_vgpu_emulate_hotplug(vgpu, false);465 else {466 gvt_vgpu_err("invalid EDID link state %d\n",467 regs->link_state);468 return -EINVAL;469 }470 regs->link_state = data;471 break;472 case offsetof(struct vfio_region_gfx_edid, edid_size):473 if (data > regs->edid_max_size) {474 gvt_vgpu_err("EDID size is bigger than %d!\n",475 regs->edid_max_size);476 return -EINVAL;477 }478 regs->edid_size = data;479 break;480 default:481 /* read-only regs */482 gvt_vgpu_err("write read-only EDID region at offset %d\n",483 offset);484 return -EPERM;485 }486 } else {487 memcpy(buf, (char *)regs + offset, count);488 }489 490 return count;491}492 493static int handle_edid_blob(struct vfio_edid_region *region, char *buf,494 size_t count, u16 offset, bool is_write)495{496 if (offset + count > region->vfio_edid_regs.edid_size)497 return -EINVAL;498 499 if (is_write)500 memcpy(region->edid_blob + offset, buf, count);501 else502 memcpy(buf, region->edid_blob + offset, count);503 504 return count;505}506 507static size_t intel_vgpu_reg_rw_edid(struct intel_vgpu *vgpu, char *buf,508 size_t count, loff_t *ppos, bool iswrite)509{510 int ret;511 unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) -512 VFIO_PCI_NUM_REGIONS;513 struct vfio_edid_region *region = vgpu->region[i].data;514 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;515 516 if (pos < region->vfio_edid_regs.edid_offset) {517 ret = handle_edid_regs(vgpu, region, buf, count, pos, iswrite);518 } else {519 pos -= EDID_BLOB_OFFSET;520 ret = handle_edid_blob(region, buf, count, pos, iswrite);521 }522 523 if (ret < 0)524 gvt_vgpu_err("failed to access EDID region\n");525 526 return ret;527}528 529static void intel_vgpu_reg_release_edid(struct intel_vgpu *vgpu,530 struct vfio_region *region)531{532 kfree(region->data);533}534 535static const struct intel_vgpu_regops intel_vgpu_regops_edid = {536 .rw = intel_vgpu_reg_rw_edid,537 .release = intel_vgpu_reg_release_edid,538};539 540static int intel_vgpu_register_reg(struct intel_vgpu *vgpu,541 unsigned int type, unsigned int subtype,542 const struct intel_vgpu_regops *ops,543 size_t size, u32 flags, void *data)544{545 struct vfio_region *region;546 547 region = krealloc(vgpu->region,548 (vgpu->num_regions + 1) * sizeof(*region),549 GFP_KERNEL);550 if (!region)551 return -ENOMEM;552 553 vgpu->region = region;554 vgpu->region[vgpu->num_regions].type = type;555 vgpu->region[vgpu->num_regions].subtype = subtype;556 vgpu->region[vgpu->num_regions].ops = ops;557 vgpu->region[vgpu->num_regions].size = size;558 vgpu->region[vgpu->num_regions].flags = flags;559 vgpu->region[vgpu->num_regions].data = data;560 vgpu->num_regions++;561 return 0;562}563 564int intel_gvt_set_opregion(struct intel_vgpu *vgpu)565{566 void *base;567 int ret;568 569 /* Each vgpu has its own opregion, although VFIO would create another570 * one later. This one is used to expose opregion to VFIO. And the571 * other one created by VFIO later, is used by guest actually.572 */573 base = vgpu_opregion(vgpu)->va;574 if (!base)575 return -ENOMEM;576 577 if (memcmp(base, OPREGION_SIGNATURE, 16)) {578 memunmap(base);579 return -EINVAL;580 }581 582 ret = intel_vgpu_register_reg(vgpu,583 PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE,584 VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION,585 &intel_vgpu_regops_opregion, INTEL_GVT_OPREGION_SIZE,586 VFIO_REGION_INFO_FLAG_READ, base);587 588 return ret;589}590 591int intel_gvt_set_edid(struct intel_vgpu *vgpu, int port_num)592{593 struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num);594 struct vfio_edid_region *base;595 int ret;596 597 base = kzalloc(sizeof(*base), GFP_KERNEL);598 if (!base)599 return -ENOMEM;600 601 /* TODO: Add multi-port and EDID extension block support */602 base->vfio_edid_regs.edid_offset = EDID_BLOB_OFFSET;603 base->vfio_edid_regs.edid_max_size = EDID_SIZE;604 base->vfio_edid_regs.edid_size = EDID_SIZE;605 base->vfio_edid_regs.max_xres = vgpu_edid_xres(port->id);606 base->vfio_edid_regs.max_yres = vgpu_edid_yres(port->id);607 base->edid_blob = port->edid->edid_block;608 609 ret = intel_vgpu_register_reg(vgpu,610 VFIO_REGION_TYPE_GFX,611 VFIO_REGION_SUBTYPE_GFX_EDID,612 &intel_vgpu_regops_edid, EDID_SIZE,613 VFIO_REGION_INFO_FLAG_READ |614 VFIO_REGION_INFO_FLAG_WRITE |615 VFIO_REGION_INFO_FLAG_CAPS, base);616 617 return ret;618}619 620static void intel_vgpu_dma_unmap(struct vfio_device *vfio_dev, u64 iova,621 u64 length)622{623 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);624 struct gvt_dma *entry;625 u64 iov_pfn = iova >> PAGE_SHIFT;626 u64 end_iov_pfn = iov_pfn + length / PAGE_SIZE;627 628 mutex_lock(&vgpu->cache_lock);629 for (; iov_pfn < end_iov_pfn; iov_pfn++) {630 entry = __gvt_cache_find_gfn(vgpu, iov_pfn);631 if (!entry)632 continue;633 634 gvt_dma_unmap_page(vgpu, entry->gfn, entry->dma_addr,635 entry->size);636 __gvt_cache_remove_entry(vgpu, entry);637 }638 mutex_unlock(&vgpu->cache_lock);639}640 641static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu)642{643 struct intel_vgpu *itr;644 int id;645 bool ret = false;646 647 mutex_lock(&vgpu->gvt->lock);648 for_each_active_vgpu(vgpu->gvt, itr, id) {649 if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, itr->status))650 continue;651 652 if (vgpu->vfio_device.kvm == itr->vfio_device.kvm) {653 ret = true;654 goto out;655 }656 }657out:658 mutex_unlock(&vgpu->gvt->lock);659 return ret;660}661 662static int intel_vgpu_open_device(struct vfio_device *vfio_dev)663{664 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);665 int ret;666 667 if (__kvmgt_vgpu_exist(vgpu))668 return -EEXIST;669 670 vgpu->track_node.track_write = kvmgt_page_track_write;671 vgpu->track_node.track_remove_region = kvmgt_page_track_remove_region;672 ret = kvm_page_track_register_notifier(vgpu->vfio_device.kvm,673 &vgpu->track_node);674 if (ret) {675 gvt_vgpu_err("KVM is required to use Intel vGPU\n");676 return ret;677 }678 679 set_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status);680 681 debugfs_create_ulong(KVMGT_DEBUGFS_FILENAME, 0444, vgpu->debugfs,682 &vgpu->nr_cache_entries);683 684 intel_gvt_activate_vgpu(vgpu);685 686 return 0;687}688 689static void intel_vgpu_release_msi_eventfd_ctx(struct intel_vgpu *vgpu)690{691 struct eventfd_ctx *trigger;692 693 trigger = vgpu->msi_trigger;694 if (trigger) {695 eventfd_ctx_put(trigger);696 vgpu->msi_trigger = NULL;697 }698}699 700static void intel_vgpu_close_device(struct vfio_device *vfio_dev)701{702 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);703 704 intel_gvt_release_vgpu(vgpu);705 706 clear_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status);707 708 debugfs_lookup_and_remove(KVMGT_DEBUGFS_FILENAME, vgpu->debugfs);709 710 kvm_page_track_unregister_notifier(vgpu->vfio_device.kvm,711 &vgpu->track_node);712 713 kvmgt_protect_table_destroy(vgpu);714 gvt_cache_destroy(vgpu);715 716 WARN_ON(vgpu->nr_cache_entries);717 718 vgpu->gfn_cache = RB_ROOT;719 vgpu->dma_addr_cache = RB_ROOT;720 721 intel_vgpu_release_msi_eventfd_ctx(vgpu);722}723 724static u64 intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar)725{726 u32 start_lo, start_hi;727 u32 mem_type;728 729 start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &730 PCI_BASE_ADDRESS_MEM_MASK;731 mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &732 PCI_BASE_ADDRESS_MEM_TYPE_MASK;733 734 switch (mem_type) {735 case PCI_BASE_ADDRESS_MEM_TYPE_64:736 start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space737 + bar + 4));738 break;739 case PCI_BASE_ADDRESS_MEM_TYPE_32:740 case PCI_BASE_ADDRESS_MEM_TYPE_1M:741 /* 1M mem BAR treated as 32-bit BAR */742 default:743 /* mem unknown type treated as 32-bit BAR */744 start_hi = 0;745 break;746 }747 748 return ((u64)start_hi << 32) | start_lo;749}750 751static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, u64 off,752 void *buf, unsigned int count, bool is_write)753{754 u64 bar_start = intel_vgpu_get_bar_addr(vgpu, bar);755 int ret;756 757 if (is_write)758 ret = intel_vgpu_emulate_mmio_write(vgpu,759 bar_start + off, buf, count);760 else761 ret = intel_vgpu_emulate_mmio_read(vgpu,762 bar_start + off, buf, count);763 return ret;764}765 766static inline bool intel_vgpu_in_aperture(struct intel_vgpu *vgpu, u64 off)767{768 return off >= vgpu_aperture_offset(vgpu) &&769 off < vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu);770}771 772static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, u64 off,773 void *buf, unsigned long count, bool is_write)774{775 void __iomem *aperture_va;776 777 if (!intel_vgpu_in_aperture(vgpu, off) ||778 !intel_vgpu_in_aperture(vgpu, off + count)) {779 gvt_vgpu_err("Invalid aperture offset %llu\n", off);780 return -EINVAL;781 }782 783 aperture_va = io_mapping_map_wc(&vgpu->gvt->gt->ggtt->iomap,784 ALIGN_DOWN(off, PAGE_SIZE),785 count + offset_in_page(off));786 if (!aperture_va)787 return -EIO;788 789 if (is_write)790 memcpy_toio(aperture_va + offset_in_page(off), buf, count);791 else792 memcpy_fromio(buf, aperture_va + offset_in_page(off), count);793 794 io_mapping_unmap(aperture_va);795 796 return 0;797}798 799static ssize_t intel_vgpu_rw(struct intel_vgpu *vgpu, char *buf,800 size_t count, loff_t *ppos, bool is_write)801{802 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);803 u64 pos = *ppos & VFIO_PCI_OFFSET_MASK;804 int ret = -EINVAL;805 806 807 if (index >= VFIO_PCI_NUM_REGIONS + vgpu->num_regions) {808 gvt_vgpu_err("invalid index: %u\n", index);809 return -EINVAL;810 }811 812 switch (index) {813 case VFIO_PCI_CONFIG_REGION_INDEX:814 if (is_write)815 ret = intel_vgpu_emulate_cfg_write(vgpu, pos,816 buf, count);817 else818 ret = intel_vgpu_emulate_cfg_read(vgpu, pos,819 buf, count);820 break;821 case VFIO_PCI_BAR0_REGION_INDEX:822 ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_0, pos,823 buf, count, is_write);824 break;825 case VFIO_PCI_BAR2_REGION_INDEX:826 ret = intel_vgpu_aperture_rw(vgpu, pos, buf, count, is_write);827 break;828 case VFIO_PCI_BAR1_REGION_INDEX:829 case VFIO_PCI_BAR3_REGION_INDEX:830 case VFIO_PCI_BAR4_REGION_INDEX:831 case VFIO_PCI_BAR5_REGION_INDEX:832 case VFIO_PCI_VGA_REGION_INDEX:833 case VFIO_PCI_ROM_REGION_INDEX:834 break;835 default:836 if (index >= VFIO_PCI_NUM_REGIONS + vgpu->num_regions)837 return -EINVAL;838 839 index -= VFIO_PCI_NUM_REGIONS;840 return vgpu->region[index].ops->rw(vgpu, buf, count,841 ppos, is_write);842 }843 844 return ret == 0 ? count : ret;845}846 847static bool gtt_entry(struct intel_vgpu *vgpu, loff_t *ppos)848{849 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);850 struct intel_gvt *gvt = vgpu->gvt;851 int offset;852 853 /* Only allow MMIO GGTT entry access */854 if (index != PCI_BASE_ADDRESS_0)855 return false;856 857 offset = (u64)(*ppos & VFIO_PCI_OFFSET_MASK) -858 intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_0);859 860 return (offset >= gvt->device_info.gtt_start_offset &&861 offset < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt)) ?862 true : false;863}864 865static ssize_t intel_vgpu_read(struct vfio_device *vfio_dev, char __user *buf,866 size_t count, loff_t *ppos)867{868 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);869 unsigned int done = 0;870 int ret;871 872 while (count) {873 size_t filled;874 875 /* Only support GGTT entry 8 bytes read */876 if (count >= 8 && !(*ppos % 8) &&877 gtt_entry(vgpu, ppos)) {878 u64 val;879 880 ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val),881 ppos, false);882 if (ret <= 0)883 goto read_err;884 885 if (copy_to_user(buf, &val, sizeof(val)))886 goto read_err;887 888 filled = 8;889 } else if (count >= 4 && !(*ppos % 4)) {890 u32 val;891 892 ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val),893 ppos, false);894 if (ret <= 0)895 goto read_err;896 897 if (copy_to_user(buf, &val, sizeof(val)))898 goto read_err;899 900 filled = 4;901 } else if (count >= 2 && !(*ppos % 2)) {902 u16 val;903 904 ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val),905 ppos, false);906 if (ret <= 0)907 goto read_err;908 909 if (copy_to_user(buf, &val, sizeof(val)))910 goto read_err;911 912 filled = 2;913 } else {914 u8 val;915 916 ret = intel_vgpu_rw(vgpu, &val, sizeof(val), ppos,917 false);918 if (ret <= 0)919 goto read_err;920 921 if (copy_to_user(buf, &val, sizeof(val)))922 goto read_err;923 924 filled = 1;925 }926 927 count -= filled;928 done += filled;929 *ppos += filled;930 buf += filled;931 }932 933 return done;934 935read_err:936 return -EFAULT;937}938 939static ssize_t intel_vgpu_write(struct vfio_device *vfio_dev,940 const char __user *buf,941 size_t count, loff_t *ppos)942{943 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);944 unsigned int done = 0;945 int ret;946 947 while (count) {948 size_t filled;949 950 /* Only support GGTT entry 8 bytes write */951 if (count >= 8 && !(*ppos % 8) &&952 gtt_entry(vgpu, ppos)) {953 u64 val;954 955 if (copy_from_user(&val, buf, sizeof(val)))956 goto write_err;957 958 ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val),959 ppos, true);960 if (ret <= 0)961 goto write_err;962 963 filled = 8;964 } else if (count >= 4 && !(*ppos % 4)) {965 u32 val;966 967 if (copy_from_user(&val, buf, sizeof(val)))968 goto write_err;969 970 ret = intel_vgpu_rw(vgpu, (char *)&val, sizeof(val),971 ppos, true);972 if (ret <= 0)973 goto write_err;974 975 filled = 4;976 } else if (count >= 2 && !(*ppos % 2)) {977 u16 val;978 979 if (copy_from_user(&val, buf, sizeof(val)))980 goto write_err;981 982 ret = intel_vgpu_rw(vgpu, (char *)&val,983 sizeof(val), ppos, true);984 if (ret <= 0)985 goto write_err;986 987 filled = 2;988 } else {989 u8 val;990 991 if (copy_from_user(&val, buf, sizeof(val)))992 goto write_err;993 994 ret = intel_vgpu_rw(vgpu, &val, sizeof(val),995 ppos, true);996 if (ret <= 0)997 goto write_err;998 999 filled = 1;1000 }1001 1002 count -= filled;1003 done += filled;1004 *ppos += filled;1005 buf += filled;1006 }1007 1008 return done;1009write_err:1010 return -EFAULT;1011}1012 1013static int intel_vgpu_mmap(struct vfio_device *vfio_dev,1014 struct vm_area_struct *vma)1015{1016 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);1017 unsigned int index;1018 u64 virtaddr;1019 unsigned long req_size, pgoff, req_start;1020 pgprot_t pg_prot;1021 1022 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);1023 if (index >= VFIO_PCI_ROM_REGION_INDEX)1024 return -EINVAL;1025 1026 if (vma->vm_end < vma->vm_start)1027 return -EINVAL;1028 if ((vma->vm_flags & VM_SHARED) == 0)1029 return -EINVAL;1030 if (index != VFIO_PCI_BAR2_REGION_INDEX)1031 return -EINVAL;1032 1033 pg_prot = vma->vm_page_prot;1034 virtaddr = vma->vm_start;1035 req_size = vma->vm_end - vma->vm_start;1036 pgoff = vma->vm_pgoff &1037 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);1038 req_start = pgoff << PAGE_SHIFT;1039 1040 if (!intel_vgpu_in_aperture(vgpu, req_start))1041 return -EINVAL;1042 if (req_start + req_size >1043 vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu))1044 return -EINVAL;1045 1046 pgoff = (gvt_aperture_pa_base(vgpu->gvt) >> PAGE_SHIFT) + pgoff;1047 1048 return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot);1049}1050 1051static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type)1052{1053 if (type == VFIO_PCI_INTX_IRQ_INDEX || type == VFIO_PCI_MSI_IRQ_INDEX)1054 return 1;1055 1056 return 0;1057}1058 1059static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu,1060 unsigned int index, unsigned int start,1061 unsigned int count, u32 flags,1062 void *data)1063{1064 return 0;1065}1066 1067static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu,1068 unsigned int index, unsigned int start,1069 unsigned int count, u32 flags, void *data)1070{1071 return 0;1072}1073 1074static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu,1075 unsigned int index, unsigned int start, unsigned int count,1076 u32 flags, void *data)1077{1078 return 0;1079}1080 1081static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu,1082 unsigned int index, unsigned int start, unsigned int count,1083 u32 flags, void *data)1084{1085 struct eventfd_ctx *trigger;1086 1087 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {1088 int fd = *(int *)data;1089 1090 trigger = eventfd_ctx_fdget(fd);1091 if (IS_ERR(trigger)) {1092 gvt_vgpu_err("eventfd_ctx_fdget failed\n");1093 return PTR_ERR(trigger);1094 }1095 vgpu->msi_trigger = trigger;1096 } else if ((flags & VFIO_IRQ_SET_DATA_NONE) && !count)1097 intel_vgpu_release_msi_eventfd_ctx(vgpu);1098 1099 return 0;1100}1101 1102static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, u32 flags,1103 unsigned int index, unsigned int start, unsigned int count,1104 void *data)1105{1106 int (*func)(struct intel_vgpu *vgpu, unsigned int index,1107 unsigned int start, unsigned int count, u32 flags,1108 void *data) = NULL;1109 1110 switch (index) {1111 case VFIO_PCI_INTX_IRQ_INDEX:1112 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {1113 case VFIO_IRQ_SET_ACTION_MASK:1114 func = intel_vgpu_set_intx_mask;1115 break;1116 case VFIO_IRQ_SET_ACTION_UNMASK:1117 func = intel_vgpu_set_intx_unmask;1118 break;1119 case VFIO_IRQ_SET_ACTION_TRIGGER:1120 func = intel_vgpu_set_intx_trigger;1121 break;1122 }1123 break;1124 case VFIO_PCI_MSI_IRQ_INDEX:1125 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {1126 case VFIO_IRQ_SET_ACTION_MASK:1127 case VFIO_IRQ_SET_ACTION_UNMASK:1128 /* XXX Need masking support exported */1129 break;1130 case VFIO_IRQ_SET_ACTION_TRIGGER:1131 func = intel_vgpu_set_msi_trigger;1132 break;1133 }1134 break;1135 }1136 1137 if (!func)1138 return -ENOTTY;1139 1140 return func(vgpu, index, start, count, flags, data);1141}1142 1143static long intel_vgpu_ioctl(struct vfio_device *vfio_dev, unsigned int cmd,1144 unsigned long arg)1145{1146 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);1147 unsigned long minsz;1148 1149 gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu->id, cmd);1150 1151 if (cmd == VFIO_DEVICE_GET_INFO) {1152 struct vfio_device_info info;1153 1154 minsz = offsetofend(struct vfio_device_info, num_irqs);1155 1156 if (copy_from_user(&info, (void __user *)arg, minsz))1157 return -EFAULT;1158 1159 if (info.argsz < minsz)1160 return -EINVAL;1161 1162 info.flags = VFIO_DEVICE_FLAGS_PCI;1163 info.flags |= VFIO_DEVICE_FLAGS_RESET;1164 info.num_regions = VFIO_PCI_NUM_REGIONS +1165 vgpu->num_regions;1166 info.num_irqs = VFIO_PCI_NUM_IRQS;1167 1168 return copy_to_user((void __user *)arg, &info, minsz) ?1169 -EFAULT : 0;1170 1171 } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {1172 struct vfio_region_info info;1173 struct vfio_info_cap caps = { .buf = NULL, .size = 0 };1174 unsigned int i;1175 int ret;1176 struct vfio_region_info_cap_sparse_mmap *sparse = NULL;1177 int nr_areas = 1;1178 int cap_type_id;1179 1180 minsz = offsetofend(struct vfio_region_info, offset);1181 1182 if (copy_from_user(&info, (void __user *)arg, minsz))1183 return -EFAULT;1184 1185 if (info.argsz < minsz)1186 return -EINVAL;1187 1188 switch (info.index) {1189 case VFIO_PCI_CONFIG_REGION_INDEX:1190 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);1191 info.size = vgpu->gvt->device_info.cfg_space_size;1192 info.flags = VFIO_REGION_INFO_FLAG_READ |1193 VFIO_REGION_INFO_FLAG_WRITE;1194 break;1195 case VFIO_PCI_BAR0_REGION_INDEX:1196 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);1197 info.size = vgpu->cfg_space.bar[info.index].size;1198 if (!info.size) {1199 info.flags = 0;1200 break;1201 }1202 1203 info.flags = VFIO_REGION_INFO_FLAG_READ |1204 VFIO_REGION_INFO_FLAG_WRITE;1205 break;1206 case VFIO_PCI_BAR1_REGION_INDEX:1207 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);1208 info.size = 0;1209 info.flags = 0;1210 break;1211 case VFIO_PCI_BAR2_REGION_INDEX:1212 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);1213 info.flags = VFIO_REGION_INFO_FLAG_CAPS |1214 VFIO_REGION_INFO_FLAG_MMAP |1215 VFIO_REGION_INFO_FLAG_READ |1216 VFIO_REGION_INFO_FLAG_WRITE;1217 info.size = gvt_aperture_sz(vgpu->gvt);1218 1219 sparse = kzalloc(struct_size(sparse, areas, nr_areas),1220 GFP_KERNEL);1221 if (!sparse)1222 return -ENOMEM;1223 1224 sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;1225 sparse->header.version = 1;1226 sparse->nr_areas = nr_areas;1227 cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;1228 sparse->areas[0].offset =1229 PAGE_ALIGN(vgpu_aperture_offset(vgpu));1230 sparse->areas[0].size = vgpu_aperture_sz(vgpu);1231 break;1232 1233 case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:1234 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);1235 info.size = 0;1236 info.flags = 0;1237 1238 gvt_dbg_core("get region info bar:%d\n", info.index);1239 break;1240 1241 case VFIO_PCI_ROM_REGION_INDEX:1242 case VFIO_PCI_VGA_REGION_INDEX:1243 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);1244 info.size = 0;1245 info.flags = 0;1246 1247 gvt_dbg_core("get region info index:%d\n", info.index);1248 break;1249 default:1250 {1251 struct vfio_region_info_cap_type cap_type = {1252 .header.id = VFIO_REGION_INFO_CAP_TYPE,1253 .header.version = 1 };1254 1255 if (info.index >= VFIO_PCI_NUM_REGIONS +1256 vgpu->num_regions)1257 return -EINVAL;1258 info.index =1259 array_index_nospec(info.index,1260 VFIO_PCI_NUM_REGIONS +1261 vgpu->num_regions);1262 1263 i = info.index - VFIO_PCI_NUM_REGIONS;1264 1265 info.offset =1266 VFIO_PCI_INDEX_TO_OFFSET(info.index);1267 info.size = vgpu->region[i].size;1268 info.flags = vgpu->region[i].flags;1269 1270 cap_type.type = vgpu->region[i].type;1271 cap_type.subtype = vgpu->region[i].subtype;1272 1273 ret = vfio_info_add_capability(&caps,1274 &cap_type.header,1275 sizeof(cap_type));1276 if (ret)1277 return ret;1278 }1279 }1280 1281 if ((info.flags & VFIO_REGION_INFO_FLAG_CAPS) && sparse) {1282 switch (cap_type_id) {1283 case VFIO_REGION_INFO_CAP_SPARSE_MMAP:1284 ret = vfio_info_add_capability(&caps,1285 &sparse->header,1286 struct_size(sparse, areas,1287 sparse->nr_areas));1288 if (ret) {1289 kfree(sparse);1290 return ret;1291 }1292 break;1293 default:1294 kfree(sparse);1295 return -EINVAL;1296 }1297 }1298 1299 if (caps.size) {1300 info.flags |= VFIO_REGION_INFO_FLAG_CAPS;1301 if (info.argsz < sizeof(info) + caps.size) {1302 info.argsz = sizeof(info) + caps.size;1303 info.cap_offset = 0;1304 } else {1305 vfio_info_cap_shift(&caps, sizeof(info));1306 if (copy_to_user((void __user *)arg +1307 sizeof(info), caps.buf,1308 caps.size)) {1309 kfree(caps.buf);1310 kfree(sparse);1311 return -EFAULT;1312 }1313 info.cap_offset = sizeof(info);1314 }1315 1316 kfree(caps.buf);1317 }1318 1319 kfree(sparse);1320 return copy_to_user((void __user *)arg, &info, minsz) ?1321 -EFAULT : 0;1322 } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {1323 struct vfio_irq_info info;1324 1325 minsz = offsetofend(struct vfio_irq_info, count);1326 1327 if (copy_from_user(&info, (void __user *)arg, minsz))1328 return -EFAULT;1329 1330 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)1331 return -EINVAL;1332 1333 switch (info.index) {1334 case VFIO_PCI_INTX_IRQ_INDEX:1335 case VFIO_PCI_MSI_IRQ_INDEX:1336 break;1337 default:1338 return -EINVAL;1339 }1340 1341 info.flags = VFIO_IRQ_INFO_EVENTFD;1342 1343 info.count = intel_vgpu_get_irq_count(vgpu, info.index);1344 1345 if (info.index == VFIO_PCI_INTX_IRQ_INDEX)1346 info.flags |= (VFIO_IRQ_INFO_MASKABLE |1347 VFIO_IRQ_INFO_AUTOMASKED);1348 else1349 info.flags |= VFIO_IRQ_INFO_NORESIZE;1350 1351 return copy_to_user((void __user *)arg, &info, minsz) ?1352 -EFAULT : 0;1353 } else if (cmd == VFIO_DEVICE_SET_IRQS) {1354 struct vfio_irq_set hdr;1355 u8 *data = NULL;1356 int ret = 0;1357 size_t data_size = 0;1358 1359 minsz = offsetofend(struct vfio_irq_set, count);1360 1361 if (copy_from_user(&hdr, (void __user *)arg, minsz))1362 return -EFAULT;1363 1364 if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) {1365 int max = intel_vgpu_get_irq_count(vgpu, hdr.index);1366 1367 ret = vfio_set_irqs_validate_and_prepare(&hdr, max,1368 VFIO_PCI_NUM_IRQS, &data_size);1369 if (ret) {1370 gvt_vgpu_err("intel:vfio_set_irqs_validate_and_prepare failed\n");1371 return -EINVAL;1372 }1373 if (data_size) {1374 data = memdup_user((void __user *)(arg + minsz),1375 data_size);1376 if (IS_ERR(data))1377 return PTR_ERR(data);1378 }1379 }1380 1381 ret = intel_vgpu_set_irqs(vgpu, hdr.flags, hdr.index,1382 hdr.start, hdr.count, data);1383 kfree(data);1384 1385 return ret;1386 } else if (cmd == VFIO_DEVICE_RESET) {1387 intel_gvt_reset_vgpu(vgpu);1388 return 0;1389 } else if (cmd == VFIO_DEVICE_QUERY_GFX_PLANE) {1390 struct vfio_device_gfx_plane_info dmabuf = {};1391 int ret = 0;1392 1393 minsz = offsetofend(struct vfio_device_gfx_plane_info,1394 dmabuf_id);1395 if (copy_from_user(&dmabuf, (void __user *)arg, minsz))1396 return -EFAULT;1397 if (dmabuf.argsz < minsz)1398 return -EINVAL;1399 1400 ret = intel_vgpu_query_plane(vgpu, &dmabuf);1401 if (ret != 0)1402 return ret;1403 1404 return copy_to_user((void __user *)arg, &dmabuf, minsz) ?1405 -EFAULT : 0;1406 } else if (cmd == VFIO_DEVICE_GET_GFX_DMABUF) {1407 __u32 dmabuf_id;1408 1409 if (get_user(dmabuf_id, (__u32 __user *)arg))1410 return -EFAULT;1411 return intel_vgpu_get_dmabuf(vgpu, dmabuf_id);1412 }1413 1414 return -ENOTTY;1415}1416 1417static ssize_t1418vgpu_id_show(struct device *dev, struct device_attribute *attr,1419 char *buf)1420{1421 struct intel_vgpu *vgpu = dev_get_drvdata(dev);1422 1423 return sprintf(buf, "%d\n", vgpu->id);1424}1425 1426static DEVICE_ATTR_RO(vgpu_id);1427 1428static struct attribute *intel_vgpu_attrs[] = {1429 &dev_attr_vgpu_id.attr,1430 NULL1431};1432 1433static const struct attribute_group intel_vgpu_group = {1434 .name = "intel_vgpu",1435 .attrs = intel_vgpu_attrs,1436};1437 1438static const struct attribute_group *intel_vgpu_groups[] = {1439 &intel_vgpu_group,1440 NULL,1441};1442 1443static int intel_vgpu_init_dev(struct vfio_device *vfio_dev)1444{1445 struct mdev_device *mdev = to_mdev_device(vfio_dev->dev);1446 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);1447 struct intel_vgpu_type *type =1448 container_of(mdev->type, struct intel_vgpu_type, type);1449 int ret;1450 1451 vgpu->gvt = kdev_to_i915(mdev->type->parent->dev)->gvt;1452 ret = intel_gvt_create_vgpu(vgpu, type->conf);1453 if (ret)1454 return ret;1455 1456 kvmgt_protect_table_init(vgpu);1457 gvt_cache_init(vgpu);1458 1459 return 0;1460}1461 1462static void intel_vgpu_release_dev(struct vfio_device *vfio_dev)1463{1464 struct intel_vgpu *vgpu = vfio_dev_to_vgpu(vfio_dev);1465 1466 intel_gvt_destroy_vgpu(vgpu);1467}1468 1469static const struct vfio_device_ops intel_vgpu_dev_ops = {1470 .init = intel_vgpu_init_dev,1471 .release = intel_vgpu_release_dev,1472 .open_device = intel_vgpu_open_device,1473 .close_device = intel_vgpu_close_device,1474 .read = intel_vgpu_read,1475 .write = intel_vgpu_write,1476 .mmap = intel_vgpu_mmap,1477 .ioctl = intel_vgpu_ioctl,1478 .dma_unmap = intel_vgpu_dma_unmap,1479 .bind_iommufd = vfio_iommufd_emulated_bind,1480 .unbind_iommufd = vfio_iommufd_emulated_unbind,1481 .attach_ioas = vfio_iommufd_emulated_attach_ioas,1482 .detach_ioas = vfio_iommufd_emulated_detach_ioas,1483};1484 1485static int intel_vgpu_probe(struct mdev_device *mdev)1486{1487 struct intel_vgpu *vgpu;1488 int ret;1489 1490 vgpu = vfio_alloc_device(intel_vgpu, vfio_device, &mdev->dev,1491 &intel_vgpu_dev_ops);1492 if (IS_ERR(vgpu)) {1493 gvt_err("failed to create intel vgpu: %ld\n", PTR_ERR(vgpu));1494 return PTR_ERR(vgpu);1495 }1496 1497 dev_set_drvdata(&mdev->dev, vgpu);1498 ret = vfio_register_emulated_iommu_dev(&vgpu->vfio_device);1499 if (ret)1500 goto out_put_vdev;1501 1502 gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n",1503 dev_name(mdev_dev(mdev)));1504 return 0;1505 1506out_put_vdev:1507 vfio_put_device(&vgpu->vfio_device);1508 return ret;1509}1510 1511static void intel_vgpu_remove(struct mdev_device *mdev)1512{1513 struct intel_vgpu *vgpu = dev_get_drvdata(&mdev->dev);1514 1515 vfio_unregister_group_dev(&vgpu->vfio_device);1516 vfio_put_device(&vgpu->vfio_device);1517}1518 1519static unsigned int intel_vgpu_get_available(struct mdev_type *mtype)1520{1521 struct intel_vgpu_type *type =1522 container_of(mtype, struct intel_vgpu_type, type);1523 struct intel_gvt *gvt = kdev_to_i915(mtype->parent->dev)->gvt;1524 unsigned int low_gm_avail, high_gm_avail, fence_avail;1525 1526 mutex_lock(&gvt->lock);1527 low_gm_avail = gvt_aperture_sz(gvt) - HOST_LOW_GM_SIZE -1528 gvt->gm.vgpu_allocated_low_gm_size;1529 high_gm_avail = gvt_hidden_sz(gvt) - HOST_HIGH_GM_SIZE -1530 gvt->gm.vgpu_allocated_high_gm_size;1531 fence_avail = gvt_fence_sz(gvt) - HOST_FENCE -1532 gvt->fence.vgpu_allocated_fence_num;1533 mutex_unlock(&gvt->lock);1534 1535 return min3(low_gm_avail / type->conf->low_mm,1536 high_gm_avail / type->conf->high_mm,1537 fence_avail / type->conf->fence);1538}1539 1540static struct mdev_driver intel_vgpu_mdev_driver = {1541 .device_api = VFIO_DEVICE_API_PCI_STRING,1542 .driver = {1543 .name = "intel_vgpu_mdev",1544 .owner = THIS_MODULE,1545 .dev_groups = intel_vgpu_groups,1546 },1547 .probe = intel_vgpu_probe,1548 .remove = intel_vgpu_remove,1549 .get_available = intel_vgpu_get_available,1550 .show_description = intel_vgpu_show_description,1551};1552 1553int intel_gvt_page_track_add(struct intel_vgpu *info, u64 gfn)1554{1555 int r;1556 1557 if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, info->status))1558 return -ESRCH;1559 1560 if (kvmgt_gfn_is_write_protected(info, gfn))1561 return 0;1562 1563 r = kvm_write_track_add_gfn(info->vfio_device.kvm, gfn);1564 if (r)1565 return r;1566 1567 kvmgt_protect_table_add(info, gfn);1568 return 0;1569}1570 1571int intel_gvt_page_track_remove(struct intel_vgpu *info, u64 gfn)1572{1573 int r;1574 1575 if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, info->status))1576 return -ESRCH;1577 1578 if (!kvmgt_gfn_is_write_protected(info, gfn))1579 return 0;1580 1581 r = kvm_write_track_remove_gfn(info->vfio_device.kvm, gfn);1582 if (r)1583 return r;1584 1585 kvmgt_protect_table_del(info, gfn);1586 return 0;1587}1588 1589static void kvmgt_page_track_write(gpa_t gpa, const u8 *val, int len,1590 struct kvm_page_track_notifier_node *node)1591{1592 struct intel_vgpu *info =1593 container_of(node, struct intel_vgpu, track_node);1594 1595 mutex_lock(&info->vgpu_lock);1596 1597 if (kvmgt_gfn_is_write_protected(info, gpa >> PAGE_SHIFT))1598 intel_vgpu_page_track_handler(info, gpa,1599 (void *)val, len);1600 1601 mutex_unlock(&info->vgpu_lock);1602}1603 1604static void kvmgt_page_track_remove_region(gfn_t gfn, unsigned long nr_pages,1605 struct kvm_page_track_notifier_node *node)1606{1607 unsigned long i;1608 struct intel_vgpu *info =1609 container_of(node, struct intel_vgpu, track_node);1610 1611 mutex_lock(&info->vgpu_lock);1612 1613 for (i = 0; i < nr_pages; i++) {1614 if (kvmgt_gfn_is_write_protected(info, gfn + i))1615 kvmgt_protect_table_del(info, gfn + i);1616 }1617 1618 mutex_unlock(&info->vgpu_lock);1619}1620 1621void intel_vgpu_detach_regions(struct intel_vgpu *vgpu)1622{1623 int i;1624 1625 if (!vgpu->region)1626 return;1627 1628 for (i = 0; i < vgpu->num_regions; i++)1629 if (vgpu->region[i].ops->release)1630 vgpu->region[i].ops->release(vgpu,1631 &vgpu->region[i]);1632 vgpu->num_regions = 0;1633 kfree(vgpu->region);1634 vgpu->region = NULL;1635}1636 1637int intel_gvt_dma_map_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,1638 unsigned long size, dma_addr_t *dma_addr)1639{1640 struct gvt_dma *entry;1641 int ret;1642 1643 if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status))1644 return -EINVAL;1645 1646 mutex_lock(&vgpu->cache_lock);1647 1648 entry = __gvt_cache_find_gfn(vgpu, gfn);1649 if (!entry) {1650 ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size);1651 if (ret)1652 goto err_unlock;1653 1654 ret = __gvt_cache_add(vgpu, gfn, *dma_addr, size);1655 if (ret)1656 goto err_unmap;1657 } else if (entry->size != size) {1658 /* the same gfn with different size: unmap and re-map */1659 gvt_dma_unmap_page(vgpu, gfn, entry->dma_addr, entry->size);1660 __gvt_cache_remove_entry(vgpu, entry);1661 1662 ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size);1663 if (ret)1664 goto err_unlock;1665 1666 ret = __gvt_cache_add(vgpu, gfn, *dma_addr, size);1667 if (ret)1668 goto err_unmap;1669 } else {1670 kref_get(&entry->ref);1671 *dma_addr = entry->dma_addr;1672 }1673 1674 mutex_unlock(&vgpu->cache_lock);1675 return 0;1676 1677err_unmap:1678 gvt_dma_unmap_page(vgpu, gfn, *dma_addr, size);1679err_unlock:1680 mutex_unlock(&vgpu->cache_lock);1681 return ret;1682}1683 1684int intel_gvt_dma_pin_guest_page(struct intel_vgpu *vgpu, dma_addr_t dma_addr)1685{1686 struct gvt_dma *entry;1687 int ret = 0;1688 1689 if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status))1690 return -EINVAL;1691 1692 mutex_lock(&vgpu->cache_lock);1693 entry = __gvt_cache_find_dma_addr(vgpu, dma_addr);1694 if (entry)1695 kref_get(&entry->ref);1696 else1697 ret = -ENOMEM;1698 mutex_unlock(&vgpu->cache_lock);1699 1700 return ret;1701}1702 1703static void __gvt_dma_release(struct kref *ref)1704{1705 struct gvt_dma *entry = container_of(ref, typeof(*entry), ref);1706 1707 gvt_dma_unmap_page(entry->vgpu, entry->gfn, entry->dma_addr,1708 entry->size);1709 __gvt_cache_remove_entry(entry->vgpu, entry);1710}1711 1712void intel_gvt_dma_unmap_guest_page(struct intel_vgpu *vgpu,1713 dma_addr_t dma_addr)1714{1715 struct gvt_dma *entry;1716 1717 if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status))1718 return;1719 1720 mutex_lock(&vgpu->cache_lock);1721 entry = __gvt_cache_find_dma_addr(vgpu, dma_addr);1722 if (entry)1723 kref_put(&entry->ref, __gvt_dma_release);1724 mutex_unlock(&vgpu->cache_lock);1725}1726 1727static void init_device_info(struct intel_gvt *gvt)1728{1729 struct intel_gvt_device_info *info = &gvt->device_info;1730 struct pci_dev *pdev = to_pci_dev(gvt->gt->i915->drm.dev);1731 1732 info->max_support_vgpus = 8;1733 info->cfg_space_size = PCI_CFG_SPACE_EXP_SIZE;1734 info->mmio_size = 2 * 1024 * 1024;1735 info->mmio_bar = 0;1736 info->gtt_start_offset = 8 * 1024 * 1024;1737 info->gtt_entry_size = 8;1738 info->gtt_entry_size_shift = 3;1739 info->gmadr_bytes_in_cmd = 8;1740 info->max_surface_size = 36 * 1024 * 1024;1741 info->msi_cap_offset = pdev->msi_cap;1742}1743 1744static void intel_gvt_test_and_emulate_vblank(struct intel_gvt *gvt)1745{1746 struct intel_vgpu *vgpu;1747 int id;1748 1749 mutex_lock(&gvt->lock);1750 idr_for_each_entry((&(gvt)->vgpu_idr), (vgpu), (id)) {1751 if (test_and_clear_bit(INTEL_GVT_REQUEST_EMULATE_VBLANK + id,1752 (void *)&gvt->service_request)) {1753 if (test_bit(INTEL_VGPU_STATUS_ACTIVE, vgpu->status))1754 intel_vgpu_emulate_vblank(vgpu);1755 }1756 }1757 mutex_unlock(&gvt->lock);1758}1759 1760static int gvt_service_thread(void *data)1761{1762 struct intel_gvt *gvt = (struct intel_gvt *)data;1763 int ret;1764 1765 gvt_dbg_core("service thread start\n");1766 1767 while (!kthread_should_stop()) {1768 ret = wait_event_interruptible(gvt->service_thread_wq,1769 kthread_should_stop() || gvt->service_request);1770 1771 if (kthread_should_stop())1772 break;1773 1774 if (WARN_ONCE(ret, "service thread is waken up by signal.\n"))1775 continue;1776 1777 intel_gvt_test_and_emulate_vblank(gvt);1778 1779 if (test_bit(INTEL_GVT_REQUEST_SCHED,1780 (void *)&gvt->service_request) ||1781 test_bit(INTEL_GVT_REQUEST_EVENT_SCHED,1782 (void *)&gvt->service_request)) {1783 intel_gvt_schedule(gvt);1784 }1785 }1786 1787 return 0;1788}1789 1790static void clean_service_thread(struct intel_gvt *gvt)1791{1792 kthread_stop(gvt->service_thread);1793}1794 1795static int init_service_thread(struct intel_gvt *gvt)1796{1797 init_waitqueue_head(&gvt->service_thread_wq);1798 1799 gvt->service_thread = kthread_run(gvt_service_thread,1800 gvt, "gvt_service_thread");1801 if (IS_ERR(gvt->service_thread)) {1802 gvt_err("fail to start service thread.\n");1803 return PTR_ERR(gvt->service_thread);1804 }1805 return 0;1806}1807 1808/**1809 * intel_gvt_clean_device - clean a GVT device1810 * @i915: i915 private1811 *1812 * This function is called at the driver unloading stage, to free the1813 * resources owned by a GVT device.1814 *1815 */1816static void intel_gvt_clean_device(struct drm_i915_private *i915)1817{1818 struct intel_gvt *gvt = fetch_and_zero(&i915->gvt);1819 1820 if (drm_WARN_ON(&i915->drm, !gvt))1821 return;1822 1823 mdev_unregister_parent(&gvt->parent);1824 intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu);1825 intel_gvt_clean_vgpu_types(gvt);1826 1827 intel_gvt_debugfs_clean(gvt);1828 clean_service_thread(gvt);1829 intel_gvt_clean_cmd_parser(gvt);1830 intel_gvt_clean_sched_policy(gvt);1831 intel_gvt_clean_workload_scheduler(gvt);1832 intel_gvt_clean_gtt(gvt);1833 intel_gvt_free_firmware(gvt);1834 intel_gvt_clean_mmio_info(gvt);1835 idr_destroy(&gvt->vgpu_idr);1836 1837 kfree(i915->gvt);1838}1839 1840/**1841 * intel_gvt_init_device - initialize a GVT device1842 * @i915: drm i915 private data1843 *1844 * This function is called at the initialization stage, to initialize1845 * necessary GVT components.1846 *1847 * Returns:1848 * Zero on success, negative error code if failed.1849 *1850 */1851static int intel_gvt_init_device(struct drm_i915_private *i915)1852{1853 struct intel_gvt *gvt;1854 struct intel_vgpu *vgpu;1855 int ret;1856 1857 if (drm_WARN_ON(&i915->drm, i915->gvt))1858 return -EEXIST;1859 1860 gvt = kzalloc(sizeof(struct intel_gvt), GFP_KERNEL);1861 if (!gvt)1862 return -ENOMEM;1863 1864 gvt_dbg_core("init gvt device\n");1865 1866 idr_init_base(&gvt->vgpu_idr, 1);1867 spin_lock_init(&gvt->scheduler.mmio_context_lock);1868 mutex_init(&gvt->lock);1869 mutex_init(&gvt->sched_lock);1870 gvt->gt = to_gt(i915);1871 i915->gvt = gvt;1872 1873 init_device_info(gvt);1874 1875 ret = intel_gvt_setup_mmio_info(gvt);1876 if (ret)1877 goto out_clean_idr;1878 1879 intel_gvt_init_engine_mmio_context(gvt);1880 1881 ret = intel_gvt_load_firmware(gvt);1882 if (ret)1883 goto out_clean_mmio_info;1884 1885 ret = intel_gvt_init_irq(gvt);1886 if (ret)1887 goto out_free_firmware;1888 1889 ret = intel_gvt_init_gtt(gvt);1890 if (ret)1891 goto out_free_firmware;1892 1893 ret = intel_gvt_init_workload_scheduler(gvt);1894 if (ret)1895 goto out_clean_gtt;1896 1897 ret = intel_gvt_init_sched_policy(gvt);1898 if (ret)1899 goto out_clean_workload_scheduler;1900 1901 ret = intel_gvt_init_cmd_parser(gvt);1902 if (ret)1903 goto out_clean_sched_policy;1904 1905 ret = init_service_thread(gvt);1906 if (ret)1907 goto out_clean_cmd_parser;1908 1909 ret = intel_gvt_init_vgpu_types(gvt);1910 if (ret)1911 goto out_clean_thread;1912 1913 vgpu = intel_gvt_create_idle_vgpu(gvt);1914 if (IS_ERR(vgpu)) {1915 ret = PTR_ERR(vgpu);1916 gvt_err("failed to create idle vgpu\n");1917 goto out_clean_types;1918 }1919 gvt->idle_vgpu = vgpu;1920 1921 intel_gvt_debugfs_init(gvt);1922 1923 ret = mdev_register_parent(&gvt->parent, i915->drm.dev,1924 &intel_vgpu_mdev_driver,1925 gvt->mdev_types, gvt->num_types);1926 if (ret)1927 goto out_destroy_idle_vgpu;1928 1929 gvt_dbg_core("gvt device initialization is done\n");1930 return 0;1931 1932out_destroy_idle_vgpu:1933 intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu);1934 intel_gvt_debugfs_clean(gvt);1935out_clean_types:1936 intel_gvt_clean_vgpu_types(gvt);1937out_clean_thread:1938 clean_service_thread(gvt);1939out_clean_cmd_parser:1940 intel_gvt_clean_cmd_parser(gvt);1941out_clean_sched_policy:1942 intel_gvt_clean_sched_policy(gvt);1943out_clean_workload_scheduler:1944 intel_gvt_clean_workload_scheduler(gvt);1945out_clean_gtt:1946 intel_gvt_clean_gtt(gvt);1947out_free_firmware:1948 intel_gvt_free_firmware(gvt);1949out_clean_mmio_info:1950 intel_gvt_clean_mmio_info(gvt);1951out_clean_idr:1952 idr_destroy(&gvt->vgpu_idr);1953 kfree(gvt);1954 i915->gvt = NULL;1955 return ret;1956}1957 1958static void intel_gvt_pm_resume(struct drm_i915_private *i915)1959{1960 struct intel_gvt *gvt = i915->gvt;1961 1962 intel_gvt_restore_fence(gvt);1963 intel_gvt_restore_mmio(gvt);1964 intel_gvt_restore_ggtt(gvt);1965}1966 1967static const struct intel_vgpu_ops intel_gvt_vgpu_ops = {1968 .init_device = intel_gvt_init_device,1969 .clean_device = intel_gvt_clean_device,1970 .pm_resume = intel_gvt_pm_resume,1971};1972 1973static int __init kvmgt_init(void)1974{1975 int ret;1976 1977 ret = intel_gvt_set_ops(&intel_gvt_vgpu_ops);1978 if (ret)1979 return ret;1980 1981 ret = mdev_register_driver(&intel_vgpu_mdev_driver);1982 if (ret)1983 intel_gvt_clear_ops(&intel_gvt_vgpu_ops);1984 return ret;1985}1986 1987static void __exit kvmgt_exit(void)1988{1989 mdev_unregister_driver(&intel_vgpu_mdev_driver);1990 intel_gvt_clear_ops(&intel_gvt_vgpu_ops);1991}1992 1993module_init(kvmgt_init);1994module_exit(kvmgt_exit);1995 1996MODULE_DESCRIPTION("Intel mediated pass-through framework for KVM");1997MODULE_LICENSE("GPL and additional rights");1998MODULE_AUTHOR("Intel Corporation");1999