899 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Intel IFC VF NIC driver for virtio dataplane offloading4 *5 * Copyright (C) 2020 Intel Corporation.6 *7 * Author: Zhu Lingshan <lingshan.zhu@intel.com>8 *9 */10 11#include <linux/interrupt.h>12#include <linux/module.h>13#include <linux/pci.h>14#include <linux/sysfs.h>15#include "ifcvf_base.h"16 17#define DRIVER_AUTHOR "Intel Corporation"18#define IFCVF_DRIVER_NAME "ifcvf"19 20static irqreturn_t ifcvf_config_changed(int irq, void *arg)21{22 struct ifcvf_hw *vf = arg;23 24 if (vf->config_cb.callback)25 return vf->config_cb.callback(vf->config_cb.private);26 27 return IRQ_HANDLED;28}29 30static irqreturn_t ifcvf_vq_intr_handler(int irq, void *arg)31{32 struct vring_info *vring = arg;33 34 if (vring->cb.callback)35 return vring->cb.callback(vring->cb.private);36 37 return IRQ_HANDLED;38}39 40static irqreturn_t ifcvf_vqs_reused_intr_handler(int irq, void *arg)41{42 struct ifcvf_hw *vf = arg;43 struct vring_info *vring;44 int i;45 46 for (i = 0; i < vf->nr_vring; i++) {47 vring = &vf->vring[i];48 if (vring->cb.callback)49 vring->cb.callback(vring->cb.private);50 }51 52 return IRQ_HANDLED;53}54 55static irqreturn_t ifcvf_dev_intr_handler(int irq, void *arg)56{57 struct ifcvf_hw *vf = arg;58 u8 isr;59 60 isr = vp_ioread8(vf->isr);61 if (isr & VIRTIO_PCI_ISR_CONFIG)62 ifcvf_config_changed(irq, arg);63 64 return ifcvf_vqs_reused_intr_handler(irq, arg);65}66 67static void ifcvf_free_irq_vectors(void *data)68{69 pci_free_irq_vectors(data);70}71 72static void ifcvf_free_per_vq_irq(struct ifcvf_hw *vf)73{74 struct pci_dev *pdev = vf->pdev;75 int i;76 77 for (i = 0; i < vf->nr_vring; i++) {78 if (vf->vring[i].irq != -EINVAL) {79 devm_free_irq(&pdev->dev, vf->vring[i].irq, &vf->vring[i]);80 vf->vring[i].irq = -EINVAL;81 }82 }83}84 85static void ifcvf_free_vqs_reused_irq(struct ifcvf_hw *vf)86{87 struct pci_dev *pdev = vf->pdev;88 89 if (vf->vqs_reused_irq != -EINVAL) {90 devm_free_irq(&pdev->dev, vf->vqs_reused_irq, vf);91 vf->vqs_reused_irq = -EINVAL;92 }93 94}95 96static void ifcvf_free_vq_irq(struct ifcvf_hw *vf)97{98 if (vf->msix_vector_status == MSIX_VECTOR_PER_VQ_AND_CONFIG)99 ifcvf_free_per_vq_irq(vf);100 else101 ifcvf_free_vqs_reused_irq(vf);102}103 104static void ifcvf_free_config_irq(struct ifcvf_hw *vf)105{106 struct pci_dev *pdev = vf->pdev;107 108 if (vf->config_irq == -EINVAL)109 return;110 111 /* If the irq is shared by all vqs and the config interrupt,112 * it is already freed in ifcvf_free_vq_irq, so here only113 * need to free config irq when msix_vector_status != MSIX_VECTOR_DEV_SHARED114 */115 if (vf->msix_vector_status != MSIX_VECTOR_DEV_SHARED) {116 devm_free_irq(&pdev->dev, vf->config_irq, vf);117 vf->config_irq = -EINVAL;118 }119}120 121static void ifcvf_free_irq(struct ifcvf_hw *vf)122{123 struct pci_dev *pdev = vf->pdev;124 125 ifcvf_free_vq_irq(vf);126 ifcvf_free_config_irq(vf);127 ifcvf_free_irq_vectors(pdev);128 vf->num_msix_vectors = 0;129}130 131/* ifcvf MSIX vectors allocator, this helper tries to allocate132 * vectors for all virtqueues and the config interrupt.133 * It returns the number of allocated vectors, negative134 * return value when fails.135 */136static int ifcvf_alloc_vectors(struct ifcvf_hw *vf)137{138 struct pci_dev *pdev = vf->pdev;139 int max_intr, ret;140 141 /* all queues and config interrupt */142 max_intr = vf->nr_vring + 1;143 ret = pci_alloc_irq_vectors(pdev, 1, max_intr, PCI_IRQ_MSIX | PCI_IRQ_AFFINITY);144 145 if (ret < 0) {146 IFCVF_ERR(pdev, "Failed to alloc IRQ vectors\n");147 return ret;148 }149 150 if (ret < max_intr)151 IFCVF_INFO(pdev,152 "Requested %u vectors, however only %u allocated, lower performance\n",153 max_intr, ret);154 155 return ret;156}157 158static int ifcvf_request_per_vq_irq(struct ifcvf_hw *vf)159{160 struct pci_dev *pdev = vf->pdev;161 int i, vector, ret, irq;162 163 vf->vqs_reused_irq = -EINVAL;164 for (i = 0; i < vf->nr_vring; i++) {165 snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n", pci_name(pdev), i);166 vector = i;167 irq = pci_irq_vector(pdev, vector);168 ret = devm_request_irq(&pdev->dev, irq,169 ifcvf_vq_intr_handler, 0,170 vf->vring[i].msix_name,171 &vf->vring[i]);172 if (ret) {173 IFCVF_ERR(pdev, "Failed to request irq for vq %d\n", i);174 goto err;175 }176 177 vf->vring[i].irq = irq;178 ret = ifcvf_set_vq_vector(vf, i, vector);179 if (ret == VIRTIO_MSI_NO_VECTOR) {180 IFCVF_ERR(pdev, "No msix vector for vq %u\n", i);181 goto err;182 }183 }184 185 return 0;186err:187 ifcvf_free_irq(vf);188 189 return -EFAULT;190}191 192static int ifcvf_request_vqs_reused_irq(struct ifcvf_hw *vf)193{194 struct pci_dev *pdev = vf->pdev;195 int i, vector, ret, irq;196 197 vector = 0;198 snprintf(vf->vring[0].msix_name, 256, "ifcvf[%s]-vqs-reused-irq\n", pci_name(pdev));199 irq = pci_irq_vector(pdev, vector);200 ret = devm_request_irq(&pdev->dev, irq,201 ifcvf_vqs_reused_intr_handler, 0,202 vf->vring[0].msix_name, vf);203 if (ret) {204 IFCVF_ERR(pdev, "Failed to request reused irq for the device\n");205 goto err;206 }207 208 vf->vqs_reused_irq = irq;209 for (i = 0; i < vf->nr_vring; i++) {210 vf->vring[i].irq = -EINVAL;211 ret = ifcvf_set_vq_vector(vf, i, vector);212 if (ret == VIRTIO_MSI_NO_VECTOR) {213 IFCVF_ERR(pdev, "No msix vector for vq %u\n", i);214 goto err;215 }216 }217 218 return 0;219err:220 ifcvf_free_irq(vf);221 222 return -EFAULT;223}224 225static int ifcvf_request_dev_irq(struct ifcvf_hw *vf)226{227 struct pci_dev *pdev = vf->pdev;228 int i, vector, ret, irq;229 230 vector = 0;231 snprintf(vf->vring[0].msix_name, 256, "ifcvf[%s]-dev-irq\n", pci_name(pdev));232 irq = pci_irq_vector(pdev, vector);233 ret = devm_request_irq(&pdev->dev, irq,234 ifcvf_dev_intr_handler, 0,235 vf->vring[0].msix_name, vf);236 if (ret) {237 IFCVF_ERR(pdev, "Failed to request irq for the device\n");238 goto err;239 }240 241 vf->vqs_reused_irq = irq;242 for (i = 0; i < vf->nr_vring; i++) {243 vf->vring[i].irq = -EINVAL;244 ret = ifcvf_set_vq_vector(vf, i, vector);245 if (ret == VIRTIO_MSI_NO_VECTOR) {246 IFCVF_ERR(pdev, "No msix vector for vq %u\n", i);247 goto err;248 }249 }250 251 vf->config_irq = irq;252 ret = ifcvf_set_config_vector(vf, vector);253 if (ret == VIRTIO_MSI_NO_VECTOR) {254 IFCVF_ERR(pdev, "No msix vector for device config\n");255 goto err;256 }257 258 return 0;259err:260 ifcvf_free_irq(vf);261 262 return -EFAULT;263 264}265 266static int ifcvf_request_vq_irq(struct ifcvf_hw *vf)267{268 int ret;269 270 if (vf->msix_vector_status == MSIX_VECTOR_PER_VQ_AND_CONFIG)271 ret = ifcvf_request_per_vq_irq(vf);272 else273 ret = ifcvf_request_vqs_reused_irq(vf);274 275 return ret;276}277 278static int ifcvf_request_config_irq(struct ifcvf_hw *vf)279{280 struct pci_dev *pdev = vf->pdev;281 int config_vector, ret;282 283 if (vf->msix_vector_status == MSIX_VECTOR_PER_VQ_AND_CONFIG)284 config_vector = vf->nr_vring;285 else if (vf->msix_vector_status == MSIX_VECTOR_SHARED_VQ_AND_CONFIG)286 /* vector 0 for vqs and 1 for config interrupt */287 config_vector = 1;288 else if (vf->msix_vector_status == MSIX_VECTOR_DEV_SHARED)289 /* re-use the vqs vector */290 return 0;291 else292 return -EINVAL;293 294 snprintf(vf->config_msix_name, 256, "ifcvf[%s]-config\n",295 pci_name(pdev));296 vf->config_irq = pci_irq_vector(pdev, config_vector);297 ret = devm_request_irq(&pdev->dev, vf->config_irq,298 ifcvf_config_changed, 0,299 vf->config_msix_name, vf);300 if (ret) {301 IFCVF_ERR(pdev, "Failed to request config irq\n");302 goto err;303 }304 305 ret = ifcvf_set_config_vector(vf, config_vector);306 if (ret == VIRTIO_MSI_NO_VECTOR) {307 IFCVF_ERR(pdev, "No msix vector for device config\n");308 goto err;309 }310 311 return 0;312err:313 ifcvf_free_irq(vf);314 315 return -EFAULT;316}317 318static int ifcvf_request_irq(struct ifcvf_hw *vf)319{320 int nvectors, ret, max_intr;321 322 nvectors = ifcvf_alloc_vectors(vf);323 if (nvectors <= 0)324 return -EFAULT;325 326 vf->msix_vector_status = MSIX_VECTOR_PER_VQ_AND_CONFIG;327 max_intr = vf->nr_vring + 1;328 if (nvectors < max_intr)329 vf->msix_vector_status = MSIX_VECTOR_SHARED_VQ_AND_CONFIG;330 331 if (nvectors == 1) {332 vf->msix_vector_status = MSIX_VECTOR_DEV_SHARED;333 ret = ifcvf_request_dev_irq(vf);334 335 return ret;336 }337 338 ret = ifcvf_request_vq_irq(vf);339 if (ret)340 return ret;341 342 ret = ifcvf_request_config_irq(vf);343 344 if (ret)345 return ret;346 347 vf->num_msix_vectors = nvectors;348 349 return 0;350}351 352static struct ifcvf_adapter *vdpa_to_adapter(struct vdpa_device *vdpa_dev)353{354 return container_of(vdpa_dev, struct ifcvf_adapter, vdpa);355}356 357static struct ifcvf_hw *vdpa_to_vf(struct vdpa_device *vdpa_dev)358{359 struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);360 361 return adapter->vf;362}363 364static u64 ifcvf_vdpa_get_device_features(struct vdpa_device *vdpa_dev)365{366 struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);367 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);368 struct pci_dev *pdev = adapter->pdev;369 u32 type = vf->dev_type;370 u64 features;371 372 if (type == VIRTIO_ID_NET || type == VIRTIO_ID_BLOCK)373 features = ifcvf_get_dev_features(vf);374 else {375 features = 0;376 IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", vf->dev_type);377 }378 379 return features;380}381 382static int ifcvf_vdpa_set_driver_features(struct vdpa_device *vdpa_dev, u64 features)383{384 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);385 int ret;386 387 ret = ifcvf_verify_min_features(vf, features);388 if (ret)389 return ret;390 391 ifcvf_set_driver_features(vf, features);392 393 return 0;394}395 396static u64 ifcvf_vdpa_get_driver_features(struct vdpa_device *vdpa_dev)397{398 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);399 u64 features;400 401 features = ifcvf_get_driver_features(vf);402 403 return features;404}405 406static u8 ifcvf_vdpa_get_status(struct vdpa_device *vdpa_dev)407{408 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);409 410 return ifcvf_get_status(vf);411}412 413static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)414{415 struct ifcvf_hw *vf;416 u8 status_old;417 int ret;418 419 vf = vdpa_to_vf(vdpa_dev);420 status_old = ifcvf_get_status(vf);421 422 if (status_old == status)423 return;424 425 if ((status & VIRTIO_CONFIG_S_DRIVER_OK) &&426 !(status_old & VIRTIO_CONFIG_S_DRIVER_OK)) {427 ret = ifcvf_request_irq(vf);428 if (ret) {429 IFCVF_ERR(vf->pdev, "failed to request irq with error %d\n", ret);430 return;431 }432 }433 434 ifcvf_set_status(vf, status);435}436 437static int ifcvf_vdpa_reset(struct vdpa_device *vdpa_dev)438{439 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);440 u8 status = ifcvf_get_status(vf);441 442 ifcvf_stop(vf);443 444 if (status & VIRTIO_CONFIG_S_DRIVER_OK)445 ifcvf_free_irq(vf);446 447 ifcvf_reset(vf);448 449 return 0;450}451 452static u16 ifcvf_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev)453{454 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);455 456 return ifcvf_get_max_vq_size(vf);457}458 459static u16 ifcvf_vdpa_get_vq_num_min(struct vdpa_device *vdpa_dev)460{461 return IFCVF_MIN_VQ_SIZE;462}463 464static int ifcvf_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,465 struct vdpa_vq_state *state)466{467 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);468 469 state->split.avail_index = ifcvf_get_vq_state(vf, qid);470 return 0;471}472 473static int ifcvf_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid,474 const struct vdpa_vq_state *state)475{476 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);477 478 return ifcvf_set_vq_state(vf, qid, state->split.avail_index);479}480 481static void ifcvf_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid,482 struct vdpa_callback *cb)483{484 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);485 486 vf->vring[qid].cb = *cb;487}488 489static void ifcvf_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev,490 u16 qid, bool ready)491{492 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);493 494 ifcvf_set_vq_ready(vf, qid, ready);495}496 497static bool ifcvf_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid)498{499 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);500 501 return ifcvf_get_vq_ready(vf, qid);502}503 504static void ifcvf_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid,505 u32 num)506{507 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);508 509 ifcvf_set_vq_num(vf, qid, num);510}511 512static int ifcvf_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid,513 u64 desc_area, u64 driver_area,514 u64 device_area)515{516 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);517 518 return ifcvf_set_vq_address(vf, qid, desc_area, driver_area, device_area);519}520 521static void ifcvf_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid)522{523 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);524 525 ifcvf_notify_queue(vf, qid);526}527 528static u32 ifcvf_vdpa_get_generation(struct vdpa_device *vdpa_dev)529{530 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);531 532 return vp_ioread8(&vf->common_cfg->config_generation);533}534 535static u32 ifcvf_vdpa_get_device_id(struct vdpa_device *vdpa_dev)536{537 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);538 539 return vf->dev_type;540}541 542static u32 ifcvf_vdpa_get_vendor_id(struct vdpa_device *vdpa_dev)543{544 struct ifcvf_adapter *adapter = vdpa_to_adapter(vdpa_dev);545 struct pci_dev *pdev = adapter->pdev;546 547 return pdev->subsystem_vendor;548}549 550static u32 ifcvf_vdpa_get_vq_align(struct vdpa_device *vdpa_dev)551{552 return IFCVF_QUEUE_ALIGNMENT;553}554 555static size_t ifcvf_vdpa_get_config_size(struct vdpa_device *vdpa_dev)556{557 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);558 559 return vf->config_size;560}561 562static u32 ifcvf_vdpa_get_vq_group(struct vdpa_device *vdpa, u16 idx)563{564 return 0;565}566 567static void ifcvf_vdpa_get_config(struct vdpa_device *vdpa_dev,568 unsigned int offset,569 void *buf, unsigned int len)570{571 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);572 573 ifcvf_read_dev_config(vf, offset, buf, len);574}575 576static void ifcvf_vdpa_set_config(struct vdpa_device *vdpa_dev,577 unsigned int offset, const void *buf,578 unsigned int len)579{580 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);581 582 ifcvf_write_dev_config(vf, offset, buf, len);583}584 585static void ifcvf_vdpa_set_config_cb(struct vdpa_device *vdpa_dev,586 struct vdpa_callback *cb)587{588 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);589 590 vf->config_cb.callback = cb->callback;591 vf->config_cb.private = cb->private;592}593 594static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev,595 u16 qid)596{597 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);598 599 if (vf->vqs_reused_irq < 0)600 return vf->vring[qid].irq;601 else602 return -EINVAL;603}604 605static u16 ifcvf_vdpa_get_vq_size(struct vdpa_device *vdpa_dev,606 u16 qid)607{608 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);609 610 return ifcvf_get_vq_size(vf, qid);611}612 613static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,614 u16 idx)615{616 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);617 struct vdpa_notification_area area;618 619 area.addr = vf->vring[idx].notify_pa;620 if (!vf->notify_off_multiplier)621 area.size = PAGE_SIZE;622 else623 area.size = vf->notify_off_multiplier;624 625 return area;626}627 628/*629 * IFCVF currently doesn't have on-chip IOMMU, so not630 * implemented set_map()/dma_map()/dma_unmap()631 */632static const struct vdpa_config_ops ifc_vdpa_ops = {633 .get_device_features = ifcvf_vdpa_get_device_features,634 .set_driver_features = ifcvf_vdpa_set_driver_features,635 .get_driver_features = ifcvf_vdpa_get_driver_features,636 .get_status = ifcvf_vdpa_get_status,637 .set_status = ifcvf_vdpa_set_status,638 .reset = ifcvf_vdpa_reset,639 .get_vq_num_max = ifcvf_vdpa_get_vq_num_max,640 .get_vq_num_min = ifcvf_vdpa_get_vq_num_min,641 .get_vq_state = ifcvf_vdpa_get_vq_state,642 .set_vq_state = ifcvf_vdpa_set_vq_state,643 .set_vq_cb = ifcvf_vdpa_set_vq_cb,644 .set_vq_ready = ifcvf_vdpa_set_vq_ready,645 .get_vq_ready = ifcvf_vdpa_get_vq_ready,646 .set_vq_num = ifcvf_vdpa_set_vq_num,647 .set_vq_address = ifcvf_vdpa_set_vq_address,648 .get_vq_irq = ifcvf_vdpa_get_vq_irq,649 .get_vq_size = ifcvf_vdpa_get_vq_size,650 .kick_vq = ifcvf_vdpa_kick_vq,651 .get_generation = ifcvf_vdpa_get_generation,652 .get_device_id = ifcvf_vdpa_get_device_id,653 .get_vendor_id = ifcvf_vdpa_get_vendor_id,654 .get_vq_align = ifcvf_vdpa_get_vq_align,655 .get_vq_group = ifcvf_vdpa_get_vq_group,656 .get_config_size = ifcvf_vdpa_get_config_size,657 .get_config = ifcvf_vdpa_get_config,658 .set_config = ifcvf_vdpa_set_config,659 .set_config_cb = ifcvf_vdpa_set_config_cb,660 .get_vq_notification = ifcvf_get_vq_notification,661};662 663static struct virtio_device_id id_table_net[] = {664 {VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},665 {0},666};667 668static struct virtio_device_id id_table_blk[] = {669 {VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID},670 {0},671};672 673static u32 get_dev_type(struct pci_dev *pdev)674{675 u32 dev_type;676 677 /* This drirver drives both modern virtio devices and transitional678 * devices in modern mode.679 * vDPA requires feature bit VIRTIO_F_ACCESS_PLATFORM,680 * so legacy devices and transitional devices in legacy681 * mode will not work for vDPA, this driver will not682 * drive devices with legacy interface.683 */684 685 if (pdev->device < 0x1040)686 dev_type = pdev->subsystem_device;687 else688 dev_type = pdev->device - 0x1040;689 690 return dev_type;691}692 693static int ifcvf_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,694 const struct vdpa_dev_set_config *config)695{696 struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;697 struct ifcvf_adapter *adapter;698 struct vdpa_device *vdpa_dev;699 struct pci_dev *pdev;700 struct ifcvf_hw *vf;701 u64 device_features;702 int ret;703 704 ifcvf_mgmt_dev = container_of(mdev, struct ifcvf_vdpa_mgmt_dev, mdev);705 vf = &ifcvf_mgmt_dev->vf;706 pdev = vf->pdev;707 adapter = vdpa_alloc_device(struct ifcvf_adapter, vdpa,708 &pdev->dev, &ifc_vdpa_ops, 1, 1, NULL, false);709 if (IS_ERR(adapter)) {710 IFCVF_ERR(pdev, "Failed to allocate vDPA structure");711 return PTR_ERR(adapter);712 }713 714 ifcvf_mgmt_dev->adapter = adapter;715 adapter->pdev = pdev;716 adapter->vdpa.dma_dev = &pdev->dev;717 adapter->vdpa.mdev = mdev;718 adapter->vf = vf;719 vdpa_dev = &adapter->vdpa;720 721 device_features = vf->hw_features;722 if (config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {723 if (config->device_features & ~device_features) {724 IFCVF_ERR(pdev, "The provisioned features 0x%llx are not supported by this device with features 0x%llx\n",725 config->device_features, device_features);726 return -EINVAL;727 }728 device_features &= config->device_features;729 }730 vf->dev_features = device_features;731 732 if (name)733 ret = dev_set_name(&vdpa_dev->dev, "%s", name);734 else735 ret = dev_set_name(&vdpa_dev->dev, "vdpa%u", vdpa_dev->index);736 737 ret = _vdpa_register_device(&adapter->vdpa, vf->nr_vring);738 if (ret) {739 put_device(&adapter->vdpa.dev);740 IFCVF_ERR(pdev, "Failed to register to vDPA bus");741 return ret;742 }743 744 return 0;745}746 747static void ifcvf_vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev)748{749 struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;750 751 ifcvf_mgmt_dev = container_of(mdev, struct ifcvf_vdpa_mgmt_dev, mdev);752 _vdpa_unregister_device(dev);753 ifcvf_mgmt_dev->adapter = NULL;754}755 756static const struct vdpa_mgmtdev_ops ifcvf_vdpa_mgmt_dev_ops = {757 .dev_add = ifcvf_vdpa_dev_add,758 .dev_del = ifcvf_vdpa_dev_del759};760 761static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)762{763 struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;764 struct device *dev = &pdev->dev;765 struct ifcvf_hw *vf;766 u32 dev_type;767 int ret, i;768 769 ret = pcim_enable_device(pdev);770 if (ret) {771 IFCVF_ERR(pdev, "Failed to enable device\n");772 return ret;773 }774 ret = pcim_iomap_regions(pdev, BIT(0) | BIT(2) | BIT(4),775 IFCVF_DRIVER_NAME);776 if (ret) {777 IFCVF_ERR(pdev, "Failed to request MMIO region\n");778 return ret;779 }780 781 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));782 if (ret) {783 IFCVF_ERR(pdev, "No usable DMA configuration\n");784 return ret;785 }786 787 ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev);788 if (ret) {789 IFCVF_ERR(pdev,790 "Failed for adding devres for freeing irq vectors\n");791 return ret;792 }793 794 pci_set_master(pdev);795 ifcvf_mgmt_dev = kzalloc(sizeof(struct ifcvf_vdpa_mgmt_dev), GFP_KERNEL);796 if (!ifcvf_mgmt_dev) {797 IFCVF_ERR(pdev, "Failed to alloc memory for the vDPA management device\n");798 return -ENOMEM;799 }800 801 vf = &ifcvf_mgmt_dev->vf;802 vf->dev_type = get_dev_type(pdev);803 vf->base = pcim_iomap_table(pdev);804 vf->pdev = pdev;805 806 ret = ifcvf_init_hw(vf, pdev);807 if (ret) {808 IFCVF_ERR(pdev, "Failed to init IFCVF hw\n");809 goto err;810 }811 812 for (i = 0; i < vf->nr_vring; i++)813 vf->vring[i].irq = -EINVAL;814 815 vf->hw_features = ifcvf_get_hw_features(vf);816 vf->config_size = ifcvf_get_config_size(vf);817 818 dev_type = get_dev_type(pdev);819 switch (dev_type) {820 case VIRTIO_ID_NET:821 ifcvf_mgmt_dev->mdev.id_table = id_table_net;822 break;823 case VIRTIO_ID_BLOCK:824 ifcvf_mgmt_dev->mdev.id_table = id_table_blk;825 break;826 default:827 IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", dev_type);828 ret = -EOPNOTSUPP;829 goto err;830 }831 832 ifcvf_mgmt_dev->mdev.ops = &ifcvf_vdpa_mgmt_dev_ops;833 ifcvf_mgmt_dev->mdev.device = dev;834 ifcvf_mgmt_dev->mdev.max_supported_vqs = vf->nr_vring;835 ifcvf_mgmt_dev->mdev.supported_features = vf->hw_features;836 ifcvf_mgmt_dev->mdev.config_attr_mask = (1 << VDPA_ATTR_DEV_FEATURES);837 838 ret = vdpa_mgmtdev_register(&ifcvf_mgmt_dev->mdev);839 if (ret) {840 IFCVF_ERR(pdev,841 "Failed to initialize the management interfaces\n");842 goto err;843 }844 845 pci_set_drvdata(pdev, ifcvf_mgmt_dev);846 847 return 0;848 849err:850 kfree(ifcvf_mgmt_dev->vf.vring);851 kfree(ifcvf_mgmt_dev);852 return ret;853}854 855static void ifcvf_remove(struct pci_dev *pdev)856{857 struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev;858 859 ifcvf_mgmt_dev = pci_get_drvdata(pdev);860 vdpa_mgmtdev_unregister(&ifcvf_mgmt_dev->mdev);861 kfree(ifcvf_mgmt_dev->vf.vring);862 kfree(ifcvf_mgmt_dev);863}864 865static struct pci_device_id ifcvf_pci_ids[] = {866 /* N3000 network device */867 { PCI_DEVICE_SUB(PCI_VENDOR_ID_REDHAT_QUMRANET,868 N3000_DEVICE_ID,869 PCI_VENDOR_ID_INTEL,870 N3000_SUBSYS_DEVICE_ID) },871 /* C5000X-PL network device872 * F2000X-PL network device873 */874 { PCI_DEVICE_SUB(PCI_VENDOR_ID_REDHAT_QUMRANET,875 VIRTIO_TRANS_ID_NET,876 PCI_VENDOR_ID_INTEL,877 VIRTIO_ID_NET) },878 /* C5000X-PL block device */879 { PCI_DEVICE_SUB(PCI_VENDOR_ID_REDHAT_QUMRANET,880 VIRTIO_TRANS_ID_BLOCK,881 PCI_VENDOR_ID_INTEL,882 VIRTIO_ID_BLOCK) },883 884 { 0 },885};886MODULE_DEVICE_TABLE(pci, ifcvf_pci_ids);887 888static struct pci_driver ifcvf_driver = {889 .name = IFCVF_DRIVER_NAME,890 .id_table = ifcvf_pci_ids,891 .probe = ifcvf_probe,892 .remove = ifcvf_remove,893};894 895module_pci_driver(ifcvf_driver);896 897MODULE_DESCRIPTION("Intel IFC VF NIC driver for virtio dataplane offloading");898MODULE_LICENSE("GPL v2");899