859 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright(c) 2023 Advanced Micro Devices, Inc */3 4#include <linux/pci.h>5#include <linux/vdpa.h>6#include <uapi/linux/vdpa.h>7#include <linux/virtio_pci_modern.h>8 9#include <linux/pds/pds_common.h>10#include <linux/pds/pds_core_if.h>11#include <linux/pds/pds_adminq.h>12#include <linux/pds/pds_auxbus.h>13 14#include "vdpa_dev.h"15#include "aux_drv.h"16#include "cmds.h"17#include "debugfs.h"18 19static u64 pds_vdpa_get_driver_features(struct vdpa_device *vdpa_dev);20 21static struct pds_vdpa_device *vdpa_to_pdsv(struct vdpa_device *vdpa_dev)22{23 return container_of(vdpa_dev, struct pds_vdpa_device, vdpa_dev);24}25 26static int pds_vdpa_notify_handler(struct notifier_block *nb,27 unsigned long ecode,28 void *data)29{30 struct pds_vdpa_device *pdsv = container_of(nb, struct pds_vdpa_device, nb);31 struct device *dev = &pdsv->vdpa_aux->padev->aux_dev.dev;32 33 dev_dbg(dev, "%s: event code %lu\n", __func__, ecode);34 35 if (ecode == PDS_EVENT_RESET || ecode == PDS_EVENT_LINK_CHANGE) {36 if (pdsv->config_cb.callback)37 pdsv->config_cb.callback(pdsv->config_cb.private);38 }39 40 return 0;41}42 43static int pds_vdpa_register_event_handler(struct pds_vdpa_device *pdsv)44{45 struct device *dev = &pdsv->vdpa_aux->padev->aux_dev.dev;46 struct notifier_block *nb = &pdsv->nb;47 int err;48 49 if (!nb->notifier_call) {50 nb->notifier_call = pds_vdpa_notify_handler;51 err = pdsc_register_notify(nb);52 if (err) {53 nb->notifier_call = NULL;54 dev_err(dev, "failed to register pds event handler: %ps\n",55 ERR_PTR(err));56 return -EINVAL;57 }58 dev_dbg(dev, "pds event handler registered\n");59 }60 61 return 0;62}63 64static void pds_vdpa_unregister_event_handler(struct pds_vdpa_device *pdsv)65{66 if (pdsv->nb.notifier_call) {67 pdsc_unregister_notify(&pdsv->nb);68 pdsv->nb.notifier_call = NULL;69 }70}71 72static int pds_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid,73 u64 desc_addr, u64 driver_addr, u64 device_addr)74{75 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);76 77 pdsv->vqs[qid].desc_addr = desc_addr;78 pdsv->vqs[qid].avail_addr = driver_addr;79 pdsv->vqs[qid].used_addr = device_addr;80 81 return 0;82}83 84static void pds_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid, u32 num)85{86 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);87 88 pdsv->vqs[qid].q_len = num;89}90 91static void pds_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid)92{93 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);94 95 iowrite16(qid, pdsv->vqs[qid].notify);96}97 98static void pds_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid,99 struct vdpa_callback *cb)100{101 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);102 103 pdsv->vqs[qid].event_cb = *cb;104}105 106static irqreturn_t pds_vdpa_isr(int irq, void *data)107{108 struct pds_vdpa_vq_info *vq;109 110 vq = data;111 if (vq->event_cb.callback)112 vq->event_cb.callback(vq->event_cb.private);113 114 return IRQ_HANDLED;115}116 117static void pds_vdpa_release_irq(struct pds_vdpa_device *pdsv, int qid)118{119 if (pdsv->vqs[qid].irq == VIRTIO_MSI_NO_VECTOR)120 return;121 122 free_irq(pdsv->vqs[qid].irq, &pdsv->vqs[qid]);123 pdsv->vqs[qid].irq = VIRTIO_MSI_NO_VECTOR;124}125 126static void pds_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev, u16 qid, bool ready)127{128 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);129 struct device *dev = &pdsv->vdpa_dev.dev;130 u64 driver_features;131 u16 invert_idx = 0;132 int err;133 134 dev_dbg(dev, "%s: qid %d ready %d => %d\n",135 __func__, qid, pdsv->vqs[qid].ready, ready);136 if (ready == pdsv->vqs[qid].ready)137 return;138 139 driver_features = pds_vdpa_get_driver_features(vdpa_dev);140 if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED))141 invert_idx = PDS_VDPA_PACKED_INVERT_IDX;142 143 if (ready) {144 /* Pass vq setup info to DSC using adminq to gather up and145 * send all info at once so FW can do its full set up in146 * one easy operation147 */148 err = pds_vdpa_cmd_init_vq(pdsv, qid, invert_idx, &pdsv->vqs[qid]);149 if (err) {150 dev_err(dev, "Failed to init vq %d: %pe\n",151 qid, ERR_PTR(err));152 ready = false;153 }154 } else {155 err = pds_vdpa_cmd_reset_vq(pdsv, qid, invert_idx, &pdsv->vqs[qid]);156 if (err)157 dev_err(dev, "%s: reset_vq failed qid %d: %pe\n",158 __func__, qid, ERR_PTR(err));159 }160 161 pdsv->vqs[qid].ready = ready;162}163 164static bool pds_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid)165{166 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);167 168 return pdsv->vqs[qid].ready;169}170 171static int pds_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid,172 const struct vdpa_vq_state *state)173{174 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);175 struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;176 struct device *dev = &padev->aux_dev.dev;177 u64 driver_features;178 u16 avail;179 u16 used;180 181 if (pdsv->vqs[qid].ready) {182 dev_err(dev, "Setting device position is denied while vq is enabled\n");183 return -EINVAL;184 }185 186 driver_features = pds_vdpa_get_driver_features(vdpa_dev);187 if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {188 avail = state->packed.last_avail_idx |189 (state->packed.last_avail_counter << 15);190 used = state->packed.last_used_idx |191 (state->packed.last_used_counter << 15);192 193 /* The avail and used index are stored with the packed wrap194 * counter bit inverted. This way, in case set_vq_state is195 * not called, the initial value can be set to zero prior to196 * feature negotiation, and it is good for both packed and197 * split vq.198 */199 avail ^= PDS_VDPA_PACKED_INVERT_IDX;200 used ^= PDS_VDPA_PACKED_INVERT_IDX;201 } else {202 avail = state->split.avail_index;203 /* state->split does not provide a used_index:204 * the vq will be set to "empty" here, and the vq will read205 * the current used index the next time the vq is kicked.206 */207 used = avail;208 }209 210 if (used != avail) {211 dev_dbg(dev, "Setting used equal to avail, for interoperability\n");212 used = avail;213 }214 215 pdsv->vqs[qid].avail_idx = avail;216 pdsv->vqs[qid].used_idx = used;217 218 return 0;219}220 221static int pds_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,222 struct vdpa_vq_state *state)223{224 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);225 struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev;226 struct device *dev = &padev->aux_dev.dev;227 u64 driver_features;228 u16 avail;229 u16 used;230 231 if (pdsv->vqs[qid].ready) {232 dev_err(dev, "Getting device position is denied while vq is enabled\n");233 return -EINVAL;234 }235 236 avail = pdsv->vqs[qid].avail_idx;237 used = pdsv->vqs[qid].used_idx;238 239 driver_features = pds_vdpa_get_driver_features(vdpa_dev);240 if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {241 avail ^= PDS_VDPA_PACKED_INVERT_IDX;242 used ^= PDS_VDPA_PACKED_INVERT_IDX;243 244 state->packed.last_avail_idx = avail & 0x7fff;245 state->packed.last_avail_counter = avail >> 15;246 state->packed.last_used_idx = used & 0x7fff;247 state->packed.last_used_counter = used >> 15;248 } else {249 state->split.avail_index = avail;250 /* state->split does not provide a used_index. */251 }252 253 return 0;254}255 256static struct vdpa_notification_area257pds_vdpa_get_vq_notification(struct vdpa_device *vdpa_dev, u16 qid)258{259 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);260 struct virtio_pci_modern_device *vd_mdev;261 struct vdpa_notification_area area;262 263 area.addr = pdsv->vqs[qid].notify_pa;264 265 vd_mdev = &pdsv->vdpa_aux->vd_mdev;266 if (!vd_mdev->notify_offset_multiplier)267 area.size = PDS_PAGE_SIZE;268 else269 area.size = vd_mdev->notify_offset_multiplier;270 271 return area;272}273 274static int pds_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev, u16 qid)275{276 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);277 278 return pdsv->vqs[qid].irq;279}280 281static u32 pds_vdpa_get_vq_align(struct vdpa_device *vdpa_dev)282{283 return PDS_PAGE_SIZE;284}285 286static u32 pds_vdpa_get_vq_group(struct vdpa_device *vdpa_dev, u16 idx)287{288 return 0;289}290 291static u64 pds_vdpa_get_device_features(struct vdpa_device *vdpa_dev)292{293 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);294 295 return pdsv->supported_features;296}297 298static int pds_vdpa_set_driver_features(struct vdpa_device *vdpa_dev, u64 features)299{300 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);301 struct device *dev = &pdsv->vdpa_dev.dev;302 u64 driver_features;303 u64 nego_features;304 u64 hw_features;305 u64 missing;306 307 if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM)) && features) {308 dev_err(dev, "VIRTIO_F_ACCESS_PLATFORM is not negotiated\n");309 return -EOPNOTSUPP;310 }311 312 /* Check for valid feature bits */313 nego_features = features & pdsv->supported_features;314 missing = features & ~nego_features;315 if (missing) {316 dev_err(dev, "Can't support all requested features in %#llx, missing %#llx features\n",317 features, missing);318 return -EOPNOTSUPP;319 }320 321 driver_features = pds_vdpa_get_driver_features(vdpa_dev);322 pdsv->negotiated_features = nego_features;323 dev_dbg(dev, "%s: %#llx => %#llx\n",324 __func__, driver_features, nego_features);325 326 /* if we're faking the F_MAC, strip it before writing to device */327 hw_features = le64_to_cpu(pdsv->vdpa_aux->ident.hw_features);328 if (!(hw_features & BIT_ULL(VIRTIO_NET_F_MAC)))329 nego_features &= ~BIT_ULL(VIRTIO_NET_F_MAC);330 331 if (driver_features == nego_features)332 return 0;333 334 vp_modern_set_features(&pdsv->vdpa_aux->vd_mdev, nego_features);335 336 return 0;337}338 339static u64 pds_vdpa_get_driver_features(struct vdpa_device *vdpa_dev)340{341 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);342 343 return pdsv->negotiated_features;344}345 346static void pds_vdpa_set_config_cb(struct vdpa_device *vdpa_dev,347 struct vdpa_callback *cb)348{349 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);350 351 pdsv->config_cb.callback = cb->callback;352 pdsv->config_cb.private = cb->private;353}354 355static u16 pds_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev)356{357 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);358 359 /* qemu has assert() that vq_num_max <= VIRTQUEUE_MAX_SIZE (1024) */360 return min_t(u16, 1024, BIT(le16_to_cpu(pdsv->vdpa_aux->ident.max_qlen)));361}362 363static u32 pds_vdpa_get_device_id(struct vdpa_device *vdpa_dev)364{365 return VIRTIO_ID_NET;366}367 368static u32 pds_vdpa_get_vendor_id(struct vdpa_device *vdpa_dev)369{370 return PCI_VENDOR_ID_PENSANDO;371}372 373static u8 pds_vdpa_get_status(struct vdpa_device *vdpa_dev)374{375 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);376 377 return vp_modern_get_status(&pdsv->vdpa_aux->vd_mdev);378}379 380static int pds_vdpa_request_irqs(struct pds_vdpa_device *pdsv)381{382 struct pci_dev *pdev = pdsv->vdpa_aux->padev->vf_pdev;383 struct pds_vdpa_aux *vdpa_aux = pdsv->vdpa_aux;384 struct device *dev = &pdsv->vdpa_dev.dev;385 int max_vq, nintrs, qid, err;386 387 max_vq = vdpa_aux->vdpa_mdev.max_supported_vqs;388 389 nintrs = pci_alloc_irq_vectors(pdev, max_vq, max_vq, PCI_IRQ_MSIX);390 if (nintrs < 0) {391 dev_err(dev, "Couldn't get %d msix vectors: %pe\n",392 max_vq, ERR_PTR(nintrs));393 return nintrs;394 }395 396 for (qid = 0; qid < pdsv->num_vqs; ++qid) {397 int irq = pci_irq_vector(pdev, qid);398 399 snprintf(pdsv->vqs[qid].irq_name, sizeof(pdsv->vqs[qid].irq_name),400 "vdpa-%s-%d", dev_name(dev), qid);401 402 err = request_irq(irq, pds_vdpa_isr, 0,403 pdsv->vqs[qid].irq_name,404 &pdsv->vqs[qid]);405 if (err) {406 dev_err(dev, "%s: no irq for qid %d: %pe\n",407 __func__, qid, ERR_PTR(err));408 goto err_release;409 }410 411 pdsv->vqs[qid].irq = irq;412 }413 414 vdpa_aux->nintrs = nintrs;415 416 return 0;417 418err_release:419 while (qid--)420 pds_vdpa_release_irq(pdsv, qid);421 422 pci_free_irq_vectors(pdev);423 424 vdpa_aux->nintrs = 0;425 426 return err;427}428 429void pds_vdpa_release_irqs(struct pds_vdpa_device *pdsv)430{431 struct pds_vdpa_aux *vdpa_aux;432 struct pci_dev *pdev;433 int qid;434 435 if (!pdsv)436 return;437 438 pdev = pdsv->vdpa_aux->padev->vf_pdev;439 vdpa_aux = pdsv->vdpa_aux;440 441 if (!vdpa_aux->nintrs)442 return;443 444 for (qid = 0; qid < pdsv->num_vqs; qid++)445 pds_vdpa_release_irq(pdsv, qid);446 447 pci_free_irq_vectors(pdev);448 449 vdpa_aux->nintrs = 0;450}451 452static void pds_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status)453{454 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);455 struct device *dev = &pdsv->vdpa_dev.dev;456 u8 old_status;457 int i;458 459 old_status = pds_vdpa_get_status(vdpa_dev);460 dev_dbg(dev, "%s: old %#x new %#x\n", __func__, old_status, status);461 462 if (status & ~old_status & VIRTIO_CONFIG_S_DRIVER_OK) {463 if (pds_vdpa_request_irqs(pdsv))464 status = old_status | VIRTIO_CONFIG_S_FAILED;465 }466 467 pds_vdpa_cmd_set_status(pdsv, status);468 469 if (status == 0) {470 struct vdpa_callback null_cb = { };471 472 pds_vdpa_set_config_cb(vdpa_dev, &null_cb);473 pds_vdpa_cmd_reset(pdsv);474 475 for (i = 0; i < pdsv->num_vqs; i++) {476 pdsv->vqs[i].avail_idx = 0;477 pdsv->vqs[i].used_idx = 0;478 }479 480 pds_vdpa_cmd_set_mac(pdsv, pdsv->mac);481 }482 483 if (status & ~old_status & VIRTIO_CONFIG_S_FEATURES_OK) {484 for (i = 0; i < pdsv->num_vqs; i++) {485 pdsv->vqs[i].notify =486 vp_modern_map_vq_notify(&pdsv->vdpa_aux->vd_mdev,487 i, &pdsv->vqs[i].notify_pa);488 }489 }490 491 if (old_status & ~status & VIRTIO_CONFIG_S_DRIVER_OK)492 pds_vdpa_release_irqs(pdsv);493}494 495static void pds_vdpa_init_vqs_entry(struct pds_vdpa_device *pdsv, int qid,496 void __iomem *notify)497{498 memset(&pdsv->vqs[qid], 0, sizeof(pdsv->vqs[0]));499 pdsv->vqs[qid].qid = qid;500 pdsv->vqs[qid].pdsv = pdsv;501 pdsv->vqs[qid].ready = false;502 pdsv->vqs[qid].irq = VIRTIO_MSI_NO_VECTOR;503 pdsv->vqs[qid].notify = notify;504}505 506static int pds_vdpa_reset(struct vdpa_device *vdpa_dev)507{508 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);509 struct device *dev;510 int err = 0;511 u8 status;512 int i;513 514 dev = &pdsv->vdpa_aux->padev->aux_dev.dev;515 status = pds_vdpa_get_status(vdpa_dev);516 517 if (status == 0)518 return 0;519 520 if (status & VIRTIO_CONFIG_S_DRIVER_OK) {521 /* Reset the vqs */522 for (i = 0; i < pdsv->num_vqs && !err; i++) {523 err = pds_vdpa_cmd_reset_vq(pdsv, i, 0, &pdsv->vqs[i]);524 if (err)525 dev_err(dev, "%s: reset_vq failed qid %d: %pe\n",526 __func__, i, ERR_PTR(err));527 }528 }529 530 pds_vdpa_set_status(vdpa_dev, 0);531 532 if (status & VIRTIO_CONFIG_S_DRIVER_OK) {533 /* Reset the vq info */534 for (i = 0; i < pdsv->num_vqs && !err; i++)535 pds_vdpa_init_vqs_entry(pdsv, i, pdsv->vqs[i].notify);536 }537 538 return 0;539}540 541static size_t pds_vdpa_get_config_size(struct vdpa_device *vdpa_dev)542{543 return sizeof(struct virtio_net_config);544}545 546static void pds_vdpa_get_config(struct vdpa_device *vdpa_dev,547 unsigned int offset,548 void *buf, unsigned int len)549{550 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);551 void __iomem *device;552 553 if (offset + len > sizeof(struct virtio_net_config)) {554 WARN(true, "%s: bad read, offset %d len %d\n", __func__, offset, len);555 return;556 }557 558 device = pdsv->vdpa_aux->vd_mdev.device;559 memcpy_fromio(buf, device + offset, len);560}561 562static void pds_vdpa_set_config(struct vdpa_device *vdpa_dev,563 unsigned int offset, const void *buf,564 unsigned int len)565{566 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);567 void __iomem *device;568 569 if (offset + len > sizeof(struct virtio_net_config)) {570 WARN(true, "%s: bad read, offset %d len %d\n", __func__, offset, len);571 return;572 }573 574 device = pdsv->vdpa_aux->vd_mdev.device;575 memcpy_toio(device + offset, buf, len);576}577 578static const struct vdpa_config_ops pds_vdpa_ops = {579 .set_vq_address = pds_vdpa_set_vq_address,580 .set_vq_num = pds_vdpa_set_vq_num,581 .kick_vq = pds_vdpa_kick_vq,582 .set_vq_cb = pds_vdpa_set_vq_cb,583 .set_vq_ready = pds_vdpa_set_vq_ready,584 .get_vq_ready = pds_vdpa_get_vq_ready,585 .set_vq_state = pds_vdpa_set_vq_state,586 .get_vq_state = pds_vdpa_get_vq_state,587 .get_vq_notification = pds_vdpa_get_vq_notification,588 .get_vq_irq = pds_vdpa_get_vq_irq,589 .get_vq_align = pds_vdpa_get_vq_align,590 .get_vq_group = pds_vdpa_get_vq_group,591 592 .get_device_features = pds_vdpa_get_device_features,593 .set_driver_features = pds_vdpa_set_driver_features,594 .get_driver_features = pds_vdpa_get_driver_features,595 .set_config_cb = pds_vdpa_set_config_cb,596 .get_vq_num_max = pds_vdpa_get_vq_num_max,597 .get_device_id = pds_vdpa_get_device_id,598 .get_vendor_id = pds_vdpa_get_vendor_id,599 .get_status = pds_vdpa_get_status,600 .set_status = pds_vdpa_set_status,601 .reset = pds_vdpa_reset,602 .get_config_size = pds_vdpa_get_config_size,603 .get_config = pds_vdpa_get_config,604 .set_config = pds_vdpa_set_config,605};606static struct virtio_device_id pds_vdpa_id_table[] = {607 {VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID},608 {0},609};610 611static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,612 const struct vdpa_dev_set_config *add_config)613{614 struct pds_vdpa_aux *vdpa_aux;615 struct pds_vdpa_device *pdsv;616 struct vdpa_mgmt_dev *mgmt;617 u16 fw_max_vqs, vq_pairs;618 struct device *dma_dev;619 struct pci_dev *pdev;620 struct device *dev;621 u8 status;622 int err;623 int i;624 625 vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev);626 dev = &vdpa_aux->padev->aux_dev.dev;627 mgmt = &vdpa_aux->vdpa_mdev;628 629 if (vdpa_aux->pdsv) {630 dev_warn(dev, "Multiple vDPA devices on a VF is not supported.\n");631 return -EOPNOTSUPP;632 }633 634 pdsv = vdpa_alloc_device(struct pds_vdpa_device, vdpa_dev,635 dev, &pds_vdpa_ops, 1, 1, name, false);636 if (IS_ERR(pdsv)) {637 dev_err(dev, "Failed to allocate vDPA structure: %pe\n", pdsv);638 return PTR_ERR(pdsv);639 }640 641 vdpa_aux->pdsv = pdsv;642 pdsv->vdpa_aux = vdpa_aux;643 644 pdev = vdpa_aux->padev->vf_pdev;645 dma_dev = &pdev->dev;646 pdsv->vdpa_dev.dma_dev = dma_dev;647 648 status = pds_vdpa_get_status(&pdsv->vdpa_dev);649 if (status == 0xff) {650 dev_err(dev, "Broken PCI - status %#x\n", status);651 err = -ENXIO;652 goto err_unmap;653 }654 655 pdsv->supported_features = mgmt->supported_features;656 657 if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {658 u64 unsupp_features =659 add_config->device_features & ~pdsv->supported_features;660 661 if (unsupp_features) {662 dev_err(dev, "Unsupported features: %#llx\n", unsupp_features);663 err = -EOPNOTSUPP;664 goto err_unmap;665 }666 667 pdsv->supported_features = add_config->device_features;668 }669 670 err = pds_vdpa_cmd_reset(pdsv);671 if (err) {672 dev_err(dev, "Failed to reset hw: %pe\n", ERR_PTR(err));673 goto err_unmap;674 }675 676 err = pds_vdpa_init_hw(pdsv);677 if (err) {678 dev_err(dev, "Failed to init hw: %pe\n", ERR_PTR(err));679 goto err_unmap;680 }681 682 fw_max_vqs = le16_to_cpu(pdsv->vdpa_aux->ident.max_vqs);683 vq_pairs = fw_max_vqs / 2;684 685 /* Make sure we have the queues being requested */686 if (add_config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MAX_VQP))687 vq_pairs = add_config->net.max_vq_pairs;688 689 pdsv->num_vqs = 2 * vq_pairs;690 if (pdsv->supported_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))691 pdsv->num_vqs++;692 693 if (pdsv->num_vqs > fw_max_vqs) {694 dev_err(dev, "%s: queue count requested %u greater than max %u\n",695 __func__, pdsv->num_vqs, fw_max_vqs);696 err = -ENOSPC;697 goto err_unmap;698 }699 700 if (pdsv->num_vqs != fw_max_vqs) {701 err = pds_vdpa_cmd_set_max_vq_pairs(pdsv, vq_pairs);702 if (err) {703 dev_err(dev, "Failed to set max_vq_pairs: %pe\n",704 ERR_PTR(err));705 goto err_unmap;706 }707 }708 709 /* Set a mac, either from the user config if provided710 * or use the device's mac if not 00:..:00711 * or set a random mac712 */713 if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR)) {714 ether_addr_copy(pdsv->mac, add_config->net.mac);715 } else {716 struct virtio_net_config __iomem *vc;717 718 vc = pdsv->vdpa_aux->vd_mdev.device;719 memcpy_fromio(pdsv->mac, vc->mac, sizeof(pdsv->mac));720 if (is_zero_ether_addr(pdsv->mac) &&721 (pdsv->supported_features & BIT_ULL(VIRTIO_NET_F_MAC))) {722 eth_random_addr(pdsv->mac);723 dev_info(dev, "setting random mac %pM\n", pdsv->mac);724 }725 }726 pds_vdpa_cmd_set_mac(pdsv, pdsv->mac);727 728 for (i = 0; i < pdsv->num_vqs; i++) {729 void __iomem *notify;730 731 notify = vp_modern_map_vq_notify(&pdsv->vdpa_aux->vd_mdev,732 i, &pdsv->vqs[i].notify_pa);733 pds_vdpa_init_vqs_entry(pdsv, i, notify);734 }735 736 pdsv->vdpa_dev.mdev = &vdpa_aux->vdpa_mdev;737 738 err = pds_vdpa_register_event_handler(pdsv);739 if (err) {740 dev_err(dev, "Failed to register for PDS events: %pe\n", ERR_PTR(err));741 goto err_unmap;742 }743 744 /* We use the _vdpa_register_device() call rather than the745 * vdpa_register_device() to avoid a deadlock because our746 * dev_add() is called with the vdpa_dev_lock already set747 * by vdpa_nl_cmd_dev_add_set_doit()748 */749 err = _vdpa_register_device(&pdsv->vdpa_dev, pdsv->num_vqs);750 if (err) {751 dev_err(dev, "Failed to register to vDPA bus: %pe\n", ERR_PTR(err));752 goto err_unevent;753 }754 755 pds_vdpa_debugfs_add_vdpadev(vdpa_aux);756 757 return 0;758 759err_unevent:760 pds_vdpa_unregister_event_handler(pdsv);761err_unmap:762 put_device(&pdsv->vdpa_dev.dev);763 vdpa_aux->pdsv = NULL;764 return err;765}766 767static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev,768 struct vdpa_device *vdpa_dev)769{770 struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev);771 struct pds_vdpa_aux *vdpa_aux;772 773 pds_vdpa_unregister_event_handler(pdsv);774 775 vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev);776 _vdpa_unregister_device(vdpa_dev);777 778 pds_vdpa_cmd_reset(vdpa_aux->pdsv);779 pds_vdpa_debugfs_reset_vdpadev(vdpa_aux);780 781 vdpa_aux->pdsv = NULL;782 783 dev_info(&vdpa_aux->padev->aux_dev.dev, "Removed vdpa device\n");784}785 786static const struct vdpa_mgmtdev_ops pds_vdpa_mgmt_dev_ops = {787 .dev_add = pds_vdpa_dev_add,788 .dev_del = pds_vdpa_dev_del789};790 791int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux)792{793 union pds_core_adminq_cmd cmd = {794 .vdpa_ident.opcode = PDS_VDPA_CMD_IDENT,795 .vdpa_ident.vf_id = cpu_to_le16(vdpa_aux->vf_id),796 };797 union pds_core_adminq_comp comp = {};798 struct vdpa_mgmt_dev *mgmt;799 struct pci_dev *pf_pdev;800 struct device *pf_dev;801 struct pci_dev *pdev;802 dma_addr_t ident_pa;803 struct device *dev;804 u16 dev_intrs;805 u16 max_vqs;806 int err;807 808 dev = &vdpa_aux->padev->aux_dev.dev;809 pdev = vdpa_aux->padev->vf_pdev;810 mgmt = &vdpa_aux->vdpa_mdev;811 812 /* Get resource info through the PF's adminq. It is a block of info,813 * so we need to map some memory for PF to make available to the814 * firmware for writing the data.815 */816 pf_pdev = pci_physfn(vdpa_aux->padev->vf_pdev);817 pf_dev = &pf_pdev->dev;818 ident_pa = dma_map_single(pf_dev, &vdpa_aux->ident,819 sizeof(vdpa_aux->ident), DMA_FROM_DEVICE);820 if (dma_mapping_error(pf_dev, ident_pa)) {821 dev_err(dev, "Failed to map ident space\n");822 return -ENOMEM;823 }824 825 cmd.vdpa_ident.ident_pa = cpu_to_le64(ident_pa);826 cmd.vdpa_ident.len = cpu_to_le32(sizeof(vdpa_aux->ident));827 err = pds_client_adminq_cmd(vdpa_aux->padev, &cmd,828 sizeof(cmd.vdpa_ident), &comp, 0);829 dma_unmap_single(pf_dev, ident_pa,830 sizeof(vdpa_aux->ident), DMA_FROM_DEVICE);831 if (err) {832 dev_err(dev, "Failed to ident hw, status %d: %pe\n",833 comp.status, ERR_PTR(err));834 return err;835 }836 837 max_vqs = le16_to_cpu(vdpa_aux->ident.max_vqs);838 dev_intrs = pci_msix_vec_count(pdev);839 dev_dbg(dev, "ident.max_vqs %d dev_intrs %d\n", max_vqs, dev_intrs);840 841 max_vqs = min_t(u16, dev_intrs, max_vqs);842 mgmt->max_supported_vqs = min_t(u16, PDS_VDPA_MAX_QUEUES, max_vqs);843 vdpa_aux->nintrs = 0;844 845 mgmt->ops = &pds_vdpa_mgmt_dev_ops;846 mgmt->id_table = pds_vdpa_id_table;847 mgmt->device = dev;848 mgmt->supported_features = le64_to_cpu(vdpa_aux->ident.hw_features);849 850 /* advertise F_MAC even if the device doesn't */851 mgmt->supported_features |= BIT_ULL(VIRTIO_NET_F_MAC);852 853 mgmt->config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR);854 mgmt->config_attr_mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP);855 mgmt->config_attr_mask |= BIT_ULL(VDPA_ATTR_DEV_FEATURES);856 857 return 0;858}859