1139 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * SolidRun DPU driver for control plane4 *5 * Copyright (C) 2022-2023 SolidRun6 *7 * Author: Alvaro Karsz <alvaro.karsz@solid-run.com>8 *9 */10#include <linux/iopoll.h>11 12#include "snet_vdpa.h"13 14/* SNET DPU device ID */15#define SNET_DEVICE_ID 0x100016/* SNET signature */17#define SNET_SIGNATURE 0xD0D0636318/* Max. config version that we can work with */19#define SNET_CFG_VERSION 0x220/* Queue align */21#define SNET_QUEUE_ALIGNMENT PAGE_SIZE22/* Kick value to notify that new data is available */23#define SNET_KICK_VAL 0x124#define SNET_CONFIG_OFF 0x025/* How long we are willing to wait for a SNET device */26#define SNET_DETECT_TIMEOUT 500000027/* How long should we wait for the DPU to read our config */28#define SNET_READ_CFG_TIMEOUT 300000029/* Size of configs written to the DPU */30#define SNET_GENERAL_CFG_LEN 3631#define SNET_GENERAL_CFG_VQ_LEN 4032 33static struct snet *vdpa_to_snet(struct vdpa_device *vdpa)34{35 return container_of(vdpa, struct snet, vdpa);36}37 38static irqreturn_t snet_cfg_irq_hndlr(int irq, void *data)39{40 struct snet *snet = data;41 /* Call callback if any */42 if (likely(snet->cb.callback))43 return snet->cb.callback(snet->cb.private);44 45 return IRQ_HANDLED;46}47 48static irqreturn_t snet_vq_irq_hndlr(int irq, void *data)49{50 struct snet_vq *vq = data;51 /* Call callback if any */52 if (likely(vq->cb.callback))53 return vq->cb.callback(vq->cb.private);54 55 return IRQ_HANDLED;56}57 58static void snet_free_irqs(struct snet *snet)59{60 struct psnet *psnet = snet->psnet;61 struct pci_dev *pdev;62 u32 i;63 64 /* Which Device allcoated the IRQs? */65 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF))66 pdev = snet->pdev->physfn;67 else68 pdev = snet->pdev;69 70 /* Free config's IRQ */71 if (snet->cfg_irq != -1) {72 devm_free_irq(&pdev->dev, snet->cfg_irq, snet);73 snet->cfg_irq = -1;74 }75 /* Free VQ IRQs */76 for (i = 0; i < snet->cfg->vq_num; i++) {77 if (snet->vqs[i] && snet->vqs[i]->irq != -1) {78 devm_free_irq(&pdev->dev, snet->vqs[i]->irq, snet->vqs[i]);79 snet->vqs[i]->irq = -1;80 }81 }82 83 /* IRQ vectors are freed when the pci remove callback is called */84}85 86static int snet_set_vq_address(struct vdpa_device *vdev, u16 idx, u64 desc_area,87 u64 driver_area, u64 device_area)88{89 struct snet *snet = vdpa_to_snet(vdev);90 /* save received parameters in vqueue sturct */91 snet->vqs[idx]->desc_area = desc_area;92 snet->vqs[idx]->driver_area = driver_area;93 snet->vqs[idx]->device_area = device_area;94 95 return 0;96}97 98static void snet_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num)99{100 struct snet *snet = vdpa_to_snet(vdev);101 /* save num in vqueue */102 snet->vqs[idx]->num = num;103}104 105static void snet_kick_vq(struct vdpa_device *vdev, u16 idx)106{107 struct snet *snet = vdpa_to_snet(vdev);108 /* not ready - ignore */109 if (unlikely(!snet->vqs[idx]->ready))110 return;111 112 iowrite32(SNET_KICK_VAL, snet->vqs[idx]->kick_ptr);113}114 115static void snet_kick_vq_with_data(struct vdpa_device *vdev, u32 data)116{117 struct snet *snet = vdpa_to_snet(vdev);118 u16 idx = data & 0xFFFF;119 120 /* not ready - ignore */121 if (unlikely(!snet->vqs[idx]->ready))122 return;123 124 iowrite32((data & 0xFFFF0000) | SNET_KICK_VAL, snet->vqs[idx]->kick_ptr);125}126 127static void snet_set_vq_cb(struct vdpa_device *vdev, u16 idx, struct vdpa_callback *cb)128{129 struct snet *snet = vdpa_to_snet(vdev);130 131 snet->vqs[idx]->cb.callback = cb->callback;132 snet->vqs[idx]->cb.private = cb->private;133}134 135static void snet_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready)136{137 struct snet *snet = vdpa_to_snet(vdev);138 139 snet->vqs[idx]->ready = ready;140}141 142static bool snet_get_vq_ready(struct vdpa_device *vdev, u16 idx)143{144 struct snet *snet = vdpa_to_snet(vdev);145 146 return snet->vqs[idx]->ready;147}148 149static bool snet_vq_state_is_initial(struct snet *snet, const struct vdpa_vq_state *state)150{151 if (SNET_HAS_FEATURE(snet, VIRTIO_F_RING_PACKED)) {152 const struct vdpa_vq_state_packed *p = &state->packed;153 154 if (p->last_avail_counter == 1 && p->last_used_counter == 1 &&155 p->last_avail_idx == 0 && p->last_used_idx == 0)156 return true;157 } else {158 const struct vdpa_vq_state_split *s = &state->split;159 160 if (s->avail_index == 0)161 return true;162 }163 164 return false;165}166 167static int snet_set_vq_state(struct vdpa_device *vdev, u16 idx, const struct vdpa_vq_state *state)168{169 struct snet *snet = vdpa_to_snet(vdev);170 171 /* We can set any state for config version 2+ */172 if (SNET_CFG_VER(snet, 2)) {173 memcpy(&snet->vqs[idx]->vq_state, state, sizeof(*state));174 return 0;175 }176 177 /* Older config - we can't set the VQ state.178 * Return 0 only if this is the initial state we use in the DPU.179 */180 if (snet_vq_state_is_initial(snet, state))181 return 0;182 183 return -EOPNOTSUPP;184}185 186static int snet_get_vq_state(struct vdpa_device *vdev, u16 idx, struct vdpa_vq_state *state)187{188 struct snet *snet = vdpa_to_snet(vdev);189 190 return snet_read_vq_state(snet, idx, state);191}192 193static int snet_get_vq_irq(struct vdpa_device *vdev, u16 idx)194{195 struct snet *snet = vdpa_to_snet(vdev);196 197 return snet->vqs[idx]->irq;198}199 200static u32 snet_get_vq_align(struct vdpa_device *vdev)201{202 return (u32)SNET_QUEUE_ALIGNMENT;203}204 205static int snet_reset_dev(struct snet *snet)206{207 struct pci_dev *pdev = snet->pdev;208 int ret = 0;209 u32 i;210 211 /* If status is 0, nothing to do */212 if (!snet->status)213 return 0;214 215 /* If DPU started, destroy it */216 if (snet->status & VIRTIO_CONFIG_S_DRIVER_OK)217 ret = snet_destroy_dev(snet);218 219 /* Clear VQs */220 for (i = 0; i < snet->cfg->vq_num; i++) {221 if (!snet->vqs[i])222 continue;223 snet->vqs[i]->cb.callback = NULL;224 snet->vqs[i]->cb.private = NULL;225 snet->vqs[i]->desc_area = 0;226 snet->vqs[i]->device_area = 0;227 snet->vqs[i]->driver_area = 0;228 snet->vqs[i]->ready = false;229 }230 231 /* Clear config callback */232 snet->cb.callback = NULL;233 snet->cb.private = NULL;234 /* Free IRQs */235 snet_free_irqs(snet);236 /* Reset status */237 snet->status = 0;238 snet->dpu_ready = false;239 240 if (ret)241 SNET_WARN(pdev, "Incomplete reset to SNET[%u] device, err: %d\n", snet->sid, ret);242 else243 SNET_DBG(pdev, "Reset SNET[%u] device\n", snet->sid);244 245 return 0;246}247 248static int snet_reset(struct vdpa_device *vdev)249{250 struct snet *snet = vdpa_to_snet(vdev);251 252 return snet_reset_dev(snet);253}254 255static size_t snet_get_config_size(struct vdpa_device *vdev)256{257 struct snet *snet = vdpa_to_snet(vdev);258 259 return (size_t)snet->cfg->cfg_size;260}261 262static u64 snet_get_features(struct vdpa_device *vdev)263{264 struct snet *snet = vdpa_to_snet(vdev);265 266 return snet->cfg->features;267}268 269static int snet_set_drv_features(struct vdpa_device *vdev, u64 features)270{271 struct snet *snet = vdpa_to_snet(vdev);272 273 snet->negotiated_features = snet->cfg->features & features;274 return 0;275}276 277static u64 snet_get_drv_features(struct vdpa_device *vdev)278{279 struct snet *snet = vdpa_to_snet(vdev);280 281 return snet->negotiated_features;282}283 284static u16 snet_get_vq_num_max(struct vdpa_device *vdev)285{286 struct snet *snet = vdpa_to_snet(vdev);287 288 return (u16)snet->cfg->vq_size;289}290 291static void snet_set_config_cb(struct vdpa_device *vdev, struct vdpa_callback *cb)292{293 struct snet *snet = vdpa_to_snet(vdev);294 295 snet->cb.callback = cb->callback;296 snet->cb.private = cb->private;297}298 299static u32 snet_get_device_id(struct vdpa_device *vdev)300{301 struct snet *snet = vdpa_to_snet(vdev);302 303 return snet->cfg->virtio_id;304}305 306static u32 snet_get_vendor_id(struct vdpa_device *vdev)307{308 return (u32)PCI_VENDOR_ID_SOLIDRUN;309}310 311static u8 snet_get_status(struct vdpa_device *vdev)312{313 struct snet *snet = vdpa_to_snet(vdev);314 315 return snet->status;316}317 318static int snet_write_conf(struct snet *snet)319{320 u32 off, i, tmp;321 int ret;322 323 /* No need to write the config twice */324 if (snet->dpu_ready)325 return true;326 327 /* Snet data :328 *329 * General data: SNET_GENERAL_CFG_LEN bytes long330 * 0 0x4 0x8 0xC 0x10 0x14 0x1C 0x24331 * | MAGIC NUMBER | CFG VER | SNET SID | NUMBER OF QUEUES | IRQ IDX | FEATURES | RSVD |332 *333 * For every VQ: SNET_GENERAL_CFG_VQ_LEN bytes long334 * 0 0x4 0x8335 * | VQ SID AND QUEUE SIZE | IRQ Index |336 * | DESC AREA |337 * | DEVICE AREA |338 * | DRIVER AREA |339 * | VQ STATE (CFG 2+) | RSVD |340 *341 * Magic number should be written last, this is the DPU indication that the data is ready342 */343 344 /* Init offset */345 off = snet->psnet->cfg.host_cfg_off;346 347 /* Ignore magic number for now */348 off += 4;349 snet_write32(snet, off, snet->psnet->negotiated_cfg_ver);350 off += 4;351 snet_write32(snet, off, snet->sid);352 off += 4;353 snet_write32(snet, off, snet->cfg->vq_num);354 off += 4;355 snet_write32(snet, off, snet->cfg_irq_idx);356 off += 4;357 snet_write64(snet, off, snet->negotiated_features);358 off += 8;359 /* Ignore reserved */360 off += 8;361 /* Write VQs */362 for (i = 0 ; i < snet->cfg->vq_num ; i++) {363 tmp = (i << 16) | (snet->vqs[i]->num & 0xFFFF);364 snet_write32(snet, off, tmp);365 off += 4;366 snet_write32(snet, off, snet->vqs[i]->irq_idx);367 off += 4;368 snet_write64(snet, off, snet->vqs[i]->desc_area);369 off += 8;370 snet_write64(snet, off, snet->vqs[i]->device_area);371 off += 8;372 snet_write64(snet, off, snet->vqs[i]->driver_area);373 off += 8;374 /* Write VQ state if config version is 2+ */375 if (SNET_CFG_VER(snet, 2))376 snet_write32(snet, off, *(u32 *)&snet->vqs[i]->vq_state);377 off += 4;378 379 /* Ignore reserved */380 off += 4;381 }382 383 /* Write magic number - data is ready */384 snet_write32(snet, snet->psnet->cfg.host_cfg_off, SNET_SIGNATURE);385 386 /* The DPU will ACK the config by clearing the signature */387 ret = readx_poll_timeout(ioread32, snet->bar + snet->psnet->cfg.host_cfg_off,388 tmp, !tmp, 10, SNET_READ_CFG_TIMEOUT);389 if (ret) {390 SNET_ERR(snet->pdev, "Timeout waiting for the DPU to read the config\n");391 return false;392 }393 394 /* set DPU flag */395 snet->dpu_ready = true;396 397 return true;398}399 400static int snet_request_irqs(struct pci_dev *pdev, struct snet *snet)401{402 int ret, i, irq;403 404 /* Request config IRQ */405 irq = pci_irq_vector(pdev, snet->cfg_irq_idx);406 ret = devm_request_irq(&pdev->dev, irq, snet_cfg_irq_hndlr, 0,407 snet->cfg_irq_name, snet);408 if (ret) {409 SNET_ERR(pdev, "Failed to request IRQ\n");410 return ret;411 }412 snet->cfg_irq = irq;413 414 /* Request IRQ for every VQ */415 for (i = 0; i < snet->cfg->vq_num; i++) {416 irq = pci_irq_vector(pdev, snet->vqs[i]->irq_idx);417 ret = devm_request_irq(&pdev->dev, irq, snet_vq_irq_hndlr, 0,418 snet->vqs[i]->irq_name, snet->vqs[i]);419 if (ret) {420 SNET_ERR(pdev, "Failed to request IRQ\n");421 return ret;422 }423 snet->vqs[i]->irq = irq;424 }425 return 0;426}427 428static void snet_set_status(struct vdpa_device *vdev, u8 status)429{430 struct snet *snet = vdpa_to_snet(vdev);431 struct psnet *psnet = snet->psnet;432 struct pci_dev *pdev = snet->pdev;433 int ret;434 bool pf_irqs;435 436 if (status == snet->status)437 return;438 439 if ((status & VIRTIO_CONFIG_S_DRIVER_OK) &&440 !(snet->status & VIRTIO_CONFIG_S_DRIVER_OK)) {441 /* Request IRQs */442 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF);443 ret = snet_request_irqs(pf_irqs ? pdev->physfn : pdev, snet);444 if (ret)445 goto set_err;446 447 /* Write config to the DPU */448 if (snet_write_conf(snet)) {449 SNET_INFO(pdev, "Create SNET[%u] device\n", snet->sid);450 } else {451 snet_free_irqs(snet);452 goto set_err;453 }454 }455 456 /* Save the new status */457 snet->status = status;458 return;459 460set_err:461 snet->status |= VIRTIO_CONFIG_S_FAILED;462}463 464static void snet_get_config(struct vdpa_device *vdev, unsigned int offset,465 void *buf, unsigned int len)466{467 struct snet *snet = vdpa_to_snet(vdev);468 void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset;469 u8 *buf_ptr = buf;470 u32 i;471 472 /* check for offset error */473 if (offset + len > snet->cfg->cfg_size)474 return;475 476 /* Write into buffer */477 for (i = 0; i < len; i++)478 *buf_ptr++ = ioread8(cfg_ptr + i);479}480 481static void snet_set_config(struct vdpa_device *vdev, unsigned int offset,482 const void *buf, unsigned int len)483{484 struct snet *snet = vdpa_to_snet(vdev);485 void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset;486 const u8 *buf_ptr = buf;487 u32 i;488 489 /* check for offset error */490 if (offset + len > snet->cfg->cfg_size)491 return;492 493 /* Write into PCI BAR */494 for (i = 0; i < len; i++)495 iowrite8(*buf_ptr++, cfg_ptr + i);496}497 498static int snet_suspend(struct vdpa_device *vdev)499{500 struct snet *snet = vdpa_to_snet(vdev);501 int ret;502 503 ret = snet_suspend_dev(snet);504 if (ret)505 SNET_ERR(snet->pdev, "SNET[%u] suspend failed, err: %d\n", snet->sid, ret);506 else507 SNET_DBG(snet->pdev, "Suspend SNET[%u] device\n", snet->sid);508 509 return ret;510}511 512static int snet_resume(struct vdpa_device *vdev)513{514 struct snet *snet = vdpa_to_snet(vdev);515 int ret;516 517 ret = snet_resume_dev(snet);518 if (ret)519 SNET_ERR(snet->pdev, "SNET[%u] resume failed, err: %d\n", snet->sid, ret);520 else521 SNET_DBG(snet->pdev, "Resume SNET[%u] device\n", snet->sid);522 523 return ret;524}525 526static const struct vdpa_config_ops snet_config_ops = {527 .set_vq_address = snet_set_vq_address,528 .set_vq_num = snet_set_vq_num,529 .kick_vq = snet_kick_vq,530 .kick_vq_with_data = snet_kick_vq_with_data,531 .set_vq_cb = snet_set_vq_cb,532 .set_vq_ready = snet_set_vq_ready,533 .get_vq_ready = snet_get_vq_ready,534 .set_vq_state = snet_set_vq_state,535 .get_vq_state = snet_get_vq_state,536 .get_vq_irq = snet_get_vq_irq,537 .get_vq_align = snet_get_vq_align,538 .reset = snet_reset,539 .get_config_size = snet_get_config_size,540 .get_device_features = snet_get_features,541 .set_driver_features = snet_set_drv_features,542 .get_driver_features = snet_get_drv_features,543 .get_vq_num_min = snet_get_vq_num_max,544 .get_vq_num_max = snet_get_vq_num_max,545 .set_config_cb = snet_set_config_cb,546 .get_device_id = snet_get_device_id,547 .get_vendor_id = snet_get_vendor_id,548 .get_status = snet_get_status,549 .set_status = snet_set_status,550 .get_config = snet_get_config,551 .set_config = snet_set_config,552 .suspend = snet_suspend,553 .resume = snet_resume,554};555 556static int psnet_open_pf_bar(struct pci_dev *pdev, struct psnet *psnet)557{558 char *name;559 int ret, i, mask = 0;560 /* We don't know which BAR will be used to communicate..561 * We will map every bar with len > 0.562 *563 * Later, we will discover the BAR and unmap all other BARs.564 */565 for (i = 0; i < PCI_STD_NUM_BARS; i++) {566 if (pci_resource_len(pdev, i))567 mask |= (1 << i);568 }569 570 /* No BAR can be used.. */571 if (!mask) {572 SNET_ERR(pdev, "Failed to find a PCI BAR\n");573 return -ENODEV;574 }575 576 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "psnet[%s]-bars", pci_name(pdev));577 if (!name)578 return -ENOMEM;579 580 ret = pcim_iomap_regions(pdev, mask, name);581 if (ret) {582 SNET_ERR(pdev, "Failed to request and map PCI BARs\n");583 return ret;584 }585 586 for (i = 0; i < PCI_STD_NUM_BARS; i++) {587 if (mask & (1 << i))588 psnet->bars[i] = pcim_iomap_table(pdev)[i];589 }590 591 return 0;592}593 594static int snet_open_vf_bar(struct pci_dev *pdev, struct snet *snet)595{596 char *name;597 int ret;598 599 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "snet[%s]-bars", pci_name(pdev));600 if (!name)601 return -ENOMEM;602 603 /* Request and map BAR */604 ret = pcim_iomap_regions(pdev, BIT(snet->psnet->cfg.vf_bar), name);605 if (ret) {606 SNET_ERR(pdev, "Failed to request and map PCI BAR for a VF\n");607 return ret;608 }609 610 snet->bar = pcim_iomap_table(pdev)[snet->psnet->cfg.vf_bar];611 612 return 0;613}614 615static void snet_free_cfg(struct snet_cfg *cfg)616{617 u32 i;618 619 if (!cfg->devs)620 return;621 622 /* Free devices */623 for (i = 0; i < cfg->devices_num; i++) {624 if (!cfg->devs[i])625 break;626 627 kfree(cfg->devs[i]);628 }629 /* Free pointers to devices */630 kfree(cfg->devs);631}632 633/* Detect which BAR is used for communication with the device. */634static int psnet_detect_bar(struct psnet *psnet, u32 off)635{636 unsigned long exit_time;637 int i;638 639 exit_time = jiffies + usecs_to_jiffies(SNET_DETECT_TIMEOUT);640 641 /* SNET DPU will write SNET's signature when the config is ready. */642 while (time_before(jiffies, exit_time)) {643 for (i = 0; i < PCI_STD_NUM_BARS; i++) {644 /* Is this BAR mapped? */645 if (!psnet->bars[i])646 continue;647 648 if (ioread32(psnet->bars[i] + off) == SNET_SIGNATURE)649 return i;650 }651 usleep_range(1000, 10000);652 }653 654 return -ENODEV;655}656 657static void psnet_unmap_unused_bars(struct pci_dev *pdev, struct psnet *psnet)658{659 int i, mask = 0;660 661 for (i = 0; i < PCI_STD_NUM_BARS; i++) {662 if (psnet->bars[i] && i != psnet->barno)663 mask |= (1 << i);664 }665 666 if (mask)667 pcim_iounmap_regions(pdev, mask);668}669 670/* Read SNET config from PCI BAR */671static int psnet_read_cfg(struct pci_dev *pdev, struct psnet *psnet)672{673 struct snet_cfg *cfg = &psnet->cfg;674 u32 i, off;675 int barno;676 677 /* Move to where the config starts */678 off = SNET_CONFIG_OFF;679 680 /* Find BAR used for communication */681 barno = psnet_detect_bar(psnet, off);682 if (barno < 0) {683 SNET_ERR(pdev, "SNET config is not ready.\n");684 return barno;685 }686 687 /* Save used BAR number and unmap all other BARs */688 psnet->barno = barno;689 SNET_DBG(pdev, "Using BAR number %d\n", barno);690 691 psnet_unmap_unused_bars(pdev, psnet);692 693 /* load config from BAR */694 cfg->key = psnet_read32(psnet, off);695 off += 4;696 cfg->cfg_size = psnet_read32(psnet, off);697 off += 4;698 cfg->cfg_ver = psnet_read32(psnet, off);699 off += 4;700 /* The negotiated config version is the lower one between this driver's config701 * and the DPU's.702 */703 psnet->negotiated_cfg_ver = min_t(u32, cfg->cfg_ver, SNET_CFG_VERSION);704 SNET_DBG(pdev, "SNET config version %u\n", psnet->negotiated_cfg_ver);705 706 cfg->vf_num = psnet_read32(psnet, off);707 off += 4;708 cfg->vf_bar = psnet_read32(psnet, off);709 off += 4;710 cfg->host_cfg_off = psnet_read32(psnet, off);711 off += 4;712 cfg->max_size_host_cfg = psnet_read32(psnet, off);713 off += 4;714 cfg->virtio_cfg_off = psnet_read32(psnet, off);715 off += 4;716 cfg->kick_off = psnet_read32(psnet, off);717 off += 4;718 cfg->hwmon_off = psnet_read32(psnet, off);719 off += 4;720 cfg->ctrl_off = psnet_read32(psnet, off);721 off += 4;722 cfg->flags = psnet_read32(psnet, off);723 off += 4;724 /* Ignore Reserved */725 off += sizeof(cfg->rsvd);726 727 cfg->devices_num = psnet_read32(psnet, off);728 off += 4;729 /* Allocate memory to hold pointer to the devices */730 cfg->devs = kcalloc(cfg->devices_num, sizeof(void *), GFP_KERNEL);731 if (!cfg->devs)732 return -ENOMEM;733 734 /* Load device configuration from BAR */735 for (i = 0; i < cfg->devices_num; i++) {736 cfg->devs[i] = kzalloc(sizeof(*cfg->devs[i]), GFP_KERNEL);737 if (!cfg->devs[i]) {738 snet_free_cfg(cfg);739 return -ENOMEM;740 }741 /* Read device config */742 cfg->devs[i]->virtio_id = psnet_read32(psnet, off);743 off += 4;744 cfg->devs[i]->vq_num = psnet_read32(psnet, off);745 off += 4;746 cfg->devs[i]->vq_size = psnet_read32(psnet, off);747 off += 4;748 cfg->devs[i]->vfid = psnet_read32(psnet, off);749 off += 4;750 cfg->devs[i]->features = psnet_read64(psnet, off);751 off += 8;752 /* Ignore Reserved */753 off += sizeof(cfg->devs[i]->rsvd);754 755 cfg->devs[i]->cfg_size = psnet_read32(psnet, off);756 off += 4;757 758 /* Is the config witten to the DPU going to be too big? */759 if (SNET_GENERAL_CFG_LEN + SNET_GENERAL_CFG_VQ_LEN * cfg->devs[i]->vq_num >760 cfg->max_size_host_cfg) {761 SNET_ERR(pdev, "Failed to read SNET config, the config is too big..\n");762 snet_free_cfg(cfg);763 return -EINVAL;764 }765 }766 return 0;767}768 769static int psnet_alloc_irq_vector(struct pci_dev *pdev, struct psnet *psnet)770{771 int ret = 0;772 u32 i, irq_num = 0;773 774 /* Let's count how many IRQs we need, 1 for every VQ + 1 for config change */775 for (i = 0; i < psnet->cfg.devices_num; i++)776 irq_num += psnet->cfg.devs[i]->vq_num + 1;777 778 ret = pci_alloc_irq_vectors(pdev, irq_num, irq_num, PCI_IRQ_MSIX);779 if (ret != irq_num) {780 SNET_ERR(pdev, "Failed to allocate IRQ vectors\n");781 return ret;782 }783 SNET_DBG(pdev, "Allocated %u IRQ vectors from physical function\n", irq_num);784 785 return 0;786}787 788static int snet_alloc_irq_vector(struct pci_dev *pdev, struct snet_dev_cfg *snet_cfg)789{790 int ret = 0;791 u32 irq_num;792 793 /* We want 1 IRQ for every VQ + 1 for config change events */794 irq_num = snet_cfg->vq_num + 1;795 796 ret = pci_alloc_irq_vectors(pdev, irq_num, irq_num, PCI_IRQ_MSIX);797 if (ret <= 0) {798 SNET_ERR(pdev, "Failed to allocate IRQ vectors\n");799 return ret;800 }801 802 return 0;803}804 805static void snet_free_vqs(struct snet *snet)806{807 u32 i;808 809 if (!snet->vqs)810 return;811 812 for (i = 0 ; i < snet->cfg->vq_num ; i++) {813 if (!snet->vqs[i])814 break;815 816 kfree(snet->vqs[i]);817 }818 kfree(snet->vqs);819}820 821static int snet_build_vqs(struct snet *snet)822{823 u32 i;824 /* Allocate the VQ pointers array */825 snet->vqs = kcalloc(snet->cfg->vq_num, sizeof(void *), GFP_KERNEL);826 if (!snet->vqs)827 return -ENOMEM;828 829 /* Allocate the VQs */830 for (i = 0; i < snet->cfg->vq_num; i++) {831 snet->vqs[i] = kzalloc(sizeof(*snet->vqs[i]), GFP_KERNEL);832 if (!snet->vqs[i]) {833 snet_free_vqs(snet);834 return -ENOMEM;835 }836 /* Reset IRQ num */837 snet->vqs[i]->irq = -1;838 /* VQ serial ID */839 snet->vqs[i]->sid = i;840 /* Kick address - every VQ gets 4B */841 snet->vqs[i]->kick_ptr = snet->bar + snet->psnet->cfg.kick_off +842 snet->vqs[i]->sid * 4;843 /* Clear kick address for this VQ */844 iowrite32(0, snet->vqs[i]->kick_ptr);845 }846 return 0;847}848 849static int psnet_get_next_irq_num(struct psnet *psnet)850{851 int irq;852 853 spin_lock(&psnet->lock);854 irq = psnet->next_irq++;855 spin_unlock(&psnet->lock);856 857 return irq;858}859 860static void snet_reserve_irq_idx(struct pci_dev *pdev, struct snet *snet)861{862 struct psnet *psnet = snet->psnet;863 int i;864 865 /* one IRQ for every VQ, and one for config changes */866 snet->cfg_irq_idx = psnet_get_next_irq_num(psnet);867 snprintf(snet->cfg_irq_name, SNET_NAME_SIZE, "snet[%s]-cfg[%d]",868 pci_name(pdev), snet->cfg_irq_idx);869 870 for (i = 0; i < snet->cfg->vq_num; i++) {871 /* Get next free IRQ ID */872 snet->vqs[i]->irq_idx = psnet_get_next_irq_num(psnet);873 /* Write IRQ name */874 snprintf(snet->vqs[i]->irq_name, SNET_NAME_SIZE, "snet[%s]-vq[%d]",875 pci_name(pdev), snet->vqs[i]->irq_idx);876 }877}878 879/* Find a device config based on virtual function id */880static struct snet_dev_cfg *snet_find_dev_cfg(struct snet_cfg *cfg, u32 vfid)881{882 u32 i;883 884 for (i = 0; i < cfg->devices_num; i++) {885 if (cfg->devs[i]->vfid == vfid)886 return cfg->devs[i];887 }888 /* Oppss.. no config found.. */889 return NULL;890}891 892/* Probe function for a physical PCI function */893static int snet_vdpa_probe_pf(struct pci_dev *pdev)894{895 struct psnet *psnet;896 int ret = 0;897 bool pf_irqs = false;898 899 ret = pcim_enable_device(pdev);900 if (ret) {901 SNET_ERR(pdev, "Failed to enable PCI device\n");902 return ret;903 }904 905 /* Allocate a PCI physical function device */906 psnet = kzalloc(sizeof(*psnet), GFP_KERNEL);907 if (!psnet)908 return -ENOMEM;909 910 /* Init PSNET spinlock */911 spin_lock_init(&psnet->lock);912 913 pci_set_master(pdev);914 pci_set_drvdata(pdev, psnet);915 916 /* Open SNET MAIN BAR */917 ret = psnet_open_pf_bar(pdev, psnet);918 if (ret)919 goto free_psnet;920 921 /* Try to read SNET's config from PCI BAR */922 ret = psnet_read_cfg(pdev, psnet);923 if (ret)924 goto free_psnet;925 926 /* If SNET_CFG_FLAG_IRQ_PF flag is set, we should use927 * PF MSI-X vectors928 */929 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF);930 931 if (pf_irqs) {932 ret = psnet_alloc_irq_vector(pdev, psnet);933 if (ret)934 goto free_cfg;935 }936 937 SNET_DBG(pdev, "Enable %u virtual functions\n", psnet->cfg.vf_num);938 ret = pci_enable_sriov(pdev, psnet->cfg.vf_num);939 if (ret) {940 SNET_ERR(pdev, "Failed to enable SR-IOV\n");941 goto free_irq;942 }943 944 /* Create HW monitor device */945 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_HWMON)) {946#if IS_ENABLED(CONFIG_HWMON)947 psnet_create_hwmon(pdev);948#else949 SNET_WARN(pdev, "Can't start HWMON, CONFIG_HWMON is not enabled\n");950#endif951 }952 953 return 0;954 955free_irq:956 if (pf_irqs)957 pci_free_irq_vectors(pdev);958free_cfg:959 snet_free_cfg(&psnet->cfg);960free_psnet:961 kfree(psnet);962 return ret;963}964 965/* Probe function for a virtual PCI function */966static int snet_vdpa_probe_vf(struct pci_dev *pdev)967{968 struct pci_dev *pdev_pf = pdev->physfn;969 struct psnet *psnet = pci_get_drvdata(pdev_pf);970 struct snet_dev_cfg *dev_cfg;971 struct snet *snet;972 u32 vfid;973 int ret;974 bool pf_irqs = false;975 976 /* Get virtual function id.977 * (the DPU counts the VFs from 1)978 */979 ret = pci_iov_vf_id(pdev);980 if (ret < 0) {981 SNET_ERR(pdev, "Failed to find a VF id\n");982 return ret;983 }984 vfid = ret + 1;985 986 /* Find the snet_dev_cfg based on vfid */987 dev_cfg = snet_find_dev_cfg(&psnet->cfg, vfid);988 if (!dev_cfg) {989 SNET_WARN(pdev, "Failed to find a VF config..\n");990 return -ENODEV;991 }992 993 /* Which PCI device should allocate the IRQs?994 * If the SNET_CFG_FLAG_IRQ_PF flag set, the PF device allocates the IRQs995 */996 pf_irqs = PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF);997 998 ret = pcim_enable_device(pdev);999 if (ret) {1000 SNET_ERR(pdev, "Failed to enable PCI VF device\n");1001 return ret;1002 }1003 1004 /* Request for MSI-X IRQs */1005 if (!pf_irqs) {1006 ret = snet_alloc_irq_vector(pdev, dev_cfg);1007 if (ret)1008 return ret;1009 }1010 1011 /* Allocate vdpa device */1012 snet = vdpa_alloc_device(struct snet, vdpa, &pdev->dev, &snet_config_ops, 1, 1, NULL,1013 false);1014 if (!snet) {1015 SNET_ERR(pdev, "Failed to allocate a vdpa device\n");1016 ret = -ENOMEM;1017 goto free_irqs;1018 }1019 1020 /* Init control mutex and spinlock */1021 mutex_init(&snet->ctrl_lock);1022 spin_lock_init(&snet->ctrl_spinlock);1023 1024 /* Save pci device pointer */1025 snet->pdev = pdev;1026 snet->psnet = psnet;1027 snet->cfg = dev_cfg;1028 snet->dpu_ready = false;1029 snet->sid = vfid;1030 /* Reset IRQ value */1031 snet->cfg_irq = -1;1032 1033 ret = snet_open_vf_bar(pdev, snet);1034 if (ret)1035 goto put_device;1036 1037 /* Create a VirtIO config pointer */1038 snet->cfg->virtio_cfg = snet->bar + snet->psnet->cfg.virtio_cfg_off;1039 1040 /* Clear control registers */1041 snet_ctrl_clear(snet);1042 1043 pci_set_master(pdev);1044 pci_set_drvdata(pdev, snet);1045 1046 ret = snet_build_vqs(snet);1047 if (ret)1048 goto put_device;1049 1050 /* Reserve IRQ indexes,1051 * The IRQs may be requested and freed multiple times,1052 * but the indexes won't change.1053 */1054 snet_reserve_irq_idx(pf_irqs ? pdev_pf : pdev, snet);1055 1056 /*set DMA device*/1057 snet->vdpa.dma_dev = &pdev->dev;1058 1059 /* Register VDPA device */1060 ret = vdpa_register_device(&snet->vdpa, snet->cfg->vq_num);1061 if (ret) {1062 SNET_ERR(pdev, "Failed to register vdpa device\n");1063 goto free_vqs;1064 }1065 1066 return 0;1067 1068free_vqs:1069 snet_free_vqs(snet);1070put_device:1071 put_device(&snet->vdpa.dev);1072free_irqs:1073 if (!pf_irqs)1074 pci_free_irq_vectors(pdev);1075 return ret;1076}1077 1078static int snet_vdpa_probe(struct pci_dev *pdev, const struct pci_device_id *id)1079{1080 if (pdev->is_virtfn)1081 return snet_vdpa_probe_vf(pdev);1082 else1083 return snet_vdpa_probe_pf(pdev);1084}1085 1086static void snet_vdpa_remove_pf(struct pci_dev *pdev)1087{1088 struct psnet *psnet = pci_get_drvdata(pdev);1089 1090 pci_disable_sriov(pdev);1091 /* If IRQs are allocated from the PF, we should free the IRQs */1092 if (PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF))1093 pci_free_irq_vectors(pdev);1094 1095 snet_free_cfg(&psnet->cfg);1096 kfree(psnet);1097}1098 1099static void snet_vdpa_remove_vf(struct pci_dev *pdev)1100{1101 struct snet *snet = pci_get_drvdata(pdev);1102 struct psnet *psnet = snet->psnet;1103 1104 vdpa_unregister_device(&snet->vdpa);1105 snet_free_vqs(snet);1106 /* If IRQs are allocated from the VF, we should free the IRQs */1107 if (!PSNET_FLAG_ON(psnet, SNET_CFG_FLAG_IRQ_PF))1108 pci_free_irq_vectors(pdev);1109}1110 1111static void snet_vdpa_remove(struct pci_dev *pdev)1112{1113 if (pdev->is_virtfn)1114 snet_vdpa_remove_vf(pdev);1115 else1116 snet_vdpa_remove_pf(pdev);1117}1118 1119static struct pci_device_id snet_driver_pci_ids[] = {1120 { PCI_DEVICE_SUB(PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID,1121 PCI_VENDOR_ID_SOLIDRUN, SNET_DEVICE_ID) },1122 { 0 },1123};1124 1125MODULE_DEVICE_TABLE(pci, snet_driver_pci_ids);1126 1127static struct pci_driver snet_vdpa_driver = {1128 .name = "snet-vdpa-driver",1129 .id_table = snet_driver_pci_ids,1130 .probe = snet_vdpa_probe,1131 .remove = snet_vdpa_remove,1132};1133 1134module_pci_driver(snet_vdpa_driver);1135 1136MODULE_AUTHOR("Alvaro Karsz <alvaro.karsz@solid-run.com>");1137MODULE_DESCRIPTION("SolidRun vDPA driver");1138MODULE_LICENSE("GPL v2");1139