228 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB2/*3 * Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved4 */5 6#include "mlx5_ib.h"7#include "data_direct.h"8 9static LIST_HEAD(mlx5_data_direct_dev_list);10static LIST_HEAD(mlx5_data_direct_reg_list);11 12/*13 * This mutex should be held when accessing either of the above lists14 */15static DEFINE_MUTEX(mlx5_data_direct_mutex);16 17struct mlx5_data_direct_registration {18 struct mlx5_ib_dev *ibdev;19 char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1];20 struct list_head list;21};22 23static const struct pci_device_id mlx5_data_direct_pci_table[] = {24 { PCI_VDEVICE(MELLANOX, 0x2100) }, /* ConnectX-8 Data Direct */25 { 0, }26};27 28static int mlx5_data_direct_vpd_get_vuid(struct mlx5_data_direct_dev *dev)29{30 struct pci_dev *pdev = dev->pdev;31 unsigned int vpd_size, kw_len;32 u8 *vpd_data;33 int start;34 int ret;35 36 vpd_data = pci_vpd_alloc(pdev, &vpd_size);37 if (IS_ERR(vpd_data)) {38 pci_err(pdev, "Unable to read VPD, err=%ld\n", PTR_ERR(vpd_data));39 return PTR_ERR(vpd_data);40 }41 42 start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, "VU", &kw_len);43 if (start < 0) {44 ret = start;45 pci_err(pdev, "VU keyword not found, err=%d\n", ret);46 goto end;47 }48 49 dev->vuid = kmemdup_nul(vpd_data + start, kw_len, GFP_KERNEL);50 ret = dev->vuid ? 0 : -ENOMEM;51 52end:53 kfree(vpd_data);54 return ret;55}56 57static void mlx5_data_direct_shutdown(struct pci_dev *pdev)58{59 pci_disable_device(pdev);60}61 62static int mlx5_data_direct_set_dma_caps(struct pci_dev *pdev)63{64 int err;65 66 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));67 if (err) {68 dev_warn(&pdev->dev,69 "Warning: couldn't set 64-bit PCI DMA mask, err=%d\n", err);70 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));71 if (err) {72 dev_err(&pdev->dev, "Can't set PCI DMA mask, err=%d\n", err);73 return err;74 }75 }76 77 dma_set_max_seg_size(&pdev->dev, SZ_2G);78 return 0;79}80 81int mlx5_data_direct_ib_reg(struct mlx5_ib_dev *ibdev, char *vuid)82{83 struct mlx5_data_direct_registration *reg;84 struct mlx5_data_direct_dev *dev;85 86 reg = kzalloc(sizeof(*reg), GFP_KERNEL);87 if (!reg)88 return -ENOMEM;89 90 reg->ibdev = ibdev;91 strcpy(reg->vuid, vuid);92 93 mutex_lock(&mlx5_data_direct_mutex);94 list_for_each_entry(dev, &mlx5_data_direct_dev_list, list) {95 if (strcmp(dev->vuid, vuid) == 0) {96 mlx5_ib_data_direct_bind(ibdev, dev);97 break;98 }99 }100 101 /* Add the registration to its global list, to be used upon bind/unbind102 * of its affiliated data direct device103 */104 list_add_tail(®->list, &mlx5_data_direct_reg_list);105 mutex_unlock(&mlx5_data_direct_mutex);106 return 0;107}108 109void mlx5_data_direct_ib_unreg(struct mlx5_ib_dev *ibdev)110{111 struct mlx5_data_direct_registration *reg;112 113 mutex_lock(&mlx5_data_direct_mutex);114 list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {115 if (reg->ibdev == ibdev) {116 list_del(®->list);117 kfree(reg);118 goto end;119 }120 }121 122 WARN_ON(true);123end:124 mutex_unlock(&mlx5_data_direct_mutex);125}126 127static void mlx5_data_direct_dev_reg(struct mlx5_data_direct_dev *dev)128{129 struct mlx5_data_direct_registration *reg;130 131 mutex_lock(&mlx5_data_direct_mutex);132 list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {133 if (strcmp(dev->vuid, reg->vuid) == 0)134 mlx5_ib_data_direct_bind(reg->ibdev, dev);135 }136 137 /* Add the data direct device to the global list, further IB devices may138 * use it later as well139 */140 list_add_tail(&dev->list, &mlx5_data_direct_dev_list);141 mutex_unlock(&mlx5_data_direct_mutex);142}143 144static void mlx5_data_direct_dev_unreg(struct mlx5_data_direct_dev *dev)145{146 struct mlx5_data_direct_registration *reg;147 148 mutex_lock(&mlx5_data_direct_mutex);149 /* Prevent any further affiliations */150 list_del(&dev->list);151 list_for_each_entry(reg, &mlx5_data_direct_reg_list, list) {152 if (strcmp(dev->vuid, reg->vuid) == 0)153 mlx5_ib_data_direct_unbind(reg->ibdev);154 }155 mutex_unlock(&mlx5_data_direct_mutex);156}157 158static int mlx5_data_direct_probe(struct pci_dev *pdev, const struct pci_device_id *id)159{160 struct mlx5_data_direct_dev *dev;161 int err;162 163 dev = kzalloc(sizeof(*dev), GFP_KERNEL);164 if (!dev)165 return -ENOMEM;166 167 dev->device = &pdev->dev;168 dev->pdev = pdev;169 170 pci_set_drvdata(dev->pdev, dev);171 err = pci_enable_device(pdev);172 if (err) {173 dev_err(dev->device, "Cannot enable PCI device, err=%d\n", err);174 goto err;175 }176 177 pci_set_master(pdev);178 err = mlx5_data_direct_set_dma_caps(pdev);179 if (err)180 goto err_disable;181 182 if (pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP32) &&183 pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP64) &&184 pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP128))185 dev_dbg(dev->device, "Enabling pci atomics failed\n");186 187 err = mlx5_data_direct_vpd_get_vuid(dev);188 if (err)189 goto err_disable;190 191 mlx5_data_direct_dev_reg(dev);192 return 0;193 194err_disable:195 pci_disable_device(pdev);196err:197 kfree(dev);198 return err;199}200 201static void mlx5_data_direct_remove(struct pci_dev *pdev)202{203 struct mlx5_data_direct_dev *dev = pci_get_drvdata(pdev);204 205 mlx5_data_direct_dev_unreg(dev);206 pci_disable_device(pdev);207 kfree(dev->vuid);208 kfree(dev);209}210 211static struct pci_driver mlx5_data_direct_driver = {212 .name = KBUILD_MODNAME,213 .id_table = mlx5_data_direct_pci_table,214 .probe = mlx5_data_direct_probe,215 .remove = mlx5_data_direct_remove,216 .shutdown = mlx5_data_direct_shutdown,217};218 219int mlx5_data_direct_driver_register(void)220{221 return pci_register_driver(&mlx5_data_direct_driver);222}223 224void mlx5_data_direct_driver_unregister(void)225{226 pci_unregister_driver(&mlx5_data_direct_driver);227}228