1445 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (c) 2021, HiSilicon Ltd.4 */5 6#include <linux/device.h>7#include <linux/eventfd.h>8#include <linux/file.h>9#include <linux/hisi_acc_qm.h>10#include <linux/interrupt.h>11#include <linux/module.h>12#include <linux/pci.h>13#include <linux/vfio.h>14#include <linux/vfio_pci_core.h>15#include <linux/anon_inodes.h>16 17#include "hisi_acc_vfio_pci.h"18 19/* Return 0 on VM acc device ready, -ETIMEDOUT hardware timeout */20static int qm_wait_dev_not_ready(struct hisi_qm *qm)21{22 u32 val;23 24 return readl_relaxed_poll_timeout(qm->io_base + QM_VF_STATE,25 val, !(val & 0x1), MB_POLL_PERIOD_US,26 MB_POLL_TIMEOUT_US);27}28 29/*30 * Each state Reg is checked 100 times,31 * with a delay of 100 microseconds after each check32 */33static u32 qm_check_reg_state(struct hisi_qm *qm, u32 regs)34{35 int check_times = 0;36 u32 state;37 38 state = readl(qm->io_base + regs);39 while (state && check_times < ERROR_CHECK_TIMEOUT) {40 udelay(CHECK_DELAY_TIME);41 state = readl(qm->io_base + regs);42 check_times++;43 }44 45 return state;46}47 48static int qm_read_regs(struct hisi_qm *qm, u32 reg_addr,49 u32 *data, u8 nums)50{51 int i;52 53 if (nums < 1 || nums > QM_REGS_MAX_LEN)54 return -EINVAL;55 56 for (i = 0; i < nums; i++) {57 data[i] = readl(qm->io_base + reg_addr);58 reg_addr += QM_REG_ADDR_OFFSET;59 }60 61 return 0;62}63 64static int qm_write_regs(struct hisi_qm *qm, u32 reg,65 u32 *data, u8 nums)66{67 int i;68 69 if (nums < 1 || nums > QM_REGS_MAX_LEN)70 return -EINVAL;71 72 for (i = 0; i < nums; i++)73 writel(data[i], qm->io_base + reg + i * QM_REG_ADDR_OFFSET);74 75 return 0;76}77 78static int qm_get_vft(struct hisi_qm *qm, u32 *base)79{80 u64 sqc_vft;81 u32 qp_num;82 int ret;83 84 ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1);85 if (ret)86 return ret;87 88 sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |89 ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) <<90 QM_XQC_ADDR_OFFSET);91 *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);92 qp_num = (QM_SQC_VFT_NUM_MASK_V2 &93 (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;94 95 return qp_num;96}97 98static int qm_get_sqc(struct hisi_qm *qm, u64 *addr)99{100 int ret;101 102 ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, 0, 0, 1);103 if (ret)104 return ret;105 106 *addr = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |107 ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) <<108 QM_XQC_ADDR_OFFSET);109 110 return 0;111}112 113static int qm_get_cqc(struct hisi_qm *qm, u64 *addr)114{115 int ret;116 117 ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, 0, 0, 1);118 if (ret)119 return ret;120 121 *addr = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |122 ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) <<123 QM_XQC_ADDR_OFFSET);124 125 return 0;126}127 128static int qm_get_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)129{130 struct device *dev = &qm->pdev->dev;131 int ret;132 133 ret = qm_read_regs(qm, QM_VF_AEQ_INT_MASK, &vf_data->aeq_int_mask, 1);134 if (ret) {135 dev_err(dev, "failed to read QM_VF_AEQ_INT_MASK\n");136 return ret;137 }138 139 ret = qm_read_regs(qm, QM_VF_EQ_INT_MASK, &vf_data->eq_int_mask, 1);140 if (ret) {141 dev_err(dev, "failed to read QM_VF_EQ_INT_MASK\n");142 return ret;143 }144 145 ret = qm_read_regs(qm, QM_IFC_INT_SOURCE_V,146 &vf_data->ifc_int_source, 1);147 if (ret) {148 dev_err(dev, "failed to read QM_IFC_INT_SOURCE_V\n");149 return ret;150 }151 152 ret = qm_read_regs(qm, QM_IFC_INT_MASK, &vf_data->ifc_int_mask, 1);153 if (ret) {154 dev_err(dev, "failed to read QM_IFC_INT_MASK\n");155 return ret;156 }157 158 ret = qm_read_regs(qm, QM_IFC_INT_SET_V, &vf_data->ifc_int_set, 1);159 if (ret) {160 dev_err(dev, "failed to read QM_IFC_INT_SET_V\n");161 return ret;162 }163 164 ret = qm_read_regs(qm, QM_PAGE_SIZE, &vf_data->page_size, 1);165 if (ret) {166 dev_err(dev, "failed to read QM_PAGE_SIZE\n");167 return ret;168 }169 170 /* QM_EQC_DW has 7 regs */171 ret = qm_read_regs(qm, QM_EQC_DW0, vf_data->qm_eqc_dw, 7);172 if (ret) {173 dev_err(dev, "failed to read QM_EQC_DW\n");174 return ret;175 }176 177 /* QM_AEQC_DW has 7 regs */178 ret = qm_read_regs(qm, QM_AEQC_DW0, vf_data->qm_aeqc_dw, 7);179 if (ret) {180 dev_err(dev, "failed to read QM_AEQC_DW\n");181 return ret;182 }183 184 return 0;185}186 187static int qm_set_regs(struct hisi_qm *qm, struct acc_vf_data *vf_data)188{189 struct device *dev = &qm->pdev->dev;190 int ret;191 192 /* Check VF state */193 if (unlikely(hisi_qm_wait_mb_ready(qm))) {194 dev_err(&qm->pdev->dev, "QM device is not ready to write\n");195 return -EBUSY;196 }197 198 ret = qm_write_regs(qm, QM_VF_AEQ_INT_MASK, &vf_data->aeq_int_mask, 1);199 if (ret) {200 dev_err(dev, "failed to write QM_VF_AEQ_INT_MASK\n");201 return ret;202 }203 204 ret = qm_write_regs(qm, QM_VF_EQ_INT_MASK, &vf_data->eq_int_mask, 1);205 if (ret) {206 dev_err(dev, "failed to write QM_VF_EQ_INT_MASK\n");207 return ret;208 }209 210 ret = qm_write_regs(qm, QM_IFC_INT_SOURCE_V,211 &vf_data->ifc_int_source, 1);212 if (ret) {213 dev_err(dev, "failed to write QM_IFC_INT_SOURCE_V\n");214 return ret;215 }216 217 ret = qm_write_regs(qm, QM_IFC_INT_MASK, &vf_data->ifc_int_mask, 1);218 if (ret) {219 dev_err(dev, "failed to write QM_IFC_INT_MASK\n");220 return ret;221 }222 223 ret = qm_write_regs(qm, QM_IFC_INT_SET_V, &vf_data->ifc_int_set, 1);224 if (ret) {225 dev_err(dev, "failed to write QM_IFC_INT_SET_V\n");226 return ret;227 }228 229 ret = qm_write_regs(qm, QM_QUE_ISO_CFG_V, &vf_data->que_iso_cfg, 1);230 if (ret) {231 dev_err(dev, "failed to write QM_QUE_ISO_CFG_V\n");232 return ret;233 }234 235 ret = qm_write_regs(qm, QM_PAGE_SIZE, &vf_data->page_size, 1);236 if (ret) {237 dev_err(dev, "failed to write QM_PAGE_SIZE\n");238 return ret;239 }240 241 /* QM_EQC_DW has 7 regs */242 ret = qm_write_regs(qm, QM_EQC_DW0, vf_data->qm_eqc_dw, 7);243 if (ret) {244 dev_err(dev, "failed to write QM_EQC_DW\n");245 return ret;246 }247 248 /* QM_AEQC_DW has 7 regs */249 ret = qm_write_regs(qm, QM_AEQC_DW0, vf_data->qm_aeqc_dw, 7);250 if (ret) {251 dev_err(dev, "failed to write QM_AEQC_DW\n");252 return ret;253 }254 255 return 0;256}257 258static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd,259 u16 index, u8 priority)260{261 u64 doorbell;262 u64 dbase;263 u16 randata = 0;264 265 if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ)266 dbase = QM_DOORBELL_SQ_CQ_BASE_V2;267 else268 dbase = QM_DOORBELL_EQ_AEQ_BASE_V2;269 270 doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) |271 ((u64)randata << QM_DB_RAND_SHIFT_V2) |272 ((u64)index << QM_DB_INDEX_SHIFT_V2) |273 ((u64)priority << QM_DB_PRIORITY_SHIFT_V2);274 275 writeq(doorbell, qm->io_base + dbase);276}277 278static int pf_qm_get_qp_num(struct hisi_qm *qm, int vf_id, u32 *rbase)279{280 unsigned int val;281 u64 sqc_vft;282 u32 qp_num;283 int ret;284 285 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,286 val & BIT(0), MB_POLL_PERIOD_US,287 MB_POLL_TIMEOUT_US);288 if (ret)289 return ret;290 291 writel(0x1, qm->io_base + QM_VFT_CFG_OP_WR);292 /* 0 mean SQC VFT */293 writel(0x0, qm->io_base + QM_VFT_CFG_TYPE);294 writel(vf_id, qm->io_base + QM_VFT_CFG);295 296 writel(0x0, qm->io_base + QM_VFT_CFG_RDY);297 writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE);298 299 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,300 val & BIT(0), MB_POLL_PERIOD_US,301 MB_POLL_TIMEOUT_US);302 if (ret)303 return ret;304 305 sqc_vft = readl(qm->io_base + QM_VFT_CFG_DATA_L) |306 ((u64)readl(qm->io_base + QM_VFT_CFG_DATA_H) <<307 QM_XQC_ADDR_OFFSET);308 *rbase = QM_SQC_VFT_BASE_MASK_V2 &309 (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);310 qp_num = (QM_SQC_VFT_NUM_MASK_V2 &311 (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;312 313 return qp_num;314}315 316static void qm_dev_cmd_init(struct hisi_qm *qm)317{318 /* Clear VF communication status registers. */319 writel(0x1, qm->io_base + QM_IFC_INT_SOURCE_V);320 321 /* Enable pf and vf communication. */322 writel(0x0, qm->io_base + QM_IFC_INT_MASK);323}324 325static int vf_qm_cache_wb(struct hisi_qm *qm)326{327 unsigned int val;328 329 writel(0x1, qm->io_base + QM_CACHE_WB_START);330 if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE,331 val, val & BIT(0), MB_POLL_PERIOD_US,332 MB_POLL_TIMEOUT_US)) {333 dev_err(&qm->pdev->dev, "vf QM writeback sqc cache fail\n");334 return -EINVAL;335 }336 337 return 0;338}339 340static void vf_qm_fun_reset(struct hisi_qm *qm)341{342 int i;343 344 for (i = 0; i < qm->qp_num; i++)345 qm_db(qm, i, QM_DOORBELL_CMD_SQ, 0, 1);346}347 348static int vf_qm_func_stop(struct hisi_qm *qm)349{350 return hisi_qm_mb(qm, QM_MB_CMD_PAUSE_QM, 0, 0, 0);351}352 353static int vf_qm_check_match(struct hisi_acc_vf_core_device *hisi_acc_vdev,354 struct hisi_acc_vf_migration_file *migf)355{356 struct acc_vf_data *vf_data = &migf->vf_data;357 struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;358 struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm;359 struct device *dev = &vf_qm->pdev->dev;360 u32 que_iso_state;361 int ret;362 363 if (migf->total_length < QM_MATCH_SIZE || hisi_acc_vdev->match_done)364 return 0;365 366 if (vf_data->acc_magic != ACC_DEV_MAGIC) {367 dev_err(dev, "failed to match ACC_DEV_MAGIC\n");368 return -EINVAL;369 }370 371 if (vf_data->dev_id != hisi_acc_vdev->vf_dev->device) {372 dev_err(dev, "failed to match VF devices\n");373 return -EINVAL;374 }375 376 /* VF qp num check */377 ret = qm_get_vft(vf_qm, &vf_qm->qp_base);378 if (ret <= 0) {379 dev_err(dev, "failed to get vft qp nums\n");380 return -EINVAL;381 }382 383 if (ret != vf_data->qp_num) {384 dev_err(dev, "failed to match VF qp num\n");385 return -EINVAL;386 }387 388 vf_qm->qp_num = ret;389 390 /* VF isolation state check */391 ret = qm_read_regs(pf_qm, QM_QUE_ISO_CFG_V, &que_iso_state, 1);392 if (ret) {393 dev_err(dev, "failed to read QM_QUE_ISO_CFG_V\n");394 return ret;395 }396 397 if (vf_data->que_iso_cfg != que_iso_state) {398 dev_err(dev, "failed to match isolation state\n");399 return -EINVAL;400 }401 402 ret = qm_write_regs(vf_qm, QM_VF_STATE, &vf_data->vf_qm_state, 1);403 if (ret) {404 dev_err(dev, "failed to write QM_VF_STATE\n");405 return ret;406 }407 408 hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state;409 hisi_acc_vdev->match_done = true;410 return 0;411}412 413static int vf_qm_get_match_data(struct hisi_acc_vf_core_device *hisi_acc_vdev,414 struct acc_vf_data *vf_data)415{416 struct hisi_qm *pf_qm = hisi_acc_vdev->pf_qm;417 struct device *dev = &pf_qm->pdev->dev;418 int vf_id = hisi_acc_vdev->vf_id;419 int ret;420 421 vf_data->acc_magic = ACC_DEV_MAGIC;422 /* Save device id */423 vf_data->dev_id = hisi_acc_vdev->vf_dev->device;424 425 /* VF qp num save from PF */426 ret = pf_qm_get_qp_num(pf_qm, vf_id, &vf_data->qp_base);427 if (ret <= 0) {428 dev_err(dev, "failed to get vft qp nums!\n");429 return -EINVAL;430 }431 432 vf_data->qp_num = ret;433 434 /* VF isolation state save from PF */435 ret = qm_read_regs(pf_qm, QM_QUE_ISO_CFG_V, &vf_data->que_iso_cfg, 1);436 if (ret) {437 dev_err(dev, "failed to read QM_QUE_ISO_CFG_V!\n");438 return ret;439 }440 441 return 0;442}443 444static int vf_qm_load_data(struct hisi_acc_vf_core_device *hisi_acc_vdev,445 struct hisi_acc_vf_migration_file *migf)446{447 struct hisi_qm *qm = &hisi_acc_vdev->vf_qm;448 struct device *dev = &qm->pdev->dev;449 struct acc_vf_data *vf_data = &migf->vf_data;450 int ret;451 452 /* Return if only match data was transferred */453 if (migf->total_length == QM_MATCH_SIZE)454 return 0;455 456 if (migf->total_length < sizeof(struct acc_vf_data))457 return -EINVAL;458 459 qm->eqe_dma = vf_data->eqe_dma;460 qm->aeqe_dma = vf_data->aeqe_dma;461 qm->sqc_dma = vf_data->sqc_dma;462 qm->cqc_dma = vf_data->cqc_dma;463 464 qm->qp_base = vf_data->qp_base;465 qm->qp_num = vf_data->qp_num;466 467 ret = qm_set_regs(qm, vf_data);468 if (ret) {469 dev_err(dev, "set VF regs failed\n");470 return ret;471 }472 473 ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0);474 if (ret) {475 dev_err(dev, "set sqc failed\n");476 return ret;477 }478 479 ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0);480 if (ret) {481 dev_err(dev, "set cqc failed\n");482 return ret;483 }484 485 qm_dev_cmd_init(qm);486 return 0;487}488 489static int vf_qm_state_save(struct hisi_acc_vf_core_device *hisi_acc_vdev,490 struct hisi_acc_vf_migration_file *migf)491{492 struct acc_vf_data *vf_data = &migf->vf_data;493 struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;494 struct device *dev = &vf_qm->pdev->dev;495 int ret;496 497 if (unlikely(qm_wait_dev_not_ready(vf_qm))) {498 /* Update state and return with match data */499 vf_data->vf_qm_state = QM_NOT_READY;500 hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state;501 migf->total_length = QM_MATCH_SIZE;502 return 0;503 }504 505 vf_data->vf_qm_state = QM_READY;506 hisi_acc_vdev->vf_qm_state = vf_data->vf_qm_state;507 508 ret = vf_qm_cache_wb(vf_qm);509 if (ret) {510 dev_err(dev, "failed to writeback QM Cache!\n");511 return ret;512 }513 514 ret = qm_get_regs(vf_qm, vf_data);515 if (ret)516 return -EINVAL;517 518 /* Every reg is 32 bit, the dma address is 64 bit. */519 vf_data->eqe_dma = vf_data->qm_eqc_dw[1];520 vf_data->eqe_dma <<= QM_XQC_ADDR_OFFSET;521 vf_data->eqe_dma |= vf_data->qm_eqc_dw[0];522 vf_data->aeqe_dma = vf_data->qm_aeqc_dw[1];523 vf_data->aeqe_dma <<= QM_XQC_ADDR_OFFSET;524 vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[0];525 526 /* Through SQC_BT/CQC_BT to get sqc and cqc address */527 ret = qm_get_sqc(vf_qm, &vf_data->sqc_dma);528 if (ret) {529 dev_err(dev, "failed to read SQC addr!\n");530 return -EINVAL;531 }532 533 ret = qm_get_cqc(vf_qm, &vf_data->cqc_dma);534 if (ret) {535 dev_err(dev, "failed to read CQC addr!\n");536 return -EINVAL;537 }538 539 migf->total_length = sizeof(struct acc_vf_data);540 return 0;541}542 543static struct hisi_acc_vf_core_device *hisi_acc_drvdata(struct pci_dev *pdev)544{545 struct vfio_pci_core_device *core_device = dev_get_drvdata(&pdev->dev);546 547 return container_of(core_device, struct hisi_acc_vf_core_device,548 core_device);549}550 551/* Check the PF's RAS state and Function INT state */552static int553hisi_acc_check_int_state(struct hisi_acc_vf_core_device *hisi_acc_vdev)554{555 struct hisi_qm *vfqm = &hisi_acc_vdev->vf_qm;556 struct hisi_qm *qm = hisi_acc_vdev->pf_qm;557 struct pci_dev *vf_pdev = hisi_acc_vdev->vf_dev;558 struct device *dev = &qm->pdev->dev;559 u32 state;560 561 /* Check RAS state */562 state = qm_check_reg_state(qm, QM_ABNORMAL_INT_STATUS);563 if (state) {564 dev_err(dev, "failed to check QM RAS state!\n");565 return -EBUSY;566 }567 568 /* Check Function Communication state between PF and VF */569 state = qm_check_reg_state(vfqm, QM_IFC_INT_STATUS);570 if (state) {571 dev_err(dev, "failed to check QM IFC INT state!\n");572 return -EBUSY;573 }574 state = qm_check_reg_state(vfqm, QM_IFC_INT_SET_V);575 if (state) {576 dev_err(dev, "failed to check QM IFC INT SET state!\n");577 return -EBUSY;578 }579 580 /* Check submodule task state */581 switch (vf_pdev->device) {582 case PCI_DEVICE_ID_HUAWEI_SEC_VF:583 state = qm_check_reg_state(qm, SEC_CORE_INT_STATUS);584 if (state) {585 dev_err(dev, "failed to check QM SEC Core INT state!\n");586 return -EBUSY;587 }588 return 0;589 case PCI_DEVICE_ID_HUAWEI_HPRE_VF:590 state = qm_check_reg_state(qm, HPRE_HAC_INT_STATUS);591 if (state) {592 dev_err(dev, "failed to check QM HPRE HAC INT state!\n");593 return -EBUSY;594 }595 return 0;596 case PCI_DEVICE_ID_HUAWEI_ZIP_VF:597 state = qm_check_reg_state(qm, HZIP_CORE_INT_STATUS);598 if (state) {599 dev_err(dev, "failed to check QM ZIP Core INT state!\n");600 return -EBUSY;601 }602 return 0;603 default:604 dev_err(dev, "failed to detect acc module type!\n");605 return -EINVAL;606 }607}608 609static void hisi_acc_vf_disable_fd(struct hisi_acc_vf_migration_file *migf)610{611 mutex_lock(&migf->lock);612 migf->disabled = true;613 migf->total_length = 0;614 migf->filp->f_pos = 0;615 mutex_unlock(&migf->lock);616}617 618static void hisi_acc_vf_disable_fds(struct hisi_acc_vf_core_device *hisi_acc_vdev)619{620 if (hisi_acc_vdev->resuming_migf) {621 hisi_acc_vf_disable_fd(hisi_acc_vdev->resuming_migf);622 fput(hisi_acc_vdev->resuming_migf->filp);623 hisi_acc_vdev->resuming_migf = NULL;624 }625 626 if (hisi_acc_vdev->saving_migf) {627 hisi_acc_vf_disable_fd(hisi_acc_vdev->saving_migf);628 fput(hisi_acc_vdev->saving_migf->filp);629 hisi_acc_vdev->saving_migf = NULL;630 }631}632 633static void hisi_acc_vf_reset(struct hisi_acc_vf_core_device *hisi_acc_vdev)634{635 hisi_acc_vdev->vf_qm_state = QM_NOT_READY;636 hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING;637 hisi_acc_vf_disable_fds(hisi_acc_vdev);638}639 640static void hisi_acc_vf_start_device(struct hisi_acc_vf_core_device *hisi_acc_vdev)641{642 struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;643 644 if (hisi_acc_vdev->vf_qm_state != QM_READY)645 return;646 647 /* Make sure the device is enabled */648 qm_dev_cmd_init(vf_qm);649 650 vf_qm_fun_reset(vf_qm);651}652 653static int hisi_acc_vf_load_state(struct hisi_acc_vf_core_device *hisi_acc_vdev)654{655 struct device *dev = &hisi_acc_vdev->vf_dev->dev;656 struct hisi_acc_vf_migration_file *migf = hisi_acc_vdev->resuming_migf;657 int ret;658 659 /* Recover data to VF */660 ret = vf_qm_load_data(hisi_acc_vdev, migf);661 if (ret) {662 dev_err(dev, "failed to recover the VF!\n");663 return ret;664 }665 666 return 0;667}668 669static int hisi_acc_vf_release_file(struct inode *inode, struct file *filp)670{671 struct hisi_acc_vf_migration_file *migf = filp->private_data;672 673 hisi_acc_vf_disable_fd(migf);674 mutex_destroy(&migf->lock);675 kfree(migf);676 return 0;677}678 679static ssize_t hisi_acc_vf_resume_write(struct file *filp, const char __user *buf,680 size_t len, loff_t *pos)681{682 struct hisi_acc_vf_migration_file *migf = filp->private_data;683 u8 *vf_data = (u8 *)&migf->vf_data;684 loff_t requested_length;685 ssize_t done = 0;686 int ret;687 688 if (pos)689 return -ESPIPE;690 pos = &filp->f_pos;691 692 if (*pos < 0 ||693 check_add_overflow((loff_t)len, *pos, &requested_length))694 return -EINVAL;695 696 if (requested_length > sizeof(struct acc_vf_data))697 return -ENOMEM;698 699 mutex_lock(&migf->lock);700 if (migf->disabled) {701 done = -ENODEV;702 goto out_unlock;703 }704 705 ret = copy_from_user(vf_data + *pos, buf, len);706 if (ret) {707 done = -EFAULT;708 goto out_unlock;709 }710 *pos += len;711 done = len;712 migf->total_length += len;713 714 ret = vf_qm_check_match(migf->hisi_acc_vdev, migf);715 if (ret)716 done = -EFAULT;717out_unlock:718 mutex_unlock(&migf->lock);719 return done;720}721 722static const struct file_operations hisi_acc_vf_resume_fops = {723 .owner = THIS_MODULE,724 .write = hisi_acc_vf_resume_write,725 .release = hisi_acc_vf_release_file,726};727 728static struct hisi_acc_vf_migration_file *729hisi_acc_vf_pci_resume(struct hisi_acc_vf_core_device *hisi_acc_vdev)730{731 struct hisi_acc_vf_migration_file *migf;732 733 migf = kzalloc(sizeof(*migf), GFP_KERNEL_ACCOUNT);734 if (!migf)735 return ERR_PTR(-ENOMEM);736 737 migf->filp = anon_inode_getfile("hisi_acc_vf_mig", &hisi_acc_vf_resume_fops, migf,738 O_WRONLY);739 if (IS_ERR(migf->filp)) {740 int err = PTR_ERR(migf->filp);741 742 kfree(migf);743 return ERR_PTR(err);744 }745 746 stream_open(migf->filp->f_inode, migf->filp);747 mutex_init(&migf->lock);748 migf->hisi_acc_vdev = hisi_acc_vdev;749 return migf;750}751 752static long hisi_acc_vf_precopy_ioctl(struct file *filp,753 unsigned int cmd, unsigned long arg)754{755 struct hisi_acc_vf_migration_file *migf = filp->private_data;756 struct hisi_acc_vf_core_device *hisi_acc_vdev = migf->hisi_acc_vdev;757 loff_t *pos = &filp->f_pos;758 struct vfio_precopy_info info;759 unsigned long minsz;760 int ret;761 762 if (cmd != VFIO_MIG_GET_PRECOPY_INFO)763 return -ENOTTY;764 765 minsz = offsetofend(struct vfio_precopy_info, dirty_bytes);766 767 if (copy_from_user(&info, (void __user *)arg, minsz))768 return -EFAULT;769 if (info.argsz < minsz)770 return -EINVAL;771 772 mutex_lock(&hisi_acc_vdev->state_mutex);773 if (hisi_acc_vdev->mig_state != VFIO_DEVICE_STATE_PRE_COPY) {774 mutex_unlock(&hisi_acc_vdev->state_mutex);775 return -EINVAL;776 }777 778 mutex_lock(&migf->lock);779 780 if (migf->disabled) {781 ret = -ENODEV;782 goto out;783 }784 785 if (*pos > migf->total_length) {786 ret = -EINVAL;787 goto out;788 }789 790 info.dirty_bytes = 0;791 info.initial_bytes = migf->total_length - *pos;792 mutex_unlock(&migf->lock);793 mutex_unlock(&hisi_acc_vdev->state_mutex);794 795 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;796out:797 mutex_unlock(&migf->lock);798 mutex_unlock(&hisi_acc_vdev->state_mutex);799 return ret;800}801 802static ssize_t hisi_acc_vf_save_read(struct file *filp, char __user *buf, size_t len,803 loff_t *pos)804{805 struct hisi_acc_vf_migration_file *migf = filp->private_data;806 ssize_t done = 0;807 int ret;808 809 if (pos)810 return -ESPIPE;811 pos = &filp->f_pos;812 813 mutex_lock(&migf->lock);814 if (*pos > migf->total_length) {815 done = -EINVAL;816 goto out_unlock;817 }818 819 if (migf->disabled) {820 done = -ENODEV;821 goto out_unlock;822 }823 824 len = min_t(size_t, migf->total_length - *pos, len);825 if (len) {826 u8 *vf_data = (u8 *)&migf->vf_data;827 828 ret = copy_to_user(buf, vf_data + *pos, len);829 if (ret) {830 done = -EFAULT;831 goto out_unlock;832 }833 *pos += len;834 done = len;835 }836out_unlock:837 mutex_unlock(&migf->lock);838 return done;839}840 841static const struct file_operations hisi_acc_vf_save_fops = {842 .owner = THIS_MODULE,843 .read = hisi_acc_vf_save_read,844 .unlocked_ioctl = hisi_acc_vf_precopy_ioctl,845 .compat_ioctl = compat_ptr_ioctl,846 .release = hisi_acc_vf_release_file,847};848 849static struct hisi_acc_vf_migration_file *850hisi_acc_open_saving_migf(struct hisi_acc_vf_core_device *hisi_acc_vdev)851{852 struct hisi_acc_vf_migration_file *migf;853 int ret;854 855 migf = kzalloc(sizeof(*migf), GFP_KERNEL_ACCOUNT);856 if (!migf)857 return ERR_PTR(-ENOMEM);858 859 migf->filp = anon_inode_getfile("hisi_acc_vf_mig", &hisi_acc_vf_save_fops, migf,860 O_RDONLY);861 if (IS_ERR(migf->filp)) {862 int err = PTR_ERR(migf->filp);863 864 kfree(migf);865 return ERR_PTR(err);866 }867 868 stream_open(migf->filp->f_inode, migf->filp);869 mutex_init(&migf->lock);870 migf->hisi_acc_vdev = hisi_acc_vdev;871 872 ret = vf_qm_get_match_data(hisi_acc_vdev, &migf->vf_data);873 if (ret) {874 fput(migf->filp);875 return ERR_PTR(ret);876 }877 878 return migf;879}880 881static struct hisi_acc_vf_migration_file *882hisi_acc_vf_pre_copy(struct hisi_acc_vf_core_device *hisi_acc_vdev)883{884 struct hisi_acc_vf_migration_file *migf;885 886 migf = hisi_acc_open_saving_migf(hisi_acc_vdev);887 if (IS_ERR(migf))888 return migf;889 890 migf->total_length = QM_MATCH_SIZE;891 return migf;892}893 894static struct hisi_acc_vf_migration_file *895hisi_acc_vf_stop_copy(struct hisi_acc_vf_core_device *hisi_acc_vdev, bool open)896{897 int ret;898 struct hisi_acc_vf_migration_file *migf = NULL;899 900 if (open) {901 /*902 * Userspace didn't use PRECOPY support. Hence saving_migf903 * is not opened yet.904 */905 migf = hisi_acc_open_saving_migf(hisi_acc_vdev);906 if (IS_ERR(migf))907 return migf;908 } else {909 migf = hisi_acc_vdev->saving_migf;910 }911 912 ret = vf_qm_state_save(hisi_acc_vdev, migf);913 if (ret)914 return ERR_PTR(ret);915 916 return open ? migf : NULL;917}918 919static int hisi_acc_vf_stop_device(struct hisi_acc_vf_core_device *hisi_acc_vdev)920{921 struct device *dev = &hisi_acc_vdev->vf_dev->dev;922 struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;923 int ret;924 925 ret = vf_qm_func_stop(vf_qm);926 if (ret) {927 dev_err(dev, "failed to stop QM VF function!\n");928 return ret;929 }930 931 ret = hisi_acc_check_int_state(hisi_acc_vdev);932 if (ret) {933 dev_err(dev, "failed to check QM INT state!\n");934 return ret;935 }936 return 0;937}938 939static struct file *940hisi_acc_vf_set_device_state(struct hisi_acc_vf_core_device *hisi_acc_vdev,941 u32 new)942{943 u32 cur = hisi_acc_vdev->mig_state;944 int ret;945 946 if (cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_PRE_COPY) {947 struct hisi_acc_vf_migration_file *migf;948 949 migf = hisi_acc_vf_pre_copy(hisi_acc_vdev);950 if (IS_ERR(migf))951 return ERR_CAST(migf);952 get_file(migf->filp);953 hisi_acc_vdev->saving_migf = migf;954 return migf->filp;955 }956 957 if (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_STOP_COPY) {958 struct hisi_acc_vf_migration_file *migf;959 960 ret = hisi_acc_vf_stop_device(hisi_acc_vdev);961 if (ret)962 return ERR_PTR(ret);963 964 migf = hisi_acc_vf_stop_copy(hisi_acc_vdev, false);965 if (IS_ERR(migf))966 return ERR_CAST(migf);967 968 return NULL;969 }970 971 if (cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_STOP) {972 ret = hisi_acc_vf_stop_device(hisi_acc_vdev);973 if (ret)974 return ERR_PTR(ret);975 return NULL;976 }977 978 if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_STOP_COPY) {979 struct hisi_acc_vf_migration_file *migf;980 981 migf = hisi_acc_vf_stop_copy(hisi_acc_vdev, true);982 if (IS_ERR(migf))983 return ERR_CAST(migf);984 get_file(migf->filp);985 hisi_acc_vdev->saving_migf = migf;986 return migf->filp;987 }988 989 if ((cur == VFIO_DEVICE_STATE_STOP_COPY && new == VFIO_DEVICE_STATE_STOP)) {990 hisi_acc_vf_disable_fds(hisi_acc_vdev);991 return NULL;992 }993 994 if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RESUMING) {995 struct hisi_acc_vf_migration_file *migf;996 997 migf = hisi_acc_vf_pci_resume(hisi_acc_vdev);998 if (IS_ERR(migf))999 return ERR_CAST(migf);1000 get_file(migf->filp);1001 hisi_acc_vdev->resuming_migf = migf;1002 return migf->filp;1003 }1004 1005 if (cur == VFIO_DEVICE_STATE_RESUMING && new == VFIO_DEVICE_STATE_STOP) {1006 ret = hisi_acc_vf_load_state(hisi_acc_vdev);1007 if (ret)1008 return ERR_PTR(ret);1009 hisi_acc_vf_disable_fds(hisi_acc_vdev);1010 return NULL;1011 }1012 1013 if (cur == VFIO_DEVICE_STATE_PRE_COPY && new == VFIO_DEVICE_STATE_RUNNING) {1014 hisi_acc_vf_disable_fds(hisi_acc_vdev);1015 return NULL;1016 }1017 1018 if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RUNNING) {1019 hisi_acc_vf_start_device(hisi_acc_vdev);1020 return NULL;1021 }1022 1023 /*1024 * vfio_mig_get_next_state() does not use arcs other than the above1025 */1026 WARN_ON(true);1027 return ERR_PTR(-EINVAL);1028}1029 1030static struct file *1031hisi_acc_vfio_pci_set_device_state(struct vfio_device *vdev,1032 enum vfio_device_mig_state new_state)1033{1034 struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(vdev,1035 struct hisi_acc_vf_core_device, core_device.vdev);1036 enum vfio_device_mig_state next_state;1037 struct file *res = NULL;1038 int ret;1039 1040 mutex_lock(&hisi_acc_vdev->state_mutex);1041 while (new_state != hisi_acc_vdev->mig_state) {1042 ret = vfio_mig_get_next_state(vdev,1043 hisi_acc_vdev->mig_state,1044 new_state, &next_state);1045 if (ret) {1046 res = ERR_PTR(-EINVAL);1047 break;1048 }1049 1050 res = hisi_acc_vf_set_device_state(hisi_acc_vdev, next_state);1051 if (IS_ERR(res))1052 break;1053 hisi_acc_vdev->mig_state = next_state;1054 if (WARN_ON(res && new_state != hisi_acc_vdev->mig_state)) {1055 fput(res);1056 res = ERR_PTR(-EINVAL);1057 break;1058 }1059 }1060 mutex_unlock(&hisi_acc_vdev->state_mutex);1061 return res;1062}1063 1064static int1065hisi_acc_vfio_pci_get_data_size(struct vfio_device *vdev,1066 unsigned long *stop_copy_length)1067{1068 *stop_copy_length = sizeof(struct acc_vf_data);1069 return 0;1070}1071 1072static int1073hisi_acc_vfio_pci_get_device_state(struct vfio_device *vdev,1074 enum vfio_device_mig_state *curr_state)1075{1076 struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(vdev,1077 struct hisi_acc_vf_core_device, core_device.vdev);1078 1079 mutex_lock(&hisi_acc_vdev->state_mutex);1080 *curr_state = hisi_acc_vdev->mig_state;1081 mutex_unlock(&hisi_acc_vdev->state_mutex);1082 return 0;1083}1084 1085static void hisi_acc_vf_pci_aer_reset_done(struct pci_dev *pdev)1086{1087 struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev);1088 1089 if (hisi_acc_vdev->core_device.vdev.migration_flags !=1090 VFIO_MIGRATION_STOP_COPY)1091 return;1092 1093 mutex_lock(&hisi_acc_vdev->state_mutex);1094 hisi_acc_vf_reset(hisi_acc_vdev);1095 mutex_unlock(&hisi_acc_vdev->state_mutex);1096}1097 1098static int hisi_acc_vf_qm_init(struct hisi_acc_vf_core_device *hisi_acc_vdev)1099{1100 struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device;1101 struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;1102 struct pci_dev *vf_dev = vdev->pdev;1103 1104 /*1105 * ACC VF dev BAR2 region consists of both functional register space1106 * and migration control register space. For migration to work, we1107 * need access to both. Hence, we map the entire BAR2 region here.1108 * But unnecessarily exposing the migration BAR region to the Guest1109 * has the potential to prevent/corrupt the Guest migration. Hence,1110 * we restrict access to the migration control space from1111 * Guest(Please see mmap/ioctl/read/write override functions).1112 *1113 * Please note that it is OK to expose the entire VF BAR if migration1114 * is not supported or required as this cannot affect the ACC PF1115 * configurations.1116 *1117 * Also the HiSilicon ACC VF devices supported by this driver on1118 * HiSilicon hardware platforms are integrated end point devices1119 * and the platform lacks the capability to perform any PCIe P2P1120 * between these devices.1121 */1122 1123 vf_qm->io_base =1124 ioremap(pci_resource_start(vf_dev, VFIO_PCI_BAR2_REGION_INDEX),1125 pci_resource_len(vf_dev, VFIO_PCI_BAR2_REGION_INDEX));1126 if (!vf_qm->io_base)1127 return -EIO;1128 1129 vf_qm->fun_type = QM_HW_VF;1130 vf_qm->pdev = vf_dev;1131 mutex_init(&vf_qm->mailbox_lock);1132 1133 return 0;1134}1135 1136static struct hisi_qm *hisi_acc_get_pf_qm(struct pci_dev *pdev)1137{1138 struct hisi_qm *pf_qm;1139 struct pci_driver *pf_driver;1140 1141 if (!pdev->is_virtfn)1142 return NULL;1143 1144 switch (pdev->device) {1145 case PCI_DEVICE_ID_HUAWEI_SEC_VF:1146 pf_driver = hisi_sec_get_pf_driver();1147 break;1148 case PCI_DEVICE_ID_HUAWEI_HPRE_VF:1149 pf_driver = hisi_hpre_get_pf_driver();1150 break;1151 case PCI_DEVICE_ID_HUAWEI_ZIP_VF:1152 pf_driver = hisi_zip_get_pf_driver();1153 break;1154 default:1155 return NULL;1156 }1157 1158 if (!pf_driver)1159 return NULL;1160 1161 pf_qm = pci_iov_get_pf_drvdata(pdev, pf_driver);1162 1163 return !IS_ERR(pf_qm) ? pf_qm : NULL;1164}1165 1166static int hisi_acc_pci_rw_access_check(struct vfio_device *core_vdev,1167 size_t count, loff_t *ppos,1168 size_t *new_count)1169{1170 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);1171 struct vfio_pci_core_device *vdev =1172 container_of(core_vdev, struct vfio_pci_core_device, vdev);1173 1174 if (index == VFIO_PCI_BAR2_REGION_INDEX) {1175 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;1176 resource_size_t end = pci_resource_len(vdev->pdev, index) / 2;1177 1178 /* Check if access is for migration control region */1179 if (pos >= end)1180 return -EINVAL;1181 1182 *new_count = min(count, (size_t)(end - pos));1183 }1184 1185 return 0;1186}1187 1188static int hisi_acc_vfio_pci_mmap(struct vfio_device *core_vdev,1189 struct vm_area_struct *vma)1190{1191 struct vfio_pci_core_device *vdev =1192 container_of(core_vdev, struct vfio_pci_core_device, vdev);1193 unsigned int index;1194 1195 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);1196 if (index == VFIO_PCI_BAR2_REGION_INDEX) {1197 u64 req_len, pgoff, req_start;1198 resource_size_t end = pci_resource_len(vdev->pdev, index) / 2;1199 1200 req_len = vma->vm_end - vma->vm_start;1201 pgoff = vma->vm_pgoff &1202 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);1203 req_start = pgoff << PAGE_SHIFT;1204 1205 if (req_start + req_len > end)1206 return -EINVAL;1207 }1208 1209 return vfio_pci_core_mmap(core_vdev, vma);1210}1211 1212static ssize_t hisi_acc_vfio_pci_write(struct vfio_device *core_vdev,1213 const char __user *buf, size_t count,1214 loff_t *ppos)1215{1216 size_t new_count = count;1217 int ret;1218 1219 ret = hisi_acc_pci_rw_access_check(core_vdev, count, ppos, &new_count);1220 if (ret)1221 return ret;1222 1223 return vfio_pci_core_write(core_vdev, buf, new_count, ppos);1224}1225 1226static ssize_t hisi_acc_vfio_pci_read(struct vfio_device *core_vdev,1227 char __user *buf, size_t count,1228 loff_t *ppos)1229{1230 size_t new_count = count;1231 int ret;1232 1233 ret = hisi_acc_pci_rw_access_check(core_vdev, count, ppos, &new_count);1234 if (ret)1235 return ret;1236 1237 return vfio_pci_core_read(core_vdev, buf, new_count, ppos);1238}1239 1240static long hisi_acc_vfio_pci_ioctl(struct vfio_device *core_vdev, unsigned int cmd,1241 unsigned long arg)1242{1243 if (cmd == VFIO_DEVICE_GET_REGION_INFO) {1244 struct vfio_pci_core_device *vdev =1245 container_of(core_vdev, struct vfio_pci_core_device, vdev);1246 struct pci_dev *pdev = vdev->pdev;1247 struct vfio_region_info info;1248 unsigned long minsz;1249 1250 minsz = offsetofend(struct vfio_region_info, offset);1251 1252 if (copy_from_user(&info, (void __user *)arg, minsz))1253 return -EFAULT;1254 1255 if (info.argsz < minsz)1256 return -EINVAL;1257 1258 if (info.index == VFIO_PCI_BAR2_REGION_INDEX) {1259 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);1260 1261 /*1262 * ACC VF dev BAR2 region consists of both functional1263 * register space and migration control register space.1264 * Report only the functional region to Guest.1265 */1266 info.size = pci_resource_len(pdev, info.index) / 2;1267 1268 info.flags = VFIO_REGION_INFO_FLAG_READ |1269 VFIO_REGION_INFO_FLAG_WRITE |1270 VFIO_REGION_INFO_FLAG_MMAP;1271 1272 return copy_to_user((void __user *)arg, &info, minsz) ?1273 -EFAULT : 0;1274 }1275 }1276 return vfio_pci_core_ioctl(core_vdev, cmd, arg);1277}1278 1279static int hisi_acc_vfio_pci_open_device(struct vfio_device *core_vdev)1280{1281 struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev,1282 struct hisi_acc_vf_core_device, core_device.vdev);1283 struct vfio_pci_core_device *vdev = &hisi_acc_vdev->core_device;1284 int ret;1285 1286 ret = vfio_pci_core_enable(vdev);1287 if (ret)1288 return ret;1289 1290 if (core_vdev->mig_ops) {1291 ret = hisi_acc_vf_qm_init(hisi_acc_vdev);1292 if (ret) {1293 vfio_pci_core_disable(vdev);1294 return ret;1295 }1296 hisi_acc_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING;1297 }1298 1299 vfio_pci_core_finish_enable(vdev);1300 return 0;1301}1302 1303static void hisi_acc_vfio_pci_close_device(struct vfio_device *core_vdev)1304{1305 struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev,1306 struct hisi_acc_vf_core_device, core_device.vdev);1307 struct hisi_qm *vf_qm = &hisi_acc_vdev->vf_qm;1308 1309 iounmap(vf_qm->io_base);1310 vfio_pci_core_close_device(core_vdev);1311}1312 1313static const struct vfio_migration_ops hisi_acc_vfio_pci_migrn_state_ops = {1314 .migration_set_state = hisi_acc_vfio_pci_set_device_state,1315 .migration_get_state = hisi_acc_vfio_pci_get_device_state,1316 .migration_get_data_size = hisi_acc_vfio_pci_get_data_size,1317};1318 1319static int hisi_acc_vfio_pci_migrn_init_dev(struct vfio_device *core_vdev)1320{1321 struct hisi_acc_vf_core_device *hisi_acc_vdev = container_of(core_vdev,1322 struct hisi_acc_vf_core_device, core_device.vdev);1323 struct pci_dev *pdev = to_pci_dev(core_vdev->dev);1324 struct hisi_qm *pf_qm = hisi_acc_get_pf_qm(pdev);1325 1326 hisi_acc_vdev->vf_id = pci_iov_vf_id(pdev) + 1;1327 hisi_acc_vdev->pf_qm = pf_qm;1328 hisi_acc_vdev->vf_dev = pdev;1329 mutex_init(&hisi_acc_vdev->state_mutex);1330 1331 core_vdev->migration_flags = VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY;1332 core_vdev->mig_ops = &hisi_acc_vfio_pci_migrn_state_ops;1333 1334 return vfio_pci_core_init_dev(core_vdev);1335}1336 1337static const struct vfio_device_ops hisi_acc_vfio_pci_migrn_ops = {1338 .name = "hisi-acc-vfio-pci-migration",1339 .init = hisi_acc_vfio_pci_migrn_init_dev,1340 .release = vfio_pci_core_release_dev,1341 .open_device = hisi_acc_vfio_pci_open_device,1342 .close_device = hisi_acc_vfio_pci_close_device,1343 .ioctl = hisi_acc_vfio_pci_ioctl,1344 .device_feature = vfio_pci_core_ioctl_feature,1345 .read = hisi_acc_vfio_pci_read,1346 .write = hisi_acc_vfio_pci_write,1347 .mmap = hisi_acc_vfio_pci_mmap,1348 .request = vfio_pci_core_request,1349 .match = vfio_pci_core_match,1350 .bind_iommufd = vfio_iommufd_physical_bind,1351 .unbind_iommufd = vfio_iommufd_physical_unbind,1352 .attach_ioas = vfio_iommufd_physical_attach_ioas,1353 .detach_ioas = vfio_iommufd_physical_detach_ioas,1354};1355 1356static const struct vfio_device_ops hisi_acc_vfio_pci_ops = {1357 .name = "hisi-acc-vfio-pci",1358 .init = vfio_pci_core_init_dev,1359 .release = vfio_pci_core_release_dev,1360 .open_device = hisi_acc_vfio_pci_open_device,1361 .close_device = vfio_pci_core_close_device,1362 .ioctl = vfio_pci_core_ioctl,1363 .device_feature = vfio_pci_core_ioctl_feature,1364 .read = vfio_pci_core_read,1365 .write = vfio_pci_core_write,1366 .mmap = vfio_pci_core_mmap,1367 .request = vfio_pci_core_request,1368 .match = vfio_pci_core_match,1369 .bind_iommufd = vfio_iommufd_physical_bind,1370 .unbind_iommufd = vfio_iommufd_physical_unbind,1371 .attach_ioas = vfio_iommufd_physical_attach_ioas,1372 .detach_ioas = vfio_iommufd_physical_detach_ioas,1373};1374 1375static int hisi_acc_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)1376{1377 struct hisi_acc_vf_core_device *hisi_acc_vdev;1378 const struct vfio_device_ops *ops = &hisi_acc_vfio_pci_ops;1379 struct hisi_qm *pf_qm;1380 int vf_id;1381 int ret;1382 1383 pf_qm = hisi_acc_get_pf_qm(pdev);1384 if (pf_qm && pf_qm->ver >= QM_HW_V3) {1385 vf_id = pci_iov_vf_id(pdev);1386 if (vf_id >= 0)1387 ops = &hisi_acc_vfio_pci_migrn_ops;1388 else1389 pci_warn(pdev, "migration support failed, continue with generic interface\n");1390 }1391 1392 hisi_acc_vdev = vfio_alloc_device(hisi_acc_vf_core_device,1393 core_device.vdev, &pdev->dev, ops);1394 if (IS_ERR(hisi_acc_vdev))1395 return PTR_ERR(hisi_acc_vdev);1396 1397 dev_set_drvdata(&pdev->dev, &hisi_acc_vdev->core_device);1398 ret = vfio_pci_core_register_device(&hisi_acc_vdev->core_device);1399 if (ret)1400 goto out_put_vdev;1401 return 0;1402 1403out_put_vdev:1404 vfio_put_device(&hisi_acc_vdev->core_device.vdev);1405 return ret;1406}1407 1408static void hisi_acc_vfio_pci_remove(struct pci_dev *pdev)1409{1410 struct hisi_acc_vf_core_device *hisi_acc_vdev = hisi_acc_drvdata(pdev);1411 1412 vfio_pci_core_unregister_device(&hisi_acc_vdev->core_device);1413 vfio_put_device(&hisi_acc_vdev->core_device.vdev);1414}1415 1416static const struct pci_device_id hisi_acc_vfio_pci_table[] = {1417 { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_SEC_VF) },1418 { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_HPRE_VF) },1419 { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_ZIP_VF) },1420 { }1421};1422 1423MODULE_DEVICE_TABLE(pci, hisi_acc_vfio_pci_table);1424 1425static const struct pci_error_handlers hisi_acc_vf_err_handlers = {1426 .reset_done = hisi_acc_vf_pci_aer_reset_done,1427 .error_detected = vfio_pci_core_aer_err_detected,1428};1429 1430static struct pci_driver hisi_acc_vfio_pci_driver = {1431 .name = KBUILD_MODNAME,1432 .id_table = hisi_acc_vfio_pci_table,1433 .probe = hisi_acc_vfio_pci_probe,1434 .remove = hisi_acc_vfio_pci_remove,1435 .err_handler = &hisi_acc_vf_err_handlers,1436 .driver_managed_dma = true,1437};1438 1439module_pci_driver(hisi_acc_vfio_pci_driver);1440 1441MODULE_LICENSE("GPL v2");1442MODULE_AUTHOR("Liu Longfang <liulongfang@huawei.com>");1443MODULE_AUTHOR("Shameer Kolothum <shameerali.kolothum.thodi@huawei.com>");1444MODULE_DESCRIPTION("HiSilicon VFIO PCI - VFIO PCI driver with live migration support for HiSilicon ACC device family");1445