511 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright(c) 2023 Advanced Micro Devices, Inc. */3 4#include <linux/io.h>5#include <linux/types.h>6#include <linux/delay.h>7 8#include <linux/pds/pds_common.h>9#include <linux/pds/pds_core_if.h>10#include <linux/pds/pds_adminq.h>11 12#include "vfio_dev.h"13#include "cmds.h"14 15#define SUSPEND_TIMEOUT_S 516#define SUSPEND_CHECK_INTERVAL_MS 117 18static int pds_vfio_client_adminq_cmd(struct pds_vfio_pci_device *pds_vfio,19 union pds_core_adminq_cmd *req,20 union pds_core_adminq_comp *resp,21 bool fast_poll)22{23 struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);24 union pds_core_adminq_cmd cmd = {};25 struct pdsc *pdsc;26 int err;27 28 /* Wrap the client request */29 cmd.client_request.opcode = PDS_AQ_CMD_CLIENT_CMD;30 cmd.client_request.client_id = cpu_to_le16(pds_vfio->client_id);31 memcpy(cmd.client_request.client_cmd, req,32 sizeof(cmd.client_request.client_cmd));33 34 pdsc = pdsc_get_pf_struct(pdev);35 if (IS_ERR(pdsc))36 return PTR_ERR(pdsc);37 38 err = pdsc_adminq_post(pdsc, &cmd, resp, fast_poll);39 if (err && err != -EAGAIN)40 dev_err(pds_vfio_to_dev(pds_vfio),41 "client admin cmd failed: %pe\n", ERR_PTR(err));42 43 return err;44}45 46int pds_vfio_register_client_cmd(struct pds_vfio_pci_device *pds_vfio)47{48 struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);49 char devname[PDS_DEVNAME_LEN];50 struct pdsc *pdsc;51 int ci;52 53 snprintf(devname, sizeof(devname), "%s.%d-%u", PDS_VFIO_LM_DEV_NAME,54 pci_domain_nr(pdev->bus),55 PCI_DEVID(pdev->bus->number, pdev->devfn));56 57 pdsc = pdsc_get_pf_struct(pdev);58 if (IS_ERR(pdsc))59 return PTR_ERR(pdsc);60 61 ci = pds_client_register(pdsc, devname);62 if (ci < 0)63 return ci;64 65 pds_vfio->client_id = ci;66 67 return 0;68}69 70void pds_vfio_unregister_client_cmd(struct pds_vfio_pci_device *pds_vfio)71{72 struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);73 struct pdsc *pdsc;74 int err;75 76 pdsc = pdsc_get_pf_struct(pdev);77 if (IS_ERR(pdsc))78 return;79 80 err = pds_client_unregister(pdsc, pds_vfio->client_id);81 if (err)82 dev_err(&pdev->dev, "unregister from DSC failed: %pe\n",83 ERR_PTR(err));84 85 pds_vfio->client_id = 0;86}87 88static int89pds_vfio_suspend_wait_device_cmd(struct pds_vfio_pci_device *pds_vfio, u8 type)90{91 union pds_core_adminq_cmd cmd = {92 .lm_suspend_status = {93 .opcode = PDS_LM_CMD_SUSPEND_STATUS,94 .vf_id = cpu_to_le16(pds_vfio->vf_id),95 .type = type,96 },97 };98 struct device *dev = pds_vfio_to_dev(pds_vfio);99 union pds_core_adminq_comp comp = {};100 unsigned long time_limit;101 unsigned long time_start;102 unsigned long time_done;103 int err;104 105 time_start = jiffies;106 time_limit = time_start + HZ * SUSPEND_TIMEOUT_S;107 do {108 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, true);109 if (err != -EAGAIN)110 break;111 112 msleep(SUSPEND_CHECK_INTERVAL_MS);113 } while (time_before(jiffies, time_limit));114 115 time_done = jiffies;116 dev_dbg(dev, "%s: vf%u: Suspend comp received in %d msecs\n", __func__,117 pds_vfio->vf_id, jiffies_to_msecs(time_done - time_start));118 119 /* Check the results */120 if (time_after_eq(time_done, time_limit)) {121 dev_err(dev, "%s: vf%u: Suspend comp timeout\n", __func__,122 pds_vfio->vf_id);123 err = -ETIMEDOUT;124 }125 126 return err;127}128 129int pds_vfio_suspend_device_cmd(struct pds_vfio_pci_device *pds_vfio, u8 type)130{131 union pds_core_adminq_cmd cmd = {132 .lm_suspend = {133 .opcode = PDS_LM_CMD_SUSPEND,134 .vf_id = cpu_to_le16(pds_vfio->vf_id),135 .type = type,136 },137 };138 struct device *dev = pds_vfio_to_dev(pds_vfio);139 union pds_core_adminq_comp comp = {};140 int err;141 142 dev_dbg(dev, "vf%u: Suspend device\n", pds_vfio->vf_id);143 144 /*145 * The initial suspend request to the firmware starts the device suspend146 * operation and the firmware returns success if it's started147 * successfully.148 */149 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, true);150 if (err) {151 dev_err(dev, "vf%u: Suspend failed: %pe\n", pds_vfio->vf_id,152 ERR_PTR(err));153 return err;154 }155 156 /*157 * The subsequent suspend status request(s) check if the firmware has158 * completed the device suspend process.159 */160 return pds_vfio_suspend_wait_device_cmd(pds_vfio, type);161}162 163int pds_vfio_resume_device_cmd(struct pds_vfio_pci_device *pds_vfio, u8 type)164{165 union pds_core_adminq_cmd cmd = {166 .lm_resume = {167 .opcode = PDS_LM_CMD_RESUME,168 .vf_id = cpu_to_le16(pds_vfio->vf_id),169 .type = type,170 },171 };172 struct device *dev = pds_vfio_to_dev(pds_vfio);173 union pds_core_adminq_comp comp = {};174 175 dev_dbg(dev, "vf%u: Resume device\n", pds_vfio->vf_id);176 177 return pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, true);178}179 180int pds_vfio_get_lm_state_size_cmd(struct pds_vfio_pci_device *pds_vfio, u64 *size)181{182 union pds_core_adminq_cmd cmd = {183 .lm_state_size = {184 .opcode = PDS_LM_CMD_STATE_SIZE,185 .vf_id = cpu_to_le16(pds_vfio->vf_id),186 },187 };188 struct device *dev = pds_vfio_to_dev(pds_vfio);189 union pds_core_adminq_comp comp = {};190 int err;191 192 dev_dbg(dev, "vf%u: Get migration status\n", pds_vfio->vf_id);193 194 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);195 if (err)196 return err;197 198 *size = le64_to_cpu(comp.lm_state_size.size);199 return 0;200}201 202static int pds_vfio_dma_map_lm_file(struct device *dev,203 enum dma_data_direction dir,204 struct pds_vfio_lm_file *lm_file)205{206 struct pds_lm_sg_elem *sgl, *sge;207 struct scatterlist *sg;208 dma_addr_t sgl_addr;209 size_t sgl_size;210 int err;211 int i;212 213 if (!lm_file)214 return -EINVAL;215 216 /* dma map file pages */217 err = dma_map_sgtable(dev, &lm_file->sg_table, dir, 0);218 if (err)219 return err;220 221 lm_file->num_sge = lm_file->sg_table.nents;222 223 /* alloc sgl */224 sgl_size = lm_file->num_sge * sizeof(struct pds_lm_sg_elem);225 sgl = kzalloc(sgl_size, GFP_KERNEL);226 if (!sgl) {227 err = -ENOMEM;228 goto out_unmap_sgtable;229 }230 231 /* fill sgl */232 sge = sgl;233 for_each_sgtable_dma_sg(&lm_file->sg_table, sg, i) {234 sge->addr = cpu_to_le64(sg_dma_address(sg));235 sge->len = cpu_to_le32(sg_dma_len(sg));236 dev_dbg(dev, "addr = %llx, len = %u\n", sge->addr, sge->len);237 sge++;238 }239 240 sgl_addr = dma_map_single(dev, sgl, sgl_size, DMA_TO_DEVICE);241 if (dma_mapping_error(dev, sgl_addr)) {242 err = -EIO;243 goto out_free_sgl;244 }245 246 lm_file->sgl = sgl;247 lm_file->sgl_addr = sgl_addr;248 249 return 0;250 251out_free_sgl:252 kfree(sgl);253out_unmap_sgtable:254 lm_file->num_sge = 0;255 dma_unmap_sgtable(dev, &lm_file->sg_table, dir, 0);256 return err;257}258 259static void pds_vfio_dma_unmap_lm_file(struct device *dev,260 enum dma_data_direction dir,261 struct pds_vfio_lm_file *lm_file)262{263 if (!lm_file)264 return;265 266 /* free sgl */267 if (lm_file->sgl) {268 dma_unmap_single(dev, lm_file->sgl_addr,269 lm_file->num_sge * sizeof(*lm_file->sgl),270 DMA_TO_DEVICE);271 kfree(lm_file->sgl);272 lm_file->sgl = NULL;273 lm_file->sgl_addr = DMA_MAPPING_ERROR;274 lm_file->num_sge = 0;275 }276 277 /* dma unmap file pages */278 dma_unmap_sgtable(dev, &lm_file->sg_table, dir, 0);279}280 281int pds_vfio_get_lm_state_cmd(struct pds_vfio_pci_device *pds_vfio)282{283 union pds_core_adminq_cmd cmd = {284 .lm_save = {285 .opcode = PDS_LM_CMD_SAVE,286 .vf_id = cpu_to_le16(pds_vfio->vf_id),287 },288 };289 struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);290 struct device *pdsc_dev = &pci_physfn(pdev)->dev;291 union pds_core_adminq_comp comp = {};292 struct pds_vfio_lm_file *lm_file;293 int err;294 295 dev_dbg(&pdev->dev, "vf%u: Get migration state\n", pds_vfio->vf_id);296 297 lm_file = pds_vfio->save_file;298 299 err = pds_vfio_dma_map_lm_file(pdsc_dev, DMA_FROM_DEVICE, lm_file);300 if (err) {301 dev_err(&pdev->dev, "failed to map save migration file: %pe\n",302 ERR_PTR(err));303 return err;304 }305 306 cmd.lm_save.sgl_addr = cpu_to_le64(lm_file->sgl_addr);307 cmd.lm_save.num_sge = cpu_to_le32(lm_file->num_sge);308 309 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);310 if (err)311 dev_err(&pdev->dev, "failed to get migration state: %pe\n",312 ERR_PTR(err));313 314 pds_vfio_dma_unmap_lm_file(pdsc_dev, DMA_FROM_DEVICE, lm_file);315 316 return err;317}318 319int pds_vfio_set_lm_state_cmd(struct pds_vfio_pci_device *pds_vfio)320{321 union pds_core_adminq_cmd cmd = {322 .lm_restore = {323 .opcode = PDS_LM_CMD_RESTORE,324 .vf_id = cpu_to_le16(pds_vfio->vf_id),325 },326 };327 struct pci_dev *pdev = pds_vfio_to_pci_dev(pds_vfio);328 struct device *pdsc_dev = &pci_physfn(pdev)->dev;329 union pds_core_adminq_comp comp = {};330 struct pds_vfio_lm_file *lm_file;331 int err;332 333 dev_dbg(&pdev->dev, "vf%u: Set migration state\n", pds_vfio->vf_id);334 335 lm_file = pds_vfio->restore_file;336 337 err = pds_vfio_dma_map_lm_file(pdsc_dev, DMA_TO_DEVICE, lm_file);338 if (err) {339 dev_err(&pdev->dev,340 "failed to map restore migration file: %pe\n",341 ERR_PTR(err));342 return err;343 }344 345 cmd.lm_restore.sgl_addr = cpu_to_le64(lm_file->sgl_addr);346 cmd.lm_restore.num_sge = cpu_to_le32(lm_file->num_sge);347 348 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);349 if (err)350 dev_err(&pdev->dev, "failed to set migration state: %pe\n",351 ERR_PTR(err));352 353 pds_vfio_dma_unmap_lm_file(pdsc_dev, DMA_TO_DEVICE, lm_file);354 355 return err;356}357 358void pds_vfio_send_host_vf_lm_status_cmd(struct pds_vfio_pci_device *pds_vfio,359 enum pds_lm_host_vf_status vf_status)360{361 union pds_core_adminq_cmd cmd = {362 .lm_host_vf_status = {363 .opcode = PDS_LM_CMD_HOST_VF_STATUS,364 .vf_id = cpu_to_le16(pds_vfio->vf_id),365 .status = vf_status,366 },367 };368 struct device *dev = pds_vfio_to_dev(pds_vfio);369 union pds_core_adminq_comp comp = {};370 int err;371 372 dev_dbg(dev, "vf%u: Set host VF LM status: %u", pds_vfio->vf_id,373 vf_status);374 if (vf_status != PDS_LM_STA_IN_PROGRESS &&375 vf_status != PDS_LM_STA_NONE) {376 dev_warn(dev, "Invalid host VF migration status, %d\n",377 vf_status);378 return;379 }380 381 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);382 if (err)383 dev_warn(dev, "failed to send host VF migration status: %pe\n",384 ERR_PTR(err));385}386 387int pds_vfio_dirty_status_cmd(struct pds_vfio_pci_device *pds_vfio,388 u64 regions_dma, u8 *max_regions, u8 *num_regions)389{390 union pds_core_adminq_cmd cmd = {391 .lm_dirty_status = {392 .opcode = PDS_LM_CMD_DIRTY_STATUS,393 .vf_id = cpu_to_le16(pds_vfio->vf_id),394 },395 };396 struct device *dev = pds_vfio_to_dev(pds_vfio);397 union pds_core_adminq_comp comp = {};398 int err;399 400 dev_dbg(dev, "vf%u: Dirty status\n", pds_vfio->vf_id);401 402 cmd.lm_dirty_status.regions_dma = cpu_to_le64(regions_dma);403 cmd.lm_dirty_status.max_regions = *max_regions;404 405 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);406 if (err) {407 dev_err(dev, "failed to get dirty status: %pe\n", ERR_PTR(err));408 return err;409 }410 411 /* only support seq_ack approach for now */412 if (!(le32_to_cpu(comp.lm_dirty_status.bmp_type_mask) &413 BIT(PDS_LM_DIRTY_BMP_TYPE_SEQ_ACK))) {414 dev_err(dev, "Dirty bitmap tracking SEQ_ACK not supported\n");415 return -EOPNOTSUPP;416 }417 418 *num_regions = comp.lm_dirty_status.num_regions;419 *max_regions = comp.lm_dirty_status.max_regions;420 421 dev_dbg(dev,422 "Page Tracking Status command successful, max_regions: %d, num_regions: %d, bmp_type: %s\n",423 *max_regions, *num_regions, "PDS_LM_DIRTY_BMP_TYPE_SEQ_ACK");424 425 return 0;426}427 428int pds_vfio_dirty_enable_cmd(struct pds_vfio_pci_device *pds_vfio,429 u64 regions_dma, u8 num_regions)430{431 union pds_core_adminq_cmd cmd = {432 .lm_dirty_enable = {433 .opcode = PDS_LM_CMD_DIRTY_ENABLE,434 .vf_id = cpu_to_le16(pds_vfio->vf_id),435 .regions_dma = cpu_to_le64(regions_dma),436 .bmp_type = PDS_LM_DIRTY_BMP_TYPE_SEQ_ACK,437 .num_regions = num_regions,438 },439 };440 struct device *dev = pds_vfio_to_dev(pds_vfio);441 union pds_core_adminq_comp comp = {};442 int err;443 444 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);445 if (err) {446 dev_err(dev, "failed dirty tracking enable: %pe\n",447 ERR_PTR(err));448 return err;449 }450 451 return 0;452}453 454int pds_vfio_dirty_disable_cmd(struct pds_vfio_pci_device *pds_vfio)455{456 union pds_core_adminq_cmd cmd = {457 .lm_dirty_disable = {458 .opcode = PDS_LM_CMD_DIRTY_DISABLE,459 .vf_id = cpu_to_le16(pds_vfio->vf_id),460 },461 };462 struct device *dev = pds_vfio_to_dev(pds_vfio);463 union pds_core_adminq_comp comp = {};464 int err;465 466 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);467 if (err || comp.lm_dirty_status.num_regions != 0) {468 /* in case num_regions is still non-zero after disable */469 err = err ? err : -EIO;470 dev_err(dev,471 "failed dirty tracking disable: %pe, num_regions %d\n",472 ERR_PTR(err), comp.lm_dirty_status.num_regions);473 return err;474 }475 476 return 0;477}478 479int pds_vfio_dirty_seq_ack_cmd(struct pds_vfio_pci_device *pds_vfio,480 u64 sgl_dma, u16 num_sge, u32 offset,481 u32 total_len, bool read_seq)482{483 const char *cmd_type_str = read_seq ? "read_seq" : "write_ack";484 union pds_core_adminq_cmd cmd = {485 .lm_dirty_seq_ack = {486 .vf_id = cpu_to_le16(pds_vfio->vf_id),487 .len_bytes = cpu_to_le32(total_len),488 .off_bytes = cpu_to_le32(offset),489 .sgl_addr = cpu_to_le64(sgl_dma),490 .num_sge = cpu_to_le16(num_sge),491 },492 };493 struct device *dev = pds_vfio_to_dev(pds_vfio);494 union pds_core_adminq_comp comp = {};495 int err;496 497 if (read_seq)498 cmd.lm_dirty_seq_ack.opcode = PDS_LM_CMD_DIRTY_READ_SEQ;499 else500 cmd.lm_dirty_seq_ack.opcode = PDS_LM_CMD_DIRTY_WRITE_ACK;501 502 err = pds_vfio_client_adminq_cmd(pds_vfio, &cmd, &comp, false);503 if (err) {504 dev_err(dev, "failed cmd Page Tracking %s: %pe\n", cmd_type_str,505 ERR_PTR(err));506 return err;507 }508 509 return 0;510}511