446 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright(c) 2023 Advanced Micro Devices, Inc. */3 4#include <linux/anon_inodes.h>5#include <linux/file.h>6#include <linux/fs.h>7#include <linux/highmem.h>8#include <linux/vfio.h>9#include <linux/vfio_pci_core.h>10 11#include "vfio_dev.h"12#include "cmds.h"13 14static struct pds_vfio_lm_file *15pds_vfio_get_lm_file(const struct file_operations *fops, int flags, u64 size)16{17 struct pds_vfio_lm_file *lm_file = NULL;18 unsigned long long npages;19 struct page **pages;20 void *page_mem;21 const void *p;22 23 if (!size)24 return NULL;25 26 /* Alloc file structure */27 lm_file = kzalloc(sizeof(*lm_file), GFP_KERNEL);28 if (!lm_file)29 return NULL;30 31 /* Create file */32 lm_file->filep =33 anon_inode_getfile("pds_vfio_lm", fops, lm_file, flags);34 if (IS_ERR(lm_file->filep))35 goto out_free_file;36 37 stream_open(lm_file->filep->f_inode, lm_file->filep);38 mutex_init(&lm_file->lock);39 40 /* prevent file from being released before we are done with it */41 get_file(lm_file->filep);42 43 /* Allocate memory for file pages */44 npages = DIV_ROUND_UP_ULL(size, PAGE_SIZE);45 pages = kmalloc_array(npages, sizeof(*pages), GFP_KERNEL);46 if (!pages)47 goto out_put_file;48 49 page_mem = kvzalloc(ALIGN(size, PAGE_SIZE), GFP_KERNEL);50 if (!page_mem)51 goto out_free_pages_array;52 53 p = page_mem - offset_in_page(page_mem);54 for (unsigned long long i = 0; i < npages; i++) {55 if (is_vmalloc_addr(p))56 pages[i] = vmalloc_to_page(p);57 else58 pages[i] = kmap_to_page((void *)p);59 if (!pages[i])60 goto out_free_page_mem;61 62 p += PAGE_SIZE;63 }64 65 /* Create scatterlist of file pages to use for DMA mapping later */66 if (sg_alloc_table_from_pages(&lm_file->sg_table, pages, npages, 0,67 size, GFP_KERNEL))68 goto out_free_page_mem;69 70 lm_file->size = size;71 lm_file->pages = pages;72 lm_file->npages = npages;73 lm_file->page_mem = page_mem;74 lm_file->alloc_size = npages * PAGE_SIZE;75 76 return lm_file;77 78out_free_page_mem:79 kvfree(page_mem);80out_free_pages_array:81 kfree(pages);82out_put_file:83 fput(lm_file->filep);84 mutex_destroy(&lm_file->lock);85out_free_file:86 kfree(lm_file);87 88 return NULL;89}90 91static void pds_vfio_put_lm_file(struct pds_vfio_lm_file *lm_file)92{93 mutex_lock(&lm_file->lock);94 95 lm_file->disabled = true;96 lm_file->size = 0;97 lm_file->alloc_size = 0;98 lm_file->filep->f_pos = 0;99 100 /* Free scatter list of file pages */101 sg_free_table(&lm_file->sg_table);102 103 kvfree(lm_file->page_mem);104 lm_file->page_mem = NULL;105 kfree(lm_file->pages);106 lm_file->pages = NULL;107 108 mutex_unlock(&lm_file->lock);109 110 /* allow file to be released since we are done with it */111 fput(lm_file->filep);112}113 114void pds_vfio_put_save_file(struct pds_vfio_pci_device *pds_vfio)115{116 if (!pds_vfio->save_file)117 return;118 119 pds_vfio_put_lm_file(pds_vfio->save_file);120 pds_vfio->save_file = NULL;121}122 123void pds_vfio_put_restore_file(struct pds_vfio_pci_device *pds_vfio)124{125 if (!pds_vfio->restore_file)126 return;127 128 pds_vfio_put_lm_file(pds_vfio->restore_file);129 pds_vfio->restore_file = NULL;130}131 132static struct page *pds_vfio_get_file_page(struct pds_vfio_lm_file *lm_file,133 unsigned long offset)134{135 unsigned long cur_offset = 0;136 struct scatterlist *sg;137 unsigned int i;138 139 /* All accesses are sequential */140 if (offset < lm_file->last_offset || !lm_file->last_offset_sg) {141 lm_file->last_offset = 0;142 lm_file->last_offset_sg = lm_file->sg_table.sgl;143 lm_file->sg_last_entry = 0;144 }145 146 cur_offset = lm_file->last_offset;147 148 for_each_sg(lm_file->last_offset_sg, sg,149 lm_file->sg_table.orig_nents - lm_file->sg_last_entry, i) {150 if (offset < sg->length + cur_offset) {151 lm_file->last_offset_sg = sg;152 lm_file->sg_last_entry += i;153 lm_file->last_offset = cur_offset;154 return nth_page(sg_page(sg),155 (offset - cur_offset) / PAGE_SIZE);156 }157 cur_offset += sg->length;158 }159 160 return NULL;161}162 163static int pds_vfio_release_file(struct inode *inode, struct file *filp)164{165 struct pds_vfio_lm_file *lm_file = filp->private_data;166 167 mutex_lock(&lm_file->lock);168 lm_file->filep->f_pos = 0;169 lm_file->size = 0;170 mutex_unlock(&lm_file->lock);171 mutex_destroy(&lm_file->lock);172 kfree(lm_file);173 174 return 0;175}176 177static ssize_t pds_vfio_save_read(struct file *filp, char __user *buf,178 size_t len, loff_t *pos)179{180 struct pds_vfio_lm_file *lm_file = filp->private_data;181 ssize_t done = 0;182 183 if (pos)184 return -ESPIPE;185 pos = &filp->f_pos;186 187 mutex_lock(&lm_file->lock);188 189 if (lm_file->disabled) {190 done = -ENODEV;191 goto out_unlock;192 }193 194 if (*pos > lm_file->size) {195 done = -EINVAL;196 goto out_unlock;197 }198 199 len = min_t(size_t, lm_file->size - *pos, len);200 while (len) {201 size_t page_offset;202 struct page *page;203 size_t page_len;204 u8 *from_buff;205 int err;206 207 page_offset = (*pos) % PAGE_SIZE;208 page = pds_vfio_get_file_page(lm_file, *pos - page_offset);209 if (!page) {210 if (done == 0)211 done = -EINVAL;212 goto out_unlock;213 }214 215 page_len = min_t(size_t, len, PAGE_SIZE - page_offset);216 from_buff = kmap_local_page(page);217 err = copy_to_user(buf, from_buff + page_offset, page_len);218 kunmap_local(from_buff);219 if (err) {220 done = -EFAULT;221 goto out_unlock;222 }223 *pos += page_len;224 len -= page_len;225 done += page_len;226 buf += page_len;227 }228 229out_unlock:230 mutex_unlock(&lm_file->lock);231 return done;232}233 234static const struct file_operations pds_vfio_save_fops = {235 .owner = THIS_MODULE,236 .read = pds_vfio_save_read,237 .release = pds_vfio_release_file,238};239 240static int pds_vfio_get_save_file(struct pds_vfio_pci_device *pds_vfio)241{242 struct device *dev = &pds_vfio->vfio_coredev.pdev->dev;243 struct pds_vfio_lm_file *lm_file;244 u64 size;245 int err;246 247 /* Get live migration state size in this state */248 err = pds_vfio_get_lm_state_size_cmd(pds_vfio, &size);249 if (err) {250 dev_err(dev, "failed to get save status: %pe\n", ERR_PTR(err));251 return err;252 }253 254 dev_dbg(dev, "save status, size = %lld\n", size);255 256 if (!size) {257 dev_err(dev, "invalid state size\n");258 return -EIO;259 }260 261 lm_file = pds_vfio_get_lm_file(&pds_vfio_save_fops, O_RDONLY, size);262 if (!lm_file) {263 dev_err(dev, "failed to create save file\n");264 return -ENOENT;265 }266 267 dev_dbg(dev, "size = %lld, alloc_size = %lld, npages = %lld\n",268 lm_file->size, lm_file->alloc_size, lm_file->npages);269 270 pds_vfio->save_file = lm_file;271 272 return 0;273}274 275static ssize_t pds_vfio_restore_write(struct file *filp, const char __user *buf,276 size_t len, loff_t *pos)277{278 struct pds_vfio_lm_file *lm_file = filp->private_data;279 loff_t requested_length;280 ssize_t done = 0;281 282 if (pos)283 return -ESPIPE;284 285 pos = &filp->f_pos;286 287 if (*pos < 0 ||288 check_add_overflow((loff_t)len, *pos, &requested_length))289 return -EINVAL;290 291 mutex_lock(&lm_file->lock);292 293 if (lm_file->disabled) {294 done = -ENODEV;295 goto out_unlock;296 }297 298 while (len) {299 size_t page_offset;300 struct page *page;301 size_t page_len;302 u8 *to_buff;303 int err;304 305 page_offset = (*pos) % PAGE_SIZE;306 page = pds_vfio_get_file_page(lm_file, *pos - page_offset);307 if (!page) {308 if (done == 0)309 done = -EINVAL;310 goto out_unlock;311 }312 313 page_len = min_t(size_t, len, PAGE_SIZE - page_offset);314 to_buff = kmap_local_page(page);315 err = copy_from_user(to_buff + page_offset, buf, page_len);316 kunmap_local(to_buff);317 if (err) {318 done = -EFAULT;319 goto out_unlock;320 }321 *pos += page_len;322 len -= page_len;323 done += page_len;324 buf += page_len;325 lm_file->size += page_len;326 }327out_unlock:328 mutex_unlock(&lm_file->lock);329 return done;330}331 332static const struct file_operations pds_vfio_restore_fops = {333 .owner = THIS_MODULE,334 .write = pds_vfio_restore_write,335 .release = pds_vfio_release_file,336};337 338static int pds_vfio_get_restore_file(struct pds_vfio_pci_device *pds_vfio)339{340 struct device *dev = &pds_vfio->vfio_coredev.pdev->dev;341 struct pds_vfio_lm_file *lm_file;342 u64 size;343 344 size = sizeof(union pds_lm_dev_state);345 dev_dbg(dev, "restore status, size = %lld\n", size);346 347 if (!size) {348 dev_err(dev, "invalid state size");349 return -EIO;350 }351 352 lm_file = pds_vfio_get_lm_file(&pds_vfio_restore_fops, O_WRONLY, size);353 if (!lm_file) {354 dev_err(dev, "failed to create restore file");355 return -ENOENT;356 }357 pds_vfio->restore_file = lm_file;358 359 return 0;360}361 362struct file *363pds_vfio_step_device_state_locked(struct pds_vfio_pci_device *pds_vfio,364 enum vfio_device_mig_state next)365{366 enum vfio_device_mig_state cur = pds_vfio->state;367 int err;368 369 if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_STOP_COPY) {370 err = pds_vfio_get_save_file(pds_vfio);371 if (err)372 return ERR_PTR(err);373 374 err = pds_vfio_get_lm_state_cmd(pds_vfio);375 if (err) {376 pds_vfio_put_save_file(pds_vfio);377 return ERR_PTR(err);378 }379 380 return pds_vfio->save_file->filep;381 }382 383 if (cur == VFIO_DEVICE_STATE_STOP_COPY && next == VFIO_DEVICE_STATE_STOP) {384 pds_vfio_put_save_file(pds_vfio);385 pds_vfio_dirty_disable(pds_vfio, true);386 return NULL;387 }388 389 if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_RESUMING) {390 err = pds_vfio_get_restore_file(pds_vfio);391 if (err)392 return ERR_PTR(err);393 394 return pds_vfio->restore_file->filep;395 }396 397 if (cur == VFIO_DEVICE_STATE_RESUMING && next == VFIO_DEVICE_STATE_STOP) {398 err = pds_vfio_set_lm_state_cmd(pds_vfio);399 if (err)400 return ERR_PTR(err);401 402 pds_vfio_put_restore_file(pds_vfio);403 return NULL;404 }405 406 if (cur == VFIO_DEVICE_STATE_RUNNING && next == VFIO_DEVICE_STATE_RUNNING_P2P) {407 pds_vfio_send_host_vf_lm_status_cmd(pds_vfio,408 PDS_LM_STA_IN_PROGRESS);409 err = pds_vfio_suspend_device_cmd(pds_vfio,410 PDS_LM_SUSPEND_RESUME_TYPE_P2P);411 if (err)412 return ERR_PTR(err);413 414 return NULL;415 }416 417 if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && next == VFIO_DEVICE_STATE_RUNNING) {418 err = pds_vfio_resume_device_cmd(pds_vfio,419 PDS_LM_SUSPEND_RESUME_TYPE_FULL);420 if (err)421 return ERR_PTR(err);422 423 pds_vfio_send_host_vf_lm_status_cmd(pds_vfio, PDS_LM_STA_NONE);424 return NULL;425 }426 427 if (cur == VFIO_DEVICE_STATE_STOP && next == VFIO_DEVICE_STATE_RUNNING_P2P) {428 err = pds_vfio_resume_device_cmd(pds_vfio,429 PDS_LM_SUSPEND_RESUME_TYPE_P2P);430 if (err)431 return ERR_PTR(err);432 433 return NULL;434 }435 436 if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && next == VFIO_DEVICE_STATE_STOP) {437 err = pds_vfio_suspend_device_cmd(pds_vfio,438 PDS_LM_SUSPEND_RESUME_TYPE_FULL);439 if (err)440 return ERR_PTR(err);441 return NULL;442 }443 444 return ERR_PTR(-EINVAL);445}446