241 lines · c
1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */2/*3 * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.4 */5 6#ifndef MLX5_VFIO_CMD_H7#define MLX5_VFIO_CMD_H8 9#include <linux/kernel.h>10#include <linux/vfio_pci_core.h>11#include <linux/mlx5/driver.h>12#include <linux/mlx5/vport.h>13#include <linux/mlx5/cq.h>14#include <linux/mlx5/qp.h>15 16enum mlx5_vf_migf_state {17 MLX5_MIGF_STATE_ERROR = 1,18 MLX5_MIGF_STATE_PRE_COPY_ERROR,19 MLX5_MIGF_STATE_PRE_COPY,20 MLX5_MIGF_STATE_SAVE_STOP_COPY_CHUNK,21 MLX5_MIGF_STATE_COMPLETE,22};23 24enum mlx5_vf_load_state {25 MLX5_VF_LOAD_STATE_READ_HEADER,26 MLX5_VF_LOAD_STATE_PREP_HEADER_DATA,27 MLX5_VF_LOAD_STATE_READ_HEADER_DATA,28 MLX5_VF_LOAD_STATE_PREP_IMAGE,29 MLX5_VF_LOAD_STATE_READ_IMAGE,30 MLX5_VF_LOAD_STATE_LOAD_IMAGE,31};32 33struct mlx5_vf_migration_tag_stop_copy_data {34 __le64 stop_copy_size;35};36 37enum mlx5_vf_migf_header_flags {38 MLX5_MIGF_HEADER_FLAGS_TAG_MANDATORY = 0,39 MLX5_MIGF_HEADER_FLAGS_TAG_OPTIONAL = 1 << 0,40};41 42enum mlx5_vf_migf_header_tag {43 MLX5_MIGF_HEADER_TAG_FW_DATA = 0,44 MLX5_MIGF_HEADER_TAG_STOP_COPY_SIZE = 1 << 0,45};46 47struct mlx5_vf_migration_header {48 __le64 record_size;49 /* For future use in case we may need to change the kernel protocol */50 __le32 flags; /* Use mlx5_vf_migf_header_flags */51 __le32 tag; /* Use mlx5_vf_migf_header_tag */52 __u8 data[]; /* Its size is given in the record_size */53};54 55struct mlx5_vhca_data_buffer {56 struct sg_append_table table;57 loff_t start_pos;58 u64 length;59 u64 allocated_length;60 u32 mkey;61 enum dma_data_direction dma_dir;62 u8 dmaed:1;63 u8 stop_copy_chunk_num;64 struct list_head buf_elm;65 struct mlx5_vf_migration_file *migf;66 /* Optimize mlx5vf_get_migration_page() for sequential access */67 struct scatterlist *last_offset_sg;68 unsigned int sg_last_entry;69 unsigned long last_offset;70};71 72struct mlx5vf_async_data {73 struct mlx5_async_work cb_work;74 struct work_struct work;75 struct mlx5_vhca_data_buffer *buf;76 struct mlx5_vhca_data_buffer *header_buf;77 int status;78 u8 stop_copy_chunk:1;79 void *out;80};81 82struct mlx5vf_save_work_data {83 struct mlx5_vf_migration_file *migf;84 size_t next_required_umem_size;85 struct work_struct work;86 u8 chunk_num;87};88 89#define MAX_NUM_CHUNKS 290 91struct mlx5_vf_migration_file {92 struct file *filp;93 struct mutex lock;94 enum mlx5_vf_migf_state state;95 96 enum mlx5_vf_load_state load_state;97 u32 pdn;98 loff_t max_pos;99 u64 record_size;100 u32 record_tag;101 u64 stop_copy_prep_size;102 u64 pre_copy_initial_bytes;103 size_t next_required_umem_size;104 u8 num_ready_chunks;105 /* Upon chunk mode preserve another set of buffers for stop_copy phase */106 struct mlx5_vhca_data_buffer *buf[MAX_NUM_CHUNKS];107 struct mlx5_vhca_data_buffer *buf_header[MAX_NUM_CHUNKS];108 struct mlx5vf_save_work_data save_data[MAX_NUM_CHUNKS];109 spinlock_t list_lock;110 struct list_head buf_list;111 struct list_head avail_list;112 struct mlx5vf_pci_core_device *mvdev;113 wait_queue_head_t poll_wait;114 struct completion save_comp;115 struct mlx5_async_ctx async_ctx;116 struct mlx5vf_async_data async_data;117};118 119struct mlx5_vhca_cq_buf {120 struct mlx5_frag_buf_ctrl fbc;121 struct mlx5_frag_buf frag_buf;122 int cqe_size;123 int nent;124};125 126struct mlx5_vhca_cq {127 struct mlx5_vhca_cq_buf buf;128 struct mlx5_db db;129 struct mlx5_core_cq mcq;130 size_t ncqe;131};132 133struct mlx5_vhca_recv_buf {134 u32 npages;135 struct page **page_list;136 dma_addr_t *dma_addrs;137 u32 next_rq_offset;138 u32 mkey;139};140 141struct mlx5_vhca_qp {142 struct mlx5_frag_buf buf;143 struct mlx5_db db;144 struct mlx5_vhca_recv_buf recv_buf;145 u32 tracked_page_size;146 u32 max_msg_size;147 u32 qpn;148 struct {149 unsigned int pc;150 unsigned int cc;151 unsigned int wqe_cnt;152 __be32 *db;153 struct mlx5_frag_buf_ctrl fbc;154 } rq;155};156 157struct mlx5_vhca_page_tracker {158 u32 id;159 u32 pdn;160 u8 is_err:1;161 u8 object_changed:1;162 struct mlx5_uars_page *uar;163 struct mlx5_vhca_cq cq;164 struct mlx5_vhca_qp *host_qp;165 struct mlx5_vhca_qp *fw_qp;166 struct mlx5_nb nb;167 int status;168};169 170struct mlx5vf_pci_core_device {171 struct vfio_pci_core_device core_device;172 int vf_id;173 u16 vhca_id;174 u8 migrate_cap:1;175 u8 deferred_reset:1;176 u8 mdev_detach:1;177 u8 log_active:1;178 u8 chunk_mode:1;179 struct completion tracker_comp;180 /* protect migration state */181 struct mutex state_mutex;182 enum vfio_device_mig_state mig_state;183 /* protect the reset_done flow */184 spinlock_t reset_lock;185 struct mlx5_vf_migration_file *resuming_migf;186 struct mlx5_vf_migration_file *saving_migf;187 struct mlx5_vhca_page_tracker tracker;188 struct workqueue_struct *cb_wq;189 struct notifier_block nb;190 struct mlx5_core_dev *mdev;191};192 193enum {194 MLX5VF_QUERY_INC = (1UL << 0),195 MLX5VF_QUERY_FINAL = (1UL << 1),196 MLX5VF_QUERY_CLEANUP = (1UL << 2),197};198 199int mlx5vf_cmd_suspend_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod);200int mlx5vf_cmd_resume_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod);201int mlx5vf_cmd_query_vhca_migration_state(struct mlx5vf_pci_core_device *mvdev,202 size_t *state_size, u64 *total_size,203 u8 query_flags);204void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev,205 const struct vfio_migration_ops *mig_ops,206 const struct vfio_log_ops *log_ops);207void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev);208void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev);209int mlx5vf_cmd_save_vhca_state(struct mlx5vf_pci_core_device *mvdev,210 struct mlx5_vf_migration_file *migf,211 struct mlx5_vhca_data_buffer *buf, bool inc,212 bool track);213int mlx5vf_cmd_load_vhca_state(struct mlx5vf_pci_core_device *mvdev,214 struct mlx5_vf_migration_file *migf,215 struct mlx5_vhca_data_buffer *buf);216int mlx5vf_cmd_alloc_pd(struct mlx5_vf_migration_file *migf);217void mlx5vf_cmd_dealloc_pd(struct mlx5_vf_migration_file *migf);218void mlx5fv_cmd_clean_migf_resources(struct mlx5_vf_migration_file *migf);219struct mlx5_vhca_data_buffer *220mlx5vf_alloc_data_buffer(struct mlx5_vf_migration_file *migf,221 size_t length, enum dma_data_direction dma_dir);222void mlx5vf_free_data_buffer(struct mlx5_vhca_data_buffer *buf);223struct mlx5_vhca_data_buffer *224mlx5vf_get_data_buffer(struct mlx5_vf_migration_file *migf,225 size_t length, enum dma_data_direction dma_dir);226void mlx5vf_put_data_buffer(struct mlx5_vhca_data_buffer *buf);227struct page *mlx5vf_get_migration_page(struct mlx5_vhca_data_buffer *buf,228 unsigned long offset);229void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev);230void mlx5vf_disable_fds(struct mlx5vf_pci_core_device *mvdev,231 enum mlx5_vf_migf_state *last_save_state);232void mlx5vf_mig_file_cleanup_cb(struct work_struct *_work);233void mlx5vf_mig_file_set_save_work(struct mlx5_vf_migration_file *migf,234 u8 chunk_num, size_t next_required_umem_size);235int mlx5vf_start_page_tracker(struct vfio_device *vdev,236 struct rb_root_cached *ranges, u32 nnodes, u64 *page_size);237int mlx5vf_stop_page_tracker(struct vfio_device *vdev);238int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova,239 unsigned long length, struct iova_bitmap *dirty);240#endif /* MLX5_VFIO_CMD_H */241