2662 lines · c
1// SPDX-License-Identifier: GPL-2.0+2/*******************************************************************************3 * Vhost kernel TCM fabric driver for virtio SCSI initiators4 *5 * (C) Copyright 2010-2013 Datera, Inc.6 * (C) Copyright 2010-2012 IBM Corp.7 *8 * Authors: Nicholas A. Bellinger <nab@daterainc.com>9 * Stefan Hajnoczi <stefanha@linux.vnet.ibm.com>10 ****************************************************************************/11 12#include <linux/module.h>13#include <linux/moduleparam.h>14#include <generated/utsrelease.h>15#include <linux/utsname.h>16#include <linux/init.h>17#include <linux/slab.h>18#include <linux/kthread.h>19#include <linux/types.h>20#include <linux/string.h>21#include <linux/configfs.h>22#include <linux/ctype.h>23#include <linux/compat.h>24#include <linux/eventfd.h>25#include <linux/fs.h>26#include <linux/vmalloc.h>27#include <linux/miscdevice.h>28#include <linux/blk_types.h>29#include <linux/bio.h>30#include <linux/unaligned.h>31#include <scsi/scsi_common.h>32#include <scsi/scsi_proto.h>33#include <target/target_core_base.h>34#include <target/target_core_fabric.h>35#include <linux/vhost.h>36#include <linux/virtio_scsi.h>37#include <linux/llist.h>38#include <linux/bitmap.h>39 40#include "vhost.h"41 42#define VHOST_SCSI_VERSION "v0.1"43#define VHOST_SCSI_NAMELEN 25644#define VHOST_SCSI_MAX_CDB_SIZE 3245#define VHOST_SCSI_PREALLOC_SGLS 204846#define VHOST_SCSI_PREALLOC_UPAGES 204847#define VHOST_SCSI_PREALLOC_PROT_SGLS 204848 49/* Max number of requests before requeueing the job.50 * Using this limit prevents one virtqueue from starving others with51 * request.52 */53#define VHOST_SCSI_WEIGHT 25654 55struct vhost_scsi_inflight {56 /* Wait for the flush operation to finish */57 struct completion comp;58 /* Refcount for the inflight reqs */59 struct kref kref;60};61 62struct vhost_scsi_cmd {63 /* Descriptor from vhost_get_vq_desc() for virt_queue segment */64 int tvc_vq_desc;65 /* virtio-scsi initiator task attribute */66 int tvc_task_attr;67 /* virtio-scsi response incoming iovecs */68 int tvc_in_iovs;69 /* virtio-scsi initiator data direction */70 enum dma_data_direction tvc_data_direction;71 /* Expected data transfer length from virtio-scsi header */72 u32 tvc_exp_data_len;73 /* The Tag from include/linux/virtio_scsi.h:struct virtio_scsi_cmd_req */74 u64 tvc_tag;75 /* The number of scatterlists associated with this cmd */76 u32 tvc_sgl_count;77 u32 tvc_prot_sgl_count;78 /* Saved unpacked SCSI LUN for vhost_scsi_target_queue_cmd() */79 u32 tvc_lun;80 u32 copied_iov:1;81 const void *saved_iter_addr;82 struct iov_iter saved_iter;83 /* Pointer to the SGL formatted memory from virtio-scsi */84 struct scatterlist *tvc_sgl;85 struct scatterlist *tvc_prot_sgl;86 struct page **tvc_upages;87 /* Pointer to response header iovec */88 struct iovec *tvc_resp_iov;89 /* Pointer to vhost_scsi for our device */90 struct vhost_scsi *tvc_vhost;91 /* Pointer to vhost_virtqueue for the cmd */92 struct vhost_virtqueue *tvc_vq;93 /* Pointer to vhost nexus memory */94 struct vhost_scsi_nexus *tvc_nexus;95 /* The TCM I/O descriptor that is accessed via container_of() */96 struct se_cmd tvc_se_cmd;97 /* Copy of the incoming SCSI command descriptor block (CDB) */98 unsigned char tvc_cdb[VHOST_SCSI_MAX_CDB_SIZE];99 /* Sense buffer that will be mapped into outgoing status */100 unsigned char tvc_sense_buf[TRANSPORT_SENSE_BUFFER];101 /* Completed commands list, serviced from vhost worker thread */102 struct llist_node tvc_completion_list;103 /* Used to track inflight cmd */104 struct vhost_scsi_inflight *inflight;105};106 107struct vhost_scsi_nexus {108 /* Pointer to TCM session for I_T Nexus */109 struct se_session *tvn_se_sess;110};111 112struct vhost_scsi_tpg {113 /* Vhost port target portal group tag for TCM */114 u16 tport_tpgt;115 /* Used to track number of TPG Port/Lun Links wrt to explict I_T Nexus shutdown */116 int tv_tpg_port_count;117 /* Used for vhost_scsi device reference to tpg_nexus, protected by tv_tpg_mutex */118 int tv_tpg_vhost_count;119 /* Used for enabling T10-PI with legacy devices */120 int tv_fabric_prot_type;121 /* list for vhost_scsi_list */122 struct list_head tv_tpg_list;123 /* Used to protect access for tpg_nexus */124 struct mutex tv_tpg_mutex;125 /* Pointer to the TCM VHost I_T Nexus for this TPG endpoint */126 struct vhost_scsi_nexus *tpg_nexus;127 /* Pointer back to vhost_scsi_tport */128 struct vhost_scsi_tport *tport;129 /* Returned by vhost_scsi_make_tpg() */130 struct se_portal_group se_tpg;131 /* Pointer back to vhost_scsi, protected by tv_tpg_mutex */132 struct vhost_scsi *vhost_scsi;133};134 135struct vhost_scsi_tport {136 /* SCSI protocol the tport is providing */137 u8 tport_proto_id;138 /* Binary World Wide unique Port Name for Vhost Target port */139 u64 tport_wwpn;140 /* ASCII formatted WWPN for Vhost Target port */141 char tport_name[VHOST_SCSI_NAMELEN];142 /* Returned by vhost_scsi_make_tport() */143 struct se_wwn tport_wwn;144};145 146struct vhost_scsi_evt {147 /* event to be sent to guest */148 struct virtio_scsi_event event;149 /* event list, serviced from vhost worker thread */150 struct llist_node list;151};152 153enum {154 VHOST_SCSI_VQ_CTL = 0,155 VHOST_SCSI_VQ_EVT = 1,156 VHOST_SCSI_VQ_IO = 2,157};158 159/* Note: can't set VIRTIO_F_VERSION_1 yet, since that implies ANY_LAYOUT. */160enum {161 VHOST_SCSI_FEATURES = VHOST_FEATURES | (1ULL << VIRTIO_SCSI_F_HOTPLUG) |162 (1ULL << VIRTIO_SCSI_F_T10_PI)163};164 165#define VHOST_SCSI_MAX_TARGET 256166#define VHOST_SCSI_MAX_IO_VQ 1024167#define VHOST_SCSI_MAX_EVENT 128168 169static unsigned vhost_scsi_max_io_vqs = 128;170module_param_named(max_io_vqs, vhost_scsi_max_io_vqs, uint, 0644);171MODULE_PARM_DESC(max_io_vqs, "Set the max number of IO virtqueues a vhost scsi device can support. The default is 128. The max is 1024.");172 173struct vhost_scsi_virtqueue {174 struct vhost_virtqueue vq;175 struct vhost_scsi *vs;176 /*177 * Reference counting for inflight reqs, used for flush operation. At178 * each time, one reference tracks new commands submitted, while we179 * wait for another one to reach 0.180 */181 struct vhost_scsi_inflight inflights[2];182 /*183 * Indicate current inflight in use, protected by vq->mutex.184 * Writers must also take dev mutex and flush under it.185 */186 int inflight_idx;187 struct vhost_scsi_cmd *scsi_cmds;188 struct sbitmap scsi_tags;189 int max_cmds;190 191 struct vhost_work completion_work;192 struct llist_head completion_list;193};194 195struct vhost_scsi {196 /* Protected by vhost_scsi->dev.mutex */197 struct vhost_scsi_tpg **vs_tpg;198 char vs_vhost_wwpn[TRANSPORT_IQN_LEN];199 200 struct vhost_dev dev;201 struct vhost_scsi_virtqueue *vqs;202 struct vhost_scsi_inflight **old_inflight;203 204 struct vhost_work vs_event_work; /* evt injection work item */205 struct llist_head vs_event_list; /* evt injection queue */206 207 bool vs_events_missed; /* any missed events, protected by vq->mutex */208 int vs_events_nr; /* num of pending events, protected by vq->mutex */209};210 211struct vhost_scsi_tmf {212 struct vhost_work vwork;213 struct work_struct flush_work;214 struct vhost_scsi *vhost;215 struct vhost_scsi_virtqueue *svq;216 217 struct se_cmd se_cmd;218 u8 scsi_resp;219 struct vhost_scsi_inflight *inflight;220 struct iovec resp_iov;221 int in_iovs;222 int vq_desc;223};224 225/*226 * Context for processing request and control queue operations.227 */228struct vhost_scsi_ctx {229 int head;230 unsigned int out, in;231 size_t req_size, rsp_size;232 size_t out_size, in_size;233 u8 *target, *lunp;234 void *req;235 struct iov_iter out_iter;236};237 238/*239 * Global mutex to protect vhost_scsi TPG list for vhost IOCTLs and LIO240 * configfs management operations.241 */242static DEFINE_MUTEX(vhost_scsi_mutex);243static LIST_HEAD(vhost_scsi_list);244 245static void vhost_scsi_done_inflight(struct kref *kref)246{247 struct vhost_scsi_inflight *inflight;248 249 inflight = container_of(kref, struct vhost_scsi_inflight, kref);250 complete(&inflight->comp);251}252 253static void vhost_scsi_init_inflight(struct vhost_scsi *vs,254 struct vhost_scsi_inflight *old_inflight[])255{256 struct vhost_scsi_inflight *new_inflight;257 struct vhost_virtqueue *vq;258 int idx, i;259 260 for (i = 0; i < vs->dev.nvqs; i++) {261 vq = &vs->vqs[i].vq;262 263 mutex_lock(&vq->mutex);264 265 /* store old infight */266 idx = vs->vqs[i].inflight_idx;267 if (old_inflight)268 old_inflight[i] = &vs->vqs[i].inflights[idx];269 270 /* setup new infight */271 vs->vqs[i].inflight_idx = idx ^ 1;272 new_inflight = &vs->vqs[i].inflights[idx ^ 1];273 kref_init(&new_inflight->kref);274 init_completion(&new_inflight->comp);275 276 mutex_unlock(&vq->mutex);277 }278}279 280static struct vhost_scsi_inflight *281vhost_scsi_get_inflight(struct vhost_virtqueue *vq)282{283 struct vhost_scsi_inflight *inflight;284 struct vhost_scsi_virtqueue *svq;285 286 svq = container_of(vq, struct vhost_scsi_virtqueue, vq);287 inflight = &svq->inflights[svq->inflight_idx];288 kref_get(&inflight->kref);289 290 return inflight;291}292 293static void vhost_scsi_put_inflight(struct vhost_scsi_inflight *inflight)294{295 kref_put(&inflight->kref, vhost_scsi_done_inflight);296}297 298static int vhost_scsi_check_true(struct se_portal_group *se_tpg)299{300 return 1;301}302 303static char *vhost_scsi_get_fabric_wwn(struct se_portal_group *se_tpg)304{305 struct vhost_scsi_tpg *tpg = container_of(se_tpg,306 struct vhost_scsi_tpg, se_tpg);307 struct vhost_scsi_tport *tport = tpg->tport;308 309 return &tport->tport_name[0];310}311 312static u16 vhost_scsi_get_tpgt(struct se_portal_group *se_tpg)313{314 struct vhost_scsi_tpg *tpg = container_of(se_tpg,315 struct vhost_scsi_tpg, se_tpg);316 return tpg->tport_tpgt;317}318 319static int vhost_scsi_check_prot_fabric_only(struct se_portal_group *se_tpg)320{321 struct vhost_scsi_tpg *tpg = container_of(se_tpg,322 struct vhost_scsi_tpg, se_tpg);323 324 return tpg->tv_fabric_prot_type;325}326 327static void vhost_scsi_release_cmd_res(struct se_cmd *se_cmd)328{329 struct vhost_scsi_cmd *tv_cmd = container_of(se_cmd,330 struct vhost_scsi_cmd, tvc_se_cmd);331 struct vhost_scsi_virtqueue *svq = container_of(tv_cmd->tvc_vq,332 struct vhost_scsi_virtqueue, vq);333 struct vhost_scsi_inflight *inflight = tv_cmd->inflight;334 int i;335 336 if (tv_cmd->tvc_sgl_count) {337 for (i = 0; i < tv_cmd->tvc_sgl_count; i++) {338 if (tv_cmd->copied_iov)339 __free_page(sg_page(&tv_cmd->tvc_sgl[i]));340 else341 put_page(sg_page(&tv_cmd->tvc_sgl[i]));342 }343 kfree(tv_cmd->saved_iter_addr);344 }345 if (tv_cmd->tvc_prot_sgl_count) {346 for (i = 0; i < tv_cmd->tvc_prot_sgl_count; i++)347 put_page(sg_page(&tv_cmd->tvc_prot_sgl[i]));348 }349 350 sbitmap_clear_bit(&svq->scsi_tags, se_cmd->map_tag);351 vhost_scsi_put_inflight(inflight);352}353 354static void vhost_scsi_release_tmf_res(struct vhost_scsi_tmf *tmf)355{356 struct vhost_scsi_inflight *inflight = tmf->inflight;357 358 kfree(tmf);359 vhost_scsi_put_inflight(inflight);360}361 362static void vhost_scsi_drop_cmds(struct vhost_scsi_virtqueue *svq)363{364 struct vhost_scsi_cmd *cmd, *t;365 struct llist_node *llnode;366 367 llnode = llist_del_all(&svq->completion_list);368 llist_for_each_entry_safe(cmd, t, llnode, tvc_completion_list)369 vhost_scsi_release_cmd_res(&cmd->tvc_se_cmd);370}371 372static void vhost_scsi_release_cmd(struct se_cmd *se_cmd)373{374 if (se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) {375 struct vhost_scsi_tmf *tmf = container_of(se_cmd,376 struct vhost_scsi_tmf, se_cmd);377 378 schedule_work(&tmf->flush_work);379 } else {380 struct vhost_scsi_cmd *cmd = container_of(se_cmd,381 struct vhost_scsi_cmd, tvc_se_cmd);382 struct vhost_scsi_virtqueue *svq = container_of(cmd->tvc_vq,383 struct vhost_scsi_virtqueue, vq);384 385 llist_add(&cmd->tvc_completion_list, &svq->completion_list);386 if (!vhost_vq_work_queue(&svq->vq, &svq->completion_work))387 vhost_scsi_drop_cmds(svq);388 }389}390 391static int vhost_scsi_write_pending(struct se_cmd *se_cmd)392{393 /* Go ahead and process the write immediately */394 target_execute_cmd(se_cmd);395 return 0;396}397 398static int vhost_scsi_queue_data_in(struct se_cmd *se_cmd)399{400 transport_generic_free_cmd(se_cmd, 0);401 return 0;402}403 404static int vhost_scsi_queue_status(struct se_cmd *se_cmd)405{406 transport_generic_free_cmd(se_cmd, 0);407 return 0;408}409 410static void vhost_scsi_queue_tm_rsp(struct se_cmd *se_cmd)411{412 struct vhost_scsi_tmf *tmf = container_of(se_cmd, struct vhost_scsi_tmf,413 se_cmd);414 415 tmf->scsi_resp = se_cmd->se_tmr_req->response;416 transport_generic_free_cmd(&tmf->se_cmd, 0);417}418 419static void vhost_scsi_aborted_task(struct se_cmd *se_cmd)420{421 return;422}423 424static void vhost_scsi_free_evt(struct vhost_scsi *vs, struct vhost_scsi_evt *evt)425{426 vs->vs_events_nr--;427 kfree(evt);428}429 430static struct vhost_scsi_evt *431vhost_scsi_allocate_evt(struct vhost_scsi *vs,432 u32 event, u32 reason)433{434 struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;435 struct vhost_scsi_evt *evt;436 437 if (vs->vs_events_nr > VHOST_SCSI_MAX_EVENT) {438 vs->vs_events_missed = true;439 return NULL;440 }441 442 evt = kzalloc(sizeof(*evt), GFP_KERNEL);443 if (!evt) {444 vq_err(vq, "Failed to allocate vhost_scsi_evt\n");445 vs->vs_events_missed = true;446 return NULL;447 }448 449 evt->event.event = cpu_to_vhost32(vq, event);450 evt->event.reason = cpu_to_vhost32(vq, reason);451 vs->vs_events_nr++;452 453 return evt;454}455 456static int vhost_scsi_check_stop_free(struct se_cmd *se_cmd)457{458 return target_put_sess_cmd(se_cmd);459}460 461static void462vhost_scsi_do_evt_work(struct vhost_scsi *vs, struct vhost_scsi_evt *evt)463{464 struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;465 struct virtio_scsi_event *event = &evt->event;466 struct virtio_scsi_event __user *eventp;467 unsigned out, in;468 int head, ret;469 470 if (!vhost_vq_get_backend(vq)) {471 vs->vs_events_missed = true;472 return;473 }474 475again:476 vhost_disable_notify(&vs->dev, vq);477 head = vhost_get_vq_desc(vq, vq->iov,478 ARRAY_SIZE(vq->iov), &out, &in,479 NULL, NULL);480 if (head < 0) {481 vs->vs_events_missed = true;482 return;483 }484 if (head == vq->num) {485 if (vhost_enable_notify(&vs->dev, vq))486 goto again;487 vs->vs_events_missed = true;488 return;489 }490 491 if ((vq->iov[out].iov_len != sizeof(struct virtio_scsi_event))) {492 vq_err(vq, "Expecting virtio_scsi_event, got %zu bytes\n",493 vq->iov[out].iov_len);494 vs->vs_events_missed = true;495 return;496 }497 498 if (vs->vs_events_missed) {499 event->event |= cpu_to_vhost32(vq, VIRTIO_SCSI_T_EVENTS_MISSED);500 vs->vs_events_missed = false;501 }502 503 eventp = vq->iov[out].iov_base;504 ret = __copy_to_user(eventp, event, sizeof(*event));505 if (!ret)506 vhost_add_used_and_signal(&vs->dev, vq, head, 0);507 else508 vq_err(vq, "Faulted on vhost_scsi_send_event\n");509}510 511static void vhost_scsi_complete_events(struct vhost_scsi *vs, bool drop)512{513 struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;514 struct vhost_scsi_evt *evt, *t;515 struct llist_node *llnode;516 517 mutex_lock(&vq->mutex);518 llnode = llist_del_all(&vs->vs_event_list);519 llist_for_each_entry_safe(evt, t, llnode, list) {520 if (!drop)521 vhost_scsi_do_evt_work(vs, evt);522 vhost_scsi_free_evt(vs, evt);523 }524 mutex_unlock(&vq->mutex);525}526 527static void vhost_scsi_evt_work(struct vhost_work *work)528{529 struct vhost_scsi *vs = container_of(work, struct vhost_scsi,530 vs_event_work);531 vhost_scsi_complete_events(vs, false);532}533 534static int vhost_scsi_copy_sgl_to_iov(struct vhost_scsi_cmd *cmd)535{536 struct iov_iter *iter = &cmd->saved_iter;537 struct scatterlist *sg = cmd->tvc_sgl;538 struct page *page;539 size_t len;540 int i;541 542 for (i = 0; i < cmd->tvc_sgl_count; i++) {543 page = sg_page(&sg[i]);544 len = sg[i].length;545 546 if (copy_page_to_iter(page, 0, len, iter) != len) {547 pr_err("Could not copy data while handling misaligned cmd. Error %zu\n",548 len);549 return -1;550 }551 }552 553 return 0;554}555 556/* Fill in status and signal that we are done processing this command557 *558 * This is scheduled in the vhost work queue so we are called with the owner559 * process mm and can access the vring.560 */561static void vhost_scsi_complete_cmd_work(struct vhost_work *work)562{563 struct vhost_scsi_virtqueue *svq = container_of(work,564 struct vhost_scsi_virtqueue, completion_work);565 struct virtio_scsi_cmd_resp v_rsp;566 struct vhost_scsi_cmd *cmd, *t;567 struct llist_node *llnode;568 struct se_cmd *se_cmd;569 struct iov_iter iov_iter;570 bool signal = false;571 int ret;572 573 llnode = llist_del_all(&svq->completion_list);574 llist_for_each_entry_safe(cmd, t, llnode, tvc_completion_list) {575 se_cmd = &cmd->tvc_se_cmd;576 577 pr_debug("%s tv_cmd %p resid %u status %#02x\n", __func__,578 cmd, se_cmd->residual_count, se_cmd->scsi_status);579 memset(&v_rsp, 0, sizeof(v_rsp));580 581 if (cmd->saved_iter_addr && vhost_scsi_copy_sgl_to_iov(cmd)) {582 v_rsp.response = VIRTIO_SCSI_S_BAD_TARGET;583 } else {584 v_rsp.resid = cpu_to_vhost32(cmd->tvc_vq,585 se_cmd->residual_count);586 /* TODO is status_qualifier field needed? */587 v_rsp.status = se_cmd->scsi_status;588 v_rsp.sense_len = cpu_to_vhost32(cmd->tvc_vq,589 se_cmd->scsi_sense_length);590 memcpy(v_rsp.sense, cmd->tvc_sense_buf,591 se_cmd->scsi_sense_length);592 }593 594 iov_iter_init(&iov_iter, ITER_DEST, cmd->tvc_resp_iov,595 cmd->tvc_in_iovs, sizeof(v_rsp));596 ret = copy_to_iter(&v_rsp, sizeof(v_rsp), &iov_iter);597 if (likely(ret == sizeof(v_rsp))) {598 signal = true;599 600 vhost_add_used(cmd->tvc_vq, cmd->tvc_vq_desc, 0);601 } else602 pr_err("Faulted on virtio_scsi_cmd_resp\n");603 604 vhost_scsi_release_cmd_res(se_cmd);605 }606 607 if (signal)608 vhost_signal(&svq->vs->dev, &svq->vq);609}610 611static struct vhost_scsi_cmd *612vhost_scsi_get_cmd(struct vhost_virtqueue *vq, struct vhost_scsi_tpg *tpg,613 unsigned char *cdb, u64 scsi_tag, u16 lun, u8 task_attr,614 u32 exp_data_len, int data_direction)615{616 struct vhost_scsi_virtqueue *svq = container_of(vq,617 struct vhost_scsi_virtqueue, vq);618 struct vhost_scsi_cmd *cmd;619 struct vhost_scsi_nexus *tv_nexus;620 struct scatterlist *sg, *prot_sg;621 struct iovec *tvc_resp_iov;622 struct page **pages;623 int tag;624 625 tv_nexus = tpg->tpg_nexus;626 if (!tv_nexus) {627 pr_err("Unable to locate active struct vhost_scsi_nexus\n");628 return ERR_PTR(-EIO);629 }630 631 tag = sbitmap_get(&svq->scsi_tags);632 if (tag < 0) {633 pr_err("Unable to obtain tag for vhost_scsi_cmd\n");634 return ERR_PTR(-ENOMEM);635 }636 637 cmd = &svq->scsi_cmds[tag];638 sg = cmd->tvc_sgl;639 prot_sg = cmd->tvc_prot_sgl;640 pages = cmd->tvc_upages;641 tvc_resp_iov = cmd->tvc_resp_iov;642 memset(cmd, 0, sizeof(*cmd));643 cmd->tvc_sgl = sg;644 cmd->tvc_prot_sgl = prot_sg;645 cmd->tvc_upages = pages;646 cmd->tvc_se_cmd.map_tag = tag;647 cmd->tvc_tag = scsi_tag;648 cmd->tvc_lun = lun;649 cmd->tvc_task_attr = task_attr;650 cmd->tvc_exp_data_len = exp_data_len;651 cmd->tvc_data_direction = data_direction;652 cmd->tvc_nexus = tv_nexus;653 cmd->inflight = vhost_scsi_get_inflight(vq);654 cmd->tvc_resp_iov = tvc_resp_iov;655 656 memcpy(cmd->tvc_cdb, cdb, VHOST_SCSI_MAX_CDB_SIZE);657 658 return cmd;659}660 661/*662 * Map a user memory range into a scatterlist663 *664 * Returns the number of scatterlist entries used or -errno on error.665 */666static int667vhost_scsi_map_to_sgl(struct vhost_scsi_cmd *cmd,668 struct iov_iter *iter,669 struct scatterlist *sgl,670 bool is_prot)671{672 struct page **pages = cmd->tvc_upages;673 struct scatterlist *sg = sgl;674 ssize_t bytes, mapped_bytes;675 size_t offset, mapped_offset;676 unsigned int npages = 0;677 678 bytes = iov_iter_get_pages2(iter, pages, LONG_MAX,679 VHOST_SCSI_PREALLOC_UPAGES, &offset);680 /* No pages were pinned */681 if (bytes <= 0)682 return bytes < 0 ? bytes : -EFAULT;683 684 mapped_bytes = bytes;685 mapped_offset = offset;686 687 while (bytes) {688 unsigned n = min_t(unsigned, PAGE_SIZE - offset, bytes);689 /*690 * The block layer requires bios/requests to be a multiple of691 * 512 bytes, but Windows can send us vecs that are misaligned.692 * This can result in bios and later requests with misaligned693 * sizes if we have to break up a cmd/scatterlist into multiple694 * bios.695 *696 * We currently only break up a command into multiple bios if697 * we hit the vec/seg limit, so check if our sgl_count is698 * greater than the max and if a vec in the cmd has a699 * misaligned offset/size.700 */701 if (!is_prot &&702 (offset & (SECTOR_SIZE - 1) || n & (SECTOR_SIZE - 1)) &&703 cmd->tvc_sgl_count > BIO_MAX_VECS) {704 WARN_ONCE(true,705 "vhost-scsi detected misaligned IO. Performance may be degraded.");706 goto revert_iter_get_pages;707 }708 709 sg_set_page(sg++, pages[npages++], n, offset);710 bytes -= n;711 offset = 0;712 }713 714 return npages;715 716revert_iter_get_pages:717 iov_iter_revert(iter, mapped_bytes);718 719 npages = 0;720 while (mapped_bytes) {721 unsigned int n = min_t(unsigned int, PAGE_SIZE - mapped_offset,722 mapped_bytes);723 724 put_page(pages[npages++]);725 726 mapped_bytes -= n;727 mapped_offset = 0;728 }729 730 return -EINVAL;731}732 733static int734vhost_scsi_calc_sgls(struct iov_iter *iter, size_t bytes, int max_sgls)735{736 int sgl_count = 0;737 738 if (!iter || !iter_iov(iter)) {739 pr_err("%s: iter->iov is NULL, but expected bytes: %zu"740 " present\n", __func__, bytes);741 return -EINVAL;742 }743 744 sgl_count = iov_iter_npages(iter, 0xffff);745 if (sgl_count > max_sgls) {746 pr_err("%s: requested sgl_count: %d exceeds pre-allocated"747 " max_sgls: %d\n", __func__, sgl_count, max_sgls);748 return -EINVAL;749 }750 return sgl_count;751}752 753static int754vhost_scsi_copy_iov_to_sgl(struct vhost_scsi_cmd *cmd, struct iov_iter *iter,755 struct scatterlist *sg, int sg_count)756{757 size_t len = iov_iter_count(iter);758 unsigned int nbytes = 0;759 struct page *page;760 int i;761 762 if (cmd->tvc_data_direction == DMA_FROM_DEVICE) {763 cmd->saved_iter_addr = dup_iter(&cmd->saved_iter, iter,764 GFP_KERNEL);765 if (!cmd->saved_iter_addr)766 return -ENOMEM;767 }768 769 for (i = 0; i < sg_count; i++) {770 page = alloc_page(GFP_KERNEL);771 if (!page) {772 i--;773 goto err;774 }775 776 nbytes = min_t(unsigned int, PAGE_SIZE, len);777 sg_set_page(&sg[i], page, nbytes, 0);778 779 if (cmd->tvc_data_direction == DMA_TO_DEVICE &&780 copy_page_from_iter(page, 0, nbytes, iter) != nbytes)781 goto err;782 783 len -= nbytes;784 }785 786 cmd->copied_iov = 1;787 return 0;788 789err:790 pr_err("Could not read %u bytes while handling misaligned cmd\n",791 nbytes);792 793 for (; i >= 0; i--)794 __free_page(sg_page(&sg[i]));795 kfree(cmd->saved_iter_addr);796 return -ENOMEM;797}798 799static int800vhost_scsi_map_iov_to_sgl(struct vhost_scsi_cmd *cmd, struct iov_iter *iter,801 struct scatterlist *sg, int sg_count, bool is_prot)802{803 struct scatterlist *p = sg;804 size_t revert_bytes;805 int ret;806 807 while (iov_iter_count(iter)) {808 ret = vhost_scsi_map_to_sgl(cmd, iter, sg, is_prot);809 if (ret < 0) {810 revert_bytes = 0;811 812 while (p < sg) {813 struct page *page = sg_page(p);814 815 if (page) {816 put_page(page);817 revert_bytes += p->length;818 }819 p++;820 }821 822 iov_iter_revert(iter, revert_bytes);823 return ret;824 }825 sg += ret;826 }827 828 return 0;829}830 831static int832vhost_scsi_mapal(struct vhost_scsi_cmd *cmd,833 size_t prot_bytes, struct iov_iter *prot_iter,834 size_t data_bytes, struct iov_iter *data_iter)835{836 int sgl_count, ret;837 838 if (prot_bytes) {839 sgl_count = vhost_scsi_calc_sgls(prot_iter, prot_bytes,840 VHOST_SCSI_PREALLOC_PROT_SGLS);841 if (sgl_count < 0)842 return sgl_count;843 844 sg_init_table(cmd->tvc_prot_sgl, sgl_count);845 cmd->tvc_prot_sgl_count = sgl_count;846 pr_debug("%s prot_sg %p prot_sgl_count %u\n", __func__,847 cmd->tvc_prot_sgl, cmd->tvc_prot_sgl_count);848 849 ret = vhost_scsi_map_iov_to_sgl(cmd, prot_iter,850 cmd->tvc_prot_sgl,851 cmd->tvc_prot_sgl_count, true);852 if (ret < 0) {853 cmd->tvc_prot_sgl_count = 0;854 return ret;855 }856 }857 sgl_count = vhost_scsi_calc_sgls(data_iter, data_bytes,858 VHOST_SCSI_PREALLOC_SGLS);859 if (sgl_count < 0)860 return sgl_count;861 862 sg_init_table(cmd->tvc_sgl, sgl_count);863 cmd->tvc_sgl_count = sgl_count;864 pr_debug("%s data_sg %p data_sgl_count %u\n", __func__,865 cmd->tvc_sgl, cmd->tvc_sgl_count);866 867 ret = vhost_scsi_map_iov_to_sgl(cmd, data_iter, cmd->tvc_sgl,868 cmd->tvc_sgl_count, false);869 if (ret == -EINVAL) {870 sg_init_table(cmd->tvc_sgl, cmd->tvc_sgl_count);871 ret = vhost_scsi_copy_iov_to_sgl(cmd, data_iter, cmd->tvc_sgl,872 cmd->tvc_sgl_count);873 }874 875 if (ret < 0) {876 cmd->tvc_sgl_count = 0;877 return ret;878 }879 return 0;880}881 882static int vhost_scsi_to_tcm_attr(int attr)883{884 switch (attr) {885 case VIRTIO_SCSI_S_SIMPLE:886 return TCM_SIMPLE_TAG;887 case VIRTIO_SCSI_S_ORDERED:888 return TCM_ORDERED_TAG;889 case VIRTIO_SCSI_S_HEAD:890 return TCM_HEAD_TAG;891 case VIRTIO_SCSI_S_ACA:892 return TCM_ACA_TAG;893 default:894 break;895 }896 return TCM_SIMPLE_TAG;897}898 899static void vhost_scsi_target_queue_cmd(struct vhost_scsi_cmd *cmd)900{901 struct se_cmd *se_cmd = &cmd->tvc_se_cmd;902 struct vhost_scsi_nexus *tv_nexus;903 struct scatterlist *sg_ptr, *sg_prot_ptr = NULL;904 905 /* FIXME: BIDI operation */906 if (cmd->tvc_sgl_count) {907 sg_ptr = cmd->tvc_sgl;908 909 if (cmd->tvc_prot_sgl_count)910 sg_prot_ptr = cmd->tvc_prot_sgl;911 else912 se_cmd->prot_pto = true;913 } else {914 sg_ptr = NULL;915 }916 tv_nexus = cmd->tvc_nexus;917 918 se_cmd->tag = 0;919 target_init_cmd(se_cmd, tv_nexus->tvn_se_sess, &cmd->tvc_sense_buf[0],920 cmd->tvc_lun, cmd->tvc_exp_data_len,921 vhost_scsi_to_tcm_attr(cmd->tvc_task_attr),922 cmd->tvc_data_direction, TARGET_SCF_ACK_KREF);923 924 if (target_submit_prep(se_cmd, cmd->tvc_cdb, sg_ptr,925 cmd->tvc_sgl_count, NULL, 0, sg_prot_ptr,926 cmd->tvc_prot_sgl_count, GFP_KERNEL))927 return;928 929 target_submit(se_cmd);930}931 932static void933vhost_scsi_send_bad_target(struct vhost_scsi *vs,934 struct vhost_virtqueue *vq,935 int head, unsigned out)936{937 struct virtio_scsi_cmd_resp __user *resp;938 struct virtio_scsi_cmd_resp rsp;939 int ret;940 941 memset(&rsp, 0, sizeof(rsp));942 rsp.response = VIRTIO_SCSI_S_BAD_TARGET;943 resp = vq->iov[out].iov_base;944 ret = __copy_to_user(resp, &rsp, sizeof(rsp));945 if (!ret)946 vhost_add_used_and_signal(&vs->dev, vq, head, 0);947 else948 pr_err("Faulted on virtio_scsi_cmd_resp\n");949}950 951static int952vhost_scsi_get_desc(struct vhost_scsi *vs, struct vhost_virtqueue *vq,953 struct vhost_scsi_ctx *vc)954{955 int ret = -ENXIO;956 957 vc->head = vhost_get_vq_desc(vq, vq->iov,958 ARRAY_SIZE(vq->iov), &vc->out, &vc->in,959 NULL, NULL);960 961 pr_debug("vhost_get_vq_desc: head: %d, out: %u in: %u\n",962 vc->head, vc->out, vc->in);963 964 /* On error, stop handling until the next kick. */965 if (unlikely(vc->head < 0))966 goto done;967 968 /* Nothing new? Wait for eventfd to tell us they refilled. */969 if (vc->head == vq->num) {970 if (unlikely(vhost_enable_notify(&vs->dev, vq))) {971 vhost_disable_notify(&vs->dev, vq);972 ret = -EAGAIN;973 }974 goto done;975 }976 977 /*978 * Get the size of request and response buffers.979 * FIXME: Not correct for BIDI operation980 */981 vc->out_size = iov_length(vq->iov, vc->out);982 vc->in_size = iov_length(&vq->iov[vc->out], vc->in);983 984 /*985 * Copy over the virtio-scsi request header, which for a986 * ANY_LAYOUT enabled guest may span multiple iovecs, or a987 * single iovec may contain both the header + outgoing988 * WRITE payloads.989 *990 * copy_from_iter() will advance out_iter, so that it will991 * point at the start of the outgoing WRITE payload, if992 * DMA_TO_DEVICE is set.993 */994 iov_iter_init(&vc->out_iter, ITER_SOURCE, vq->iov, vc->out, vc->out_size);995 ret = 0;996 997done:998 return ret;999}1000 1001static int1002vhost_scsi_chk_size(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc)1003{1004 if (unlikely(vc->in_size < vc->rsp_size)) {1005 vq_err(vq,1006 "Response buf too small, need min %zu bytes got %zu",1007 vc->rsp_size, vc->in_size);1008 return -EINVAL;1009 } else if (unlikely(vc->out_size < vc->req_size)) {1010 vq_err(vq,1011 "Request buf too small, need min %zu bytes got %zu",1012 vc->req_size, vc->out_size);1013 return -EIO;1014 }1015 1016 return 0;1017}1018 1019static int1020vhost_scsi_get_req(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc,1021 struct vhost_scsi_tpg **tpgp)1022{1023 int ret = -EIO;1024 1025 if (unlikely(!copy_from_iter_full(vc->req, vc->req_size,1026 &vc->out_iter))) {1027 vq_err(vq, "Faulted on copy_from_iter_full\n");1028 } else if (unlikely(*vc->lunp != 1)) {1029 /* virtio-scsi spec requires byte 0 of the lun to be 1 */1030 vq_err(vq, "Illegal virtio-scsi lun: %u\n", *vc->lunp);1031 } else {1032 struct vhost_scsi_tpg **vs_tpg, *tpg = NULL;1033 1034 if (vc->target) {1035 /* validated at handler entry */1036 vs_tpg = vhost_vq_get_backend(vq);1037 tpg = READ_ONCE(vs_tpg[*vc->target]);1038 if (unlikely(!tpg)) {1039 vq_err(vq, "Target 0x%x does not exist\n", *vc->target);1040 goto out;1041 }1042 }1043 1044 if (tpgp)1045 *tpgp = tpg;1046 ret = 0;1047 }1048out:1049 return ret;1050}1051 1052static u16 vhost_buf_to_lun(u8 *lun_buf)1053{1054 return ((lun_buf[2] << 8) | lun_buf[3]) & 0x3FFF;1055}1056 1057static void1058vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)1059{1060 struct vhost_scsi_tpg **vs_tpg, *tpg;1061 struct virtio_scsi_cmd_req v_req;1062 struct virtio_scsi_cmd_req_pi v_req_pi;1063 struct vhost_scsi_ctx vc;1064 struct vhost_scsi_cmd *cmd;1065 struct iov_iter in_iter, prot_iter, data_iter;1066 u64 tag;1067 u32 exp_data_len, data_direction;1068 int ret, prot_bytes, i, c = 0;1069 u16 lun;1070 u8 task_attr;1071 bool t10_pi = vhost_has_feature(vq, VIRTIO_SCSI_F_T10_PI);1072 void *cdb;1073 1074 mutex_lock(&vq->mutex);1075 /*1076 * We can handle the vq only after the endpoint is setup by calling the1077 * VHOST_SCSI_SET_ENDPOINT ioctl.1078 */1079 vs_tpg = vhost_vq_get_backend(vq);1080 if (!vs_tpg)1081 goto out;1082 1083 memset(&vc, 0, sizeof(vc));1084 vc.rsp_size = sizeof(struct virtio_scsi_cmd_resp);1085 1086 vhost_disable_notify(&vs->dev, vq);1087 1088 do {1089 ret = vhost_scsi_get_desc(vs, vq, &vc);1090 if (ret)1091 goto err;1092 1093 /*1094 * Setup pointers and values based upon different virtio-scsi1095 * request header if T10_PI is enabled in KVM guest.1096 */1097 if (t10_pi) {1098 vc.req = &v_req_pi;1099 vc.req_size = sizeof(v_req_pi);1100 vc.lunp = &v_req_pi.lun[0];1101 vc.target = &v_req_pi.lun[1];1102 } else {1103 vc.req = &v_req;1104 vc.req_size = sizeof(v_req);1105 vc.lunp = &v_req.lun[0];1106 vc.target = &v_req.lun[1];1107 }1108 1109 /*1110 * Validate the size of request and response buffers.1111 * Check for a sane response buffer so we can report1112 * early errors back to the guest.1113 */1114 ret = vhost_scsi_chk_size(vq, &vc);1115 if (ret)1116 goto err;1117 1118 ret = vhost_scsi_get_req(vq, &vc, &tpg);1119 if (ret)1120 goto err;1121 1122 ret = -EIO; /* bad target on any error from here on */1123 1124 /*1125 * Determine data_direction by calculating the total outgoing1126 * iovec sizes + incoming iovec sizes vs. virtio-scsi request +1127 * response headers respectively.1128 *1129 * For DMA_TO_DEVICE this is out_iter, which is already pointing1130 * to the right place.1131 *1132 * For DMA_FROM_DEVICE, the iovec will be just past the end1133 * of the virtio-scsi response header in either the same1134 * or immediately following iovec.1135 *1136 * Any associated T10_PI bytes for the outgoing / incoming1137 * payloads are included in calculation of exp_data_len here.1138 */1139 prot_bytes = 0;1140 1141 if (vc.out_size > vc.req_size) {1142 data_direction = DMA_TO_DEVICE;1143 exp_data_len = vc.out_size - vc.req_size;1144 data_iter = vc.out_iter;1145 } else if (vc.in_size > vc.rsp_size) {1146 data_direction = DMA_FROM_DEVICE;1147 exp_data_len = vc.in_size - vc.rsp_size;1148 1149 iov_iter_init(&in_iter, ITER_DEST, &vq->iov[vc.out], vc.in,1150 vc.rsp_size + exp_data_len);1151 iov_iter_advance(&in_iter, vc.rsp_size);1152 data_iter = in_iter;1153 } else {1154 data_direction = DMA_NONE;1155 exp_data_len = 0;1156 }1157 /*1158 * If T10_PI header + payload is present, setup prot_iter values1159 * and recalculate data_iter for vhost_scsi_mapal() mapping to1160 * host scatterlists via get_user_pages_fast().1161 */1162 if (t10_pi) {1163 if (v_req_pi.pi_bytesout) {1164 if (data_direction != DMA_TO_DEVICE) {1165 vq_err(vq, "Received non zero pi_bytesout,"1166 " but wrong data_direction\n");1167 goto err;1168 }1169 prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesout);1170 } else if (v_req_pi.pi_bytesin) {1171 if (data_direction != DMA_FROM_DEVICE) {1172 vq_err(vq, "Received non zero pi_bytesin,"1173 " but wrong data_direction\n");1174 goto err;1175 }1176 prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesin);1177 }1178 /*1179 * Set prot_iter to data_iter and truncate it to1180 * prot_bytes, and advance data_iter past any1181 * preceding prot_bytes that may be present.1182 *1183 * Also fix up the exp_data_len to reflect only the1184 * actual data payload length.1185 */1186 if (prot_bytes) {1187 exp_data_len -= prot_bytes;1188 prot_iter = data_iter;1189 iov_iter_truncate(&prot_iter, prot_bytes);1190 iov_iter_advance(&data_iter, prot_bytes);1191 }1192 tag = vhost64_to_cpu(vq, v_req_pi.tag);1193 task_attr = v_req_pi.task_attr;1194 cdb = &v_req_pi.cdb[0];1195 lun = vhost_buf_to_lun(v_req_pi.lun);1196 } else {1197 tag = vhost64_to_cpu(vq, v_req.tag);1198 task_attr = v_req.task_attr;1199 cdb = &v_req.cdb[0];1200 lun = vhost_buf_to_lun(v_req.lun);1201 }1202 /*1203 * Check that the received CDB size does not exceeded our1204 * hardcoded max for vhost-scsi, then get a pre-allocated1205 * cmd descriptor for the new virtio-scsi tag.1206 *1207 * TODO what if cdb was too small for varlen cdb header?1208 */1209 if (unlikely(scsi_command_size(cdb) > VHOST_SCSI_MAX_CDB_SIZE)) {1210 vq_err(vq, "Received SCSI CDB with command_size: %d that"1211 " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",1212 scsi_command_size(cdb), VHOST_SCSI_MAX_CDB_SIZE);1213 goto err;1214 }1215 cmd = vhost_scsi_get_cmd(vq, tpg, cdb, tag, lun, task_attr,1216 exp_data_len + prot_bytes,1217 data_direction);1218 if (IS_ERR(cmd)) {1219 vq_err(vq, "vhost_scsi_get_cmd failed %ld\n",1220 PTR_ERR(cmd));1221 goto err;1222 }1223 cmd->tvc_vhost = vs;1224 cmd->tvc_vq = vq;1225 for (i = 0; i < vc.in ; i++)1226 cmd->tvc_resp_iov[i] = vq->iov[vc.out + i];1227 cmd->tvc_in_iovs = vc.in;1228 1229 pr_debug("vhost_scsi got command opcode: %#02x, lun: %d\n",1230 cmd->tvc_cdb[0], cmd->tvc_lun);1231 pr_debug("cmd: %p exp_data_len: %d, prot_bytes: %d data_direction:"1232 " %d\n", cmd, exp_data_len, prot_bytes, data_direction);1233 1234 if (data_direction != DMA_NONE) {1235 if (unlikely(vhost_scsi_mapal(cmd, prot_bytes,1236 &prot_iter, exp_data_len,1237 &data_iter))) {1238 vq_err(vq, "Failed to map iov to sgl\n");1239 vhost_scsi_release_cmd_res(&cmd->tvc_se_cmd);1240 goto err;1241 }1242 }1243 /*1244 * Save the descriptor from vhost_get_vq_desc() to be used to1245 * complete the virtio-scsi request in TCM callback context via1246 * vhost_scsi_queue_data_in() and vhost_scsi_queue_status()1247 */1248 cmd->tvc_vq_desc = vc.head;1249 vhost_scsi_target_queue_cmd(cmd);1250 ret = 0;1251err:1252 /*1253 * ENXIO: No more requests, or read error, wait for next kick1254 * EINVAL: Invalid response buffer, drop the request1255 * EIO: Respond with bad target1256 * EAGAIN: Pending request1257 */1258 if (ret == -ENXIO)1259 break;1260 else if (ret == -EIO)1261 vhost_scsi_send_bad_target(vs, vq, vc.head, vc.out);1262 } while (likely(!vhost_exceeds_weight(vq, ++c, 0)));1263out:1264 mutex_unlock(&vq->mutex);1265}1266 1267static void1268vhost_scsi_send_tmf_resp(struct vhost_scsi *vs, struct vhost_virtqueue *vq,1269 int in_iovs, int vq_desc, struct iovec *resp_iov,1270 int tmf_resp_code)1271{1272 struct virtio_scsi_ctrl_tmf_resp rsp;1273 struct iov_iter iov_iter;1274 int ret;1275 1276 pr_debug("%s\n", __func__);1277 memset(&rsp, 0, sizeof(rsp));1278 rsp.response = tmf_resp_code;1279 1280 iov_iter_init(&iov_iter, ITER_DEST, resp_iov, in_iovs, sizeof(rsp));1281 1282 ret = copy_to_iter(&rsp, sizeof(rsp), &iov_iter);1283 if (likely(ret == sizeof(rsp)))1284 vhost_add_used_and_signal(&vs->dev, vq, vq_desc, 0);1285 else1286 pr_err("Faulted on virtio_scsi_ctrl_tmf_resp\n");1287}1288 1289static void vhost_scsi_tmf_resp_work(struct vhost_work *work)1290{1291 struct vhost_scsi_tmf *tmf = container_of(work, struct vhost_scsi_tmf,1292 vwork);1293 int resp_code;1294 1295 if (tmf->scsi_resp == TMR_FUNCTION_COMPLETE)1296 resp_code = VIRTIO_SCSI_S_FUNCTION_SUCCEEDED;1297 else1298 resp_code = VIRTIO_SCSI_S_FUNCTION_REJECTED;1299 1300 vhost_scsi_send_tmf_resp(tmf->vhost, &tmf->svq->vq, tmf->in_iovs,1301 tmf->vq_desc, &tmf->resp_iov, resp_code);1302 vhost_scsi_release_tmf_res(tmf);1303}1304 1305static void vhost_scsi_tmf_flush_work(struct work_struct *work)1306{1307 struct vhost_scsi_tmf *tmf = container_of(work, struct vhost_scsi_tmf,1308 flush_work);1309 struct vhost_virtqueue *vq = &tmf->svq->vq;1310 /*1311 * Make sure we have sent responses for other commands before we1312 * send our response.1313 */1314 vhost_dev_flush(vq->dev);1315 if (!vhost_vq_work_queue(vq, &tmf->vwork))1316 vhost_scsi_release_tmf_res(tmf);1317}1318 1319static void1320vhost_scsi_handle_tmf(struct vhost_scsi *vs, struct vhost_scsi_tpg *tpg,1321 struct vhost_virtqueue *vq,1322 struct virtio_scsi_ctrl_tmf_req *vtmf,1323 struct vhost_scsi_ctx *vc)1324{1325 struct vhost_scsi_virtqueue *svq = container_of(vq,1326 struct vhost_scsi_virtqueue, vq);1327 struct vhost_scsi_tmf *tmf;1328 1329 if (vhost32_to_cpu(vq, vtmf->subtype) !=1330 VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET)1331 goto send_reject;1332 1333 if (!tpg->tpg_nexus || !tpg->tpg_nexus->tvn_se_sess) {1334 pr_err("Unable to locate active struct vhost_scsi_nexus for LUN RESET.\n");1335 goto send_reject;1336 }1337 1338 tmf = kzalloc(sizeof(*tmf), GFP_KERNEL);1339 if (!tmf)1340 goto send_reject;1341 1342 INIT_WORK(&tmf->flush_work, vhost_scsi_tmf_flush_work);1343 vhost_work_init(&tmf->vwork, vhost_scsi_tmf_resp_work);1344 tmf->vhost = vs;1345 tmf->svq = svq;1346 tmf->resp_iov = vq->iov[vc->out];1347 tmf->vq_desc = vc->head;1348 tmf->in_iovs = vc->in;1349 tmf->inflight = vhost_scsi_get_inflight(vq);1350 1351 if (target_submit_tmr(&tmf->se_cmd, tpg->tpg_nexus->tvn_se_sess, NULL,1352 vhost_buf_to_lun(vtmf->lun), NULL,1353 TMR_LUN_RESET, GFP_KERNEL, 0,1354 TARGET_SCF_ACK_KREF) < 0) {1355 vhost_scsi_release_tmf_res(tmf);1356 goto send_reject;1357 }1358 1359 return;1360 1361send_reject:1362 vhost_scsi_send_tmf_resp(vs, vq, vc->in, vc->head, &vq->iov[vc->out],1363 VIRTIO_SCSI_S_FUNCTION_REJECTED);1364}1365 1366static void1367vhost_scsi_send_an_resp(struct vhost_scsi *vs,1368 struct vhost_virtqueue *vq,1369 struct vhost_scsi_ctx *vc)1370{1371 struct virtio_scsi_ctrl_an_resp rsp;1372 struct iov_iter iov_iter;1373 int ret;1374 1375 pr_debug("%s\n", __func__);1376 memset(&rsp, 0, sizeof(rsp)); /* event_actual = 0 */1377 rsp.response = VIRTIO_SCSI_S_OK;1378 1379 iov_iter_init(&iov_iter, ITER_DEST, &vq->iov[vc->out], vc->in, sizeof(rsp));1380 1381 ret = copy_to_iter(&rsp, sizeof(rsp), &iov_iter);1382 if (likely(ret == sizeof(rsp)))1383 vhost_add_used_and_signal(&vs->dev, vq, vc->head, 0);1384 else1385 pr_err("Faulted on virtio_scsi_ctrl_an_resp\n");1386}1387 1388static void1389vhost_scsi_ctl_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)1390{1391 struct vhost_scsi_tpg *tpg;1392 union {1393 __virtio32 type;1394 struct virtio_scsi_ctrl_an_req an;1395 struct virtio_scsi_ctrl_tmf_req tmf;1396 } v_req;1397 struct vhost_scsi_ctx vc;1398 size_t typ_size;1399 int ret, c = 0;1400 1401 mutex_lock(&vq->mutex);1402 /*1403 * We can handle the vq only after the endpoint is setup by calling the1404 * VHOST_SCSI_SET_ENDPOINT ioctl.1405 */1406 if (!vhost_vq_get_backend(vq))1407 goto out;1408 1409 memset(&vc, 0, sizeof(vc));1410 1411 vhost_disable_notify(&vs->dev, vq);1412 1413 do {1414 ret = vhost_scsi_get_desc(vs, vq, &vc);1415 if (ret)1416 goto err;1417 1418 /*1419 * Get the request type first in order to setup1420 * other parameters dependent on the type.1421 */1422 vc.req = &v_req.type;1423 typ_size = sizeof(v_req.type);1424 1425 if (unlikely(!copy_from_iter_full(vc.req, typ_size,1426 &vc.out_iter))) {1427 vq_err(vq, "Faulted on copy_from_iter tmf type\n");1428 /*1429 * The size of the response buffer depends on the1430 * request type and must be validated against it.1431 * Since the request type is not known, don't send1432 * a response.1433 */1434 continue;1435 }1436 1437 switch (vhost32_to_cpu(vq, v_req.type)) {1438 case VIRTIO_SCSI_T_TMF:1439 vc.req = &v_req.tmf;1440 vc.req_size = sizeof(struct virtio_scsi_ctrl_tmf_req);1441 vc.rsp_size = sizeof(struct virtio_scsi_ctrl_tmf_resp);1442 vc.lunp = &v_req.tmf.lun[0];1443 vc.target = &v_req.tmf.lun[1];1444 break;1445 case VIRTIO_SCSI_T_AN_QUERY:1446 case VIRTIO_SCSI_T_AN_SUBSCRIBE:1447 vc.req = &v_req.an;1448 vc.req_size = sizeof(struct virtio_scsi_ctrl_an_req);1449 vc.rsp_size = sizeof(struct virtio_scsi_ctrl_an_resp);1450 vc.lunp = &v_req.an.lun[0];1451 vc.target = NULL;1452 break;1453 default:1454 vq_err(vq, "Unknown control request %d", v_req.type);1455 continue;1456 }1457 1458 /*1459 * Validate the size of request and response buffers.1460 * Check for a sane response buffer so we can report1461 * early errors back to the guest.1462 */1463 ret = vhost_scsi_chk_size(vq, &vc);1464 if (ret)1465 goto err;1466 1467 /*1468 * Get the rest of the request now that its size is known.1469 */1470 vc.req += typ_size;1471 vc.req_size -= typ_size;1472 1473 ret = vhost_scsi_get_req(vq, &vc, &tpg);1474 if (ret)1475 goto err;1476 1477 if (v_req.type == VIRTIO_SCSI_T_TMF)1478 vhost_scsi_handle_tmf(vs, tpg, vq, &v_req.tmf, &vc);1479 else1480 vhost_scsi_send_an_resp(vs, vq, &vc);1481err:1482 /*1483 * ENXIO: No more requests, or read error, wait for next kick1484 * EINVAL: Invalid response buffer, drop the request1485 * EIO: Respond with bad target1486 * EAGAIN: Pending request1487 */1488 if (ret == -ENXIO)1489 break;1490 else if (ret == -EIO)1491 vhost_scsi_send_bad_target(vs, vq, vc.head, vc.out);1492 } while (likely(!vhost_exceeds_weight(vq, ++c, 0)));1493out:1494 mutex_unlock(&vq->mutex);1495}1496 1497static void vhost_scsi_ctl_handle_kick(struct vhost_work *work)1498{1499 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,1500 poll.work);1501 struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);1502 1503 pr_debug("%s: The handling func for control queue.\n", __func__);1504 vhost_scsi_ctl_handle_vq(vs, vq);1505}1506 1507static void1508vhost_scsi_send_evt(struct vhost_scsi *vs, struct vhost_virtqueue *vq,1509 struct vhost_scsi_tpg *tpg, struct se_lun *lun,1510 u32 event, u32 reason)1511{1512 struct vhost_scsi_evt *evt;1513 1514 evt = vhost_scsi_allocate_evt(vs, event, reason);1515 if (!evt)1516 return;1517 1518 if (tpg && lun) {1519 /* TODO: share lun setup code with virtio-scsi.ko */1520 /*1521 * Note: evt->event is zeroed when we allocate it and1522 * lun[4-7] need to be zero according to virtio-scsi spec.1523 */1524 evt->event.lun[0] = 0x01;1525 evt->event.lun[1] = tpg->tport_tpgt;1526 if (lun->unpacked_lun >= 256)1527 evt->event.lun[2] = lun->unpacked_lun >> 8 | 0x40 ;1528 evt->event.lun[3] = lun->unpacked_lun & 0xFF;1529 }1530 1531 llist_add(&evt->list, &vs->vs_event_list);1532 if (!vhost_vq_work_queue(vq, &vs->vs_event_work))1533 vhost_scsi_complete_events(vs, true);1534}1535 1536static void vhost_scsi_evt_handle_kick(struct vhost_work *work)1537{1538 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,1539 poll.work);1540 struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);1541 1542 mutex_lock(&vq->mutex);1543 if (!vhost_vq_get_backend(vq))1544 goto out;1545 1546 if (vs->vs_events_missed)1547 vhost_scsi_send_evt(vs, vq, NULL, NULL, VIRTIO_SCSI_T_NO_EVENT,1548 0);1549out:1550 mutex_unlock(&vq->mutex);1551}1552 1553static void vhost_scsi_handle_kick(struct vhost_work *work)1554{1555 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,1556 poll.work);1557 struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);1558 1559 vhost_scsi_handle_vq(vs, vq);1560}1561 1562/* Callers must hold dev mutex */1563static void vhost_scsi_flush(struct vhost_scsi *vs)1564{1565 int i;1566 1567 /* Init new inflight and remember the old inflight */1568 vhost_scsi_init_inflight(vs, vs->old_inflight);1569 1570 /*1571 * The inflight->kref was initialized to 1. We decrement it here to1572 * indicate the start of the flush operation so that it will reach 01573 * when all the reqs are finished.1574 */1575 for (i = 0; i < vs->dev.nvqs; i++)1576 kref_put(&vs->old_inflight[i]->kref, vhost_scsi_done_inflight);1577 1578 /* Flush both the vhost poll and vhost work */1579 vhost_dev_flush(&vs->dev);1580 1581 /* Wait for all reqs issued before the flush to be finished */1582 for (i = 0; i < vs->dev.nvqs; i++)1583 wait_for_completion(&vs->old_inflight[i]->comp);1584}1585 1586static void vhost_scsi_destroy_vq_cmds(struct vhost_virtqueue *vq)1587{1588 struct vhost_scsi_virtqueue *svq = container_of(vq,1589 struct vhost_scsi_virtqueue, vq);1590 struct vhost_scsi_cmd *tv_cmd;1591 unsigned int i;1592 1593 if (!svq->scsi_cmds)1594 return;1595 1596 for (i = 0; i < svq->max_cmds; i++) {1597 tv_cmd = &svq->scsi_cmds[i];1598 1599 kfree(tv_cmd->tvc_sgl);1600 kfree(tv_cmd->tvc_prot_sgl);1601 kfree(tv_cmd->tvc_upages);1602 kfree(tv_cmd->tvc_resp_iov);1603 }1604 1605 sbitmap_free(&svq->scsi_tags);1606 kfree(svq->scsi_cmds);1607 svq->scsi_cmds = NULL;1608}1609 1610static int vhost_scsi_setup_vq_cmds(struct vhost_virtqueue *vq, int max_cmds)1611{1612 struct vhost_scsi_virtqueue *svq = container_of(vq,1613 struct vhost_scsi_virtqueue, vq);1614 struct vhost_scsi_cmd *tv_cmd;1615 unsigned int i;1616 1617 if (svq->scsi_cmds)1618 return 0;1619 1620 if (sbitmap_init_node(&svq->scsi_tags, max_cmds, -1, GFP_KERNEL,1621 NUMA_NO_NODE, false, true))1622 return -ENOMEM;1623 svq->max_cmds = max_cmds;1624 1625 svq->scsi_cmds = kcalloc(max_cmds, sizeof(*tv_cmd), GFP_KERNEL);1626 if (!svq->scsi_cmds) {1627 sbitmap_free(&svq->scsi_tags);1628 return -ENOMEM;1629 }1630 1631 for (i = 0; i < max_cmds; i++) {1632 tv_cmd = &svq->scsi_cmds[i];1633 1634 tv_cmd->tvc_sgl = kcalloc(VHOST_SCSI_PREALLOC_SGLS,1635 sizeof(struct scatterlist),1636 GFP_KERNEL);1637 if (!tv_cmd->tvc_sgl) {1638 pr_err("Unable to allocate tv_cmd->tvc_sgl\n");1639 goto out;1640 }1641 1642 tv_cmd->tvc_upages = kcalloc(VHOST_SCSI_PREALLOC_UPAGES,1643 sizeof(struct page *),1644 GFP_KERNEL);1645 if (!tv_cmd->tvc_upages) {1646 pr_err("Unable to allocate tv_cmd->tvc_upages\n");1647 goto out;1648 }1649 1650 tv_cmd->tvc_resp_iov = kcalloc(UIO_MAXIOV,1651 sizeof(struct iovec),1652 GFP_KERNEL);1653 if (!tv_cmd->tvc_resp_iov) {1654 pr_err("Unable to allocate tv_cmd->tvc_resp_iov\n");1655 goto out;1656 }1657 1658 tv_cmd->tvc_prot_sgl = kcalloc(VHOST_SCSI_PREALLOC_PROT_SGLS,1659 sizeof(struct scatterlist),1660 GFP_KERNEL);1661 if (!tv_cmd->tvc_prot_sgl) {1662 pr_err("Unable to allocate tv_cmd->tvc_prot_sgl\n");1663 goto out;1664 }1665 }1666 return 0;1667out:1668 vhost_scsi_destroy_vq_cmds(vq);1669 return -ENOMEM;1670}1671 1672/*1673 * Called from vhost_scsi_ioctl() context to walk the list of available1674 * vhost_scsi_tpg with an active struct vhost_scsi_nexus1675 *1676 * The lock nesting rule is:1677 * vs->dev.mutex -> vhost_scsi_mutex -> tpg->tv_tpg_mutex -> vq->mutex1678 */1679static int1680vhost_scsi_set_endpoint(struct vhost_scsi *vs,1681 struct vhost_scsi_target *t)1682{1683 struct se_portal_group *se_tpg;1684 struct vhost_scsi_tport *tv_tport;1685 struct vhost_scsi_tpg *tpg;1686 struct vhost_scsi_tpg **vs_tpg;1687 struct vhost_virtqueue *vq;1688 int index, ret, i, len;1689 bool match = false;1690 1691 mutex_lock(&vs->dev.mutex);1692 1693 /* Verify that ring has been setup correctly. */1694 for (index = 0; index < vs->dev.nvqs; ++index) {1695 /* Verify that ring has been setup correctly. */1696 if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {1697 ret = -EFAULT;1698 goto out;1699 }1700 }1701 1702 len = sizeof(vs_tpg[0]) * VHOST_SCSI_MAX_TARGET;1703 vs_tpg = kzalloc(len, GFP_KERNEL);1704 if (!vs_tpg) {1705 ret = -ENOMEM;1706 goto out;1707 }1708 if (vs->vs_tpg)1709 memcpy(vs_tpg, vs->vs_tpg, len);1710 1711 mutex_lock(&vhost_scsi_mutex);1712 list_for_each_entry(tpg, &vhost_scsi_list, tv_tpg_list) {1713 mutex_lock(&tpg->tv_tpg_mutex);1714 if (!tpg->tpg_nexus) {1715 mutex_unlock(&tpg->tv_tpg_mutex);1716 continue;1717 }1718 if (tpg->tv_tpg_vhost_count != 0) {1719 mutex_unlock(&tpg->tv_tpg_mutex);1720 continue;1721 }1722 tv_tport = tpg->tport;1723 1724 if (!strcmp(tv_tport->tport_name, t->vhost_wwpn)) {1725 if (vs->vs_tpg && vs->vs_tpg[tpg->tport_tpgt]) {1726 mutex_unlock(&tpg->tv_tpg_mutex);1727 mutex_unlock(&vhost_scsi_mutex);1728 ret = -EEXIST;1729 goto undepend;1730 }1731 /*1732 * In order to ensure individual vhost-scsi configfs1733 * groups cannot be removed while in use by vhost ioctl,1734 * go ahead and take an explicit se_tpg->tpg_group.cg_item1735 * dependency now.1736 */1737 se_tpg = &tpg->se_tpg;1738 ret = target_depend_item(&se_tpg->tpg_group.cg_item);1739 if (ret) {1740 pr_warn("target_depend_item() failed: %d\n", ret);1741 mutex_unlock(&tpg->tv_tpg_mutex);1742 mutex_unlock(&vhost_scsi_mutex);1743 goto undepend;1744 }1745 tpg->tv_tpg_vhost_count++;1746 tpg->vhost_scsi = vs;1747 vs_tpg[tpg->tport_tpgt] = tpg;1748 match = true;1749 }1750 mutex_unlock(&tpg->tv_tpg_mutex);1751 }1752 mutex_unlock(&vhost_scsi_mutex);1753 1754 if (match) {1755 memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn,1756 sizeof(vs->vs_vhost_wwpn));1757 1758 for (i = VHOST_SCSI_VQ_IO; i < vs->dev.nvqs; i++) {1759 vq = &vs->vqs[i].vq;1760 if (!vhost_vq_is_setup(vq))1761 continue;1762 1763 ret = vhost_scsi_setup_vq_cmds(vq, vq->num);1764 if (ret)1765 goto destroy_vq_cmds;1766 }1767 1768 for (i = 0; i < vs->dev.nvqs; i++) {1769 vq = &vs->vqs[i].vq;1770 mutex_lock(&vq->mutex);1771 vhost_vq_set_backend(vq, vs_tpg);1772 vhost_vq_init_access(vq);1773 mutex_unlock(&vq->mutex);1774 }1775 ret = 0;1776 } else {1777 ret = -EEXIST;1778 }1779 1780 /*1781 * Act as synchronize_rcu to make sure access to1782 * old vs->vs_tpg is finished.1783 */1784 vhost_scsi_flush(vs);1785 kfree(vs->vs_tpg);1786 vs->vs_tpg = vs_tpg;1787 goto out;1788 1789destroy_vq_cmds:1790 for (i--; i >= VHOST_SCSI_VQ_IO; i--) {1791 if (!vhost_vq_get_backend(&vs->vqs[i].vq))1792 vhost_scsi_destroy_vq_cmds(&vs->vqs[i].vq);1793 }1794undepend:1795 for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {1796 tpg = vs_tpg[i];1797 if (tpg) {1798 mutex_lock(&tpg->tv_tpg_mutex);1799 tpg->vhost_scsi = NULL;1800 tpg->tv_tpg_vhost_count--;1801 mutex_unlock(&tpg->tv_tpg_mutex);1802 target_undepend_item(&tpg->se_tpg.tpg_group.cg_item);1803 }1804 }1805 kfree(vs_tpg);1806out:1807 mutex_unlock(&vs->dev.mutex);1808 return ret;1809}1810 1811static int1812vhost_scsi_clear_endpoint(struct vhost_scsi *vs,1813 struct vhost_scsi_target *t)1814{1815 struct se_portal_group *se_tpg;1816 struct vhost_scsi_tport *tv_tport;1817 struct vhost_scsi_tpg *tpg;1818 struct vhost_virtqueue *vq;1819 bool match = false;1820 int index, ret, i;1821 u8 target;1822 1823 mutex_lock(&vs->dev.mutex);1824 /* Verify that ring has been setup correctly. */1825 for (index = 0; index < vs->dev.nvqs; ++index) {1826 if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {1827 ret = -EFAULT;1828 goto err_dev;1829 }1830 }1831 1832 if (!vs->vs_tpg) {1833 ret = 0;1834 goto err_dev;1835 }1836 1837 for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {1838 target = i;1839 tpg = vs->vs_tpg[target];1840 if (!tpg)1841 continue;1842 1843 tv_tport = tpg->tport;1844 if (!tv_tport) {1845 ret = -ENODEV;1846 goto err_dev;1847 }1848 1849 if (strcmp(tv_tport->tport_name, t->vhost_wwpn)) {1850 pr_warn("tv_tport->tport_name: %s, tpg->tport_tpgt: %hu"1851 " does not match t->vhost_wwpn: %s, t->vhost_tpgt: %hu\n",1852 tv_tport->tport_name, tpg->tport_tpgt,1853 t->vhost_wwpn, t->vhost_tpgt);1854 ret = -EINVAL;1855 goto err_dev;1856 }1857 match = true;1858 }1859 if (!match)1860 goto free_vs_tpg;1861 1862 /* Prevent new cmds from starting and accessing the tpgs/sessions */1863 for (i = 0; i < vs->dev.nvqs; i++) {1864 vq = &vs->vqs[i].vq;1865 mutex_lock(&vq->mutex);1866 vhost_vq_set_backend(vq, NULL);1867 mutex_unlock(&vq->mutex);1868 }1869 /* Make sure cmds are not running before tearing them down. */1870 vhost_scsi_flush(vs);1871 1872 for (i = 0; i < vs->dev.nvqs; i++) {1873 vq = &vs->vqs[i].vq;1874 vhost_scsi_destroy_vq_cmds(vq);1875 }1876 1877 /*1878 * We can now release our hold on the tpg and sessions and userspace1879 * can free them after this point.1880 */1881 for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) {1882 target = i;1883 tpg = vs->vs_tpg[target];1884 if (!tpg)1885 continue;1886 1887 mutex_lock(&tpg->tv_tpg_mutex);1888 1889 tpg->tv_tpg_vhost_count--;1890 tpg->vhost_scsi = NULL;1891 vs->vs_tpg[target] = NULL;1892 1893 mutex_unlock(&tpg->tv_tpg_mutex);1894 1895 se_tpg = &tpg->se_tpg;1896 target_undepend_item(&se_tpg->tpg_group.cg_item);1897 }1898 1899free_vs_tpg:1900 /*1901 * Act as synchronize_rcu to make sure access to1902 * old vs->vs_tpg is finished.1903 */1904 vhost_scsi_flush(vs);1905 kfree(vs->vs_tpg);1906 vs->vs_tpg = NULL;1907 WARN_ON(vs->vs_events_nr);1908 mutex_unlock(&vs->dev.mutex);1909 return 0;1910 1911err_dev:1912 mutex_unlock(&vs->dev.mutex);1913 return ret;1914}1915 1916static int vhost_scsi_set_features(struct vhost_scsi *vs, u64 features)1917{1918 struct vhost_virtqueue *vq;1919 int i;1920 1921 if (features & ~VHOST_SCSI_FEATURES)1922 return -EOPNOTSUPP;1923 1924 mutex_lock(&vs->dev.mutex);1925 if ((features & (1 << VHOST_F_LOG_ALL)) &&1926 !vhost_log_access_ok(&vs->dev)) {1927 mutex_unlock(&vs->dev.mutex);1928 return -EFAULT;1929 }1930 1931 for (i = 0; i < vs->dev.nvqs; i++) {1932 vq = &vs->vqs[i].vq;1933 mutex_lock(&vq->mutex);1934 vq->acked_features = features;1935 mutex_unlock(&vq->mutex);1936 }1937 mutex_unlock(&vs->dev.mutex);1938 return 0;1939}1940 1941static int vhost_scsi_open(struct inode *inode, struct file *f)1942{1943 struct vhost_scsi_virtqueue *svq;1944 struct vhost_scsi *vs;1945 struct vhost_virtqueue **vqs;1946 int r = -ENOMEM, i, nvqs = vhost_scsi_max_io_vqs;1947 1948 vs = kvzalloc(sizeof(*vs), GFP_KERNEL);1949 if (!vs)1950 goto err_vs;1951 1952 if (nvqs > VHOST_SCSI_MAX_IO_VQ) {1953 pr_err("Invalid max_io_vqs of %d. Using %d.\n", nvqs,1954 VHOST_SCSI_MAX_IO_VQ);1955 nvqs = VHOST_SCSI_MAX_IO_VQ;1956 } else if (nvqs == 0) {1957 pr_err("Invalid max_io_vqs of %d. Using 1.\n", nvqs);1958 nvqs = 1;1959 }1960 nvqs += VHOST_SCSI_VQ_IO;1961 1962 vs->old_inflight = kmalloc_array(nvqs, sizeof(*vs->old_inflight),1963 GFP_KERNEL | __GFP_ZERO);1964 if (!vs->old_inflight)1965 goto err_inflight;1966 1967 vs->vqs = kmalloc_array(nvqs, sizeof(*vs->vqs),1968 GFP_KERNEL | __GFP_ZERO);1969 if (!vs->vqs)1970 goto err_vqs;1971 1972 vqs = kmalloc_array(nvqs, sizeof(*vqs), GFP_KERNEL);1973 if (!vqs)1974 goto err_local_vqs;1975 1976 vhost_work_init(&vs->vs_event_work, vhost_scsi_evt_work);1977 1978 vs->vs_events_nr = 0;1979 vs->vs_events_missed = false;1980 1981 vqs[VHOST_SCSI_VQ_CTL] = &vs->vqs[VHOST_SCSI_VQ_CTL].vq;1982 vqs[VHOST_SCSI_VQ_EVT] = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;1983 vs->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick;1984 vs->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick;1985 for (i = VHOST_SCSI_VQ_IO; i < nvqs; i++) {1986 svq = &vs->vqs[i];1987 1988 vqs[i] = &svq->vq;1989 svq->vs = vs;1990 init_llist_head(&svq->completion_list);1991 vhost_work_init(&svq->completion_work,1992 vhost_scsi_complete_cmd_work);1993 svq->vq.handle_kick = vhost_scsi_handle_kick;1994 }1995 vhost_dev_init(&vs->dev, vqs, nvqs, UIO_MAXIOV,1996 VHOST_SCSI_WEIGHT, 0, true, NULL);1997 1998 vhost_scsi_init_inflight(vs, NULL);1999 2000 f->private_data = vs;2001 return 0;2002 2003err_local_vqs:2004 kfree(vs->vqs);2005err_vqs:2006 kfree(vs->old_inflight);2007err_inflight:2008 kvfree(vs);2009err_vs:2010 return r;2011}2012 2013static int vhost_scsi_release(struct inode *inode, struct file *f)2014{2015 struct vhost_scsi *vs = f->private_data;2016 struct vhost_scsi_target t;2017 2018 mutex_lock(&vs->dev.mutex);2019 memcpy(t.vhost_wwpn, vs->vs_vhost_wwpn, sizeof(t.vhost_wwpn));2020 mutex_unlock(&vs->dev.mutex);2021 vhost_scsi_clear_endpoint(vs, &t);2022 vhost_dev_stop(&vs->dev);2023 vhost_dev_cleanup(&vs->dev);2024 kfree(vs->dev.vqs);2025 kfree(vs->vqs);2026 kfree(vs->old_inflight);2027 kvfree(vs);2028 return 0;2029}2030 2031static long2032vhost_scsi_ioctl(struct file *f,2033 unsigned int ioctl,2034 unsigned long arg)2035{2036 struct vhost_scsi *vs = f->private_data;2037 struct vhost_scsi_target backend;2038 void __user *argp = (void __user *)arg;2039 u64 __user *featurep = argp;2040 u32 __user *eventsp = argp;2041 u32 events_missed;2042 u64 features;2043 int r, abi_version = VHOST_SCSI_ABI_VERSION;2044 struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;2045 2046 switch (ioctl) {2047 case VHOST_SCSI_SET_ENDPOINT:2048 if (copy_from_user(&backend, argp, sizeof backend))2049 return -EFAULT;2050 if (backend.reserved != 0)2051 return -EOPNOTSUPP;2052 2053 return vhost_scsi_set_endpoint(vs, &backend);2054 case VHOST_SCSI_CLEAR_ENDPOINT:2055 if (copy_from_user(&backend, argp, sizeof backend))2056 return -EFAULT;2057 if (backend.reserved != 0)2058 return -EOPNOTSUPP;2059 2060 return vhost_scsi_clear_endpoint(vs, &backend);2061 case VHOST_SCSI_GET_ABI_VERSION:2062 if (copy_to_user(argp, &abi_version, sizeof abi_version))2063 return -EFAULT;2064 return 0;2065 case VHOST_SCSI_SET_EVENTS_MISSED:2066 if (get_user(events_missed, eventsp))2067 return -EFAULT;2068 mutex_lock(&vq->mutex);2069 vs->vs_events_missed = events_missed;2070 mutex_unlock(&vq->mutex);2071 return 0;2072 case VHOST_SCSI_GET_EVENTS_MISSED:2073 mutex_lock(&vq->mutex);2074 events_missed = vs->vs_events_missed;2075 mutex_unlock(&vq->mutex);2076 if (put_user(events_missed, eventsp))2077 return -EFAULT;2078 return 0;2079 case VHOST_GET_FEATURES:2080 features = VHOST_SCSI_FEATURES;2081 if (copy_to_user(featurep, &features, sizeof features))2082 return -EFAULT;2083 return 0;2084 case VHOST_SET_FEATURES:2085 if (copy_from_user(&features, featurep, sizeof features))2086 return -EFAULT;2087 return vhost_scsi_set_features(vs, features);2088 case VHOST_NEW_WORKER:2089 case VHOST_FREE_WORKER:2090 case VHOST_ATTACH_VRING_WORKER:2091 case VHOST_GET_VRING_WORKER:2092 mutex_lock(&vs->dev.mutex);2093 r = vhost_worker_ioctl(&vs->dev, ioctl, argp);2094 mutex_unlock(&vs->dev.mutex);2095 return r;2096 default:2097 mutex_lock(&vs->dev.mutex);2098 r = vhost_dev_ioctl(&vs->dev, ioctl, argp);2099 /* TODO: flush backend after dev ioctl. */2100 if (r == -ENOIOCTLCMD)2101 r = vhost_vring_ioctl(&vs->dev, ioctl, argp);2102 mutex_unlock(&vs->dev.mutex);2103 return r;2104 }2105}2106 2107static const struct file_operations vhost_scsi_fops = {2108 .owner = THIS_MODULE,2109 .release = vhost_scsi_release,2110 .unlocked_ioctl = vhost_scsi_ioctl,2111 .compat_ioctl = compat_ptr_ioctl,2112 .open = vhost_scsi_open,2113 .llseek = noop_llseek,2114};2115 2116static struct miscdevice vhost_scsi_misc = {2117 MISC_DYNAMIC_MINOR,2118 "vhost-scsi",2119 &vhost_scsi_fops,2120};2121 2122static int __init vhost_scsi_register(void)2123{2124 return misc_register(&vhost_scsi_misc);2125}2126 2127static void vhost_scsi_deregister(void)2128{2129 misc_deregister(&vhost_scsi_misc);2130}2131 2132static char *vhost_scsi_dump_proto_id(struct vhost_scsi_tport *tport)2133{2134 switch (tport->tport_proto_id) {2135 case SCSI_PROTOCOL_SAS:2136 return "SAS";2137 case SCSI_PROTOCOL_FCP:2138 return "FCP";2139 case SCSI_PROTOCOL_ISCSI:2140 return "iSCSI";2141 default:2142 break;2143 }2144 2145 return "Unknown";2146}2147 2148static void2149vhost_scsi_do_plug(struct vhost_scsi_tpg *tpg,2150 struct se_lun *lun, bool plug)2151{2152 2153 struct vhost_scsi *vs = tpg->vhost_scsi;2154 struct vhost_virtqueue *vq;2155 u32 reason;2156 2157 if (!vs)2158 return;2159 2160 if (plug)2161 reason = VIRTIO_SCSI_EVT_RESET_RESCAN;2162 else2163 reason = VIRTIO_SCSI_EVT_RESET_REMOVED;2164 2165 vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;2166 mutex_lock(&vq->mutex);2167 /*2168 * We can't queue events if the backend has been cleared, because2169 * we could end up queueing an event after the flush.2170 */2171 if (!vhost_vq_get_backend(vq))2172 goto unlock;2173 2174 if (vhost_has_feature(vq, VIRTIO_SCSI_F_HOTPLUG))2175 vhost_scsi_send_evt(vs, vq, tpg, lun,2176 VIRTIO_SCSI_T_TRANSPORT_RESET, reason);2177unlock:2178 mutex_unlock(&vq->mutex);2179}2180 2181static void vhost_scsi_hotplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun)2182{2183 vhost_scsi_do_plug(tpg, lun, true);2184}2185 2186static void vhost_scsi_hotunplug(struct vhost_scsi_tpg *tpg, struct se_lun *lun)2187{2188 vhost_scsi_do_plug(tpg, lun, false);2189}2190 2191static int vhost_scsi_port_link(struct se_portal_group *se_tpg,2192 struct se_lun *lun)2193{2194 struct vhost_scsi_tpg *tpg = container_of(se_tpg,2195 struct vhost_scsi_tpg, se_tpg);2196 2197 mutex_lock(&tpg->tv_tpg_mutex);2198 tpg->tv_tpg_port_count++;2199 vhost_scsi_hotplug(tpg, lun);2200 mutex_unlock(&tpg->tv_tpg_mutex);2201 2202 return 0;2203}2204 2205static void vhost_scsi_port_unlink(struct se_portal_group *se_tpg,2206 struct se_lun *lun)2207{2208 struct vhost_scsi_tpg *tpg = container_of(se_tpg,2209 struct vhost_scsi_tpg, se_tpg);2210 2211 mutex_lock(&tpg->tv_tpg_mutex);2212 tpg->tv_tpg_port_count--;2213 vhost_scsi_hotunplug(tpg, lun);2214 mutex_unlock(&tpg->tv_tpg_mutex);2215}2216 2217static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_store(2218 struct config_item *item, const char *page, size_t count)2219{2220 struct se_portal_group *se_tpg = attrib_to_tpg(item);2221 struct vhost_scsi_tpg *tpg = container_of(se_tpg,2222 struct vhost_scsi_tpg, se_tpg);2223 unsigned long val;2224 int ret = kstrtoul(page, 0, &val);2225 2226 if (ret) {2227 pr_err("kstrtoul() returned %d for fabric_prot_type\n", ret);2228 return ret;2229 }2230 if (val != 0 && val != 1 && val != 3) {2231 pr_err("Invalid vhost_scsi fabric_prot_type: %lu\n", val);2232 return -EINVAL;2233 }2234 tpg->tv_fabric_prot_type = val;2235 2236 return count;2237}2238 2239static ssize_t vhost_scsi_tpg_attrib_fabric_prot_type_show(2240 struct config_item *item, char *page)2241{2242 struct se_portal_group *se_tpg = attrib_to_tpg(item);2243 struct vhost_scsi_tpg *tpg = container_of(se_tpg,2244 struct vhost_scsi_tpg, se_tpg);2245 2246 return sysfs_emit(page, "%d\n", tpg->tv_fabric_prot_type);2247}2248 2249CONFIGFS_ATTR(vhost_scsi_tpg_attrib_, fabric_prot_type);2250 2251static struct configfs_attribute *vhost_scsi_tpg_attrib_attrs[] = {2252 &vhost_scsi_tpg_attrib_attr_fabric_prot_type,2253 NULL,2254};2255 2256static int vhost_scsi_make_nexus(struct vhost_scsi_tpg *tpg,2257 const char *name)2258{2259 struct vhost_scsi_nexus *tv_nexus;2260 2261 mutex_lock(&tpg->tv_tpg_mutex);2262 if (tpg->tpg_nexus) {2263 mutex_unlock(&tpg->tv_tpg_mutex);2264 pr_debug("tpg->tpg_nexus already exists\n");2265 return -EEXIST;2266 }2267 2268 tv_nexus = kzalloc(sizeof(*tv_nexus), GFP_KERNEL);2269 if (!tv_nexus) {2270 mutex_unlock(&tpg->tv_tpg_mutex);2271 pr_err("Unable to allocate struct vhost_scsi_nexus\n");2272 return -ENOMEM;2273 }2274 /*2275 * Since we are running in 'demo mode' this call with generate a2276 * struct se_node_acl for the vhost_scsi struct se_portal_group with2277 * the SCSI Initiator port name of the passed configfs group 'name'.2278 */2279 tv_nexus->tvn_se_sess = target_setup_session(&tpg->se_tpg, 0, 0,2280 TARGET_PROT_DIN_PASS | TARGET_PROT_DOUT_PASS,2281 (unsigned char *)name, tv_nexus, NULL);2282 if (IS_ERR(tv_nexus->tvn_se_sess)) {2283 mutex_unlock(&tpg->tv_tpg_mutex);2284 kfree(tv_nexus);2285 return -ENOMEM;2286 }2287 tpg->tpg_nexus = tv_nexus;2288 2289 mutex_unlock(&tpg->tv_tpg_mutex);2290 return 0;2291}2292 2293static int vhost_scsi_drop_nexus(struct vhost_scsi_tpg *tpg)2294{2295 struct se_session *se_sess;2296 struct vhost_scsi_nexus *tv_nexus;2297 2298 mutex_lock(&tpg->tv_tpg_mutex);2299 tv_nexus = tpg->tpg_nexus;2300 if (!tv_nexus) {2301 mutex_unlock(&tpg->tv_tpg_mutex);2302 return -ENODEV;2303 }2304 2305 se_sess = tv_nexus->tvn_se_sess;2306 if (!se_sess) {2307 mutex_unlock(&tpg->tv_tpg_mutex);2308 return -ENODEV;2309 }2310 2311 if (tpg->tv_tpg_port_count != 0) {2312 mutex_unlock(&tpg->tv_tpg_mutex);2313 pr_err("Unable to remove TCM_vhost I_T Nexus with"2314 " active TPG port count: %d\n",2315 tpg->tv_tpg_port_count);2316 return -EBUSY;2317 }2318 2319 if (tpg->tv_tpg_vhost_count != 0) {2320 mutex_unlock(&tpg->tv_tpg_mutex);2321 pr_err("Unable to remove TCM_vhost I_T Nexus with"2322 " active TPG vhost count: %d\n",2323 tpg->tv_tpg_vhost_count);2324 return -EBUSY;2325 }2326 2327 pr_debug("TCM_vhost_ConfigFS: Removing I_T Nexus to emulated"2328 " %s Initiator Port: %s\n", vhost_scsi_dump_proto_id(tpg->tport),2329 tv_nexus->tvn_se_sess->se_node_acl->initiatorname);2330 2331 /*2332 * Release the SCSI I_T Nexus to the emulated vhost Target Port2333 */2334 target_remove_session(se_sess);2335 tpg->tpg_nexus = NULL;2336 mutex_unlock(&tpg->tv_tpg_mutex);2337 2338 kfree(tv_nexus);2339 return 0;2340}2341 2342static ssize_t vhost_scsi_tpg_nexus_show(struct config_item *item, char *page)2343{2344 struct se_portal_group *se_tpg = to_tpg(item);2345 struct vhost_scsi_tpg *tpg = container_of(se_tpg,2346 struct vhost_scsi_tpg, se_tpg);2347 struct vhost_scsi_nexus *tv_nexus;2348 ssize_t ret;2349 2350 mutex_lock(&tpg->tv_tpg_mutex);2351 tv_nexus = tpg->tpg_nexus;2352 if (!tv_nexus) {2353 mutex_unlock(&tpg->tv_tpg_mutex);2354 return -ENODEV;2355 }2356 ret = sysfs_emit(page, "%s\n",2357 tv_nexus->tvn_se_sess->se_node_acl->initiatorname);2358 mutex_unlock(&tpg->tv_tpg_mutex);2359 2360 return ret;2361}2362 2363static ssize_t vhost_scsi_tpg_nexus_store(struct config_item *item,2364 const char *page, size_t count)2365{2366 struct se_portal_group *se_tpg = to_tpg(item);2367 struct vhost_scsi_tpg *tpg = container_of(se_tpg,2368 struct vhost_scsi_tpg, se_tpg);2369 struct vhost_scsi_tport *tport_wwn = tpg->tport;2370 unsigned char i_port[VHOST_SCSI_NAMELEN], *ptr, *port_ptr;2371 int ret;2372 /*2373 * Shutdown the active I_T nexus if 'NULL' is passed..2374 */2375 if (!strncmp(page, "NULL", 4)) {2376 ret = vhost_scsi_drop_nexus(tpg);2377 return (!ret) ? count : ret;2378 }2379 /*2380 * Otherwise make sure the passed virtual Initiator port WWN matches2381 * the fabric protocol_id set in vhost_scsi_make_tport(), and call2382 * vhost_scsi_make_nexus().2383 */2384 if (strlen(page) >= VHOST_SCSI_NAMELEN) {2385 pr_err("Emulated NAA Sas Address: %s, exceeds"2386 " max: %d\n", page, VHOST_SCSI_NAMELEN);2387 return -EINVAL;2388 }2389 snprintf(&i_port[0], VHOST_SCSI_NAMELEN, "%s", page);2390 2391 ptr = strstr(i_port, "naa.");2392 if (ptr) {2393 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_SAS) {2394 pr_err("Passed SAS Initiator Port %s does not"2395 " match target port protoid: %s\n", i_port,2396 vhost_scsi_dump_proto_id(tport_wwn));2397 return -EINVAL;2398 }2399 port_ptr = &i_port[0];2400 goto check_newline;2401 }2402 ptr = strstr(i_port, "fc.");2403 if (ptr) {2404 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_FCP) {2405 pr_err("Passed FCP Initiator Port %s does not"2406 " match target port protoid: %s\n", i_port,2407 vhost_scsi_dump_proto_id(tport_wwn));2408 return -EINVAL;2409 }2410 port_ptr = &i_port[3]; /* Skip over "fc." */2411 goto check_newline;2412 }2413 ptr = strstr(i_port, "iqn.");2414 if (ptr) {2415 if (tport_wwn->tport_proto_id != SCSI_PROTOCOL_ISCSI) {2416 pr_err("Passed iSCSI Initiator Port %s does not"2417 " match target port protoid: %s\n", i_port,2418 vhost_scsi_dump_proto_id(tport_wwn));2419 return -EINVAL;2420 }2421 port_ptr = &i_port[0];2422 goto check_newline;2423 }2424 pr_err("Unable to locate prefix for emulated Initiator Port:"2425 " %s\n", i_port);2426 return -EINVAL;2427 /*2428 * Clear any trailing newline for the NAA WWN2429 */2430check_newline:2431 if (i_port[strlen(i_port)-1] == '\n')2432 i_port[strlen(i_port)-1] = '\0';2433 2434 ret = vhost_scsi_make_nexus(tpg, port_ptr);2435 if (ret < 0)2436 return ret;2437 2438 return count;2439}2440 2441CONFIGFS_ATTR(vhost_scsi_tpg_, nexus);2442 2443static struct configfs_attribute *vhost_scsi_tpg_attrs[] = {2444 &vhost_scsi_tpg_attr_nexus,2445 NULL,2446};2447 2448static struct se_portal_group *2449vhost_scsi_make_tpg(struct se_wwn *wwn, const char *name)2450{2451 struct vhost_scsi_tport *tport = container_of(wwn,2452 struct vhost_scsi_tport, tport_wwn);2453 2454 struct vhost_scsi_tpg *tpg;2455 u16 tpgt;2456 int ret;2457 2458 if (strstr(name, "tpgt_") != name)2459 return ERR_PTR(-EINVAL);2460 if (kstrtou16(name + 5, 10, &tpgt) || tpgt >= VHOST_SCSI_MAX_TARGET)2461 return ERR_PTR(-EINVAL);2462 2463 tpg = kzalloc(sizeof(*tpg), GFP_KERNEL);2464 if (!tpg) {2465 pr_err("Unable to allocate struct vhost_scsi_tpg");2466 return ERR_PTR(-ENOMEM);2467 }2468 mutex_init(&tpg->tv_tpg_mutex);2469 INIT_LIST_HEAD(&tpg->tv_tpg_list);2470 tpg->tport = tport;2471 tpg->tport_tpgt = tpgt;2472 2473 ret = core_tpg_register(wwn, &tpg->se_tpg, tport->tport_proto_id);2474 if (ret < 0) {2475 kfree(tpg);2476 return NULL;2477 }2478 mutex_lock(&vhost_scsi_mutex);2479 list_add_tail(&tpg->tv_tpg_list, &vhost_scsi_list);2480 mutex_unlock(&vhost_scsi_mutex);2481 2482 return &tpg->se_tpg;2483}2484 2485static void vhost_scsi_drop_tpg(struct se_portal_group *se_tpg)2486{2487 struct vhost_scsi_tpg *tpg = container_of(se_tpg,2488 struct vhost_scsi_tpg, se_tpg);2489 2490 mutex_lock(&vhost_scsi_mutex);2491 list_del(&tpg->tv_tpg_list);2492 mutex_unlock(&vhost_scsi_mutex);2493 /*2494 * Release the virtual I_T Nexus for this vhost TPG2495 */2496 vhost_scsi_drop_nexus(tpg);2497 /*2498 * Deregister the se_tpg from TCM..2499 */2500 core_tpg_deregister(se_tpg);2501 kfree(tpg);2502}2503 2504static struct se_wwn *2505vhost_scsi_make_tport(struct target_fabric_configfs *tf,2506 struct config_group *group,2507 const char *name)2508{2509 struct vhost_scsi_tport *tport;2510 char *ptr;2511 u64 wwpn = 0;2512 int off = 0;2513 2514 /* if (vhost_scsi_parse_wwn(name, &wwpn, 1) < 0)2515 return ERR_PTR(-EINVAL); */2516 2517 tport = kzalloc(sizeof(*tport), GFP_KERNEL);2518 if (!tport) {2519 pr_err("Unable to allocate struct vhost_scsi_tport");2520 return ERR_PTR(-ENOMEM);2521 }2522 tport->tport_wwpn = wwpn;2523 /*2524 * Determine the emulated Protocol Identifier and Target Port Name2525 * based on the incoming configfs directory name.2526 */2527 ptr = strstr(name, "naa.");2528 if (ptr) {2529 tport->tport_proto_id = SCSI_PROTOCOL_SAS;2530 goto check_len;2531 }2532 ptr = strstr(name, "fc.");2533 if (ptr) {2534 tport->tport_proto_id = SCSI_PROTOCOL_FCP;2535 off = 3; /* Skip over "fc." */2536 goto check_len;2537 }2538 ptr = strstr(name, "iqn.");2539 if (ptr) {2540 tport->tport_proto_id = SCSI_PROTOCOL_ISCSI;2541 goto check_len;2542 }2543 2544 pr_err("Unable to locate prefix for emulated Target Port:"2545 " %s\n", name);2546 kfree(tport);2547 return ERR_PTR(-EINVAL);2548 2549check_len:2550 if (strlen(name) >= VHOST_SCSI_NAMELEN) {2551 pr_err("Emulated %s Address: %s, exceeds"2552 " max: %d\n", name, vhost_scsi_dump_proto_id(tport),2553 VHOST_SCSI_NAMELEN);2554 kfree(tport);2555 return ERR_PTR(-EINVAL);2556 }2557 snprintf(&tport->tport_name[0], VHOST_SCSI_NAMELEN, "%s", &name[off]);2558 2559 pr_debug("TCM_VHost_ConfigFS: Allocated emulated Target"2560 " %s Address: %s\n", vhost_scsi_dump_proto_id(tport), name);2561 2562 return &tport->tport_wwn;2563}2564 2565static void vhost_scsi_drop_tport(struct se_wwn *wwn)2566{2567 struct vhost_scsi_tport *tport = container_of(wwn,2568 struct vhost_scsi_tport, tport_wwn);2569 2570 pr_debug("TCM_VHost_ConfigFS: Deallocating emulated Target"2571 " %s Address: %s\n", vhost_scsi_dump_proto_id(tport),2572 tport->tport_name);2573 2574 kfree(tport);2575}2576 2577static ssize_t2578vhost_scsi_wwn_version_show(struct config_item *item, char *page)2579{2580 return sysfs_emit(page, "TCM_VHOST fabric module %s on %s/%s"2581 "on "UTS_RELEASE"\n", VHOST_SCSI_VERSION, utsname()->sysname,2582 utsname()->machine);2583}2584 2585CONFIGFS_ATTR_RO(vhost_scsi_wwn_, version);2586 2587static struct configfs_attribute *vhost_scsi_wwn_attrs[] = {2588 &vhost_scsi_wwn_attr_version,2589 NULL,2590};2591 2592static const struct target_core_fabric_ops vhost_scsi_ops = {2593 .module = THIS_MODULE,2594 .fabric_name = "vhost",2595 .max_data_sg_nents = VHOST_SCSI_PREALLOC_SGLS,2596 .tpg_get_wwn = vhost_scsi_get_fabric_wwn,2597 .tpg_get_tag = vhost_scsi_get_tpgt,2598 .tpg_check_demo_mode = vhost_scsi_check_true,2599 .tpg_check_demo_mode_cache = vhost_scsi_check_true,2600 .tpg_check_prot_fabric_only = vhost_scsi_check_prot_fabric_only,2601 .release_cmd = vhost_scsi_release_cmd,2602 .check_stop_free = vhost_scsi_check_stop_free,2603 .sess_get_initiator_sid = NULL,2604 .write_pending = vhost_scsi_write_pending,2605 .queue_data_in = vhost_scsi_queue_data_in,2606 .queue_status = vhost_scsi_queue_status,2607 .queue_tm_rsp = vhost_scsi_queue_tm_rsp,2608 .aborted_task = vhost_scsi_aborted_task,2609 /*2610 * Setup callers for generic logic in target_core_fabric_configfs.c2611 */2612 .fabric_make_wwn = vhost_scsi_make_tport,2613 .fabric_drop_wwn = vhost_scsi_drop_tport,2614 .fabric_make_tpg = vhost_scsi_make_tpg,2615 .fabric_drop_tpg = vhost_scsi_drop_tpg,2616 .fabric_post_link = vhost_scsi_port_link,2617 .fabric_pre_unlink = vhost_scsi_port_unlink,2618 2619 .tfc_wwn_attrs = vhost_scsi_wwn_attrs,2620 .tfc_tpg_base_attrs = vhost_scsi_tpg_attrs,2621 .tfc_tpg_attrib_attrs = vhost_scsi_tpg_attrib_attrs,2622 2623 .default_submit_type = TARGET_QUEUE_SUBMIT,2624 .direct_submit_supp = 1,2625};2626 2627static int __init vhost_scsi_init(void)2628{2629 int ret = -ENOMEM;2630 2631 pr_debug("TCM_VHOST fabric module %s on %s/%s"2632 " on "UTS_RELEASE"\n", VHOST_SCSI_VERSION, utsname()->sysname,2633 utsname()->machine);2634 2635 ret = vhost_scsi_register();2636 if (ret < 0)2637 goto out;2638 2639 ret = target_register_template(&vhost_scsi_ops);2640 if (ret < 0)2641 goto out_vhost_scsi_deregister;2642 2643 return 0;2644 2645out_vhost_scsi_deregister:2646 vhost_scsi_deregister();2647out:2648 return ret;2649};2650 2651static void vhost_scsi_exit(void)2652{2653 target_unregister_template(&vhost_scsi_ops);2654 vhost_scsi_deregister();2655};2656 2657MODULE_DESCRIPTION("VHOST_SCSI series fabric driver");2658MODULE_ALIAS("tcm_vhost");2659MODULE_LICENSE("GPL");2660module_init(vhost_scsi_init);2661module_exit(vhost_scsi_exit);2662