1158 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright (C) 2021 Broadcom. All Rights Reserved. The term4 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.5 */6 7#include "efct_driver.h"8#include "efct_hw.h"9 10#define enable_tsend_auto_resp(efct) 111#define enable_treceive_auto_resp(efct) 012 13#define SCSI_IOFMT "[%04x][i:%04x t:%04x h:%04x]"14 15#define scsi_io_printf(io, fmt, ...) \16 efc_log_debug(io->efct, "[%s]" SCSI_IOFMT fmt, \17 io->node->display_name, io->instance_index,\18 io->init_task_tag, io->tgt_task_tag, io->hw_tag, ##__VA_ARGS__)19 20#define EFCT_LOG_ENABLE_SCSI_TRACE(efct) \21 (((efct) != NULL) ? (((efct)->logmask & (1U << 2)) != 0) : 0)22 23#define scsi_io_trace(io, fmt, ...) \24 do { \25 if (EFCT_LOG_ENABLE_SCSI_TRACE(io->efct)) \26 scsi_io_printf(io, fmt, ##__VA_ARGS__); \27 } while (0)28 29struct efct_io *30efct_scsi_io_alloc(struct efct_node *node)31{32 struct efct *efct;33 struct efct_xport *xport;34 struct efct_io *io;35 unsigned long flags;36 37 efct = node->efct;38 39 xport = efct->xport;40 41 io = efct_io_pool_io_alloc(efct->xport->io_pool);42 if (!io) {43 efc_log_err(efct, "IO alloc Failed\n");44 atomic_add_return(1, &xport->io_alloc_failed_count);45 return NULL;46 }47 48 /* initialize refcount */49 kref_init(&io->ref);50 io->release = _efct_scsi_io_free;51 52 /* set generic fields */53 io->efct = efct;54 io->node = node;55 kref_get(&node->ref);56 57 /* set type and name */58 io->io_type = EFCT_IO_TYPE_IO;59 io->display_name = "scsi_io";60 61 io->cmd_ini = false;62 io->cmd_tgt = true;63 64 /* Add to node's active_ios list */65 INIT_LIST_HEAD(&io->list_entry);66 spin_lock_irqsave(&node->active_ios_lock, flags);67 list_add(&io->list_entry, &node->active_ios);68 69 spin_unlock_irqrestore(&node->active_ios_lock, flags);70 71 return io;72}73 74void75_efct_scsi_io_free(struct kref *arg)76{77 struct efct_io *io = container_of(arg, struct efct_io, ref);78 struct efct *efct = io->efct;79 struct efct_node *node = io->node;80 unsigned long flags = 0;81 82 scsi_io_trace(io, "freeing io 0x%p %s\n", io, io->display_name);83 84 if (io->io_free) {85 efc_log_err(efct, "IO already freed.\n");86 return;87 }88 89 spin_lock_irqsave(&node->active_ios_lock, flags);90 list_del_init(&io->list_entry);91 spin_unlock_irqrestore(&node->active_ios_lock, flags);92 93 kref_put(&node->ref, node->release);94 io->node = NULL;95 efct_io_pool_io_free(efct->xport->io_pool, io);96}97 98void99efct_scsi_io_free(struct efct_io *io)100{101 scsi_io_trace(io, "freeing io 0x%p %s\n", io, io->display_name);102 WARN_ON(!refcount_read(&io->ref.refcount));103 kref_put(&io->ref, io->release);104}105 106static void107efct_target_io_cb(struct efct_hw_io *hio, u32 length, int status,108 u32 ext_status, void *app)109{110 u32 flags = 0;111 struct efct_io *io = app;112 struct efct *efct;113 enum efct_scsi_io_status scsi_stat = EFCT_SCSI_STATUS_GOOD;114 efct_scsi_io_cb_t cb;115 116 if (!io || !io->efct) {117 pr_err("%s: IO can not be NULL\n", __func__);118 return;119 }120 121 scsi_io_trace(io, "status x%x ext_status x%x\n", status, ext_status);122 123 efct = io->efct;124 125 io->transferred += length;126 127 if (!io->scsi_tgt_cb) {128 efct_scsi_check_pending(efct);129 return;130 }131 132 /* Call target server completion */133 cb = io->scsi_tgt_cb;134 135 /* Clear the callback before invoking the callback */136 io->scsi_tgt_cb = NULL;137 138 /* if status was good, and auto-good-response was set,139 * then callback target-server with IO_CMPL_RSP_SENT,140 * otherwise send IO_CMPL141 */142 if (status == 0 && io->auto_resp)143 flags |= EFCT_SCSI_IO_CMPL_RSP_SENT;144 else145 flags |= EFCT_SCSI_IO_CMPL;146 147 switch (status) {148 case SLI4_FC_WCQE_STATUS_SUCCESS:149 scsi_stat = EFCT_SCSI_STATUS_GOOD;150 break;151 case SLI4_FC_WCQE_STATUS_DI_ERROR:152 if (ext_status & SLI4_FC_DI_ERROR_GE)153 scsi_stat = EFCT_SCSI_STATUS_DIF_GUARD_ERR;154 else if (ext_status & SLI4_FC_DI_ERROR_AE)155 scsi_stat = EFCT_SCSI_STATUS_DIF_APP_TAG_ERROR;156 else if (ext_status & SLI4_FC_DI_ERROR_RE)157 scsi_stat = EFCT_SCSI_STATUS_DIF_REF_TAG_ERROR;158 else159 scsi_stat = EFCT_SCSI_STATUS_DIF_UNKNOWN_ERROR;160 break;161 case SLI4_FC_WCQE_STATUS_LOCAL_REJECT:162 switch (ext_status) {163 case SLI4_FC_LOCAL_REJECT_INVALID_RELOFFSET:164 case SLI4_FC_LOCAL_REJECT_ABORT_REQUESTED:165 scsi_stat = EFCT_SCSI_STATUS_ABORTED;166 break;167 case SLI4_FC_LOCAL_REJECT_INVALID_RPI:168 scsi_stat = EFCT_SCSI_STATUS_NEXUS_LOST;169 break;170 case SLI4_FC_LOCAL_REJECT_NO_XRI:171 scsi_stat = EFCT_SCSI_STATUS_NO_IO;172 break;173 default:174 /*we have seen 0x0d(TX_DMA_FAILED err)*/175 scsi_stat = EFCT_SCSI_STATUS_ERROR;176 break;177 }178 break;179 180 case SLI4_FC_WCQE_STATUS_TARGET_WQE_TIMEOUT:181 /* target IO timed out */182 scsi_stat = EFCT_SCSI_STATUS_TIMEDOUT_AND_ABORTED;183 break;184 185 case SLI4_FC_WCQE_STATUS_SHUTDOWN:186 /* Target IO cancelled by HW */187 scsi_stat = EFCT_SCSI_STATUS_SHUTDOWN;188 break;189 190 default:191 scsi_stat = EFCT_SCSI_STATUS_ERROR;192 break;193 }194 195 cb(io, scsi_stat, flags, io->scsi_tgt_cb_arg);196 197 efct_scsi_check_pending(efct);198}199 200static int201efct_scsi_build_sgls(struct efct_hw *hw, struct efct_hw_io *hio,202 struct efct_scsi_sgl *sgl, u32 sgl_count,203 enum efct_hw_io_type type)204{205 int rc;206 u32 i;207 struct efct *efct = hw->os;208 209 /* Initialize HW SGL */210 rc = efct_hw_io_init_sges(hw, hio, type);211 if (rc) {212 efc_log_err(efct, "efct_hw_io_init_sges failed: %d\n", rc);213 return -EIO;214 }215 216 for (i = 0; i < sgl_count; i++) {217 /* Add data SGE */218 rc = efct_hw_io_add_sge(hw, hio, sgl[i].addr, sgl[i].len);219 if (rc) {220 efc_log_err(efct, "add sge failed cnt=%d rc=%d\n",221 sgl_count, rc);222 return rc;223 }224 }225 226 return 0;227}228 229static void efc_log_sgl(struct efct_io *io)230{231 struct efct_hw_io *hio = io->hio;232 struct sli4_sge *data = NULL;233 u32 *dword = NULL;234 u32 i;235 u32 n_sge;236 237 scsi_io_trace(io, "def_sgl at 0x%x 0x%08x\n",238 upper_32_bits(hio->def_sgl.phys),239 lower_32_bits(hio->def_sgl.phys));240 n_sge = (hio->sgl == &hio->def_sgl) ? hio->n_sge : hio->def_sgl_count;241 for (i = 0, data = hio->def_sgl.virt; i < n_sge; i++, data++) {242 dword = (u32 *)data;243 244 scsi_io_trace(io, "SGL %2d 0x%08x 0x%08x 0x%08x 0x%08x\n",245 i, dword[0], dword[1], dword[2], dword[3]);246 247 if (dword[2] & (1U << 31))248 break;249 }250}251 252static void253efct_scsi_check_pending_async_cb(struct efct_hw *hw, int status,254 u8 *mqe, void *arg)255{256 struct efct_io *io = arg;257 258 if (io) {259 efct_hw_done_t cb = io->hw_cb;260 261 if (!io->hw_cb)262 return;263 264 io->hw_cb = NULL;265 (cb)(io->hio, 0, SLI4_FC_WCQE_STATUS_DISPATCH_ERROR, 0, io);266 }267}268 269static int270efct_scsi_io_dispatch_hw_io(struct efct_io *io, struct efct_hw_io *hio)271{272 int rc = 0;273 struct efct *efct = io->efct;274 275 /* Got a HW IO;276 * update ini/tgt_task_tag with HW IO info and dispatch277 */278 io->hio = hio;279 if (io->cmd_tgt)280 io->tgt_task_tag = hio->indicator;281 else if (io->cmd_ini)282 io->init_task_tag = hio->indicator;283 io->hw_tag = hio->reqtag;284 285 hio->eq = io->hw_priv;286 287 /* Copy WQ steering */288 switch (io->wq_steering) {289 case EFCT_SCSI_WQ_STEERING_CLASS >> EFCT_SCSI_WQ_STEERING_SHIFT:290 hio->wq_steering = EFCT_HW_WQ_STEERING_CLASS;291 break;292 case EFCT_SCSI_WQ_STEERING_REQUEST >> EFCT_SCSI_WQ_STEERING_SHIFT:293 hio->wq_steering = EFCT_HW_WQ_STEERING_REQUEST;294 break;295 case EFCT_SCSI_WQ_STEERING_CPU >> EFCT_SCSI_WQ_STEERING_SHIFT:296 hio->wq_steering = EFCT_HW_WQ_STEERING_CPU;297 break;298 }299 300 switch (io->io_type) {301 case EFCT_IO_TYPE_IO:302 rc = efct_scsi_build_sgls(&efct->hw, io->hio,303 io->sgl, io->sgl_count, io->hio_type);304 if (rc)305 break;306 307 if (EFCT_LOG_ENABLE_SCSI_TRACE(efct))308 efc_log_sgl(io);309 310 if (io->app_id)311 io->iparam.fcp_tgt.app_id = io->app_id;312 313 io->iparam.fcp_tgt.vpi = io->node->vpi;314 io->iparam.fcp_tgt.rpi = io->node->rpi;315 io->iparam.fcp_tgt.s_id = io->node->port_fc_id;316 io->iparam.fcp_tgt.d_id = io->node->node_fc_id;317 io->iparam.fcp_tgt.xmit_len = io->wire_len;318 319 rc = efct_hw_io_send(&io->efct->hw, io->hio_type, io->hio,320 &io->iparam, io->hw_cb, io);321 break;322 default:323 scsi_io_printf(io, "Unknown IO type=%d\n", io->io_type);324 rc = -EIO;325 break;326 }327 return rc;328}329 330static int331efct_scsi_io_dispatch_no_hw_io(struct efct_io *io)332{333 int rc;334 335 switch (io->io_type) {336 case EFCT_IO_TYPE_ABORT: {337 struct efct_hw_io *hio_to_abort = NULL;338 339 hio_to_abort = io->io_to_abort->hio;340 341 if (!hio_to_abort) {342 /*343 * If "IO to abort" does not have an344 * associated HW IO, immediately make callback with345 * success. The command must have been sent to346 * the backend, but the data phase has not yet347 * started, so we don't have a HW IO.348 *349 * Note: since the backend shims should be350 * taking a reference on io_to_abort, it should not351 * be possible to have been completed and freed by352 * the backend before the abort got here.353 */354 scsi_io_printf(io, "IO: not active\n");355 ((efct_hw_done_t)io->hw_cb)(io->hio, 0,356 SLI4_FC_WCQE_STATUS_SUCCESS, 0, io);357 rc = 0;358 break;359 }360 361 /* HW IO is valid, abort it */362 scsi_io_printf(io, "aborting\n");363 rc = efct_hw_io_abort(&io->efct->hw, hio_to_abort,364 io->send_abts, io->hw_cb, io);365 if (rc) {366 int status = SLI4_FC_WCQE_STATUS_SUCCESS;367 efct_hw_done_t cb = io->hw_cb;368 369 if (rc != -ENOENT && rc != -EINPROGRESS) {370 status = -1;371 scsi_io_printf(io, "Failed to abort IO rc=%d\n",372 rc);373 }374 cb(io->hio, 0, status, 0, io);375 rc = 0;376 }377 378 break;379 }380 default:381 scsi_io_printf(io, "Unknown IO type=%d\n", io->io_type);382 rc = -EIO;383 break;384 }385 return rc;386}387 388static struct efct_io *389efct_scsi_dispatch_pending(struct efct *efct)390{391 struct efct_xport *xport = efct->xport;392 struct efct_io *io = NULL;393 struct efct_hw_io *hio;394 unsigned long flags = 0;395 int status;396 397 spin_lock_irqsave(&xport->io_pending_lock, flags);398 399 if (!list_empty(&xport->io_pending_list)) {400 io = list_first_entry(&xport->io_pending_list, struct efct_io,401 io_pending_link);402 list_del_init(&io->io_pending_link);403 }404 405 if (!io) {406 spin_unlock_irqrestore(&xport->io_pending_lock, flags);407 return NULL;408 }409 410 if (io->io_type == EFCT_IO_TYPE_ABORT) {411 hio = NULL;412 } else {413 hio = efct_hw_io_alloc(&efct->hw);414 if (!hio) {415 /*416 * No HW IO available.Put IO back on417 * the front of pending list418 */419 list_add(&xport->io_pending_list, &io->io_pending_link);420 io = NULL;421 } else {422 hio->eq = io->hw_priv;423 }424 }425 426 /* Must drop the lock before dispatching the IO */427 spin_unlock_irqrestore(&xport->io_pending_lock, flags);428 429 if (!io)430 return NULL;431 432 /*433 * We pulled an IO off the pending list,434 * and either got an HW IO or don't need one435 */436 atomic_sub_return(1, &xport->io_pending_count);437 if (!hio)438 status = efct_scsi_io_dispatch_no_hw_io(io);439 else440 status = efct_scsi_io_dispatch_hw_io(io, hio);441 if (status) {442 /*443 * Invoke the HW callback, but do so in the444 * separate execution context,provided by the445 * NOP mailbox completion processing context446 * by using efct_hw_async_call()447 */448 if (efct_hw_async_call(&efct->hw,449 efct_scsi_check_pending_async_cb, io)) {450 efc_log_debug(efct, "call hw async failed\n");451 }452 }453 454 return io;455}456 457void458efct_scsi_check_pending(struct efct *efct)459{460 struct efct_xport *xport = efct->xport;461 struct efct_io *io = NULL;462 int count = 0;463 unsigned long flags = 0;464 int dispatch = 0;465 466 /* Guard against recursion */467 if (atomic_add_return(1, &xport->io_pending_recursing)) {468 /* This function is already running. Decrement and return. */469 atomic_sub_return(1, &xport->io_pending_recursing);470 return;471 }472 473 while (efct_scsi_dispatch_pending(efct))474 count++;475 476 if (count) {477 atomic_sub_return(1, &xport->io_pending_recursing);478 return;479 }480 481 /*482 * If nothing was removed from the list,483 * we might be in a case where we need to abort an484 * active IO and the abort is on the pending list.485 * Look for an abort we can dispatch.486 */487 488 spin_lock_irqsave(&xport->io_pending_lock, flags);489 490 list_for_each_entry(io, &xport->io_pending_list, io_pending_link) {491 if (io->io_type == EFCT_IO_TYPE_ABORT && io->io_to_abort->hio) {492 /* This IO has a HW IO, so it is493 * active. Dispatch the abort.494 */495 dispatch = 1;496 list_del_init(&io->io_pending_link);497 atomic_sub_return(1, &xport->io_pending_count);498 break;499 }500 }501 502 spin_unlock_irqrestore(&xport->io_pending_lock, flags);503 504 if (dispatch) {505 if (efct_scsi_io_dispatch_no_hw_io(io)) {506 if (efct_hw_async_call(&efct->hw,507 efct_scsi_check_pending_async_cb, io)) {508 efc_log_debug(efct, "hw async failed\n");509 }510 }511 }512 513 atomic_sub_return(1, &xport->io_pending_recursing);514}515 516int517efct_scsi_io_dispatch(struct efct_io *io, void *cb)518{519 struct efct_hw_io *hio;520 struct efct *efct = io->efct;521 struct efct_xport *xport = efct->xport;522 unsigned long flags = 0;523 524 io->hw_cb = cb;525 526 /*527 * if this IO already has a HW IO, then this is either528 * not the first phase of the IO. Send it to the HW.529 */530 if (io->hio)531 return efct_scsi_io_dispatch_hw_io(io, io->hio);532 533 /*534 * We don't already have a HW IO associated with the IO. First check535 * the pending list. If not empty, add IO to the tail and process the536 * pending list.537 */538 spin_lock_irqsave(&xport->io_pending_lock, flags);539 if (!list_empty(&xport->io_pending_list)) {540 /*541 * If this is a low latency request,542 * the put at the front of the IO pending543 * queue, otherwise put it at the end of the queue.544 */545 if (io->low_latency) {546 INIT_LIST_HEAD(&io->io_pending_link);547 list_add(&xport->io_pending_list, &io->io_pending_link);548 } else {549 INIT_LIST_HEAD(&io->io_pending_link);550 list_add_tail(&io->io_pending_link,551 &xport->io_pending_list);552 }553 spin_unlock_irqrestore(&xport->io_pending_lock, flags);554 atomic_add_return(1, &xport->io_pending_count);555 atomic_add_return(1, &xport->io_total_pending);556 557 /* process pending list */558 efct_scsi_check_pending(efct);559 return 0;560 }561 spin_unlock_irqrestore(&xport->io_pending_lock, flags);562 563 /*564 * We don't have a HW IO associated with the IO and there's nothing565 * on the pending list. Attempt to allocate a HW IO and dispatch it.566 */567 hio = efct_hw_io_alloc(&io->efct->hw);568 if (!hio) {569 /* Couldn't get a HW IO. Save this IO on the pending list */570 spin_lock_irqsave(&xport->io_pending_lock, flags);571 INIT_LIST_HEAD(&io->io_pending_link);572 list_add_tail(&io->io_pending_link, &xport->io_pending_list);573 spin_unlock_irqrestore(&xport->io_pending_lock, flags);574 575 atomic_add_return(1, &xport->io_total_pending);576 atomic_add_return(1, &xport->io_pending_count);577 return 0;578 }579 580 /* We successfully allocated a HW IO; dispatch to HW */581 return efct_scsi_io_dispatch_hw_io(io, hio);582}583 584int585efct_scsi_io_dispatch_abort(struct efct_io *io, void *cb)586{587 struct efct *efct = io->efct;588 struct efct_xport *xport = efct->xport;589 unsigned long flags = 0;590 591 io->hw_cb = cb;592 593 /*594 * For aborts, we don't need a HW IO, but we still want595 * to pass through the pending list to preserve ordering.596 * Thus, if the pending list is not empty, add this abort597 * to the pending list and process the pending list.598 */599 spin_lock_irqsave(&xport->io_pending_lock, flags);600 if (!list_empty(&xport->io_pending_list)) {601 INIT_LIST_HEAD(&io->io_pending_link);602 list_add_tail(&io->io_pending_link, &xport->io_pending_list);603 spin_unlock_irqrestore(&xport->io_pending_lock, flags);604 atomic_add_return(1, &xport->io_pending_count);605 atomic_add_return(1, &xport->io_total_pending);606 607 /* process pending list */608 efct_scsi_check_pending(efct);609 return 0;610 }611 spin_unlock_irqrestore(&xport->io_pending_lock, flags);612 613 /* nothing on pending list, dispatch abort */614 return efct_scsi_io_dispatch_no_hw_io(io);615}616 617static inline int618efct_scsi_xfer_data(struct efct_io *io, u32 flags,619 struct efct_scsi_sgl *sgl, u32 sgl_count, u64 xwire_len,620 enum efct_hw_io_type type, int enable_ar,621 efct_scsi_io_cb_t cb, void *arg)622{623 struct efct *efct;624 size_t residual = 0;625 626 io->sgl_count = sgl_count;627 628 efct = io->efct;629 630 scsi_io_trace(io, "%s wire_len %llu\n",631 (type == EFCT_HW_IO_TARGET_READ) ? "send" : "recv",632 xwire_len);633 634 io->hio_type = type;635 636 io->scsi_tgt_cb = cb;637 io->scsi_tgt_cb_arg = arg;638 639 residual = io->exp_xfer_len - io->transferred;640 io->wire_len = (xwire_len < residual) ? xwire_len : residual;641 residual = (xwire_len - io->wire_len);642 643 memset(&io->iparam, 0, sizeof(io->iparam));644 io->iparam.fcp_tgt.ox_id = io->init_task_tag;645 io->iparam.fcp_tgt.offset = io->transferred;646 io->iparam.fcp_tgt.cs_ctl = io->cs_ctl;647 io->iparam.fcp_tgt.timeout = io->timeout;648 649 /* if this is the last data phase and there is no residual, enable650 * auto-good-response651 */652 if (enable_ar && (flags & EFCT_SCSI_LAST_DATAPHASE) && residual == 0 &&653 ((io->transferred + io->wire_len) == io->exp_xfer_len) &&654 (!(flags & EFCT_SCSI_NO_AUTO_RESPONSE))) {655 io->iparam.fcp_tgt.flags |= SLI4_IO_AUTO_GOOD_RESPONSE;656 io->auto_resp = true;657 } else {658 io->auto_resp = false;659 }660 661 /* save this transfer length */662 io->xfer_req = io->wire_len;663 664 /* Adjust the transferred count to account for overrun665 * when the residual is calculated in efct_scsi_send_resp666 */667 io->transferred += residual;668 669 /* Adjust the SGL size if there is overrun */670 671 if (residual) {672 struct efct_scsi_sgl *sgl_ptr = &io->sgl[sgl_count - 1];673 674 while (residual) {675 size_t len = sgl_ptr->len;676 677 if (len > residual) {678 sgl_ptr->len = len - residual;679 residual = 0;680 } else {681 sgl_ptr->len = 0;682 residual -= len;683 io->sgl_count--;684 }685 sgl_ptr--;686 }687 }688 689 /* Set latency and WQ steering */690 io->low_latency = (flags & EFCT_SCSI_LOW_LATENCY) != 0;691 io->wq_steering = (flags & EFCT_SCSI_WQ_STEERING_MASK) >>692 EFCT_SCSI_WQ_STEERING_SHIFT;693 io->wq_class = (flags & EFCT_SCSI_WQ_CLASS_MASK) >>694 EFCT_SCSI_WQ_CLASS_SHIFT;695 696 if (efct->xport) {697 struct efct_xport *xport = efct->xport;698 699 if (type == EFCT_HW_IO_TARGET_READ) {700 xport->fcp_stats.input_requests++;701 xport->fcp_stats.input_bytes += xwire_len;702 } else if (type == EFCT_HW_IO_TARGET_WRITE) {703 xport->fcp_stats.output_requests++;704 xport->fcp_stats.output_bytes += xwire_len;705 }706 }707 return efct_scsi_io_dispatch(io, efct_target_io_cb);708}709 710int711efct_scsi_send_rd_data(struct efct_io *io, u32 flags,712 struct efct_scsi_sgl *sgl, u32 sgl_count, u64 len,713 efct_scsi_io_cb_t cb, void *arg)714{715 return efct_scsi_xfer_data(io, flags, sgl, sgl_count,716 len, EFCT_HW_IO_TARGET_READ,717 enable_tsend_auto_resp(io->efct), cb, arg);718}719 720int721efct_scsi_recv_wr_data(struct efct_io *io, u32 flags,722 struct efct_scsi_sgl *sgl, u32 sgl_count, u64 len,723 efct_scsi_io_cb_t cb, void *arg)724{725 return efct_scsi_xfer_data(io, flags, sgl, sgl_count, len,726 EFCT_HW_IO_TARGET_WRITE,727 enable_treceive_auto_resp(io->efct), cb, arg);728}729 730int731efct_scsi_send_resp(struct efct_io *io, u32 flags,732 struct efct_scsi_cmd_resp *rsp,733 efct_scsi_io_cb_t cb, void *arg)734{735 struct efct *efct;736 int residual;737 /* Always try auto resp */738 bool auto_resp = true;739 u8 scsi_status = 0;740 u16 scsi_status_qualifier = 0;741 u8 *sense_data = NULL;742 u32 sense_data_length = 0;743 744 efct = io->efct;745 746 if (rsp) {747 scsi_status = rsp->scsi_status;748 scsi_status_qualifier = rsp->scsi_status_qualifier;749 sense_data = rsp->sense_data;750 sense_data_length = rsp->sense_data_length;751 residual = rsp->residual;752 } else {753 residual = io->exp_xfer_len - io->transferred;754 }755 756 io->wire_len = 0;757 io->hio_type = EFCT_HW_IO_TARGET_RSP;758 759 io->scsi_tgt_cb = cb;760 io->scsi_tgt_cb_arg = arg;761 762 memset(&io->iparam, 0, sizeof(io->iparam));763 io->iparam.fcp_tgt.ox_id = io->init_task_tag;764 io->iparam.fcp_tgt.offset = 0;765 io->iparam.fcp_tgt.cs_ctl = io->cs_ctl;766 io->iparam.fcp_tgt.timeout = io->timeout;767 768 /* Set low latency queueing request */769 io->low_latency = (flags & EFCT_SCSI_LOW_LATENCY) != 0;770 io->wq_steering = (flags & EFCT_SCSI_WQ_STEERING_MASK) >>771 EFCT_SCSI_WQ_STEERING_SHIFT;772 io->wq_class = (flags & EFCT_SCSI_WQ_CLASS_MASK) >>773 EFCT_SCSI_WQ_CLASS_SHIFT;774 775 if (scsi_status != 0 || residual || sense_data_length) {776 struct fcp_resp_with_ext *fcprsp = io->rspbuf.virt;777 u8 *sns_data;778 779 if (!fcprsp) {780 efc_log_err(efct, "NULL response buffer\n");781 return -EIO;782 }783 784 sns_data = (u8 *)io->rspbuf.virt + sizeof(*fcprsp);785 786 auto_resp = false;787 788 memset(fcprsp, 0, sizeof(*fcprsp));789 790 io->wire_len += sizeof(*fcprsp);791 792 fcprsp->resp.fr_status = scsi_status;793 fcprsp->resp.fr_retry_delay =794 cpu_to_be16(scsi_status_qualifier);795 796 /* set residual status if necessary */797 if (residual != 0) {798 /* FCP: if data transferred is less than the799 * amount expected, then this is an underflow.800 * If data transferred would have been greater801 * than the amount expected this is an overflow802 */803 if (residual > 0) {804 fcprsp->resp.fr_flags |= FCP_RESID_UNDER;805 fcprsp->ext.fr_resid = cpu_to_be32(residual);806 } else {807 fcprsp->resp.fr_flags |= FCP_RESID_OVER;808 fcprsp->ext.fr_resid = cpu_to_be32(-residual);809 }810 }811 812 if (EFCT_SCSI_SNS_BUF_VALID(sense_data) && sense_data_length) {813 if (sense_data_length > SCSI_SENSE_BUFFERSIZE) {814 efc_log_err(efct, "Sense exceeds max size.\n");815 return -EIO;816 }817 818 fcprsp->resp.fr_flags |= FCP_SNS_LEN_VAL;819 memcpy(sns_data, sense_data, sense_data_length);820 fcprsp->ext.fr_sns_len = cpu_to_be32(sense_data_length);821 io->wire_len += sense_data_length;822 }823 824 io->sgl[0].addr = io->rspbuf.phys;825 io->sgl[0].dif_addr = 0;826 io->sgl[0].len = io->wire_len;827 io->sgl_count = 1;828 }829 830 if (auto_resp)831 io->iparam.fcp_tgt.flags |= SLI4_IO_AUTO_GOOD_RESPONSE;832 833 return efct_scsi_io_dispatch(io, efct_target_io_cb);834}835 836static int837efct_target_bls_resp_cb(struct efct_hw_io *hio, u32 length, int status,838 u32 ext_status, void *app)839{840 struct efct_io *io = app;841 struct efct *efct;842 enum efct_scsi_io_status bls_status;843 844 efct = io->efct;845 846 /* BLS isn't really a "SCSI" concept, but use SCSI status */847 if (status) {848 io_error_log(io, "s=%#x x=%#x\n", status, ext_status);849 bls_status = EFCT_SCSI_STATUS_ERROR;850 } else {851 bls_status = EFCT_SCSI_STATUS_GOOD;852 }853 854 if (io->bls_cb) {855 efct_scsi_io_cb_t bls_cb = io->bls_cb;856 void *bls_cb_arg = io->bls_cb_arg;857 858 io->bls_cb = NULL;859 io->bls_cb_arg = NULL;860 861 /* invoke callback */862 bls_cb(io, bls_status, 0, bls_cb_arg);863 }864 865 efct_scsi_check_pending(efct);866 return 0;867}868 869static int870efct_target_send_bls_resp(struct efct_io *io,871 efct_scsi_io_cb_t cb, void *arg)872{873 struct efct_node *node = io->node;874 struct sli_bls_params *bls = &io->iparam.bls;875 struct efct *efct = node->efct;876 struct fc_ba_acc *acc;877 int rc;878 879 /* fill out IO structure with everything needed to send BA_ACC */880 memset(&io->iparam, 0, sizeof(io->iparam));881 bls->ox_id = io->init_task_tag;882 bls->rx_id = io->abort_rx_id;883 bls->vpi = io->node->vpi;884 bls->rpi = io->node->rpi;885 bls->s_id = U32_MAX;886 bls->d_id = io->node->node_fc_id;887 bls->rpi_registered = true;888 889 acc = (void *)bls->payload;890 acc->ba_ox_id = cpu_to_be16(bls->ox_id);891 acc->ba_rx_id = cpu_to_be16(bls->rx_id);892 acc->ba_high_seq_cnt = cpu_to_be16(U16_MAX);893 894 /* generic io fields have already been populated */895 896 /* set type and BLS-specific fields */897 io->io_type = EFCT_IO_TYPE_BLS_RESP;898 io->display_name = "bls_rsp";899 io->hio_type = EFCT_HW_BLS_ACC;900 io->bls_cb = cb;901 io->bls_cb_arg = arg;902 903 /* dispatch IO */904 rc = efct_hw_bls_send(efct, FC_RCTL_BA_ACC, bls,905 efct_target_bls_resp_cb, io);906 return rc;907}908 909static int efct_bls_send_rjt_cb(struct efct_hw_io *hio, u32 length, int status,910 u32 ext_status, void *app)911{912 struct efct_io *io = app;913 914 efct_scsi_io_free(io);915 return 0;916}917 918struct efct_io *919efct_bls_send_rjt(struct efct_io *io, struct fc_frame_header *hdr)920{921 struct efct_node *node = io->node;922 struct sli_bls_params *bls = &io->iparam.bls;923 struct efct *efct = node->efct;924 struct fc_ba_rjt *acc;925 int rc;926 927 /* fill out BLS Response-specific fields */928 io->io_type = EFCT_IO_TYPE_BLS_RESP;929 io->display_name = "ba_rjt";930 io->hio_type = EFCT_HW_BLS_RJT;931 io->init_task_tag = be16_to_cpu(hdr->fh_ox_id);932 933 /* fill out iparam fields */934 memset(&io->iparam, 0, sizeof(io->iparam));935 bls->ox_id = be16_to_cpu(hdr->fh_ox_id);936 bls->rx_id = be16_to_cpu(hdr->fh_rx_id);937 bls->vpi = io->node->vpi;938 bls->rpi = io->node->rpi;939 bls->s_id = U32_MAX;940 bls->d_id = io->node->node_fc_id;941 bls->rpi_registered = true;942 943 acc = (void *)bls->payload;944 acc->br_reason = ELS_RJT_UNAB;945 acc->br_explan = ELS_EXPL_NONE;946 947 rc = efct_hw_bls_send(efct, FC_RCTL_BA_RJT, bls, efct_bls_send_rjt_cb,948 io);949 if (rc) {950 efc_log_err(efct, "efct_scsi_io_dispatch() failed: %d\n", rc);951 efct_scsi_io_free(io);952 io = NULL;953 }954 return io;955}956 957int958efct_scsi_send_tmf_resp(struct efct_io *io,959 enum efct_scsi_tmf_resp rspcode,960 u8 addl_rsp_info[3],961 efct_scsi_io_cb_t cb, void *arg)962{963 int rc;964 struct {965 struct fcp_resp_with_ext rsp_ext;966 struct fcp_resp_rsp_info info;967 } *fcprsp;968 u8 fcp_rspcode;969 970 io->wire_len = 0;971 972 switch (rspcode) {973 case EFCT_SCSI_TMF_FUNCTION_COMPLETE:974 fcp_rspcode = FCP_TMF_CMPL;975 break;976 case EFCT_SCSI_TMF_FUNCTION_SUCCEEDED:977 case EFCT_SCSI_TMF_FUNCTION_IO_NOT_FOUND:978 fcp_rspcode = FCP_TMF_CMPL;979 break;980 case EFCT_SCSI_TMF_FUNCTION_REJECTED:981 fcp_rspcode = FCP_TMF_REJECTED;982 break;983 case EFCT_SCSI_TMF_INCORRECT_LOGICAL_UNIT_NUMBER:984 fcp_rspcode = FCP_TMF_INVALID_LUN;985 break;986 case EFCT_SCSI_TMF_SERVICE_DELIVERY:987 fcp_rspcode = FCP_TMF_FAILED;988 break;989 default:990 fcp_rspcode = FCP_TMF_REJECTED;991 break;992 }993 994 io->hio_type = EFCT_HW_IO_TARGET_RSP;995 996 io->scsi_tgt_cb = cb;997 io->scsi_tgt_cb_arg = arg;998 999 if (io->tmf_cmd == EFCT_SCSI_TMF_ABORT_TASK) {1000 rc = efct_target_send_bls_resp(io, cb, arg);1001 return rc;1002 }1003 1004 /* populate the FCP TMF response */1005 fcprsp = io->rspbuf.virt;1006 memset(fcprsp, 0, sizeof(*fcprsp));1007 1008 fcprsp->rsp_ext.resp.fr_flags |= FCP_SNS_LEN_VAL;1009 1010 if (addl_rsp_info) {1011 memcpy(fcprsp->info._fr_resvd, addl_rsp_info,1012 sizeof(fcprsp->info._fr_resvd));1013 }1014 fcprsp->info.rsp_code = fcp_rspcode;1015 1016 io->wire_len = sizeof(*fcprsp);1017 1018 fcprsp->rsp_ext.ext.fr_rsp_len =1019 cpu_to_be32(sizeof(struct fcp_resp_rsp_info));1020 1021 io->sgl[0].addr = io->rspbuf.phys;1022 io->sgl[0].dif_addr = 0;1023 io->sgl[0].len = io->wire_len;1024 io->sgl_count = 1;1025 1026 memset(&io->iparam, 0, sizeof(io->iparam));1027 io->iparam.fcp_tgt.ox_id = io->init_task_tag;1028 io->iparam.fcp_tgt.offset = 0;1029 io->iparam.fcp_tgt.cs_ctl = io->cs_ctl;1030 io->iparam.fcp_tgt.timeout = io->timeout;1031 1032 rc = efct_scsi_io_dispatch(io, efct_target_io_cb);1033 1034 return rc;1035}1036 1037static int1038efct_target_abort_cb(struct efct_hw_io *hio, u32 length, int status,1039 u32 ext_status, void *app)1040{1041 struct efct_io *io = app;1042 struct efct *efct;1043 enum efct_scsi_io_status scsi_status;1044 efct_scsi_io_cb_t abort_cb;1045 void *abort_cb_arg;1046 1047 efct = io->efct;1048 1049 if (!io->abort_cb)1050 goto done;1051 1052 abort_cb = io->abort_cb;1053 abort_cb_arg = io->abort_cb_arg;1054 1055 io->abort_cb = NULL;1056 io->abort_cb_arg = NULL;1057 1058 switch (status) {1059 case SLI4_FC_WCQE_STATUS_SUCCESS:1060 scsi_status = EFCT_SCSI_STATUS_GOOD;1061 break;1062 case SLI4_FC_WCQE_STATUS_LOCAL_REJECT:1063 switch (ext_status) {1064 case SLI4_FC_LOCAL_REJECT_NO_XRI:1065 scsi_status = EFCT_SCSI_STATUS_NO_IO;1066 break;1067 case SLI4_FC_LOCAL_REJECT_ABORT_IN_PROGRESS:1068 scsi_status = EFCT_SCSI_STATUS_ABORT_IN_PROGRESS;1069 break;1070 default:1071 /*we have seen 0x15 (abort in progress)*/1072 scsi_status = EFCT_SCSI_STATUS_ERROR;1073 break;1074 }1075 break;1076 case SLI4_FC_WCQE_STATUS_FCP_RSP_FAILURE:1077 scsi_status = EFCT_SCSI_STATUS_CHECK_RESPONSE;1078 break;1079 default:1080 scsi_status = EFCT_SCSI_STATUS_ERROR;1081 break;1082 }1083 /* invoke callback */1084 abort_cb(io->io_to_abort, scsi_status, 0, abort_cb_arg);1085 1086done:1087 /* done with IO to abort,efct_ref_get(): efct_scsi_tgt_abort_io() */1088 kref_put(&io->io_to_abort->ref, io->io_to_abort->release);1089 1090 efct_io_pool_io_free(efct->xport->io_pool, io);1091 1092 efct_scsi_check_pending(efct);1093 return 0;1094}1095 1096int1097efct_scsi_tgt_abort_io(struct efct_io *io, efct_scsi_io_cb_t cb, void *arg)1098{1099 struct efct *efct;1100 struct efct_xport *xport;1101 int rc;1102 struct efct_io *abort_io = NULL;1103 1104 efct = io->efct;1105 xport = efct->xport;1106 1107 /* take a reference on IO being aborted */1108 if (kref_get_unless_zero(&io->ref) == 0) {1109 /* command no longer active */1110 scsi_io_printf(io, "command no longer active\n");1111 return -EIO;1112 }1113 1114 /*1115 * allocate a new IO to send the abort request. Use efct_io_alloc()1116 * directly, as we need an IO object that will not fail allocation1117 * due to allocations being disabled (in efct_scsi_io_alloc())1118 */1119 abort_io = efct_io_pool_io_alloc(efct->xport->io_pool);1120 if (!abort_io) {1121 atomic_add_return(1, &xport->io_alloc_failed_count);1122 kref_put(&io->ref, io->release);1123 return -EIO;1124 }1125 1126 /* Save the target server callback and argument */1127 /* set generic fields */1128 abort_io->cmd_tgt = true;1129 abort_io->node = io->node;1130 1131 /* set type and abort-specific fields */1132 abort_io->io_type = EFCT_IO_TYPE_ABORT;1133 abort_io->display_name = "tgt_abort";1134 abort_io->io_to_abort = io;1135 abort_io->send_abts = false;1136 abort_io->abort_cb = cb;1137 abort_io->abort_cb_arg = arg;1138 1139 /* now dispatch IO */1140 rc = efct_scsi_io_dispatch_abort(abort_io, efct_target_abort_cb);1141 if (rc)1142 kref_put(&io->ref, io->release);1143 return rc;1144}1145 1146void1147efct_scsi_io_complete(struct efct_io *io)1148{1149 if (io->io_free) {1150 efc_log_debug(io->efct, "completion for non-busy io tag 0x%x\n",1151 io->tag);1152 return;1153 }1154 1155 scsi_io_trace(io, "freeing io 0x%p %s\n", io, io->display_name);1156 kref_put(&io->ref, io->release);1157}1158