3655 lines · c
1/*******************************************************************2 * This file is part of the Emulex Linux Device Driver for *3 * Fibre Channel Host Bus Adapters. *4 * Copyright (C) 2017-2024 Broadcom. All Rights Reserved. The term *5 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. *6 * Copyright (C) 2004-2016 Emulex. All rights reserved. *7 * EMULEX and SLI are trademarks of Emulex. *8 * www.broadcom.com *9 * Portions Copyright (C) 2004-2005 Christoph Hellwig *10 * *11 * This program is free software; you can redistribute it and/or *12 * modify it under the terms of version 2 of the GNU General *13 * Public License as published by the Free Software Foundation. *14 * This program is distributed in the hope that it will be useful. *15 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *16 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *17 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *18 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *19 * TO BE LEGALLY INVALID. See the GNU General Public License for *20 * more details, a copy of which can be found in the file COPYING *21 * included with this package. *22 ********************************************************************/23#include <linux/pci.h>24#include <linux/slab.h>25#include <linux/interrupt.h>26#include <linux/delay.h>27#include <linux/unaligned.h>28#include <linux/crc-t10dif.h>29#include <net/checksum.h>30 31#include <scsi/scsi.h>32#include <scsi/scsi_device.h>33#include <scsi/scsi_eh.h>34#include <scsi/scsi_host.h>35#include <scsi/scsi_tcq.h>36#include <scsi/scsi_transport_fc.h>37#include <scsi/fc/fc_fs.h>38 39#include "lpfc_version.h"40#include "lpfc_hw4.h"41#include "lpfc_hw.h"42#include "lpfc_sli.h"43#include "lpfc_sli4.h"44#include "lpfc_nl.h"45#include "lpfc_disc.h"46#include "lpfc.h"47#include "lpfc_scsi.h"48#include "lpfc_nvme.h"49#include "lpfc_logmsg.h"50#include "lpfc_crtn.h"51#include "lpfc_vport.h"52#include "lpfc_debugfs.h"53 54static struct lpfc_iocbq *lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *,55 struct lpfc_async_xchg_ctx *,56 dma_addr_t rspbuf,57 uint16_t rspsize);58static struct lpfc_iocbq *lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *,59 struct lpfc_async_xchg_ctx *);60static int lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *,61 struct lpfc_async_xchg_ctx *,62 uint32_t, uint16_t);63static int lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *,64 struct lpfc_async_xchg_ctx *,65 uint32_t, uint16_t);66static void lpfc_nvmet_wqfull_flush(struct lpfc_hba *, struct lpfc_queue *,67 struct lpfc_async_xchg_ctx *);68static void lpfc_nvmet_fcp_rqst_defer_work(struct work_struct *);69 70static void lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf);71 72static union lpfc_wqe128 lpfc_tsend_cmd_template;73static union lpfc_wqe128 lpfc_treceive_cmd_template;74static union lpfc_wqe128 lpfc_trsp_cmd_template;75 76/* Setup WQE templates for NVME IOs */77void78lpfc_nvmet_cmd_template(void)79{80 union lpfc_wqe128 *wqe;81 82 /* TSEND template */83 wqe = &lpfc_tsend_cmd_template;84 memset(wqe, 0, sizeof(union lpfc_wqe128));85 86 /* Word 0, 1, 2 - BDE is variable */87 88 /* Word 3 - payload_offset_len is zero */89 90 /* Word 4 - relative_offset is variable */91 92 /* Word 5 - is zero */93 94 /* Word 6 - ctxt_tag, xri_tag is variable */95 96 /* Word 7 - wqe_ar is variable */97 bf_set(wqe_cmnd, &wqe->fcp_tsend.wqe_com, CMD_FCP_TSEND64_WQE);98 bf_set(wqe_pu, &wqe->fcp_tsend.wqe_com, PARM_REL_OFF);99 bf_set(wqe_class, &wqe->fcp_tsend.wqe_com, CLASS3);100 bf_set(wqe_ct, &wqe->fcp_tsend.wqe_com, SLI4_CT_RPI);101 bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 1);102 103 /* Word 8 - abort_tag is variable */104 105 /* Word 9 - reqtag, rcvoxid is variable */106 107 /* Word 10 - wqes, xc is variable */108 bf_set(wqe_xchg, &wqe->fcp_tsend.wqe_com, LPFC_NVME_XCHG);109 bf_set(wqe_dbde, &wqe->fcp_tsend.wqe_com, 1);110 bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0);111 bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 1);112 bf_set(wqe_iod, &wqe->fcp_tsend.wqe_com, LPFC_WQE_IOD_WRITE);113 bf_set(wqe_lenloc, &wqe->fcp_tsend.wqe_com, LPFC_WQE_LENLOC_WORD12);114 115 /* Word 11 - sup, irsp, irsplen is variable */116 bf_set(wqe_cmd_type, &wqe->fcp_tsend.wqe_com, FCP_COMMAND_TSEND);117 bf_set(wqe_cqid, &wqe->fcp_tsend.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);118 bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);119 bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0);120 bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0);121 bf_set(wqe_pbde, &wqe->fcp_tsend.wqe_com, 0);122 123 /* Word 12 - fcp_data_len is variable */124 125 /* Word 13, 14, 15 - PBDE is zero */126 127 /* TRECEIVE template */128 wqe = &lpfc_treceive_cmd_template;129 memset(wqe, 0, sizeof(union lpfc_wqe128));130 131 /* Word 0, 1, 2 - BDE is variable */132 133 /* Word 3 */134 wqe->fcp_treceive.payload_offset_len = TXRDY_PAYLOAD_LEN;135 136 /* Word 4 - relative_offset is variable */137 138 /* Word 5 - is zero */139 140 /* Word 6 - ctxt_tag, xri_tag is variable */141 142 /* Word 7 */143 bf_set(wqe_cmnd, &wqe->fcp_treceive.wqe_com, CMD_FCP_TRECEIVE64_WQE);144 bf_set(wqe_pu, &wqe->fcp_treceive.wqe_com, PARM_REL_OFF);145 bf_set(wqe_class, &wqe->fcp_treceive.wqe_com, CLASS3);146 bf_set(wqe_ct, &wqe->fcp_treceive.wqe_com, SLI4_CT_RPI);147 bf_set(wqe_ar, &wqe->fcp_treceive.wqe_com, 0);148 149 /* Word 8 - abort_tag is variable */150 151 /* Word 9 - reqtag, rcvoxid is variable */152 153 /* Word 10 - xc is variable */154 bf_set(wqe_dbde, &wqe->fcp_treceive.wqe_com, 1);155 bf_set(wqe_wqes, &wqe->fcp_treceive.wqe_com, 0);156 bf_set(wqe_xchg, &wqe->fcp_treceive.wqe_com, LPFC_NVME_XCHG);157 bf_set(wqe_iod, &wqe->fcp_treceive.wqe_com, LPFC_WQE_IOD_READ);158 bf_set(wqe_lenloc, &wqe->fcp_treceive.wqe_com, LPFC_WQE_LENLOC_WORD12);159 bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 1);160 161 /* Word 11 - pbde is variable */162 bf_set(wqe_cmd_type, &wqe->fcp_treceive.wqe_com, FCP_COMMAND_TRECEIVE);163 bf_set(wqe_cqid, &wqe->fcp_treceive.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);164 bf_set(wqe_sup, &wqe->fcp_treceive.wqe_com, 0);165 bf_set(wqe_irsp, &wqe->fcp_treceive.wqe_com, 0);166 bf_set(wqe_irsplen, &wqe->fcp_treceive.wqe_com, 0);167 bf_set(wqe_pbde, &wqe->fcp_treceive.wqe_com, 1);168 169 /* Word 12 - fcp_data_len is variable */170 171 /* Word 13, 14, 15 - PBDE is variable */172 173 /* TRSP template */174 wqe = &lpfc_trsp_cmd_template;175 memset(wqe, 0, sizeof(union lpfc_wqe128));176 177 /* Word 0, 1, 2 - BDE is variable */178 179 /* Word 3 - response_len is variable */180 181 /* Word 4, 5 - is zero */182 183 /* Word 6 - ctxt_tag, xri_tag is variable */184 185 /* Word 7 */186 bf_set(wqe_cmnd, &wqe->fcp_trsp.wqe_com, CMD_FCP_TRSP64_WQE);187 bf_set(wqe_pu, &wqe->fcp_trsp.wqe_com, PARM_UNUSED);188 bf_set(wqe_class, &wqe->fcp_trsp.wqe_com, CLASS3);189 bf_set(wqe_ct, &wqe->fcp_trsp.wqe_com, SLI4_CT_RPI);190 bf_set(wqe_ag, &wqe->fcp_trsp.wqe_com, 1); /* wqe_ar */191 192 /* Word 8 - abort_tag is variable */193 194 /* Word 9 - reqtag is variable */195 196 /* Word 10 wqes, xc is variable */197 bf_set(wqe_dbde, &wqe->fcp_trsp.wqe_com, 1);198 bf_set(wqe_xchg, &wqe->fcp_trsp.wqe_com, LPFC_NVME_XCHG);199 bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 0);200 bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, 0);201 bf_set(wqe_iod, &wqe->fcp_trsp.wqe_com, LPFC_WQE_IOD_NONE);202 bf_set(wqe_lenloc, &wqe->fcp_trsp.wqe_com, LPFC_WQE_LENLOC_WORD3);203 204 /* Word 11 irsp, irsplen is variable */205 bf_set(wqe_cmd_type, &wqe->fcp_trsp.wqe_com, FCP_COMMAND_TRSP);206 bf_set(wqe_cqid, &wqe->fcp_trsp.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);207 bf_set(wqe_sup, &wqe->fcp_trsp.wqe_com, 0);208 bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 0);209 bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com, 0);210 bf_set(wqe_pbde, &wqe->fcp_trsp.wqe_com, 0);211 212 /* Word 12, 13, 14, 15 - is zero */213}214 215#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))216static struct lpfc_async_xchg_ctx *217lpfc_nvmet_get_ctx_for_xri(struct lpfc_hba *phba, u16 xri)218{219 struct lpfc_async_xchg_ctx *ctxp;220 unsigned long iflag;221 bool found = false;222 223 spin_lock_irqsave(&phba->sli4_hba.t_active_list_lock, iflag);224 list_for_each_entry(ctxp, &phba->sli4_hba.t_active_ctx_list, list) {225 if (ctxp->ctxbuf->sglq->sli4_xritag != xri)226 continue;227 228 found = true;229 break;230 }231 spin_unlock_irqrestore(&phba->sli4_hba.t_active_list_lock, iflag);232 if (found)233 return ctxp;234 235 return NULL;236}237 238static struct lpfc_async_xchg_ctx *239lpfc_nvmet_get_ctx_for_oxid(struct lpfc_hba *phba, u16 oxid, u32 sid)240{241 struct lpfc_async_xchg_ctx *ctxp;242 unsigned long iflag;243 bool found = false;244 245 spin_lock_irqsave(&phba->sli4_hba.t_active_list_lock, iflag);246 list_for_each_entry(ctxp, &phba->sli4_hba.t_active_ctx_list, list) {247 if (ctxp->oxid != oxid || ctxp->sid != sid)248 continue;249 250 found = true;251 break;252 }253 spin_unlock_irqrestore(&phba->sli4_hba.t_active_list_lock, iflag);254 if (found)255 return ctxp;256 257 return NULL;258}259#endif260 261static void262lpfc_nvmet_defer_release(struct lpfc_hba *phba,263 struct lpfc_async_xchg_ctx *ctxp)264{265 lockdep_assert_held(&ctxp->ctxlock);266 267 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,268 "6313 NVMET Defer ctx release oxid x%x flg x%x\n",269 ctxp->oxid, ctxp->flag);270 271 if (ctxp->flag & LPFC_NVME_CTX_RLS)272 return;273 274 ctxp->flag |= LPFC_NVME_CTX_RLS;275 spin_lock(&phba->sli4_hba.t_active_list_lock);276 list_del(&ctxp->list);277 spin_unlock(&phba->sli4_hba.t_active_list_lock);278 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock);279 list_add_tail(&ctxp->list, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list);280 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock);281}282 283/**284 * __lpfc_nvme_xmt_ls_rsp_cmp - Generic completion handler for the285 * transmission of an NVME LS response.286 * @phba: Pointer to HBA context object.287 * @cmdwqe: Pointer to driver command WQE object.288 * @rspwqe: Pointer to driver response WQE object.289 *290 * The function is called from SLI ring event handler with no291 * lock held. The function frees memory resources used for the command292 * used to send the NVME LS RSP.293 **/294void295__lpfc_nvme_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,296 struct lpfc_iocbq *rspwqe)297{298 struct lpfc_async_xchg_ctx *axchg = cmdwqe->context_un.axchg;299 struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;300 struct nvmefc_ls_rsp *ls_rsp = &axchg->ls_rsp;301 uint32_t status, result;302 303 status = bf_get(lpfc_wcqe_c_status, wcqe);304 result = wcqe->parameter;305 306 if (axchg->state != LPFC_NVME_STE_LS_RSP || axchg->entry_cnt != 2) {307 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,308 "6410 NVMEx LS cmpl state mismatch IO x%x: "309 "%d %d\n",310 axchg->oxid, axchg->state, axchg->entry_cnt);311 }312 313 lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x result x%x\n",314 axchg->oxid, status, result);315 316 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,317 "6038 NVMEx LS rsp cmpl: %d %d oxid x%x\n",318 status, result, axchg->oxid);319 320 lpfc_nlp_put(cmdwqe->ndlp);321 cmdwqe->context_un.axchg = NULL;322 cmdwqe->bpl_dmabuf = NULL;323 lpfc_sli_release_iocbq(phba, cmdwqe);324 ls_rsp->done(ls_rsp);325 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,326 "6200 NVMEx LS rsp cmpl done status %d oxid x%x\n",327 status, axchg->oxid);328 kfree(axchg);329}330 331/**332 * lpfc_nvmet_xmt_ls_rsp_cmp - Completion handler for LS Response333 * @phba: Pointer to HBA context object.334 * @cmdwqe: Pointer to driver command WQE object.335 * @rspwqe: Pointer to driver response WQE object.336 *337 * The function is called from SLI ring event handler with no338 * lock held. This function is the completion handler for NVME LS commands339 * The function updates any states and statistics, then calls the340 * generic completion handler to free resources.341 **/342static void343lpfc_nvmet_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,344 struct lpfc_iocbq *rspwqe)345{346 struct lpfc_nvmet_tgtport *tgtp;347 uint32_t status, result;348 struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;349 350 if (!phba->targetport)351 goto finish;352 353 status = bf_get(lpfc_wcqe_c_status, wcqe);354 result = wcqe->parameter;355 356 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;357 if (tgtp) {358 if (status) {359 atomic_inc(&tgtp->xmt_ls_rsp_error);360 if (result == IOERR_ABORT_REQUESTED)361 atomic_inc(&tgtp->xmt_ls_rsp_aborted);362 if (bf_get(lpfc_wcqe_c_xb, wcqe))363 atomic_inc(&tgtp->xmt_ls_rsp_xb_set);364 } else {365 atomic_inc(&tgtp->xmt_ls_rsp_cmpl);366 }367 }368 369finish:370 __lpfc_nvme_xmt_ls_rsp_cmp(phba, cmdwqe, rspwqe);371}372 373/**374 * lpfc_nvmet_ctxbuf_post - Repost a NVMET RQ DMA buffer and clean up context375 * @phba: HBA buffer is associated with376 * @ctx_buf: ctx buffer context377 *378 * Description: Frees the given DMA buffer in the appropriate way given by379 * reposting it to its associated RQ so it can be reused.380 *381 * Notes: Takes phba->hbalock. Can be called with or without other locks held.382 *383 * Returns: None384 **/385void386lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf)387{388#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))389 struct lpfc_async_xchg_ctx *ctxp = ctx_buf->context;390 struct lpfc_nvmet_tgtport *tgtp;391 struct fc_frame_header *fc_hdr;392 struct rqb_dmabuf *nvmebuf;393 struct lpfc_nvmet_ctx_info *infop;394 uint32_t size, oxid, sid;395 int cpu;396 unsigned long iflag;397 398 if (ctxp->state == LPFC_NVME_STE_FREE) {399 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,400 "6411 NVMET free, already free IO x%x: %d %d\n",401 ctxp->oxid, ctxp->state, ctxp->entry_cnt);402 }403 404 if (ctxp->rqb_buffer) {405 spin_lock_irqsave(&ctxp->ctxlock, iflag);406 nvmebuf = ctxp->rqb_buffer;407 /* check if freed in another path whilst acquiring lock */408 if (nvmebuf) {409 ctxp->rqb_buffer = NULL;410 if (ctxp->flag & LPFC_NVME_CTX_REUSE_WQ) {411 ctxp->flag &= ~LPFC_NVME_CTX_REUSE_WQ;412 spin_unlock_irqrestore(&ctxp->ctxlock, iflag);413 nvmebuf->hrq->rqbp->rqb_free_buffer(phba,414 nvmebuf);415 } else {416 spin_unlock_irqrestore(&ctxp->ctxlock, iflag);417 /* repost */418 lpfc_rq_buf_free(phba, &nvmebuf->hbuf);419 }420 } else {421 spin_unlock_irqrestore(&ctxp->ctxlock, iflag);422 }423 }424 ctxp->state = LPFC_NVME_STE_FREE;425 426 spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag);427 if (phba->sli4_hba.nvmet_io_wait_cnt) {428 list_remove_head(&phba->sli4_hba.lpfc_nvmet_io_wait_list,429 nvmebuf, struct rqb_dmabuf,430 hbuf.list);431 phba->sli4_hba.nvmet_io_wait_cnt--;432 spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock,433 iflag);434 435 fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);436 oxid = be16_to_cpu(fc_hdr->fh_ox_id);437 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;438 size = nvmebuf->bytes_recv;439 sid = sli4_sid_from_fc_hdr(fc_hdr);440 441 ctxp = (struct lpfc_async_xchg_ctx *)ctx_buf->context;442 ctxp->wqeq = NULL;443 ctxp->offset = 0;444 ctxp->phba = phba;445 ctxp->size = size;446 ctxp->oxid = oxid;447 ctxp->sid = sid;448 ctxp->state = LPFC_NVME_STE_RCV;449 ctxp->entry_cnt = 1;450 ctxp->flag = 0;451 ctxp->ctxbuf = ctx_buf;452 ctxp->rqb_buffer = (void *)nvmebuf;453 spin_lock_init(&ctxp->ctxlock);454 455#ifdef CONFIG_SCSI_LPFC_DEBUG_FS456 /* NOTE: isr time stamp is stale when context is re-assigned*/457 if (ctxp->ts_isr_cmd) {458 ctxp->ts_cmd_nvme = 0;459 ctxp->ts_nvme_data = 0;460 ctxp->ts_data_wqput = 0;461 ctxp->ts_isr_data = 0;462 ctxp->ts_data_nvme = 0;463 ctxp->ts_nvme_status = 0;464 ctxp->ts_status_wqput = 0;465 ctxp->ts_isr_status = 0;466 ctxp->ts_status_nvme = 0;467 }468#endif469 atomic_inc(&tgtp->rcv_fcp_cmd_in);470 471 /* Indicate that a replacement buffer has been posted */472 spin_lock_irqsave(&ctxp->ctxlock, iflag);473 ctxp->flag |= LPFC_NVME_CTX_REUSE_WQ;474 spin_unlock_irqrestore(&ctxp->ctxlock, iflag);475 476 if (!queue_work(phba->wq, &ctx_buf->defer_work)) {477 atomic_inc(&tgtp->rcv_fcp_cmd_drop);478 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,479 "6181 Unable to queue deferred work "480 "for oxid x%x. "481 "FCP Drop IO [x%x x%x x%x]\n",482 ctxp->oxid,483 atomic_read(&tgtp->rcv_fcp_cmd_in),484 atomic_read(&tgtp->rcv_fcp_cmd_out),485 atomic_read(&tgtp->xmt_fcp_release));486 487 spin_lock_irqsave(&ctxp->ctxlock, iflag);488 lpfc_nvmet_defer_release(phba, ctxp);489 spin_unlock_irqrestore(&ctxp->ctxlock, iflag);490 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid);491 }492 return;493 }494 spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock, iflag);495 496 /*497 * Use the CPU context list, from the MRQ the IO was received on498 * (ctxp->idx), to save context structure.499 */500 spin_lock_irqsave(&phba->sli4_hba.t_active_list_lock, iflag);501 list_del_init(&ctxp->list);502 spin_unlock_irqrestore(&phba->sli4_hba.t_active_list_lock, iflag);503 cpu = raw_smp_processor_id();504 infop = lpfc_get_ctx_list(phba, cpu, ctxp->idx);505 spin_lock_irqsave(&infop->nvmet_ctx_list_lock, iflag);506 list_add_tail(&ctx_buf->list, &infop->nvmet_ctx_list);507 infop->nvmet_ctx_list_cnt++;508 spin_unlock_irqrestore(&infop->nvmet_ctx_list_lock, iflag);509#endif510}511 512#ifdef CONFIG_SCSI_LPFC_DEBUG_FS513static void514lpfc_nvmet_ktime(struct lpfc_hba *phba,515 struct lpfc_async_xchg_ctx *ctxp)516{517 uint64_t seg1, seg2, seg3, seg4, seg5;518 uint64_t seg6, seg7, seg8, seg9, seg10;519 uint64_t segsum;520 521 if (!ctxp->ts_isr_cmd || !ctxp->ts_cmd_nvme ||522 !ctxp->ts_nvme_data || !ctxp->ts_data_wqput ||523 !ctxp->ts_isr_data || !ctxp->ts_data_nvme ||524 !ctxp->ts_nvme_status || !ctxp->ts_status_wqput ||525 !ctxp->ts_isr_status || !ctxp->ts_status_nvme)526 return;527 528 if (ctxp->ts_status_nvme < ctxp->ts_isr_cmd)529 return;530 if (ctxp->ts_isr_cmd > ctxp->ts_cmd_nvme)531 return;532 if (ctxp->ts_cmd_nvme > ctxp->ts_nvme_data)533 return;534 if (ctxp->ts_nvme_data > ctxp->ts_data_wqput)535 return;536 if (ctxp->ts_data_wqput > ctxp->ts_isr_data)537 return;538 if (ctxp->ts_isr_data > ctxp->ts_data_nvme)539 return;540 if (ctxp->ts_data_nvme > ctxp->ts_nvme_status)541 return;542 if (ctxp->ts_nvme_status > ctxp->ts_status_wqput)543 return;544 if (ctxp->ts_status_wqput > ctxp->ts_isr_status)545 return;546 if (ctxp->ts_isr_status > ctxp->ts_status_nvme)547 return;548 /*549 * Segment 1 - Time from FCP command received by MSI-X ISR550 * to FCP command is passed to NVME Layer.551 * Segment 2 - Time from FCP command payload handed552 * off to NVME Layer to Driver receives a Command op553 * from NVME Layer.554 * Segment 3 - Time from Driver receives a Command op555 * from NVME Layer to Command is put on WQ.556 * Segment 4 - Time from Driver WQ put is done557 * to MSI-X ISR for Command cmpl.558 * Segment 5 - Time from MSI-X ISR for Command cmpl to559 * Command cmpl is passed to NVME Layer.560 * Segment 6 - Time from Command cmpl is passed to NVME561 * Layer to Driver receives a RSP op from NVME Layer.562 * Segment 7 - Time from Driver receives a RSP op from563 * NVME Layer to WQ put is done on TRSP FCP Status.564 * Segment 8 - Time from Driver WQ put is done on TRSP565 * FCP Status to MSI-X ISR for TRSP cmpl.566 * Segment 9 - Time from MSI-X ISR for TRSP cmpl to567 * TRSP cmpl is passed to NVME Layer.568 * Segment 10 - Time from FCP command received by569 * MSI-X ISR to command is completed on wire.570 * (Segments 1 thru 8) for READDATA / WRITEDATA571 * (Segments 1 thru 4) for READDATA_RSP572 */573 seg1 = ctxp->ts_cmd_nvme - ctxp->ts_isr_cmd;574 segsum = seg1;575 576 seg2 = ctxp->ts_nvme_data - ctxp->ts_isr_cmd;577 if (segsum > seg2)578 return;579 seg2 -= segsum;580 segsum += seg2;581 582 seg3 = ctxp->ts_data_wqput - ctxp->ts_isr_cmd;583 if (segsum > seg3)584 return;585 seg3 -= segsum;586 segsum += seg3;587 588 seg4 = ctxp->ts_isr_data - ctxp->ts_isr_cmd;589 if (segsum > seg4)590 return;591 seg4 -= segsum;592 segsum += seg4;593 594 seg5 = ctxp->ts_data_nvme - ctxp->ts_isr_cmd;595 if (segsum > seg5)596 return;597 seg5 -= segsum;598 segsum += seg5;599 600 601 /* For auto rsp commands seg6 thru seg10 will be 0 */602 if (ctxp->ts_nvme_status > ctxp->ts_data_nvme) {603 seg6 = ctxp->ts_nvme_status - ctxp->ts_isr_cmd;604 if (segsum > seg6)605 return;606 seg6 -= segsum;607 segsum += seg6;608 609 seg7 = ctxp->ts_status_wqput - ctxp->ts_isr_cmd;610 if (segsum > seg7)611 return;612 seg7 -= segsum;613 segsum += seg7;614 615 seg8 = ctxp->ts_isr_status - ctxp->ts_isr_cmd;616 if (segsum > seg8)617 return;618 seg8 -= segsum;619 segsum += seg8;620 621 seg9 = ctxp->ts_status_nvme - ctxp->ts_isr_cmd;622 if (segsum > seg9)623 return;624 seg9 -= segsum;625 segsum += seg9;626 627 if (ctxp->ts_isr_status < ctxp->ts_isr_cmd)628 return;629 seg10 = (ctxp->ts_isr_status -630 ctxp->ts_isr_cmd);631 } else {632 if (ctxp->ts_isr_data < ctxp->ts_isr_cmd)633 return;634 seg6 = 0;635 seg7 = 0;636 seg8 = 0;637 seg9 = 0;638 seg10 = (ctxp->ts_isr_data - ctxp->ts_isr_cmd);639 }640 641 phba->ktime_seg1_total += seg1;642 if (seg1 < phba->ktime_seg1_min)643 phba->ktime_seg1_min = seg1;644 else if (seg1 > phba->ktime_seg1_max)645 phba->ktime_seg1_max = seg1;646 647 phba->ktime_seg2_total += seg2;648 if (seg2 < phba->ktime_seg2_min)649 phba->ktime_seg2_min = seg2;650 else if (seg2 > phba->ktime_seg2_max)651 phba->ktime_seg2_max = seg2;652 653 phba->ktime_seg3_total += seg3;654 if (seg3 < phba->ktime_seg3_min)655 phba->ktime_seg3_min = seg3;656 else if (seg3 > phba->ktime_seg3_max)657 phba->ktime_seg3_max = seg3;658 659 phba->ktime_seg4_total += seg4;660 if (seg4 < phba->ktime_seg4_min)661 phba->ktime_seg4_min = seg4;662 else if (seg4 > phba->ktime_seg4_max)663 phba->ktime_seg4_max = seg4;664 665 phba->ktime_seg5_total += seg5;666 if (seg5 < phba->ktime_seg5_min)667 phba->ktime_seg5_min = seg5;668 else if (seg5 > phba->ktime_seg5_max)669 phba->ktime_seg5_max = seg5;670 671 phba->ktime_data_samples++;672 if (!seg6)673 goto out;674 675 phba->ktime_seg6_total += seg6;676 if (seg6 < phba->ktime_seg6_min)677 phba->ktime_seg6_min = seg6;678 else if (seg6 > phba->ktime_seg6_max)679 phba->ktime_seg6_max = seg6;680 681 phba->ktime_seg7_total += seg7;682 if (seg7 < phba->ktime_seg7_min)683 phba->ktime_seg7_min = seg7;684 else if (seg7 > phba->ktime_seg7_max)685 phba->ktime_seg7_max = seg7;686 687 phba->ktime_seg8_total += seg8;688 if (seg8 < phba->ktime_seg8_min)689 phba->ktime_seg8_min = seg8;690 else if (seg8 > phba->ktime_seg8_max)691 phba->ktime_seg8_max = seg8;692 693 phba->ktime_seg9_total += seg9;694 if (seg9 < phba->ktime_seg9_min)695 phba->ktime_seg9_min = seg9;696 else if (seg9 > phba->ktime_seg9_max)697 phba->ktime_seg9_max = seg9;698out:699 phba->ktime_seg10_total += seg10;700 if (seg10 < phba->ktime_seg10_min)701 phba->ktime_seg10_min = seg10;702 else if (seg10 > phba->ktime_seg10_max)703 phba->ktime_seg10_max = seg10;704 phba->ktime_status_samples++;705}706#endif707 708/**709 * lpfc_nvmet_xmt_fcp_op_cmp - Completion handler for FCP Response710 * @phba: Pointer to HBA context object.711 * @cmdwqe: Pointer to driver command WQE object.712 * @rspwqe: Pointer to driver response WQE object.713 *714 * The function is called from SLI ring event handler with no715 * lock held. This function is the completion handler for NVME FCP commands716 * The function frees memory resources used for the NVME commands.717 **/718static void719lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,720 struct lpfc_iocbq *rspwqe)721{722 struct lpfc_nvmet_tgtport *tgtp;723 struct nvmefc_tgt_fcp_req *rsp;724 struct lpfc_async_xchg_ctx *ctxp;725 uint32_t status, result, op, logerr;726 struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;727#ifdef CONFIG_SCSI_LPFC_DEBUG_FS728 int id;729#endif730 731 ctxp = cmdwqe->context_un.axchg;732 ctxp->flag &= ~LPFC_NVME_IO_INP;733 734 rsp = &ctxp->hdlrctx.fcp_req;735 op = rsp->op;736 737 status = bf_get(lpfc_wcqe_c_status, wcqe);738 result = wcqe->parameter;739 740 if (phba->targetport)741 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;742 else743 tgtp = NULL;744 745 lpfc_nvmeio_data(phba, "NVMET FCP CMPL: xri x%x op x%x status x%x\n",746 ctxp->oxid, op, status);747 748 if (status) {749 rsp->fcp_error = NVME_SC_DATA_XFER_ERROR;750 rsp->transferred_length = 0;751 if (tgtp) {752 atomic_inc(&tgtp->xmt_fcp_rsp_error);753 if (result == IOERR_ABORT_REQUESTED)754 atomic_inc(&tgtp->xmt_fcp_rsp_aborted);755 }756 757 logerr = LOG_NVME_IOERR;758 759 /* pick up SLI4 exhange busy condition */760 if (bf_get(lpfc_wcqe_c_xb, wcqe)) {761 ctxp->flag |= LPFC_NVME_XBUSY;762 logerr |= LOG_NVME_ABTS;763 if (tgtp)764 atomic_inc(&tgtp->xmt_fcp_rsp_xb_set);765 766 } else {767 ctxp->flag &= ~LPFC_NVME_XBUSY;768 }769 770 lpfc_printf_log(phba, KERN_INFO, logerr,771 "6315 IO Error Cmpl oxid: x%x xri: x%x %x/%x "772 "XBUSY:x%x\n",773 ctxp->oxid, ctxp->ctxbuf->sglq->sli4_xritag,774 status, result, ctxp->flag);775 776 } else {777 rsp->fcp_error = NVME_SC_SUCCESS;778 if (op == NVMET_FCOP_RSP)779 rsp->transferred_length = rsp->rsplen;780 else781 rsp->transferred_length = rsp->transfer_length;782 if (tgtp)783 atomic_inc(&tgtp->xmt_fcp_rsp_cmpl);784 }785 786 if ((op == NVMET_FCOP_READDATA_RSP) ||787 (op == NVMET_FCOP_RSP)) {788 /* Sanity check */789 ctxp->state = LPFC_NVME_STE_DONE;790 ctxp->entry_cnt++;791 792#ifdef CONFIG_SCSI_LPFC_DEBUG_FS793 if (ctxp->ts_cmd_nvme) {794 if (rsp->op == NVMET_FCOP_READDATA_RSP) {795 ctxp->ts_isr_data =796 cmdwqe->isr_timestamp;797 ctxp->ts_data_nvme =798 ktime_get_ns();799 ctxp->ts_nvme_status =800 ctxp->ts_data_nvme;801 ctxp->ts_status_wqput =802 ctxp->ts_data_nvme;803 ctxp->ts_isr_status =804 ctxp->ts_data_nvme;805 ctxp->ts_status_nvme =806 ctxp->ts_data_nvme;807 } else {808 ctxp->ts_isr_status =809 cmdwqe->isr_timestamp;810 ctxp->ts_status_nvme =811 ktime_get_ns();812 }813 }814#endif815 rsp->done(rsp);816#ifdef CONFIG_SCSI_LPFC_DEBUG_FS817 if (ctxp->ts_cmd_nvme)818 lpfc_nvmet_ktime(phba, ctxp);819#endif820 /* lpfc_nvmet_xmt_fcp_release() will recycle the context */821 } else {822 ctxp->entry_cnt++;823 memset_startat(cmdwqe, 0, cmd_flag);824#ifdef CONFIG_SCSI_LPFC_DEBUG_FS825 if (ctxp->ts_cmd_nvme) {826 ctxp->ts_isr_data = cmdwqe->isr_timestamp;827 ctxp->ts_data_nvme = ktime_get_ns();828 }829#endif830 rsp->done(rsp);831 }832#ifdef CONFIG_SCSI_LPFC_DEBUG_FS833 if (phba->hdwqstat_on & LPFC_CHECK_NVMET_IO) {834 id = raw_smp_processor_id();835 this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io);836 if (ctxp->cpu != id)837 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,838 "6704 CPU Check cmdcmpl: "839 "cpu %d expect %d\n",840 id, ctxp->cpu);841 }842#endif843}844 845/**846 * __lpfc_nvme_xmt_ls_rsp - Generic service routine to issue transmit847 * an NVME LS rsp for a prior NVME LS request that was received.848 * @axchg: pointer to exchange context for the NVME LS request the response849 * is for.850 * @ls_rsp: pointer to the transport LS RSP that is to be sent851 * @xmt_ls_rsp_cmp: completion routine to call upon RSP transmit done852 *853 * This routine is used to format and send a WQE to transmit a NVME LS854 * Response. The response is for a prior NVME LS request that was855 * received and posted to the transport.856 *857 * Returns:858 * 0 : if response successfully transmit859 * non-zero : if response failed to transmit, of the form -Exxx.860 **/861int862__lpfc_nvme_xmt_ls_rsp(struct lpfc_async_xchg_ctx *axchg,863 struct nvmefc_ls_rsp *ls_rsp,864 void (*xmt_ls_rsp_cmp)(struct lpfc_hba *phba,865 struct lpfc_iocbq *cmdwqe,866 struct lpfc_iocbq *rspwqe))867{868 struct lpfc_hba *phba = axchg->phba;869 struct hbq_dmabuf *nvmebuf = (struct hbq_dmabuf *)axchg->rqb_buffer;870 struct lpfc_iocbq *nvmewqeq;871 struct lpfc_dmabuf dmabuf;872 struct ulp_bde64 bpl;873 int rc;874 875 if (test_bit(FC_UNLOADING, &phba->pport->load_flag))876 return -ENODEV;877 878 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,879 "6023 NVMEx LS rsp oxid x%x\n", axchg->oxid);880 881 if (axchg->state != LPFC_NVME_STE_LS_RCV || axchg->entry_cnt != 1) {882 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,883 "6412 NVMEx LS rsp state mismatch "884 "oxid x%x: %d %d\n",885 axchg->oxid, axchg->state, axchg->entry_cnt);886 return -EALREADY;887 }888 axchg->state = LPFC_NVME_STE_LS_RSP;889 axchg->entry_cnt++;890 891 nvmewqeq = lpfc_nvmet_prep_ls_wqe(phba, axchg, ls_rsp->rspdma,892 ls_rsp->rsplen);893 if (nvmewqeq == NULL) {894 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,895 "6150 NVMEx LS Drop Rsp x%x: Prep\n",896 axchg->oxid);897 rc = -ENOMEM;898 goto out_free_buf;899 }900 901 /* Save numBdes for bpl2sgl */902 nvmewqeq->num_bdes = 1;903 nvmewqeq->hba_wqidx = 0;904 nvmewqeq->bpl_dmabuf = &dmabuf;905 dmabuf.virt = &bpl;906 bpl.addrLow = nvmewqeq->wqe.xmit_sequence.bde.addrLow;907 bpl.addrHigh = nvmewqeq->wqe.xmit_sequence.bde.addrHigh;908 bpl.tus.f.bdeSize = ls_rsp->rsplen;909 bpl.tus.f.bdeFlags = 0;910 bpl.tus.w = le32_to_cpu(bpl.tus.w);911 /*912 * Note: although we're using stack space for the dmabuf, the913 * call to lpfc_sli4_issue_wqe is synchronous, so it will not914 * be referenced after it returns back to this routine.915 */916 917 nvmewqeq->cmd_cmpl = xmt_ls_rsp_cmp;918 nvmewqeq->context_un.axchg = axchg;919 920 lpfc_nvmeio_data(phba, "NVMEx LS RSP: xri x%x wqidx x%x len x%x\n",921 axchg->oxid, nvmewqeq->hba_wqidx, ls_rsp->rsplen);922 923 rc = lpfc_sli4_issue_wqe(phba, axchg->hdwq, nvmewqeq);924 925 /* clear to be sure there's no reference */926 nvmewqeq->bpl_dmabuf = NULL;927 928 if (rc == WQE_SUCCESS) {929 /*930 * Okay to repost buffer here, but wait till cmpl931 * before freeing ctxp and iocbq.932 */933 lpfc_in_buf_free(phba, &nvmebuf->dbuf);934 return 0;935 }936 937 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,938 "6151 NVMEx LS RSP x%x: failed to transmit %d\n",939 axchg->oxid, rc);940 941 rc = -ENXIO;942 943 lpfc_nlp_put(nvmewqeq->ndlp);944 945out_free_buf:946 /* Give back resources */947 lpfc_in_buf_free(phba, &nvmebuf->dbuf);948 949 /*950 * As transport doesn't track completions of responses, if the rsp951 * fails to send, the transport will effectively ignore the rsp952 * and consider the LS done. However, the driver has an active953 * exchange open for the LS - so be sure to abort the exchange954 * if the response isn't sent.955 */956 lpfc_nvme_unsol_ls_issue_abort(phba, axchg, axchg->sid, axchg->oxid);957 return rc;958}959 960/**961 * lpfc_nvmet_xmt_ls_rsp - Transmit NVME LS response962 * @tgtport: pointer to target port that NVME LS is to be transmit from.963 * @ls_rsp: pointer to the transport LS RSP that is to be sent964 *965 * Driver registers this routine to transmit responses for received NVME966 * LS requests.967 *968 * This routine is used to format and send a WQE to transmit a NVME LS969 * Response. The ls_rsp is used to reverse-map the LS to the original970 * NVME LS request sequence, which provides addressing information for971 * the remote port the LS to be sent to, as well as the exchange id972 * that is the LS is bound to.973 *974 * Returns:975 * 0 : if response successfully transmit976 * non-zero : if response failed to transmit, of the form -Exxx.977 **/978static int979lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport,980 struct nvmefc_ls_rsp *ls_rsp)981{982 struct lpfc_async_xchg_ctx *axchg =983 container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp);984 struct lpfc_nvmet_tgtport *nvmep = tgtport->private;985 int rc;986 987 if (test_bit(FC_UNLOADING, &axchg->phba->pport->load_flag))988 return -ENODEV;989 990 rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, lpfc_nvmet_xmt_ls_rsp_cmp);991 992 if (rc) {993 atomic_inc(&nvmep->xmt_ls_drop);994 /*995 * unless the failure is due to having already sent996 * the response, an abort will be generated for the997 * exchange if the rsp can't be sent.998 */999 if (rc != -EALREADY)1000 atomic_inc(&nvmep->xmt_ls_abort);1001 return rc;1002 }1003 1004 atomic_inc(&nvmep->xmt_ls_rsp);1005 return 0;1006}1007 1008static int1009lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport,1010 struct nvmefc_tgt_fcp_req *rsp)1011{1012 struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;1013 struct lpfc_async_xchg_ctx *ctxp =1014 container_of(rsp, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req);1015 struct lpfc_hba *phba = ctxp->phba;1016 struct lpfc_queue *wq;1017 struct lpfc_iocbq *nvmewqeq;1018 struct lpfc_sli_ring *pring;1019 unsigned long iflags;1020 int rc;1021#ifdef CONFIG_SCSI_LPFC_DEBUG_FS1022 int id;1023#endif1024 1025 if (test_bit(FC_UNLOADING, &phba->pport->load_flag)) {1026 rc = -ENODEV;1027 goto aerr;1028 }1029 1030#ifdef CONFIG_SCSI_LPFC_DEBUG_FS1031 if (ctxp->ts_cmd_nvme) {1032 if (rsp->op == NVMET_FCOP_RSP)1033 ctxp->ts_nvme_status = ktime_get_ns();1034 else1035 ctxp->ts_nvme_data = ktime_get_ns();1036 }1037 1038 /* Setup the hdw queue if not already set */1039 if (!ctxp->hdwq)1040 ctxp->hdwq = &phba->sli4_hba.hdwq[rsp->hwqid];1041 1042 if (phba->hdwqstat_on & LPFC_CHECK_NVMET_IO) {1043 id = raw_smp_processor_id();1044 this_cpu_inc(phba->sli4_hba.c_stat->xmt_io);1045 if (rsp->hwqid != id)1046 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,1047 "6705 CPU Check OP: "1048 "cpu %d expect %d\n",1049 id, rsp->hwqid);1050 ctxp->cpu = id; /* Setup cpu for cmpl check */1051 }1052#endif1053 1054 /* Sanity check */1055 if ((ctxp->flag & LPFC_NVME_ABTS_RCV) ||1056 (ctxp->state == LPFC_NVME_STE_ABORT)) {1057 atomic_inc(&lpfc_nvmep->xmt_fcp_drop);1058 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,1059 "6102 IO oxid x%x aborted\n",1060 ctxp->oxid);1061 rc = -ENXIO;1062 goto aerr;1063 }1064 1065 nvmewqeq = lpfc_nvmet_prep_fcp_wqe(phba, ctxp);1066 if (nvmewqeq == NULL) {1067 atomic_inc(&lpfc_nvmep->xmt_fcp_drop);1068 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,1069 "6152 FCP Drop IO x%x: Prep\n",1070 ctxp->oxid);1071 rc = -ENXIO;1072 goto aerr;1073 }1074 1075 nvmewqeq->cmd_cmpl = lpfc_nvmet_xmt_fcp_op_cmp;1076 nvmewqeq->context_un.axchg = ctxp;1077 nvmewqeq->cmd_flag |= LPFC_IO_NVMET;1078 ctxp->wqeq->hba_wqidx = rsp->hwqid;1079 1080 lpfc_nvmeio_data(phba, "NVMET FCP CMND: xri x%x op x%x len x%x\n",1081 ctxp->oxid, rsp->op, rsp->rsplen);1082 1083 ctxp->flag |= LPFC_NVME_IO_INP;1084 rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, nvmewqeq);1085 if (rc == WQE_SUCCESS) {1086#ifdef CONFIG_SCSI_LPFC_DEBUG_FS1087 if (!ctxp->ts_cmd_nvme)1088 return 0;1089 if (rsp->op == NVMET_FCOP_RSP)1090 ctxp->ts_status_wqput = ktime_get_ns();1091 else1092 ctxp->ts_data_wqput = ktime_get_ns();1093#endif1094 return 0;1095 }1096 1097 if (rc == -EBUSY) {1098 /*1099 * WQ was full, so queue nvmewqeq to be sent after1100 * WQE release CQE1101 */1102 ctxp->flag |= LPFC_NVME_DEFER_WQFULL;1103 wq = ctxp->hdwq->io_wq;1104 pring = wq->pring;1105 spin_lock_irqsave(&pring->ring_lock, iflags);1106 list_add_tail(&nvmewqeq->list, &wq->wqfull_list);1107 wq->q_flag |= HBA_NVMET_WQFULL;1108 spin_unlock_irqrestore(&pring->ring_lock, iflags);1109 atomic_inc(&lpfc_nvmep->defer_wqfull);1110 return 0;1111 }1112 1113 /* Give back resources */1114 atomic_inc(&lpfc_nvmep->xmt_fcp_drop);1115 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,1116 "6153 FCP Drop IO x%x: Issue: %d\n",1117 ctxp->oxid, rc);1118 1119 ctxp->wqeq->hba_wqidx = 0;1120 nvmewqeq->context_un.axchg = NULL;1121 nvmewqeq->bpl_dmabuf = NULL;1122 rc = -EBUSY;1123aerr:1124 return rc;1125}1126 1127static void1128lpfc_nvmet_targetport_delete(struct nvmet_fc_target_port *targetport)1129{1130 struct lpfc_nvmet_tgtport *tport = targetport->private;1131 1132 /* release any threads waiting for the unreg to complete */1133 if (tport->phba->targetport)1134 complete(tport->tport_unreg_cmp);1135}1136 1137static void1138lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport,1139 struct nvmefc_tgt_fcp_req *req)1140{1141 struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;1142 struct lpfc_async_xchg_ctx *ctxp =1143 container_of(req, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req);1144 struct lpfc_hba *phba = ctxp->phba;1145 struct lpfc_queue *wq;1146 unsigned long flags;1147 1148 if (test_bit(FC_UNLOADING, &phba->pport->load_flag))1149 return;1150 1151 if (!ctxp->hdwq)1152 ctxp->hdwq = &phba->sli4_hba.hdwq[0];1153 1154 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,1155 "6103 NVMET Abort op: oxid x%x flg x%x ste %d\n",1156 ctxp->oxid, ctxp->flag, ctxp->state);1157 1158 lpfc_nvmeio_data(phba, "NVMET FCP ABRT: xri x%x flg x%x ste x%x\n",1159 ctxp->oxid, ctxp->flag, ctxp->state);1160 1161 atomic_inc(&lpfc_nvmep->xmt_fcp_abort);1162 1163 spin_lock_irqsave(&ctxp->ctxlock, flags);1164 1165 /* Since iaab/iaar are NOT set, we need to check1166 * if the firmware is in process of aborting IO1167 */1168 if (ctxp->flag & (LPFC_NVME_XBUSY | LPFC_NVME_ABORT_OP)) {1169 spin_unlock_irqrestore(&ctxp->ctxlock, flags);1170 return;1171 }1172 ctxp->flag |= LPFC_NVME_ABORT_OP;1173 1174 if (ctxp->flag & LPFC_NVME_DEFER_WQFULL) {1175 spin_unlock_irqrestore(&ctxp->ctxlock, flags);1176 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid,1177 ctxp->oxid);1178 wq = ctxp->hdwq->io_wq;1179 lpfc_nvmet_wqfull_flush(phba, wq, ctxp);1180 return;1181 }1182 spin_unlock_irqrestore(&ctxp->ctxlock, flags);1183 1184 /* A state of LPFC_NVME_STE_RCV means we have just received1185 * the NVME command and have not started processing it.1186 * (by issuing any IO WQEs on this exchange yet)1187 */1188 if (ctxp->state == LPFC_NVME_STE_RCV)1189 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid,1190 ctxp->oxid);1191 else1192 lpfc_nvmet_sol_fcp_issue_abort(phba, ctxp, ctxp->sid,1193 ctxp->oxid);1194}1195 1196static void1197lpfc_nvmet_xmt_fcp_release(struct nvmet_fc_target_port *tgtport,1198 struct nvmefc_tgt_fcp_req *rsp)1199{1200 struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;1201 struct lpfc_async_xchg_ctx *ctxp =1202 container_of(rsp, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req);1203 struct lpfc_hba *phba = ctxp->phba;1204 unsigned long flags;1205 bool aborting = false;1206 1207 spin_lock_irqsave(&ctxp->ctxlock, flags);1208 if (ctxp->flag & LPFC_NVME_XBUSY)1209 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,1210 "6027 NVMET release with XBUSY flag x%x"1211 " oxid x%x\n",1212 ctxp->flag, ctxp->oxid);1213 else if (ctxp->state != LPFC_NVME_STE_DONE &&1214 ctxp->state != LPFC_NVME_STE_ABORT)1215 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,1216 "6413 NVMET release bad state %d %d oxid x%x\n",1217 ctxp->state, ctxp->entry_cnt, ctxp->oxid);1218 1219 if ((ctxp->flag & LPFC_NVME_ABORT_OP) ||1220 (ctxp->flag & LPFC_NVME_XBUSY)) {1221 aborting = true;1222 /* let the abort path do the real release */1223 lpfc_nvmet_defer_release(phba, ctxp);1224 }1225 spin_unlock_irqrestore(&ctxp->ctxlock, flags);1226 1227 lpfc_nvmeio_data(phba, "NVMET FCP FREE: xri x%x ste %d abt %d\n", ctxp->oxid,1228 ctxp->state, aborting);1229 1230 atomic_inc(&lpfc_nvmep->xmt_fcp_release);1231 ctxp->flag &= ~LPFC_NVME_TNOTIFY;1232 1233 if (aborting)1234 return;1235 1236 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);1237}1238 1239static void1240lpfc_nvmet_defer_rcv(struct nvmet_fc_target_port *tgtport,1241 struct nvmefc_tgt_fcp_req *rsp)1242{1243 struct lpfc_nvmet_tgtport *tgtp;1244 struct lpfc_async_xchg_ctx *ctxp =1245 container_of(rsp, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req);1246 struct rqb_dmabuf *nvmebuf = ctxp->rqb_buffer;1247 struct lpfc_hba *phba = ctxp->phba;1248 unsigned long iflag;1249 1250 1251 lpfc_nvmeio_data(phba, "NVMET DEFERRCV: xri x%x sz %d CPU %02x\n",1252 ctxp->oxid, ctxp->size, raw_smp_processor_id());1253 1254 if (!nvmebuf) {1255 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,1256 "6425 Defer rcv: no buffer oxid x%x: "1257 "flg %x ste %x\n",1258 ctxp->oxid, ctxp->flag, ctxp->state);1259 return;1260 }1261 1262 tgtp = phba->targetport->private;1263 if (tgtp)1264 atomic_inc(&tgtp->rcv_fcp_cmd_defer);1265 1266 /* Free the nvmebuf since a new buffer already replaced it */1267 nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf);1268 spin_lock_irqsave(&ctxp->ctxlock, iflag);1269 ctxp->rqb_buffer = NULL;1270 spin_unlock_irqrestore(&ctxp->ctxlock, iflag);1271}1272 1273/**1274 * lpfc_nvmet_ls_req_cmp - completion handler for a nvme ls request1275 * @phba: Pointer to HBA context object1276 * @cmdwqe: Pointer to driver command WQE object.1277 * @rspwqe: Pointer to driver response WQE object.1278 *1279 * This function is the completion handler for NVME LS requests.1280 * The function updates any states and statistics, then calls the1281 * generic completion handler to finish completion of the request.1282 **/1283static void1284lpfc_nvmet_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,1285 struct lpfc_iocbq *rspwqe)1286{1287 struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;1288 __lpfc_nvme_ls_req_cmp(phba, cmdwqe->vport, cmdwqe, wcqe);1289}1290 1291/**1292 * lpfc_nvmet_ls_req - Issue an Link Service request1293 * @targetport: pointer to target instance registered with nvmet transport.1294 * @hosthandle: hosthandle set by the driver in a prior ls_rqst_rcv.1295 * Driver sets this value to the ndlp pointer.1296 * @pnvme_lsreq: the transport nvme_ls_req structure for the LS1297 *1298 * Driver registers this routine to handle any link service request1299 * from the nvme_fc transport to a remote nvme-aware port.1300 *1301 * Return value :1302 * 0 - Success1303 * non-zero: various error codes, in form of -Exxx1304 **/1305static int1306lpfc_nvmet_ls_req(struct nvmet_fc_target_port *targetport,1307 void *hosthandle,1308 struct nvmefc_ls_req *pnvme_lsreq)1309{1310 struct lpfc_nvmet_tgtport *lpfc_nvmet = targetport->private;1311 struct lpfc_hba *phba;1312 struct lpfc_nodelist *ndlp;1313 int ret;1314 u32 hstate;1315 1316 if (!lpfc_nvmet)1317 return -EINVAL;1318 1319 phba = lpfc_nvmet->phba;1320 if (test_bit(FC_UNLOADING, &phba->pport->load_flag))1321 return -EINVAL;1322 1323 hstate = atomic_read(&lpfc_nvmet->state);1324 if (hstate == LPFC_NVMET_INV_HOST_ACTIVE)1325 return -EACCES;1326 1327 ndlp = (struct lpfc_nodelist *)hosthandle;1328 1329 ret = __lpfc_nvme_ls_req(phba->pport, ndlp, pnvme_lsreq,1330 lpfc_nvmet_ls_req_cmp);1331 1332 return ret;1333}1334 1335/**1336 * lpfc_nvmet_ls_abort - Abort a prior NVME LS request1337 * @targetport: Transport targetport, that LS was issued from.1338 * @hosthandle: hosthandle set by the driver in a prior ls_rqst_rcv.1339 * Driver sets this value to the ndlp pointer.1340 * @pnvme_lsreq: the transport nvme_ls_req structure for LS to be aborted1341 *1342 * Driver registers this routine to abort an NVME LS request that is1343 * in progress (from the transports perspective).1344 **/1345static void1346lpfc_nvmet_ls_abort(struct nvmet_fc_target_port *targetport,1347 void *hosthandle,1348 struct nvmefc_ls_req *pnvme_lsreq)1349{1350 struct lpfc_nvmet_tgtport *lpfc_nvmet = targetport->private;1351 struct lpfc_hba *phba;1352 struct lpfc_nodelist *ndlp;1353 int ret;1354 1355 phba = lpfc_nvmet->phba;1356 if (test_bit(FC_UNLOADING, &phba->pport->load_flag))1357 return;1358 1359 ndlp = (struct lpfc_nodelist *)hosthandle;1360 1361 ret = __lpfc_nvme_ls_abort(phba->pport, ndlp, pnvme_lsreq);1362 if (!ret)1363 atomic_inc(&lpfc_nvmet->xmt_ls_abort);1364}1365 1366static int1367lpfc_nvmet_host_traddr(void *hosthandle, u64 *wwnn, u64 *wwpn)1368{1369 struct lpfc_nodelist *ndlp = hosthandle;1370 1371 *wwnn = wwn_to_u64(ndlp->nlp_nodename.u.wwn);1372 *wwpn = wwn_to_u64(ndlp->nlp_portname.u.wwn);1373 return 0;1374}1375 1376static void1377lpfc_nvmet_host_release(void *hosthandle)1378{1379 struct lpfc_nodelist *ndlp = hosthandle;1380 struct lpfc_hba *phba = ndlp->phba;1381 struct lpfc_nvmet_tgtport *tgtp;1382 1383 if (!phba->targetport || !phba->targetport->private)1384 return;1385 1386 lpfc_printf_log(phba, KERN_ERR, LOG_NVME,1387 "6202 NVMET XPT releasing hosthandle x%px "1388 "DID x%x xflags x%x refcnt %d\n",1389 hosthandle, ndlp->nlp_DID, ndlp->fc4_xpt_flags,1390 kref_read(&ndlp->kref));1391 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;1392 spin_lock_irq(&ndlp->lock);1393 ndlp->fc4_xpt_flags &= ~NLP_XPT_HAS_HH;1394 spin_unlock_irq(&ndlp->lock);1395 lpfc_nlp_put(ndlp);1396 atomic_set(&tgtp->state, 0);1397}1398 1399static void1400lpfc_nvmet_discovery_event(struct nvmet_fc_target_port *tgtport)1401{1402 struct lpfc_nvmet_tgtport *tgtp;1403 struct lpfc_hba *phba;1404 uint32_t rc;1405 1406 tgtp = tgtport->private;1407 phba = tgtp->phba;1408 1409 rc = lpfc_issue_els_rscn(phba->pport, 0);1410 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,1411 "6420 NVMET subsystem change: Notification %s\n",1412 (rc) ? "Failed" : "Sent");1413}1414 1415static struct nvmet_fc_target_template lpfc_tgttemplate = {1416 .targetport_delete = lpfc_nvmet_targetport_delete,1417 .xmt_ls_rsp = lpfc_nvmet_xmt_ls_rsp,1418 .fcp_op = lpfc_nvmet_xmt_fcp_op,1419 .fcp_abort = lpfc_nvmet_xmt_fcp_abort,1420 .fcp_req_release = lpfc_nvmet_xmt_fcp_release,1421 .defer_rcv = lpfc_nvmet_defer_rcv,1422 .discovery_event = lpfc_nvmet_discovery_event,1423 .ls_req = lpfc_nvmet_ls_req,1424 .ls_abort = lpfc_nvmet_ls_abort,1425 .host_release = lpfc_nvmet_host_release,1426 .host_traddr = lpfc_nvmet_host_traddr,1427 1428 .max_hw_queues = 1,1429 .max_sgl_segments = LPFC_NVMET_DEFAULT_SEGS,1430 .max_dif_sgl_segments = LPFC_NVMET_DEFAULT_SEGS,1431 .dma_boundary = 0xFFFFFFFF,1432 1433 /* optional features */1434 .target_features = 0,1435 /* sizes of additional private data for data structures */1436 .target_priv_sz = sizeof(struct lpfc_nvmet_tgtport),1437 .lsrqst_priv_sz = 0,1438};1439 1440static void1441__lpfc_nvmet_clean_io_for_cpu(struct lpfc_hba *phba,1442 struct lpfc_nvmet_ctx_info *infop)1443{1444 struct lpfc_nvmet_ctxbuf *ctx_buf, *next_ctx_buf;1445 unsigned long flags;1446 1447 spin_lock_irqsave(&infop->nvmet_ctx_list_lock, flags);1448 list_for_each_entry_safe(ctx_buf, next_ctx_buf,1449 &infop->nvmet_ctx_list, list) {1450 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock);1451 list_del_init(&ctx_buf->list);1452 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock);1453 1454 spin_lock(&phba->hbalock);1455 __lpfc_clear_active_sglq(phba, ctx_buf->sglq->sli4_lxritag);1456 spin_unlock(&phba->hbalock);1457 1458 ctx_buf->sglq->state = SGL_FREED;1459 ctx_buf->sglq->ndlp = NULL;1460 1461 spin_lock(&phba->sli4_hba.sgl_list_lock);1462 list_add_tail(&ctx_buf->sglq->list,1463 &phba->sli4_hba.lpfc_nvmet_sgl_list);1464 spin_unlock(&phba->sli4_hba.sgl_list_lock);1465 1466 lpfc_sli_release_iocbq(phba, ctx_buf->iocbq);1467 kfree(ctx_buf->context);1468 }1469 spin_unlock_irqrestore(&infop->nvmet_ctx_list_lock, flags);1470}1471 1472static void1473lpfc_nvmet_cleanup_io_context(struct lpfc_hba *phba)1474{1475 struct lpfc_nvmet_ctx_info *infop;1476 int i, j;1477 1478 /* The first context list, MRQ 0 CPU 0 */1479 infop = phba->sli4_hba.nvmet_ctx_info;1480 if (!infop)1481 return;1482 1483 /* Cycle the entire CPU context list for every MRQ */1484 for (i = 0; i < phba->cfg_nvmet_mrq; i++) {1485 for_each_present_cpu(j) {1486 infop = lpfc_get_ctx_list(phba, j, i);1487 __lpfc_nvmet_clean_io_for_cpu(phba, infop);1488 }1489 }1490 kfree(phba->sli4_hba.nvmet_ctx_info);1491 phba->sli4_hba.nvmet_ctx_info = NULL;1492}1493 1494static int1495lpfc_nvmet_setup_io_context(struct lpfc_hba *phba)1496{1497 struct lpfc_nvmet_ctxbuf *ctx_buf;1498 struct lpfc_iocbq *nvmewqe;1499 union lpfc_wqe128 *wqe;1500 struct lpfc_nvmet_ctx_info *last_infop;1501 struct lpfc_nvmet_ctx_info *infop;1502 int i, j, idx, cpu;1503 1504 lpfc_printf_log(phba, KERN_INFO, LOG_NVME,1505 "6403 Allocate NVMET resources for %d XRIs\n",1506 phba->sli4_hba.nvmet_xri_cnt);1507 1508 phba->sli4_hba.nvmet_ctx_info = kcalloc(1509 phba->sli4_hba.num_possible_cpu * phba->cfg_nvmet_mrq,1510 sizeof(struct lpfc_nvmet_ctx_info), GFP_KERNEL);1511 if (!phba->sli4_hba.nvmet_ctx_info) {1512 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,1513 "6419 Failed allocate memory for "1514 "nvmet context lists\n");1515 return -ENOMEM;1516 }1517 1518 /*1519 * Assuming X CPUs in the system, and Y MRQs, allocate some1520 * lpfc_nvmet_ctx_info structures as follows:1521 *1522 * cpu0/mrq0 cpu1/mrq0 ... cpuX/mrq01523 * cpu0/mrq1 cpu1/mrq1 ... cpuX/mrq11524 * ...1525 * cpuX/mrqY cpuX/mrqY ... cpuX/mrqY1526 *1527 * Each line represents a MRQ "silo" containing an entry for1528 * every CPU.1529 *1530 * MRQ X is initially assumed to be associated with CPU X, thus1531 * contexts are initially distributed across all MRQs using1532 * the MRQ index (N) as follows cpuN/mrqN. When contexts are1533 * freed, the are freed to the MRQ silo based on the CPU number1534 * of the IO completion. Thus a context that was allocated for MRQ A1535 * whose IO completed on CPU B will be freed to cpuB/mrqA.1536 */1537 for_each_possible_cpu(i) {1538 for (j = 0; j < phba->cfg_nvmet_mrq; j++) {1539 infop = lpfc_get_ctx_list(phba, i, j);1540 INIT_LIST_HEAD(&infop->nvmet_ctx_list);1541 spin_lock_init(&infop->nvmet_ctx_list_lock);1542 infop->nvmet_ctx_list_cnt = 0;1543 }1544 }1545 1546 /*1547 * Setup the next CPU context info ptr for each MRQ.1548 * MRQ 0 will cycle thru CPUs 0 - X separately from1549 * MRQ 1 cycling thru CPUs 0 - X, and so on.1550 */1551 for (j = 0; j < phba->cfg_nvmet_mrq; j++) {1552 last_infop = lpfc_get_ctx_list(phba,1553 cpumask_first(cpu_present_mask),1554 j);1555 for (i = phba->sli4_hba.num_possible_cpu - 1; i >= 0; i--) {1556 infop = lpfc_get_ctx_list(phba, i, j);1557 infop->nvmet_ctx_next_cpu = last_infop;1558 last_infop = infop;1559 }1560 }1561 1562 /* For all nvmet xris, allocate resources needed to process a1563 * received command on a per xri basis.1564 */1565 idx = 0;1566 cpu = cpumask_first(cpu_present_mask);1567 for (i = 0; i < phba->sli4_hba.nvmet_xri_cnt; i++) {1568 ctx_buf = kzalloc(sizeof(*ctx_buf), GFP_KERNEL);1569 if (!ctx_buf) {1570 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,1571 "6404 Ran out of memory for NVMET\n");1572 return -ENOMEM;1573 }1574 1575 ctx_buf->context = kzalloc(sizeof(*ctx_buf->context),1576 GFP_KERNEL);1577 if (!ctx_buf->context) {1578 kfree(ctx_buf);1579 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,1580 "6405 Ran out of NVMET "1581 "context memory\n");1582 return -ENOMEM;1583 }1584 ctx_buf->context->ctxbuf = ctx_buf;1585 ctx_buf->context->state = LPFC_NVME_STE_FREE;1586 1587 ctx_buf->iocbq = lpfc_sli_get_iocbq(phba);1588 if (!ctx_buf->iocbq) {1589 kfree(ctx_buf->context);1590 kfree(ctx_buf);1591 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,1592 "6406 Ran out of NVMET iocb/WQEs\n");1593 return -ENOMEM;1594 }1595 ctx_buf->iocbq->cmd_flag = LPFC_IO_NVMET;1596 nvmewqe = ctx_buf->iocbq;1597 wqe = &nvmewqe->wqe;1598 1599 /* Initialize WQE */1600 memset(wqe, 0, sizeof(*wqe));1601 1602 ctx_buf->iocbq->cmd_dmabuf = NULL;1603 spin_lock(&phba->sli4_hba.sgl_list_lock);1604 ctx_buf->sglq = __lpfc_sli_get_nvmet_sglq(phba, ctx_buf->iocbq);1605 spin_unlock(&phba->sli4_hba.sgl_list_lock);1606 if (!ctx_buf->sglq) {1607 lpfc_sli_release_iocbq(phba, ctx_buf->iocbq);1608 kfree(ctx_buf->context);1609 kfree(ctx_buf);1610 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,1611 "6407 Ran out of NVMET XRIs\n");1612 return -ENOMEM;1613 }1614 INIT_WORK(&ctx_buf->defer_work, lpfc_nvmet_fcp_rqst_defer_work);1615 1616 /*1617 * Add ctx to MRQidx context list. Our initial assumption1618 * is MRQidx will be associated with CPUidx. This association1619 * can change on the fly.1620 */1621 infop = lpfc_get_ctx_list(phba, cpu, idx);1622 spin_lock(&infop->nvmet_ctx_list_lock);1623 list_add_tail(&ctx_buf->list, &infop->nvmet_ctx_list);1624 infop->nvmet_ctx_list_cnt++;1625 spin_unlock(&infop->nvmet_ctx_list_lock);1626 1627 /* Spread ctx structures evenly across all MRQs */1628 idx++;1629 if (idx >= phba->cfg_nvmet_mrq) {1630 idx = 0;1631 cpu = cpumask_first(cpu_present_mask);1632 continue;1633 }1634 cpu = lpfc_next_present_cpu(cpu);1635 }1636 1637 for_each_present_cpu(i) {1638 for (j = 0; j < phba->cfg_nvmet_mrq; j++) {1639 infop = lpfc_get_ctx_list(phba, i, j);1640 lpfc_printf_log(phba, KERN_INFO, LOG_NVME | LOG_INIT,1641 "6408 TOTAL NVMET ctx for CPU %d "1642 "MRQ %d: cnt %d nextcpu x%px\n",1643 i, j, infop->nvmet_ctx_list_cnt,1644 infop->nvmet_ctx_next_cpu);1645 }1646 }1647 return 0;1648}1649 1650int1651lpfc_nvmet_create_targetport(struct lpfc_hba *phba)1652{1653 struct lpfc_vport *vport = phba->pport;1654 struct lpfc_nvmet_tgtport *tgtp;1655 struct nvmet_fc_port_info pinfo;1656 int error;1657 1658 if (phba->targetport)1659 return 0;1660 1661 error = lpfc_nvmet_setup_io_context(phba);1662 if (error)1663 return error;1664 1665 memset(&pinfo, 0, sizeof(struct nvmet_fc_port_info));1666 pinfo.node_name = wwn_to_u64(vport->fc_nodename.u.wwn);1667 pinfo.port_name = wwn_to_u64(vport->fc_portname.u.wwn);1668 pinfo.port_id = vport->fc_myDID;1669 1670 /* We need to tell the transport layer + 1 because it takes page1671 * alignment into account. When space for the SGL is allocated we1672 * allocate + 3, one for cmd, one for rsp and one for this alignment1673 */1674 lpfc_tgttemplate.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;1675 lpfc_tgttemplate.max_hw_queues = phba->cfg_hdw_queue;1676 lpfc_tgttemplate.target_features = NVMET_FCTGTFEAT_READDATA_RSP;1677 1678#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))1679 error = nvmet_fc_register_targetport(&pinfo, &lpfc_tgttemplate,1680 &phba->pcidev->dev,1681 &phba->targetport);1682#else1683 error = -ENOENT;1684#endif1685 if (error) {1686 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,1687 "6025 Cannot register NVME targetport x%x: "1688 "portnm %llx nodenm %llx segs %d qs %d\n",1689 error,1690 pinfo.port_name, pinfo.node_name,1691 lpfc_tgttemplate.max_sgl_segments,1692 lpfc_tgttemplate.max_hw_queues);1693 phba->targetport = NULL;1694 phba->nvmet_support = 0;1695 1696 lpfc_nvmet_cleanup_io_context(phba);1697 1698 } else {1699 tgtp = (struct lpfc_nvmet_tgtport *)1700 phba->targetport->private;1701 tgtp->phba = phba;1702 1703 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,1704 "6026 Registered NVME "1705 "targetport: x%px, private x%px "1706 "portnm %llx nodenm %llx segs %d qs %d\n",1707 phba->targetport, tgtp,1708 pinfo.port_name, pinfo.node_name,1709 lpfc_tgttemplate.max_sgl_segments,1710 lpfc_tgttemplate.max_hw_queues);1711 1712 atomic_set(&tgtp->rcv_ls_req_in, 0);1713 atomic_set(&tgtp->rcv_ls_req_out, 0);1714 atomic_set(&tgtp->rcv_ls_req_drop, 0);1715 atomic_set(&tgtp->xmt_ls_abort, 0);1716 atomic_set(&tgtp->xmt_ls_abort_cmpl, 0);1717 atomic_set(&tgtp->xmt_ls_rsp, 0);1718 atomic_set(&tgtp->xmt_ls_drop, 0);1719 atomic_set(&tgtp->xmt_ls_rsp_error, 0);1720 atomic_set(&tgtp->xmt_ls_rsp_xb_set, 0);1721 atomic_set(&tgtp->xmt_ls_rsp_aborted, 0);1722 atomic_set(&tgtp->xmt_ls_rsp_cmpl, 0);1723 atomic_set(&tgtp->rcv_fcp_cmd_in, 0);1724 atomic_set(&tgtp->rcv_fcp_cmd_out, 0);1725 atomic_set(&tgtp->rcv_fcp_cmd_drop, 0);1726 atomic_set(&tgtp->xmt_fcp_drop, 0);1727 atomic_set(&tgtp->xmt_fcp_read_rsp, 0);1728 atomic_set(&tgtp->xmt_fcp_read, 0);1729 atomic_set(&tgtp->xmt_fcp_write, 0);1730 atomic_set(&tgtp->xmt_fcp_rsp, 0);1731 atomic_set(&tgtp->xmt_fcp_release, 0);1732 atomic_set(&tgtp->xmt_fcp_rsp_cmpl, 0);1733 atomic_set(&tgtp->xmt_fcp_rsp_error, 0);1734 atomic_set(&tgtp->xmt_fcp_rsp_xb_set, 0);1735 atomic_set(&tgtp->xmt_fcp_rsp_aborted, 0);1736 atomic_set(&tgtp->xmt_fcp_rsp_drop, 0);1737 atomic_set(&tgtp->xmt_fcp_xri_abort_cqe, 0);1738 atomic_set(&tgtp->xmt_fcp_abort, 0);1739 atomic_set(&tgtp->xmt_fcp_abort_cmpl, 0);1740 atomic_set(&tgtp->xmt_abort_unsol, 0);1741 atomic_set(&tgtp->xmt_abort_sol, 0);1742 atomic_set(&tgtp->xmt_abort_rsp, 0);1743 atomic_set(&tgtp->xmt_abort_rsp_error, 0);1744 atomic_set(&tgtp->defer_ctx, 0);1745 atomic_set(&tgtp->defer_fod, 0);1746 atomic_set(&tgtp->defer_wqfull, 0);1747 }1748 return error;1749}1750 1751int1752lpfc_nvmet_update_targetport(struct lpfc_hba *phba)1753{1754 struct lpfc_vport *vport = phba->pport;1755 1756 if (!phba->targetport)1757 return 0;1758 1759 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,1760 "6007 Update NVMET port x%px did x%x\n",1761 phba->targetport, vport->fc_myDID);1762 1763 phba->targetport->port_id = vport->fc_myDID;1764 return 0;1765}1766 1767/**1768 * lpfc_sli4_nvmet_xri_aborted - Fast-path process of nvmet xri abort1769 * @phba: pointer to lpfc hba data structure.1770 * @axri: pointer to the nvmet xri abort wcqe structure.1771 *1772 * This routine is invoked by the worker thread to process a SLI4 fast-path1773 * NVMET aborted xri.1774 **/1775void1776lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba,1777 struct sli4_wcqe_xri_aborted *axri)1778{1779#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))1780 uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri);1781 uint16_t rxid = bf_get(lpfc_wcqe_xa_remote_xid, axri);1782 struct lpfc_async_xchg_ctx *ctxp, *next_ctxp;1783 struct lpfc_nvmet_tgtport *tgtp;1784 struct nvmefc_tgt_fcp_req *req = NULL;1785 struct lpfc_nodelist *ndlp;1786 unsigned long iflag = 0;1787 int rrq_empty = 0;1788 bool released = false;1789 1790 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,1791 "6317 XB aborted xri x%x rxid x%x\n", xri, rxid);1792 1793 if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME))1794 return;1795 1796 if (phba->targetport) {1797 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;1798 atomic_inc(&tgtp->xmt_fcp_xri_abort_cqe);1799 }1800 1801 spin_lock_irqsave(&phba->sli4_hba.abts_nvmet_buf_list_lock, iflag);1802 list_for_each_entry_safe(ctxp, next_ctxp,1803 &phba->sli4_hba.lpfc_abts_nvmet_ctx_list,1804 list) {1805 if (ctxp->ctxbuf->sglq->sli4_xritag != xri)1806 continue;1807 1808 spin_unlock_irqrestore(&phba->sli4_hba.abts_nvmet_buf_list_lock,1809 iflag);1810 1811 spin_lock_irqsave(&ctxp->ctxlock, iflag);1812 /* Check if we already received a free context call1813 * and we have completed processing an abort situation.1814 */1815 if (ctxp->flag & LPFC_NVME_CTX_RLS &&1816 !(ctxp->flag & LPFC_NVME_ABORT_OP)) {1817 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock);1818 list_del_init(&ctxp->list);1819 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock);1820 released = true;1821 }1822 ctxp->flag &= ~LPFC_NVME_XBUSY;1823 spin_unlock_irqrestore(&ctxp->ctxlock, iflag);1824 1825 spin_lock_irqsave(&phba->rrq_list_lock, iflag);1826 rrq_empty = list_empty(&phba->active_rrq_list);1827 spin_unlock_irqrestore(&phba->rrq_list_lock, iflag);1828 ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);1829 if (ndlp &&1830 (ndlp->nlp_state == NLP_STE_UNMAPPED_NODE ||1831 ndlp->nlp_state == NLP_STE_MAPPED_NODE)) {1832 lpfc_set_rrq_active(phba, ndlp,1833 ctxp->ctxbuf->sglq->sli4_lxritag,1834 rxid, 1);1835 lpfc_sli4_abts_err_handler(phba, ndlp, axri);1836 }1837 1838 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,1839 "6318 XB aborted oxid x%x flg x%x (%x)\n",1840 ctxp->oxid, ctxp->flag, released);1841 if (released)1842 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);1843 1844 if (rrq_empty)1845 lpfc_worker_wake_up(phba);1846 return;1847 }1848 spin_unlock_irqrestore(&phba->sli4_hba.abts_nvmet_buf_list_lock, iflag);1849 ctxp = lpfc_nvmet_get_ctx_for_xri(phba, xri);1850 if (ctxp) {1851 /*1852 * Abort already done by FW, so BA_ACC sent.1853 * However, the transport may be unaware.1854 */1855 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,1856 "6323 NVMET Rcv ABTS xri x%x ctxp state x%x "1857 "flag x%x oxid x%x rxid x%x\n",1858 xri, ctxp->state, ctxp->flag, ctxp->oxid,1859 rxid);1860 1861 spin_lock_irqsave(&ctxp->ctxlock, iflag);1862 ctxp->flag |= LPFC_NVME_ABTS_RCV;1863 ctxp->state = LPFC_NVME_STE_ABORT;1864 spin_unlock_irqrestore(&ctxp->ctxlock, iflag);1865 1866 lpfc_nvmeio_data(phba,1867 "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n",1868 xri, raw_smp_processor_id(), 0);1869 1870 req = &ctxp->hdlrctx.fcp_req;1871 if (req)1872 nvmet_fc_rcv_fcp_abort(phba->targetport, req);1873 }1874#endif1875}1876 1877int1878lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport,1879 struct fc_frame_header *fc_hdr)1880{1881#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))1882 struct lpfc_hba *phba = vport->phba;1883 struct lpfc_async_xchg_ctx *ctxp, *next_ctxp;1884 struct nvmefc_tgt_fcp_req *rsp;1885 uint32_t sid;1886 uint16_t oxid, xri;1887 unsigned long iflag = 0;1888 1889 sid = sli4_sid_from_fc_hdr(fc_hdr);1890 oxid = be16_to_cpu(fc_hdr->fh_ox_id);1891 1892 spin_lock_irqsave(&phba->sli4_hba.abts_nvmet_buf_list_lock, iflag);1893 list_for_each_entry_safe(ctxp, next_ctxp,1894 &phba->sli4_hba.lpfc_abts_nvmet_ctx_list,1895 list) {1896 if (ctxp->oxid != oxid || ctxp->sid != sid)1897 continue;1898 1899 xri = ctxp->ctxbuf->sglq->sli4_xritag;1900 1901 spin_unlock_irqrestore(&phba->sli4_hba.abts_nvmet_buf_list_lock,1902 iflag);1903 spin_lock_irqsave(&ctxp->ctxlock, iflag);1904 ctxp->flag |= LPFC_NVME_ABTS_RCV;1905 spin_unlock_irqrestore(&ctxp->ctxlock, iflag);1906 1907 lpfc_nvmeio_data(phba,1908 "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n",1909 xri, raw_smp_processor_id(), 0);1910 1911 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,1912 "6319 NVMET Rcv ABTS:acc xri x%x\n", xri);1913 1914 rsp = &ctxp->hdlrctx.fcp_req;1915 nvmet_fc_rcv_fcp_abort(phba->targetport, rsp);1916 1917 /* Respond with BA_ACC accordingly */1918 lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 1);1919 return 0;1920 }1921 spin_unlock_irqrestore(&phba->sli4_hba.abts_nvmet_buf_list_lock, iflag);1922 /* check the wait list */1923 if (phba->sli4_hba.nvmet_io_wait_cnt) {1924 struct rqb_dmabuf *nvmebuf;1925 struct fc_frame_header *fc_hdr_tmp;1926 u32 sid_tmp;1927 u16 oxid_tmp;1928 bool found = false;1929 1930 spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag);1931 1932 /* match by oxid and s_id */1933 list_for_each_entry(nvmebuf,1934 &phba->sli4_hba.lpfc_nvmet_io_wait_list,1935 hbuf.list) {1936 fc_hdr_tmp = (struct fc_frame_header *)1937 (nvmebuf->hbuf.virt);1938 oxid_tmp = be16_to_cpu(fc_hdr_tmp->fh_ox_id);1939 sid_tmp = sli4_sid_from_fc_hdr(fc_hdr_tmp);1940 if (oxid_tmp != oxid || sid_tmp != sid)1941 continue;1942 1943 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,1944 "6321 NVMET Rcv ABTS oxid x%x from x%x "1945 "is waiting for a ctxp\n",1946 oxid, sid);1947 1948 list_del_init(&nvmebuf->hbuf.list);1949 phba->sli4_hba.nvmet_io_wait_cnt--;1950 found = true;1951 break;1952 }1953 spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock,1954 iflag);1955 1956 /* free buffer since already posted a new DMA buffer to RQ */1957 if (found) {1958 nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf);1959 /* Respond with BA_ACC accordingly */1960 lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 1);1961 return 0;1962 }1963 }1964 1965 /* check active list */1966 ctxp = lpfc_nvmet_get_ctx_for_oxid(phba, oxid, sid);1967 if (ctxp) {1968 xri = ctxp->ctxbuf->sglq->sli4_xritag;1969 1970 spin_lock_irqsave(&ctxp->ctxlock, iflag);1971 ctxp->flag |= (LPFC_NVME_ABTS_RCV | LPFC_NVME_ABORT_OP);1972 spin_unlock_irqrestore(&ctxp->ctxlock, iflag);1973 1974 lpfc_nvmeio_data(phba,1975 "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n",1976 xri, raw_smp_processor_id(), 0);1977 1978 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,1979 "6322 NVMET Rcv ABTS:acc oxid x%x xri x%x "1980 "flag x%x state x%x\n",1981 ctxp->oxid, xri, ctxp->flag, ctxp->state);1982 1983 if (ctxp->flag & LPFC_NVME_TNOTIFY) {1984 /* Notify the transport */1985 nvmet_fc_rcv_fcp_abort(phba->targetport,1986 &ctxp->hdlrctx.fcp_req);1987 } else {1988 cancel_work_sync(&ctxp->ctxbuf->defer_work);1989 spin_lock_irqsave(&ctxp->ctxlock, iflag);1990 lpfc_nvmet_defer_release(phba, ctxp);1991 spin_unlock_irqrestore(&ctxp->ctxlock, iflag);1992 }1993 lpfc_nvmet_sol_fcp_issue_abort(phba, ctxp, ctxp->sid,1994 ctxp->oxid);1995 1996 lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 1);1997 return 0;1998 }1999 2000 lpfc_nvmeio_data(phba, "NVMET ABTS RCV: oxid x%x CPU %02x rjt %d\n",2001 oxid, raw_smp_processor_id(), 1);2002 2003 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,2004 "6320 NVMET Rcv ABTS:rjt oxid x%x\n", oxid);2005 2006 /* Respond with BA_RJT accordingly */2007 lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 0);2008#endif2009 return 0;2010}2011 2012static void2013lpfc_nvmet_wqfull_flush(struct lpfc_hba *phba, struct lpfc_queue *wq,2014 struct lpfc_async_xchg_ctx *ctxp)2015{2016 struct lpfc_sli_ring *pring;2017 struct lpfc_iocbq *nvmewqeq;2018 struct lpfc_iocbq *next_nvmewqeq;2019 unsigned long iflags;2020 struct lpfc_wcqe_complete wcqe;2021 struct lpfc_wcqe_complete *wcqep;2022 2023 pring = wq->pring;2024 wcqep = &wcqe;2025 2026 /* Fake an ABORT error code back to cmpl routine */2027 memset(wcqep, 0, sizeof(struct lpfc_wcqe_complete));2028 bf_set(lpfc_wcqe_c_status, wcqep, IOSTAT_LOCAL_REJECT);2029 wcqep->parameter = IOERR_ABORT_REQUESTED;2030 2031 spin_lock_irqsave(&pring->ring_lock, iflags);2032 list_for_each_entry_safe(nvmewqeq, next_nvmewqeq,2033 &wq->wqfull_list, list) {2034 if (ctxp) {2035 /* Checking for a specific IO to flush */2036 if (nvmewqeq->context_un.axchg == ctxp) {2037 list_del(&nvmewqeq->list);2038 spin_unlock_irqrestore(&pring->ring_lock,2039 iflags);2040 memcpy(&nvmewqeq->wcqe_cmpl, wcqep,2041 sizeof(*wcqep));2042 lpfc_nvmet_xmt_fcp_op_cmp(phba, nvmewqeq,2043 nvmewqeq);2044 return;2045 }2046 continue;2047 } else {2048 /* Flush all IOs */2049 list_del(&nvmewqeq->list);2050 spin_unlock_irqrestore(&pring->ring_lock, iflags);2051 memcpy(&nvmewqeq->wcqe_cmpl, wcqep, sizeof(*wcqep));2052 lpfc_nvmet_xmt_fcp_op_cmp(phba, nvmewqeq, nvmewqeq);2053 spin_lock_irqsave(&pring->ring_lock, iflags);2054 }2055 }2056 if (!ctxp)2057 wq->q_flag &= ~HBA_NVMET_WQFULL;2058 spin_unlock_irqrestore(&pring->ring_lock, iflags);2059}2060 2061void2062lpfc_nvmet_wqfull_process(struct lpfc_hba *phba,2063 struct lpfc_queue *wq)2064{2065#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))2066 struct lpfc_sli_ring *pring;2067 struct lpfc_iocbq *nvmewqeq;2068 struct lpfc_async_xchg_ctx *ctxp;2069 unsigned long iflags;2070 int rc;2071 2072 /*2073 * Some WQE slots are available, so try to re-issue anything2074 * on the WQ wqfull_list.2075 */2076 pring = wq->pring;2077 spin_lock_irqsave(&pring->ring_lock, iflags);2078 while (!list_empty(&wq->wqfull_list)) {2079 list_remove_head(&wq->wqfull_list, nvmewqeq, struct lpfc_iocbq,2080 list);2081 spin_unlock_irqrestore(&pring->ring_lock, iflags);2082 ctxp = nvmewqeq->context_un.axchg;2083 rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, nvmewqeq);2084 spin_lock_irqsave(&pring->ring_lock, iflags);2085 if (rc == -EBUSY) {2086 /* WQ was full again, so put it back on the list */2087 list_add(&nvmewqeq->list, &wq->wqfull_list);2088 spin_unlock_irqrestore(&pring->ring_lock, iflags);2089 return;2090 }2091 if (rc == WQE_SUCCESS) {2092#ifdef CONFIG_SCSI_LPFC_DEBUG_FS2093 if (ctxp->ts_cmd_nvme) {2094 if (ctxp->hdlrctx.fcp_req.op == NVMET_FCOP_RSP)2095 ctxp->ts_status_wqput = ktime_get_ns();2096 else2097 ctxp->ts_data_wqput = ktime_get_ns();2098 }2099#endif2100 } else {2101 WARN_ON(rc);2102 }2103 }2104 wq->q_flag &= ~HBA_NVMET_WQFULL;2105 spin_unlock_irqrestore(&pring->ring_lock, iflags);2106 2107#endif2108}2109 2110void2111lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba)2112{2113#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))2114 struct lpfc_nvmet_tgtport *tgtp;2115 struct lpfc_queue *wq;2116 uint32_t qidx;2117 DECLARE_COMPLETION_ONSTACK(tport_unreg_cmp);2118 2119 if (phba->nvmet_support == 0)2120 return;2121 if (phba->targetport) {2122 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;2123 for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) {2124 wq = phba->sli4_hba.hdwq[qidx].io_wq;2125 lpfc_nvmet_wqfull_flush(phba, wq, NULL);2126 }2127 tgtp->tport_unreg_cmp = &tport_unreg_cmp;2128 nvmet_fc_unregister_targetport(phba->targetport);2129 if (!wait_for_completion_timeout(&tport_unreg_cmp,2130 msecs_to_jiffies(LPFC_NVMET_WAIT_TMO)))2131 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,2132 "6179 Unreg targetport x%px timeout "2133 "reached.\n", phba->targetport);2134 lpfc_nvmet_cleanup_io_context(phba);2135 }2136 phba->targetport = NULL;2137#endif2138}2139 2140/**2141 * lpfc_nvmet_handle_lsreq - Process an NVME LS request2142 * @phba: pointer to lpfc hba data structure.2143 * @axchg: pointer to exchange context for the NVME LS request2144 *2145 * This routine is used for processing an asychronously received NVME LS2146 * request. Any remaining validation is done and the LS is then forwarded2147 * to the nvmet-fc transport via nvmet_fc_rcv_ls_req().2148 *2149 * The calling sequence should be: nvmet_fc_rcv_ls_req() -> (processing)2150 * -> lpfc_nvmet_xmt_ls_rsp/cmp -> req->done.2151 * lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg.2152 *2153 * Returns 0 if LS was handled and delivered to the transport2154 * Returns 1 if LS failed to be handled and should be dropped2155 */2156int2157lpfc_nvmet_handle_lsreq(struct lpfc_hba *phba,2158 struct lpfc_async_xchg_ctx *axchg)2159{2160#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))2161 struct lpfc_nvmet_tgtport *tgtp = phba->targetport->private;2162 uint32_t *payload = axchg->payload;2163 int rc;2164 2165 atomic_inc(&tgtp->rcv_ls_req_in);2166 2167 /*2168 * Driver passes the ndlp as the hosthandle argument allowing2169 * the transport to generate LS requests for any associateions2170 * that are created.2171 */2172 rc = nvmet_fc_rcv_ls_req(phba->targetport, axchg->ndlp, &axchg->ls_rsp,2173 axchg->payload, axchg->size);2174 2175 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,2176 "6037 NVMET Unsol rcv: sz %d rc %d: %08x %08x %08x "2177 "%08x %08x %08x\n", axchg->size, rc,2178 *payload, *(payload+1), *(payload+2),2179 *(payload+3), *(payload+4), *(payload+5));2180 2181 if (!rc) {2182 atomic_inc(&tgtp->rcv_ls_req_out);2183 return 0;2184 }2185 2186 atomic_inc(&tgtp->rcv_ls_req_drop);2187#endif2188 return 1;2189}2190 2191static void2192lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf)2193{2194#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))2195 struct lpfc_async_xchg_ctx *ctxp = ctx_buf->context;2196 struct lpfc_hba *phba = ctxp->phba;2197 struct rqb_dmabuf *nvmebuf = ctxp->rqb_buffer;2198 struct lpfc_nvmet_tgtport *tgtp;2199 uint32_t *payload, qno;2200 uint32_t rc;2201 unsigned long iflags;2202 2203 if (!nvmebuf) {2204 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,2205 "6159 process_rcv_fcp_req, nvmebuf is NULL, "2206 "oxid: x%x flg: x%x state: x%x\n",2207 ctxp->oxid, ctxp->flag, ctxp->state);2208 spin_lock_irqsave(&ctxp->ctxlock, iflags);2209 lpfc_nvmet_defer_release(phba, ctxp);2210 spin_unlock_irqrestore(&ctxp->ctxlock, iflags);2211 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid,2212 ctxp->oxid);2213 return;2214 }2215 2216 if (ctxp->flag & LPFC_NVME_ABTS_RCV) {2217 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,2218 "6324 IO oxid x%x aborted\n",2219 ctxp->oxid);2220 return;2221 }2222 2223 payload = (uint32_t *)(nvmebuf->dbuf.virt);2224 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;2225 ctxp->flag |= LPFC_NVME_TNOTIFY;2226#ifdef CONFIG_SCSI_LPFC_DEBUG_FS2227 if (ctxp->ts_isr_cmd)2228 ctxp->ts_cmd_nvme = ktime_get_ns();2229#endif2230 /*2231 * The calling sequence should be:2232 * nvmet_fc_rcv_fcp_req->lpfc_nvmet_xmt_fcp_op/cmp- req->done2233 * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp.2234 * When we return from nvmet_fc_rcv_fcp_req, all relevant info2235 * the NVME command / FC header is stored.2236 * A buffer has already been reposted for this IO, so just free2237 * the nvmebuf.2238 */2239 rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->hdlrctx.fcp_req,2240 payload, ctxp->size);2241 /* Process FCP command */2242 if (rc == 0) {2243 atomic_inc(&tgtp->rcv_fcp_cmd_out);2244 spin_lock_irqsave(&ctxp->ctxlock, iflags);2245 if ((ctxp->flag & LPFC_NVME_CTX_REUSE_WQ) ||2246 (nvmebuf != ctxp->rqb_buffer)) {2247 spin_unlock_irqrestore(&ctxp->ctxlock, iflags);2248 return;2249 }2250 ctxp->rqb_buffer = NULL;2251 spin_unlock_irqrestore(&ctxp->ctxlock, iflags);2252 lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */2253 return;2254 }2255 2256 /* Processing of FCP command is deferred */2257 if (rc == -EOVERFLOW) {2258 lpfc_nvmeio_data(phba, "NVMET RCV BUSY: xri x%x sz %d "2259 "from %06x\n",2260 ctxp->oxid, ctxp->size, ctxp->sid);2261 atomic_inc(&tgtp->rcv_fcp_cmd_out);2262 atomic_inc(&tgtp->defer_fod);2263 spin_lock_irqsave(&ctxp->ctxlock, iflags);2264 if (ctxp->flag & LPFC_NVME_CTX_REUSE_WQ) {2265 spin_unlock_irqrestore(&ctxp->ctxlock, iflags);2266 return;2267 }2268 spin_unlock_irqrestore(&ctxp->ctxlock, iflags);2269 /*2270 * Post a replacement DMA buffer to RQ and defer2271 * freeing rcv buffer till .defer_rcv callback2272 */2273 qno = nvmebuf->idx;2274 lpfc_post_rq_buffer(2275 phba, phba->sli4_hba.nvmet_mrq_hdr[qno],2276 phba->sli4_hba.nvmet_mrq_data[qno], 1, qno);2277 return;2278 }2279 ctxp->flag &= ~LPFC_NVME_TNOTIFY;2280 atomic_inc(&tgtp->rcv_fcp_cmd_drop);2281 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,2282 "2582 FCP Drop IO x%x: err x%x: x%x x%x x%x\n",2283 ctxp->oxid, rc,2284 atomic_read(&tgtp->rcv_fcp_cmd_in),2285 atomic_read(&tgtp->rcv_fcp_cmd_out),2286 atomic_read(&tgtp->xmt_fcp_release));2287 lpfc_nvmeio_data(phba, "NVMET FCP DROP: xri x%x sz %d from %06x\n",2288 ctxp->oxid, ctxp->size, ctxp->sid);2289 spin_lock_irqsave(&ctxp->ctxlock, iflags);2290 lpfc_nvmet_defer_release(phba, ctxp);2291 spin_unlock_irqrestore(&ctxp->ctxlock, iflags);2292 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid);2293#endif2294}2295 2296static void2297lpfc_nvmet_fcp_rqst_defer_work(struct work_struct *work)2298{2299#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))2300 struct lpfc_nvmet_ctxbuf *ctx_buf =2301 container_of(work, struct lpfc_nvmet_ctxbuf, defer_work);2302 2303 lpfc_nvmet_process_rcv_fcp_req(ctx_buf);2304#endif2305}2306 2307static struct lpfc_nvmet_ctxbuf *2308lpfc_nvmet_replenish_context(struct lpfc_hba *phba,2309 struct lpfc_nvmet_ctx_info *current_infop)2310{2311#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))2312 struct lpfc_nvmet_ctxbuf *ctx_buf = NULL;2313 struct lpfc_nvmet_ctx_info *get_infop;2314 int i;2315 2316 /*2317 * The current_infop for the MRQ a NVME command IU was received2318 * on is empty. Our goal is to replenish this MRQs context2319 * list from a another CPUs.2320 *2321 * First we need to pick a context list to start looking on.2322 * nvmet_ctx_start_cpu has available context the last time2323 * we needed to replenish this CPU where nvmet_ctx_next_cpu2324 * is just the next sequential CPU for this MRQ.2325 */2326 if (current_infop->nvmet_ctx_start_cpu)2327 get_infop = current_infop->nvmet_ctx_start_cpu;2328 else2329 get_infop = current_infop->nvmet_ctx_next_cpu;2330 2331 for (i = 0; i < phba->sli4_hba.num_possible_cpu; i++) {2332 if (get_infop == current_infop) {2333 get_infop = get_infop->nvmet_ctx_next_cpu;2334 continue;2335 }2336 spin_lock(&get_infop->nvmet_ctx_list_lock);2337 2338 /* Just take the entire context list, if there are any */2339 if (get_infop->nvmet_ctx_list_cnt) {2340 list_splice_init(&get_infop->nvmet_ctx_list,2341 ¤t_infop->nvmet_ctx_list);2342 current_infop->nvmet_ctx_list_cnt =2343 get_infop->nvmet_ctx_list_cnt - 1;2344 get_infop->nvmet_ctx_list_cnt = 0;2345 spin_unlock(&get_infop->nvmet_ctx_list_lock);2346 2347 current_infop->nvmet_ctx_start_cpu = get_infop;2348 list_remove_head(¤t_infop->nvmet_ctx_list,2349 ctx_buf, struct lpfc_nvmet_ctxbuf,2350 list);2351 return ctx_buf;2352 }2353 2354 /* Otherwise, move on to the next CPU for this MRQ */2355 spin_unlock(&get_infop->nvmet_ctx_list_lock);2356 get_infop = get_infop->nvmet_ctx_next_cpu;2357 }2358 2359#endif2360 /* Nothing found, all contexts for the MRQ are in-flight */2361 return NULL;2362}2363 2364/**2365 * lpfc_nvmet_unsol_fcp_buffer - Process an unsolicited event data buffer2366 * @phba: pointer to lpfc hba data structure.2367 * @idx: relative index of MRQ vector2368 * @nvmebuf: pointer to lpfc nvme command HBQ data structure.2369 * @isr_timestamp: in jiffies.2370 * @cqflag: cq processing information regarding workload.2371 *2372 * This routine is used for processing the WQE associated with a unsolicited2373 * event. It first determines whether there is an existing ndlp that matches2374 * the DID from the unsolicited WQE. If not, it will create a new one with2375 * the DID from the unsolicited WQE. The ELS command from the unsolicited2376 * WQE is then used to invoke the proper routine and to set up proper state2377 * of the discovery state machine.2378 **/2379static void2380lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba,2381 uint32_t idx,2382 struct rqb_dmabuf *nvmebuf,2383 uint64_t isr_timestamp,2384 uint8_t cqflag)2385{2386 struct lpfc_async_xchg_ctx *ctxp;2387 struct lpfc_nvmet_tgtport *tgtp;2388 struct fc_frame_header *fc_hdr;2389 struct lpfc_nvmet_ctxbuf *ctx_buf;2390 struct lpfc_nvmet_ctx_info *current_infop;2391 uint32_t size, oxid, sid, qno;2392 unsigned long iflag;2393 int current_cpu;2394 2395 if (!IS_ENABLED(CONFIG_NVME_TARGET_FC))2396 return;2397 2398 ctx_buf = NULL;2399 if (!nvmebuf || !phba->targetport) {2400 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,2401 "6157 NVMET FCP Drop IO\n");2402 if (nvmebuf)2403 lpfc_rq_buf_free(phba, &nvmebuf->hbuf);2404 return;2405 }2406 2407 /*2408 * Get a pointer to the context list for this MRQ based on2409 * the CPU this MRQ IRQ is associated with. If the CPU association2410 * changes from our initial assumption, the context list could2411 * be empty, thus it would need to be replenished with the2412 * context list from another CPU for this MRQ.2413 */2414 current_cpu = raw_smp_processor_id();2415 current_infop = lpfc_get_ctx_list(phba, current_cpu, idx);2416 spin_lock_irqsave(¤t_infop->nvmet_ctx_list_lock, iflag);2417 if (current_infop->nvmet_ctx_list_cnt) {2418 list_remove_head(¤t_infop->nvmet_ctx_list,2419 ctx_buf, struct lpfc_nvmet_ctxbuf, list);2420 current_infop->nvmet_ctx_list_cnt--;2421 } else {2422 ctx_buf = lpfc_nvmet_replenish_context(phba, current_infop);2423 }2424 spin_unlock_irqrestore(¤t_infop->nvmet_ctx_list_lock, iflag);2425 2426 fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);2427 oxid = be16_to_cpu(fc_hdr->fh_ox_id);2428 size = nvmebuf->bytes_recv;2429 2430#ifdef CONFIG_SCSI_LPFC_DEBUG_FS2431 if (phba->hdwqstat_on & LPFC_CHECK_NVMET_IO) {2432 this_cpu_inc(phba->sli4_hba.c_stat->rcv_io);2433 if (idx != current_cpu)2434 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,2435 "6703 CPU Check rcv: "2436 "cpu %d expect %d\n",2437 current_cpu, idx);2438 }2439#endif2440 2441 lpfc_nvmeio_data(phba, "NVMET FCP RCV: xri x%x sz %d CPU %02x\n",2442 oxid, size, raw_smp_processor_id());2443 2444 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;2445 2446 if (!ctx_buf) {2447 /* Queue this NVME IO to process later */2448 spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag);2449 list_add_tail(&nvmebuf->hbuf.list,2450 &phba->sli4_hba.lpfc_nvmet_io_wait_list);2451 phba->sli4_hba.nvmet_io_wait_cnt++;2452 phba->sli4_hba.nvmet_io_wait_total++;2453 spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock,2454 iflag);2455 2456 /* Post a brand new DMA buffer to RQ */2457 qno = nvmebuf->idx;2458 lpfc_post_rq_buffer(2459 phba, phba->sli4_hba.nvmet_mrq_hdr[qno],2460 phba->sli4_hba.nvmet_mrq_data[qno], 1, qno);2461 2462 atomic_inc(&tgtp->defer_ctx);2463 return;2464 }2465 2466 sid = sli4_sid_from_fc_hdr(fc_hdr);2467 2468 ctxp = (struct lpfc_async_xchg_ctx *)ctx_buf->context;2469 spin_lock_irqsave(&phba->sli4_hba.t_active_list_lock, iflag);2470 list_add_tail(&ctxp->list, &phba->sli4_hba.t_active_ctx_list);2471 spin_unlock_irqrestore(&phba->sli4_hba.t_active_list_lock, iflag);2472 if (ctxp->state != LPFC_NVME_STE_FREE) {2473 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,2474 "6414 NVMET Context corrupt %d %d oxid x%x\n",2475 ctxp->state, ctxp->entry_cnt, ctxp->oxid);2476 }2477 ctxp->wqeq = NULL;2478 ctxp->offset = 0;2479 ctxp->phba = phba;2480 ctxp->size = size;2481 ctxp->oxid = oxid;2482 ctxp->sid = sid;2483 ctxp->idx = idx;2484 ctxp->state = LPFC_NVME_STE_RCV;2485 ctxp->entry_cnt = 1;2486 ctxp->flag = 0;2487 ctxp->ctxbuf = ctx_buf;2488 ctxp->rqb_buffer = (void *)nvmebuf;2489 ctxp->hdwq = NULL;2490 spin_lock_init(&ctxp->ctxlock);2491 2492#ifdef CONFIG_SCSI_LPFC_DEBUG_FS2493 if (isr_timestamp)2494 ctxp->ts_isr_cmd = isr_timestamp;2495 ctxp->ts_cmd_nvme = 0;2496 ctxp->ts_nvme_data = 0;2497 ctxp->ts_data_wqput = 0;2498 ctxp->ts_isr_data = 0;2499 ctxp->ts_data_nvme = 0;2500 ctxp->ts_nvme_status = 0;2501 ctxp->ts_status_wqput = 0;2502 ctxp->ts_isr_status = 0;2503 ctxp->ts_status_nvme = 0;2504#endif2505 2506 atomic_inc(&tgtp->rcv_fcp_cmd_in);2507 /* check for cq processing load */2508 if (!cqflag) {2509 lpfc_nvmet_process_rcv_fcp_req(ctx_buf);2510 return;2511 }2512 2513 if (!queue_work(phba->wq, &ctx_buf->defer_work)) {2514 atomic_inc(&tgtp->rcv_fcp_cmd_drop);2515 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,2516 "6325 Unable to queue work for oxid x%x. "2517 "FCP Drop IO [x%x x%x x%x]\n",2518 ctxp->oxid,2519 atomic_read(&tgtp->rcv_fcp_cmd_in),2520 atomic_read(&tgtp->rcv_fcp_cmd_out),2521 atomic_read(&tgtp->xmt_fcp_release));2522 2523 spin_lock_irqsave(&ctxp->ctxlock, iflag);2524 lpfc_nvmet_defer_release(phba, ctxp);2525 spin_unlock_irqrestore(&ctxp->ctxlock, iflag);2526 lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid);2527 }2528}2529 2530/**2531 * lpfc_nvmet_unsol_fcp_event - Process an unsolicited event from an nvme nport2532 * @phba: pointer to lpfc hba data structure.2533 * @idx: relative index of MRQ vector2534 * @nvmebuf: pointer to received nvme data structure.2535 * @isr_timestamp: in jiffies.2536 * @cqflag: cq processing information regarding workload.2537 *2538 * This routine is used to process an unsolicited event received from a SLI2539 * (Service Level Interface) ring. The actual processing of the data buffer2540 * associated with the unsolicited event is done by invoking the routine2541 * lpfc_nvmet_unsol_fcp_buffer() after properly set up the buffer from the2542 * SLI RQ on which the unsolicited event was received.2543 **/2544void2545lpfc_nvmet_unsol_fcp_event(struct lpfc_hba *phba,2546 uint32_t idx,2547 struct rqb_dmabuf *nvmebuf,2548 uint64_t isr_timestamp,2549 uint8_t cqflag)2550{2551 if (!nvmebuf) {2552 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,2553 "3167 NVMET FCP Drop IO\n");2554 return;2555 }2556 if (phba->nvmet_support == 0) {2557 lpfc_rq_buf_free(phba, &nvmebuf->hbuf);2558 return;2559 }2560 lpfc_nvmet_unsol_fcp_buffer(phba, idx, nvmebuf, isr_timestamp, cqflag);2561}2562 2563/**2564 * lpfc_nvmet_prep_ls_wqe - Allocate and prepare a lpfc wqe data structure2565 * @phba: pointer to a host N_Port data structure.2566 * @ctxp: Context info for NVME LS Request2567 * @rspbuf: DMA buffer of NVME command.2568 * @rspsize: size of the NVME command.2569 *2570 * This routine is used for allocating a lpfc-WQE data structure from2571 * the driver lpfc-WQE free-list and prepare the WQE with the parameters2572 * passed into the routine for discovery state machine to issue an Extended2573 * Link Service (NVME) commands. It is a generic lpfc-WQE allocation2574 * and preparation routine that is used by all the discovery state machine2575 * routines and the NVME command-specific fields will be later set up by2576 * the individual discovery machine routines after calling this routine2577 * allocating and preparing a generic WQE data structure. It fills in the2578 * Buffer Descriptor Entries (BDEs), allocates buffers for both command2579 * payload and response payload (if expected). The reference count on the2580 * ndlp is incremented by 1 and the reference to the ndlp is put into2581 * context1 of the WQE data structure for this WQE to hold the ndlp2582 * reference for the command's callback function to access later.2583 *2584 * Return code2585 * Pointer to the newly allocated/prepared nvme wqe data structure2586 * NULL - when nvme wqe data structure allocation/preparation failed2587 **/2588static struct lpfc_iocbq *2589lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *phba,2590 struct lpfc_async_xchg_ctx *ctxp,2591 dma_addr_t rspbuf, uint16_t rspsize)2592{2593 struct lpfc_nodelist *ndlp;2594 struct lpfc_iocbq *nvmewqe;2595 union lpfc_wqe128 *wqe;2596 2597 if (!lpfc_is_link_up(phba)) {2598 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,2599 "6104 NVMET prep LS wqe: link err: "2600 "NPORT x%x oxid:x%x ste %d\n",2601 ctxp->sid, ctxp->oxid, ctxp->state);2602 return NULL;2603 }2604 2605 /* Allocate buffer for command wqe */2606 nvmewqe = lpfc_sli_get_iocbq(phba);2607 if (nvmewqe == NULL) {2608 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,2609 "6105 NVMET prep LS wqe: No WQE: "2610 "NPORT x%x oxid x%x ste %d\n",2611 ctxp->sid, ctxp->oxid, ctxp->state);2612 return NULL;2613 }2614 2615 ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);2616 if (!ndlp ||2617 ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&2618 (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {2619 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,2620 "6106 NVMET prep LS wqe: No ndlp: "2621 "NPORT x%x oxid x%x ste %d\n",2622 ctxp->sid, ctxp->oxid, ctxp->state);2623 goto nvme_wqe_free_wqeq_exit;2624 }2625 ctxp->wqeq = nvmewqe;2626 2627 /* prevent preparing wqe with NULL ndlp reference */2628 nvmewqe->ndlp = lpfc_nlp_get(ndlp);2629 if (!nvmewqe->ndlp)2630 goto nvme_wqe_free_wqeq_exit;2631 nvmewqe->context_un.axchg = ctxp;2632 2633 wqe = &nvmewqe->wqe;2634 memset(wqe, 0, sizeof(union lpfc_wqe));2635 2636 /* Words 0 - 2 */2637 wqe->xmit_sequence.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;2638 wqe->xmit_sequence.bde.tus.f.bdeSize = rspsize;2639 wqe->xmit_sequence.bde.addrLow = le32_to_cpu(putPaddrLow(rspbuf));2640 wqe->xmit_sequence.bde.addrHigh = le32_to_cpu(putPaddrHigh(rspbuf));2641 2642 /* Word 3 */2643 2644 /* Word 4 */2645 2646 /* Word 5 */2647 bf_set(wqe_dfctl, &wqe->xmit_sequence.wge_ctl, 0);2648 bf_set(wqe_ls, &wqe->xmit_sequence.wge_ctl, 1);2649 bf_set(wqe_la, &wqe->xmit_sequence.wge_ctl, 0);2650 bf_set(wqe_rctl, &wqe->xmit_sequence.wge_ctl, FC_RCTL_ELS4_REP);2651 bf_set(wqe_type, &wqe->xmit_sequence.wge_ctl, FC_TYPE_NVME);2652 2653 /* Word 6 */2654 bf_set(wqe_ctxt_tag, &wqe->xmit_sequence.wqe_com,2655 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);2656 bf_set(wqe_xri_tag, &wqe->xmit_sequence.wqe_com, nvmewqe->sli4_xritag);2657 2658 /* Word 7 */2659 bf_set(wqe_cmnd, &wqe->xmit_sequence.wqe_com,2660 CMD_XMIT_SEQUENCE64_WQE);2661 bf_set(wqe_ct, &wqe->xmit_sequence.wqe_com, SLI4_CT_RPI);2662 bf_set(wqe_class, &wqe->xmit_sequence.wqe_com, CLASS3);2663 bf_set(wqe_pu, &wqe->xmit_sequence.wqe_com, 0);2664 2665 /* Word 8 */2666 wqe->xmit_sequence.wqe_com.abort_tag = nvmewqe->iotag;2667 2668 /* Word 9 */2669 bf_set(wqe_reqtag, &wqe->xmit_sequence.wqe_com, nvmewqe->iotag);2670 /* Needs to be set by caller */2671 bf_set(wqe_rcvoxid, &wqe->xmit_sequence.wqe_com, ctxp->oxid);2672 2673 /* Word 10 */2674 bf_set(wqe_dbde, &wqe->xmit_sequence.wqe_com, 1);2675 bf_set(wqe_iod, &wqe->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE);2676 bf_set(wqe_lenloc, &wqe->xmit_sequence.wqe_com,2677 LPFC_WQE_LENLOC_WORD12);2678 bf_set(wqe_ebde_cnt, &wqe->xmit_sequence.wqe_com, 0);2679 2680 /* Word 11 */2681 bf_set(wqe_cqid, &wqe->xmit_sequence.wqe_com,2682 LPFC_WQE_CQ_ID_DEFAULT);2683 bf_set(wqe_cmd_type, &wqe->xmit_sequence.wqe_com,2684 OTHER_COMMAND);2685 2686 /* Word 12 */2687 wqe->xmit_sequence.xmit_len = rspsize;2688 2689 nvmewqe->retry = 1;2690 nvmewqe->vport = phba->pport;2691 nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT;2692 nvmewqe->cmd_flag |= LPFC_IO_NVME_LS;2693 2694 /* Xmit NVMET response to remote NPORT <did> */2695 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,2696 "6039 Xmit NVMET LS response to remote "2697 "NPORT x%x iotag:x%x oxid:x%x size:x%x\n",2698 ndlp->nlp_DID, nvmewqe->iotag, ctxp->oxid,2699 rspsize);2700 return nvmewqe;2701 2702nvme_wqe_free_wqeq_exit:2703 nvmewqe->context_un.axchg = NULL;2704 nvmewqe->ndlp = NULL;2705 nvmewqe->bpl_dmabuf = NULL;2706 lpfc_sli_release_iocbq(phba, nvmewqe);2707 return NULL;2708}2709 2710 2711static struct lpfc_iocbq *2712lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,2713 struct lpfc_async_xchg_ctx *ctxp)2714{2715 struct nvmefc_tgt_fcp_req *rsp = &ctxp->hdlrctx.fcp_req;2716 struct lpfc_nvmet_tgtport *tgtp;2717 struct sli4_sge *sgl;2718 struct lpfc_nodelist *ndlp;2719 struct lpfc_iocbq *nvmewqe;2720 struct scatterlist *sgel;2721 union lpfc_wqe128 *wqe;2722 struct ulp_bde64 *bde;2723 dma_addr_t physaddr;2724 int i, cnt, nsegs;2725 bool use_pbde = false;2726 int xc = 1;2727 2728 if (!lpfc_is_link_up(phba)) {2729 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,2730 "6107 NVMET prep FCP wqe: link err:"2731 "NPORT x%x oxid x%x ste %d\n",2732 ctxp->sid, ctxp->oxid, ctxp->state);2733 return NULL;2734 }2735 2736 ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);2737 if (!ndlp ||2738 ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&2739 (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {2740 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,2741 "6108 NVMET prep FCP wqe: no ndlp: "2742 "NPORT x%x oxid x%x ste %d\n",2743 ctxp->sid, ctxp->oxid, ctxp->state);2744 return NULL;2745 }2746 2747 if (rsp->sg_cnt > lpfc_tgttemplate.max_sgl_segments) {2748 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,2749 "6109 NVMET prep FCP wqe: seg cnt err: "2750 "NPORT x%x oxid x%x ste %d cnt %d\n",2751 ctxp->sid, ctxp->oxid, ctxp->state,2752 phba->cfg_nvme_seg_cnt);2753 return NULL;2754 }2755 nsegs = rsp->sg_cnt;2756 2757 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;2758 nvmewqe = ctxp->wqeq;2759 if (nvmewqe == NULL) {2760 /* Allocate buffer for command wqe */2761 nvmewqe = ctxp->ctxbuf->iocbq;2762 if (nvmewqe == NULL) {2763 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,2764 "6110 NVMET prep FCP wqe: No "2765 "WQE: NPORT x%x oxid x%x ste %d\n",2766 ctxp->sid, ctxp->oxid, ctxp->state);2767 return NULL;2768 }2769 ctxp->wqeq = nvmewqe;2770 xc = 0; /* create new XRI */2771 nvmewqe->sli4_lxritag = NO_XRI;2772 nvmewqe->sli4_xritag = NO_XRI;2773 }2774 2775 /* Sanity check */2776 if (((ctxp->state == LPFC_NVME_STE_RCV) &&2777 (ctxp->entry_cnt == 1)) ||2778 (ctxp->state == LPFC_NVME_STE_DATA)) {2779 wqe = &nvmewqe->wqe;2780 } else {2781 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,2782 "6111 Wrong state NVMET FCP: %d cnt %d\n",2783 ctxp->state, ctxp->entry_cnt);2784 return NULL;2785 }2786 2787 sgl = (struct sli4_sge *)ctxp->ctxbuf->sglq->sgl;2788 switch (rsp->op) {2789 case NVMET_FCOP_READDATA:2790 case NVMET_FCOP_READDATA_RSP:2791 /* From the tsend template, initialize words 7 - 11 */2792 memcpy(&wqe->words[7],2793 &lpfc_tsend_cmd_template.words[7],2794 sizeof(uint32_t) * 5);2795 2796 /* Words 0 - 2 : The first sg segment */2797 sgel = &rsp->sg[0];2798 physaddr = sg_dma_address(sgel);2799 wqe->fcp_tsend.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;2800 wqe->fcp_tsend.bde.tus.f.bdeSize = sg_dma_len(sgel);2801 wqe->fcp_tsend.bde.addrLow = cpu_to_le32(putPaddrLow(physaddr));2802 wqe->fcp_tsend.bde.addrHigh =2803 cpu_to_le32(putPaddrHigh(physaddr));2804 2805 /* Word 3 */2806 wqe->fcp_tsend.payload_offset_len = 0;2807 2808 /* Word 4 */2809 wqe->fcp_tsend.relative_offset = ctxp->offset;2810 2811 /* Word 5 */2812 wqe->fcp_tsend.reserved = 0;2813 2814 /* Word 6 */2815 bf_set(wqe_ctxt_tag, &wqe->fcp_tsend.wqe_com,2816 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);2817 bf_set(wqe_xri_tag, &wqe->fcp_tsend.wqe_com,2818 nvmewqe->sli4_xritag);2819 2820 /* Word 7 - set ar later */2821 2822 /* Word 8 */2823 wqe->fcp_tsend.wqe_com.abort_tag = nvmewqe->iotag;2824 2825 /* Word 9 */2826 bf_set(wqe_reqtag, &wqe->fcp_tsend.wqe_com, nvmewqe->iotag);2827 bf_set(wqe_rcvoxid, &wqe->fcp_tsend.wqe_com, ctxp->oxid);2828 2829 /* Word 10 - set wqes later, in template xc=1 */2830 if (!xc)2831 bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 0);2832 2833 /* Word 12 */2834 wqe->fcp_tsend.fcp_data_len = rsp->transfer_length;2835 2836 /* Setup 2 SKIP SGEs */2837 sgl->addr_hi = 0;2838 sgl->addr_lo = 0;2839 sgl->word2 = 0;2840 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);2841 sgl->word2 = cpu_to_le32(sgl->word2);2842 sgl->sge_len = 0;2843 sgl++;2844 sgl->addr_hi = 0;2845 sgl->addr_lo = 0;2846 sgl->word2 = 0;2847 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);2848 sgl->word2 = cpu_to_le32(sgl->word2);2849 sgl->sge_len = 0;2850 sgl++;2851 if (rsp->op == NVMET_FCOP_READDATA_RSP) {2852 atomic_inc(&tgtp->xmt_fcp_read_rsp);2853 2854 /* In template ar=1 wqes=0 sup=0 irsp=0 irsplen=0 */2855 2856 if (rsp->rsplen == LPFC_NVMET_SUCCESS_LEN) {2857 if (ndlp->nlp_flag & NLP_SUPPRESS_RSP)2858 bf_set(wqe_sup,2859 &wqe->fcp_tsend.wqe_com, 1);2860 } else {2861 bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 1);2862 bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 1);2863 bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com,2864 ((rsp->rsplen >> 2) - 1));2865 memcpy(&wqe->words[16], rsp->rspaddr,2866 rsp->rsplen);2867 }2868 } else {2869 atomic_inc(&tgtp->xmt_fcp_read);2870 2871 /* In template ar=1 wqes=0 sup=0 irsp=0 irsplen=0 */2872 bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 0);2873 }2874 break;2875 2876 case NVMET_FCOP_WRITEDATA:2877 /* From the treceive template, initialize words 3 - 11 */2878 memcpy(&wqe->words[3],2879 &lpfc_treceive_cmd_template.words[3],2880 sizeof(uint32_t) * 9);2881 2882 /* Words 0 - 2 : First SGE is skipped, set invalid BDE type */2883 wqe->fcp_treceive.bde.tus.f.bdeFlags = LPFC_SGE_TYPE_SKIP;2884 wqe->fcp_treceive.bde.tus.f.bdeSize = 0;2885 wqe->fcp_treceive.bde.addrLow = 0;2886 wqe->fcp_treceive.bde.addrHigh = 0;2887 2888 /* Word 4 */2889 wqe->fcp_treceive.relative_offset = ctxp->offset;2890 2891 /* Word 6 */2892 bf_set(wqe_ctxt_tag, &wqe->fcp_treceive.wqe_com,2893 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);2894 bf_set(wqe_xri_tag, &wqe->fcp_treceive.wqe_com,2895 nvmewqe->sli4_xritag);2896 2897 /* Word 7 */2898 2899 /* Word 8 */2900 wqe->fcp_treceive.wqe_com.abort_tag = nvmewqe->iotag;2901 2902 /* Word 9 */2903 bf_set(wqe_reqtag, &wqe->fcp_treceive.wqe_com, nvmewqe->iotag);2904 bf_set(wqe_rcvoxid, &wqe->fcp_treceive.wqe_com, ctxp->oxid);2905 2906 /* Word 10 - in template xc=1 */2907 if (!xc)2908 bf_set(wqe_xc, &wqe->fcp_treceive.wqe_com, 0);2909 2910 /* Word 11 - check for pbde */2911 if (nsegs == 1 && phba->cfg_enable_pbde) {2912 use_pbde = true;2913 /* Word 11 - PBDE bit already preset by template */2914 } else {2915 /* Overwrite default template setting */2916 bf_set(wqe_pbde, &wqe->fcp_treceive.wqe_com, 0);2917 }2918 2919 /* Word 12 */2920 wqe->fcp_tsend.fcp_data_len = rsp->transfer_length;2921 2922 /* Setup 2 SKIP SGEs */2923 sgl->addr_hi = 0;2924 sgl->addr_lo = 0;2925 sgl->word2 = 0;2926 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);2927 sgl->word2 = cpu_to_le32(sgl->word2);2928 sgl->sge_len = 0;2929 sgl++;2930 sgl->addr_hi = 0;2931 sgl->addr_lo = 0;2932 sgl->word2 = 0;2933 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);2934 sgl->word2 = cpu_to_le32(sgl->word2);2935 sgl->sge_len = 0;2936 sgl++;2937 atomic_inc(&tgtp->xmt_fcp_write);2938 break;2939 2940 case NVMET_FCOP_RSP:2941 /* From the treceive template, initialize words 4 - 11 */2942 memcpy(&wqe->words[4],2943 &lpfc_trsp_cmd_template.words[4],2944 sizeof(uint32_t) * 8);2945 2946 /* Words 0 - 2 */2947 physaddr = rsp->rspdma;2948 wqe->fcp_trsp.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;2949 wqe->fcp_trsp.bde.tus.f.bdeSize = rsp->rsplen;2950 wqe->fcp_trsp.bde.addrLow =2951 cpu_to_le32(putPaddrLow(physaddr));2952 wqe->fcp_trsp.bde.addrHigh =2953 cpu_to_le32(putPaddrHigh(physaddr));2954 2955 /* Word 3 */2956 wqe->fcp_trsp.response_len = rsp->rsplen;2957 2958 /* Word 6 */2959 bf_set(wqe_ctxt_tag, &wqe->fcp_trsp.wqe_com,2960 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);2961 bf_set(wqe_xri_tag, &wqe->fcp_trsp.wqe_com,2962 nvmewqe->sli4_xritag);2963 2964 /* Word 7 */2965 2966 /* Word 8 */2967 wqe->fcp_trsp.wqe_com.abort_tag = nvmewqe->iotag;2968 2969 /* Word 9 */2970 bf_set(wqe_reqtag, &wqe->fcp_trsp.wqe_com, nvmewqe->iotag);2971 bf_set(wqe_rcvoxid, &wqe->fcp_trsp.wqe_com, ctxp->oxid);2972 2973 /* Word 10 */2974 if (xc)2975 bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, 1);2976 2977 /* Word 11 */2978 /* In template wqes=0 irsp=0 irsplen=0 - good response */2979 if (rsp->rsplen != LPFC_NVMET_SUCCESS_LEN) {2980 /* Bad response - embed it */2981 bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 1);2982 bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 1);2983 bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com,2984 ((rsp->rsplen >> 2) - 1));2985 memcpy(&wqe->words[16], rsp->rspaddr, rsp->rsplen);2986 }2987 2988 /* Word 12 */2989 wqe->fcp_trsp.rsvd_12_15[0] = 0;2990 2991 /* Use rspbuf, NOT sg list */2992 nsegs = 0;2993 sgl->word2 = 0;2994 atomic_inc(&tgtp->xmt_fcp_rsp);2995 break;2996 2997 default:2998 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,2999 "6064 Unknown Rsp Op %d\n",3000 rsp->op);3001 return NULL;3002 }3003 3004 nvmewqe->retry = 1;3005 nvmewqe->vport = phba->pport;3006 nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT;3007 nvmewqe->ndlp = ndlp;3008 3009 for_each_sg(rsp->sg, sgel, nsegs, i) {3010 physaddr = sg_dma_address(sgel);3011 cnt = sg_dma_len(sgel);3012 sgl->addr_hi = putPaddrHigh(physaddr);3013 sgl->addr_lo = putPaddrLow(physaddr);3014 sgl->word2 = 0;3015 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA);3016 bf_set(lpfc_sli4_sge_offset, sgl, ctxp->offset);3017 if ((i+1) == rsp->sg_cnt)3018 bf_set(lpfc_sli4_sge_last, sgl, 1);3019 sgl->word2 = cpu_to_le32(sgl->word2);3020 sgl->sge_len = cpu_to_le32(cnt);3021 sgl++;3022 ctxp->offset += cnt;3023 }3024 3025 bde = (struct ulp_bde64 *)&wqe->words[13];3026 if (use_pbde) {3027 /* decrement sgl ptr backwards once to first data sge */3028 sgl--;3029 3030 /* Words 13-15 (PBDE) */3031 bde->addrLow = sgl->addr_lo;3032 bde->addrHigh = sgl->addr_hi;3033 bde->tus.f.bdeSize = le32_to_cpu(sgl->sge_len);3034 bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;3035 bde->tus.w = cpu_to_le32(bde->tus.w);3036 } else {3037 memset(bde, 0, sizeof(struct ulp_bde64));3038 }3039 ctxp->state = LPFC_NVME_STE_DATA;3040 ctxp->entry_cnt++;3041 return nvmewqe;3042}3043 3044/**3045 * lpfc_nvmet_sol_fcp_abort_cmp - Completion handler for ABTS3046 * @phba: Pointer to HBA context object.3047 * @cmdwqe: Pointer to driver command WQE object.3048 * @rspwqe: Pointer to driver response WQE object.3049 *3050 * The function is called from SLI ring event handler with no3051 * lock held. This function is the completion handler for NVME ABTS for FCP cmds3052 * The function frees memory resources used for the NVME commands.3053 **/3054static void3055lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,3056 struct lpfc_iocbq *rspwqe)3057{3058 struct lpfc_async_xchg_ctx *ctxp;3059 struct lpfc_nvmet_tgtport *tgtp;3060 uint32_t result;3061 unsigned long flags;3062 bool released = false;3063 struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;3064 3065 ctxp = cmdwqe->context_un.axchg;3066 result = wcqe->parameter;3067 3068 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;3069 if (ctxp->flag & LPFC_NVME_ABORT_OP)3070 atomic_inc(&tgtp->xmt_fcp_abort_cmpl);3071 3072 spin_lock_irqsave(&ctxp->ctxlock, flags);3073 ctxp->state = LPFC_NVME_STE_DONE;3074 3075 /* Check if we already received a free context call3076 * and we have completed processing an abort situation.3077 */3078 if ((ctxp->flag & LPFC_NVME_CTX_RLS) &&3079 !(ctxp->flag & LPFC_NVME_XBUSY)) {3080 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock);3081 list_del_init(&ctxp->list);3082 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock);3083 released = true;3084 }3085 ctxp->flag &= ~LPFC_NVME_ABORT_OP;3086 spin_unlock_irqrestore(&ctxp->ctxlock, flags);3087 atomic_inc(&tgtp->xmt_abort_rsp);3088 3089 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,3090 "6165 ABORT cmpl: oxid x%x flg x%x (%d) "3091 "WCQE: %08x %08x %08x %08x\n",3092 ctxp->oxid, ctxp->flag, released,3093 wcqe->word0, wcqe->total_data_placed,3094 result, wcqe->word3);3095 3096 cmdwqe->rsp_dmabuf = NULL;3097 cmdwqe->bpl_dmabuf = NULL;3098 /*3099 * if transport has released ctx, then can reuse it. Otherwise,3100 * will be recycled by transport release call.3101 */3102 if (released)3103 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);3104 3105 /* This is the iocbq for the abort, not the command */3106 lpfc_sli_release_iocbq(phba, cmdwqe);3107 3108 /* Since iaab/iaar are NOT set, there is no work left.3109 * For LPFC_NVME_XBUSY, lpfc_sli4_nvmet_xri_aborted3110 * should have been called already.3111 */3112}3113 3114/**3115 * lpfc_nvmet_unsol_fcp_abort_cmp - Completion handler for ABTS3116 * @phba: Pointer to HBA context object.3117 * @cmdwqe: Pointer to driver command WQE object.3118 * @rspwqe: Pointer to driver response WQE object.3119 *3120 * The function is called from SLI ring event handler with no3121 * lock held. This function is the completion handler for NVME ABTS for FCP cmds3122 * The function frees memory resources used for the NVME commands.3123 **/3124static void3125lpfc_nvmet_unsol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,3126 struct lpfc_iocbq *rspwqe)3127{3128 struct lpfc_async_xchg_ctx *ctxp;3129 struct lpfc_nvmet_tgtport *tgtp;3130 unsigned long flags;3131 uint32_t result;3132 bool released = false;3133 struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;3134 3135 ctxp = cmdwqe->context_un.axchg;3136 result = wcqe->parameter;3137 3138 if (!ctxp) {3139 /* if context is clear, related io alrady complete */3140 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,3141 "6070 ABTS cmpl: WCQE: %08x %08x %08x %08x\n",3142 wcqe->word0, wcqe->total_data_placed,3143 result, wcqe->word3);3144 return;3145 }3146 3147 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;3148 spin_lock_irqsave(&ctxp->ctxlock, flags);3149 if (ctxp->flag & LPFC_NVME_ABORT_OP)3150 atomic_inc(&tgtp->xmt_fcp_abort_cmpl);3151 3152 /* Sanity check */3153 if (ctxp->state != LPFC_NVME_STE_ABORT) {3154 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,3155 "6112 ABTS Wrong state:%d oxid x%x\n",3156 ctxp->state, ctxp->oxid);3157 }3158 3159 /* Check if we already received a free context call3160 * and we have completed processing an abort situation.3161 */3162 ctxp->state = LPFC_NVME_STE_DONE;3163 if ((ctxp->flag & LPFC_NVME_CTX_RLS) &&3164 !(ctxp->flag & LPFC_NVME_XBUSY)) {3165 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock);3166 list_del_init(&ctxp->list);3167 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock);3168 released = true;3169 }3170 ctxp->flag &= ~LPFC_NVME_ABORT_OP;3171 spin_unlock_irqrestore(&ctxp->ctxlock, flags);3172 atomic_inc(&tgtp->xmt_abort_rsp);3173 3174 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,3175 "6316 ABTS cmpl oxid x%x flg x%x (%x) "3176 "WCQE: %08x %08x %08x %08x\n",3177 ctxp->oxid, ctxp->flag, released,3178 wcqe->word0, wcqe->total_data_placed,3179 result, wcqe->word3);3180 3181 cmdwqe->rsp_dmabuf = NULL;3182 cmdwqe->bpl_dmabuf = NULL;3183 /*3184 * if transport has released ctx, then can reuse it. Otherwise,3185 * will be recycled by transport release call.3186 */3187 if (released)3188 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);3189 3190 /* Since iaab/iaar are NOT set, there is no work left.3191 * For LPFC_NVME_XBUSY, lpfc_sli4_nvmet_xri_aborted3192 * should have been called already.3193 */3194}3195 3196/**3197 * lpfc_nvmet_xmt_ls_abort_cmp - Completion handler for ABTS3198 * @phba: Pointer to HBA context object.3199 * @cmdwqe: Pointer to driver command WQE object.3200 * @rspwqe: Pointer to driver response WQE object.3201 *3202 * The function is called from SLI ring event handler with no3203 * lock held. This function is the completion handler for NVME ABTS for LS cmds3204 * The function frees memory resources used for the NVME commands.3205 **/3206static void3207lpfc_nvmet_xmt_ls_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,3208 struct lpfc_iocbq *rspwqe)3209{3210 struct lpfc_async_xchg_ctx *ctxp;3211 struct lpfc_nvmet_tgtport *tgtp;3212 uint32_t result;3213 struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;3214 3215 ctxp = cmdwqe->context_un.axchg;3216 result = wcqe->parameter;3217 3218 if (phba->nvmet_support) {3219 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;3220 atomic_inc(&tgtp->xmt_ls_abort_cmpl);3221 }3222 3223 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,3224 "6083 Abort cmpl: ctx x%px WCQE:%08x %08x %08x %08x\n",3225 ctxp, wcqe->word0, wcqe->total_data_placed,3226 result, wcqe->word3);3227 3228 if (!ctxp) {3229 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,3230 "6415 NVMET LS Abort No ctx: WCQE: "3231 "%08x %08x %08x %08x\n",3232 wcqe->word0, wcqe->total_data_placed,3233 result, wcqe->word3);3234 3235 lpfc_sli_release_iocbq(phba, cmdwqe);3236 return;3237 }3238 3239 if (ctxp->state != LPFC_NVME_STE_LS_ABORT) {3240 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,3241 "6416 NVMET LS abort cmpl state mismatch: "3242 "oxid x%x: %d %d\n",3243 ctxp->oxid, ctxp->state, ctxp->entry_cnt);3244 }3245 3246 cmdwqe->rsp_dmabuf = NULL;3247 cmdwqe->bpl_dmabuf = NULL;3248 lpfc_sli_release_iocbq(phba, cmdwqe);3249 kfree(ctxp);3250}3251 3252static int3253lpfc_nvmet_unsol_issue_abort(struct lpfc_hba *phba,3254 struct lpfc_async_xchg_ctx *ctxp,3255 uint32_t sid, uint16_t xri)3256{3257 struct lpfc_nvmet_tgtport *tgtp = NULL;3258 struct lpfc_iocbq *abts_wqeq;3259 union lpfc_wqe128 *wqe_abts;3260 struct lpfc_nodelist *ndlp;3261 3262 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,3263 "6067 ABTS: sid %x xri x%x/x%x\n",3264 sid, xri, ctxp->wqeq->sli4_xritag);3265 3266 if (phba->nvmet_support && phba->targetport)3267 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;3268 3269 ndlp = lpfc_findnode_did(phba->pport, sid);3270 if (!ndlp ||3271 ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&3272 (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {3273 if (tgtp)3274 atomic_inc(&tgtp->xmt_abort_rsp_error);3275 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,3276 "6134 Drop ABTS - wrong NDLP state x%x.\n",3277 (ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE);3278 3279 /* No failure to an ABTS request. */3280 return 0;3281 }3282 3283 abts_wqeq = ctxp->wqeq;3284 wqe_abts = &abts_wqeq->wqe;3285 3286 /*3287 * Since we zero the whole WQE, we need to ensure we set the WQE fields3288 * that were initialized in lpfc_sli4_nvmet_alloc.3289 */3290 memset(wqe_abts, 0, sizeof(union lpfc_wqe));3291 3292 /* Word 5 */3293 bf_set(wqe_dfctl, &wqe_abts->xmit_sequence.wge_ctl, 0);3294 bf_set(wqe_ls, &wqe_abts->xmit_sequence.wge_ctl, 1);3295 bf_set(wqe_la, &wqe_abts->xmit_sequence.wge_ctl, 0);3296 bf_set(wqe_rctl, &wqe_abts->xmit_sequence.wge_ctl, FC_RCTL_BA_ABTS);3297 bf_set(wqe_type, &wqe_abts->xmit_sequence.wge_ctl, FC_TYPE_BLS);3298 3299 /* Word 6 */3300 bf_set(wqe_ctxt_tag, &wqe_abts->xmit_sequence.wqe_com,3301 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);3302 bf_set(wqe_xri_tag, &wqe_abts->xmit_sequence.wqe_com,3303 abts_wqeq->sli4_xritag);3304 3305 /* Word 7 */3306 bf_set(wqe_cmnd, &wqe_abts->xmit_sequence.wqe_com,3307 CMD_XMIT_SEQUENCE64_WQE);3308 bf_set(wqe_ct, &wqe_abts->xmit_sequence.wqe_com, SLI4_CT_RPI);3309 bf_set(wqe_class, &wqe_abts->xmit_sequence.wqe_com, CLASS3);3310 bf_set(wqe_pu, &wqe_abts->xmit_sequence.wqe_com, 0);3311 3312 /* Word 8 */3313 wqe_abts->xmit_sequence.wqe_com.abort_tag = abts_wqeq->iotag;3314 3315 /* Word 9 */3316 bf_set(wqe_reqtag, &wqe_abts->xmit_sequence.wqe_com, abts_wqeq->iotag);3317 /* Needs to be set by caller */3318 bf_set(wqe_rcvoxid, &wqe_abts->xmit_sequence.wqe_com, xri);3319 3320 /* Word 10 */3321 bf_set(wqe_iod, &wqe_abts->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE);3322 bf_set(wqe_lenloc, &wqe_abts->xmit_sequence.wqe_com,3323 LPFC_WQE_LENLOC_WORD12);3324 bf_set(wqe_ebde_cnt, &wqe_abts->xmit_sequence.wqe_com, 0);3325 bf_set(wqe_qosd, &wqe_abts->xmit_sequence.wqe_com, 0);3326 3327 /* Word 11 */3328 bf_set(wqe_cqid, &wqe_abts->xmit_sequence.wqe_com,3329 LPFC_WQE_CQ_ID_DEFAULT);3330 bf_set(wqe_cmd_type, &wqe_abts->xmit_sequence.wqe_com,3331 OTHER_COMMAND);3332 3333 abts_wqeq->vport = phba->pport;3334 abts_wqeq->ndlp = ndlp;3335 abts_wqeq->context_un.axchg = ctxp;3336 abts_wqeq->bpl_dmabuf = NULL;3337 abts_wqeq->num_bdes = 0;3338 /* hba_wqidx should already be setup from command we are aborting */3339 abts_wqeq->iocb.ulpCommand = CMD_XMIT_SEQUENCE64_CR;3340 abts_wqeq->iocb.ulpLe = 1;3341 3342 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,3343 "6069 Issue ABTS to xri x%x reqtag x%x\n",3344 xri, abts_wqeq->iotag);3345 return 1;3346}3347 3348static int3349lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba,3350 struct lpfc_async_xchg_ctx *ctxp,3351 uint32_t sid, uint16_t xri)3352{3353 struct lpfc_nvmet_tgtport *tgtp;3354 struct lpfc_iocbq *abts_wqeq;3355 struct lpfc_nodelist *ndlp;3356 unsigned long flags;3357 bool ia;3358 int rc;3359 3360 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;3361 if (!ctxp->wqeq) {3362 ctxp->wqeq = ctxp->ctxbuf->iocbq;3363 ctxp->wqeq->hba_wqidx = 0;3364 }3365 3366 ndlp = lpfc_findnode_did(phba->pport, sid);3367 if (!ndlp ||3368 ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&3369 (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {3370 atomic_inc(&tgtp->xmt_abort_rsp_error);3371 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,3372 "6160 Drop ABORT - wrong NDLP state x%x.\n",3373 (ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE);3374 3375 /* No failure to an ABTS request. */3376 spin_lock_irqsave(&ctxp->ctxlock, flags);3377 ctxp->flag &= ~LPFC_NVME_ABORT_OP;3378 spin_unlock_irqrestore(&ctxp->ctxlock, flags);3379 return 0;3380 }3381 3382 /* Issue ABTS for this WQE based on iotag */3383 ctxp->abort_wqeq = lpfc_sli_get_iocbq(phba);3384 spin_lock_irqsave(&ctxp->ctxlock, flags);3385 if (!ctxp->abort_wqeq) {3386 atomic_inc(&tgtp->xmt_abort_rsp_error);3387 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,3388 "6161 ABORT failed: No wqeqs: "3389 "xri: x%x\n", ctxp->oxid);3390 /* No failure to an ABTS request. */3391 ctxp->flag &= ~LPFC_NVME_ABORT_OP;3392 spin_unlock_irqrestore(&ctxp->ctxlock, flags);3393 return 0;3394 }3395 abts_wqeq = ctxp->abort_wqeq;3396 ctxp->state = LPFC_NVME_STE_ABORT;3397 ia = (ctxp->flag & LPFC_NVME_ABTS_RCV) ? true : false;3398 spin_unlock_irqrestore(&ctxp->ctxlock, flags);3399 3400 /* Announce entry to new IO submit field. */3401 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,3402 "6162 ABORT Request to rport DID x%06x "3403 "for xri x%x x%x\n",3404 ctxp->sid, ctxp->oxid, ctxp->wqeq->sli4_xritag);3405 3406 /* If the hba is getting reset, this flag is set. It is3407 * cleared when the reset is complete and rings reestablished.3408 */3409 /* driver queued commands are in process of being flushed */3410 if (test_bit(HBA_IOQ_FLUSH, &phba->hba_flag)) {3411 atomic_inc(&tgtp->xmt_abort_rsp_error);3412 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,3413 "6163 Driver in reset cleanup - flushing "3414 "NVME Req now. hba_flag x%lx oxid x%x\n",3415 phba->hba_flag, ctxp->oxid);3416 lpfc_sli_release_iocbq(phba, abts_wqeq);3417 spin_lock_irqsave(&ctxp->ctxlock, flags);3418 ctxp->flag &= ~LPFC_NVME_ABORT_OP;3419 spin_unlock_irqrestore(&ctxp->ctxlock, flags);3420 return 0;3421 }3422 3423 spin_lock_irqsave(&phba->hbalock, flags);3424 /* Outstanding abort is in progress */3425 if (abts_wqeq->cmd_flag & LPFC_DRIVER_ABORTED) {3426 spin_unlock_irqrestore(&phba->hbalock, flags);3427 atomic_inc(&tgtp->xmt_abort_rsp_error);3428 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,3429 "6164 Outstanding NVME I/O Abort Request "3430 "still pending on oxid x%x\n",3431 ctxp->oxid);3432 lpfc_sli_release_iocbq(phba, abts_wqeq);3433 spin_lock_irqsave(&ctxp->ctxlock, flags);3434 ctxp->flag &= ~LPFC_NVME_ABORT_OP;3435 spin_unlock_irqrestore(&ctxp->ctxlock, flags);3436 return 0;3437 }3438 3439 /* Ready - mark outstanding as aborted by driver. */3440 abts_wqeq->cmd_flag |= LPFC_DRIVER_ABORTED;3441 3442 lpfc_sli_prep_abort_xri(phba, abts_wqeq, ctxp->wqeq->sli4_xritag,3443 abts_wqeq->iotag, CLASS3,3444 LPFC_WQE_CQ_ID_DEFAULT, ia, true);3445 3446 /* ABTS WQE must go to the same WQ as the WQE to be aborted */3447 abts_wqeq->hba_wqidx = ctxp->wqeq->hba_wqidx;3448 abts_wqeq->cmd_cmpl = lpfc_nvmet_sol_fcp_abort_cmp;3449 abts_wqeq->cmd_flag |= LPFC_IO_NVME;3450 abts_wqeq->context_un.axchg = ctxp;3451 abts_wqeq->vport = phba->pport;3452 if (!ctxp->hdwq)3453 ctxp->hdwq = &phba->sli4_hba.hdwq[abts_wqeq->hba_wqidx];3454 3455 rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, abts_wqeq);3456 spin_unlock_irqrestore(&phba->hbalock, flags);3457 if (rc == WQE_SUCCESS) {3458 atomic_inc(&tgtp->xmt_abort_sol);3459 return 0;3460 }3461 3462 atomic_inc(&tgtp->xmt_abort_rsp_error);3463 spin_lock_irqsave(&ctxp->ctxlock, flags);3464 ctxp->flag &= ~LPFC_NVME_ABORT_OP;3465 spin_unlock_irqrestore(&ctxp->ctxlock, flags);3466 lpfc_sli_release_iocbq(phba, abts_wqeq);3467 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,3468 "6166 Failed ABORT issue_wqe with status x%x "3469 "for oxid x%x.\n",3470 rc, ctxp->oxid);3471 return 1;3472}3473 3474static int3475lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *phba,3476 struct lpfc_async_xchg_ctx *ctxp,3477 uint32_t sid, uint16_t xri)3478{3479 struct lpfc_nvmet_tgtport *tgtp;3480 struct lpfc_iocbq *abts_wqeq;3481 unsigned long flags;3482 bool released = false;3483 int rc;3484 3485 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;3486 if (!ctxp->wqeq) {3487 ctxp->wqeq = ctxp->ctxbuf->iocbq;3488 ctxp->wqeq->hba_wqidx = 0;3489 }3490 3491 if (ctxp->state == LPFC_NVME_STE_FREE) {3492 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,3493 "6417 NVMET ABORT ctx freed %d %d oxid x%x\n",3494 ctxp->state, ctxp->entry_cnt, ctxp->oxid);3495 rc = WQE_BUSY;3496 goto aerr;3497 }3498 ctxp->state = LPFC_NVME_STE_ABORT;3499 ctxp->entry_cnt++;3500 rc = lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri);3501 if (rc == 0)3502 goto aerr;3503 3504 spin_lock_irqsave(&phba->hbalock, flags);3505 abts_wqeq = ctxp->wqeq;3506 abts_wqeq->cmd_cmpl = lpfc_nvmet_unsol_fcp_abort_cmp;3507 abts_wqeq->cmd_flag |= LPFC_IO_NVMET;3508 if (!ctxp->hdwq)3509 ctxp->hdwq = &phba->sli4_hba.hdwq[abts_wqeq->hba_wqidx];3510 3511 rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, abts_wqeq);3512 spin_unlock_irqrestore(&phba->hbalock, flags);3513 if (rc == WQE_SUCCESS) {3514 return 0;3515 }3516 3517aerr:3518 spin_lock_irqsave(&ctxp->ctxlock, flags);3519 if (ctxp->flag & LPFC_NVME_CTX_RLS) {3520 spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock);3521 list_del_init(&ctxp->list);3522 spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock);3523 released = true;3524 }3525 ctxp->flag &= ~(LPFC_NVME_ABORT_OP | LPFC_NVME_CTX_RLS);3526 spin_unlock_irqrestore(&ctxp->ctxlock, flags);3527 3528 atomic_inc(&tgtp->xmt_abort_rsp_error);3529 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,3530 "6135 Failed to Issue ABTS for oxid x%x. Status x%x "3531 "(%x)\n",3532 ctxp->oxid, rc, released);3533 if (released)3534 lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);3535 return 1;3536}3537 3538/**3539 * lpfc_nvme_unsol_ls_issue_abort - issue ABTS on an exchange received3540 * via async frame receive where the frame is not handled.3541 * @phba: pointer to adapter structure3542 * @ctxp: pointer to the asynchronously received received sequence3543 * @sid: address of the remote port to send the ABTS to3544 * @xri: oxid value to for the ABTS (other side's exchange id).3545 **/3546int3547lpfc_nvme_unsol_ls_issue_abort(struct lpfc_hba *phba,3548 struct lpfc_async_xchg_ctx *ctxp,3549 uint32_t sid, uint16_t xri)3550{3551 struct lpfc_nvmet_tgtport *tgtp = NULL;3552 struct lpfc_iocbq *abts_wqeq;3553 unsigned long flags;3554 int rc;3555 3556 if ((ctxp->state == LPFC_NVME_STE_LS_RCV && ctxp->entry_cnt == 1) ||3557 (ctxp->state == LPFC_NVME_STE_LS_RSP && ctxp->entry_cnt == 2)) {3558 ctxp->state = LPFC_NVME_STE_LS_ABORT;3559 ctxp->entry_cnt++;3560 } else {3561 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,3562 "6418 NVMET LS abort state mismatch "3563 "IO x%x: %d %d\n",3564 ctxp->oxid, ctxp->state, ctxp->entry_cnt);3565 ctxp->state = LPFC_NVME_STE_LS_ABORT;3566 }3567 3568 if (phba->nvmet_support && phba->targetport)3569 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;3570 3571 if (!ctxp->wqeq) {3572 /* Issue ABTS for this WQE based on iotag */3573 ctxp->wqeq = lpfc_sli_get_iocbq(phba);3574 if (!ctxp->wqeq) {3575 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,3576 "6068 Abort failed: No wqeqs: "3577 "xri: x%x\n", xri);3578 /* No failure to an ABTS request. */3579 kfree(ctxp);3580 return 0;3581 }3582 }3583 abts_wqeq = ctxp->wqeq;3584 3585 if (lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri) == 0) {3586 rc = WQE_BUSY;3587 goto out;3588 }3589 3590 spin_lock_irqsave(&phba->hbalock, flags);3591 abts_wqeq->cmd_cmpl = lpfc_nvmet_xmt_ls_abort_cmp;3592 abts_wqeq->cmd_flag |= LPFC_IO_NVME_LS;3593 rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, abts_wqeq);3594 spin_unlock_irqrestore(&phba->hbalock, flags);3595 if (rc == WQE_SUCCESS) {3596 if (tgtp)3597 atomic_inc(&tgtp->xmt_abort_unsol);3598 return 0;3599 }3600out:3601 if (tgtp)3602 atomic_inc(&tgtp->xmt_abort_rsp_error);3603 abts_wqeq->rsp_dmabuf = NULL;3604 abts_wqeq->bpl_dmabuf = NULL;3605 lpfc_sli_release_iocbq(phba, abts_wqeq);3606 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,3607 "6056 Failed to Issue ABTS. Status x%x\n", rc);3608 return 1;3609}3610 3611/**3612 * lpfc_nvmet_invalidate_host3613 *3614 * @phba: pointer to the driver instance bound to an adapter port.3615 * @ndlp: pointer to an lpfc_nodelist type3616 *3617 * This routine upcalls the nvmet transport to invalidate an NVME3618 * host to which this target instance had active connections.3619 */3620void3621lpfc_nvmet_invalidate_host(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp)3622{3623 u32 ndlp_has_hh;3624 struct lpfc_nvmet_tgtport *tgtp;3625 3626 lpfc_printf_log(phba, KERN_INFO,3627 LOG_NVME | LOG_NVME_ABTS | LOG_NVME_DISC,3628 "6203 Invalidating hosthandle x%px\n",3629 ndlp);3630 3631 tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;3632 atomic_set(&tgtp->state, LPFC_NVMET_INV_HOST_ACTIVE);3633 3634 spin_lock_irq(&ndlp->lock);3635 ndlp_has_hh = ndlp->fc4_xpt_flags & NLP_XPT_HAS_HH;3636 spin_unlock_irq(&ndlp->lock);3637 3638 /* Do not invalidate any nodes that do not have a hosthandle.3639 * The host_release callbk will cause a node reference3640 * count imbalance and a crash.3641 */3642 if (!ndlp_has_hh) {3643 lpfc_printf_log(phba, KERN_INFO,3644 LOG_NVME | LOG_NVME_ABTS | LOG_NVME_DISC,3645 "6204 Skip invalidate on node x%px DID x%x\n",3646 ndlp, ndlp->nlp_DID);3647 return;3648 }3649 3650#if (IS_ENABLED(CONFIG_NVME_TARGET_FC))3651 /* Need to get the nvmet_fc_target_port pointer here.*/3652 nvmet_fc_invalidate_host(phba->targetport, ndlp);3653#endif3654}3655