2844 lines · c
1/*******************************************************************2 * This file is part of the Emulex Linux Device Driver for *3 * Fibre Channel Host Bus Adapters. *4 * Copyright (C) 2017-2024 Broadcom. All Rights Reserved. The term *5 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. *6 * Copyright (C) 2004-2016 Emulex. All rights reserved. *7 * EMULEX and SLI are trademarks of Emulex. *8 * www.broadcom.com *9 * Portions Copyright (C) 2004-2005 Christoph Hellwig *10 * *11 * This program is free software; you can redistribute it and/or *12 * modify it under the terms of version 2 of the GNU General *13 * Public License as published by the Free Software Foundation. *14 * This program is distributed in the hope that it will be useful. *15 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *16 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *17 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *18 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *19 * TO BE LEGALLY INVALID. See the GNU General Public License for *20 * more details, a copy of which can be found in the file COPYING *21 * included with this package. *22 ********************************************************************/23#include <linux/pci.h>24#include <linux/slab.h>25#include <linux/interrupt.h>26#include <linux/delay.h>27#include <linux/unaligned.h>28#include <linux/crc-t10dif.h>29#include <net/checksum.h>30 31#include <scsi/scsi.h>32#include <scsi/scsi_device.h>33#include <scsi/scsi_eh.h>34#include <scsi/scsi_host.h>35#include <scsi/scsi_tcq.h>36#include <scsi/scsi_transport_fc.h>37#include <scsi/fc/fc_fs.h>38 39#include "lpfc_version.h"40#include "lpfc_hw4.h"41#include "lpfc_hw.h"42#include "lpfc_sli.h"43#include "lpfc_sli4.h"44#include "lpfc_nl.h"45#include "lpfc_disc.h"46#include "lpfc.h"47#include "lpfc_nvme.h"48#include "lpfc_scsi.h"49#include "lpfc_logmsg.h"50#include "lpfc_crtn.h"51#include "lpfc_vport.h"52#include "lpfc_debugfs.h"53 54/* NVME initiator-based functions */55 56static struct lpfc_io_buf *57lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,58 int idx, int expedite);59 60static void61lpfc_release_nvme_buf(struct lpfc_hba *, struct lpfc_io_buf *);62 63static struct nvme_fc_port_template lpfc_nvme_template;64 65/**66 * lpfc_nvme_create_queue -67 * @pnvme_lport: Transport localport that LS is to be issued from68 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.69 * @qsize: Size of the queue in bytes70 * @handle: An opaque driver handle used in follow-up calls.71 *72 * Driver registers this routine to preallocate and initialize any73 * internal data structures to bind the @qidx to its internal IO queues.74 * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ.75 *76 * Return value :77 * 0 - Success78 * -EINVAL - Unsupported input value.79 * -ENOMEM - Could not alloc necessary memory80 **/81static int82lpfc_nvme_create_queue(struct nvme_fc_local_port *pnvme_lport,83 unsigned int qidx, u16 qsize,84 void **handle)85{86 struct lpfc_nvme_lport *lport;87 struct lpfc_vport *vport;88 struct lpfc_nvme_qhandle *qhandle;89 char *str;90 91 if (!pnvme_lport->private)92 return -ENOMEM;93 94 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;95 vport = lport->vport;96 97 if (!vport || test_bit(FC_UNLOADING, &vport->load_flag) ||98 test_bit(HBA_IOQ_FLUSH, &vport->phba->hba_flag))99 return -ENODEV;100 101 qhandle = kzalloc(sizeof(struct lpfc_nvme_qhandle), GFP_KERNEL);102 if (qhandle == NULL)103 return -ENOMEM;104 105 qhandle->cpu_id = raw_smp_processor_id();106 qhandle->qidx = qidx;107 /*108 * NVME qidx == 0 is the admin queue, so both admin queue109 * and first IO queue will use MSI-X vector and associated110 * EQ/CQ/WQ at index 0. After that they are sequentially assigned.111 */112 if (qidx) {113 str = "IO "; /* IO queue */114 qhandle->index = ((qidx - 1) %115 lpfc_nvme_template.max_hw_queues);116 } else {117 str = "ADM"; /* Admin queue */118 qhandle->index = qidx;119 }120 121 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,122 "6073 Binding %s HdwQueue %d (cpu %d) to "123 "hdw_queue %d qhandle x%px\n", str,124 qidx, qhandle->cpu_id, qhandle->index, qhandle);125 *handle = (void *)qhandle;126 return 0;127}128 129/**130 * lpfc_nvme_delete_queue -131 * @pnvme_lport: Transport localport that LS is to be issued from132 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.133 * @handle: An opaque driver handle from lpfc_nvme_create_queue134 *135 * Driver registers this routine to free136 * any internal data structures to bind the @qidx to its internal137 * IO queues.138 *139 * Return value :140 * 0 - Success141 * TODO: What are the failure codes.142 **/143static void144lpfc_nvme_delete_queue(struct nvme_fc_local_port *pnvme_lport,145 unsigned int qidx,146 void *handle)147{148 struct lpfc_nvme_lport *lport;149 struct lpfc_vport *vport;150 151 if (!pnvme_lport->private)152 return;153 154 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;155 vport = lport->vport;156 157 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,158 "6001 ENTER. lpfc_pnvme x%px, qidx x%x qhandle x%px\n",159 lport, qidx, handle);160 kfree(handle);161}162 163static void164lpfc_nvme_localport_delete(struct nvme_fc_local_port *localport)165{166 struct lpfc_nvme_lport *lport = localport->private;167 168 lpfc_printf_vlog(lport->vport, KERN_INFO, LOG_NVME,169 "6173 localport x%px delete complete\n",170 lport);171 172 /* release any threads waiting for the unreg to complete */173 if (lport->vport->localport)174 complete(lport->lport_unreg_cmp);175}176 177/* lpfc_nvme_remoteport_delete178 *179 * @remoteport: Pointer to an nvme transport remoteport instance.180 *181 * This is a template downcall. NVME transport calls this function182 * when it has completed the unregistration of a previously183 * registered remoteport.184 *185 * Return value :186 * None187 */188static void189lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port *remoteport)190{191 struct lpfc_nvme_rport *rport = remoteport->private;192 struct lpfc_vport *vport;193 struct lpfc_nodelist *ndlp;194 u32 fc4_xpt_flags;195 196 ndlp = rport->ndlp;197 if (!ndlp) {198 pr_err("**** %s: NULL ndlp on rport x%px remoteport x%px\n",199 __func__, rport, remoteport);200 goto rport_err;201 }202 203 vport = ndlp->vport;204 if (!vport) {205 pr_err("**** %s: Null vport on ndlp x%px, ste x%x rport x%px\n",206 __func__, ndlp, ndlp->nlp_state, rport);207 goto rport_err;208 }209 210 fc4_xpt_flags = NVME_XPT_REGD | SCSI_XPT_REGD;211 212 /* Remove this rport from the lport's list - memory is owned by the213 * transport. Remove the ndlp reference for the NVME transport before214 * calling state machine to remove the node.215 */216 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,217 "6146 remoteport delete of remoteport x%px, ndlp x%px "218 "DID x%x xflags x%x\n",219 remoteport, ndlp, ndlp->nlp_DID, ndlp->fc4_xpt_flags);220 spin_lock_irq(&ndlp->lock);221 222 /* The register rebind might have occurred before the delete223 * downcall. Guard against this race.224 */225 if (ndlp->fc4_xpt_flags & NVME_XPT_UNREG_WAIT)226 ndlp->fc4_xpt_flags &= ~(NVME_XPT_UNREG_WAIT | NVME_XPT_REGD);227 228 spin_unlock_irq(&ndlp->lock);229 230 /* On a devloss timeout event, one more put is executed provided the231 * NVME and SCSI rport unregister requests are complete.232 */233 if (!(ndlp->fc4_xpt_flags & fc4_xpt_flags))234 lpfc_disc_state_machine(vport, ndlp, NULL, NLP_EVT_DEVICE_RM);235 236 rport_err:237 return;238}239 240/**241 * lpfc_nvme_handle_lsreq - Process an unsolicited NVME LS request242 * @phba: pointer to lpfc hba data structure.243 * @axchg: pointer to exchange context for the NVME LS request244 *245 * This routine is used for processing an asychronously received NVME LS246 * request. Any remaining validation is done and the LS is then forwarded247 * to the nvme-fc transport via nvme_fc_rcv_ls_req().248 *249 * The calling sequence should be: nvme_fc_rcv_ls_req() -> (processing)250 * -> lpfc_nvme_xmt_ls_rsp/cmp -> req->done.251 * __lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg.252 *253 * Returns 0 if LS was handled and delivered to the transport254 * Returns 1 if LS failed to be handled and should be dropped255 */256int257lpfc_nvme_handle_lsreq(struct lpfc_hba *phba,258 struct lpfc_async_xchg_ctx *axchg)259{260#if (IS_ENABLED(CONFIG_NVME_FC))261 struct lpfc_vport *vport;262 struct lpfc_nvme_rport *lpfc_rport;263 struct nvme_fc_remote_port *remoteport;264 struct lpfc_nvme_lport *lport;265 uint32_t *payload = axchg->payload;266 int rc;267 268 vport = axchg->ndlp->vport;269 lpfc_rport = axchg->ndlp->nrport;270 if (!lpfc_rport)271 return -EINVAL;272 273 remoteport = lpfc_rport->remoteport;274 if (!vport->localport ||275 test_bit(HBA_IOQ_FLUSH, &vport->phba->hba_flag))276 return -EINVAL;277 278 lport = vport->localport->private;279 if (!lport)280 return -EINVAL;281 282 rc = nvme_fc_rcv_ls_req(remoteport, &axchg->ls_rsp, axchg->payload,283 axchg->size);284 285 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,286 "6205 NVME Unsol rcv: sz %d rc %d: %08x %08x %08x "287 "%08x %08x %08x\n",288 axchg->size, rc,289 *payload, *(payload+1), *(payload+2),290 *(payload+3), *(payload+4), *(payload+5));291 292 if (!rc)293 return 0;294#endif295 return 1;296}297 298/**299 * __lpfc_nvme_ls_req_cmp - Generic completion handler for a NVME300 * LS request.301 * @phba: Pointer to HBA context object302 * @vport: The local port that issued the LS303 * @cmdwqe: Pointer to driver command WQE object.304 * @wcqe: Pointer to driver response CQE object.305 *306 * This function is the generic completion handler for NVME LS requests.307 * The function updates any states and statistics, calls the transport308 * ls_req done() routine, then tears down the command and buffers used309 * for the LS request.310 **/311void312__lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_vport *vport,313 struct lpfc_iocbq *cmdwqe,314 struct lpfc_wcqe_complete *wcqe)315{316 struct nvmefc_ls_req *pnvme_lsreq;317 struct lpfc_dmabuf *buf_ptr;318 struct lpfc_nodelist *ndlp;319 int status;320 321 pnvme_lsreq = cmdwqe->context_un.nvme_lsreq;322 ndlp = cmdwqe->ndlp;323 buf_ptr = cmdwqe->bpl_dmabuf;324 325 status = bf_get(lpfc_wcqe_c_status, wcqe);326 327 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,328 "6047 NVMEx LS REQ x%px cmpl DID %x Xri: %x "329 "status %x reason x%x cmd:x%px lsreg:x%px bmp:x%px "330 "ndlp:x%px\n",331 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,332 cmdwqe->sli4_xritag, status,333 (wcqe->parameter & 0xffff),334 cmdwqe, pnvme_lsreq, cmdwqe->bpl_dmabuf,335 ndlp);336 337 lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x parm x%x\n",338 cmdwqe->sli4_xritag, status, wcqe->parameter);339 340 if (buf_ptr) {341 lpfc_mbuf_free(phba, buf_ptr->virt, buf_ptr->phys);342 kfree(buf_ptr);343 cmdwqe->bpl_dmabuf = NULL;344 }345 if (pnvme_lsreq->done) {346 if (status != CQE_STATUS_SUCCESS)347 status = -ENXIO;348 pnvme_lsreq->done(pnvme_lsreq, status);349 } else {350 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,351 "6046 NVMEx cmpl without done call back? "352 "Data x%px DID %x Xri: %x status %x\n",353 pnvme_lsreq, ndlp ? ndlp->nlp_DID : 0,354 cmdwqe->sli4_xritag, status);355 }356 if (ndlp) {357 lpfc_nlp_put(ndlp);358 cmdwqe->ndlp = NULL;359 }360 lpfc_sli_release_iocbq(phba, cmdwqe);361}362 363static void364lpfc_nvme_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,365 struct lpfc_iocbq *rspwqe)366{367 struct lpfc_vport *vport = cmdwqe->vport;368 struct lpfc_nvme_lport *lport;369 uint32_t status;370 struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;371 372 status = bf_get(lpfc_wcqe_c_status, wcqe);373 374 if (vport->localport) {375 lport = (struct lpfc_nvme_lport *)vport->localport->private;376 if (lport) {377 atomic_inc(&lport->fc4NvmeLsCmpls);378 if (status) {379 if (bf_get(lpfc_wcqe_c_xb, wcqe))380 atomic_inc(&lport->cmpl_ls_xb);381 atomic_inc(&lport->cmpl_ls_err);382 }383 }384 }385 386 __lpfc_nvme_ls_req_cmp(phba, vport, cmdwqe, wcqe);387}388 389static int390lpfc_nvme_gen_req(struct lpfc_vport *vport, struct lpfc_dmabuf *bmp,391 struct lpfc_dmabuf *inp,392 struct nvmefc_ls_req *pnvme_lsreq,393 void (*cmpl)(struct lpfc_hba *, struct lpfc_iocbq *,394 struct lpfc_iocbq *),395 struct lpfc_nodelist *ndlp, uint32_t num_entry,396 uint32_t tmo, uint8_t retry)397{398 struct lpfc_hba *phba = vport->phba;399 union lpfc_wqe128 *wqe;400 struct lpfc_iocbq *genwqe;401 struct ulp_bde64 *bpl;402 struct ulp_bde64 bde;403 int i, rc, xmit_len, first_len;404 405 /* Allocate buffer for command WQE */406 genwqe = lpfc_sli_get_iocbq(phba);407 if (genwqe == NULL)408 return 1;409 410 wqe = &genwqe->wqe;411 /* Initialize only 64 bytes */412 memset(wqe, 0, sizeof(union lpfc_wqe));413 414 genwqe->bpl_dmabuf = bmp;415 genwqe->cmd_flag |= LPFC_IO_NVME_LS;416 417 /* Save for completion so we can release these resources */418 genwqe->ndlp = lpfc_nlp_get(ndlp);419 if (!genwqe->ndlp) {420 dev_warn(&phba->pcidev->dev,421 "Warning: Failed node ref, not sending LS_REQ\n");422 lpfc_sli_release_iocbq(phba, genwqe);423 return 1;424 }425 426 genwqe->context_un.nvme_lsreq = pnvme_lsreq;427 /* Fill in payload, bp points to frame payload */428 429 if (!tmo)430 /* FC spec states we need 3 * ratov for CT requests */431 tmo = (3 * phba->fc_ratov);432 433 /* For this command calculate the xmit length of the request bde. */434 xmit_len = 0;435 first_len = 0;436 bpl = (struct ulp_bde64 *)bmp->virt;437 for (i = 0; i < num_entry; i++) {438 bde.tus.w = bpl[i].tus.w;439 if (bde.tus.f.bdeFlags != BUFF_TYPE_BDE_64)440 break;441 xmit_len += bde.tus.f.bdeSize;442 if (i == 0)443 first_len = xmit_len;444 }445 446 genwqe->num_bdes = num_entry;447 genwqe->hba_wqidx = 0;448 449 /* Words 0 - 2 */450 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;451 wqe->generic.bde.tus.f.bdeSize = first_len;452 wqe->generic.bde.addrLow = bpl[0].addrLow;453 wqe->generic.bde.addrHigh = bpl[0].addrHigh;454 455 /* Word 3 */456 wqe->gen_req.request_payload_len = first_len;457 458 /* Word 4 */459 460 /* Word 5 */461 bf_set(wqe_dfctl, &wqe->gen_req.wge_ctl, 0);462 bf_set(wqe_si, &wqe->gen_req.wge_ctl, 1);463 bf_set(wqe_la, &wqe->gen_req.wge_ctl, 1);464 bf_set(wqe_rctl, &wqe->gen_req.wge_ctl, FC_RCTL_ELS4_REQ);465 bf_set(wqe_type, &wqe->gen_req.wge_ctl, FC_TYPE_NVME);466 467 /* Word 6 */468 bf_set(wqe_ctxt_tag, &wqe->gen_req.wqe_com,469 phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);470 bf_set(wqe_xri_tag, &wqe->gen_req.wqe_com, genwqe->sli4_xritag);471 472 /* Word 7 */473 bf_set(wqe_tmo, &wqe->gen_req.wqe_com, tmo);474 bf_set(wqe_class, &wqe->gen_req.wqe_com, CLASS3);475 bf_set(wqe_cmnd, &wqe->gen_req.wqe_com, CMD_GEN_REQUEST64_WQE);476 bf_set(wqe_ct, &wqe->gen_req.wqe_com, SLI4_CT_RPI);477 478 /* Word 8 */479 wqe->gen_req.wqe_com.abort_tag = genwqe->iotag;480 481 /* Word 9 */482 bf_set(wqe_reqtag, &wqe->gen_req.wqe_com, genwqe->iotag);483 484 /* Word 10 */485 bf_set(wqe_dbde, &wqe->gen_req.wqe_com, 1);486 bf_set(wqe_iod, &wqe->gen_req.wqe_com, LPFC_WQE_IOD_READ);487 bf_set(wqe_qosd, &wqe->gen_req.wqe_com, 1);488 bf_set(wqe_lenloc, &wqe->gen_req.wqe_com, LPFC_WQE_LENLOC_NONE);489 bf_set(wqe_ebde_cnt, &wqe->gen_req.wqe_com, 0);490 491 /* Word 11 */492 bf_set(wqe_cqid, &wqe->gen_req.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);493 bf_set(wqe_cmd_type, &wqe->gen_req.wqe_com, OTHER_COMMAND);494 495 496 /* Issue GEN REQ WQE for NPORT <did> */497 genwqe->cmd_cmpl = cmpl;498 genwqe->drvrTimeout = tmo + LPFC_DRVR_TIMEOUT;499 genwqe->vport = vport;500 genwqe->retry = retry;501 502 lpfc_nvmeio_data(phba, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n",503 genwqe->sli4_xritag, genwqe->iotag, ndlp->nlp_DID);504 505 rc = lpfc_sli4_issue_wqe(phba, &phba->sli4_hba.hdwq[0], genwqe);506 if (rc) {507 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,508 "6045 Issue GEN REQ WQE to NPORT x%x "509 "Data: x%x x%x rc x%x\n",510 ndlp->nlp_DID, genwqe->iotag,511 vport->port_state, rc);512 lpfc_nlp_put(ndlp);513 lpfc_sli_release_iocbq(phba, genwqe);514 return 1;515 }516 517 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_ELS,518 "6050 Issue GEN REQ WQE to NPORT x%x "519 "Data: oxid: x%x state: x%x wq:x%px lsreq:x%px "520 "bmp:x%px xmit:%d 1st:%d\n",521 ndlp->nlp_DID, genwqe->sli4_xritag,522 vport->port_state,523 genwqe, pnvme_lsreq, bmp, xmit_len, first_len);524 return 0;525}526 527 528/**529 * __lpfc_nvme_ls_req - Generic service routine to issue an NVME LS request530 * @vport: The local port issuing the LS531 * @ndlp: The remote port to send the LS to532 * @pnvme_lsreq: Pointer to LS request structure from the transport533 * @gen_req_cmp: Completion call-back534 *535 * Routine validates the ndlp, builds buffers and sends a GEN_REQUEST536 * WQE to perform the LS operation.537 *538 * Return value :539 * 0 - Success540 * non-zero: various error codes, in form of -Exxx541 **/542int543__lpfc_nvme_ls_req(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,544 struct nvmefc_ls_req *pnvme_lsreq,545 void (*gen_req_cmp)(struct lpfc_hba *phba,546 struct lpfc_iocbq *cmdwqe,547 struct lpfc_iocbq *rspwqe))548{549 struct lpfc_dmabuf *bmp;550 struct ulp_bde64 *bpl;551 int ret;552 uint16_t ntype, nstate;553 554 if (!ndlp) {555 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,556 "6051 NVMEx LS REQ: Bad NDLP x%px, Failing "557 "LS Req\n",558 ndlp);559 return -ENODEV;560 }561 562 ntype = ndlp->nlp_type;563 nstate = ndlp->nlp_state;564 if ((ntype & NLP_NVME_TARGET && nstate != NLP_STE_MAPPED_NODE) ||565 (ntype & NLP_NVME_INITIATOR && nstate != NLP_STE_UNMAPPED_NODE)) {566 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,567 "6088 NVMEx LS REQ: Fail DID x%06x not "568 "ready for IO. Type x%x, State x%x\n",569 ndlp->nlp_DID, ntype, nstate);570 return -ENODEV;571 }572 if (test_bit(HBA_IOQ_FLUSH, &vport->phba->hba_flag))573 return -ENODEV;574 575 if (!vport->phba->sli4_hba.nvmels_wq)576 return -ENOMEM;577 578 /*579 * there are two dma buf in the request, actually there is one and580 * the second one is just the start address + cmd size.581 * Before calling lpfc_nvme_gen_req these buffers need to be wrapped582 * in a lpfc_dmabuf struct. When freeing we just free the wrapper583 * because the nvem layer owns the data bufs.584 * We do not have to break these packets open, we don't care what is585 * in them. And we do not have to look at the resonse data, we only586 * care that we got a response. All of the caring is going to happen587 * in the nvme-fc layer.588 */589 590 bmp = kmalloc(sizeof(*bmp), GFP_KERNEL);591 if (!bmp) {592 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,593 "6044 NVMEx LS REQ: Could not alloc LS buf "594 "for DID %x\n",595 ndlp->nlp_DID);596 return -ENOMEM;597 }598 599 bmp->virt = lpfc_mbuf_alloc(vport->phba, MEM_PRI, &(bmp->phys));600 if (!bmp->virt) {601 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,602 "6042 NVMEx LS REQ: Could not alloc mbuf "603 "for DID %x\n",604 ndlp->nlp_DID);605 kfree(bmp);606 return -ENOMEM;607 }608 609 INIT_LIST_HEAD(&bmp->list);610 611 bpl = (struct ulp_bde64 *)bmp->virt;612 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rqstdma));613 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rqstdma));614 bpl->tus.f.bdeFlags = 0;615 bpl->tus.f.bdeSize = pnvme_lsreq->rqstlen;616 bpl->tus.w = le32_to_cpu(bpl->tus.w);617 bpl++;618 619 bpl->addrHigh = le32_to_cpu(putPaddrHigh(pnvme_lsreq->rspdma));620 bpl->addrLow = le32_to_cpu(putPaddrLow(pnvme_lsreq->rspdma));621 bpl->tus.f.bdeFlags = BUFF_TYPE_BDE_64I;622 bpl->tus.f.bdeSize = pnvme_lsreq->rsplen;623 bpl->tus.w = le32_to_cpu(bpl->tus.w);624 625 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,626 "6149 NVMEx LS REQ: Issue to DID 0x%06x lsreq x%px, "627 "rqstlen:%d rsplen:%d %pad %pad\n",628 ndlp->nlp_DID, pnvme_lsreq, pnvme_lsreq->rqstlen,629 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,630 &pnvme_lsreq->rspdma);631 632 ret = lpfc_nvme_gen_req(vport, bmp, pnvme_lsreq->rqstaddr,633 pnvme_lsreq, gen_req_cmp, ndlp, 2,634 pnvme_lsreq->timeout, 0);635 if (ret != WQE_SUCCESS) {636 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,637 "6052 NVMEx REQ: EXIT. issue ls wqe failed "638 "lsreq x%px Status %x DID %x\n",639 pnvme_lsreq, ret, ndlp->nlp_DID);640 lpfc_mbuf_free(vport->phba, bmp->virt, bmp->phys);641 kfree(bmp);642 return -EIO;643 }644 645 return 0;646}647 648/**649 * lpfc_nvme_ls_req - Issue an NVME Link Service request650 * @pnvme_lport: Transport localport that LS is to be issued from.651 * @pnvme_rport: Transport remoteport that LS is to be sent to.652 * @pnvme_lsreq: the transport nvme_ls_req structure for the LS653 *654 * Driver registers this routine to handle any link service request655 * from the nvme_fc transport to a remote nvme-aware port.656 *657 * Return value :658 * 0 - Success659 * non-zero: various error codes, in form of -Exxx660 **/661static int662lpfc_nvme_ls_req(struct nvme_fc_local_port *pnvme_lport,663 struct nvme_fc_remote_port *pnvme_rport,664 struct nvmefc_ls_req *pnvme_lsreq)665{666 struct lpfc_nvme_lport *lport;667 struct lpfc_nvme_rport *rport;668 struct lpfc_vport *vport;669 int ret;670 671 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;672 rport = (struct lpfc_nvme_rport *)pnvme_rport->private;673 if (unlikely(!lport) || unlikely(!rport))674 return -EINVAL;675 676 vport = lport->vport;677 if (test_bit(FC_UNLOADING, &vport->load_flag) ||678 test_bit(HBA_IOQ_FLUSH, &vport->phba->hba_flag))679 return -ENODEV;680 681 atomic_inc(&lport->fc4NvmeLsRequests);682 683 ret = __lpfc_nvme_ls_req(vport, rport->ndlp, pnvme_lsreq,684 lpfc_nvme_ls_req_cmp);685 if (ret)686 atomic_inc(&lport->xmt_ls_err);687 688 return ret;689}690 691/**692 * __lpfc_nvme_ls_abort - Generic service routine to abort a prior693 * NVME LS request694 * @vport: The local port that issued the LS695 * @ndlp: The remote port the LS was sent to696 * @pnvme_lsreq: Pointer to LS request structure from the transport697 *698 * The driver validates the ndlp, looks for the LS, and aborts the699 * LS if found.700 *701 * Returns:702 * 0 : if LS found and aborted703 * non-zero: various error conditions in form -Exxx704 **/705int706__lpfc_nvme_ls_abort(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,707 struct nvmefc_ls_req *pnvme_lsreq)708{709 struct lpfc_hba *phba = vport->phba;710 struct lpfc_sli_ring *pring;711 struct lpfc_iocbq *wqe, *next_wqe;712 bool foundit = false;713 714 if (!ndlp) {715 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,716 "6049 NVMEx LS REQ Abort: Bad NDLP x%px DID "717 "x%06x, Failing LS Req\n",718 ndlp, ndlp ? ndlp->nlp_DID : 0);719 return -EINVAL;720 }721 722 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,723 "6040 NVMEx LS REQ Abort: Issue LS_ABORT for lsreq "724 "x%px rqstlen:%d rsplen:%d %pad %pad\n",725 pnvme_lsreq, pnvme_lsreq->rqstlen,726 pnvme_lsreq->rsplen, &pnvme_lsreq->rqstdma,727 &pnvme_lsreq->rspdma);728 729 /*730 * Lock the ELS ring txcmplq and look for the wqe that matches731 * this ELS. If found, issue an abort on the wqe.732 */733 pring = phba->sli4_hba.nvmels_wq->pring;734 spin_lock_irq(&phba->hbalock);735 spin_lock(&pring->ring_lock);736 list_for_each_entry_safe(wqe, next_wqe, &pring->txcmplq, list) {737 if (wqe->context_un.nvme_lsreq == pnvme_lsreq) {738 wqe->cmd_flag |= LPFC_DRIVER_ABORTED;739 foundit = true;740 break;741 }742 }743 spin_unlock(&pring->ring_lock);744 745 if (foundit)746 lpfc_sli_issue_abort_iotag(phba, pring, wqe, NULL);747 spin_unlock_irq(&phba->hbalock);748 749 if (foundit)750 return 0;751 752 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC | LOG_NVME_ABTS,753 "6213 NVMEx LS REQ Abort: Unable to locate req x%px\n",754 pnvme_lsreq);755 return -EINVAL;756}757 758static int759lpfc_nvme_xmt_ls_rsp(struct nvme_fc_local_port *localport,760 struct nvme_fc_remote_port *remoteport,761 struct nvmefc_ls_rsp *ls_rsp)762{763 struct lpfc_async_xchg_ctx *axchg =764 container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp);765 struct lpfc_nvme_lport *lport;766 int rc;767 768 if (test_bit(FC_UNLOADING, &axchg->phba->pport->load_flag))769 return -ENODEV;770 771 lport = (struct lpfc_nvme_lport *)localport->private;772 773 rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, __lpfc_nvme_xmt_ls_rsp_cmp);774 775 if (rc) {776 /*777 * unless the failure is due to having already sent778 * the response, an abort will be generated for the779 * exchange if the rsp can't be sent.780 */781 if (rc != -EALREADY)782 atomic_inc(&lport->xmt_ls_abort);783 return rc;784 }785 786 return 0;787}788 789/**790 * lpfc_nvme_ls_abort - Abort a prior NVME LS request791 * @pnvme_lport: Transport localport that LS is to be issued from.792 * @pnvme_rport: Transport remoteport that LS is to be sent to.793 * @pnvme_lsreq: the transport nvme_ls_req structure for the LS794 *795 * Driver registers this routine to abort a NVME LS request that is796 * in progress (from the transports perspective).797 **/798static void799lpfc_nvme_ls_abort(struct nvme_fc_local_port *pnvme_lport,800 struct nvme_fc_remote_port *pnvme_rport,801 struct nvmefc_ls_req *pnvme_lsreq)802{803 struct lpfc_nvme_lport *lport;804 struct lpfc_vport *vport;805 struct lpfc_nodelist *ndlp;806 int ret;807 808 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;809 if (unlikely(!lport))810 return;811 vport = lport->vport;812 813 if (test_bit(FC_UNLOADING, &vport->load_flag))814 return;815 816 ndlp = lpfc_findnode_did(vport, pnvme_rport->port_id);817 818 ret = __lpfc_nvme_ls_abort(vport, ndlp, pnvme_lsreq);819 if (!ret)820 atomic_inc(&lport->xmt_ls_abort);821}822 823/* Fix up the existing sgls for NVME IO. */824static inline void825lpfc_nvme_adj_fcp_sgls(struct lpfc_vport *vport,826 struct lpfc_io_buf *lpfc_ncmd,827 struct nvmefc_fcp_req *nCmd)828{829 struct lpfc_hba *phba = vport->phba;830 struct sli4_sge *sgl;831 union lpfc_wqe128 *wqe;832 uint32_t *wptr, *dptr;833 834 /*835 * Get a local pointer to the built-in wqe and correct836 * the cmd size to match NVME's 96 bytes and fix837 * the dma address.838 */839 840 wqe = &lpfc_ncmd->cur_iocbq.wqe;841 842 /*843 * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to844 * match NVME. NVME sends 96 bytes. Also, use the845 * nvme commands command and response dma addresses846 * rather than the virtual memory to ease the restore847 * operation.848 */849 sgl = lpfc_ncmd->dma_sgl;850 sgl->sge_len = cpu_to_le32(nCmd->cmdlen);851 if (phba->cfg_nvme_embed_cmd) {852 sgl->addr_hi = 0;853 sgl->addr_lo = 0;854 855 /* Word 0-2 - NVME CMND IU (embedded payload) */856 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_IMMED;857 wqe->generic.bde.tus.f.bdeSize = 56;858 wqe->generic.bde.addrHigh = 0;859 wqe->generic.bde.addrLow = 64; /* Word 16 */860 861 /* Word 10 - dbde is 0, wqes is 1 in template */862 863 /*864 * Embed the payload in the last half of the WQE865 * WQE words 16-30 get the NVME CMD IU payload866 *867 * WQE words 16-19 get payload Words 1-4868 * WQE words 20-21 get payload Words 6-7869 * WQE words 22-29 get payload Words 16-23870 */871 wptr = &wqe->words[16]; /* WQE ptr */872 dptr = (uint32_t *)nCmd->cmdaddr; /* payload ptr */873 dptr++; /* Skip Word 0 in payload */874 875 *wptr++ = *dptr++; /* Word 1 */876 *wptr++ = *dptr++; /* Word 2 */877 *wptr++ = *dptr++; /* Word 3 */878 *wptr++ = *dptr++; /* Word 4 */879 dptr++; /* Skip Word 5 in payload */880 *wptr++ = *dptr++; /* Word 6 */881 *wptr++ = *dptr++; /* Word 7 */882 dptr += 8; /* Skip Words 8-15 in payload */883 *wptr++ = *dptr++; /* Word 16 */884 *wptr++ = *dptr++; /* Word 17 */885 *wptr++ = *dptr++; /* Word 18 */886 *wptr++ = *dptr++; /* Word 19 */887 *wptr++ = *dptr++; /* Word 20 */888 *wptr++ = *dptr++; /* Word 21 */889 *wptr++ = *dptr++; /* Word 22 */890 *wptr = *dptr; /* Word 23 */891 } else {892 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->cmddma));893 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->cmddma));894 895 /* Word 0-2 - NVME CMND IU Inline BDE */896 wqe->generic.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;897 wqe->generic.bde.tus.f.bdeSize = nCmd->cmdlen;898 wqe->generic.bde.addrHigh = sgl->addr_hi;899 wqe->generic.bde.addrLow = sgl->addr_lo;900 901 /* Word 10 */902 bf_set(wqe_dbde, &wqe->generic.wqe_com, 1);903 bf_set(wqe_wqes, &wqe->generic.wqe_com, 0);904 }905 906 sgl++;907 908 /* Setup the physical region for the FCP RSP */909 sgl->addr_hi = cpu_to_le32(putPaddrHigh(nCmd->rspdma));910 sgl->addr_lo = cpu_to_le32(putPaddrLow(nCmd->rspdma));911 sgl->word2 = le32_to_cpu(sgl->word2);912 if (nCmd->sg_cnt)913 bf_set(lpfc_sli4_sge_last, sgl, 0);914 else915 bf_set(lpfc_sli4_sge_last, sgl, 1);916 sgl->word2 = cpu_to_le32(sgl->word2);917 sgl->sge_len = cpu_to_le32(nCmd->rsplen);918}919 920 921/*922 * lpfc_nvme_io_cmd_cmpl - Complete an NVME-over-FCP IO923 *924 * Driver registers this routine as it io request handler. This925 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq926 * data structure to the rport indicated in @lpfc_nvme_rport.927 *928 * Return value :929 * 0 - Success930 * TODO: What are the failure codes.931 **/932static void933lpfc_nvme_io_cmd_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,934 struct lpfc_iocbq *pwqeOut)935{936 struct lpfc_io_buf *lpfc_ncmd = pwqeIn->io_buf;937 struct lpfc_wcqe_complete *wcqe = &pwqeOut->wcqe_cmpl;938 struct lpfc_vport *vport = pwqeIn->vport;939 struct nvmefc_fcp_req *nCmd;940 struct nvme_fc_ersp_iu *ep;941 struct nvme_fc_cmd_iu *cp;942 struct lpfc_nodelist *ndlp;943 struct lpfc_nvme_fcpreq_priv *freqpriv;944 struct lpfc_nvme_lport *lport;945 uint32_t code, status, idx;946 uint16_t cid, sqhd, data;947 uint32_t *ptr;948 uint32_t lat;949 bool call_done = false;950#ifdef CONFIG_SCSI_LPFC_DEBUG_FS951 int cpu;952#endif953 bool offline = false;954 955 /* Sanity check on return of outstanding command */956 if (!lpfc_ncmd) {957 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,958 "6071 Null lpfc_ncmd pointer. No "959 "release, skip completion\n");960 return;961 }962 963 /* Guard against abort handler being called at same time */964 spin_lock(&lpfc_ncmd->buf_lock);965 966 if (!lpfc_ncmd->nvmeCmd) {967 spin_unlock(&lpfc_ncmd->buf_lock);968 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,969 "6066 Missing cmpl ptrs: lpfc_ncmd x%px, "970 "nvmeCmd x%px\n",971 lpfc_ncmd, lpfc_ncmd->nvmeCmd);972 973 /* Release the lpfc_ncmd regardless of the missing elements. */974 lpfc_release_nvme_buf(phba, lpfc_ncmd);975 return;976 }977 nCmd = lpfc_ncmd->nvmeCmd;978 status = bf_get(lpfc_wcqe_c_status, wcqe);979 980 idx = lpfc_ncmd->cur_iocbq.hba_wqidx;981 phba->sli4_hba.hdwq[idx].nvme_cstat.io_cmpls++;982 983 if (unlikely(status && vport->localport)) {984 lport = (struct lpfc_nvme_lport *)vport->localport->private;985 if (lport) {986 if (bf_get(lpfc_wcqe_c_xb, wcqe))987 atomic_inc(&lport->cmpl_fcp_xb);988 atomic_inc(&lport->cmpl_fcp_err);989 }990 }991 992 lpfc_nvmeio_data(phba, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n",993 lpfc_ncmd->cur_iocbq.sli4_xritag,994 status, wcqe->parameter);995 /*996 * Catch race where our node has transitioned, but the997 * transport is still transitioning.998 */999 ndlp = lpfc_ncmd->ndlp;1000 if (!ndlp) {1001 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,1002 "6062 Ignoring NVME cmpl. No ndlp\n");1003 goto out_err;1004 }1005 1006 code = bf_get(lpfc_wcqe_c_code, wcqe);1007 if (code == CQE_CODE_NVME_ERSP) {1008 /* For this type of CQE, we need to rebuild the rsp */1009 ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr;1010 1011 /*1012 * Get Command Id from cmd to plug into response. This1013 * code is not needed in the next NVME Transport drop.1014 */1015 cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr;1016 cid = cp->sqe.common.command_id;1017 1018 /*1019 * RSN is in CQE word 21020 * SQHD is in CQE Word 3 bits 15:01021 * Cmd Specific info is in CQE Word 11022 * and in CQE Word 0 bits 15:01023 */1024 sqhd = bf_get(lpfc_wcqe_c_sqhead, wcqe);1025 1026 /* Now lets build the NVME ERSP IU */1027 ep->iu_len = cpu_to_be16(8);1028 ep->rsn = wcqe->parameter;1029 ep->xfrd_len = cpu_to_be32(nCmd->payload_length);1030 ep->rsvd12 = 0;1031 ptr = (uint32_t *)&ep->cqe.result.u64;1032 *ptr++ = wcqe->total_data_placed;1033 data = bf_get(lpfc_wcqe_c_ersp0, wcqe);1034 *ptr = (uint32_t)data;1035 ep->cqe.sq_head = sqhd;1036 ep->cqe.sq_id = nCmd->sqid;1037 ep->cqe.command_id = cid;1038 ep->cqe.status = 0;1039 1040 lpfc_ncmd->status = IOSTAT_SUCCESS;1041 lpfc_ncmd->result = 0;1042 nCmd->rcv_rsplen = LPFC_NVME_ERSP_LEN;1043 nCmd->transferred_length = nCmd->payload_length;1044 } else {1045 lpfc_ncmd->status = status;1046 lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK);1047 1048 /* For NVME, the only failure path that results in an1049 * IO error is when the adapter rejects it. All other1050 * conditions are a success case and resolved by the1051 * transport.1052 * IOSTAT_FCP_RSP_ERROR means:1053 * 1. Length of data received doesn't match total1054 * transfer length in WQE1055 * 2. If the RSP payload does NOT match these cases:1056 * a. RSP length 12/24 bytes and all zeros1057 * b. NVME ERSP1058 */1059 switch (lpfc_ncmd->status) {1060 case IOSTAT_SUCCESS:1061 nCmd->transferred_length = wcqe->total_data_placed;1062 nCmd->rcv_rsplen = 0;1063 nCmd->status = 0;1064 break;1065 case IOSTAT_FCP_RSP_ERROR:1066 nCmd->transferred_length = wcqe->total_data_placed;1067 nCmd->rcv_rsplen = wcqe->parameter;1068 nCmd->status = 0;1069 1070 /* Get the NVME cmd details for this unique error. */1071 cp = (struct nvme_fc_cmd_iu *)nCmd->cmdaddr;1072 ep = (struct nvme_fc_ersp_iu *)nCmd->rspaddr;1073 1074 /* Check if this is really an ERSP */1075 if (nCmd->rcv_rsplen == LPFC_NVME_ERSP_LEN) {1076 lpfc_ncmd->status = IOSTAT_SUCCESS;1077 lpfc_ncmd->result = 0;1078 1079 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,1080 "6084 NVME FCP_ERR ERSP: "1081 "xri %x placed x%x opcode x%x cmd_id "1082 "x%x cqe_status x%x\n",1083 lpfc_ncmd->cur_iocbq.sli4_xritag,1084 wcqe->total_data_placed,1085 cp->sqe.common.opcode,1086 cp->sqe.common.command_id,1087 ep->cqe.status);1088 break;1089 }1090 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,1091 "6081 NVME Completion Protocol Error: "1092 "xri %x status x%x result x%x "1093 "placed x%x opcode x%x cmd_id x%x, "1094 "cqe_status x%x\n",1095 lpfc_ncmd->cur_iocbq.sli4_xritag,1096 lpfc_ncmd->status, lpfc_ncmd->result,1097 wcqe->total_data_placed,1098 cp->sqe.common.opcode,1099 cp->sqe.common.command_id,1100 ep->cqe.status);1101 break;1102 case IOSTAT_LOCAL_REJECT:1103 /* Let fall through to set command final state. */1104 if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED)1105 lpfc_printf_vlog(vport, KERN_INFO,1106 LOG_NVME_IOERR,1107 "6032 Delay Aborted cmd x%px "1108 "nvme cmd x%px, xri x%x, "1109 "xb %d\n",1110 lpfc_ncmd, nCmd,1111 lpfc_ncmd->cur_iocbq.sli4_xritag,1112 bf_get(lpfc_wcqe_c_xb, wcqe));1113 fallthrough;1114 default:1115out_err:1116 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,1117 "6072 NVME Completion Error: xri %x "1118 "status x%x result x%x [x%x] "1119 "placed x%x\n",1120 lpfc_ncmd->cur_iocbq.sli4_xritag,1121 lpfc_ncmd->status, lpfc_ncmd->result,1122 wcqe->parameter,1123 wcqe->total_data_placed);1124 nCmd->transferred_length = 0;1125 nCmd->rcv_rsplen = 0;1126 nCmd->status = NVME_SC_INTERNAL;1127 if (pci_channel_offline(vport->phba->pcidev) ||1128 lpfc_ncmd->result == IOERR_SLI_DOWN)1129 offline = true;1130 }1131 }1132 1133 /* pick up SLI4 exhange busy condition */1134 if (bf_get(lpfc_wcqe_c_xb, wcqe) && !offline)1135 lpfc_ncmd->flags |= LPFC_SBUF_XBUSY;1136 else1137 lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY;1138 1139 /* Update stats and complete the IO. There is1140 * no need for dma unprep because the nvme_transport1141 * owns the dma address.1142 */1143#ifdef CONFIG_SCSI_LPFC_DEBUG_FS1144 if (lpfc_ncmd->ts_cmd_start) {1145 lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp;1146 lpfc_ncmd->ts_data_io = ktime_get_ns();1147 phba->ktime_last_cmd = lpfc_ncmd->ts_data_io;1148 lpfc_io_ktime(phba, lpfc_ncmd);1149 }1150 if (unlikely(phba->hdwqstat_on & LPFC_CHECK_NVME_IO)) {1151 cpu = raw_smp_processor_id();1152 this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io);1153 if (lpfc_ncmd->cpu != cpu)1154 lpfc_printf_vlog(vport,1155 KERN_INFO, LOG_NVME_IOERR,1156 "6701 CPU Check cmpl: "1157 "cpu %d expect %d\n",1158 cpu, lpfc_ncmd->cpu);1159 }1160#endif1161 1162 /* NVME targets need completion held off until the abort exchange1163 * completes unless the NVME Rport is getting unregistered.1164 */1165 1166 if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {1167 freqpriv = nCmd->private;1168 freqpriv->nvme_buf = NULL;1169 lpfc_ncmd->nvmeCmd = NULL;1170 call_done = true;1171 }1172 spin_unlock(&lpfc_ncmd->buf_lock);1173 1174 /* Check if IO qualified for CMF */1175 if (phba->cmf_active_mode != LPFC_CFG_OFF &&1176 nCmd->io_dir == NVMEFC_FCP_READ &&1177 nCmd->payload_length) {1178 /* Used when calculating average latency */1179 lat = ktime_get_ns() - lpfc_ncmd->rx_cmd_start;1180 lpfc_update_cmf_cmpl(phba, lat, nCmd->payload_length, NULL);1181 }1182 1183 if (call_done)1184 nCmd->done(nCmd);1185 1186 /* Call release with XB=1 to queue the IO into the abort list. */1187 lpfc_release_nvme_buf(phba, lpfc_ncmd);1188}1189 1190 1191/**1192 * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO1193 * @vport: pointer to a host virtual N_Port data structure1194 * @lpfc_ncmd: Pointer to lpfc scsi command1195 * @pnode: pointer to a node-list data structure1196 * @cstat: pointer to the control status structure1197 *1198 * Driver registers this routine as it io request handler. This1199 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq1200 * data structure to the rport indicated in @lpfc_nvme_rport.1201 *1202 * Return value :1203 * 0 - Success1204 * TODO: What are the failure codes.1205 **/1206static int1207lpfc_nvme_prep_io_cmd(struct lpfc_vport *vport,1208 struct lpfc_io_buf *lpfc_ncmd,1209 struct lpfc_nodelist *pnode,1210 struct lpfc_fc4_ctrl_stat *cstat)1211{1212 struct lpfc_hba *phba = vport->phba;1213 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;1214 struct nvme_common_command *sqe;1215 struct lpfc_iocbq *pwqeq = &lpfc_ncmd->cur_iocbq;1216 union lpfc_wqe128 *wqe = &pwqeq->wqe;1217 uint32_t req_len;1218 1219 /*1220 * There are three possibilities here - use scatter-gather segment, use1221 * the single mapping, or neither.1222 */1223 if (nCmd->sg_cnt) {1224 if (nCmd->io_dir == NVMEFC_FCP_WRITE) {1225 /* From the iwrite template, initialize words 7 - 11 */1226 memcpy(&wqe->words[7],1227 &lpfc_iwrite_cmd_template.words[7],1228 sizeof(uint32_t) * 5);1229 1230 /* Word 4 */1231 wqe->fcp_iwrite.total_xfer_len = nCmd->payload_length;1232 1233 /* Word 5 */1234 if ((phba->cfg_nvme_enable_fb) &&1235 (pnode->nlp_flag & NLP_FIRSTBURST)) {1236 req_len = lpfc_ncmd->nvmeCmd->payload_length;1237 if (req_len < pnode->nvme_fb_size)1238 wqe->fcp_iwrite.initial_xfer_len =1239 req_len;1240 else1241 wqe->fcp_iwrite.initial_xfer_len =1242 pnode->nvme_fb_size;1243 } else {1244 wqe->fcp_iwrite.initial_xfer_len = 0;1245 }1246 cstat->output_requests++;1247 } else {1248 /* From the iread template, initialize words 7 - 11 */1249 memcpy(&wqe->words[7],1250 &lpfc_iread_cmd_template.words[7],1251 sizeof(uint32_t) * 5);1252 1253 /* Word 4 */1254 wqe->fcp_iread.total_xfer_len = nCmd->payload_length;1255 1256 /* Word 5 */1257 wqe->fcp_iread.rsrvd5 = 0;1258 1259 /* For a CMF Managed port, iod must be zero'ed */1260 if (phba->cmf_active_mode == LPFC_CFG_MANAGED)1261 bf_set(wqe_iod, &wqe->fcp_iread.wqe_com,1262 LPFC_WQE_IOD_NONE);1263 cstat->input_requests++;1264 }1265 } else {1266 /* From the icmnd template, initialize words 4 - 11 */1267 memcpy(&wqe->words[4], &lpfc_icmnd_cmd_template.words[4],1268 sizeof(uint32_t) * 8);1269 cstat->control_requests++;1270 }1271 1272 if (pnode->nlp_nvme_info & NLP_NVME_NSLER) {1273 bf_set(wqe_erp, &wqe->generic.wqe_com, 1);1274 sqe = &((struct nvme_fc_cmd_iu *)1275 nCmd->cmdaddr)->sqe.common;1276 if (sqe->opcode == nvme_admin_async_event)1277 bf_set(wqe_ffrq, &wqe->generic.wqe_com, 1);1278 }1279 1280 /*1281 * Finish initializing those WQE fields that are independent1282 * of the nvme_cmnd request_buffer1283 */1284 1285 /* Word 3 */1286 bf_set(payload_offset_len, &wqe->fcp_icmd,1287 (nCmd->rsplen + nCmd->cmdlen));1288 1289 /* Word 6 */1290 bf_set(wqe_ctxt_tag, &wqe->generic.wqe_com,1291 phba->sli4_hba.rpi_ids[pnode->nlp_rpi]);1292 bf_set(wqe_xri_tag, &wqe->generic.wqe_com, pwqeq->sli4_xritag);1293 1294 /* Word 8 */1295 wqe->generic.wqe_com.abort_tag = pwqeq->iotag;1296 1297 /* Word 9 */1298 bf_set(wqe_reqtag, &wqe->generic.wqe_com, pwqeq->iotag);1299 1300 /* Word 10 */1301 bf_set(wqe_xchg, &wqe->fcp_iwrite.wqe_com, LPFC_NVME_XCHG);1302 1303 /* Words 13 14 15 are for PBDE support */1304 1305 /* add the VMID tags as per switch response */1306 if (unlikely(lpfc_ncmd->cur_iocbq.cmd_flag & LPFC_IO_VMID)) {1307 if (phba->pport->vmid_priority_tagging) {1308 bf_set(wqe_ccpe, &wqe->fcp_iwrite.wqe_com, 1);1309 bf_set(wqe_ccp, &wqe->fcp_iwrite.wqe_com,1310 lpfc_ncmd->cur_iocbq.vmid_tag.cs_ctl_vmid);1311 } else {1312 bf_set(wqe_appid, &wqe->fcp_iwrite.wqe_com, 1);1313 bf_set(wqe_wqes, &wqe->fcp_iwrite.wqe_com, 1);1314 wqe->words[31] = lpfc_ncmd->cur_iocbq.vmid_tag.app_id;1315 }1316 }1317 1318 pwqeq->vport = vport;1319 return 0;1320}1321 1322 1323/**1324 * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO1325 * @vport: pointer to a host virtual N_Port data structure1326 * @lpfc_ncmd: Pointer to lpfc scsi command1327 *1328 * Driver registers this routine as it io request handler. This1329 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq1330 * data structure to the rport indicated in @lpfc_nvme_rport.1331 *1332 * Return value :1333 * 0 - Success1334 * TODO: What are the failure codes.1335 **/1336static int1337lpfc_nvme_prep_io_dma(struct lpfc_vport *vport,1338 struct lpfc_io_buf *lpfc_ncmd)1339{1340 struct lpfc_hba *phba = vport->phba;1341 struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;1342 union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe;1343 struct sli4_sge *sgl = lpfc_ncmd->dma_sgl;1344 struct sli4_hybrid_sgl *sgl_xtra = NULL;1345 struct scatterlist *data_sg;1346 struct sli4_sge *first_data_sgl;1347 struct ulp_bde64 *bde;1348 dma_addr_t physaddr = 0;1349 uint32_t dma_len = 0;1350 uint32_t dma_offset = 0;1351 int nseg, i, j;1352 bool lsp_just_set = false;1353 1354 /* Fix up the command and response DMA stuff. */1355 lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd);1356 1357 /*1358 * There are three possibilities here - use scatter-gather segment, use1359 * the single mapping, or neither.1360 */1361 if (nCmd->sg_cnt) {1362 /*1363 * Jump over the cmd and rsp SGEs. The fix routine1364 * has already adjusted for this.1365 */1366 sgl += 2;1367 1368 first_data_sgl = sgl;1369 lpfc_ncmd->seg_cnt = nCmd->sg_cnt;1370 if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) {1371 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,1372 "6058 Too many sg segments from "1373 "NVME Transport. Max %d, "1374 "nvmeIO sg_cnt %d\n",1375 phba->cfg_nvme_seg_cnt + 1,1376 lpfc_ncmd->seg_cnt);1377 lpfc_ncmd->seg_cnt = 0;1378 return 1;1379 }1380 1381 /*1382 * The driver established a maximum scatter-gather segment count1383 * during probe that limits the number of sg elements in any1384 * single nvme command. Just run through the seg_cnt and format1385 * the sge's.1386 */1387 nseg = nCmd->sg_cnt;1388 data_sg = nCmd->first_sgl;1389 1390 /* for tracking the segment boundaries */1391 j = 2;1392 for (i = 0; i < nseg; i++) {1393 if (data_sg == NULL) {1394 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,1395 "6059 dptr err %d, nseg %d\n",1396 i, nseg);1397 lpfc_ncmd->seg_cnt = 0;1398 return 1;1399 }1400 1401 sgl->word2 = 0;1402 if (nseg == 1) {1403 bf_set(lpfc_sli4_sge_last, sgl, 1);1404 bf_set(lpfc_sli4_sge_type, sgl,1405 LPFC_SGE_TYPE_DATA);1406 } else {1407 bf_set(lpfc_sli4_sge_last, sgl, 0);1408 1409 /* expand the segment */1410 if (!lsp_just_set &&1411 !((j + 1) % phba->border_sge_num) &&1412 ((nseg - 1) != i)) {1413 /* set LSP type */1414 bf_set(lpfc_sli4_sge_type, sgl,1415 LPFC_SGE_TYPE_LSP);1416 1417 sgl_xtra = lpfc_get_sgl_per_hdwq(1418 phba, lpfc_ncmd);1419 1420 if (unlikely(!sgl_xtra)) {1421 lpfc_ncmd->seg_cnt = 0;1422 return 1;1423 }1424 sgl->addr_lo = cpu_to_le32(putPaddrLow(1425 sgl_xtra->dma_phys_sgl));1426 sgl->addr_hi = cpu_to_le32(putPaddrHigh(1427 sgl_xtra->dma_phys_sgl));1428 1429 } else {1430 bf_set(lpfc_sli4_sge_type, sgl,1431 LPFC_SGE_TYPE_DATA);1432 }1433 }1434 1435 if (!(bf_get(lpfc_sli4_sge_type, sgl) &1436 LPFC_SGE_TYPE_LSP)) {1437 if ((nseg - 1) == i)1438 bf_set(lpfc_sli4_sge_last, sgl, 1);1439 1440 physaddr = sg_dma_address(data_sg);1441 dma_len = sg_dma_len(data_sg);1442 sgl->addr_lo = cpu_to_le32(1443 putPaddrLow(physaddr));1444 sgl->addr_hi = cpu_to_le32(1445 putPaddrHigh(physaddr));1446 1447 bf_set(lpfc_sli4_sge_offset, sgl, dma_offset);1448 sgl->word2 = cpu_to_le32(sgl->word2);1449 sgl->sge_len = cpu_to_le32(dma_len);1450 1451 dma_offset += dma_len;1452 data_sg = sg_next(data_sg);1453 1454 sgl++;1455 1456 lsp_just_set = false;1457 } else {1458 sgl->word2 = cpu_to_le32(sgl->word2);1459 1460 sgl->sge_len = cpu_to_le32(1461 phba->cfg_sg_dma_buf_size);1462 1463 sgl = (struct sli4_sge *)sgl_xtra->dma_sgl;1464 i = i - 1;1465 1466 lsp_just_set = true;1467 }1468 1469 j++;1470 }1471 1472 /* PBDE support for first data SGE only */1473 if (nseg == 1 && phba->cfg_enable_pbde) {1474 /* Words 13-15 */1475 bde = (struct ulp_bde64 *)1476 &wqe->words[13];1477 bde->addrLow = first_data_sgl->addr_lo;1478 bde->addrHigh = first_data_sgl->addr_hi;1479 bde->tus.f.bdeSize =1480 le32_to_cpu(first_data_sgl->sge_len);1481 bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;1482 bde->tus.w = cpu_to_le32(bde->tus.w);1483 1484 /* Word 11 - set PBDE bit */1485 bf_set(wqe_pbde, &wqe->generic.wqe_com, 1);1486 } else {1487 memset(&wqe->words[13], 0, (sizeof(uint32_t) * 3));1488 /* Word 11 - PBDE bit disabled by default template */1489 }1490 1491 } else {1492 lpfc_ncmd->seg_cnt = 0;1493 1494 /* For this clause to be valid, the payload_length1495 * and sg_cnt must zero.1496 */1497 if (nCmd->payload_length != 0) {1498 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,1499 "6063 NVME DMA Prep Err: sg_cnt %d "1500 "payload_length x%x\n",1501 nCmd->sg_cnt, nCmd->payload_length);1502 return 1;1503 }1504 }1505 return 0;1506}1507 1508/**1509 * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO1510 * @pnvme_lport: Pointer to the driver's local port data1511 * @pnvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq1512 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue1513 * @pnvme_fcreq: IO request from nvme fc to driver.1514 *1515 * Driver registers this routine as it io request handler. This1516 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq1517 * data structure to the rport indicated in @lpfc_nvme_rport.1518 *1519 * Return value :1520 * 0 - Success1521 * TODO: What are the failure codes.1522 **/1523static int1524lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port *pnvme_lport,1525 struct nvme_fc_remote_port *pnvme_rport,1526 void *hw_queue_handle,1527 struct nvmefc_fcp_req *pnvme_fcreq)1528{1529 int ret = 0;1530 int expedite = 0;1531 int idx, cpu;1532 struct lpfc_nvme_lport *lport;1533 struct lpfc_fc4_ctrl_stat *cstat;1534 struct lpfc_vport *vport;1535 struct lpfc_hba *phba;1536 struct lpfc_nodelist *ndlp;1537 struct lpfc_io_buf *lpfc_ncmd;1538 struct lpfc_nvme_rport *rport;1539 struct lpfc_nvme_qhandle *lpfc_queue_info;1540 struct lpfc_nvme_fcpreq_priv *freqpriv;1541 struct nvme_common_command *sqe;1542 uint64_t start = 0;1543#if (IS_ENABLED(CONFIG_NVME_FC))1544 u8 *uuid = NULL;1545 int err;1546 enum dma_data_direction iodir;1547#endif1548 1549 /* Validate pointers. LLDD fault handling with transport does1550 * have timing races.1551 */1552 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;1553 if (unlikely(!lport)) {1554 ret = -EINVAL;1555 goto out_fail;1556 }1557 1558 vport = lport->vport;1559 1560 if (unlikely(!hw_queue_handle)) {1561 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,1562 "6117 Fail IO, NULL hw_queue_handle\n");1563 atomic_inc(&lport->xmt_fcp_err);1564 ret = -EBUSY;1565 goto out_fail;1566 }1567 1568 phba = vport->phba;1569 1570 if ((unlikely(test_bit(FC_UNLOADING, &vport->load_flag))) ||1571 test_bit(HBA_IOQ_FLUSH, &phba->hba_flag)) {1572 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,1573 "6124 Fail IO, Driver unload\n");1574 atomic_inc(&lport->xmt_fcp_err);1575 ret = -ENODEV;1576 goto out_fail;1577 }1578 1579 freqpriv = pnvme_fcreq->private;1580 if (unlikely(!freqpriv)) {1581 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,1582 "6158 Fail IO, NULL request data\n");1583 atomic_inc(&lport->xmt_fcp_err);1584 ret = -EINVAL;1585 goto out_fail;1586 }1587 1588#ifdef CONFIG_SCSI_LPFC_DEBUG_FS1589 if (phba->ktime_on)1590 start = ktime_get_ns();1591#endif1592 rport = (struct lpfc_nvme_rport *)pnvme_rport->private;1593 lpfc_queue_info = (struct lpfc_nvme_qhandle *)hw_queue_handle;1594 1595 /*1596 * Catch race where our node has transitioned, but the1597 * transport is still transitioning.1598 */1599 ndlp = rport->ndlp;1600 if (!ndlp) {1601 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,1602 "6053 Busy IO, ndlp not ready: rport x%px "1603 "ndlp x%px, DID x%06x\n",1604 rport, ndlp, pnvme_rport->port_id);1605 atomic_inc(&lport->xmt_fcp_err);1606 ret = -EBUSY;1607 goto out_fail;1608 }1609 1610 /* The remote node has to be a mapped target or it's an error. */1611 if ((ndlp->nlp_type & NLP_NVME_TARGET) &&1612 (ndlp->nlp_state != NLP_STE_MAPPED_NODE)) {1613 lpfc_printf_vlog(vport, KERN_INFO, LOG_NODE | LOG_NVME_IOERR,1614 "6036 Fail IO, DID x%06x not ready for "1615 "IO. State x%x, Type x%x Flg x%x\n",1616 pnvme_rport->port_id,1617 ndlp->nlp_state, ndlp->nlp_type,1618 ndlp->fc4_xpt_flags);1619 atomic_inc(&lport->xmt_fcp_bad_ndlp);1620 ret = -EBUSY;1621 goto out_fail;1622 1623 }1624 1625 /* Currently only NVME Keep alive commands should be expedited1626 * if the driver runs out of a resource. These should only be1627 * issued on the admin queue, qidx 01628 */1629 if (!lpfc_queue_info->qidx && !pnvme_fcreq->sg_cnt) {1630 sqe = &((struct nvme_fc_cmd_iu *)1631 pnvme_fcreq->cmdaddr)->sqe.common;1632 if (sqe->opcode == nvme_admin_keep_alive)1633 expedite = 1;1634 }1635 1636 /* Check if IO qualifies for CMF */1637 if (phba->cmf_active_mode != LPFC_CFG_OFF &&1638 pnvme_fcreq->io_dir == NVMEFC_FCP_READ &&1639 pnvme_fcreq->payload_length) {1640 ret = lpfc_update_cmf_cmd(phba, pnvme_fcreq->payload_length);1641 if (ret) {1642 ret = -EBUSY;1643 goto out_fail;1644 }1645 /* Get start time for IO latency */1646 start = ktime_get_ns();1647 }1648 1649 /* The node is shared with FCP IO, make sure the IO pending count does1650 * not exceed the programmed depth.1651 */1652 if (lpfc_ndlp_check_qdepth(phba, ndlp)) {1653 if ((atomic_read(&ndlp->cmd_pending) >= ndlp->cmd_qdepth) &&1654 !expedite) {1655 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,1656 "6174 Fail IO, ndlp qdepth exceeded: "1657 "idx %d DID %x pend %d qdepth %d\n",1658 lpfc_queue_info->index, ndlp->nlp_DID,1659 atomic_read(&ndlp->cmd_pending),1660 ndlp->cmd_qdepth);1661 atomic_inc(&lport->xmt_fcp_qdepth);1662 ret = -EBUSY;1663 goto out_fail1;1664 }1665 }1666 1667 /* Lookup Hardware Queue index based on fcp_io_sched module parameter */1668 if (phba->cfg_fcp_io_sched == LPFC_FCP_SCHED_BY_HDWQ) {1669 idx = lpfc_queue_info->index;1670 } else {1671 cpu = raw_smp_processor_id();1672 idx = phba->sli4_hba.cpu_map[cpu].hdwq;1673 }1674 1675 lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite);1676 if (lpfc_ncmd == NULL) {1677 atomic_inc(&lport->xmt_fcp_noxri);1678 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,1679 "6065 Fail IO, driver buffer pool is empty: "1680 "idx %d DID %x\n",1681 lpfc_queue_info->index, ndlp->nlp_DID);1682 ret = -EBUSY;1683 goto out_fail1;1684 }1685#ifdef CONFIG_SCSI_LPFC_DEBUG_FS1686 if (start) {1687 lpfc_ncmd->ts_cmd_start = start;1688 lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd;1689 } else {1690 lpfc_ncmd->ts_cmd_start = 0;1691 }1692#endif1693 lpfc_ncmd->rx_cmd_start = start;1694 1695 /*1696 * Store the data needed by the driver to issue, abort, and complete1697 * an IO.1698 * Do not let the IO hang out forever. There is no midlayer issuing1699 * an abort so inform the FW of the maximum IO pending time.1700 */1701 freqpriv->nvme_buf = lpfc_ncmd;1702 lpfc_ncmd->nvmeCmd = pnvme_fcreq;1703 lpfc_ncmd->ndlp = ndlp;1704 lpfc_ncmd->qidx = lpfc_queue_info->qidx;1705 1706#if (IS_ENABLED(CONFIG_NVME_FC))1707 /* check the necessary and sufficient condition to support VMID */1708 if (lpfc_is_vmid_enabled(phba) &&1709 (ndlp->vmid_support ||1710 phba->pport->vmid_priority_tagging ==1711 LPFC_VMID_PRIO_TAG_ALL_TARGETS)) {1712 /* is the I/O generated by a VM, get the associated virtual */1713 /* entity id */1714 uuid = nvme_fc_io_getuuid(pnvme_fcreq);1715 1716 if (uuid) {1717 if (pnvme_fcreq->io_dir == NVMEFC_FCP_WRITE)1718 iodir = DMA_TO_DEVICE;1719 else if (pnvme_fcreq->io_dir == NVMEFC_FCP_READ)1720 iodir = DMA_FROM_DEVICE;1721 else1722 iodir = DMA_NONE;1723 1724 err = lpfc_vmid_get_appid(vport, uuid, iodir,1725 (union lpfc_vmid_io_tag *)1726 &lpfc_ncmd->cur_iocbq.vmid_tag);1727 if (!err)1728 lpfc_ncmd->cur_iocbq.cmd_flag |= LPFC_IO_VMID;1729 }1730 }1731#endif1732 1733 /*1734 * Issue the IO on the WQ indicated by index in the hw_queue_handle.1735 * This identfier was create in our hardware queue create callback1736 * routine. The driver now is dependent on the IO queue steering from1737 * the transport. We are trusting the upper NVME layers know which1738 * index to use and that they have affinitized a CPU to this hardware1739 * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ.1740 */1741 lpfc_ncmd->cur_iocbq.hba_wqidx = idx;1742 cstat = &phba->sli4_hba.hdwq[idx].nvme_cstat;1743 1744 lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat);1745 ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd);1746 if (ret) {1747 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,1748 "6175 Fail IO, Prep DMA: "1749 "idx %d DID %x\n",1750 lpfc_queue_info->index, ndlp->nlp_DID);1751 atomic_inc(&lport->xmt_fcp_err);1752 ret = -ENOMEM;1753 goto out_free_nvme_buf;1754 }1755 1756 lpfc_nvmeio_data(phba, "NVME FCP XMIT: xri x%x idx %d to %06x\n",1757 lpfc_ncmd->cur_iocbq.sli4_xritag,1758 lpfc_queue_info->index, ndlp->nlp_DID);1759 1760 ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq);1761 if (ret) {1762 atomic_inc(&lport->xmt_fcp_wqerr);1763 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,1764 "6113 Fail IO, Could not issue WQE err %x "1765 "sid: x%x did: x%x oxid: x%x\n",1766 ret, vport->fc_myDID, ndlp->nlp_DID,1767 lpfc_ncmd->cur_iocbq.sli4_xritag);1768 goto out_free_nvme_buf;1769 }1770 1771 if (phba->cfg_xri_rebalancing)1772 lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no);1773 1774#ifdef CONFIG_SCSI_LPFC_DEBUG_FS1775 if (lpfc_ncmd->ts_cmd_start)1776 lpfc_ncmd->ts_cmd_wqput = ktime_get_ns();1777 1778 if (phba->hdwqstat_on & LPFC_CHECK_NVME_IO) {1779 cpu = raw_smp_processor_id();1780 this_cpu_inc(phba->sli4_hba.c_stat->xmt_io);1781 lpfc_ncmd->cpu = cpu;1782 if (idx != cpu)1783 lpfc_printf_vlog(vport,1784 KERN_INFO, LOG_NVME_IOERR,1785 "6702 CPU Check cmd: "1786 "cpu %d wq %d\n",1787 lpfc_ncmd->cpu,1788 lpfc_queue_info->index);1789 }1790#endif1791 return 0;1792 1793 out_free_nvme_buf:1794 if (lpfc_ncmd->nvmeCmd->sg_cnt) {1795 if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE)1796 cstat->output_requests--;1797 else1798 cstat->input_requests--;1799 } else1800 cstat->control_requests--;1801 lpfc_release_nvme_buf(phba, lpfc_ncmd);1802 out_fail1:1803 lpfc_update_cmf_cmpl(phba, LPFC_CGN_NOT_SENT,1804 pnvme_fcreq->payload_length, NULL);1805 out_fail:1806 return ret;1807}1808 1809/**1810 * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request.1811 * @phba: Pointer to HBA context object1812 * @cmdiocb: Pointer to command iocb object.1813 * @rspiocb: Pointer to response iocb object.1814 *1815 * This is the callback function for any NVME FCP IO that was aborted.1816 *1817 * Return value:1818 * None1819 **/1820void1821lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba *phba, struct lpfc_iocbq *cmdiocb,1822 struct lpfc_iocbq *rspiocb)1823{1824 struct lpfc_wcqe_complete *abts_cmpl = &rspiocb->wcqe_cmpl;1825 1826 lpfc_printf_log(phba, KERN_INFO, LOG_NVME,1827 "6145 ABORT_XRI_CN completing on rpi x%x "1828 "original iotag x%x, abort cmd iotag x%x "1829 "req_tag x%x, status x%x, hwstatus x%x\n",1830 bf_get(wqe_ctxt_tag, &cmdiocb->wqe.generic.wqe_com),1831 get_job_abtsiotag(phba, cmdiocb), cmdiocb->iotag,1832 bf_get(lpfc_wcqe_c_request_tag, abts_cmpl),1833 bf_get(lpfc_wcqe_c_status, abts_cmpl),1834 bf_get(lpfc_wcqe_c_hw_status, abts_cmpl));1835 lpfc_sli_release_iocbq(phba, cmdiocb);1836}1837 1838/**1839 * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS1840 * @pnvme_lport: Pointer to the driver's local port data1841 * @pnvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq1842 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue1843 * @pnvme_fcreq: IO request from nvme fc to driver.1844 *1845 * Driver registers this routine as its nvme request io abort handler. This1846 * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq1847 * data structure to the rport indicated in @lpfc_nvme_rport. This routine1848 * is executed asynchronously - one the target is validated as "MAPPED" and1849 * ready for IO, the driver issues the abort request and returns.1850 *1851 * Return value:1852 * None1853 **/1854static void1855lpfc_nvme_fcp_abort(struct nvme_fc_local_port *pnvme_lport,1856 struct nvme_fc_remote_port *pnvme_rport,1857 void *hw_queue_handle,1858 struct nvmefc_fcp_req *pnvme_fcreq)1859{1860 struct lpfc_nvme_lport *lport;1861 struct lpfc_vport *vport;1862 struct lpfc_hba *phba;1863 struct lpfc_io_buf *lpfc_nbuf;1864 struct lpfc_iocbq *nvmereq_wqe;1865 struct lpfc_nvme_fcpreq_priv *freqpriv;1866 unsigned long flags;1867 int ret_val;1868 1869 /* Validate pointers. LLDD fault handling with transport does1870 * have timing races.1871 */1872 lport = (struct lpfc_nvme_lport *)pnvme_lport->private;1873 if (unlikely(!lport))1874 return;1875 1876 vport = lport->vport;1877 1878 if (unlikely(!hw_queue_handle)) {1879 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,1880 "6129 Fail Abort, HW Queue Handle NULL.\n");1881 return;1882 }1883 1884 phba = vport->phba;1885 freqpriv = pnvme_fcreq->private;1886 1887 if (unlikely(!freqpriv))1888 return;1889 if (test_bit(FC_UNLOADING, &vport->load_flag))1890 return;1891 1892 /* Announce entry to new IO submit field. */1893 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,1894 "6002 Abort Request to rport DID x%06x "1895 "for nvme_fc_req x%px\n",1896 pnvme_rport->port_id,1897 pnvme_fcreq);1898 1899 lpfc_nbuf = freqpriv->nvme_buf;1900 if (!lpfc_nbuf) {1901 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,1902 "6140 NVME IO req has no matching lpfc nvme "1903 "io buffer. Skipping abort req.\n");1904 return;1905 } else if (!lpfc_nbuf->nvmeCmd) {1906 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,1907 "6141 lpfc NVME IO req has no nvme_fcreq "1908 "io buffer. Skipping abort req.\n");1909 return;1910 }1911 1912 /* driver queued commands are in process of being flushed */1913 if (test_bit(HBA_IOQ_FLUSH, &phba->hba_flag)) {1914 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,1915 "6139 Driver in reset cleanup - flushing "1916 "NVME Req now. hba_flag x%lx\n",1917 phba->hba_flag);1918 return;1919 }1920 1921 /* Guard against IO completion being called at same time */1922 spin_lock_irqsave(&lpfc_nbuf->buf_lock, flags);1923 spin_lock(&phba->hbalock);1924 1925 nvmereq_wqe = &lpfc_nbuf->cur_iocbq;1926 1927 /*1928 * The lpfc_nbuf and the mapped nvme_fcreq in the driver's1929 * state must match the nvme_fcreq passed by the nvme1930 * transport. If they don't match, it is likely the driver1931 * has already completed the NVME IO and the nvme transport1932 * has not seen it yet.1933 */1934 if (lpfc_nbuf->nvmeCmd != pnvme_fcreq) {1935 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,1936 "6143 NVME req mismatch: "1937 "lpfc_nbuf x%px nvmeCmd x%px, "1938 "pnvme_fcreq x%px. Skipping Abort xri x%x\n",1939 lpfc_nbuf, lpfc_nbuf->nvmeCmd,1940 pnvme_fcreq, nvmereq_wqe->sli4_xritag);1941 goto out_unlock;1942 }1943 1944 /* Don't abort IOs no longer on the pending queue. */1945 if (!(nvmereq_wqe->cmd_flag & LPFC_IO_ON_TXCMPLQ)) {1946 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,1947 "6142 NVME IO req x%px not queued - skipping "1948 "abort req xri x%x\n",1949 pnvme_fcreq, nvmereq_wqe->sli4_xritag);1950 goto out_unlock;1951 }1952 1953 atomic_inc(&lport->xmt_fcp_abort);1954 lpfc_nvmeio_data(phba, "NVME FCP ABORT: xri x%x idx %d to %06x\n",1955 nvmereq_wqe->sli4_xritag,1956 nvmereq_wqe->hba_wqidx, pnvme_rport->port_id);1957 1958 /* Outstanding abort is in progress */1959 if (nvmereq_wqe->cmd_flag & LPFC_DRIVER_ABORTED) {1960 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,1961 "6144 Outstanding NVME I/O Abort Request "1962 "still pending on nvme_fcreq x%px, "1963 "lpfc_ncmd x%px xri x%x\n",1964 pnvme_fcreq, lpfc_nbuf,1965 nvmereq_wqe->sli4_xritag);1966 goto out_unlock;1967 }1968 1969 ret_val = lpfc_sli4_issue_abort_iotag(phba, nvmereq_wqe,1970 lpfc_nvme_abort_fcreq_cmpl);1971 1972 spin_unlock(&phba->hbalock);1973 spin_unlock_irqrestore(&lpfc_nbuf->buf_lock, flags);1974 1975 /* Make sure HBA is alive */1976 lpfc_issue_hb_tmo(phba);1977 1978 if (ret_val != WQE_SUCCESS) {1979 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,1980 "6137 Failed abts issue_wqe with status x%x "1981 "for nvme_fcreq x%px.\n",1982 ret_val, pnvme_fcreq);1983 return;1984 }1985 1986 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_ABTS,1987 "6138 Transport Abort NVME Request Issued for "1988 "ox_id x%x\n",1989 nvmereq_wqe->sli4_xritag);1990 return;1991 1992out_unlock:1993 spin_unlock(&phba->hbalock);1994 spin_unlock_irqrestore(&lpfc_nbuf->buf_lock, flags);1995 return;1996}1997 1998/* Declare and initialization an instance of the FC NVME template. */1999static struct nvme_fc_port_template lpfc_nvme_template = {2000 /* initiator-based functions */2001 .localport_delete = lpfc_nvme_localport_delete,2002 .remoteport_delete = lpfc_nvme_remoteport_delete,2003 .create_queue = lpfc_nvme_create_queue,2004 .delete_queue = lpfc_nvme_delete_queue,2005 .ls_req = lpfc_nvme_ls_req,2006 .fcp_io = lpfc_nvme_fcp_io_submit,2007 .ls_abort = lpfc_nvme_ls_abort,2008 .fcp_abort = lpfc_nvme_fcp_abort,2009 .xmt_ls_rsp = lpfc_nvme_xmt_ls_rsp,2010 2011 .max_hw_queues = 1,2012 .max_sgl_segments = LPFC_NVME_DEFAULT_SEGS,2013 .max_dif_sgl_segments = LPFC_NVME_DEFAULT_SEGS,2014 .dma_boundary = 0xFFFFFFFF,2015 2016 /* Sizes of additional private data for data structures.2017 * No use for the last two sizes at this time.2018 */2019 .local_priv_sz = sizeof(struct lpfc_nvme_lport),2020 .remote_priv_sz = sizeof(struct lpfc_nvme_rport),2021 .lsrqst_priv_sz = 0,2022 .fcprqst_priv_sz = sizeof(struct lpfc_nvme_fcpreq_priv),2023};2024 2025/*2026 * lpfc_get_nvme_buf - Get a nvme buffer from io_buf_list of the HBA2027 *2028 * This routine removes a nvme buffer from head of @hdwq io_buf_list2029 * and returns to caller.2030 *2031 * Return codes:2032 * NULL - Error2033 * Pointer to lpfc_nvme_buf - Success2034 **/2035static struct lpfc_io_buf *2036lpfc_get_nvme_buf(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp,2037 int idx, int expedite)2038{2039 struct lpfc_io_buf *lpfc_ncmd;2040 struct lpfc_sli4_hdw_queue *qp;2041 struct sli4_sge *sgl;2042 struct lpfc_iocbq *pwqeq;2043 union lpfc_wqe128 *wqe;2044 2045 lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite);2046 2047 if (lpfc_ncmd) {2048 pwqeq = &(lpfc_ncmd->cur_iocbq);2049 wqe = &pwqeq->wqe;2050 2051 /* Setup key fields in buffer that may have been changed2052 * if other protocols used this buffer.2053 */2054 pwqeq->cmd_flag = LPFC_IO_NVME;2055 pwqeq->cmd_cmpl = lpfc_nvme_io_cmd_cmpl;2056 lpfc_ncmd->start_time = jiffies;2057 lpfc_ncmd->flags = 0;2058 2059 /* Rsp SGE will be filled in when we rcv an IO2060 * from the NVME Layer to be sent.2061 * The cmd is going to be embedded so we need a SKIP SGE.2062 */2063 sgl = lpfc_ncmd->dma_sgl;2064 bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);2065 bf_set(lpfc_sli4_sge_last, sgl, 0);2066 sgl->word2 = cpu_to_le32(sgl->word2);2067 /* Fill in word 3 / sgl_len during cmd submission */2068 2069 /* Initialize 64 bytes only */2070 memset(wqe, 0, sizeof(union lpfc_wqe));2071 2072 if (lpfc_ndlp_check_qdepth(phba, ndlp)) {2073 atomic_inc(&ndlp->cmd_pending);2074 lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH;2075 }2076 2077 } else {2078 qp = &phba->sli4_hba.hdwq[idx];2079 qp->empty_io_bufs++;2080 }2081 2082 return lpfc_ncmd;2083}2084 2085/**2086 * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list.2087 * @phba: The Hba for which this call is being executed.2088 * @lpfc_ncmd: The nvme buffer which is being released.2089 *2090 * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba2091 * lpfc_io_buf_list list. For SLI4 XRI's are tied to the nvme buffer2092 * and cannot be reused for at least RA_TOV amount of time if it was2093 * aborted.2094 **/2095static void2096lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd)2097{2098 struct lpfc_sli4_hdw_queue *qp;2099 unsigned long iflag = 0;2100 2101 if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp)2102 atomic_dec(&lpfc_ncmd->ndlp->cmd_pending);2103 2104 lpfc_ncmd->ndlp = NULL;2105 lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH;2106 2107 qp = lpfc_ncmd->hdwq;2108 if (unlikely(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {2109 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,2110 "6310 XB release deferred for "2111 "ox_id x%x on reqtag x%x\n",2112 lpfc_ncmd->cur_iocbq.sli4_xritag,2113 lpfc_ncmd->cur_iocbq.iotag);2114 2115 spin_lock_irqsave(&qp->abts_io_buf_list_lock, iflag);2116 list_add_tail(&lpfc_ncmd->list,2117 &qp->lpfc_abts_io_buf_list);2118 qp->abts_nvme_io_bufs++;2119 spin_unlock_irqrestore(&qp->abts_io_buf_list_lock, iflag);2120 } else2121 lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp);2122}2123 2124/**2125 * lpfc_nvme_create_localport - Create/Bind an nvme localport instance.2126 * @vport: the lpfc_vport instance requesting a localport.2127 *2128 * This routine is invoked to create an nvme localport instance to bind2129 * to the nvme_fc_transport. It is called once during driver load2130 * like lpfc_create_shost after all other services are initialized.2131 * It requires a vport, vpi, and wwns at call time. Other localport2132 * parameters are modified as the driver's FCID and the Fabric WWN2133 * are established.2134 *2135 * Return codes2136 * 0 - successful2137 * -ENOMEM - no heap memory available2138 * other values - from nvme registration upcall2139 **/2140int2141lpfc_nvme_create_localport(struct lpfc_vport *vport)2142{2143 int ret = 0;2144 struct lpfc_hba *phba = vport->phba;2145 struct nvme_fc_port_info nfcp_info;2146 struct nvme_fc_local_port *localport;2147 struct lpfc_nvme_lport *lport;2148 2149 /* Initialize this localport instance. The vport wwn usage ensures2150 * that NPIV is accounted for.2151 */2152 memset(&nfcp_info, 0, sizeof(struct nvme_fc_port_info));2153 nfcp_info.port_role = FC_PORT_ROLE_NVME_INITIATOR;2154 nfcp_info.node_name = wwn_to_u64(vport->fc_nodename.u.wwn);2155 nfcp_info.port_name = wwn_to_u64(vport->fc_portname.u.wwn);2156 2157 /* We need to tell the transport layer + 1 because it takes page2158 * alignment into account. When space for the SGL is allocated we2159 * allocate + 3, one for cmd, one for rsp and one for this alignment2160 */2161 lpfc_nvme_template.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;2162 2163 /* Advertise how many hw queues we support based on cfg_hdw_queue,2164 * which will not exceed cpu count.2165 */2166 lpfc_nvme_template.max_hw_queues = phba->cfg_hdw_queue;2167 2168 if (!IS_ENABLED(CONFIG_NVME_FC))2169 return ret;2170 2171 /* localport is allocated from the stack, but the registration2172 * call allocates heap memory as well as the private area.2173 */2174 2175 ret = nvme_fc_register_localport(&nfcp_info, &lpfc_nvme_template,2176 &vport->phba->pcidev->dev, &localport);2177 if (!ret) {2178 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME | LOG_NVME_DISC,2179 "6005 Successfully registered local "2180 "NVME port num %d, localP x%px, private "2181 "x%px, sg_seg %d\n",2182 localport->port_num, localport,2183 localport->private,2184 lpfc_nvme_template.max_sgl_segments);2185 2186 /* Private is our lport size declared in the template. */2187 lport = (struct lpfc_nvme_lport *)localport->private;2188 vport->localport = localport;2189 lport->vport = vport;2190 vport->nvmei_support = 1;2191 2192 atomic_set(&lport->xmt_fcp_noxri, 0);2193 atomic_set(&lport->xmt_fcp_bad_ndlp, 0);2194 atomic_set(&lport->xmt_fcp_qdepth, 0);2195 atomic_set(&lport->xmt_fcp_err, 0);2196 atomic_set(&lport->xmt_fcp_wqerr, 0);2197 atomic_set(&lport->xmt_fcp_abort, 0);2198 atomic_set(&lport->xmt_ls_abort, 0);2199 atomic_set(&lport->xmt_ls_err, 0);2200 atomic_set(&lport->cmpl_fcp_xb, 0);2201 atomic_set(&lport->cmpl_fcp_err, 0);2202 atomic_set(&lport->cmpl_ls_xb, 0);2203 atomic_set(&lport->cmpl_ls_err, 0);2204 2205 atomic_set(&lport->fc4NvmeLsRequests, 0);2206 atomic_set(&lport->fc4NvmeLsCmpls, 0);2207 }2208 2209 return ret;2210}2211 2212#if (IS_ENABLED(CONFIG_NVME_FC))2213/* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg.2214 *2215 * The driver has to wait for the host nvme transport to callback2216 * indicating the localport has successfully unregistered all2217 * resources. Since this is an uninterruptible wait, loop every ten2218 * seconds and print a message indicating no progress.2219 *2220 * An uninterruptible wait is used because of the risk of transport-to-2221 * driver state mismatch.2222 */2223static void2224lpfc_nvme_lport_unreg_wait(struct lpfc_vport *vport,2225 struct lpfc_nvme_lport *lport,2226 struct completion *lport_unreg_cmp)2227{2228 u32 wait_tmo;2229 int ret, i, pending = 0;2230 struct lpfc_sli_ring *pring;2231 struct lpfc_hba *phba = vport->phba;2232 struct lpfc_sli4_hdw_queue *qp;2233 int abts_scsi, abts_nvme;2234 2235 /* Host transport has to clean up and confirm requiring an indefinite2236 * wait. Print a message if a 10 second wait expires and renew the2237 * wait. This is unexpected.2238 */2239 wait_tmo = msecs_to_jiffies(LPFC_NVME_WAIT_TMO * 1000);2240 while (true) {2241 ret = wait_for_completion_timeout(lport_unreg_cmp, wait_tmo);2242 if (unlikely(!ret)) {2243 pending = 0;2244 abts_scsi = 0;2245 abts_nvme = 0;2246 for (i = 0; i < phba->cfg_hdw_queue; i++) {2247 qp = &phba->sli4_hba.hdwq[i];2248 if (!vport->localport || !qp || !qp->io_wq)2249 return;2250 2251 pring = qp->io_wq->pring;2252 if (!pring)2253 continue;2254 pending += pring->txcmplq_cnt;2255 abts_scsi += qp->abts_scsi_io_bufs;2256 abts_nvme += qp->abts_nvme_io_bufs;2257 }2258 if (!vport->localport ||2259 test_bit(HBA_PCI_ERR, &vport->phba->bit_flags) ||2260 phba->link_state == LPFC_HBA_ERROR ||2261 test_bit(FC_UNLOADING, &vport->load_flag))2262 return;2263 2264 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,2265 "6176 Lport x%px Localport x%px wait "2266 "timed out. Pending %d [%d:%d]. "2267 "Renewing.\n",2268 lport, vport->localport, pending,2269 abts_scsi, abts_nvme);2270 continue;2271 }2272 break;2273 }2274 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_IOERR,2275 "6177 Lport x%px Localport x%px Complete Success\n",2276 lport, vport->localport);2277}2278#endif2279 2280/**2281 * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport.2282 * @vport: pointer to a host virtual N_Port data structure2283 *2284 * This routine is invoked to destroy all lports bound to the phba.2285 * The lport memory was allocated by the nvme fc transport and is2286 * released there. This routine ensures all rports bound to the2287 * lport have been disconnected.2288 *2289 **/2290void2291lpfc_nvme_destroy_localport(struct lpfc_vport *vport)2292{2293#if (IS_ENABLED(CONFIG_NVME_FC))2294 struct nvme_fc_local_port *localport;2295 struct lpfc_nvme_lport *lport;2296 int ret;2297 DECLARE_COMPLETION_ONSTACK(lport_unreg_cmp);2298 2299 if (vport->nvmei_support == 0)2300 return;2301 2302 localport = vport->localport;2303 if (!localport)2304 return;2305 lport = (struct lpfc_nvme_lport *)localport->private;2306 2307 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,2308 "6011 Destroying NVME localport x%px\n",2309 localport);2310 2311 /* lport's rport list is clear. Unregister2312 * lport and release resources.2313 */2314 lport->lport_unreg_cmp = &lport_unreg_cmp;2315 ret = nvme_fc_unregister_localport(localport);2316 2317 /* Wait for completion. This either blocks2318 * indefinitely or succeeds2319 */2320 lpfc_nvme_lport_unreg_wait(vport, lport, &lport_unreg_cmp);2321 vport->localport = NULL;2322 2323 /* Regardless of the unregister upcall response, clear2324 * nvmei_support. All rports are unregistered and the2325 * driver will clean up.2326 */2327 vport->nvmei_support = 0;2328 if (ret == 0) {2329 lpfc_printf_vlog(vport,2330 KERN_INFO, LOG_NVME_DISC,2331 "6009 Unregistered lport Success\n");2332 } else {2333 lpfc_printf_vlog(vport,2334 KERN_INFO, LOG_NVME_DISC,2335 "6010 Unregistered lport "2336 "Failed, status x%x\n",2337 ret);2338 }2339#endif2340}2341 2342void2343lpfc_nvme_update_localport(struct lpfc_vport *vport)2344{2345#if (IS_ENABLED(CONFIG_NVME_FC))2346 struct nvme_fc_local_port *localport;2347 struct lpfc_nvme_lport *lport;2348 2349 localport = vport->localport;2350 if (!localport) {2351 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,2352 "6710 Update NVME fail. No localport\n");2353 return;2354 }2355 lport = (struct lpfc_nvme_lport *)localport->private;2356 if (!lport) {2357 lpfc_printf_vlog(vport, KERN_WARNING, LOG_NVME,2358 "6171 Update NVME fail. localP x%px, No lport\n",2359 localport);2360 return;2361 }2362 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,2363 "6012 Update NVME lport x%px did x%x\n",2364 localport, vport->fc_myDID);2365 2366 localport->port_id = vport->fc_myDID;2367 if (localport->port_id == 0)2368 localport->port_role = FC_PORT_ROLE_NVME_DISCOVERY;2369 else2370 localport->port_role = FC_PORT_ROLE_NVME_INITIATOR;2371 2372 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,2373 "6030 bound lport x%px to DID x%06x\n",2374 lport, localport->port_id);2375#endif2376}2377 2378int2379lpfc_nvme_register_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)2380{2381#if (IS_ENABLED(CONFIG_NVME_FC))2382 int ret = 0;2383 struct nvme_fc_local_port *localport;2384 struct lpfc_nvme_lport *lport;2385 struct lpfc_nvme_rport *rport;2386 struct lpfc_nvme_rport *oldrport;2387 struct nvme_fc_remote_port *remote_port;2388 struct nvme_fc_port_info rpinfo;2389 struct lpfc_nodelist *prev_ndlp = NULL;2390 struct fc_rport *srport = ndlp->rport;2391 2392 lpfc_printf_vlog(ndlp->vport, KERN_INFO, LOG_NVME_DISC,2393 "6006 Register NVME PORT. DID x%06x nlptype x%x\n",2394 ndlp->nlp_DID, ndlp->nlp_type);2395 2396 localport = vport->localport;2397 if (!localport)2398 return 0;2399 2400 lport = (struct lpfc_nvme_lport *)localport->private;2401 2402 /* NVME rports are not preserved across devloss.2403 * Just register this instance. Note, rpinfo->dev_loss_tmo2404 * is left 0 to indicate accept transport defaults. The2405 * driver communicates port role capabilities consistent2406 * with the PRLI response data.2407 */2408 memset(&rpinfo, 0, sizeof(struct nvme_fc_port_info));2409 rpinfo.port_id = ndlp->nlp_DID;2410 if (ndlp->nlp_type & NLP_NVME_TARGET)2411 rpinfo.port_role |= FC_PORT_ROLE_NVME_TARGET;2412 if (ndlp->nlp_type & NLP_NVME_INITIATOR)2413 rpinfo.port_role |= FC_PORT_ROLE_NVME_INITIATOR;2414 2415 if (ndlp->nlp_type & NLP_NVME_DISCOVERY)2416 rpinfo.port_role |= FC_PORT_ROLE_NVME_DISCOVERY;2417 2418 rpinfo.port_name = wwn_to_u64(ndlp->nlp_portname.u.wwn);2419 rpinfo.node_name = wwn_to_u64(ndlp->nlp_nodename.u.wwn);2420 if (srport)2421 rpinfo.dev_loss_tmo = srport->dev_loss_tmo;2422 else2423 rpinfo.dev_loss_tmo = vport->cfg_devloss_tmo;2424 2425 spin_lock_irq(&ndlp->lock);2426 2427 /* If an oldrport exists, so does the ndlp reference. If not2428 * a new reference is needed because either the node has never2429 * been registered or it's been unregistered and getting deleted.2430 */2431 oldrport = lpfc_ndlp_get_nrport(ndlp);2432 if (oldrport) {2433 prev_ndlp = oldrport->ndlp;2434 spin_unlock_irq(&ndlp->lock);2435 } else {2436 spin_unlock_irq(&ndlp->lock);2437 if (!lpfc_nlp_get(ndlp)) {2438 dev_warn(&vport->phba->pcidev->dev,2439 "Warning - No node ref - exit register\n");2440 return 0;2441 }2442 }2443 2444 ret = nvme_fc_register_remoteport(localport, &rpinfo, &remote_port);2445 if (!ret) {2446 /* If the ndlp already has an nrport, this is just2447 * a resume of the existing rport. Else this is a2448 * new rport.2449 */2450 /* Guard against an unregister/reregister2451 * race that leaves the WAIT flag set.2452 */2453 spin_lock_irq(&ndlp->lock);2454 ndlp->fc4_xpt_flags &= ~NVME_XPT_UNREG_WAIT;2455 ndlp->fc4_xpt_flags |= NVME_XPT_REGD;2456 spin_unlock_irq(&ndlp->lock);2457 rport = remote_port->private;2458 if (oldrport) {2459 2460 /* Sever the ndlp<->rport association2461 * before dropping the ndlp ref from2462 * register.2463 */2464 spin_lock_irq(&ndlp->lock);2465 ndlp->nrport = NULL;2466 ndlp->fc4_xpt_flags &= ~NVME_XPT_UNREG_WAIT;2467 spin_unlock_irq(&ndlp->lock);2468 rport->ndlp = NULL;2469 rport->remoteport = NULL;2470 2471 /* Reference only removed if previous NDLP is no longer2472 * active. It might be just a swap and removing the2473 * reference would cause a premature cleanup.2474 */2475 if (prev_ndlp && prev_ndlp != ndlp) {2476 if (!prev_ndlp->nrport)2477 lpfc_nlp_put(prev_ndlp);2478 }2479 }2480 2481 /* Clean bind the rport to the ndlp. */2482 rport->remoteport = remote_port;2483 rport->lport = lport;2484 rport->ndlp = ndlp;2485 spin_lock_irq(&ndlp->lock);2486 ndlp->nrport = rport;2487 spin_unlock_irq(&ndlp->lock);2488 lpfc_printf_vlog(vport, KERN_INFO,2489 LOG_NVME_DISC | LOG_NODE,2490 "6022 Bind lport x%px to remoteport x%px "2491 "rport x%px WWNN 0x%llx, "2492 "Rport WWPN 0x%llx DID "2493 "x%06x Role x%x, ndlp %p prev_ndlp x%px\n",2494 lport, remote_port, rport,2495 rpinfo.node_name, rpinfo.port_name,2496 rpinfo.port_id, rpinfo.port_role,2497 ndlp, prev_ndlp);2498 } else {2499 lpfc_printf_vlog(vport, KERN_ERR,2500 LOG_TRACE_EVENT,2501 "6031 RemotePort Registration failed "2502 "err: %d, DID x%06x ref %u\n",2503 ret, ndlp->nlp_DID, kref_read(&ndlp->kref));2504 lpfc_nlp_put(ndlp);2505 }2506 2507 return ret;2508#else2509 return 0;2510#endif2511}2512 2513/*2514 * lpfc_nvme_rescan_port - Check to see if we should rescan this remoteport2515 *2516 * If the ndlp represents an NVME Target, that we are logged into,2517 * ping the NVME FC Transport layer to initiate a device rescan2518 * on this remote NPort.2519 */2520void2521lpfc_nvme_rescan_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)2522{2523#if (IS_ENABLED(CONFIG_NVME_FC))2524 struct lpfc_nvme_rport *nrport;2525 struct nvme_fc_remote_port *remoteport = NULL;2526 2527 spin_lock_irq(&ndlp->lock);2528 nrport = lpfc_ndlp_get_nrport(ndlp);2529 if (nrport)2530 remoteport = nrport->remoteport;2531 spin_unlock_irq(&ndlp->lock);2532 2533 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,2534 "6170 Rescan NPort DID x%06x type x%x "2535 "state x%x nrport x%px remoteport x%px\n",2536 ndlp->nlp_DID, ndlp->nlp_type, ndlp->nlp_state,2537 nrport, remoteport);2538 2539 if (!nrport || !remoteport)2540 goto rescan_exit;2541 2542 /* Rescan an NVME target in MAPPED state with DISCOVERY role set */2543 if (remoteport->port_role & FC_PORT_ROLE_NVME_DISCOVERY &&2544 ndlp->nlp_state == NLP_STE_MAPPED_NODE) {2545 nvme_fc_rescan_remoteport(remoteport);2546 2547 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,2548 "6172 NVME rescanned DID x%06x "2549 "port_state x%x\n",2550 ndlp->nlp_DID, remoteport->port_state);2551 }2552 return;2553 rescan_exit:2554 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,2555 "6169 Skip NVME Rport Rescan, NVME remoteport "2556 "unregistered\n");2557#endif2558}2559 2560/* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport.2561 *2562 * There is no notion of Devloss or rport recovery from the current2563 * nvme_transport perspective. Loss of an rport just means IO cannot2564 * be sent and recovery is completely up to the initator.2565 * For now, the driver just unbinds the DID and port_role so that2566 * no further IO can be issued.2567 */2568void2569lpfc_nvme_unregister_port(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp)2570{2571#if (IS_ENABLED(CONFIG_NVME_FC))2572 int ret;2573 struct nvme_fc_local_port *localport;2574 struct lpfc_nvme_lport *lport;2575 struct lpfc_nvme_rport *rport;2576 struct nvme_fc_remote_port *remoteport = NULL;2577 2578 localport = vport->localport;2579 2580 /* This is fundamental error. The localport is always2581 * available until driver unload. Just exit.2582 */2583 if (!localport)2584 return;2585 2586 lport = (struct lpfc_nvme_lport *)localport->private;2587 if (!lport)2588 goto input_err;2589 2590 spin_lock_irq(&ndlp->lock);2591 rport = lpfc_ndlp_get_nrport(ndlp);2592 if (rport)2593 remoteport = rport->remoteport;2594 spin_unlock_irq(&ndlp->lock);2595 if (!remoteport)2596 goto input_err;2597 2598 lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME_DISC,2599 "6033 Unreg nvme remoteport x%px, portname x%llx, "2600 "port_id x%06x, portstate x%x port type x%x "2601 "refcnt %d\n",2602 remoteport, remoteport->port_name,2603 remoteport->port_id, remoteport->port_state,2604 ndlp->nlp_type, kref_read(&ndlp->kref));2605 2606 /* Sanity check ndlp type. Only call for NVME ports. Don't2607 * clear any rport state until the transport calls back.2608 */2609 2610 if (ndlp->nlp_type & NLP_NVME_TARGET) {2611 /* No concern about the role change on the nvme remoteport.2612 * The transport will update it.2613 */2614 spin_lock_irq(&ndlp->lock);2615 ndlp->fc4_xpt_flags |= NVME_XPT_UNREG_WAIT;2616 spin_unlock_irq(&ndlp->lock);2617 2618 /* Don't let the host nvme transport keep sending keep-alives2619 * on this remoteport. Vport is unloading, no recovery. The2620 * return values is ignored. The upcall is a courtesy to the2621 * transport.2622 */2623 if (test_bit(FC_UNLOADING, &vport->load_flag) ||2624 unlikely(vport->phba->link_state == LPFC_HBA_ERROR))2625 (void)nvme_fc_set_remoteport_devloss(remoteport, 0);2626 2627 ret = nvme_fc_unregister_remoteport(remoteport);2628 2629 /* The driver no longer knows if the nrport memory is valid.2630 * because the controller teardown process has begun and2631 * is asynchronous. Break the binding in the ndlp. Also2632 * remove the register ndlp reference to setup node release.2633 */2634 ndlp->nrport = NULL;2635 lpfc_nlp_put(ndlp);2636 if (ret != 0) {2637 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,2638 "6167 NVME unregister failed %d "2639 "port_state x%x\n",2640 ret, remoteport->port_state);2641 2642 if (test_bit(FC_UNLOADING, &vport->load_flag)) {2643 /* Only 1 thread can drop the initial node2644 * reference. Check if another thread has set2645 * NLP_DROPPED.2646 */2647 spin_lock_irq(&ndlp->lock);2648 if (!(ndlp->nlp_flag & NLP_DROPPED)) {2649 ndlp->nlp_flag |= NLP_DROPPED;2650 spin_unlock_irq(&ndlp->lock);2651 lpfc_nlp_put(ndlp);2652 return;2653 }2654 spin_unlock_irq(&ndlp->lock);2655 }2656 }2657 }2658 return;2659 2660 input_err:2661#endif2662 lpfc_printf_vlog(vport, KERN_ERR, LOG_TRACE_EVENT,2663 "6168 State error: lport x%px, rport x%px FCID x%06x\n",2664 vport->localport, ndlp->rport, ndlp->nlp_DID);2665}2666 2667/**2668 * lpfc_sli4_nvme_pci_offline_aborted - Fast-path process of NVME xri abort2669 * @phba: pointer to lpfc hba data structure.2670 * @lpfc_ncmd: The nvme job structure for the request being aborted.2671 *2672 * This routine is invoked by the worker thread to process a SLI4 fast-path2673 * NVME aborted xri. Aborted NVME IO commands are completed to the transport2674 * here.2675 **/2676void2677lpfc_sli4_nvme_pci_offline_aborted(struct lpfc_hba *phba,2678 struct lpfc_io_buf *lpfc_ncmd)2679{2680 struct nvmefc_fcp_req *nvme_cmd = NULL;2681 2682 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,2683 "6533 %s nvme_cmd %p tag x%x abort complete and "2684 "xri released\n", __func__,2685 lpfc_ncmd->nvmeCmd,2686 lpfc_ncmd->cur_iocbq.iotag);2687 2688 /* Aborted NVME commands are required to not complete2689 * before the abort exchange command fully completes.2690 * Once completed, it is available via the put list.2691 */2692 if (lpfc_ncmd->nvmeCmd) {2693 nvme_cmd = lpfc_ncmd->nvmeCmd;2694 nvme_cmd->transferred_length = 0;2695 nvme_cmd->rcv_rsplen = 0;2696 nvme_cmd->status = NVME_SC_INTERNAL;2697 nvme_cmd->done(nvme_cmd);2698 lpfc_ncmd->nvmeCmd = NULL;2699 }2700 lpfc_release_nvme_buf(phba, lpfc_ncmd);2701}2702 2703/**2704 * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort2705 * @phba: pointer to lpfc hba data structure.2706 * @axri: pointer to the fcp xri abort wcqe structure.2707 * @lpfc_ncmd: The nvme job structure for the request being aborted.2708 *2709 * This routine is invoked by the worker thread to process a SLI4 fast-path2710 * NVME aborted xri. Aborted NVME IO commands are completed to the transport2711 * here.2712 **/2713void2714lpfc_sli4_nvme_xri_aborted(struct lpfc_hba *phba,2715 struct sli4_wcqe_xri_aborted *axri,2716 struct lpfc_io_buf *lpfc_ncmd)2717{2718 uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri);2719 struct nvmefc_fcp_req *nvme_cmd = NULL;2720 struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp;2721 2722 2723 if (ndlp)2724 lpfc_sli4_abts_err_handler(phba, ndlp, axri);2725 2726 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,2727 "6311 nvme_cmd %p xri x%x tag x%x abort complete and "2728 "xri released\n",2729 lpfc_ncmd->nvmeCmd, xri,2730 lpfc_ncmd->cur_iocbq.iotag);2731 2732 /* Aborted NVME commands are required to not complete2733 * before the abort exchange command fully completes.2734 * Once completed, it is available via the put list.2735 */2736 if (lpfc_ncmd->nvmeCmd) {2737 nvme_cmd = lpfc_ncmd->nvmeCmd;2738 nvme_cmd->done(nvme_cmd);2739 lpfc_ncmd->nvmeCmd = NULL;2740 }2741 lpfc_release_nvme_buf(phba, lpfc_ncmd);2742}2743 2744/**2745 * lpfc_nvme_wait_for_io_drain - Wait for all NVME wqes to complete2746 * @phba: Pointer to HBA context object.2747 *2748 * This function flushes all wqes in the nvme rings and frees all resources2749 * in the txcmplq. This function does not issue abort wqes for the IO2750 * commands in txcmplq, they will just be returned with2751 * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI2752 * slot has been permanently disabled.2753 **/2754void2755lpfc_nvme_wait_for_io_drain(struct lpfc_hba *phba)2756{2757 struct lpfc_sli_ring *pring;2758 u32 i, wait_cnt = 0;2759 2760 if (phba->sli_rev < LPFC_SLI_REV4 || !phba->sli4_hba.hdwq)2761 return;2762 2763 /* Cycle through all IO rings and make sure all outstanding2764 * WQEs have been removed from the txcmplqs.2765 */2766 for (i = 0; i < phba->cfg_hdw_queue; i++) {2767 if (!phba->sli4_hba.hdwq[i].io_wq)2768 continue;2769 pring = phba->sli4_hba.hdwq[i].io_wq->pring;2770 2771 if (!pring)2772 continue;2773 2774 /* Retrieve everything on the txcmplq */2775 while (!list_empty(&pring->txcmplq)) {2776 msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1);2777 wait_cnt++;2778 2779 /* The sleep is 10mS. Every ten seconds,2780 * dump a message. Something is wrong.2781 */2782 if ((wait_cnt % 1000) == 0) {2783 lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,2784 "6178 NVME IO not empty, "2785 "cnt %d\n", wait_cnt);2786 }2787 }2788 }2789 2790 /* Make sure HBA is alive */2791 lpfc_issue_hb_tmo(phba);2792 2793}2794 2795void2796lpfc_nvme_cancel_iocb(struct lpfc_hba *phba, struct lpfc_iocbq *pwqeIn,2797 uint32_t stat, uint32_t param)2798{2799#if (IS_ENABLED(CONFIG_NVME_FC))2800 struct lpfc_io_buf *lpfc_ncmd;2801 struct nvmefc_fcp_req *nCmd;2802 struct lpfc_wcqe_complete wcqe;2803 struct lpfc_wcqe_complete *wcqep = &wcqe;2804 2805 lpfc_ncmd = pwqeIn->io_buf;2806 if (!lpfc_ncmd) {2807 lpfc_sli_release_iocbq(phba, pwqeIn);2808 return;2809 }2810 /* For abort iocb just return, IO iocb will do a done call */2811 if (bf_get(wqe_cmnd, &pwqeIn->wqe.gen_req.wqe_com) ==2812 CMD_ABORT_XRI_CX) {2813 lpfc_sli_release_iocbq(phba, pwqeIn);2814 return;2815 }2816 2817 spin_lock(&lpfc_ncmd->buf_lock);2818 nCmd = lpfc_ncmd->nvmeCmd;2819 if (!nCmd) {2820 spin_unlock(&lpfc_ncmd->buf_lock);2821 lpfc_release_nvme_buf(phba, lpfc_ncmd);2822 return;2823 }2824 spin_unlock(&lpfc_ncmd->buf_lock);2825 2826 lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,2827 "6194 NVME Cancel xri %x\n",2828 lpfc_ncmd->cur_iocbq.sli4_xritag);2829 2830 wcqep->word0 = 0;2831 bf_set(lpfc_wcqe_c_status, wcqep, stat);2832 wcqep->parameter = param;2833 wcqep->total_data_placed = 0;2834 wcqep->word3 = 0; /* xb is 0 */2835 2836 /* Call release with XB=1 to queue the IO into the abort list. */2837 if (phba->sli.sli_flag & LPFC_SLI_ACTIVE)2838 bf_set(lpfc_wcqe_c_xb, wcqep, 1);2839 2840 memcpy(&pwqeIn->wcqe_cmpl, wcqep, sizeof(*wcqep));2841 (pwqeIn->cmd_cmpl)(phba, pwqeIn, pwqeIn);2842#endif2843}2844