3581 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright (C) 2021 Broadcom. All Rights Reserved. The term4 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.5 */6 7#include "efct_driver.h"8#include "efct_hw.h"9#include "efct_unsol.h"10 11struct efct_hw_link_stat_cb_arg {12 void (*cb)(int status, u32 num_counters,13 struct efct_hw_link_stat_counts *counters, void *arg);14 void *arg;15};16 17struct efct_hw_host_stat_cb_arg {18 void (*cb)(int status, u32 num_counters,19 struct efct_hw_host_stat_counts *counters, void *arg);20 void *arg;21};22 23struct efct_hw_fw_wr_cb_arg {24 void (*cb)(int status, u32 bytes_written, u32 change_status, void *arg);25 void *arg;26};27 28struct efct_mbox_rqst_ctx {29 int (*callback)(struct efc *efc, int status, u8 *mqe, void *arg);30 void *arg;31};32 33static int34efct_hw_link_event_init(struct efct_hw *hw)35{36 hw->link.status = SLI4_LINK_STATUS_MAX;37 hw->link.topology = SLI4_LINK_TOPO_NONE;38 hw->link.medium = SLI4_LINK_MEDIUM_MAX;39 hw->link.speed = 0;40 hw->link.loop_map = NULL;41 hw->link.fc_id = U32_MAX;42 43 return 0;44}45 46static int47efct_hw_read_max_dump_size(struct efct_hw *hw)48{49 u8 buf[SLI4_BMBX_SIZE];50 struct efct *efct = hw->os;51 int rc = 0;52 struct sli4_rsp_cmn_set_dump_location *rsp;53 54 /* attempt to detemine the dump size for function 0 only. */55 if (PCI_FUNC(efct->pci->devfn) != 0)56 return rc;57 58 if (sli_cmd_common_set_dump_location(&hw->sli, buf, 1, 0, NULL, 0))59 return -EIO;60 61 rsp = (struct sli4_rsp_cmn_set_dump_location *)62 (buf + offsetof(struct sli4_cmd_sli_config, payload.embed));63 64 rc = efct_hw_command(hw, buf, EFCT_CMD_POLL, NULL, NULL);65 if (rc != 0) {66 efc_log_debug(hw->os, "set dump location cmd failed\n");67 return rc;68 }69 70 hw->dump_size =71 le32_to_cpu(rsp->buffer_length_dword) & SLI4_CMN_SET_DUMP_BUFFER_LEN;72 73 efc_log_debug(hw->os, "Dump size %x\n", hw->dump_size);74 75 return rc;76}77 78static int79__efct_read_topology_cb(struct efct_hw *hw, int status, u8 *mqe, void *arg)80{81 struct sli4_cmd_read_topology *read_topo =82 (struct sli4_cmd_read_topology *)mqe;83 u8 speed;84 struct efc_domain_record drec = {0};85 struct efct *efct = hw->os;86 87 if (status || le16_to_cpu(read_topo->hdr.status)) {88 efc_log_debug(hw->os, "bad status cqe=%#x mqe=%#x\n", status,89 le16_to_cpu(read_topo->hdr.status));90 return -EIO;91 }92 93 switch (le32_to_cpu(read_topo->dw2_attentype) &94 SLI4_READTOPO_ATTEN_TYPE) {95 case SLI4_READ_TOPOLOGY_LINK_UP:96 hw->link.status = SLI4_LINK_STATUS_UP;97 break;98 case SLI4_READ_TOPOLOGY_LINK_DOWN:99 hw->link.status = SLI4_LINK_STATUS_DOWN;100 break;101 case SLI4_READ_TOPOLOGY_LINK_NO_ALPA:102 hw->link.status = SLI4_LINK_STATUS_NO_ALPA;103 break;104 default:105 hw->link.status = SLI4_LINK_STATUS_MAX;106 break;107 }108 109 switch (read_topo->topology) {110 case SLI4_READ_TOPO_NON_FC_AL:111 hw->link.topology = SLI4_LINK_TOPO_NON_FC_AL;112 break;113 case SLI4_READ_TOPO_FC_AL:114 hw->link.topology = SLI4_LINK_TOPO_FC_AL;115 if (hw->link.status == SLI4_LINK_STATUS_UP)116 hw->link.loop_map = hw->loop_map.virt;117 hw->link.fc_id = read_topo->acquired_al_pa;118 break;119 default:120 hw->link.topology = SLI4_LINK_TOPO_MAX;121 break;122 }123 124 hw->link.medium = SLI4_LINK_MEDIUM_FC;125 126 speed = (le32_to_cpu(read_topo->currlink_state) &127 SLI4_READTOPO_LINKSTATE_SPEED) >> 8;128 switch (speed) {129 case SLI4_READ_TOPOLOGY_SPEED_1G:130 hw->link.speed = 1 * 1000;131 break;132 case SLI4_READ_TOPOLOGY_SPEED_2G:133 hw->link.speed = 2 * 1000;134 break;135 case SLI4_READ_TOPOLOGY_SPEED_4G:136 hw->link.speed = 4 * 1000;137 break;138 case SLI4_READ_TOPOLOGY_SPEED_8G:139 hw->link.speed = 8 * 1000;140 break;141 case SLI4_READ_TOPOLOGY_SPEED_16G:142 hw->link.speed = 16 * 1000;143 break;144 case SLI4_READ_TOPOLOGY_SPEED_32G:145 hw->link.speed = 32 * 1000;146 break;147 case SLI4_READ_TOPOLOGY_SPEED_64G:148 hw->link.speed = 64 * 1000;149 break;150 case SLI4_READ_TOPOLOGY_SPEED_128G:151 hw->link.speed = 128 * 1000;152 break;153 }154 155 drec.speed = hw->link.speed;156 drec.fc_id = hw->link.fc_id;157 drec.is_nport = true;158 efc_domain_cb(efct->efcport, EFC_HW_DOMAIN_FOUND, &drec);159 160 return 0;161}162 163static int164efct_hw_cb_link(void *ctx, void *e)165{166 struct efct_hw *hw = ctx;167 struct sli4_link_event *event = e;168 struct efc_domain *d = NULL;169 int rc = 0;170 struct efct *efct = hw->os;171 172 efct_hw_link_event_init(hw);173 174 switch (event->status) {175 case SLI4_LINK_STATUS_UP:176 177 hw->link = *event;178 efct->efcport->link_status = EFC_LINK_STATUS_UP;179 180 if (event->topology == SLI4_LINK_TOPO_NON_FC_AL) {181 struct efc_domain_record drec = {0};182 183 efc_log_info(hw->os, "Link Up, NPORT, speed is %d\n",184 event->speed);185 drec.speed = event->speed;186 drec.fc_id = event->fc_id;187 drec.is_nport = true;188 efc_domain_cb(efct->efcport, EFC_HW_DOMAIN_FOUND,189 &drec);190 } else if (event->topology == SLI4_LINK_TOPO_FC_AL) {191 u8 buf[SLI4_BMBX_SIZE];192 193 efc_log_info(hw->os, "Link Up, LOOP, speed is %d\n",194 event->speed);195 196 if (!sli_cmd_read_topology(&hw->sli, buf,197 &hw->loop_map)) {198 rc = efct_hw_command(hw, buf, EFCT_CMD_NOWAIT,199 __efct_read_topology_cb, NULL);200 }201 202 if (rc)203 efc_log_debug(hw->os, "READ_TOPOLOGY failed\n");204 } else {205 efc_log_info(hw->os, "%s(%#x), speed is %d\n",206 "Link Up, unsupported topology ",207 event->topology, event->speed);208 }209 break;210 case SLI4_LINK_STATUS_DOWN:211 efc_log_info(hw->os, "Link down\n");212 213 hw->link.status = event->status;214 efct->efcport->link_status = EFC_LINK_STATUS_DOWN;215 216 d = efct->efcport->domain;217 if (d)218 efc_domain_cb(efct->efcport, EFC_HW_DOMAIN_LOST, d);219 break;220 default:221 efc_log_debug(hw->os, "unhandled link status %#x\n",222 event->status);223 break;224 }225 226 return 0;227}228 229int230efct_hw_setup(struct efct_hw *hw, void *os, struct pci_dev *pdev)231{232 u32 i, max_sgl, cpus;233 234 if (hw->hw_setup_called)235 return 0;236 237 /*238 * efct_hw_init() relies on NULL pointers indicating that a structure239 * needs allocation. If a structure is non-NULL, efct_hw_init() won't240 * free/realloc that memory241 */242 memset(hw, 0, sizeof(struct efct_hw));243 244 hw->hw_setup_called = true;245 246 hw->os = os;247 248 mutex_init(&hw->bmbx_lock);249 spin_lock_init(&hw->cmd_lock);250 INIT_LIST_HEAD(&hw->cmd_head);251 INIT_LIST_HEAD(&hw->cmd_pending);252 hw->cmd_head_count = 0;253 254 /* Create mailbox command ctx pool */255 hw->cmd_ctx_pool = mempool_create_kmalloc_pool(EFCT_CMD_CTX_POOL_SZ,256 sizeof(struct efct_command_ctx));257 if (!hw->cmd_ctx_pool) {258 efc_log_err(hw->os, "failed to allocate mailbox buffer pool\n");259 return -EIO;260 }261 262 /* Create mailbox request ctx pool for library callback */263 hw->mbox_rqst_pool = mempool_create_kmalloc_pool(EFCT_CMD_CTX_POOL_SZ,264 sizeof(struct efct_mbox_rqst_ctx));265 if (!hw->mbox_rqst_pool) {266 efc_log_err(hw->os, "failed to allocate mbox request pool\n");267 return -EIO;268 }269 270 spin_lock_init(&hw->io_lock);271 INIT_LIST_HEAD(&hw->io_inuse);272 INIT_LIST_HEAD(&hw->io_free);273 INIT_LIST_HEAD(&hw->io_wait_free);274 275 atomic_set(&hw->io_alloc_failed_count, 0);276 277 hw->config.speed = SLI4_LINK_SPEED_AUTO_16_8_4;278 if (sli_setup(&hw->sli, hw->os, pdev, ((struct efct *)os)->reg)) {279 efc_log_err(hw->os, "SLI setup failed\n");280 return -EIO;281 }282 283 efct_hw_link_event_init(hw);284 285 sli_callback(&hw->sli, SLI4_CB_LINK, efct_hw_cb_link, hw);286 287 /*288 * Set all the queue sizes to the maximum allowed.289 */290 for (i = 0; i < ARRAY_SIZE(hw->num_qentries); i++)291 hw->num_qentries[i] = hw->sli.qinfo.max_qentries[i];292 /*293 * Adjust the size of the WQs so that the CQ is twice as big as294 * the WQ to allow for 2 completions per IO. This allows us to295 * handle multi-phase as well as aborts.296 */297 hw->num_qentries[SLI4_QTYPE_WQ] = hw->num_qentries[SLI4_QTYPE_CQ] / 2;298 299 /*300 * The RQ assignment for RQ pair mode.301 */302 303 hw->config.rq_default_buffer_size = EFCT_HW_RQ_SIZE_PAYLOAD;304 hw->config.n_io = hw->sli.ext[SLI4_RSRC_XRI].size;305 306 cpus = num_possible_cpus();307 hw->config.n_eq = cpus > EFCT_HW_MAX_NUM_EQ ? EFCT_HW_MAX_NUM_EQ : cpus;308 309 max_sgl = sli_get_max_sgl(&hw->sli) - SLI4_SGE_MAX_RESERVED;310 max_sgl = (max_sgl > EFCT_FC_MAX_SGL) ? EFCT_FC_MAX_SGL : max_sgl;311 hw->config.n_sgl = max_sgl;312 313 (void)efct_hw_read_max_dump_size(hw);314 315 return 0;316}317 318static void319efct_logfcfi(struct efct_hw *hw, u32 j, u32 i, u32 id)320{321 efc_log_info(hw->os,322 "REG_FCFI: filter[%d] %08X -> RQ[%d] id=%d\n",323 j, hw->config.filter_def[j], i, id);324}325 326static inline void327efct_hw_init_free_io(struct efct_hw_io *io)328{329 /*330 * Set io->done to NULL, to avoid any callbacks, should331 * a completion be received for one of these IOs332 */333 io->done = NULL;334 io->abort_done = NULL;335 io->status_saved = false;336 io->abort_in_progress = false;337 io->type = 0xFFFF;338 io->wq = NULL;339}340 341static bool efct_hw_iotype_is_originator(u16 io_type)342{343 switch (io_type) {344 case EFCT_HW_FC_CT:345 case EFCT_HW_ELS_REQ:346 return true;347 default:348 return false;349 }350}351 352static void353efct_hw_io_restore_sgl(struct efct_hw *hw, struct efct_hw_io *io)354{355 /* Restore the default */356 io->sgl = &io->def_sgl;357 io->sgl_count = io->def_sgl_count;358}359 360static void361efct_hw_wq_process_io(void *arg, u8 *cqe, int status)362{363 struct efct_hw_io *io = arg;364 struct efct_hw *hw = io->hw;365 struct sli4_fc_wcqe *wcqe = (void *)cqe;366 u32 len = 0;367 u32 ext = 0;368 369 /* clear xbusy flag if WCQE[XB] is clear */370 if (io->xbusy && (wcqe->flags & SLI4_WCQE_XB) == 0)371 io->xbusy = false;372 373 /* get extended CQE status */374 switch (io->type) {375 case EFCT_HW_BLS_ACC:376 case EFCT_HW_BLS_RJT:377 break;378 case EFCT_HW_ELS_REQ:379 sli_fc_els_did(&hw->sli, cqe, &ext);380 len = sli_fc_response_length(&hw->sli, cqe);381 break;382 case EFCT_HW_ELS_RSP:383 case EFCT_HW_FC_CT_RSP:384 break;385 case EFCT_HW_FC_CT:386 len = sli_fc_response_length(&hw->sli, cqe);387 break;388 case EFCT_HW_IO_TARGET_WRITE:389 len = sli_fc_io_length(&hw->sli, cqe);390 break;391 case EFCT_HW_IO_TARGET_READ:392 len = sli_fc_io_length(&hw->sli, cqe);393 break;394 case EFCT_HW_IO_TARGET_RSP:395 break;396 case EFCT_HW_IO_DNRX_REQUEUE:397 /* release the count for re-posting the buffer */398 /* efct_hw_io_free(hw, io); */399 break;400 default:401 efc_log_err(hw->os, "unhandled io type %#x for XRI 0x%x\n",402 io->type, io->indicator);403 break;404 }405 if (status) {406 ext = sli_fc_ext_status(&hw->sli, cqe);407 /*408 * If we're not an originator IO, and XB is set, then issue409 * abort for the IO from within the HW410 */411 if (efct_hw_iotype_is_originator(io->type) &&412 wcqe->flags & SLI4_WCQE_XB) {413 int rc;414 415 efc_log_debug(hw->os, "aborting xri=%#x tag=%#x\n",416 io->indicator, io->reqtag);417 418 /*419 * Because targets may send a response when the IO420 * completes using the same XRI, we must wait for the421 * XRI_ABORTED CQE to issue the IO callback422 */423 rc = efct_hw_io_abort(hw, io, false, NULL, NULL);424 if (rc == 0) {425 /*426 * latch status to return after abort is427 * complete428 */429 io->status_saved = true;430 io->saved_status = status;431 io->saved_ext = ext;432 io->saved_len = len;433 goto exit_efct_hw_wq_process_io;434 } else if (rc == -EINPROGRESS) {435 /*436 * Already being aborted by someone else (ABTS437 * perhaps). Just return original438 * error.439 */440 efc_log_debug(hw->os, "%s%#x tag=%#x\n",441 "abort in progress xri=",442 io->indicator, io->reqtag);443 444 } else {445 /* Failed to abort for some other reason, log446 * error447 */448 efc_log_debug(hw->os, "%s%#x tag=%#x rc=%d\n",449 "Failed to abort xri=",450 io->indicator, io->reqtag, rc);451 }452 }453 }454 455 if (io->done) {456 efct_hw_done_t done = io->done;457 458 io->done = NULL;459 460 if (io->status_saved) {461 /* use latched status if exists */462 status = io->saved_status;463 len = io->saved_len;464 ext = io->saved_ext;465 io->status_saved = false;466 }467 468 /* Restore default SGL */469 efct_hw_io_restore_sgl(hw, io);470 done(io, len, status, ext, io->arg);471 }472 473exit_efct_hw_wq_process_io:474 return;475}476 477static int478efct_hw_setup_io(struct efct_hw *hw)479{480 u32 i = 0;481 struct efct_hw_io *io = NULL;482 uintptr_t xfer_virt = 0;483 uintptr_t xfer_phys = 0;484 u32 index;485 bool new_alloc = true;486 struct efc_dma *dma;487 struct efct *efct = hw->os;488 489 if (!hw->io) {490 hw->io = kmalloc_array(hw->config.n_io, sizeof(io), GFP_KERNEL);491 if (!hw->io)492 return -ENOMEM;493 494 memset(hw->io, 0, hw->config.n_io * sizeof(io));495 496 for (i = 0; i < hw->config.n_io; i++) {497 hw->io[i] = kzalloc(sizeof(*io), GFP_KERNEL);498 if (!hw->io[i])499 goto error;500 }501 502 /* Create WQE buffs for IO */503 hw->wqe_buffs = kzalloc((hw->config.n_io * hw->sli.wqe_size),504 GFP_KERNEL);505 if (!hw->wqe_buffs) {506 kfree(hw->io);507 return -ENOMEM;508 }509 510 } else {511 /* re-use existing IOs, including SGLs */512 new_alloc = false;513 }514 515 if (new_alloc) {516 dma = &hw->xfer_rdy;517 dma->size = sizeof(struct fcp_txrdy) * hw->config.n_io;518 dma->virt = dma_alloc_coherent(&efct->pci->dev,519 dma->size, &dma->phys, GFP_KERNEL);520 if (!dma->virt)521 return -ENOMEM;522 }523 xfer_virt = (uintptr_t)hw->xfer_rdy.virt;524 xfer_phys = hw->xfer_rdy.phys;525 526 /* Initialize the pool of HW IO objects */527 for (i = 0; i < hw->config.n_io; i++) {528 struct hw_wq_callback *wqcb;529 530 io = hw->io[i];531 532 /* initialize IO fields */533 io->hw = hw;534 535 /* Assign a WQE buff */536 io->wqe.wqebuf = &hw->wqe_buffs[i * hw->sli.wqe_size];537 538 /* Allocate the request tag for this IO */539 wqcb = efct_hw_reqtag_alloc(hw, efct_hw_wq_process_io, io);540 if (!wqcb) {541 efc_log_err(hw->os, "can't allocate request tag\n");542 return -ENOSPC;543 }544 io->reqtag = wqcb->instance_index;545 546 /* Now for the fields that are initialized on each free */547 efct_hw_init_free_io(io);548 549 /* The XB flag isn't cleared on IO free, so init to zero */550 io->xbusy = 0;551 552 if (sli_resource_alloc(&hw->sli, SLI4_RSRC_XRI,553 &io->indicator, &index)) {554 efc_log_err(hw->os,555 "sli_resource_alloc failed @ %d\n", i);556 return -ENOMEM;557 }558 559 if (new_alloc) {560 dma = &io->def_sgl;561 dma->size = hw->config.n_sgl *562 sizeof(struct sli4_sge);563 dma->virt = dma_alloc_coherent(&efct->pci->dev,564 dma->size, &dma->phys,565 GFP_KERNEL);566 if (!dma->virt) {567 efc_log_err(hw->os, "dma_alloc fail %d\n", i);568 memset(&io->def_sgl, 0,569 sizeof(struct efc_dma));570 return -ENOMEM;571 }572 }573 io->def_sgl_count = hw->config.n_sgl;574 io->sgl = &io->def_sgl;575 io->sgl_count = io->def_sgl_count;576 577 if (hw->xfer_rdy.size) {578 io->xfer_rdy.virt = (void *)xfer_virt;579 io->xfer_rdy.phys = xfer_phys;580 io->xfer_rdy.size = sizeof(struct fcp_txrdy);581 582 xfer_virt += sizeof(struct fcp_txrdy);583 xfer_phys += sizeof(struct fcp_txrdy);584 }585 }586 587 return 0;588error:589 for (i = 0; i < hw->config.n_io && hw->io[i]; i++) {590 kfree(hw->io[i]);591 hw->io[i] = NULL;592 }593 594 kfree(hw->io);595 hw->io = NULL;596 597 return -ENOMEM;598}599 600static int601efct_hw_init_prereg_io(struct efct_hw *hw)602{603 u32 i, idx = 0;604 struct efct_hw_io *io = NULL;605 u8 cmd[SLI4_BMBX_SIZE];606 int rc = 0;607 u32 n_rem;608 u32 n = 0;609 u32 sgls_per_request = 256;610 struct efc_dma **sgls = NULL;611 struct efc_dma req;612 struct efct *efct = hw->os;613 614 sgls = kmalloc_array(sgls_per_request, sizeof(*sgls), GFP_KERNEL);615 if (!sgls)616 return -ENOMEM;617 618 memset(&req, 0, sizeof(struct efc_dma));619 req.size = 32 + sgls_per_request * 16;620 req.virt = dma_alloc_coherent(&efct->pci->dev, req.size, &req.phys,621 GFP_KERNEL);622 if (!req.virt) {623 kfree(sgls);624 return -ENOMEM;625 }626 627 for (n_rem = hw->config.n_io; n_rem; n_rem -= n) {628 /* Copy address of SGL's into local sgls[] array, break629 * out if the xri is not contiguous.630 */631 u32 min = (sgls_per_request < n_rem) ? sgls_per_request : n_rem;632 633 for (n = 0; n < min; n++) {634 /* Check that we have contiguous xri values */635 if (n > 0) {636 if (hw->io[idx + n]->indicator !=637 hw->io[idx + n - 1]->indicator + 1)638 break;639 }640 641 sgls[n] = hw->io[idx + n]->sgl;642 }643 644 if (sli_cmd_post_sgl_pages(&hw->sli, cmd,645 hw->io[idx]->indicator, n, sgls, NULL, &req)) {646 rc = -EIO;647 break;648 }649 650 rc = efct_hw_command(hw, cmd, EFCT_CMD_POLL, NULL, NULL);651 if (rc) {652 efc_log_err(hw->os, "SGL post failed, rc=%d\n", rc);653 break;654 }655 656 /* Add to tail if successful */657 for (i = 0; i < n; i++, idx++) {658 io = hw->io[idx];659 io->state = EFCT_HW_IO_STATE_FREE;660 INIT_LIST_HEAD(&io->list_entry);661 list_add_tail(&io->list_entry, &hw->io_free);662 }663 }664 665 dma_free_coherent(&efct->pci->dev, req.size, req.virt, req.phys);666 memset(&req, 0, sizeof(struct efc_dma));667 kfree(sgls);668 669 return rc;670}671 672static int673efct_hw_init_io(struct efct_hw *hw)674{675 u32 i, idx = 0;676 bool prereg = false;677 struct efct_hw_io *io = NULL;678 int rc = 0;679 680 prereg = hw->sli.params.sgl_pre_registered;681 682 if (prereg)683 return efct_hw_init_prereg_io(hw);684 685 for (i = 0; i < hw->config.n_io; i++, idx++) {686 io = hw->io[idx];687 io->state = EFCT_HW_IO_STATE_FREE;688 INIT_LIST_HEAD(&io->list_entry);689 list_add_tail(&io->list_entry, &hw->io_free);690 }691 692 return rc;693}694 695static int696efct_hw_config_set_fdt_xfer_hint(struct efct_hw *hw, u32 fdt_xfer_hint)697{698 int rc = 0;699 u8 buf[SLI4_BMBX_SIZE];700 struct sli4_rqst_cmn_set_features_set_fdt_xfer_hint param;701 702 memset(¶m, 0, sizeof(param));703 param.fdt_xfer_hint = cpu_to_le32(fdt_xfer_hint);704 /* build the set_features command */705 sli_cmd_common_set_features(&hw->sli, buf,706 SLI4_SET_FEATURES_SET_FTD_XFER_HINT, sizeof(param), ¶m);707 708 rc = efct_hw_command(hw, buf, EFCT_CMD_POLL, NULL, NULL);709 if (rc)710 efc_log_warn(hw->os, "set FDT hint %d failed: %d\n",711 fdt_xfer_hint, rc);712 else713 efc_log_info(hw->os, "Set FTD transfer hint to %d\n",714 le32_to_cpu(param.fdt_xfer_hint));715 716 return rc;717}718 719static int720efct_hw_config_rq(struct efct_hw *hw)721{722 u32 min_rq_count, i, rc;723 struct sli4_cmd_rq_cfg rq_cfg[SLI4_CMD_REG_FCFI_NUM_RQ_CFG];724 u8 buf[SLI4_BMBX_SIZE];725 726 efc_log_info(hw->os, "using REG_FCFI standard\n");727 728 /*729 * Set the filter match/mask values from hw's730 * filter_def values731 */732 for (i = 0; i < SLI4_CMD_REG_FCFI_NUM_RQ_CFG; i++) {733 rq_cfg[i].rq_id = cpu_to_le16(0xffff);734 rq_cfg[i].r_ctl_mask = (u8)hw->config.filter_def[i];735 rq_cfg[i].r_ctl_match = (u8)(hw->config.filter_def[i] >> 8);736 rq_cfg[i].type_mask = (u8)(hw->config.filter_def[i] >> 16);737 rq_cfg[i].type_match = (u8)(hw->config.filter_def[i] >> 24);738 }739 740 /*741 * Update the rq_id's of the FCF configuration742 * (don't update more than the number of rq_cfg743 * elements)744 */745 min_rq_count = (hw->hw_rq_count < SLI4_CMD_REG_FCFI_NUM_RQ_CFG) ?746 hw->hw_rq_count : SLI4_CMD_REG_FCFI_NUM_RQ_CFG;747 for (i = 0; i < min_rq_count; i++) {748 struct hw_rq *rq = hw->hw_rq[i];749 u32 j;750 751 for (j = 0; j < SLI4_CMD_REG_FCFI_NUM_RQ_CFG; j++) {752 u32 mask = (rq->filter_mask != 0) ?753 rq->filter_mask : 1;754 755 if (!(mask & (1U << j)))756 continue;757 758 rq_cfg[i].rq_id = cpu_to_le16(rq->hdr->id);759 efct_logfcfi(hw, j, i, rq->hdr->id);760 }761 }762 763 rc = -EIO;764 if (!sli_cmd_reg_fcfi(&hw->sli, buf, 0, rq_cfg))765 rc = efct_hw_command(hw, buf, EFCT_CMD_POLL, NULL, NULL);766 767 if (rc != 0) {768 efc_log_err(hw->os, "FCFI registration failed\n");769 return rc;770 }771 hw->fcf_indicator =772 le16_to_cpu(((struct sli4_cmd_reg_fcfi *)buf)->fcfi);773 774 return rc;775}776 777static int778efct_hw_config_mrq(struct efct_hw *hw, u8 mode, u16 fcf_index)779{780 u8 buf[SLI4_BMBX_SIZE], mrq_bitmask = 0;781 struct hw_rq *rq;782 struct sli4_cmd_reg_fcfi_mrq *rsp = NULL;783 struct sli4_cmd_rq_cfg rq_filter[SLI4_CMD_REG_FCFI_MRQ_NUM_RQ_CFG];784 u32 rc, i;785 786 if (mode == SLI4_CMD_REG_FCFI_SET_FCFI_MODE)787 goto issue_cmd;788 789 /* Set the filter match/mask values from hw's filter_def values */790 for (i = 0; i < SLI4_CMD_REG_FCFI_NUM_RQ_CFG; i++) {791 rq_filter[i].rq_id = cpu_to_le16(0xffff);792 rq_filter[i].type_mask = (u8)hw->config.filter_def[i];793 rq_filter[i].type_match = (u8)(hw->config.filter_def[i] >> 8);794 rq_filter[i].r_ctl_mask = (u8)(hw->config.filter_def[i] >> 16);795 rq_filter[i].r_ctl_match = (u8)(hw->config.filter_def[i] >> 24);796 }797 798 rq = hw->hw_rq[0];799 rq_filter[0].rq_id = cpu_to_le16(rq->hdr->id);800 rq_filter[1].rq_id = cpu_to_le16(rq->hdr->id);801 802 mrq_bitmask = 0x2;803issue_cmd:804 efc_log_debug(hw->os, "Issue reg_fcfi_mrq count:%d policy:%d mode:%d\n",805 hw->hw_rq_count, hw->config.rq_selection_policy, mode);806 /* Invoke REG_FCFI_MRQ */807 rc = sli_cmd_reg_fcfi_mrq(&hw->sli, buf, mode, fcf_index,808 hw->config.rq_selection_policy, mrq_bitmask,809 hw->hw_mrq_count, rq_filter);810 if (rc) {811 efc_log_err(hw->os, "sli_cmd_reg_fcfi_mrq() failed\n");812 return -EIO;813 }814 815 rc = efct_hw_command(hw, buf, EFCT_CMD_POLL, NULL, NULL);816 817 rsp = (struct sli4_cmd_reg_fcfi_mrq *)buf;818 819 if ((rc) || (le16_to_cpu(rsp->hdr.status))) {820 efc_log_err(hw->os, "FCFI MRQ reg failed. cmd=%x status=%x\n",821 rsp->hdr.command, le16_to_cpu(rsp->hdr.status));822 return -EIO;823 }824 825 if (mode == SLI4_CMD_REG_FCFI_SET_FCFI_MODE)826 hw->fcf_indicator = le16_to_cpu(rsp->fcfi);827 828 return 0;829}830 831static void832efct_hw_queue_hash_add(struct efct_queue_hash *hash,833 u16 id, u16 index)834{835 u32 hash_index = id & (EFCT_HW_Q_HASH_SIZE - 1);836 837 /*838 * Since the hash is always bigger than the number of queues, then we839 * never have to worry about an infinite loop.840 */841 while (hash[hash_index].in_use)842 hash_index = (hash_index + 1) & (EFCT_HW_Q_HASH_SIZE - 1);843 844 /* not used, claim the entry */845 hash[hash_index].id = id;846 hash[hash_index].in_use = true;847 hash[hash_index].index = index;848}849 850static int851efct_hw_config_sli_port_health_check(struct efct_hw *hw, u8 query, u8 enable)852{853 int rc = 0;854 u8 buf[SLI4_BMBX_SIZE];855 struct sli4_rqst_cmn_set_features_health_check param;856 u32 health_check_flag = 0;857 858 memset(¶m, 0, sizeof(param));859 860 if (enable)861 health_check_flag |= SLI4_RQ_HEALTH_CHECK_ENABLE;862 863 if (query)864 health_check_flag |= SLI4_RQ_HEALTH_CHECK_QUERY;865 866 param.health_check_dword = cpu_to_le32(health_check_flag);867 868 /* build the set_features command */869 sli_cmd_common_set_features(&hw->sli, buf,870 SLI4_SET_FEATURES_SLI_PORT_HEALTH_CHECK, sizeof(param), ¶m);871 872 rc = efct_hw_command(hw, buf, EFCT_CMD_POLL, NULL, NULL);873 if (rc)874 efc_log_err(hw->os, "efct_hw_command returns %d\n", rc);875 else876 efc_log_debug(hw->os, "SLI Port Health Check is enabled\n");877 878 return rc;879}880 881int882efct_hw_init(struct efct_hw *hw)883{884 int rc;885 u32 i = 0;886 int rem_count;887 unsigned long flags = 0;888 struct efct_hw_io *temp;889 struct efc_dma *dma;890 891 /*892 * Make sure the command lists are empty. If this is start-of-day,893 * they'll be empty since they were just initialized in efct_hw_setup.894 * If we've just gone through a reset, the command and command pending895 * lists should have been cleaned up as part of the reset896 * (efct_hw_reset()).897 */898 spin_lock_irqsave(&hw->cmd_lock, flags);899 if (!list_empty(&hw->cmd_head)) {900 spin_unlock_irqrestore(&hw->cmd_lock, flags);901 efc_log_err(hw->os, "command found on cmd list\n");902 return -EIO;903 }904 if (!list_empty(&hw->cmd_pending)) {905 spin_unlock_irqrestore(&hw->cmd_lock, flags);906 efc_log_err(hw->os, "command found on pending list\n");907 return -EIO;908 }909 spin_unlock_irqrestore(&hw->cmd_lock, flags);910 911 /* Free RQ buffers if prevously allocated */912 efct_hw_rx_free(hw);913 914 /*915 * The IO queues must be initialized here for the reset case. The916 * efct_hw_init_io() function will re-add the IOs to the free list.917 * The cmd_head list should be OK since we free all entries in918 * efct_hw_command_cancel() that is called in the efct_hw_reset().919 */920 921 /* If we are in this function due to a reset, there may be stale items922 * on lists that need to be removed. Clean them up.923 */924 rem_count = 0;925 while ((!list_empty(&hw->io_wait_free))) {926 rem_count++;927 temp = list_first_entry(&hw->io_wait_free, struct efct_hw_io,928 list_entry);929 list_del_init(&temp->list_entry);930 }931 if (rem_count > 0)932 efc_log_debug(hw->os, "rmvd %d items from io_wait_free list\n",933 rem_count);934 935 rem_count = 0;936 while ((!list_empty(&hw->io_inuse))) {937 rem_count++;938 temp = list_first_entry(&hw->io_inuse, struct efct_hw_io,939 list_entry);940 list_del_init(&temp->list_entry);941 }942 if (rem_count > 0)943 efc_log_debug(hw->os, "rmvd %d items from io_inuse list\n",944 rem_count);945 946 rem_count = 0;947 while ((!list_empty(&hw->io_free))) {948 rem_count++;949 temp = list_first_entry(&hw->io_free, struct efct_hw_io,950 list_entry);951 list_del_init(&temp->list_entry);952 }953 if (rem_count > 0)954 efc_log_debug(hw->os, "rmvd %d items from io_free list\n",955 rem_count);956 957 /* If MRQ not required, Make sure we dont request feature. */958 if (hw->config.n_rq == 1)959 hw->sli.features &= (~SLI4_REQFEAT_MRQP);960 961 if (sli_init(&hw->sli)) {962 efc_log_err(hw->os, "SLI failed to initialize\n");963 return -EIO;964 }965 966 if (hw->sliport_healthcheck) {967 rc = efct_hw_config_sli_port_health_check(hw, 0, 1);968 if (rc != 0) {969 efc_log_err(hw->os, "Enable port Health check fail\n");970 return rc;971 }972 }973 974 /*975 * Set FDT transfer hint, only works on Lancer976 */977 if (hw->sli.if_type == SLI4_INTF_IF_TYPE_2) {978 /*979 * Non-fatal error. In particular, we can disregard failure to980 * set EFCT_HW_FDT_XFER_HINT on devices with legacy firmware981 * that do not support EFCT_HW_FDT_XFER_HINT feature.982 */983 efct_hw_config_set_fdt_xfer_hint(hw, EFCT_HW_FDT_XFER_HINT);984 }985 986 /* zero the hashes */987 memset(hw->cq_hash, 0, sizeof(hw->cq_hash));988 efc_log_debug(hw->os, "Max CQs %d, hash size = %d\n",989 EFCT_HW_MAX_NUM_CQ, EFCT_HW_Q_HASH_SIZE);990 991 memset(hw->rq_hash, 0, sizeof(hw->rq_hash));992 efc_log_debug(hw->os, "Max RQs %d, hash size = %d\n",993 EFCT_HW_MAX_NUM_RQ, EFCT_HW_Q_HASH_SIZE);994 995 memset(hw->wq_hash, 0, sizeof(hw->wq_hash));996 efc_log_debug(hw->os, "Max WQs %d, hash size = %d\n",997 EFCT_HW_MAX_NUM_WQ, EFCT_HW_Q_HASH_SIZE);998 999 rc = efct_hw_init_queues(hw);1000 if (rc)1001 return rc;1002 1003 rc = efct_hw_map_wq_cpu(hw);1004 if (rc)1005 return rc;1006 1007 /* Allocate and p_st RQ buffers */1008 rc = efct_hw_rx_allocate(hw);1009 if (rc) {1010 efc_log_err(hw->os, "rx_allocate failed\n");1011 return rc;1012 }1013 1014 rc = efct_hw_rx_post(hw);1015 if (rc) {1016 efc_log_err(hw->os, "WARNING - error posting RQ buffers\n");1017 return rc;1018 }1019 1020 if (hw->config.n_eq == 1) {1021 rc = efct_hw_config_rq(hw);1022 if (rc) {1023 efc_log_err(hw->os, "config rq failed %d\n", rc);1024 return rc;1025 }1026 } else {1027 rc = efct_hw_config_mrq(hw, SLI4_CMD_REG_FCFI_SET_FCFI_MODE, 0);1028 if (rc != 0) {1029 efc_log_err(hw->os, "REG_FCFI_MRQ FCFI reg failed\n");1030 return rc;1031 }1032 1033 rc = efct_hw_config_mrq(hw, SLI4_CMD_REG_FCFI_SET_MRQ_MODE, 0);1034 if (rc != 0) {1035 efc_log_err(hw->os, "REG_FCFI_MRQ MRQ reg failed\n");1036 return rc;1037 }1038 }1039 1040 /*1041 * Allocate the WQ request tag pool, if not previously allocated1042 * (the request tag value is 16 bits, thus the pool allocation size1043 * of 64k)1044 */1045 hw->wq_reqtag_pool = efct_hw_reqtag_pool_alloc(hw);1046 if (!hw->wq_reqtag_pool) {1047 efc_log_err(hw->os, "efct_hw_reqtag_pool_alloc failed\n");1048 return -ENOMEM;1049 }1050 1051 rc = efct_hw_setup_io(hw);1052 if (rc) {1053 efc_log_err(hw->os, "IO allocation failure\n");1054 return rc;1055 }1056 1057 rc = efct_hw_init_io(hw);1058 if (rc) {1059 efc_log_err(hw->os, "IO initialization failure\n");1060 return rc;1061 }1062 1063 dma = &hw->loop_map;1064 dma->size = SLI4_MIN_LOOP_MAP_BYTES;1065 dma->virt = dma_alloc_coherent(&hw->os->pci->dev, dma->size, &dma->phys,1066 GFP_KERNEL);1067 if (!dma->virt)1068 return -EIO;1069 1070 /*1071 * Arming the EQ allows (e.g.) interrupts when CQ completions write EQ1072 * entries1073 */1074 for (i = 0; i < hw->eq_count; i++)1075 sli_queue_arm(&hw->sli, &hw->eq[i], true);1076 1077 /*1078 * Initialize RQ hash1079 */1080 for (i = 0; i < hw->rq_count; i++)1081 efct_hw_queue_hash_add(hw->rq_hash, hw->rq[i].id, i);1082 1083 /*1084 * Initialize WQ hash1085 */1086 for (i = 0; i < hw->wq_count; i++)1087 efct_hw_queue_hash_add(hw->wq_hash, hw->wq[i].id, i);1088 1089 /*1090 * Arming the CQ allows (e.g.) MQ completions to write CQ entries1091 */1092 for (i = 0; i < hw->cq_count; i++) {1093 efct_hw_queue_hash_add(hw->cq_hash, hw->cq[i].id, i);1094 sli_queue_arm(&hw->sli, &hw->cq[i], true);1095 }1096 1097 /* Set RQ process limit*/1098 for (i = 0; i < hw->hw_rq_count; i++) {1099 struct hw_rq *rq = hw->hw_rq[i];1100 1101 hw->cq[rq->cq->instance].proc_limit = hw->config.n_io / 2;1102 }1103 1104 /* record the fact that the queues are functional */1105 hw->state = EFCT_HW_STATE_ACTIVE;1106 /*1107 * Allocate a HW IOs for send frame.1108 */1109 hw->hw_wq[0]->send_frame_io = efct_hw_io_alloc(hw);1110 if (!hw->hw_wq[0]->send_frame_io)1111 efc_log_err(hw->os, "alloc for send_frame_io failed\n");1112 1113 /* Initialize send frame sequence id */1114 atomic_set(&hw->send_frame_seq_id, 0);1115 1116 return 0;1117}1118 1119int1120efct_hw_parse_filter(struct efct_hw *hw, void *value)1121{1122 int rc = 0;1123 char *p = NULL;1124 char *token;1125 u32 idx = 0;1126 1127 for (idx = 0; idx < ARRAY_SIZE(hw->config.filter_def); idx++)1128 hw->config.filter_def[idx] = 0;1129 1130 p = kstrdup(value, GFP_KERNEL);1131 if (!p || !*p) {1132 efc_log_err(hw->os, "p is NULL\n");1133 return -ENOMEM;1134 }1135 1136 idx = 0;1137 while ((token = strsep(&p, ",")) && *token) {1138 if (kstrtou32(token, 0, &hw->config.filter_def[idx++]))1139 efc_log_err(hw->os, "kstrtoint failed\n");1140 1141 if (!p || !*p)1142 break;1143 1144 if (idx == ARRAY_SIZE(hw->config.filter_def))1145 break;1146 }1147 kfree(p);1148 1149 return rc;1150}1151 1152u641153efct_get_wwnn(struct efct_hw *hw)1154{1155 struct sli4 *sli = &hw->sli;1156 u8 p[8];1157 1158 memcpy(p, sli->wwnn, sizeof(p));1159 return get_unaligned_be64(p);1160}1161 1162u641163efct_get_wwpn(struct efct_hw *hw)1164{1165 struct sli4 *sli = &hw->sli;1166 u8 p[8];1167 1168 memcpy(p, sli->wwpn, sizeof(p));1169 return get_unaligned_be64(p);1170}1171 1172static struct efc_hw_rq_buffer *1173efct_hw_rx_buffer_alloc(struct efct_hw *hw, u32 rqindex, u32 count,1174 u32 size)1175{1176 struct efct *efct = hw->os;1177 struct efc_hw_rq_buffer *rq_buf = NULL;1178 struct efc_hw_rq_buffer *prq;1179 u32 i;1180 1181 if (!count)1182 return NULL;1183 1184 rq_buf = kmalloc_array(count, sizeof(*rq_buf), GFP_KERNEL);1185 if (!rq_buf)1186 return NULL;1187 memset(rq_buf, 0, sizeof(*rq_buf) * count);1188 1189 for (i = 0, prq = rq_buf; i < count; i ++, prq++) {1190 prq->rqindex = rqindex;1191 prq->dma.size = size;1192 prq->dma.virt = dma_alloc_coherent(&efct->pci->dev,1193 prq->dma.size,1194 &prq->dma.phys,1195 GFP_KERNEL);1196 if (!prq->dma.virt) {1197 efc_log_err(hw->os, "DMA allocation failed\n");1198 kfree(rq_buf);1199 return NULL;1200 }1201 }1202 return rq_buf;1203}1204 1205static void1206efct_hw_rx_buffer_free(struct efct_hw *hw,1207 struct efc_hw_rq_buffer *rq_buf,1208 u32 count)1209{1210 struct efct *efct = hw->os;1211 u32 i;1212 struct efc_hw_rq_buffer *prq;1213 1214 if (rq_buf) {1215 for (i = 0, prq = rq_buf; i < count; i++, prq++) {1216 dma_free_coherent(&efct->pci->dev,1217 prq->dma.size, prq->dma.virt,1218 prq->dma.phys);1219 memset(&prq->dma, 0, sizeof(struct efc_dma));1220 }1221 1222 kfree(rq_buf);1223 }1224}1225 1226int1227efct_hw_rx_allocate(struct efct_hw *hw)1228{1229 struct efct *efct = hw->os;1230 u32 i;1231 int rc = 0;1232 u32 rqindex = 0;1233 u32 hdr_size = EFCT_HW_RQ_SIZE_HDR;1234 u32 payload_size = hw->config.rq_default_buffer_size;1235 1236 rqindex = 0;1237 1238 for (i = 0; i < hw->hw_rq_count; i++) {1239 struct hw_rq *rq = hw->hw_rq[i];1240 1241 /* Allocate header buffers */1242 rq->hdr_buf = efct_hw_rx_buffer_alloc(hw, rqindex,1243 rq->entry_count,1244 hdr_size);1245 if (!rq->hdr_buf) {1246 efc_log_err(efct, "rx_buffer_alloc hdr_buf failed\n");1247 rc = -EIO;1248 break;1249 }1250 1251 efc_log_debug(hw->os,1252 "rq[%2d] rq_id %02d header %4d by %4d bytes\n",1253 i, rq->hdr->id, rq->entry_count, hdr_size);1254 1255 rqindex++;1256 1257 /* Allocate payload buffers */1258 rq->payload_buf = efct_hw_rx_buffer_alloc(hw, rqindex,1259 rq->entry_count,1260 payload_size);1261 if (!rq->payload_buf) {1262 efc_log_err(efct, "rx_buffer_alloc fb_buf failed\n");1263 rc = -EIO;1264 break;1265 }1266 efc_log_debug(hw->os,1267 "rq[%2d] rq_id %02d default %4d by %4d bytes\n",1268 i, rq->data->id, rq->entry_count, payload_size);1269 rqindex++;1270 }1271 1272 return rc ? -EIO : 0;1273}1274 1275int1276efct_hw_rx_post(struct efct_hw *hw)1277{1278 u32 i;1279 u32 idx;1280 u32 rq_idx;1281 int rc = 0;1282 1283 if (!hw->seq_pool) {1284 u32 count = 0;1285 1286 for (i = 0; i < hw->hw_rq_count; i++)1287 count += hw->hw_rq[i]->entry_count;1288 1289 hw->seq_pool = kmalloc_array(count,1290 sizeof(struct efc_hw_sequence), GFP_KERNEL);1291 if (!hw->seq_pool)1292 return -ENOMEM;1293 }1294 1295 /*1296 * In RQ pair mode, we MUST post the header and payload buffer at the1297 * same time.1298 */1299 for (rq_idx = 0, idx = 0; rq_idx < hw->hw_rq_count; rq_idx++) {1300 struct hw_rq *rq = hw->hw_rq[rq_idx];1301 1302 for (i = 0; i < rq->entry_count - 1; i++) {1303 struct efc_hw_sequence *seq;1304 1305 seq = hw->seq_pool + idx;1306 idx++;1307 seq->header = &rq->hdr_buf[i];1308 seq->payload = &rq->payload_buf[i];1309 rc = efct_hw_sequence_free(hw, seq);1310 if (rc)1311 break;1312 }1313 if (rc)1314 break;1315 }1316 1317 if (rc && hw->seq_pool)1318 kfree(hw->seq_pool);1319 1320 return rc;1321}1322 1323void1324efct_hw_rx_free(struct efct_hw *hw)1325{1326 u32 i;1327 1328 /* Free hw_rq buffers */1329 for (i = 0; i < hw->hw_rq_count; i++) {1330 struct hw_rq *rq = hw->hw_rq[i];1331 1332 if (rq) {1333 efct_hw_rx_buffer_free(hw, rq->hdr_buf,1334 rq->entry_count);1335 rq->hdr_buf = NULL;1336 efct_hw_rx_buffer_free(hw, rq->payload_buf,1337 rq->entry_count);1338 rq->payload_buf = NULL;1339 }1340 }1341}1342 1343static int1344efct_hw_cmd_submit_pending(struct efct_hw *hw)1345{1346 int rc = 0;1347 1348 /* Assumes lock held */1349 1350 /* Only submit MQE if there's room */1351 while (hw->cmd_head_count < (EFCT_HW_MQ_DEPTH - 1) &&1352 !list_empty(&hw->cmd_pending)) {1353 struct efct_command_ctx *ctx;1354 1355 ctx = list_first_entry(&hw->cmd_pending,1356 struct efct_command_ctx, list_entry);1357 if (!ctx)1358 break;1359 1360 list_del_init(&ctx->list_entry);1361 1362 list_add_tail(&ctx->list_entry, &hw->cmd_head);1363 hw->cmd_head_count++;1364 if (sli_mq_write(&hw->sli, hw->mq, ctx->buf) < 0) {1365 efc_log_debug(hw->os,1366 "sli_queue_write failed: %d\n", rc);1367 rc = -EIO;1368 break;1369 }1370 }1371 return rc;1372}1373 1374int1375efct_hw_command(struct efct_hw *hw, u8 *cmd, u32 opts, void *cb, void *arg)1376{1377 int rc = -EIO;1378 unsigned long flags = 0;1379 void *bmbx = NULL;1380 1381 /*1382 * If the chip is in an error state (UE'd) then reject this mailbox1383 * command.1384 */1385 if (sli_fw_error_status(&hw->sli) > 0) {1386 efc_log_crit(hw->os, "Chip in an error state - reset needed\n");1387 efc_log_crit(hw->os, "status=%#x error1=%#x error2=%#x\n",1388 sli_reg_read_status(&hw->sli),1389 sli_reg_read_err1(&hw->sli),1390 sli_reg_read_err2(&hw->sli));1391 1392 return -EIO;1393 }1394 1395 /*1396 * Send a mailbox command to the hardware, and either wait for1397 * a completion (EFCT_CMD_POLL) or get an optional asynchronous1398 * completion (EFCT_CMD_NOWAIT).1399 */1400 1401 if (opts == EFCT_CMD_POLL) {1402 mutex_lock(&hw->bmbx_lock);1403 bmbx = hw->sli.bmbx.virt;1404 1405 memcpy(bmbx, cmd, SLI4_BMBX_SIZE);1406 1407 if (sli_bmbx_command(&hw->sli) == 0) {1408 rc = 0;1409 memcpy(cmd, bmbx, SLI4_BMBX_SIZE);1410 }1411 mutex_unlock(&hw->bmbx_lock);1412 } else if (opts == EFCT_CMD_NOWAIT) {1413 struct efct_command_ctx *ctx = NULL;1414 1415 if (hw->state != EFCT_HW_STATE_ACTIVE) {1416 efc_log_err(hw->os, "Can't send command, HW state=%d\n",1417 hw->state);1418 return -EIO;1419 }1420 1421 ctx = mempool_alloc(hw->cmd_ctx_pool, GFP_ATOMIC);1422 if (!ctx)1423 return -ENOSPC;1424 1425 memset(ctx, 0, sizeof(struct efct_command_ctx));1426 1427 if (cb) {1428 ctx->cb = cb;1429 ctx->arg = arg;1430 }1431 1432 memcpy(ctx->buf, cmd, SLI4_BMBX_SIZE);1433 ctx->ctx = hw;1434 1435 spin_lock_irqsave(&hw->cmd_lock, flags);1436 1437 /* Add to pending list */1438 INIT_LIST_HEAD(&ctx->list_entry);1439 list_add_tail(&ctx->list_entry, &hw->cmd_pending);1440 1441 /* Submit as much of the pending list as we can */1442 rc = efct_hw_cmd_submit_pending(hw);1443 1444 spin_unlock_irqrestore(&hw->cmd_lock, flags);1445 }1446 1447 return rc;1448}1449 1450static int1451efct_hw_command_process(struct efct_hw *hw, int status, u8 *mqe,1452 size_t size)1453{1454 struct efct_command_ctx *ctx = NULL;1455 unsigned long flags = 0;1456 1457 spin_lock_irqsave(&hw->cmd_lock, flags);1458 if (!list_empty(&hw->cmd_head)) {1459 ctx = list_first_entry(&hw->cmd_head,1460 struct efct_command_ctx, list_entry);1461 list_del_init(&ctx->list_entry);1462 }1463 if (!ctx) {1464 efc_log_err(hw->os, "no command context\n");1465 spin_unlock_irqrestore(&hw->cmd_lock, flags);1466 return -EIO;1467 }1468 1469 hw->cmd_head_count--;1470 1471 /* Post any pending requests */1472 efct_hw_cmd_submit_pending(hw);1473 1474 spin_unlock_irqrestore(&hw->cmd_lock, flags);1475 1476 if (ctx->cb) {1477 memcpy(ctx->buf, mqe, size);1478 ctx->cb(hw, status, ctx->buf, ctx->arg);1479 }1480 1481 mempool_free(ctx, hw->cmd_ctx_pool);1482 1483 return 0;1484}1485 1486static int1487efct_hw_mq_process(struct efct_hw *hw,1488 int status, struct sli4_queue *mq)1489{1490 u8 mqe[SLI4_BMBX_SIZE];1491 int rc;1492 1493 rc = sli_mq_read(&hw->sli, mq, mqe);1494 if (!rc)1495 rc = efct_hw_command_process(hw, status, mqe, mq->size);1496 1497 return rc;1498}1499 1500static int1501efct_hw_command_cancel(struct efct_hw *hw)1502{1503 unsigned long flags = 0;1504 int rc = 0;1505 1506 spin_lock_irqsave(&hw->cmd_lock, flags);1507 1508 /*1509 * Manually clean up remaining commands. Note: since this calls1510 * efct_hw_command_process(), we'll also process the cmd_pending1511 * list, so no need to manually clean that out.1512 */1513 while (!list_empty(&hw->cmd_head)) {1514 u8 mqe[SLI4_BMBX_SIZE] = { 0 };1515 struct efct_command_ctx *ctx;1516 1517 ctx = list_first_entry(&hw->cmd_head,1518 struct efct_command_ctx, list_entry);1519 1520 efc_log_debug(hw->os, "hung command %08x\n",1521 !ctx ? U32_MAX : *((u32 *)ctx->buf));1522 spin_unlock_irqrestore(&hw->cmd_lock, flags);1523 rc = efct_hw_command_process(hw, -1, mqe, SLI4_BMBX_SIZE);1524 spin_lock_irqsave(&hw->cmd_lock, flags);1525 }1526 1527 spin_unlock_irqrestore(&hw->cmd_lock, flags);1528 1529 return rc;1530}1531 1532static void1533efct_mbox_rsp_cb(struct efct_hw *hw, int status, u8 *mqe, void *arg)1534{1535 struct efct_mbox_rqst_ctx *ctx = arg;1536 1537 if (ctx) {1538 if (ctx->callback)1539 (*ctx->callback)(hw->os->efcport, status, mqe,1540 ctx->arg);1541 1542 mempool_free(ctx, hw->mbox_rqst_pool);1543 }1544}1545 1546int1547efct_issue_mbox_rqst(void *base, void *cmd, void *cb, void *arg)1548{1549 struct efct_mbox_rqst_ctx *ctx;1550 struct efct *efct = base;1551 struct efct_hw *hw = &efct->hw;1552 int rc;1553 1554 /*1555 * Allocate a callback context (which includes the mbox cmd buffer),1556 * we need this to be persistent as the mbox cmd submission may be1557 * queued and executed later execution.1558 */1559 ctx = mempool_alloc(hw->mbox_rqst_pool, GFP_ATOMIC);1560 if (!ctx)1561 return -EIO;1562 1563 ctx->callback = cb;1564 ctx->arg = arg;1565 1566 rc = efct_hw_command(hw, cmd, EFCT_CMD_NOWAIT, efct_mbox_rsp_cb, ctx);1567 if (rc) {1568 efc_log_err(efct, "issue mbox rqst failure rc:%d\n", rc);1569 mempool_free(ctx, hw->mbox_rqst_pool);1570 return -EIO;1571 }1572 1573 return 0;1574}1575 1576static inline struct efct_hw_io *1577_efct_hw_io_alloc(struct efct_hw *hw)1578{1579 struct efct_hw_io *io = NULL;1580 1581 if (!list_empty(&hw->io_free)) {1582 io = list_first_entry(&hw->io_free, struct efct_hw_io,1583 list_entry);1584 list_del(&io->list_entry);1585 }1586 if (io) {1587 INIT_LIST_HEAD(&io->list_entry);1588 list_add_tail(&io->list_entry, &hw->io_inuse);1589 io->state = EFCT_HW_IO_STATE_INUSE;1590 io->abort_reqtag = U32_MAX;1591 io->wq = hw->wq_cpu_array[raw_smp_processor_id()];1592 if (!io->wq) {1593 efc_log_err(hw->os, "WQ not assigned for cpu:%d\n",1594 raw_smp_processor_id());1595 io->wq = hw->hw_wq[0];1596 }1597 kref_init(&io->ref);1598 io->release = efct_hw_io_free_internal;1599 } else {1600 atomic_add(1, &hw->io_alloc_failed_count);1601 }1602 1603 return io;1604}1605 1606struct efct_hw_io *1607efct_hw_io_alloc(struct efct_hw *hw)1608{1609 struct efct_hw_io *io = NULL;1610 unsigned long flags = 0;1611 1612 spin_lock_irqsave(&hw->io_lock, flags);1613 io = _efct_hw_io_alloc(hw);1614 spin_unlock_irqrestore(&hw->io_lock, flags);1615 1616 return io;1617}1618 1619static void1620efct_hw_io_free_move_correct_list(struct efct_hw *hw,1621 struct efct_hw_io *io)1622{1623 /*1624 * When an IO is freed, depending on the exchange busy flag,1625 * move it to the correct list.1626 */1627 if (io->xbusy) {1628 /*1629 * add to wait_free list and wait for XRI_ABORTED CQEs to clean1630 * up1631 */1632 INIT_LIST_HEAD(&io->list_entry);1633 list_add_tail(&io->list_entry, &hw->io_wait_free);1634 io->state = EFCT_HW_IO_STATE_WAIT_FREE;1635 } else {1636 /* IO not busy, add to free list */1637 INIT_LIST_HEAD(&io->list_entry);1638 list_add_tail(&io->list_entry, &hw->io_free);1639 io->state = EFCT_HW_IO_STATE_FREE;1640 }1641}1642 1643static inline void1644efct_hw_io_free_common(struct efct_hw *hw, struct efct_hw_io *io)1645{1646 /* initialize IO fields */1647 efct_hw_init_free_io(io);1648 1649 /* Restore default SGL */1650 efct_hw_io_restore_sgl(hw, io);1651}1652 1653void1654efct_hw_io_free_internal(struct kref *arg)1655{1656 unsigned long flags = 0;1657 struct efct_hw_io *io = container_of(arg, struct efct_hw_io, ref);1658 struct efct_hw *hw = io->hw;1659 1660 /* perform common cleanup */1661 efct_hw_io_free_common(hw, io);1662 1663 spin_lock_irqsave(&hw->io_lock, flags);1664 /* remove from in-use list */1665 if (!list_empty(&io->list_entry) && !list_empty(&hw->io_inuse)) {1666 list_del_init(&io->list_entry);1667 efct_hw_io_free_move_correct_list(hw, io);1668 }1669 spin_unlock_irqrestore(&hw->io_lock, flags);1670}1671 1672int1673efct_hw_io_free(struct efct_hw *hw, struct efct_hw_io *io)1674{1675 return kref_put(&io->ref, io->release);1676}1677 1678struct efct_hw_io *1679efct_hw_io_lookup(struct efct_hw *hw, u32 xri)1680{1681 u32 ioindex;1682 1683 ioindex = xri - hw->sli.ext[SLI4_RSRC_XRI].base[0];1684 return hw->io[ioindex];1685}1686 1687int1688efct_hw_io_init_sges(struct efct_hw *hw, struct efct_hw_io *io,1689 enum efct_hw_io_type type)1690{1691 struct sli4_sge *data = NULL;1692 u32 i = 0;1693 u32 skips = 0;1694 u32 sge_flags = 0;1695 1696 if (!io) {1697 efc_log_err(hw->os, "bad parameter hw=%p io=%p\n", hw, io);1698 return -EIO;1699 }1700 1701 /* Clear / reset the scatter-gather list */1702 io->sgl = &io->def_sgl;1703 io->sgl_count = io->def_sgl_count;1704 io->first_data_sge = 0;1705 1706 memset(io->sgl->virt, 0, 2 * sizeof(struct sli4_sge));1707 io->n_sge = 0;1708 io->sge_offset = 0;1709 1710 io->type = type;1711 1712 data = io->sgl->virt;1713 1714 /*1715 * Some IO types have underlying hardware requirements on the order1716 * of SGEs. Process all special entries here.1717 */1718 switch (type) {1719 case EFCT_HW_IO_TARGET_WRITE:1720 1721 /* populate host resident XFER_RDY buffer */1722 sge_flags = le32_to_cpu(data->dw2_flags);1723 sge_flags &= (~SLI4_SGE_TYPE_MASK);1724 sge_flags |= (SLI4_SGE_TYPE_DATA << SLI4_SGE_TYPE_SHIFT);1725 data->buffer_address_high =1726 cpu_to_le32(upper_32_bits(io->xfer_rdy.phys));1727 data->buffer_address_low =1728 cpu_to_le32(lower_32_bits(io->xfer_rdy.phys));1729 data->buffer_length = cpu_to_le32(io->xfer_rdy.size);1730 data->dw2_flags = cpu_to_le32(sge_flags);1731 data++;1732 1733 skips = EFCT_TARGET_WRITE_SKIPS;1734 1735 io->n_sge = 1;1736 break;1737 case EFCT_HW_IO_TARGET_READ:1738 /*1739 * For FCP_TSEND64, the first 2 entries are SKIP SGE's1740 */1741 skips = EFCT_TARGET_READ_SKIPS;1742 break;1743 case EFCT_HW_IO_TARGET_RSP:1744 /*1745 * No skips, etc. for FCP_TRSP641746 */1747 break;1748 default:1749 efc_log_err(hw->os, "unsupported IO type %#x\n", type);1750 return -EIO;1751 }1752 1753 /*1754 * Write skip entries1755 */1756 for (i = 0; i < skips; i++) {1757 sge_flags = le32_to_cpu(data->dw2_flags);1758 sge_flags &= (~SLI4_SGE_TYPE_MASK);1759 sge_flags |= (SLI4_SGE_TYPE_SKIP << SLI4_SGE_TYPE_SHIFT);1760 data->dw2_flags = cpu_to_le32(sge_flags);1761 data++;1762 }1763 1764 io->n_sge += skips;1765 1766 /*1767 * Set last1768 */1769 sge_flags = le32_to_cpu(data->dw2_flags);1770 sge_flags |= SLI4_SGE_LAST;1771 data->dw2_flags = cpu_to_le32(sge_flags);1772 1773 return 0;1774}1775 1776int1777efct_hw_io_add_sge(struct efct_hw *hw, struct efct_hw_io *io,1778 uintptr_t addr, u32 length)1779{1780 struct sli4_sge *data = NULL;1781 u32 sge_flags = 0;1782 1783 if (!io || !addr || !length) {1784 efc_log_err(hw->os,1785 "bad parameter hw=%p io=%p addr=%lx length=%u\n",1786 hw, io, addr, length);1787 return -EIO;1788 }1789 1790 if (length > hw->sli.sge_supported_length) {1791 efc_log_err(hw->os,1792 "length of SGE %d bigger than allowed %d\n",1793 length, hw->sli.sge_supported_length);1794 return -EIO;1795 }1796 1797 data = io->sgl->virt;1798 data += io->n_sge;1799 1800 sge_flags = le32_to_cpu(data->dw2_flags);1801 sge_flags &= ~SLI4_SGE_TYPE_MASK;1802 sge_flags |= SLI4_SGE_TYPE_DATA << SLI4_SGE_TYPE_SHIFT;1803 sge_flags &= ~SLI4_SGE_DATA_OFFSET_MASK;1804 sge_flags |= SLI4_SGE_DATA_OFFSET_MASK & io->sge_offset;1805 1806 data->buffer_address_high = cpu_to_le32(upper_32_bits(addr));1807 data->buffer_address_low = cpu_to_le32(lower_32_bits(addr));1808 data->buffer_length = cpu_to_le32(length);1809 1810 /*1811 * Always assume this is the last entry and mark as such.1812 * If this is not the first entry unset the "last SGE"1813 * indication for the previous entry1814 */1815 sge_flags |= SLI4_SGE_LAST;1816 data->dw2_flags = cpu_to_le32(sge_flags);1817 1818 if (io->n_sge) {1819 sge_flags = le32_to_cpu(data[-1].dw2_flags);1820 sge_flags &= ~SLI4_SGE_LAST;1821 data[-1].dw2_flags = cpu_to_le32(sge_flags);1822 }1823 1824 /* Set first_data_bde if not previously set */1825 if (io->first_data_sge == 0)1826 io->first_data_sge = io->n_sge;1827 1828 io->sge_offset += length;1829 io->n_sge++;1830 1831 return 0;1832}1833 1834void1835efct_hw_io_abort_all(struct efct_hw *hw)1836{1837 struct efct_hw_io *io_to_abort = NULL;1838 struct efct_hw_io *next_io = NULL;1839 1840 list_for_each_entry_safe(io_to_abort, next_io,1841 &hw->io_inuse, list_entry) {1842 efct_hw_io_abort(hw, io_to_abort, true, NULL, NULL);1843 }1844}1845 1846static void1847efct_hw_wq_process_abort(void *arg, u8 *cqe, int status)1848{1849 struct efct_hw_io *io = arg;1850 struct efct_hw *hw = io->hw;1851 u32 ext = 0;1852 u32 len = 0;1853 struct hw_wq_callback *wqcb;1854 1855 /*1856 * For IOs that were aborted internally, we may need to issue the1857 * callback here depending on whether a XRI_ABORTED CQE is expected ot1858 * not. If the status is Local Reject/No XRI, then1859 * issue the callback now.1860 */1861 ext = sli_fc_ext_status(&hw->sli, cqe);1862 if (status == SLI4_FC_WCQE_STATUS_LOCAL_REJECT &&1863 ext == SLI4_FC_LOCAL_REJECT_NO_XRI && io->done) {1864 efct_hw_done_t done = io->done;1865 1866 io->done = NULL;1867 1868 /*1869 * Use latched status as this is always saved for an internal1870 * abort Note: We won't have both a done and abort_done1871 * function, so don't worry about1872 * clobbering the len, status and ext fields.1873 */1874 status = io->saved_status;1875 len = io->saved_len;1876 ext = io->saved_ext;1877 io->status_saved = false;1878 done(io, len, status, ext, io->arg);1879 }1880 1881 if (io->abort_done) {1882 efct_hw_done_t done = io->abort_done;1883 1884 io->abort_done = NULL;1885 done(io, len, status, ext, io->abort_arg);1886 }1887 1888 /* clear abort bit to indicate abort is complete */1889 io->abort_in_progress = false;1890 1891 /* Free the WQ callback */1892 if (io->abort_reqtag == U32_MAX) {1893 efc_log_err(hw->os, "HW IO already freed\n");1894 return;1895 }1896 1897 wqcb = efct_hw_reqtag_get_instance(hw, io->abort_reqtag);1898 efct_hw_reqtag_free(hw, wqcb);1899 1900 /*1901 * Call efct_hw_io_free() because this releases the WQ reservation as1902 * well as doing the refcount put. Don't duplicate the code here.1903 */1904 (void)efct_hw_io_free(hw, io);1905}1906 1907static void1908efct_hw_fill_abort_wqe(struct efct_hw *hw, struct efct_hw_wqe *wqe)1909{1910 struct sli4_abort_wqe *abort = (void *)wqe->wqebuf;1911 1912 memset(abort, 0, hw->sli.wqe_size);1913 1914 abort->criteria = SLI4_ABORT_CRITERIA_XRI_TAG;1915 abort->ia_ir_byte |= wqe->send_abts ? 0 : 1;1916 1917 /* Suppress ABTS retries */1918 abort->ia_ir_byte |= SLI4_ABRT_WQE_IR;1919 1920 abort->t_tag = cpu_to_le32(wqe->id);1921 abort->command = SLI4_WQE_ABORT;1922 abort->request_tag = cpu_to_le16(wqe->abort_reqtag);1923 1924 abort->dw10w0_flags = cpu_to_le16(SLI4_ABRT_WQE_QOSD);1925 1926 abort->cq_id = cpu_to_le16(SLI4_CQ_DEFAULT);1927}1928 1929int1930efct_hw_io_abort(struct efct_hw *hw, struct efct_hw_io *io_to_abort,1931 bool send_abts, void *cb, void *arg)1932{1933 struct hw_wq_callback *wqcb;1934 unsigned long flags = 0;1935 1936 if (!io_to_abort) {1937 efc_log_err(hw->os, "bad parameter hw=%p io=%p\n",1938 hw, io_to_abort);1939 return -EIO;1940 }1941 1942 if (hw->state != EFCT_HW_STATE_ACTIVE) {1943 efc_log_err(hw->os, "cannot send IO abort, HW state=%d\n",1944 hw->state);1945 return -EIO;1946 }1947 1948 /* take a reference on IO being aborted */1949 if (kref_get_unless_zero(&io_to_abort->ref) == 0) {1950 /* command no longer active */1951 efc_log_debug(hw->os,1952 "io not active xri=0x%x tag=0x%x\n",1953 io_to_abort->indicator, io_to_abort->reqtag);1954 return -ENOENT;1955 }1956 1957 /* Must have a valid WQ reference */1958 if (!io_to_abort->wq) {1959 efc_log_debug(hw->os, "io_to_abort xri=0x%x not active on WQ\n",1960 io_to_abort->indicator);1961 /* efct_ref_get(): same function */1962 kref_put(&io_to_abort->ref, io_to_abort->release);1963 return -ENOENT;1964 }1965 1966 /*1967 * Validation checks complete; now check to see if already being1968 * aborted, if not set the flag.1969 */1970 if (cmpxchg(&io_to_abort->abort_in_progress, false, true)) {1971 /* efct_ref_get(): same function */1972 kref_put(&io_to_abort->ref, io_to_abort->release);1973 efc_log_debug(hw->os,1974 "io already being aborted xri=0x%x tag=0x%x\n",1975 io_to_abort->indicator, io_to_abort->reqtag);1976 return -EINPROGRESS;1977 }1978 1979 /*1980 * If we got here, the possibilities are:1981 * - host owned xri1982 * - io_to_abort->wq_index != U32_MAX1983 * - submit ABORT_WQE to same WQ1984 * - port owned xri:1985 * - rxri: io_to_abort->wq_index == U32_MAX1986 * - submit ABORT_WQE to any WQ1987 * - non-rxri1988 * - io_to_abort->index != U32_MAX1989 * - submit ABORT_WQE to same WQ1990 * - io_to_abort->index == U32_MAX1991 * - submit ABORT_WQE to any WQ1992 */1993 io_to_abort->abort_done = cb;1994 io_to_abort->abort_arg = arg;1995 1996 /* Allocate a request tag for the abort portion of this IO */1997 wqcb = efct_hw_reqtag_alloc(hw, efct_hw_wq_process_abort, io_to_abort);1998 if (!wqcb) {1999 efc_log_err(hw->os, "can't allocate request tag\n");2000 return -ENOSPC;2001 }2002 2003 io_to_abort->abort_reqtag = wqcb->instance_index;2004 io_to_abort->wqe.send_abts = send_abts;2005 io_to_abort->wqe.id = io_to_abort->indicator;2006 io_to_abort->wqe.abort_reqtag = io_to_abort->abort_reqtag;2007 2008 /*2009 * If the wqe is on the pending list, then set this wqe to be2010 * aborted when the IO's wqe is removed from the list.2011 */2012 if (io_to_abort->wq) {2013 spin_lock_irqsave(&io_to_abort->wq->queue->lock, flags);2014 if (io_to_abort->wqe.list_entry.next) {2015 io_to_abort->wqe.abort_wqe_submit_needed = true;2016 spin_unlock_irqrestore(&io_to_abort->wq->queue->lock,2017 flags);2018 return 0;2019 }2020 spin_unlock_irqrestore(&io_to_abort->wq->queue->lock, flags);2021 }2022 2023 efct_hw_fill_abort_wqe(hw, &io_to_abort->wqe);2024 2025 /* ABORT_WQE does not actually utilize an XRI on the Port,2026 * therefore, keep xbusy as-is to track the exchange's state,2027 * not the ABORT_WQE's state2028 */2029 if (efct_hw_wq_write(io_to_abort->wq, &io_to_abort->wqe)) {2030 io_to_abort->abort_in_progress = false;2031 /* efct_ref_get(): same function */2032 kref_put(&io_to_abort->ref, io_to_abort->release);2033 return -EIO;2034 }2035 2036 return 0;2037}2038 2039void2040efct_hw_reqtag_pool_free(struct efct_hw *hw)2041{2042 u32 i;2043 struct reqtag_pool *reqtag_pool = hw->wq_reqtag_pool;2044 struct hw_wq_callback *wqcb = NULL;2045 2046 if (reqtag_pool) {2047 for (i = 0; i < U16_MAX; i++) {2048 wqcb = reqtag_pool->tags[i];2049 if (!wqcb)2050 continue;2051 2052 kfree(wqcb);2053 }2054 kfree(reqtag_pool);2055 hw->wq_reqtag_pool = NULL;2056 }2057}2058 2059struct reqtag_pool *2060efct_hw_reqtag_pool_alloc(struct efct_hw *hw)2061{2062 u32 i = 0;2063 struct reqtag_pool *reqtag_pool;2064 struct hw_wq_callback *wqcb;2065 2066 reqtag_pool = kzalloc(sizeof(*reqtag_pool), GFP_KERNEL);2067 if (!reqtag_pool)2068 return NULL;2069 2070 INIT_LIST_HEAD(&reqtag_pool->freelist);2071 /* initialize reqtag pool lock */2072 spin_lock_init(&reqtag_pool->lock);2073 for (i = 0; i < U16_MAX; i++) {2074 wqcb = kmalloc(sizeof(*wqcb), GFP_KERNEL);2075 if (!wqcb)2076 break;2077 2078 reqtag_pool->tags[i] = wqcb;2079 wqcb->instance_index = i;2080 wqcb->callback = NULL;2081 wqcb->arg = NULL;2082 INIT_LIST_HEAD(&wqcb->list_entry);2083 list_add_tail(&wqcb->list_entry, &reqtag_pool->freelist);2084 }2085 2086 return reqtag_pool;2087}2088 2089struct hw_wq_callback *2090efct_hw_reqtag_alloc(struct efct_hw *hw,2091 void (*callback)(void *arg, u8 *cqe, int status),2092 void *arg)2093{2094 struct hw_wq_callback *wqcb = NULL;2095 struct reqtag_pool *reqtag_pool = hw->wq_reqtag_pool;2096 unsigned long flags = 0;2097 2098 if (!callback)2099 return wqcb;2100 2101 spin_lock_irqsave(&reqtag_pool->lock, flags);2102 2103 if (!list_empty(&reqtag_pool->freelist)) {2104 wqcb = list_first_entry(&reqtag_pool->freelist,2105 struct hw_wq_callback, list_entry);2106 }2107 2108 if (wqcb) {2109 list_del_init(&wqcb->list_entry);2110 spin_unlock_irqrestore(&reqtag_pool->lock, flags);2111 wqcb->callback = callback;2112 wqcb->arg = arg;2113 } else {2114 spin_unlock_irqrestore(&reqtag_pool->lock, flags);2115 }2116 2117 return wqcb;2118}2119 2120void2121efct_hw_reqtag_free(struct efct_hw *hw, struct hw_wq_callback *wqcb)2122{2123 unsigned long flags = 0;2124 struct reqtag_pool *reqtag_pool = hw->wq_reqtag_pool;2125 2126 if (!wqcb->callback)2127 efc_log_err(hw->os, "WQCB is already freed\n");2128 2129 spin_lock_irqsave(&reqtag_pool->lock, flags);2130 wqcb->callback = NULL;2131 wqcb->arg = NULL;2132 INIT_LIST_HEAD(&wqcb->list_entry);2133 list_add(&wqcb->list_entry, &hw->wq_reqtag_pool->freelist);2134 spin_unlock_irqrestore(&reqtag_pool->lock, flags);2135}2136 2137struct hw_wq_callback *2138efct_hw_reqtag_get_instance(struct efct_hw *hw, u32 instance_index)2139{2140 struct hw_wq_callback *wqcb;2141 2142 wqcb = hw->wq_reqtag_pool->tags[instance_index];2143 if (!wqcb)2144 efc_log_err(hw->os, "wqcb for instance %d is null\n",2145 instance_index);2146 2147 return wqcb;2148}2149 2150int2151efct_hw_queue_hash_find(struct efct_queue_hash *hash, u16 id)2152{2153 int index = -1;2154 int i = id & (EFCT_HW_Q_HASH_SIZE - 1);2155 2156 /*2157 * Since the hash is always bigger than the maximum number of Qs, then2158 * we never have to worry about an infinite loop. We will always find2159 * an unused entry.2160 */2161 do {2162 if (hash[i].in_use && hash[i].id == id)2163 index = hash[i].index;2164 else2165 i = (i + 1) & (EFCT_HW_Q_HASH_SIZE - 1);2166 } while (index == -1 && hash[i].in_use);2167 2168 return index;2169}2170 2171int2172efct_hw_process(struct efct_hw *hw, u32 vector,2173 u32 max_isr_time_msec)2174{2175 struct hw_eq *eq;2176 2177 /*2178 * The caller should disable interrupts if they wish to prevent us2179 * from processing during a shutdown. The following states are defined:2180 * EFCT_HW_STATE_UNINITIALIZED - No queues allocated2181 * EFCT_HW_STATE_QUEUES_ALLOCATED - The state after a chip reset,2182 * queues are cleared.2183 * EFCT_HW_STATE_ACTIVE - Chip and queues are operational2184 * EFCT_HW_STATE_RESET_IN_PROGRESS - reset, we still want completions2185 * EFCT_HW_STATE_TEARDOWN_IN_PROGRESS - We still want mailbox2186 * completions.2187 */2188 if (hw->state == EFCT_HW_STATE_UNINITIALIZED)2189 return 0;2190 2191 /* Get pointer to struct hw_eq */2192 eq = hw->hw_eq[vector];2193 if (!eq)2194 return 0;2195 2196 eq->use_count++;2197 2198 return efct_hw_eq_process(hw, eq, max_isr_time_msec);2199}2200 2201int2202efct_hw_eq_process(struct efct_hw *hw, struct hw_eq *eq,2203 u32 max_isr_time_msec)2204{2205 u8 eqe[sizeof(struct sli4_eqe)] = { 0 };2206 u32 tcheck_count;2207 u64 tstart;2208 u64 telapsed;2209 bool done = false;2210 2211 tcheck_count = EFCT_HW_TIMECHECK_ITERATIONS;2212 tstart = jiffies_to_msecs(jiffies);2213 2214 while (!done && !sli_eq_read(&hw->sli, eq->queue, eqe)) {2215 u16 cq_id = 0;2216 int rc;2217 2218 rc = sli_eq_parse(&hw->sli, eqe, &cq_id);2219 if (unlikely(rc)) {2220 if (rc == SLI4_EQE_STATUS_EQ_FULL) {2221 u32 i;2222 2223 /*2224 * Received a sentinel EQE indicating the2225 * EQ is full. Process all CQs2226 */2227 for (i = 0; i < hw->cq_count; i++)2228 efct_hw_cq_process(hw, hw->hw_cq[i]);2229 continue;2230 } else {2231 return rc;2232 }2233 } else {2234 int index;2235 2236 index = efct_hw_queue_hash_find(hw->cq_hash, cq_id);2237 2238 if (likely(index >= 0))2239 efct_hw_cq_process(hw, hw->hw_cq[index]);2240 else2241 efc_log_err(hw->os, "bad CQ_ID %#06x\n", cq_id);2242 }2243 2244 if (eq->queue->n_posted > eq->queue->posted_limit)2245 sli_queue_arm(&hw->sli, eq->queue, false);2246 2247 if (tcheck_count && (--tcheck_count == 0)) {2248 tcheck_count = EFCT_HW_TIMECHECK_ITERATIONS;2249 telapsed = jiffies_to_msecs(jiffies) - tstart;2250 if (telapsed >= max_isr_time_msec)2251 done = true;2252 }2253 }2254 sli_queue_eq_arm(&hw->sli, eq->queue, true);2255 2256 return 0;2257}2258 2259static int2260_efct_hw_wq_write(struct hw_wq *wq, struct efct_hw_wqe *wqe)2261{2262 int queue_rc;2263 2264 /* Every so often, set the wqec bit to generate comsummed completions */2265 if (wq->wqec_count)2266 wq->wqec_count--;2267 2268 if (wq->wqec_count == 0) {2269 struct sli4_generic_wqe *genwqe = (void *)wqe->wqebuf;2270 2271 genwqe->cmdtype_wqec_byte |= SLI4_GEN_WQE_WQEC;2272 wq->wqec_count = wq->wqec_set_count;2273 }2274 2275 /* Decrement WQ free count */2276 wq->free_count--;2277 2278 queue_rc = sli_wq_write(&wq->hw->sli, wq->queue, wqe->wqebuf);2279 2280 return (queue_rc < 0) ? -EIO : 0;2281}2282 2283static void2284hw_wq_submit_pending(struct hw_wq *wq, u32 update_free_count)2285{2286 struct efct_hw_wqe *wqe;2287 unsigned long flags = 0;2288 2289 spin_lock_irqsave(&wq->queue->lock, flags);2290 2291 /* Update free count with value passed in */2292 wq->free_count += update_free_count;2293 2294 while ((wq->free_count > 0) && (!list_empty(&wq->pending_list))) {2295 wqe = list_first_entry(&wq->pending_list,2296 struct efct_hw_wqe, list_entry);2297 list_del_init(&wqe->list_entry);2298 _efct_hw_wq_write(wq, wqe);2299 2300 if (wqe->abort_wqe_submit_needed) {2301 wqe->abort_wqe_submit_needed = false;2302 efct_hw_fill_abort_wqe(wq->hw, wqe);2303 INIT_LIST_HEAD(&wqe->list_entry);2304 list_add_tail(&wqe->list_entry, &wq->pending_list);2305 wq->wq_pending_count++;2306 }2307 }2308 2309 spin_unlock_irqrestore(&wq->queue->lock, flags);2310}2311 2312void2313efct_hw_cq_process(struct efct_hw *hw, struct hw_cq *cq)2314{2315 u8 cqe[sizeof(struct sli4_mcqe)];2316 u16 rid = U16_MAX;2317 /* completion type */2318 enum sli4_qentry ctype;2319 u32 n_processed = 0;2320 u32 tstart, telapsed;2321 2322 tstart = jiffies_to_msecs(jiffies);2323 2324 while (!sli_cq_read(&hw->sli, cq->queue, cqe)) {2325 int status;2326 2327 status = sli_cq_parse(&hw->sli, cq->queue, cqe, &ctype, &rid);2328 /*2329 * The sign of status is significant. If status is:2330 * == 0 : call completed correctly and2331 * the CQE indicated success2332 * > 0 : call completed correctly and2333 * the CQE indicated an error2334 * < 0 : call failed and no information is available about the2335 * CQE2336 */2337 if (status < 0) {2338 if (status == SLI4_MCQE_STATUS_NOT_COMPLETED)2339 /*2340 * Notification that an entry was consumed,2341 * but not completed2342 */2343 continue;2344 2345 break;2346 }2347 2348 switch (ctype) {2349 case SLI4_QENTRY_ASYNC:2350 sli_cqe_async(&hw->sli, cqe);2351 break;2352 case SLI4_QENTRY_MQ:2353 /*2354 * Process MQ entry. Note there is no way to determine2355 * the MQ_ID from the completion entry.2356 */2357 efct_hw_mq_process(hw, status, hw->mq);2358 break;2359 case SLI4_QENTRY_WQ:2360 efct_hw_wq_process(hw, cq, cqe, status, rid);2361 break;2362 case SLI4_QENTRY_WQ_RELEASE: {2363 u32 wq_id = rid;2364 int index;2365 struct hw_wq *wq = NULL;2366 2367 index = efct_hw_queue_hash_find(hw->wq_hash, wq_id);2368 2369 if (likely(index >= 0)) {2370 wq = hw->hw_wq[index];2371 } else {2372 efc_log_err(hw->os, "bad WQ_ID %#06x\n", wq_id);2373 break;2374 }2375 /* Submit any HW IOs that are on the WQ pending list */2376 hw_wq_submit_pending(wq, wq->wqec_set_count);2377 2378 break;2379 }2380 2381 case SLI4_QENTRY_RQ:2382 efct_hw_rqpair_process_rq(hw, cq, cqe);2383 break;2384 case SLI4_QENTRY_XABT: {2385 efct_hw_xabt_process(hw, cq, cqe, rid);2386 break;2387 }2388 default:2389 efc_log_debug(hw->os, "unhandled ctype=%#x rid=%#x\n",2390 ctype, rid);2391 break;2392 }2393 2394 n_processed++;2395 if (n_processed == cq->queue->proc_limit)2396 break;2397 2398 if (cq->queue->n_posted >= cq->queue->posted_limit)2399 sli_queue_arm(&hw->sli, cq->queue, false);2400 }2401 2402 sli_queue_arm(&hw->sli, cq->queue, true);2403 2404 if (n_processed > cq->queue->max_num_processed)2405 cq->queue->max_num_processed = n_processed;2406 telapsed = jiffies_to_msecs(jiffies) - tstart;2407 if (telapsed > cq->queue->max_process_time)2408 cq->queue->max_process_time = telapsed;2409}2410 2411void2412efct_hw_wq_process(struct efct_hw *hw, struct hw_cq *cq,2413 u8 *cqe, int status, u16 rid)2414{2415 struct hw_wq_callback *wqcb;2416 2417 if (rid == EFCT_HW_REQUE_XRI_REGTAG) {2418 if (status)2419 efc_log_err(hw->os, "reque xri failed, status = %d\n",2420 status);2421 return;2422 }2423 2424 wqcb = efct_hw_reqtag_get_instance(hw, rid);2425 if (!wqcb) {2426 efc_log_err(hw->os, "invalid request tag: x%x\n", rid);2427 return;2428 }2429 2430 if (!wqcb->callback) {2431 efc_log_err(hw->os, "wqcb callback is NULL\n");2432 return;2433 }2434 2435 (*wqcb->callback)(wqcb->arg, cqe, status);2436}2437 2438void2439efct_hw_xabt_process(struct efct_hw *hw, struct hw_cq *cq,2440 u8 *cqe, u16 rid)2441{2442 /* search IOs wait free list */2443 struct efct_hw_io *io = NULL;2444 unsigned long flags = 0;2445 2446 io = efct_hw_io_lookup(hw, rid);2447 if (!io) {2448 /* IO lookup failure should never happen */2449 efc_log_err(hw->os, "xabt io lookup failed rid=%#x\n", rid);2450 return;2451 }2452 2453 if (!io->xbusy)2454 efc_log_debug(hw->os, "xabt io not busy rid=%#x\n", rid);2455 else2456 /* mark IO as no longer busy */2457 io->xbusy = false;2458 2459 /*2460 * For IOs that were aborted internally, we need to issue any pending2461 * callback here.2462 */2463 if (io->done) {2464 efct_hw_done_t done = io->done;2465 void *arg = io->arg;2466 2467 /*2468 * Use latched status as this is always saved for an internal2469 * abort2470 */2471 int status = io->saved_status;2472 u32 len = io->saved_len;2473 u32 ext = io->saved_ext;2474 2475 io->done = NULL;2476 io->status_saved = false;2477 2478 done(io, len, status, ext, arg);2479 }2480 2481 spin_lock_irqsave(&hw->io_lock, flags);2482 if (io->state == EFCT_HW_IO_STATE_INUSE ||2483 io->state == EFCT_HW_IO_STATE_WAIT_FREE) {2484 /* if on wait_free list, caller has already freed IO;2485 * remove from wait_free list and add to free list.2486 * if on in-use list, already marked as no longer busy;2487 * just leave there and wait for caller to free.2488 */2489 if (io->state == EFCT_HW_IO_STATE_WAIT_FREE) {2490 io->state = EFCT_HW_IO_STATE_FREE;2491 list_del_init(&io->list_entry);2492 efct_hw_io_free_move_correct_list(hw, io);2493 }2494 }2495 spin_unlock_irqrestore(&hw->io_lock, flags);2496}2497 2498static int2499efct_hw_flush(struct efct_hw *hw)2500{2501 u32 i = 0;2502 2503 /* Process any remaining completions */2504 for (i = 0; i < hw->eq_count; i++)2505 efct_hw_process(hw, i, ~0);2506 2507 return 0;2508}2509 2510int2511efct_hw_wq_write(struct hw_wq *wq, struct efct_hw_wqe *wqe)2512{2513 int rc = 0;2514 unsigned long flags = 0;2515 2516 spin_lock_irqsave(&wq->queue->lock, flags);2517 if (list_empty(&wq->pending_list)) {2518 if (wq->free_count > 0) {2519 rc = _efct_hw_wq_write(wq, wqe);2520 } else {2521 INIT_LIST_HEAD(&wqe->list_entry);2522 list_add_tail(&wqe->list_entry, &wq->pending_list);2523 wq->wq_pending_count++;2524 }2525 2526 spin_unlock_irqrestore(&wq->queue->lock, flags);2527 return rc;2528 }2529 2530 INIT_LIST_HEAD(&wqe->list_entry);2531 list_add_tail(&wqe->list_entry, &wq->pending_list);2532 wq->wq_pending_count++;2533 while (wq->free_count > 0) {2534 wqe = list_first_entry(&wq->pending_list, struct efct_hw_wqe,2535 list_entry);2536 if (!wqe)2537 break;2538 2539 list_del_init(&wqe->list_entry);2540 rc = _efct_hw_wq_write(wq, wqe);2541 if (rc)2542 break;2543 2544 if (wqe->abort_wqe_submit_needed) {2545 wqe->abort_wqe_submit_needed = false;2546 efct_hw_fill_abort_wqe(wq->hw, wqe);2547 2548 INIT_LIST_HEAD(&wqe->list_entry);2549 list_add_tail(&wqe->list_entry, &wq->pending_list);2550 wq->wq_pending_count++;2551 }2552 }2553 2554 spin_unlock_irqrestore(&wq->queue->lock, flags);2555 2556 return rc;2557}2558 2559int2560efct_efc_bls_send(struct efc *efc, u32 type, struct sli_bls_params *bls)2561{2562 struct efct *efct = efc->base;2563 2564 return efct_hw_bls_send(efct, type, bls, NULL, NULL);2565}2566 2567int2568efct_hw_bls_send(struct efct *efct, u32 type, struct sli_bls_params *bls_params,2569 void *cb, void *arg)2570{2571 struct efct_hw *hw = &efct->hw;2572 struct efct_hw_io *hio;2573 struct sli_bls_payload bls;2574 int rc;2575 2576 if (hw->state != EFCT_HW_STATE_ACTIVE) {2577 efc_log_err(hw->os,2578 "cannot send BLS, HW state=%d\n", hw->state);2579 return -EIO;2580 }2581 2582 hio = efct_hw_io_alloc(hw);2583 if (!hio) {2584 efc_log_err(hw->os, "HIO allocation failed\n");2585 return -EIO;2586 }2587 2588 hio->done = cb;2589 hio->arg = arg;2590 2591 bls_params->xri = hio->indicator;2592 bls_params->tag = hio->reqtag;2593 2594 if (type == FC_RCTL_BA_ACC) {2595 hio->type = EFCT_HW_BLS_ACC;2596 bls.type = SLI4_SLI_BLS_ACC;2597 memcpy(&bls.u.acc, bls_params->payload, sizeof(bls.u.acc));2598 } else {2599 hio->type = EFCT_HW_BLS_RJT;2600 bls.type = SLI4_SLI_BLS_RJT;2601 memcpy(&bls.u.rjt, bls_params->payload, sizeof(bls.u.rjt));2602 }2603 2604 bls.ox_id = cpu_to_le16(bls_params->ox_id);2605 bls.rx_id = cpu_to_le16(bls_params->rx_id);2606 2607 if (sli_xmit_bls_rsp64_wqe(&hw->sli, hio->wqe.wqebuf,2608 &bls, bls_params)) {2609 efc_log_err(hw->os, "XMIT_BLS_RSP64 WQE error\n");2610 return -EIO;2611 }2612 2613 hio->xbusy = true;2614 2615 /*2616 * Add IO to active io wqe list before submitting, in case the2617 * wcqe processing preempts this thread.2618 */2619 hio->wq->use_count++;2620 rc = efct_hw_wq_write(hio->wq, &hio->wqe);2621 if (rc >= 0) {2622 /* non-negative return is success */2623 rc = 0;2624 } else {2625 /* failed to write wqe, remove from active wqe list */2626 efc_log_err(hw->os,2627 "sli_queue_write failed: %d\n", rc);2628 hio->xbusy = false;2629 }2630 2631 return rc;2632}2633 2634static int2635efct_els_ssrs_send_cb(struct efct_hw_io *hio, u32 length, int status,2636 u32 ext_status, void *arg)2637{2638 struct efc_disc_io *io = arg;2639 2640 efc_disc_io_complete(io, length, status, ext_status);2641 return 0;2642}2643 2644static inline void2645efct_fill_els_params(struct efc_disc_io *io, struct sli_els_params *params)2646{2647 u8 *cmd = io->req.virt;2648 2649 params->cmd = *cmd;2650 params->s_id = io->s_id;2651 params->d_id = io->d_id;2652 params->ox_id = io->iparam.els.ox_id;2653 params->rpi = io->rpi;2654 params->vpi = io->vpi;2655 params->rpi_registered = io->rpi_registered;2656 params->xmit_len = io->xmit_len;2657 params->rsp_len = io->rsp_len;2658 params->timeout = io->iparam.els.timeout;2659}2660 2661static inline void2662efct_fill_ct_params(struct efc_disc_io *io, struct sli_ct_params *params)2663{2664 params->r_ctl = io->iparam.ct.r_ctl;2665 params->type = io->iparam.ct.type;2666 params->df_ctl = io->iparam.ct.df_ctl;2667 params->d_id = io->d_id;2668 params->ox_id = io->iparam.ct.ox_id;2669 params->rpi = io->rpi;2670 params->vpi = io->vpi;2671 params->rpi_registered = io->rpi_registered;2672 params->xmit_len = io->xmit_len;2673 params->rsp_len = io->rsp_len;2674 params->timeout = io->iparam.ct.timeout;2675}2676 2677/**2678 * efct_els_hw_srrs_send() - Send a single request and response cmd.2679 * @efc: efc library structure2680 * @io: Discovery IO used to hold els and ct cmd context.2681 *2682 * This routine supports communication sequences consisting of a single2683 * request and single response between two endpoints. Examples include:2684 * - Sending an ELS request.2685 * - Sending an ELS response - To send an ELS response, the caller must provide2686 * the OX_ID from the received request.2687 * - Sending a FC Common Transport (FC-CT) request - To send a FC-CT request,2688 * the caller must provide the R_CTL, TYPE, and DF_CTL2689 * values to place in the FC frame header.2690 *2691 * Return: Status of the request.2692 */2693int2694efct_els_hw_srrs_send(struct efc *efc, struct efc_disc_io *io)2695{2696 struct efct *efct = efc->base;2697 struct efct_hw_io *hio;2698 struct efct_hw *hw = &efct->hw;2699 struct efc_dma *send = &io->req;2700 struct efc_dma *receive = &io->rsp;2701 struct sli4_sge *sge = NULL;2702 int rc = 0;2703 u32 len = io->xmit_len;2704 u32 sge0_flags;2705 u32 sge1_flags;2706 2707 hio = efct_hw_io_alloc(hw);2708 if (!hio) {2709 pr_err("HIO alloc failed\n");2710 return -EIO;2711 }2712 2713 if (hw->state != EFCT_HW_STATE_ACTIVE) {2714 efc_log_debug(hw->os,2715 "cannot send SRRS, HW state=%d\n", hw->state);2716 return -EIO;2717 }2718 2719 hio->done = efct_els_ssrs_send_cb;2720 hio->arg = io;2721 2722 sge = hio->sgl->virt;2723 2724 /* clear both SGE */2725 memset(hio->sgl->virt, 0, 2 * sizeof(struct sli4_sge));2726 2727 sge0_flags = le32_to_cpu(sge[0].dw2_flags);2728 sge1_flags = le32_to_cpu(sge[1].dw2_flags);2729 if (send->size) {2730 sge[0].buffer_address_high =2731 cpu_to_le32(upper_32_bits(send->phys));2732 sge[0].buffer_address_low =2733 cpu_to_le32(lower_32_bits(send->phys));2734 2735 sge0_flags |= (SLI4_SGE_TYPE_DATA << SLI4_SGE_TYPE_SHIFT);2736 2737 sge[0].buffer_length = cpu_to_le32(len);2738 }2739 2740 if (io->io_type == EFC_DISC_IO_ELS_REQ ||2741 io->io_type == EFC_DISC_IO_CT_REQ) {2742 sge[1].buffer_address_high =2743 cpu_to_le32(upper_32_bits(receive->phys));2744 sge[1].buffer_address_low =2745 cpu_to_le32(lower_32_bits(receive->phys));2746 2747 sge1_flags |= (SLI4_SGE_TYPE_DATA << SLI4_SGE_TYPE_SHIFT);2748 sge1_flags |= SLI4_SGE_LAST;2749 2750 sge[1].buffer_length = cpu_to_le32(receive->size);2751 } else {2752 sge0_flags |= SLI4_SGE_LAST;2753 }2754 2755 sge[0].dw2_flags = cpu_to_le32(sge0_flags);2756 sge[1].dw2_flags = cpu_to_le32(sge1_flags);2757 2758 switch (io->io_type) {2759 case EFC_DISC_IO_ELS_REQ: {2760 struct sli_els_params els_params;2761 2762 hio->type = EFCT_HW_ELS_REQ;2763 efct_fill_els_params(io, &els_params);2764 els_params.xri = hio->indicator;2765 els_params.tag = hio->reqtag;2766 2767 if (sli_els_request64_wqe(&hw->sli, hio->wqe.wqebuf, hio->sgl,2768 &els_params)) {2769 efc_log_err(hw->os, "REQ WQE error\n");2770 rc = -EIO;2771 }2772 break;2773 }2774 case EFC_DISC_IO_ELS_RESP: {2775 struct sli_els_params els_params;2776 2777 hio->type = EFCT_HW_ELS_RSP;2778 efct_fill_els_params(io, &els_params);2779 els_params.xri = hio->indicator;2780 els_params.tag = hio->reqtag;2781 if (sli_xmit_els_rsp64_wqe(&hw->sli, hio->wqe.wqebuf, send,2782 &els_params)){2783 efc_log_err(hw->os, "RSP WQE error\n");2784 rc = -EIO;2785 }2786 break;2787 }2788 case EFC_DISC_IO_CT_REQ: {2789 struct sli_ct_params ct_params;2790 2791 hio->type = EFCT_HW_FC_CT;2792 efct_fill_ct_params(io, &ct_params);2793 ct_params.xri = hio->indicator;2794 ct_params.tag = hio->reqtag;2795 if (sli_gen_request64_wqe(&hw->sli, hio->wqe.wqebuf, hio->sgl,2796 &ct_params)){2797 efc_log_err(hw->os, "GEN WQE error\n");2798 rc = -EIO;2799 }2800 break;2801 }2802 case EFC_DISC_IO_CT_RESP: {2803 struct sli_ct_params ct_params;2804 2805 hio->type = EFCT_HW_FC_CT_RSP;2806 efct_fill_ct_params(io, &ct_params);2807 ct_params.xri = hio->indicator;2808 ct_params.tag = hio->reqtag;2809 if (sli_xmit_sequence64_wqe(&hw->sli, hio->wqe.wqebuf, hio->sgl,2810 &ct_params)){2811 efc_log_err(hw->os, "XMIT SEQ WQE error\n");2812 rc = -EIO;2813 }2814 break;2815 }2816 default:2817 efc_log_err(hw->os, "bad SRRS type %#x\n", io->io_type);2818 rc = -EIO;2819 }2820 2821 if (rc == 0) {2822 hio->xbusy = true;2823 2824 /*2825 * Add IO to active io wqe list before submitting, in case the2826 * wcqe processing preempts this thread.2827 */2828 hio->wq->use_count++;2829 rc = efct_hw_wq_write(hio->wq, &hio->wqe);2830 if (rc >= 0) {2831 /* non-negative return is success */2832 rc = 0;2833 } else {2834 /* failed to write wqe, remove from active wqe list */2835 efc_log_err(hw->os,2836 "sli_queue_write failed: %d\n", rc);2837 hio->xbusy = false;2838 }2839 }2840 2841 return rc;2842}2843 2844int2845efct_hw_io_send(struct efct_hw *hw, enum efct_hw_io_type type,2846 struct efct_hw_io *io, union efct_hw_io_param_u *iparam,2847 void *cb, void *arg)2848{2849 int rc = 0;2850 bool send_wqe = true;2851 2852 if (!io) {2853 pr_err("bad parm hw=%p io=%p\n", hw, io);2854 return -EIO;2855 }2856 2857 if (hw->state != EFCT_HW_STATE_ACTIVE) {2858 efc_log_err(hw->os, "cannot send IO, HW state=%d\n", hw->state);2859 return -EIO;2860 }2861 2862 /*2863 * Save state needed during later stages2864 */2865 io->type = type;2866 io->done = cb;2867 io->arg = arg;2868 2869 /*2870 * Format the work queue entry used to send the IO2871 */2872 switch (type) {2873 case EFCT_HW_IO_TARGET_WRITE: {2874 u16 *flags = &iparam->fcp_tgt.flags;2875 struct fcp_txrdy *xfer = io->xfer_rdy.virt;2876 2877 /*2878 * Fill in the XFER_RDY for IF_TYPE 0 devices2879 */2880 xfer->ft_data_ro = cpu_to_be32(iparam->fcp_tgt.offset);2881 xfer->ft_burst_len = cpu_to_be32(iparam->fcp_tgt.xmit_len);2882 2883 if (io->xbusy)2884 *flags |= SLI4_IO_CONTINUATION;2885 else2886 *flags &= ~SLI4_IO_CONTINUATION;2887 iparam->fcp_tgt.xri = io->indicator;2888 iparam->fcp_tgt.tag = io->reqtag;2889 2890 if (sli_fcp_treceive64_wqe(&hw->sli, io->wqe.wqebuf,2891 &io->def_sgl, io->first_data_sge,2892 SLI4_CQ_DEFAULT,2893 0, 0, &iparam->fcp_tgt)) {2894 efc_log_err(hw->os, "TRECEIVE WQE error\n");2895 rc = -EIO;2896 }2897 break;2898 }2899 case EFCT_HW_IO_TARGET_READ: {2900 u16 *flags = &iparam->fcp_tgt.flags;2901 2902 if (io->xbusy)2903 *flags |= SLI4_IO_CONTINUATION;2904 else2905 *flags &= ~SLI4_IO_CONTINUATION;2906 2907 iparam->fcp_tgt.xri = io->indicator;2908 iparam->fcp_tgt.tag = io->reqtag;2909 2910 if (sli_fcp_tsend64_wqe(&hw->sli, io->wqe.wqebuf,2911 &io->def_sgl, io->first_data_sge,2912 SLI4_CQ_DEFAULT,2913 0, 0, &iparam->fcp_tgt)) {2914 efc_log_err(hw->os, "TSEND WQE error\n");2915 rc = -EIO;2916 }2917 break;2918 }2919 case EFCT_HW_IO_TARGET_RSP: {2920 u16 *flags = &iparam->fcp_tgt.flags;2921 2922 if (io->xbusy)2923 *flags |= SLI4_IO_CONTINUATION;2924 else2925 *flags &= ~SLI4_IO_CONTINUATION;2926 2927 iparam->fcp_tgt.xri = io->indicator;2928 iparam->fcp_tgt.tag = io->reqtag;2929 2930 if (sli_fcp_trsp64_wqe(&hw->sli, io->wqe.wqebuf,2931 &io->def_sgl, SLI4_CQ_DEFAULT,2932 0, &iparam->fcp_tgt)) {2933 efc_log_err(hw->os, "TRSP WQE error\n");2934 rc = -EIO;2935 }2936 2937 break;2938 }2939 default:2940 efc_log_err(hw->os, "unsupported IO type %#x\n", type);2941 rc = -EIO;2942 }2943 2944 if (send_wqe && rc == 0) {2945 io->xbusy = true;2946 2947 /*2948 * Add IO to active io wqe list before submitting, in case the2949 * wcqe processing preempts this thread.2950 */2951 hw->tcmd_wq_submit[io->wq->instance]++;2952 io->wq->use_count++;2953 rc = efct_hw_wq_write(io->wq, &io->wqe);2954 if (rc >= 0) {2955 /* non-negative return is success */2956 rc = 0;2957 } else {2958 /* failed to write wqe, remove from active wqe list */2959 efc_log_err(hw->os,2960 "sli_queue_write failed: %d\n", rc);2961 io->xbusy = false;2962 }2963 }2964 2965 return rc;2966}2967 2968int2969efct_hw_send_frame(struct efct_hw *hw, struct fc_frame_header *hdr,2970 u8 sof, u8 eof, struct efc_dma *payload,2971 struct efct_hw_send_frame_context *ctx,2972 void (*callback)(void *arg, u8 *cqe, int status),2973 void *arg)2974{2975 int rc;2976 struct efct_hw_wqe *wqe;2977 u32 xri;2978 struct hw_wq *wq;2979 2980 wqe = &ctx->wqe;2981 2982 /* populate the callback object */2983 ctx->hw = hw;2984 2985 /* Fetch and populate request tag */2986 ctx->wqcb = efct_hw_reqtag_alloc(hw, callback, arg);2987 if (!ctx->wqcb) {2988 efc_log_err(hw->os, "can't allocate request tag\n");2989 return -ENOSPC;2990 }2991 2992 wq = hw->hw_wq[0];2993 2994 /* Set XRI and RX_ID in the header based on which WQ, and which2995 * send_frame_io we are using2996 */2997 xri = wq->send_frame_io->indicator;2998 2999 /* Build the send frame WQE */3000 rc = sli_send_frame_wqe(&hw->sli, wqe->wqebuf,3001 sof, eof, (u32 *)hdr, payload, payload->len,3002 EFCT_HW_SEND_FRAME_TIMEOUT, xri,3003 ctx->wqcb->instance_index);3004 if (rc) {3005 efc_log_err(hw->os, "sli_send_frame_wqe failed: %d\n", rc);3006 return -EIO;3007 }3008 3009 /* Write to WQ */3010 rc = efct_hw_wq_write(wq, wqe);3011 if (rc) {3012 efc_log_err(hw->os, "efct_hw_wq_write failed: %d\n", rc);3013 return -EIO;3014 }3015 3016 wq->use_count++;3017 3018 return 0;3019}3020 3021static int3022efct_hw_cb_link_stat(struct efct_hw *hw, int status,3023 u8 *mqe, void *arg)3024{3025 struct sli4_cmd_read_link_stats *mbox_rsp;3026 struct efct_hw_link_stat_cb_arg *cb_arg = arg;3027 struct efct_hw_link_stat_counts counts[EFCT_HW_LINK_STAT_MAX];3028 u32 num_counters, i;3029 u32 mbox_rsp_flags = 0;3030 3031 mbox_rsp = (struct sli4_cmd_read_link_stats *)mqe;3032 mbox_rsp_flags = le32_to_cpu(mbox_rsp->dw1_flags);3033 num_counters = (mbox_rsp_flags & SLI4_READ_LNKSTAT_GEC) ? 20 : 13;3034 memset(counts, 0, sizeof(struct efct_hw_link_stat_counts) *3035 EFCT_HW_LINK_STAT_MAX);3036 3037 /* Fill overflow counts, mask starts from SLI4_READ_LNKSTAT_W02OF*/3038 for (i = 0; i < EFCT_HW_LINK_STAT_MAX; i++)3039 counts[i].overflow = (mbox_rsp_flags & (1 << (i + 2)));3040 3041 counts[EFCT_HW_LINK_STAT_LINK_FAILURE_COUNT].counter =3042 le32_to_cpu(mbox_rsp->linkfail_errcnt);3043 counts[EFCT_HW_LINK_STAT_LOSS_OF_SYNC_COUNT].counter =3044 le32_to_cpu(mbox_rsp->losssync_errcnt);3045 counts[EFCT_HW_LINK_STAT_LOSS_OF_SIGNAL_COUNT].counter =3046 le32_to_cpu(mbox_rsp->losssignal_errcnt);3047 counts[EFCT_HW_LINK_STAT_PRIMITIVE_SEQ_COUNT].counter =3048 le32_to_cpu(mbox_rsp->primseq_errcnt);3049 counts[EFCT_HW_LINK_STAT_INVALID_XMIT_WORD_COUNT].counter =3050 le32_to_cpu(mbox_rsp->inval_txword_errcnt);3051 counts[EFCT_HW_LINK_STAT_CRC_COUNT].counter =3052 le32_to_cpu(mbox_rsp->crc_errcnt);3053 counts[EFCT_HW_LINK_STAT_PRIMITIVE_SEQ_TIMEOUT_COUNT].counter =3054 le32_to_cpu(mbox_rsp->primseq_eventtimeout_cnt);3055 counts[EFCT_HW_LINK_STAT_ELASTIC_BUFFER_OVERRUN_COUNT].counter =3056 le32_to_cpu(mbox_rsp->elastic_bufoverrun_errcnt);3057 counts[EFCT_HW_LINK_STAT_ARB_TIMEOUT_COUNT].counter =3058 le32_to_cpu(mbox_rsp->arbit_fc_al_timeout_cnt);3059 counts[EFCT_HW_LINK_STAT_ADVERTISED_RCV_B2B_CREDIT].counter =3060 le32_to_cpu(mbox_rsp->adv_rx_buftor_to_buf_credit);3061 counts[EFCT_HW_LINK_STAT_CURR_RCV_B2B_CREDIT].counter =3062 le32_to_cpu(mbox_rsp->curr_rx_buf_to_buf_credit);3063 counts[EFCT_HW_LINK_STAT_ADVERTISED_XMIT_B2B_CREDIT].counter =3064 le32_to_cpu(mbox_rsp->adv_tx_buf_to_buf_credit);3065 counts[EFCT_HW_LINK_STAT_CURR_XMIT_B2B_CREDIT].counter =3066 le32_to_cpu(mbox_rsp->curr_tx_buf_to_buf_credit);3067 counts[EFCT_HW_LINK_STAT_RCV_EOFA_COUNT].counter =3068 le32_to_cpu(mbox_rsp->rx_eofa_cnt);3069 counts[EFCT_HW_LINK_STAT_RCV_EOFDTI_COUNT].counter =3070 le32_to_cpu(mbox_rsp->rx_eofdti_cnt);3071 counts[EFCT_HW_LINK_STAT_RCV_EOFNI_COUNT].counter =3072 le32_to_cpu(mbox_rsp->rx_eofni_cnt);3073 counts[EFCT_HW_LINK_STAT_RCV_SOFF_COUNT].counter =3074 le32_to_cpu(mbox_rsp->rx_soff_cnt);3075 counts[EFCT_HW_LINK_STAT_RCV_DROPPED_NO_AER_COUNT].counter =3076 le32_to_cpu(mbox_rsp->rx_dropped_no_aer_cnt);3077 counts[EFCT_HW_LINK_STAT_RCV_DROPPED_NO_RPI_COUNT].counter =3078 le32_to_cpu(mbox_rsp->rx_dropped_no_avail_rpi_rescnt);3079 counts[EFCT_HW_LINK_STAT_RCV_DROPPED_NO_XRI_COUNT].counter =3080 le32_to_cpu(mbox_rsp->rx_dropped_no_avail_xri_rescnt);3081 3082 if (cb_arg) {3083 if (cb_arg->cb) {3084 if (status == 0 && le16_to_cpu(mbox_rsp->hdr.status))3085 status = le16_to_cpu(mbox_rsp->hdr.status);3086 cb_arg->cb(status, num_counters, counts, cb_arg->arg);3087 }3088 3089 kfree(cb_arg);3090 }3091 3092 return 0;3093}3094 3095int3096efct_hw_get_link_stats(struct efct_hw *hw, u8 req_ext_counters,3097 u8 clear_overflow_flags, u8 clear_all_counters,3098 void (*cb)(int status, u32 num_counters,3099 struct efct_hw_link_stat_counts *counters,3100 void *arg),3101 void *arg)3102{3103 int rc = -EIO;3104 struct efct_hw_link_stat_cb_arg *cb_arg;3105 u8 mbxdata[SLI4_BMBX_SIZE];3106 3107 cb_arg = kzalloc(sizeof(*cb_arg), GFP_ATOMIC);3108 if (!cb_arg)3109 return -ENOMEM;3110 3111 cb_arg->cb = cb;3112 cb_arg->arg = arg;3113 3114 /* Send the HW command */3115 if (!sli_cmd_read_link_stats(&hw->sli, mbxdata, req_ext_counters,3116 clear_overflow_flags, clear_all_counters))3117 rc = efct_hw_command(hw, mbxdata, EFCT_CMD_NOWAIT,3118 efct_hw_cb_link_stat, cb_arg);3119 3120 if (rc)3121 kfree(cb_arg);3122 3123 return rc;3124}3125 3126static int3127efct_hw_cb_host_stat(struct efct_hw *hw, int status, u8 *mqe, void *arg)3128{3129 struct sli4_cmd_read_status *mbox_rsp =3130 (struct sli4_cmd_read_status *)mqe;3131 struct efct_hw_host_stat_cb_arg *cb_arg = arg;3132 struct efct_hw_host_stat_counts counts[EFCT_HW_HOST_STAT_MAX];3133 u32 num_counters = EFCT_HW_HOST_STAT_MAX;3134 3135 memset(counts, 0, sizeof(struct efct_hw_host_stat_counts) *3136 EFCT_HW_HOST_STAT_MAX);3137 3138 counts[EFCT_HW_HOST_STAT_TX_KBYTE_COUNT].counter =3139 le32_to_cpu(mbox_rsp->trans_kbyte_cnt);3140 counts[EFCT_HW_HOST_STAT_RX_KBYTE_COUNT].counter =3141 le32_to_cpu(mbox_rsp->recv_kbyte_cnt);3142 counts[EFCT_HW_HOST_STAT_TX_FRAME_COUNT].counter =3143 le32_to_cpu(mbox_rsp->trans_frame_cnt);3144 counts[EFCT_HW_HOST_STAT_RX_FRAME_COUNT].counter =3145 le32_to_cpu(mbox_rsp->recv_frame_cnt);3146 counts[EFCT_HW_HOST_STAT_TX_SEQ_COUNT].counter =3147 le32_to_cpu(mbox_rsp->trans_seq_cnt);3148 counts[EFCT_HW_HOST_STAT_RX_SEQ_COUNT].counter =3149 le32_to_cpu(mbox_rsp->recv_seq_cnt);3150 counts[EFCT_HW_HOST_STAT_TOTAL_EXCH_ORIG].counter =3151 le32_to_cpu(mbox_rsp->tot_exchanges_orig);3152 counts[EFCT_HW_HOST_STAT_TOTAL_EXCH_RESP].counter =3153 le32_to_cpu(mbox_rsp->tot_exchanges_resp);3154 counts[EFCT_HW_HOSY_STAT_RX_P_BSY_COUNT].counter =3155 le32_to_cpu(mbox_rsp->recv_p_bsy_cnt);3156 counts[EFCT_HW_HOST_STAT_RX_F_BSY_COUNT].counter =3157 le32_to_cpu(mbox_rsp->recv_f_bsy_cnt);3158 counts[EFCT_HW_HOST_STAT_DROP_FRM_DUE_TO_NO_RQ_BUF_COUNT].counter =3159 le32_to_cpu(mbox_rsp->no_rq_buf_dropped_frames_cnt);3160 counts[EFCT_HW_HOST_STAT_EMPTY_RQ_TIMEOUT_COUNT].counter =3161 le32_to_cpu(mbox_rsp->empty_rq_timeout_cnt);3162 counts[EFCT_HW_HOST_STAT_DROP_FRM_DUE_TO_NO_XRI_COUNT].counter =3163 le32_to_cpu(mbox_rsp->no_xri_dropped_frames_cnt);3164 counts[EFCT_HW_HOST_STAT_EMPTY_XRI_POOL_COUNT].counter =3165 le32_to_cpu(mbox_rsp->empty_xri_pool_cnt);3166 3167 if (cb_arg) {3168 if (cb_arg->cb) {3169 if (status == 0 && le16_to_cpu(mbox_rsp->hdr.status))3170 status = le16_to_cpu(mbox_rsp->hdr.status);3171 cb_arg->cb(status, num_counters, counts, cb_arg->arg);3172 }3173 3174 kfree(cb_arg);3175 }3176 3177 return 0;3178}3179 3180int3181efct_hw_get_host_stats(struct efct_hw *hw, u8 cc,3182 void (*cb)(int status, u32 num_counters,3183 struct efct_hw_host_stat_counts *counters,3184 void *arg),3185 void *arg)3186{3187 int rc = -EIO;3188 struct efct_hw_host_stat_cb_arg *cb_arg;3189 u8 mbxdata[SLI4_BMBX_SIZE];3190 3191 cb_arg = kmalloc(sizeof(*cb_arg), GFP_ATOMIC);3192 if (!cb_arg)3193 return -ENOMEM;3194 3195 cb_arg->cb = cb;3196 cb_arg->arg = arg;3197 3198 /* Send the HW command to get the host stats */3199 if (!sli_cmd_read_status(&hw->sli, mbxdata, cc))3200 rc = efct_hw_command(hw, mbxdata, EFCT_CMD_NOWAIT,3201 efct_hw_cb_host_stat, cb_arg);3202 3203 if (rc) {3204 efc_log_debug(hw->os, "READ_HOST_STATS failed\n");3205 kfree(cb_arg);3206 }3207 3208 return rc;3209}3210 3211struct efct_hw_async_call_ctx {3212 efct_hw_async_cb_t callback;3213 void *arg;3214 u8 cmd[SLI4_BMBX_SIZE];3215};3216 3217static void3218efct_hw_async_cb(struct efct_hw *hw, int status, u8 *mqe, void *arg)3219{3220 struct efct_hw_async_call_ctx *ctx = arg;3221 3222 if (ctx) {3223 if (ctx->callback)3224 (*ctx->callback)(hw, status, mqe, ctx->arg);3225 3226 kfree(ctx);3227 }3228}3229 3230int3231efct_hw_async_call(struct efct_hw *hw, efct_hw_async_cb_t callback, void *arg)3232{3233 struct efct_hw_async_call_ctx *ctx;3234 int rc;3235 3236 /*3237 * Allocate a callback context (which includes the mbox cmd buffer),3238 * we need this to be persistent as the mbox cmd submission may be3239 * queued and executed later execution.3240 */3241 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);3242 if (!ctx)3243 return -ENOMEM;3244 3245 ctx->callback = callback;3246 ctx->arg = arg;3247 3248 /* Build and send a NOP mailbox command */3249 if (sli_cmd_common_nop(&hw->sli, ctx->cmd, 0)) {3250 efc_log_err(hw->os, "COMMON_NOP format failure\n");3251 kfree(ctx);3252 return -EIO;3253 }3254 3255 rc = efct_hw_command(hw, ctx->cmd, EFCT_CMD_NOWAIT, efct_hw_async_cb,3256 ctx);3257 if (rc) {3258 efc_log_err(hw->os, "COMMON_NOP command failure, rc=%d\n", rc);3259 kfree(ctx);3260 return -EIO;3261 }3262 return 0;3263}3264 3265static int3266efct_hw_cb_fw_write(struct efct_hw *hw, int status, u8 *mqe, void *arg)3267{3268 struct sli4_cmd_sli_config *mbox_rsp =3269 (struct sli4_cmd_sli_config *)mqe;3270 struct sli4_rsp_cmn_write_object *wr_obj_rsp;3271 struct efct_hw_fw_wr_cb_arg *cb_arg = arg;3272 u32 bytes_written;3273 u16 mbox_status;3274 u32 change_status;3275 3276 wr_obj_rsp = (struct sli4_rsp_cmn_write_object *)3277 &mbox_rsp->payload.embed;3278 bytes_written = le32_to_cpu(wr_obj_rsp->actual_write_length);3279 mbox_status = le16_to_cpu(mbox_rsp->hdr.status);3280 change_status = (le32_to_cpu(wr_obj_rsp->change_status_dword) &3281 RSP_CHANGE_STATUS);3282 3283 if (cb_arg) {3284 if (cb_arg->cb) {3285 if (!status && mbox_status)3286 status = mbox_status;3287 cb_arg->cb(status, bytes_written, change_status,3288 cb_arg->arg);3289 }3290 3291 kfree(cb_arg);3292 }3293 3294 return 0;3295}3296 3297int3298efct_hw_firmware_write(struct efct_hw *hw, struct efc_dma *dma, u32 size,3299 u32 offset, int last,3300 void (*cb)(int status, u32 bytes_written,3301 u32 change_status, void *arg),3302 void *arg)3303{3304 int rc = -EIO;3305 u8 mbxdata[SLI4_BMBX_SIZE];3306 struct efct_hw_fw_wr_cb_arg *cb_arg;3307 int noc = 0;3308 3309 cb_arg = kzalloc(sizeof(*cb_arg), GFP_KERNEL);3310 if (!cb_arg)3311 return -ENOMEM;3312 3313 cb_arg->cb = cb;3314 cb_arg->arg = arg;3315 3316 /* Write a portion of a firmware image to the device */3317 if (!sli_cmd_common_write_object(&hw->sli, mbxdata,3318 noc, last, size, offset, "/prg/",3319 dma))3320 rc = efct_hw_command(hw, mbxdata, EFCT_CMD_NOWAIT,3321 efct_hw_cb_fw_write, cb_arg);3322 3323 if (rc != 0) {3324 efc_log_debug(hw->os, "COMMON_WRITE_OBJECT failed\n");3325 kfree(cb_arg);3326 }3327 3328 return rc;3329}3330 3331static int3332efct_hw_cb_port_control(struct efct_hw *hw, int status, u8 *mqe,3333 void *arg)3334{3335 return 0;3336}3337 3338int3339efct_hw_port_control(struct efct_hw *hw, enum efct_hw_port ctrl,3340 uintptr_t value,3341 void (*cb)(int status, uintptr_t value, void *arg),3342 void *arg)3343{3344 int rc = -EIO;3345 u8 link[SLI4_BMBX_SIZE];3346 u32 speed = 0;3347 u8 reset_alpa = 0;3348 3349 switch (ctrl) {3350 case EFCT_HW_PORT_INIT:3351 if (!sli_cmd_config_link(&hw->sli, link))3352 rc = efct_hw_command(hw, link, EFCT_CMD_NOWAIT,3353 efct_hw_cb_port_control, NULL);3354 3355 if (rc != 0) {3356 efc_log_err(hw->os, "CONFIG_LINK failed\n");3357 break;3358 }3359 speed = hw->config.speed;3360 reset_alpa = (u8)(value & 0xff);3361 3362 rc = -EIO;3363 if (!sli_cmd_init_link(&hw->sli, link, speed, reset_alpa))3364 rc = efct_hw_command(hw, link, EFCT_CMD_NOWAIT,3365 efct_hw_cb_port_control, NULL);3366 /* Free buffer on error, since no callback is coming */3367 if (rc)3368 efc_log_err(hw->os, "INIT_LINK failed\n");3369 break;3370 3371 case EFCT_HW_PORT_SHUTDOWN:3372 if (!sli_cmd_down_link(&hw->sli, link))3373 rc = efct_hw_command(hw, link, EFCT_CMD_NOWAIT,3374 efct_hw_cb_port_control, NULL);3375 /* Free buffer on error, since no callback is coming */3376 if (rc)3377 efc_log_err(hw->os, "DOWN_LINK failed\n");3378 break;3379 3380 default:3381 efc_log_debug(hw->os, "unhandled control %#x\n", ctrl);3382 break;3383 }3384 3385 return rc;3386}3387 3388void3389efct_hw_teardown(struct efct_hw *hw)3390{3391 u32 i = 0;3392 u32 destroy_queues;3393 u32 free_memory;3394 struct efc_dma *dma;3395 struct efct *efct = hw->os;3396 3397 destroy_queues = (hw->state == EFCT_HW_STATE_ACTIVE);3398 free_memory = (hw->state != EFCT_HW_STATE_UNINITIALIZED);3399 3400 /* Cancel Sliport Healthcheck */3401 if (hw->sliport_healthcheck) {3402 hw->sliport_healthcheck = 0;3403 efct_hw_config_sli_port_health_check(hw, 0, 0);3404 }3405 3406 if (hw->state != EFCT_HW_STATE_QUEUES_ALLOCATED) {3407 hw->state = EFCT_HW_STATE_TEARDOWN_IN_PROGRESS;3408 3409 efct_hw_flush(hw);3410 3411 if (list_empty(&hw->cmd_head))3412 efc_log_debug(hw->os,3413 "All commands completed on MQ queue\n");3414 else3415 efc_log_debug(hw->os,3416 "Some cmds still pending on MQ queue\n");3417 3418 /* Cancel any remaining commands */3419 efct_hw_command_cancel(hw);3420 } else {3421 hw->state = EFCT_HW_STATE_TEARDOWN_IN_PROGRESS;3422 }3423 3424 dma_free_coherent(&efct->pci->dev,3425 hw->rnode_mem.size, hw->rnode_mem.virt,3426 hw->rnode_mem.phys);3427 memset(&hw->rnode_mem, 0, sizeof(struct efc_dma));3428 3429 if (hw->io) {3430 for (i = 0; i < hw->config.n_io; i++) {3431 if (hw->io[i] && hw->io[i]->sgl &&3432 hw->io[i]->sgl->virt) {3433 dma_free_coherent(&efct->pci->dev,3434 hw->io[i]->sgl->size,3435 hw->io[i]->sgl->virt,3436 hw->io[i]->sgl->phys);3437 }3438 kfree(hw->io[i]);3439 hw->io[i] = NULL;3440 }3441 kfree(hw->io);3442 hw->io = NULL;3443 kfree(hw->wqe_buffs);3444 hw->wqe_buffs = NULL;3445 }3446 3447 dma = &hw->xfer_rdy;3448 dma_free_coherent(&efct->pci->dev,3449 dma->size, dma->virt, dma->phys);3450 memset(dma, 0, sizeof(struct efc_dma));3451 3452 dma = &hw->loop_map;3453 dma_free_coherent(&efct->pci->dev,3454 dma->size, dma->virt, dma->phys);3455 memset(dma, 0, sizeof(struct efc_dma));3456 3457 for (i = 0; i < hw->wq_count; i++)3458 sli_queue_free(&hw->sli, &hw->wq[i], destroy_queues,3459 free_memory);3460 3461 for (i = 0; i < hw->rq_count; i++)3462 sli_queue_free(&hw->sli, &hw->rq[i], destroy_queues,3463 free_memory);3464 3465 for (i = 0; i < hw->mq_count; i++)3466 sli_queue_free(&hw->sli, &hw->mq[i], destroy_queues,3467 free_memory);3468 3469 for (i = 0; i < hw->cq_count; i++)3470 sli_queue_free(&hw->sli, &hw->cq[i], destroy_queues,3471 free_memory);3472 3473 for (i = 0; i < hw->eq_count; i++)3474 sli_queue_free(&hw->sli, &hw->eq[i], destroy_queues,3475 free_memory);3476 3477 /* Free rq buffers */3478 efct_hw_rx_free(hw);3479 3480 efct_hw_queue_teardown(hw);3481 3482 kfree(hw->wq_cpu_array);3483 3484 sli_teardown(&hw->sli);3485 3486 /* record the fact that the queues are non-functional */3487 hw->state = EFCT_HW_STATE_UNINITIALIZED;3488 3489 /* free sequence free pool */3490 kfree(hw->seq_pool);3491 hw->seq_pool = NULL;3492 3493 /* free hw_wq_callback pool */3494 efct_hw_reqtag_pool_free(hw);3495 3496 mempool_destroy(hw->cmd_ctx_pool);3497 mempool_destroy(hw->mbox_rqst_pool);3498 3499 /* Mark HW setup as not having been called */3500 hw->hw_setup_called = false;3501}3502 3503static int3504efct_hw_sli_reset(struct efct_hw *hw, enum efct_hw_reset reset,3505 enum efct_hw_state prev_state)3506{3507 int rc = 0;3508 3509 switch (reset) {3510 case EFCT_HW_RESET_FUNCTION:3511 efc_log_debug(hw->os, "issuing function level reset\n");3512 if (sli_reset(&hw->sli)) {3513 efc_log_err(hw->os, "sli_reset failed\n");3514 rc = -EIO;3515 }3516 break;3517 case EFCT_HW_RESET_FIRMWARE:3518 efc_log_debug(hw->os, "issuing firmware reset\n");3519 if (sli_fw_reset(&hw->sli)) {3520 efc_log_err(hw->os, "sli_soft_reset failed\n");3521 rc = -EIO;3522 }3523 /*3524 * Because the FW reset leaves the FW in a non-running state,3525 * follow that with a regular reset.3526 */3527 efc_log_debug(hw->os, "issuing function level reset\n");3528 if (sli_reset(&hw->sli)) {3529 efc_log_err(hw->os, "sli_reset failed\n");3530 rc = -EIO;3531 }3532 break;3533 default:3534 efc_log_err(hw->os, "unknown type - no reset performed\n");3535 hw->state = prev_state;3536 rc = -EINVAL;3537 break;3538 }3539 3540 return rc;3541}3542 3543int3544efct_hw_reset(struct efct_hw *hw, enum efct_hw_reset reset)3545{3546 int rc = 0;3547 enum efct_hw_state prev_state = hw->state;3548 3549 if (hw->state != EFCT_HW_STATE_ACTIVE)3550 efc_log_debug(hw->os,3551 "HW state %d is not active\n", hw->state);3552 3553 hw->state = EFCT_HW_STATE_RESET_IN_PROGRESS;3554 3555 /*3556 * If the prev_state is already reset/teardown in progress,3557 * don't continue further3558 */3559 if (prev_state == EFCT_HW_STATE_RESET_IN_PROGRESS ||3560 prev_state == EFCT_HW_STATE_TEARDOWN_IN_PROGRESS)3561 return efct_hw_sli_reset(hw, reset, prev_state);3562 3563 if (prev_state != EFCT_HW_STATE_UNINITIALIZED) {3564 efct_hw_flush(hw);3565 3566 if (list_empty(&hw->cmd_head))3567 efc_log_debug(hw->os,3568 "All commands completed on MQ queue\n");3569 else3570 efc_log_err(hw->os,3571 "Some commands still pending on MQ queue\n");3572 }3573 3574 /* Reset the chip */3575 rc = efct_hw_sli_reset(hw, reset, prev_state);3576 if (rc == -EINVAL)3577 return -EIO;3578 3579 return rc;3580}3581