1052 lines · c
1// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)2/* QLogic qed NIC Driver3 * Copyright (c) 2015-2017 QLogic Corporation4 * Copyright (c) 2019-2020 Marvell International Ltd.5 */6 7#include <linux/types.h>8#include <asm/byteorder.h>9#include <linux/io.h>10#include <linux/delay.h>11#include <linux/dma-mapping.h>12#include <linux/errno.h>13#include <linux/kernel.h>14#include <linux/list.h>15#include <linux/pci.h>16#include <linux/slab.h>17#include <linux/spinlock.h>18#include <linux/string.h>19#include "qed.h"20#include "qed_cxt.h"21#include "qed_dev_api.h"22#include "qed_hsi.h"23#include "qed_iro_hsi.h"24#include "qed_hw.h"25#include "qed_int.h"26#include "qed_iscsi.h"27#include "qed_mcp.h"28#include "qed_ooo.h"29#include "qed_reg_addr.h"30#include "qed_sp.h"31#include "qed_sriov.h"32#include "qed_rdma.h"33 34/***************************************************************************35 * Structures & Definitions36 ***************************************************************************/37 38#define SPQ_HIGH_PRI_RESERVE_DEFAULT (1)39 40#define SPQ_BLOCK_DELAY_MAX_ITER (10)41#define SPQ_BLOCK_DELAY_US (10)42#define SPQ_BLOCK_SLEEP_MAX_ITER (1000)43#define SPQ_BLOCK_SLEEP_MS (5)44 45/***************************************************************************46 * Blocking Imp. (BLOCK/EBLOCK mode)47 ***************************************************************************/48static void qed_spq_blocking_cb(struct qed_hwfn *p_hwfn,49 void *cookie,50 union event_ring_data *data, u8 fw_return_code)51{52 struct qed_spq_comp_done *comp_done;53 54 comp_done = (struct qed_spq_comp_done *)cookie;55 56 comp_done->fw_return_code = fw_return_code;57 58 /* Make sure completion done is visible on waiting thread */59 smp_store_release(&comp_done->done, 0x1);60}61 62static int __qed_spq_block(struct qed_hwfn *p_hwfn,63 struct qed_spq_entry *p_ent,64 u8 *p_fw_ret, bool sleep_between_iter)65{66 struct qed_spq_comp_done *comp_done;67 u32 iter_cnt;68 69 comp_done = (struct qed_spq_comp_done *)p_ent->comp_cb.cookie;70 iter_cnt = sleep_between_iter ? SPQ_BLOCK_SLEEP_MAX_ITER71 : SPQ_BLOCK_DELAY_MAX_ITER;72 73 while (iter_cnt--) {74 /* Validate we receive completion update */75 if (smp_load_acquire(&comp_done->done) == 1) { /* ^^^ */76 if (p_fw_ret)77 *p_fw_ret = comp_done->fw_return_code;78 return 0;79 }80 81 if (sleep_between_iter)82 msleep(SPQ_BLOCK_SLEEP_MS);83 else84 udelay(SPQ_BLOCK_DELAY_US);85 }86 87 return -EBUSY;88}89 90static int qed_spq_block(struct qed_hwfn *p_hwfn,91 struct qed_spq_entry *p_ent,92 u8 *p_fw_ret, bool skip_quick_poll)93{94 struct qed_spq_comp_done *comp_done;95 struct qed_ptt *p_ptt;96 int rc;97 98 /* A relatively short polling period w/o sleeping, to allow the FW to99 * complete the ramrod and thus possibly to avoid the following sleeps.100 */101 if (!skip_quick_poll) {102 rc = __qed_spq_block(p_hwfn, p_ent, p_fw_ret, false);103 if (!rc)104 return 0;105 }106 107 /* Move to polling with a sleeping period between iterations */108 rc = __qed_spq_block(p_hwfn, p_ent, p_fw_ret, true);109 if (!rc)110 return 0;111 112 p_ptt = qed_ptt_acquire(p_hwfn);113 if (!p_ptt) {114 DP_NOTICE(p_hwfn, "ptt, failed to acquire\n");115 return -EAGAIN;116 }117 118 DP_INFO(p_hwfn, "Ramrod is stuck, requesting MCP drain\n");119 rc = qed_mcp_drain(p_hwfn, p_ptt);120 qed_ptt_release(p_hwfn, p_ptt);121 if (rc) {122 DP_NOTICE(p_hwfn, "MCP drain failed\n");123 goto err;124 }125 126 /* Retry after drain */127 rc = __qed_spq_block(p_hwfn, p_ent, p_fw_ret, true);128 if (!rc)129 return 0;130 131 comp_done = (struct qed_spq_comp_done *)p_ent->comp_cb.cookie;132 if (comp_done->done == 1) {133 if (p_fw_ret)134 *p_fw_ret = comp_done->fw_return_code;135 return 0;136 }137err:138 p_ptt = qed_ptt_acquire(p_hwfn);139 if (!p_ptt)140 return -EBUSY;141 qed_hw_err_notify(p_hwfn, p_ptt, QED_HW_ERR_RAMROD_FAIL,142 "Ramrod is stuck [CID %08x %s:%02x %s:%02x echo %04x]\n",143 le32_to_cpu(p_ent->elem.hdr.cid),144 qed_get_ramrod_cmd_id_str(p_ent->elem.hdr.protocol_id,145 p_ent->elem.hdr.cmd_id),146 p_ent->elem.hdr.cmd_id,147 qed_get_protocol_type_str(p_ent->elem.hdr.protocol_id),148 p_ent->elem.hdr.protocol_id,149 le16_to_cpu(p_ent->elem.hdr.echo));150 qed_ptt_release(p_hwfn, p_ptt);151 152 return -EBUSY;153}154 155/***************************************************************************156 * SPQ entries inner API157 ***************************************************************************/158static int qed_spq_fill_entry(struct qed_hwfn *p_hwfn,159 struct qed_spq_entry *p_ent)160{161 p_ent->flags = 0;162 163 switch (p_ent->comp_mode) {164 case QED_SPQ_MODE_EBLOCK:165 case QED_SPQ_MODE_BLOCK:166 p_ent->comp_cb.function = qed_spq_blocking_cb;167 break;168 case QED_SPQ_MODE_CB:169 break;170 default:171 DP_NOTICE(p_hwfn, "Unknown SPQE completion mode %d\n",172 p_ent->comp_mode);173 return -EINVAL;174 }175 176 DP_VERBOSE(p_hwfn,177 QED_MSG_SPQ,178 "Ramrod hdr: [CID 0x%08x %s:0x%02x %s:0x%02x] Data ptr: [%08x:%08x] Cmpltion Mode: %s\n",179 p_ent->elem.hdr.cid,180 qed_get_ramrod_cmd_id_str(p_ent->elem.hdr.protocol_id,181 p_ent->elem.hdr.cmd_id),182 p_ent->elem.hdr.cmd_id,183 qed_get_protocol_type_str(p_ent->elem.hdr.protocol_id),184 p_ent->elem.hdr.protocol_id,185 p_ent->elem.data_ptr.hi, p_ent->elem.data_ptr.lo,186 D_TRINE(p_ent->comp_mode, QED_SPQ_MODE_EBLOCK,187 QED_SPQ_MODE_BLOCK, "MODE_EBLOCK", "MODE_BLOCK",188 "MODE_CB"));189 190 return 0;191}192 193/***************************************************************************194 * HSI access195 ***************************************************************************/196static void qed_spq_hw_initialize(struct qed_hwfn *p_hwfn,197 struct qed_spq *p_spq)198{199 struct core_conn_context *p_cxt;200 struct qed_cxt_info cxt_info;201 u16 physical_q;202 int rc;203 204 cxt_info.iid = p_spq->cid;205 206 rc = qed_cxt_get_cid_info(p_hwfn, &cxt_info);207 208 if (rc < 0) {209 DP_NOTICE(p_hwfn, "Cannot find context info for cid=%d\n",210 p_spq->cid);211 return;212 }213 214 p_cxt = cxt_info.p_cxt;215 216 SET_FIELD(p_cxt->xstorm_ag_context.flags10,217 XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN, 1);218 SET_FIELD(p_cxt->xstorm_ag_context.flags1,219 XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE, 1);220 SET_FIELD(p_cxt->xstorm_ag_context.flags9,221 XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN, 1);222 223 /* QM physical queue */224 physical_q = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_LB);225 p_cxt->xstorm_ag_context.physical_q0 = cpu_to_le16(physical_q);226 227 p_cxt->xstorm_st_context.spq_base_addr.lo =228 DMA_LO_LE(p_spq->chain.p_phys_addr);229 p_cxt->xstorm_st_context.spq_base_addr.hi =230 DMA_HI_LE(p_spq->chain.p_phys_addr);231}232 233static int qed_spq_hw_post(struct qed_hwfn *p_hwfn,234 struct qed_spq *p_spq, struct qed_spq_entry *p_ent)235{236 struct qed_chain *p_chain = &p_hwfn->p_spq->chain;237 struct core_db_data *p_db_data = &p_spq->db_data;238 u16 echo = qed_chain_get_prod_idx(p_chain);239 struct slow_path_element *elem;240 241 p_ent->elem.hdr.echo = cpu_to_le16(echo);242 elem = qed_chain_produce(p_chain);243 if (!elem) {244 DP_NOTICE(p_hwfn, "Failed to produce from SPQ chain\n");245 return -EINVAL;246 }247 248 *elem = p_ent->elem; /* struct assignment */249 250 /* send a doorbell on the slow hwfn session */251 p_db_data->spq_prod = cpu_to_le16(qed_chain_get_prod_idx(p_chain));252 253 /* make sure the SPQE is updated before the doorbell */254 wmb();255 256 DOORBELL(p_hwfn, p_spq->db_addr_offset, *(u32 *)p_db_data);257 258 /* make sure doorbell is rang */259 wmb();260 261 DP_VERBOSE(p_hwfn, QED_MSG_SPQ,262 "Doorbelled [0x%08x, CID 0x%08x] with Flags: %02x agg_params: %02x, prod: %04x\n",263 p_spq->db_addr_offset,264 p_spq->cid,265 p_db_data->params,266 p_db_data->agg_flags, qed_chain_get_prod_idx(p_chain));267 268 return 0;269}270 271/***************************************************************************272 * Asynchronous events273 ***************************************************************************/274static int275qed_async_event_completion(struct qed_hwfn *p_hwfn,276 struct event_ring_entry *p_eqe)277{278 qed_spq_async_comp_cb cb;279 280 if (!p_hwfn->p_spq)281 return -EINVAL;282 283 if (p_eqe->protocol_id >= MAX_PROTOCOL_TYPE) {284 DP_ERR(p_hwfn, "Wrong protocol: %s:%d\n",285 qed_get_protocol_type_str(p_eqe->protocol_id),286 p_eqe->protocol_id);287 288 return -EINVAL;289 }290 291 cb = p_hwfn->p_spq->async_comp_cb[p_eqe->protocol_id];292 if (cb) {293 return cb(p_hwfn, p_eqe->opcode, p_eqe->echo,294 &p_eqe->data, p_eqe->fw_return_code);295 } else {296 DP_NOTICE(p_hwfn,297 "Unknown Async completion for %s:%d\n",298 qed_get_protocol_type_str(p_eqe->protocol_id),299 p_eqe->protocol_id);300 301 return -EINVAL;302 }303}304 305int306qed_spq_register_async_cb(struct qed_hwfn *p_hwfn,307 enum protocol_type protocol_id,308 qed_spq_async_comp_cb cb)309{310 if (!p_hwfn->p_spq || (protocol_id >= MAX_PROTOCOL_TYPE))311 return -EINVAL;312 313 p_hwfn->p_spq->async_comp_cb[protocol_id] = cb;314 return 0;315}316 317void318qed_spq_unregister_async_cb(struct qed_hwfn *p_hwfn,319 enum protocol_type protocol_id)320{321 if (!p_hwfn->p_spq || (protocol_id >= MAX_PROTOCOL_TYPE))322 return;323 324 p_hwfn->p_spq->async_comp_cb[protocol_id] = NULL;325}326 327/***************************************************************************328 * EQ API329 ***************************************************************************/330void qed_eq_prod_update(struct qed_hwfn *p_hwfn, u16 prod)331{332 u32 addr = GET_GTT_REG_ADDR(GTT_BAR0_MAP_REG_USDM_RAM,333 USTORM_EQE_CONS, p_hwfn->rel_pf_id);334 335 REG_WR16(p_hwfn, addr, prod);336}337 338int qed_eq_completion(struct qed_hwfn *p_hwfn, void *cookie)339{340 struct qed_eq *p_eq = cookie;341 struct qed_chain *p_chain = &p_eq->chain;342 int rc = 0;343 344 /* take a snapshot of the FW consumer */345 u16 fw_cons_idx = le16_to_cpu(*p_eq->p_fw_cons);346 347 DP_VERBOSE(p_hwfn, QED_MSG_SPQ, "fw_cons_idx %x\n", fw_cons_idx);348 349 /* Need to guarantee the fw_cons index we use points to a usuable350 * element (to comply with our chain), so our macros would comply351 */352 if ((fw_cons_idx & qed_chain_get_usable_per_page(p_chain)) ==353 qed_chain_get_usable_per_page(p_chain))354 fw_cons_idx += qed_chain_get_unusable_per_page(p_chain);355 356 /* Complete current segment of eq entries */357 while (fw_cons_idx != qed_chain_get_cons_idx(p_chain)) {358 struct event_ring_entry *p_eqe = qed_chain_consume(p_chain);359 360 if (!p_eqe) {361 rc = -EINVAL;362 break;363 }364 365 DP_VERBOSE(p_hwfn, QED_MSG_SPQ,366 "op %x prot %x res0 %x echo %x fwret %x flags %x\n",367 p_eqe->opcode,368 p_eqe->protocol_id,369 p_eqe->reserved0,370 le16_to_cpu(p_eqe->echo),371 p_eqe->fw_return_code,372 p_eqe->flags);373 374 if (GET_FIELD(p_eqe->flags, EVENT_RING_ENTRY_ASYNC)) {375 if (qed_async_event_completion(p_hwfn, p_eqe))376 rc = -EINVAL;377 } else if (qed_spq_completion(p_hwfn,378 p_eqe->echo,379 p_eqe->fw_return_code,380 &p_eqe->data)) {381 rc = -EINVAL;382 }383 384 qed_chain_recycle_consumed(p_chain);385 }386 387 qed_eq_prod_update(p_hwfn, qed_chain_get_prod_idx(p_chain));388 389 /* Attempt to post pending requests */390 spin_lock_bh(&p_hwfn->p_spq->lock);391 rc = qed_spq_pend_post(p_hwfn);392 spin_unlock_bh(&p_hwfn->p_spq->lock);393 394 return rc;395}396 397int qed_eq_alloc(struct qed_hwfn *p_hwfn, u16 num_elem)398{399 struct qed_chain_init_params params = {400 .mode = QED_CHAIN_MODE_PBL,401 .intended_use = QED_CHAIN_USE_TO_PRODUCE,402 .cnt_type = QED_CHAIN_CNT_TYPE_U16,403 .num_elems = num_elem,404 .elem_size = sizeof(union event_ring_element),405 };406 struct qed_eq *p_eq;407 int ret;408 409 /* Allocate EQ struct */410 p_eq = kzalloc(sizeof(*p_eq), GFP_KERNEL);411 if (!p_eq)412 return -ENOMEM;413 414 ret = qed_chain_alloc(p_hwfn->cdev, &p_eq->chain, ¶ms);415 if (ret) {416 DP_NOTICE(p_hwfn, "Failed to allocate EQ chain\n");417 goto eq_allocate_fail;418 }419 420 /* register EQ completion on the SP SB */421 qed_int_register_cb(p_hwfn, qed_eq_completion,422 p_eq, &p_eq->eq_sb_index, &p_eq->p_fw_cons);423 424 p_hwfn->p_eq = p_eq;425 return 0;426 427eq_allocate_fail:428 kfree(p_eq);429 430 return ret;431}432 433void qed_eq_setup(struct qed_hwfn *p_hwfn)434{435 qed_chain_reset(&p_hwfn->p_eq->chain);436}437 438void qed_eq_free(struct qed_hwfn *p_hwfn)439{440 if (!p_hwfn->p_eq)441 return;442 443 qed_chain_free(p_hwfn->cdev, &p_hwfn->p_eq->chain);444 445 kfree(p_hwfn->p_eq);446 p_hwfn->p_eq = NULL;447}448 449/***************************************************************************450 * CQE API - manipulate EQ functionality451 ***************************************************************************/452static int qed_cqe_completion(struct qed_hwfn *p_hwfn,453 struct eth_slow_path_rx_cqe *cqe,454 enum protocol_type protocol)455{456 if (IS_VF(p_hwfn->cdev))457 return 0;458 459 /* @@@tmp - it's possible we'll eventually want to handle some460 * actual commands that can arrive here, but for now this is only461 * used to complete the ramrod using the echo value on the cqe462 */463 return qed_spq_completion(p_hwfn, cqe->echo, 0, NULL);464}465 466int qed_eth_cqe_completion(struct qed_hwfn *p_hwfn,467 struct eth_slow_path_rx_cqe *cqe)468{469 int rc;470 471 rc = qed_cqe_completion(p_hwfn, cqe, PROTOCOLID_ETH);472 if (rc)473 DP_NOTICE(p_hwfn,474 "Failed to handle RXQ CQE [cmd 0x%02x]\n",475 cqe->ramrod_cmd_id);476 477 return rc;478}479 480/***************************************************************************481 * Slow hwfn Queue (spq)482 ***************************************************************************/483void qed_spq_setup(struct qed_hwfn *p_hwfn)484{485 struct qed_spq *p_spq = p_hwfn->p_spq;486 struct qed_spq_entry *p_virt = NULL;487 struct core_db_data *p_db_data;488 void __iomem *db_addr;489 dma_addr_t p_phys = 0;490 u32 i, capacity;491 int rc;492 493 INIT_LIST_HEAD(&p_spq->pending);494 INIT_LIST_HEAD(&p_spq->completion_pending);495 INIT_LIST_HEAD(&p_spq->free_pool);496 INIT_LIST_HEAD(&p_spq->unlimited_pending);497 spin_lock_init(&p_spq->lock);498 499 /* SPQ empty pool */500 p_phys = p_spq->p_phys + offsetof(struct qed_spq_entry, ramrod);501 p_virt = p_spq->p_virt;502 503 capacity = qed_chain_get_capacity(&p_spq->chain);504 for (i = 0; i < capacity; i++) {505 DMA_REGPAIR_LE(p_virt->elem.data_ptr, p_phys);506 507 list_add_tail(&p_virt->list, &p_spq->free_pool);508 509 p_virt++;510 p_phys += sizeof(struct qed_spq_entry);511 }512 513 /* Statistics */514 p_spq->normal_count = 0;515 p_spq->comp_count = 0;516 p_spq->comp_sent_count = 0;517 p_spq->unlimited_pending_count = 0;518 519 bitmap_zero(p_spq->p_comp_bitmap, SPQ_RING_SIZE);520 p_spq->comp_bitmap_idx = 0;521 522 /* SPQ cid, cannot fail */523 qed_cxt_acquire_cid(p_hwfn, PROTOCOLID_CORE, &p_spq->cid);524 qed_spq_hw_initialize(p_hwfn, p_spq);525 526 /* reset the chain itself */527 qed_chain_reset(&p_spq->chain);528 529 /* Initialize the address/data of the SPQ doorbell */530 p_spq->db_addr_offset = qed_db_addr(p_spq->cid, DQ_DEMS_LEGACY);531 p_db_data = &p_spq->db_data;532 memset(p_db_data, 0, sizeof(*p_db_data));533 SET_FIELD(p_db_data->params, CORE_DB_DATA_DEST, DB_DEST_XCM);534 SET_FIELD(p_db_data->params, CORE_DB_DATA_AGG_CMD, DB_AGG_CMD_MAX);535 SET_FIELD(p_db_data->params, CORE_DB_DATA_AGG_VAL_SEL,536 DQ_XCM_CORE_SPQ_PROD_CMD);537 p_db_data->agg_flags = DQ_XCM_CORE_DQ_CF_CMD;538 539 /* Register the SPQ doorbell with the doorbell recovery mechanism */540 db_addr = (void __iomem *)((u8 __iomem *)p_hwfn->doorbells +541 p_spq->db_addr_offset);542 rc = qed_db_recovery_add(p_hwfn->cdev, db_addr, &p_spq->db_data,543 DB_REC_WIDTH_32B, DB_REC_KERNEL);544 if (rc)545 DP_INFO(p_hwfn,546 "Failed to register the SPQ doorbell with the doorbell recovery mechanism\n");547}548 549int qed_spq_alloc(struct qed_hwfn *p_hwfn)550{551 struct qed_chain_init_params params = {552 .mode = QED_CHAIN_MODE_SINGLE,553 .intended_use = QED_CHAIN_USE_TO_PRODUCE,554 .cnt_type = QED_CHAIN_CNT_TYPE_U16,555 .elem_size = sizeof(struct slow_path_element),556 };557 struct qed_dev *cdev = p_hwfn->cdev;558 struct qed_spq_entry *p_virt = NULL;559 struct qed_spq *p_spq = NULL;560 dma_addr_t p_phys = 0;561 u32 capacity;562 int ret;563 564 /* SPQ struct */565 p_spq = kzalloc(sizeof(*p_spq), GFP_KERNEL);566 if (!p_spq)567 return -ENOMEM;568 569 /* SPQ ring */570 ret = qed_chain_alloc(cdev, &p_spq->chain, ¶ms);571 if (ret) {572 DP_NOTICE(p_hwfn, "Failed to allocate SPQ chain\n");573 goto spq_chain_alloc_fail;574 }575 576 /* allocate and fill the SPQ elements (incl. ramrod data list) */577 capacity = qed_chain_get_capacity(&p_spq->chain);578 ret = -ENOMEM;579 580 p_virt = dma_alloc_coherent(&cdev->pdev->dev,581 capacity * sizeof(struct qed_spq_entry),582 &p_phys, GFP_KERNEL);583 if (!p_virt)584 goto spq_alloc_fail;585 586 p_spq->p_virt = p_virt;587 p_spq->p_phys = p_phys;588 p_hwfn->p_spq = p_spq;589 590 return 0;591 592spq_alloc_fail:593 qed_chain_free(cdev, &p_spq->chain);594spq_chain_alloc_fail:595 kfree(p_spq);596 597 return ret;598}599 600void qed_spq_free(struct qed_hwfn *p_hwfn)601{602 struct qed_spq *p_spq = p_hwfn->p_spq;603 void __iomem *db_addr;604 u32 capacity;605 606 if (!p_spq)607 return;608 609 /* Delete the SPQ doorbell from the doorbell recovery mechanism */610 db_addr = (void __iomem *)((u8 __iomem *)p_hwfn->doorbells +611 p_spq->db_addr_offset);612 qed_db_recovery_del(p_hwfn->cdev, db_addr, &p_spq->db_data);613 614 if (p_spq->p_virt) {615 capacity = qed_chain_get_capacity(&p_spq->chain);616 dma_free_coherent(&p_hwfn->cdev->pdev->dev,617 capacity *618 sizeof(struct qed_spq_entry),619 p_spq->p_virt, p_spq->p_phys);620 }621 622 qed_chain_free(p_hwfn->cdev, &p_spq->chain);623 kfree(p_spq);624 p_hwfn->p_spq = NULL;625}626 627int qed_spq_get_entry(struct qed_hwfn *p_hwfn, struct qed_spq_entry **pp_ent)628{629 struct qed_spq *p_spq = p_hwfn->p_spq;630 struct qed_spq_entry *p_ent = NULL;631 int rc = 0;632 633 spin_lock_bh(&p_spq->lock);634 635 if (list_empty(&p_spq->free_pool)) {636 p_ent = kzalloc(sizeof(*p_ent), GFP_ATOMIC);637 if (!p_ent) {638 DP_NOTICE(p_hwfn,639 "Failed to allocate an SPQ entry for a pending ramrod\n");640 rc = -ENOMEM;641 goto out_unlock;642 }643 p_ent->queue = &p_spq->unlimited_pending;644 } else {645 p_ent = list_first_entry(&p_spq->free_pool,646 struct qed_spq_entry, list);647 list_del(&p_ent->list);648 p_ent->queue = &p_spq->pending;649 }650 651 *pp_ent = p_ent;652 653out_unlock:654 spin_unlock_bh(&p_spq->lock);655 return rc;656}657 658/* Locked variant; Should be called while the SPQ lock is taken */659static void __qed_spq_return_entry(struct qed_hwfn *p_hwfn,660 struct qed_spq_entry *p_ent)661{662 list_add_tail(&p_ent->list, &p_hwfn->p_spq->free_pool);663}664 665void qed_spq_return_entry(struct qed_hwfn *p_hwfn, struct qed_spq_entry *p_ent)666{667 spin_lock_bh(&p_hwfn->p_spq->lock);668 __qed_spq_return_entry(p_hwfn, p_ent);669 spin_unlock_bh(&p_hwfn->p_spq->lock);670}671 672/**673 * qed_spq_add_entry() - Add a new entry to the pending list.674 * Should be used while lock is being held.675 *676 * @p_hwfn: HW device data.677 * @p_ent: An entry to add.678 * @priority: Desired priority.679 *680 * Adds an entry to the pending list is there is room (an empty681 * element is available in the free_pool), or else places the682 * entry in the unlimited_pending pool.683 *684 * Return: zero on success, -EINVAL on invalid @priority.685 */686static int qed_spq_add_entry(struct qed_hwfn *p_hwfn,687 struct qed_spq_entry *p_ent,688 enum spq_priority priority)689{690 struct qed_spq *p_spq = p_hwfn->p_spq;691 692 if (p_ent->queue == &p_spq->unlimited_pending) {693 if (list_empty(&p_spq->free_pool)) {694 list_add_tail(&p_ent->list, &p_spq->unlimited_pending);695 p_spq->unlimited_pending_count++;696 697 return 0;698 } else {699 struct qed_spq_entry *p_en2;700 701 p_en2 = list_first_entry(&p_spq->free_pool,702 struct qed_spq_entry, list);703 list_del(&p_en2->list);704 705 /* Copy the ring element physical pointer to the new706 * entry, since we are about to override the entire ring707 * entry and don't want to lose the pointer.708 */709 p_ent->elem.data_ptr = p_en2->elem.data_ptr;710 711 *p_en2 = *p_ent;712 713 /* EBLOCK responsible to free the allocated p_ent */714 if (p_ent->comp_mode != QED_SPQ_MODE_EBLOCK)715 kfree(p_ent);716 else717 p_ent->post_ent = p_en2;718 719 p_ent = p_en2;720 }721 }722 723 /* entry is to be placed in 'pending' queue */724 switch (priority) {725 case QED_SPQ_PRIORITY_NORMAL:726 list_add_tail(&p_ent->list, &p_spq->pending);727 p_spq->normal_count++;728 break;729 case QED_SPQ_PRIORITY_HIGH:730 list_add(&p_ent->list, &p_spq->pending);731 p_spq->high_count++;732 break;733 default:734 return -EINVAL;735 }736 737 return 0;738}739 740/***************************************************************************741 * Accessor742 ***************************************************************************/743u32 qed_spq_get_cid(struct qed_hwfn *p_hwfn)744{745 if (!p_hwfn->p_spq)746 return 0xffffffff; /* illegal */747 return p_hwfn->p_spq->cid;748}749 750/***************************************************************************751 * Posting new Ramrods752 ***************************************************************************/753static int qed_spq_post_list(struct qed_hwfn *p_hwfn,754 struct list_head *head, u32 keep_reserve)755{756 struct qed_spq *p_spq = p_hwfn->p_spq;757 int rc;758 759 while (qed_chain_get_elem_left(&p_spq->chain) > keep_reserve &&760 !list_empty(head)) {761 struct qed_spq_entry *p_ent =762 list_first_entry(head, struct qed_spq_entry, list);763 list_move_tail(&p_ent->list, &p_spq->completion_pending);764 p_spq->comp_sent_count++;765 766 rc = qed_spq_hw_post(p_hwfn, p_spq, p_ent);767 if (rc) {768 list_del(&p_ent->list);769 __qed_spq_return_entry(p_hwfn, p_ent);770 return rc;771 }772 }773 774 return 0;775}776 777int qed_spq_pend_post(struct qed_hwfn *p_hwfn)778{779 struct qed_spq *p_spq = p_hwfn->p_spq;780 struct qed_spq_entry *p_ent = NULL;781 782 while (!list_empty(&p_spq->free_pool)) {783 if (list_empty(&p_spq->unlimited_pending))784 break;785 786 p_ent = list_first_entry(&p_spq->unlimited_pending,787 struct qed_spq_entry, list);788 if (!p_ent)789 return -EINVAL;790 791 list_del(&p_ent->list);792 793 qed_spq_add_entry(p_hwfn, p_ent, p_ent->priority);794 }795 796 return qed_spq_post_list(p_hwfn, &p_spq->pending,797 SPQ_HIGH_PRI_RESERVE_DEFAULT);798}799 800static void qed_spq_recov_set_ret_code(struct qed_spq_entry *p_ent,801 u8 *fw_return_code)802{803 if (!fw_return_code)804 return;805 806 if (p_ent->elem.hdr.protocol_id == PROTOCOLID_ROCE ||807 p_ent->elem.hdr.protocol_id == PROTOCOLID_IWARP)808 *fw_return_code = RDMA_RETURN_OK;809}810 811/* Avoid overriding of SPQ entries when getting out-of-order completions, by812 * marking the completions in a bitmap and increasing the chain consumer only813 * for the first successive completed entries.814 */815static void qed_spq_comp_bmap_update(struct qed_hwfn *p_hwfn, __le16 echo)816{817 u16 pos = le16_to_cpu(echo) % SPQ_RING_SIZE;818 struct qed_spq *p_spq = p_hwfn->p_spq;819 820 __set_bit(pos, p_spq->p_comp_bitmap);821 while (test_bit(p_spq->comp_bitmap_idx,822 p_spq->p_comp_bitmap)) {823 __clear_bit(p_spq->comp_bitmap_idx,824 p_spq->p_comp_bitmap);825 p_spq->comp_bitmap_idx++;826 qed_chain_return_produced(&p_spq->chain);827 }828}829 830int qed_spq_post(struct qed_hwfn *p_hwfn,831 struct qed_spq_entry *p_ent, u8 *fw_return_code)832{833 int rc = 0;834 struct qed_spq *p_spq = p_hwfn ? p_hwfn->p_spq : NULL;835 bool b_ret_ent = true;836 bool eblock;837 838 if (!p_hwfn)839 return -EINVAL;840 841 if (!p_ent) {842 DP_NOTICE(p_hwfn, "Got a NULL pointer\n");843 return -EINVAL;844 }845 846 if (p_hwfn->cdev->recov_in_prog) {847 DP_VERBOSE(p_hwfn,848 QED_MSG_SPQ,849 "Recovery is in progress. Skip spq post [%s:%02x %s:%02x]\n",850 qed_get_ramrod_cmd_id_str(p_ent->elem.hdr.protocol_id,851 p_ent->elem.hdr.cmd_id),852 p_ent->elem.hdr.cmd_id,853 qed_get_protocol_type_str(p_ent->elem.hdr.protocol_id),854 p_ent->elem.hdr.protocol_id);855 856 /* Let the flow complete w/o any error handling */857 qed_spq_recov_set_ret_code(p_ent, fw_return_code);858 return 0;859 }860 861 /* Complete the entry */862 rc = qed_spq_fill_entry(p_hwfn, p_ent);863 864 spin_lock_bh(&p_spq->lock);865 866 /* Check return value after LOCK is taken for cleaner error flow */867 if (rc)868 goto spq_post_fail;869 870 /* Check if entry is in block mode before qed_spq_add_entry,871 * which might kfree p_ent.872 */873 eblock = (p_ent->comp_mode == QED_SPQ_MODE_EBLOCK);874 875 /* Add the request to the pending queue */876 rc = qed_spq_add_entry(p_hwfn, p_ent, p_ent->priority);877 if (rc)878 goto spq_post_fail;879 880 rc = qed_spq_pend_post(p_hwfn);881 if (rc) {882 /* Since it's possible that pending failed for a different883 * entry [although unlikely], the failed entry was already884 * dealt with; No need to return it here.885 */886 b_ret_ent = false;887 goto spq_post_fail;888 }889 890 spin_unlock_bh(&p_spq->lock);891 892 if (eblock) {893 /* For entries in QED BLOCK mode, the completion code cannot894 * perform the necessary cleanup - if it did, we couldn't895 * access p_ent here to see whether it's successful or not.896 * Thus, after gaining the answer perform the cleanup here.897 */898 rc = qed_spq_block(p_hwfn, p_ent, fw_return_code,899 p_ent->queue == &p_spq->unlimited_pending);900 901 if (p_ent->queue == &p_spq->unlimited_pending) {902 struct qed_spq_entry *p_post_ent = p_ent->post_ent;903 904 kfree(p_ent);905 906 /* Return the entry which was actually posted */907 p_ent = p_post_ent;908 }909 910 if (rc)911 goto spq_post_fail2;912 913 /* return to pool */914 qed_spq_return_entry(p_hwfn, p_ent);915 }916 return rc;917 918spq_post_fail2:919 spin_lock_bh(&p_spq->lock);920 list_del(&p_ent->list);921 qed_spq_comp_bmap_update(p_hwfn, p_ent->elem.hdr.echo);922 923spq_post_fail:924 /* return to the free pool */925 if (b_ret_ent)926 __qed_spq_return_entry(p_hwfn, p_ent);927 spin_unlock_bh(&p_spq->lock);928 929 return rc;930}931 932int qed_spq_completion(struct qed_hwfn *p_hwfn,933 __le16 echo,934 u8 fw_return_code,935 union event_ring_data *p_data)936{937 struct qed_spq *p_spq;938 struct qed_spq_entry *p_ent = NULL;939 struct qed_spq_entry *tmp;940 struct qed_spq_entry *found = NULL;941 942 if (!p_hwfn)943 return -EINVAL;944 945 p_spq = p_hwfn->p_spq;946 if (!p_spq)947 return -EINVAL;948 949 spin_lock_bh(&p_spq->lock);950 list_for_each_entry_safe(p_ent, tmp, &p_spq->completion_pending, list) {951 if (p_ent->elem.hdr.echo == echo) {952 list_del(&p_ent->list);953 qed_spq_comp_bmap_update(p_hwfn, echo);954 p_spq->comp_count++;955 found = p_ent;956 break;957 }958 959 /* This is relatively uncommon - depends on scenarios960 * which have mutliple per-PF sent ramrods.961 */962 DP_VERBOSE(p_hwfn, QED_MSG_SPQ,963 "Got completion for echo %04x - doesn't match echo %04x in completion pending list\n",964 le16_to_cpu(echo),965 le16_to_cpu(p_ent->elem.hdr.echo));966 }967 968 /* Release lock before callback, as callback may post969 * an additional ramrod.970 */971 spin_unlock_bh(&p_spq->lock);972 973 if (!found) {974 DP_NOTICE(p_hwfn,975 "Failed to find an entry this EQE [echo %04x] completes\n",976 le16_to_cpu(echo));977 return -EEXIST;978 }979 980 DP_VERBOSE(p_hwfn, QED_MSG_SPQ,981 "Complete EQE [echo %04x]: func %p cookie %p)\n",982 le16_to_cpu(echo),983 p_ent->comp_cb.function, p_ent->comp_cb.cookie);984 if (found->comp_cb.function)985 found->comp_cb.function(p_hwfn, found->comp_cb.cookie, p_data,986 fw_return_code);987 else988 DP_VERBOSE(p_hwfn,989 QED_MSG_SPQ,990 "Got a completion without a callback function\n");991 992 if (found->comp_mode != QED_SPQ_MODE_EBLOCK)993 /* EBLOCK is responsible for returning its own entry into the994 * free list.995 */996 qed_spq_return_entry(p_hwfn, found);997 998 return 0;999}1000 1001#define QED_SPQ_CONSQ_ELEM_SIZE 0x801002 1003int qed_consq_alloc(struct qed_hwfn *p_hwfn)1004{1005 struct qed_chain_init_params params = {1006 .mode = QED_CHAIN_MODE_PBL,1007 .intended_use = QED_CHAIN_USE_TO_PRODUCE,1008 .cnt_type = QED_CHAIN_CNT_TYPE_U16,1009 .num_elems = QED_CHAIN_PAGE_SIZE / QED_SPQ_CONSQ_ELEM_SIZE,1010 .elem_size = QED_SPQ_CONSQ_ELEM_SIZE,1011 };1012 struct qed_consq *p_consq;1013 int ret;1014 1015 /* Allocate ConsQ struct */1016 p_consq = kzalloc(sizeof(*p_consq), GFP_KERNEL);1017 if (!p_consq)1018 return -ENOMEM;1019 1020 /* Allocate and initialize ConsQ chain */1021 ret = qed_chain_alloc(p_hwfn->cdev, &p_consq->chain, ¶ms);1022 if (ret) {1023 DP_NOTICE(p_hwfn, "Failed to allocate ConsQ chain");1024 goto consq_alloc_fail;1025 }1026 1027 p_hwfn->p_consq = p_consq;1028 1029 return 0;1030 1031consq_alloc_fail:1032 kfree(p_consq);1033 1034 return ret;1035}1036 1037void qed_consq_setup(struct qed_hwfn *p_hwfn)1038{1039 qed_chain_reset(&p_hwfn->p_consq->chain);1040}1041 1042void qed_consq_free(struct qed_hwfn *p_hwfn)1043{1044 if (!p_hwfn->p_consq)1045 return;1046 1047 qed_chain_free(p_hwfn->cdev, &p_hwfn->p_consq->chain);1048 1049 kfree(p_hwfn->p_consq);1050 p_hwfn->p_consq = NULL;1051}1052