1261 lines · c
1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) 2018, Intel Corporation. */3 4#include "ice_common.h"5 6#define ICE_CQ_INIT_REGS(qinfo, prefix) \7do { \8 (qinfo)->sq.head = prefix##_ATQH; \9 (qinfo)->sq.tail = prefix##_ATQT; \10 (qinfo)->sq.len = prefix##_ATQLEN; \11 (qinfo)->sq.bah = prefix##_ATQBAH; \12 (qinfo)->sq.bal = prefix##_ATQBAL; \13 (qinfo)->sq.len_mask = prefix##_ATQLEN_ATQLEN_M; \14 (qinfo)->sq.len_ena_mask = prefix##_ATQLEN_ATQENABLE_M; \15 (qinfo)->sq.len_crit_mask = prefix##_ATQLEN_ATQCRIT_M; \16 (qinfo)->sq.head_mask = prefix##_ATQH_ATQH_M; \17 (qinfo)->rq.head = prefix##_ARQH; \18 (qinfo)->rq.tail = prefix##_ARQT; \19 (qinfo)->rq.len = prefix##_ARQLEN; \20 (qinfo)->rq.bah = prefix##_ARQBAH; \21 (qinfo)->rq.bal = prefix##_ARQBAL; \22 (qinfo)->rq.len_mask = prefix##_ARQLEN_ARQLEN_M; \23 (qinfo)->rq.len_ena_mask = prefix##_ARQLEN_ARQENABLE_M; \24 (qinfo)->rq.len_crit_mask = prefix##_ARQLEN_ARQCRIT_M; \25 (qinfo)->rq.head_mask = prefix##_ARQH_ARQH_M; \26} while (0)27 28/**29 * ice_adminq_init_regs - Initialize AdminQ registers30 * @hw: pointer to the hardware structure31 *32 * This assumes the alloc_sq and alloc_rq functions have already been called33 */34static void ice_adminq_init_regs(struct ice_hw *hw)35{36 struct ice_ctl_q_info *cq = &hw->adminq;37 38 ICE_CQ_INIT_REGS(cq, PF_FW);39}40 41/**42 * ice_mailbox_init_regs - Initialize Mailbox registers43 * @hw: pointer to the hardware structure44 *45 * This assumes the alloc_sq and alloc_rq functions have already been called46 */47static void ice_mailbox_init_regs(struct ice_hw *hw)48{49 struct ice_ctl_q_info *cq = &hw->mailboxq;50 51 ICE_CQ_INIT_REGS(cq, PF_MBX);52}53 54/**55 * ice_sb_init_regs - Initialize Sideband registers56 * @hw: pointer to the hardware structure57 *58 * This assumes the alloc_sq and alloc_rq functions have already been called59 */60static void ice_sb_init_regs(struct ice_hw *hw)61{62 struct ice_ctl_q_info *cq = &hw->sbq;63 64 ICE_CQ_INIT_REGS(cq, PF_SB);65}66 67/**68 * ice_check_sq_alive69 * @hw: pointer to the HW struct70 * @cq: pointer to the specific Control queue71 *72 * Returns true if Queue is enabled else false.73 */74bool ice_check_sq_alive(struct ice_hw *hw, struct ice_ctl_q_info *cq)75{76 /* check both queue-length and queue-enable fields */77 if (cq->sq.len && cq->sq.len_mask && cq->sq.len_ena_mask)78 return (rd32(hw, cq->sq.len) & (cq->sq.len_mask |79 cq->sq.len_ena_mask)) ==80 (cq->num_sq_entries | cq->sq.len_ena_mask);81 82 return false;83}84 85/**86 * ice_alloc_ctrlq_sq_ring - Allocate Control Transmit Queue (ATQ) rings87 * @hw: pointer to the hardware structure88 * @cq: pointer to the specific Control queue89 */90static int91ice_alloc_ctrlq_sq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq)92{93 size_t size = cq->num_sq_entries * sizeof(struct ice_aq_desc);94 95 cq->sq.desc_buf.va = dmam_alloc_coherent(ice_hw_to_dev(hw), size,96 &cq->sq.desc_buf.pa,97 GFP_KERNEL | __GFP_ZERO);98 if (!cq->sq.desc_buf.va)99 return -ENOMEM;100 cq->sq.desc_buf.size = size;101 102 return 0;103}104 105/**106 * ice_alloc_ctrlq_rq_ring - Allocate Control Receive Queue (ARQ) rings107 * @hw: pointer to the hardware structure108 * @cq: pointer to the specific Control queue109 */110static int111ice_alloc_ctrlq_rq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq)112{113 size_t size = cq->num_rq_entries * sizeof(struct ice_aq_desc);114 115 cq->rq.desc_buf.va = dmam_alloc_coherent(ice_hw_to_dev(hw), size,116 &cq->rq.desc_buf.pa,117 GFP_KERNEL | __GFP_ZERO);118 if (!cq->rq.desc_buf.va)119 return -ENOMEM;120 cq->rq.desc_buf.size = size;121 return 0;122}123 124/**125 * ice_free_cq_ring - Free control queue ring126 * @hw: pointer to the hardware structure127 * @ring: pointer to the specific control queue ring128 *129 * This assumes the posted buffers have already been cleaned130 * and de-allocated131 */132static void ice_free_cq_ring(struct ice_hw *hw, struct ice_ctl_q_ring *ring)133{134 dmam_free_coherent(ice_hw_to_dev(hw), ring->desc_buf.size,135 ring->desc_buf.va, ring->desc_buf.pa);136 ring->desc_buf.va = NULL;137 ring->desc_buf.pa = 0;138 ring->desc_buf.size = 0;139}140 141/**142 * ice_alloc_rq_bufs - Allocate pre-posted buffers for the ARQ143 * @hw: pointer to the hardware structure144 * @cq: pointer to the specific Control queue145 */146static int147ice_alloc_rq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq)148{149 int i;150 151 /* We'll be allocating the buffer info memory first, then we can152 * allocate the mapped buffers for the event processing153 */154 cq->rq.dma_head = devm_kcalloc(ice_hw_to_dev(hw), cq->num_rq_entries,155 sizeof(cq->rq.desc_buf), GFP_KERNEL);156 if (!cq->rq.dma_head)157 return -ENOMEM;158 cq->rq.r.rq_bi = (struct ice_dma_mem *)cq->rq.dma_head;159 160 /* allocate the mapped buffers */161 for (i = 0; i < cq->num_rq_entries; i++) {162 struct ice_aq_desc *desc;163 struct ice_dma_mem *bi;164 165 bi = &cq->rq.r.rq_bi[i];166 bi->va = dmam_alloc_coherent(ice_hw_to_dev(hw),167 cq->rq_buf_size, &bi->pa,168 GFP_KERNEL | __GFP_ZERO);169 if (!bi->va)170 goto unwind_alloc_rq_bufs;171 bi->size = cq->rq_buf_size;172 173 /* now configure the descriptors for use */174 desc = ICE_CTL_Q_DESC(cq->rq, i);175 176 desc->flags = cpu_to_le16(ICE_AQ_FLAG_BUF);177 if (cq->rq_buf_size > ICE_AQ_LG_BUF)178 desc->flags |= cpu_to_le16(ICE_AQ_FLAG_LB);179 desc->opcode = 0;180 /* This is in accordance with control queue design, there is no181 * register for buffer size configuration182 */183 desc->datalen = cpu_to_le16(bi->size);184 desc->retval = 0;185 desc->cookie_high = 0;186 desc->cookie_low = 0;187 desc->params.generic.addr_high =188 cpu_to_le32(upper_32_bits(bi->pa));189 desc->params.generic.addr_low =190 cpu_to_le32(lower_32_bits(bi->pa));191 desc->params.generic.param0 = 0;192 desc->params.generic.param1 = 0;193 }194 return 0;195 196unwind_alloc_rq_bufs:197 /* don't try to free the one that failed... */198 i--;199 for (; i >= 0; i--) {200 dmam_free_coherent(ice_hw_to_dev(hw), cq->rq.r.rq_bi[i].size,201 cq->rq.r.rq_bi[i].va, cq->rq.r.rq_bi[i].pa);202 cq->rq.r.rq_bi[i].va = NULL;203 cq->rq.r.rq_bi[i].pa = 0;204 cq->rq.r.rq_bi[i].size = 0;205 }206 cq->rq.r.rq_bi = NULL;207 devm_kfree(ice_hw_to_dev(hw), cq->rq.dma_head);208 cq->rq.dma_head = NULL;209 210 return -ENOMEM;211}212 213/**214 * ice_alloc_sq_bufs - Allocate empty buffer structs for the ATQ215 * @hw: pointer to the hardware structure216 * @cq: pointer to the specific Control queue217 */218static int219ice_alloc_sq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq)220{221 int i;222 223 /* No mapped memory needed yet, just the buffer info structures */224 cq->sq.dma_head = devm_kcalloc(ice_hw_to_dev(hw), cq->num_sq_entries,225 sizeof(cq->sq.desc_buf), GFP_KERNEL);226 if (!cq->sq.dma_head)227 return -ENOMEM;228 cq->sq.r.sq_bi = (struct ice_dma_mem *)cq->sq.dma_head;229 230 /* allocate the mapped buffers */231 for (i = 0; i < cq->num_sq_entries; i++) {232 struct ice_dma_mem *bi;233 234 bi = &cq->sq.r.sq_bi[i];235 bi->va = dmam_alloc_coherent(ice_hw_to_dev(hw),236 cq->sq_buf_size, &bi->pa,237 GFP_KERNEL | __GFP_ZERO);238 if (!bi->va)239 goto unwind_alloc_sq_bufs;240 bi->size = cq->sq_buf_size;241 }242 return 0;243 244unwind_alloc_sq_bufs:245 /* don't try to free the one that failed... */246 i--;247 for (; i >= 0; i--) {248 dmam_free_coherent(ice_hw_to_dev(hw), cq->sq.r.sq_bi[i].size,249 cq->sq.r.sq_bi[i].va, cq->sq.r.sq_bi[i].pa);250 cq->sq.r.sq_bi[i].va = NULL;251 cq->sq.r.sq_bi[i].pa = 0;252 cq->sq.r.sq_bi[i].size = 0;253 }254 cq->sq.r.sq_bi = NULL;255 devm_kfree(ice_hw_to_dev(hw), cq->sq.dma_head);256 cq->sq.dma_head = NULL;257 258 return -ENOMEM;259}260 261static int262ice_cfg_cq_regs(struct ice_hw *hw, struct ice_ctl_q_ring *ring, u16 num_entries)263{264 /* Clear Head and Tail */265 wr32(hw, ring->head, 0);266 wr32(hw, ring->tail, 0);267 268 /* set starting point */269 wr32(hw, ring->len, (num_entries | ring->len_ena_mask));270 wr32(hw, ring->bal, lower_32_bits(ring->desc_buf.pa));271 wr32(hw, ring->bah, upper_32_bits(ring->desc_buf.pa));272 273 /* Check one register to verify that config was applied */274 if (rd32(hw, ring->bal) != lower_32_bits(ring->desc_buf.pa))275 return -EIO;276 277 return 0;278}279 280/**281 * ice_cfg_sq_regs - configure Control ATQ registers282 * @hw: pointer to the hardware structure283 * @cq: pointer to the specific Control queue284 *285 * Configure base address and length registers for the transmit queue286 */287static int ice_cfg_sq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq)288{289 return ice_cfg_cq_regs(hw, &cq->sq, cq->num_sq_entries);290}291 292/**293 * ice_cfg_rq_regs - configure Control ARQ register294 * @hw: pointer to the hardware structure295 * @cq: pointer to the specific Control queue296 *297 * Configure base address and length registers for the receive (event queue)298 */299static int ice_cfg_rq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq)300{301 int status;302 303 status = ice_cfg_cq_regs(hw, &cq->rq, cq->num_rq_entries);304 if (status)305 return status;306 307 /* Update tail in the HW to post pre-allocated buffers */308 wr32(hw, cq->rq.tail, (u32)(cq->num_rq_entries - 1));309 310 return 0;311}312 313#define ICE_FREE_CQ_BUFS(hw, qi, ring) \314do { \315 /* free descriptors */ \316 if ((qi)->ring.r.ring##_bi) { \317 int i; \318 \319 for (i = 0; i < (qi)->num_##ring##_entries; i++) \320 if ((qi)->ring.r.ring##_bi[i].pa) { \321 dmam_free_coherent(ice_hw_to_dev(hw), \322 (qi)->ring.r.ring##_bi[i].size, \323 (qi)->ring.r.ring##_bi[i].va, \324 (qi)->ring.r.ring##_bi[i].pa); \325 (qi)->ring.r.ring##_bi[i].va = NULL;\326 (qi)->ring.r.ring##_bi[i].pa = 0;\327 (qi)->ring.r.ring##_bi[i].size = 0;\328 } \329 } \330 /* free DMA head */ \331 devm_kfree(ice_hw_to_dev(hw), (qi)->ring.dma_head); \332} while (0)333 334/**335 * ice_init_sq - main initialization routine for Control ATQ336 * @hw: pointer to the hardware structure337 * @cq: pointer to the specific Control queue338 *339 * This is the main initialization routine for the Control Send Queue340 * Prior to calling this function, the driver *MUST* set the following fields341 * in the cq->structure:342 * - cq->num_sq_entries343 * - cq->sq_buf_size344 *345 * Do *NOT* hold the lock when calling this as the memory allocation routines346 * called are not going to be atomic context safe347 */348static int ice_init_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)349{350 int ret_code;351 352 if (cq->sq.count > 0) {353 /* queue already initialized */354 ret_code = -EBUSY;355 goto init_ctrlq_exit;356 }357 358 /* verify input for valid configuration */359 if (!cq->num_sq_entries || !cq->sq_buf_size) {360 ret_code = -EIO;361 goto init_ctrlq_exit;362 }363 364 cq->sq.next_to_use = 0;365 cq->sq.next_to_clean = 0;366 367 /* allocate the ring memory */368 ret_code = ice_alloc_ctrlq_sq_ring(hw, cq);369 if (ret_code)370 goto init_ctrlq_exit;371 372 /* allocate buffers in the rings */373 ret_code = ice_alloc_sq_bufs(hw, cq);374 if (ret_code)375 goto init_ctrlq_free_rings;376 377 /* initialize base registers */378 ret_code = ice_cfg_sq_regs(hw, cq);379 if (ret_code)380 goto init_ctrlq_free_rings;381 382 /* success! */383 cq->sq.count = cq->num_sq_entries;384 goto init_ctrlq_exit;385 386init_ctrlq_free_rings:387 ICE_FREE_CQ_BUFS(hw, cq, sq);388 ice_free_cq_ring(hw, &cq->sq);389 390init_ctrlq_exit:391 return ret_code;392}393 394/**395 * ice_init_rq - initialize receive side of a control queue396 * @hw: pointer to the hardware structure397 * @cq: pointer to the specific Control queue398 *399 * The main initialization routine for Receive side of a control queue.400 * Prior to calling this function, the driver *MUST* set the following fields401 * in the cq->structure:402 * - cq->num_rq_entries403 * - cq->rq_buf_size404 *405 * Do *NOT* hold the lock when calling this as the memory allocation routines406 * called are not going to be atomic context safe407 */408static int ice_init_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq)409{410 int ret_code;411 412 if (cq->rq.count > 0) {413 /* queue already initialized */414 ret_code = -EBUSY;415 goto init_ctrlq_exit;416 }417 418 /* verify input for valid configuration */419 if (!cq->num_rq_entries || !cq->rq_buf_size) {420 ret_code = -EIO;421 goto init_ctrlq_exit;422 }423 424 cq->rq.next_to_use = 0;425 cq->rq.next_to_clean = 0;426 427 /* allocate the ring memory */428 ret_code = ice_alloc_ctrlq_rq_ring(hw, cq);429 if (ret_code)430 goto init_ctrlq_exit;431 432 /* allocate buffers in the rings */433 ret_code = ice_alloc_rq_bufs(hw, cq);434 if (ret_code)435 goto init_ctrlq_free_rings;436 437 /* initialize base registers */438 ret_code = ice_cfg_rq_regs(hw, cq);439 if (ret_code)440 goto init_ctrlq_free_rings;441 442 /* success! */443 cq->rq.count = cq->num_rq_entries;444 goto init_ctrlq_exit;445 446init_ctrlq_free_rings:447 ICE_FREE_CQ_BUFS(hw, cq, rq);448 ice_free_cq_ring(hw, &cq->rq);449 450init_ctrlq_exit:451 return ret_code;452}453 454/**455 * ice_shutdown_sq - shutdown the transmit side of a control queue456 * @hw: pointer to the hardware structure457 * @cq: pointer to the specific Control queue458 *459 * The main shutdown routine for the Control Transmit Queue460 */461static int ice_shutdown_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)462{463 int ret_code = 0;464 465 mutex_lock(&cq->sq_lock);466 467 if (!cq->sq.count) {468 ret_code = -EBUSY;469 goto shutdown_sq_out;470 }471 472 /* Stop processing of the control queue */473 wr32(hw, cq->sq.head, 0);474 wr32(hw, cq->sq.tail, 0);475 wr32(hw, cq->sq.len, 0);476 wr32(hw, cq->sq.bal, 0);477 wr32(hw, cq->sq.bah, 0);478 479 cq->sq.count = 0; /* to indicate uninitialized queue */480 481 /* free ring buffers and the ring itself */482 ICE_FREE_CQ_BUFS(hw, cq, sq);483 ice_free_cq_ring(hw, &cq->sq);484 485shutdown_sq_out:486 mutex_unlock(&cq->sq_lock);487 return ret_code;488}489 490/**491 * ice_aq_ver_check - Check the reported AQ API version492 * @hw: pointer to the hardware structure493 *494 * Checks if the driver should load on a given AQ API version.495 *496 * Return: 'true' iff the driver should attempt to load. 'false' otherwise.497 */498static bool ice_aq_ver_check(struct ice_hw *hw)499{500 u8 exp_fw_api_ver_major = EXP_FW_API_VER_MAJOR_BY_MAC(hw);501 u8 exp_fw_api_ver_minor = EXP_FW_API_VER_MINOR_BY_MAC(hw);502 503 if (hw->api_maj_ver > exp_fw_api_ver_major) {504 /* Major API version is newer than expected, don't load */505 dev_warn(ice_hw_to_dev(hw),506 "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");507 return false;508 } else if (hw->api_maj_ver == exp_fw_api_ver_major) {509 if (hw->api_min_ver > (exp_fw_api_ver_minor + 2))510 dev_info(ice_hw_to_dev(hw),511 "The driver for the device detected a newer version (%u.%u) of the NVM image than expected (%u.%u). Please install the most recent version of the network driver.\n",512 hw->api_maj_ver, hw->api_min_ver,513 exp_fw_api_ver_major, exp_fw_api_ver_minor);514 else if ((hw->api_min_ver + 2) < exp_fw_api_ver_minor)515 dev_info(ice_hw_to_dev(hw),516 "The driver for the device detected an older version (%u.%u) of the NVM image than expected (%u.%u). Please update the NVM image.\n",517 hw->api_maj_ver, hw->api_min_ver,518 exp_fw_api_ver_major, exp_fw_api_ver_minor);519 } else {520 /* Major API version is older than expected, log a warning */521 dev_info(ice_hw_to_dev(hw),522 "The driver for the device detected an older version (%u.%u) of the NVM image than expected (%u.%u). Please update the NVM image.\n",523 hw->api_maj_ver, hw->api_min_ver,524 exp_fw_api_ver_major, exp_fw_api_ver_minor);525 }526 return true;527}528 529/**530 * ice_shutdown_rq - shutdown Control ARQ531 * @hw: pointer to the hardware structure532 * @cq: pointer to the specific Control queue533 *534 * The main shutdown routine for the Control Receive Queue535 */536static int ice_shutdown_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq)537{538 int ret_code = 0;539 540 mutex_lock(&cq->rq_lock);541 542 if (!cq->rq.count) {543 ret_code = -EBUSY;544 goto shutdown_rq_out;545 }546 547 /* Stop Control Queue processing */548 wr32(hw, cq->rq.head, 0);549 wr32(hw, cq->rq.tail, 0);550 wr32(hw, cq->rq.len, 0);551 wr32(hw, cq->rq.bal, 0);552 wr32(hw, cq->rq.bah, 0);553 554 /* set rq.count to 0 to indicate uninitialized queue */555 cq->rq.count = 0;556 557 /* free ring buffers and the ring itself */558 ICE_FREE_CQ_BUFS(hw, cq, rq);559 ice_free_cq_ring(hw, &cq->rq);560 561shutdown_rq_out:562 mutex_unlock(&cq->rq_lock);563 return ret_code;564}565 566/**567 * ice_init_check_adminq - Check version for Admin Queue to know if its alive568 * @hw: pointer to the hardware structure569 */570static int ice_init_check_adminq(struct ice_hw *hw)571{572 struct ice_ctl_q_info *cq = &hw->adminq;573 int status;574 575 status = ice_aq_get_fw_ver(hw, NULL);576 if (status)577 goto init_ctrlq_free_rq;578 579 if (!ice_aq_ver_check(hw)) {580 status = -EIO;581 goto init_ctrlq_free_rq;582 }583 584 return 0;585 586init_ctrlq_free_rq:587 ice_shutdown_rq(hw, cq);588 ice_shutdown_sq(hw, cq);589 return status;590}591 592/**593 * ice_init_ctrlq - main initialization routine for any control Queue594 * @hw: pointer to the hardware structure595 * @q_type: specific Control queue type596 *597 * Prior to calling this function, the driver *MUST* set the following fields598 * in the cq->structure:599 * - cq->num_sq_entries600 * - cq->num_rq_entries601 * - cq->rq_buf_size602 * - cq->sq_buf_size603 *604 * NOTE: this function does not initialize the controlq locks605 */606static int ice_init_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type)607{608 struct ice_ctl_q_info *cq;609 int ret_code;610 611 switch (q_type) {612 case ICE_CTL_Q_ADMIN:613 ice_adminq_init_regs(hw);614 cq = &hw->adminq;615 break;616 case ICE_CTL_Q_SB:617 ice_sb_init_regs(hw);618 cq = &hw->sbq;619 break;620 case ICE_CTL_Q_MAILBOX:621 ice_mailbox_init_regs(hw);622 cq = &hw->mailboxq;623 break;624 default:625 return -EINVAL;626 }627 cq->qtype = q_type;628 629 /* verify input for valid configuration */630 if (!cq->num_rq_entries || !cq->num_sq_entries ||631 !cq->rq_buf_size || !cq->sq_buf_size) {632 return -EIO;633 }634 635 /* allocate the ATQ */636 ret_code = ice_init_sq(hw, cq);637 if (ret_code)638 return ret_code;639 640 /* allocate the ARQ */641 ret_code = ice_init_rq(hw, cq);642 if (ret_code)643 goto init_ctrlq_free_sq;644 645 /* success! */646 return 0;647 648init_ctrlq_free_sq:649 ice_shutdown_sq(hw, cq);650 return ret_code;651}652 653/**654 * ice_is_sbq_supported - is the sideband queue supported655 * @hw: pointer to the hardware structure656 *657 * Returns true if the sideband control queue interface is658 * supported for the device, false otherwise659 */660bool ice_is_sbq_supported(struct ice_hw *hw)661{662 /* The device sideband queue is only supported on devices with the663 * generic MAC type.664 */665 return ice_is_generic_mac(hw);666}667 668/**669 * ice_get_sbq - returns the right control queue to use for sideband670 * @hw: pointer to the hardware structure671 */672struct ice_ctl_q_info *ice_get_sbq(struct ice_hw *hw)673{674 if (ice_is_sbq_supported(hw))675 return &hw->sbq;676 return &hw->adminq;677}678 679/**680 * ice_shutdown_ctrlq - shutdown routine for any control queue681 * @hw: pointer to the hardware structure682 * @q_type: specific Control queue type683 * @unloading: is the driver unloading itself684 *685 * NOTE: this function does not destroy the control queue locks.686 */687static void ice_shutdown_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type,688 bool unloading)689{690 struct ice_ctl_q_info *cq;691 692 switch (q_type) {693 case ICE_CTL_Q_ADMIN:694 cq = &hw->adminq;695 if (ice_check_sq_alive(hw, cq))696 ice_aq_q_shutdown(hw, unloading);697 break;698 case ICE_CTL_Q_SB:699 cq = &hw->sbq;700 break;701 case ICE_CTL_Q_MAILBOX:702 cq = &hw->mailboxq;703 break;704 default:705 return;706 }707 708 ice_shutdown_sq(hw, cq);709 ice_shutdown_rq(hw, cq);710}711 712/**713 * ice_shutdown_all_ctrlq - shutdown routine for all control queues714 * @hw: pointer to the hardware structure715 * @unloading: is the driver unloading itself716 *717 * NOTE: this function does not destroy the control queue locks. The driver718 * may call this at runtime to shutdown and later restart control queues, such719 * as in response to a reset event.720 */721void ice_shutdown_all_ctrlq(struct ice_hw *hw, bool unloading)722{723 /* Shutdown FW admin queue */724 ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN, unloading);725 /* Shutdown PHY Sideband */726 if (ice_is_sbq_supported(hw))727 ice_shutdown_ctrlq(hw, ICE_CTL_Q_SB, unloading);728 /* Shutdown PF-VF Mailbox */729 ice_shutdown_ctrlq(hw, ICE_CTL_Q_MAILBOX, unloading);730}731 732/**733 * ice_init_all_ctrlq - main initialization routine for all control queues734 * @hw: pointer to the hardware structure735 *736 * Prior to calling this function, the driver MUST* set the following fields737 * in the cq->structure for all control queues:738 * - cq->num_sq_entries739 * - cq->num_rq_entries740 * - cq->rq_buf_size741 * - cq->sq_buf_size742 *743 * NOTE: this function does not initialize the controlq locks.744 */745int ice_init_all_ctrlq(struct ice_hw *hw)746{747 u32 retry = 0;748 int status;749 750 /* Init FW admin queue */751 do {752 status = ice_init_ctrlq(hw, ICE_CTL_Q_ADMIN);753 if (status)754 return status;755 756 status = ice_init_check_adminq(hw);757 if (status != -EIO)758 break;759 760 ice_debug(hw, ICE_DBG_AQ_MSG, "Retry Admin Queue init due to FW critical error\n");761 ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN, true);762 msleep(ICE_CTL_Q_ADMIN_INIT_MSEC);763 } while (retry++ < ICE_CTL_Q_ADMIN_INIT_TIMEOUT);764 765 if (status)766 return status;767 /* sideband control queue (SBQ) interface is not supported on some768 * devices. Initialize if supported, else fallback to the admin queue769 * interface770 */771 if (ice_is_sbq_supported(hw)) {772 status = ice_init_ctrlq(hw, ICE_CTL_Q_SB);773 if (status)774 return status;775 }776 /* Init Mailbox queue */777 return ice_init_ctrlq(hw, ICE_CTL_Q_MAILBOX);778}779 780/**781 * ice_init_ctrlq_locks - Initialize locks for a control queue782 * @cq: pointer to the control queue783 *784 * Initializes the send and receive queue locks for a given control queue.785 */786static void ice_init_ctrlq_locks(struct ice_ctl_q_info *cq)787{788 mutex_init(&cq->sq_lock);789 mutex_init(&cq->rq_lock);790}791 792/**793 * ice_create_all_ctrlq - main initialization routine for all control queues794 * @hw: pointer to the hardware structure795 *796 * Prior to calling this function, the driver *MUST* set the following fields797 * in the cq->structure for all control queues:798 * - cq->num_sq_entries799 * - cq->num_rq_entries800 * - cq->rq_buf_size801 * - cq->sq_buf_size802 *803 * This function creates all the control queue locks and then calls804 * ice_init_all_ctrlq. It should be called once during driver load. If the805 * driver needs to re-initialize control queues at run time it should call806 * ice_init_all_ctrlq instead.807 */808int ice_create_all_ctrlq(struct ice_hw *hw)809{810 ice_init_ctrlq_locks(&hw->adminq);811 if (ice_is_sbq_supported(hw))812 ice_init_ctrlq_locks(&hw->sbq);813 ice_init_ctrlq_locks(&hw->mailboxq);814 815 return ice_init_all_ctrlq(hw);816}817 818/**819 * ice_destroy_ctrlq_locks - Destroy locks for a control queue820 * @cq: pointer to the control queue821 *822 * Destroys the send and receive queue locks for a given control queue.823 */824static void ice_destroy_ctrlq_locks(struct ice_ctl_q_info *cq)825{826 mutex_destroy(&cq->sq_lock);827 mutex_destroy(&cq->rq_lock);828}829 830/**831 * ice_destroy_all_ctrlq - exit routine for all control queues832 * @hw: pointer to the hardware structure833 *834 * This function shuts down all the control queues and then destroys the835 * control queue locks. It should be called once during driver unload. The836 * driver should call ice_shutdown_all_ctrlq if it needs to shut down and837 * reinitialize control queues, such as in response to a reset event.838 */839void ice_destroy_all_ctrlq(struct ice_hw *hw)840{841 /* shut down all the control queues first */842 ice_shutdown_all_ctrlq(hw, true);843 844 ice_destroy_ctrlq_locks(&hw->adminq);845 if (ice_is_sbq_supported(hw))846 ice_destroy_ctrlq_locks(&hw->sbq);847 ice_destroy_ctrlq_locks(&hw->mailboxq);848}849 850/**851 * ice_clean_sq - cleans send side of a control queue852 * @hw: pointer to the hardware structure853 * @cq: pointer to the specific Control queue854 *855 * returns the number of free desc856 */857static u16 ice_clean_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq)858{859 struct ice_ctl_q_ring *sq = &cq->sq;860 u16 ntc = sq->next_to_clean;861 struct ice_aq_desc *desc;862 863 desc = ICE_CTL_Q_DESC(*sq, ntc);864 865 while (rd32(hw, cq->sq.head) != ntc) {866 ice_debug(hw, ICE_DBG_AQ_MSG, "ntc %d head %d.\n", ntc, rd32(hw, cq->sq.head));867 memset(desc, 0, sizeof(*desc));868 ntc++;869 if (ntc == sq->count)870 ntc = 0;871 desc = ICE_CTL_Q_DESC(*sq, ntc);872 }873 874 sq->next_to_clean = ntc;875 876 return ICE_CTL_Q_DESC_UNUSED(sq);877}878 879/**880 * ice_ctl_q_str - Convert control queue type to string881 * @qtype: the control queue type882 *883 * Return: A string name for the given control queue type.884 */885static const char *ice_ctl_q_str(enum ice_ctl_q qtype)886{887 switch (qtype) {888 case ICE_CTL_Q_UNKNOWN:889 return "Unknown CQ";890 case ICE_CTL_Q_ADMIN:891 return "AQ";892 case ICE_CTL_Q_MAILBOX:893 return "MBXQ";894 case ICE_CTL_Q_SB:895 return "SBQ";896 default:897 return "Unrecognized CQ";898 }899}900 901/**902 * ice_debug_cq903 * @hw: pointer to the hardware structure904 * @cq: pointer to the specific Control queue905 * @desc: pointer to control queue descriptor906 * @buf: pointer to command buffer907 * @buf_len: max length of buf908 * @response: true if this is the writeback response909 *910 * Dumps debug log about control command with descriptor contents.911 */912static void ice_debug_cq(struct ice_hw *hw, struct ice_ctl_q_info *cq,913 void *desc, void *buf, u16 buf_len, bool response)914{915 struct ice_aq_desc *cq_desc = desc;916 u16 datalen, flags;917 918 if (!IS_ENABLED(CONFIG_DYNAMIC_DEBUG) &&919 !((ICE_DBG_AQ_DESC | ICE_DBG_AQ_DESC_BUF) & hw->debug_mask))920 return;921 922 if (!desc)923 return;924 925 datalen = le16_to_cpu(cq_desc->datalen);926 flags = le16_to_cpu(cq_desc->flags);927 928 ice_debug(hw, ICE_DBG_AQ_DESC, "%s %s: opcode 0x%04X, flags 0x%04X, datalen 0x%04X, retval 0x%04X\n\tcookie (h,l) 0x%08X 0x%08X\n\tparam (0,1) 0x%08X 0x%08X\n\taddr (h,l) 0x%08X 0x%08X\n",929 ice_ctl_q_str(cq->qtype), response ? "Response" : "Command",930 le16_to_cpu(cq_desc->opcode), flags, datalen,931 le16_to_cpu(cq_desc->retval),932 le32_to_cpu(cq_desc->cookie_high),933 le32_to_cpu(cq_desc->cookie_low),934 le32_to_cpu(cq_desc->params.generic.param0),935 le32_to_cpu(cq_desc->params.generic.param1),936 le32_to_cpu(cq_desc->params.generic.addr_high),937 le32_to_cpu(cq_desc->params.generic.addr_low));938 /* Dump buffer iff 1) one exists and 2) is either a response indicated939 * by the DD and/or CMP flag set or a command with the RD flag set.940 */941 if (buf && cq_desc->datalen &&942 (flags & (ICE_AQ_FLAG_DD | ICE_AQ_FLAG_CMP | ICE_AQ_FLAG_RD))) {943 char prefix[] = KBUILD_MODNAME " 0x12341234 0x12341234 ";944 945 sprintf(prefix, KBUILD_MODNAME " 0x%08X 0x%08X ",946 le32_to_cpu(cq_desc->params.generic.addr_high),947 le32_to_cpu(cq_desc->params.generic.addr_low));948 ice_debug_array_w_prefix(hw, ICE_DBG_AQ_DESC_BUF, prefix,949 buf,950 min_t(u16, buf_len, datalen));951 }952}953 954/**955 * ice_sq_done - poll until the last send on a control queue has completed956 * @hw: pointer to the HW struct957 * @cq: pointer to the specific Control queue958 *959 * Use read_poll_timeout to poll the control queue head, checking until it960 * matches next_to_use. According to the control queue designers, this has961 * better timing reliability than the DD bit.962 *963 * Return: true if all the descriptors on the send side of a control queue964 * are finished processing, false otherwise.965 */966static bool ice_sq_done(struct ice_hw *hw, struct ice_ctl_q_info *cq)967{968 u32 head;969 970 /* Wait a short time before the initial check, to allow hardware time971 * for completion.972 */973 udelay(5);974 975 return !rd32_poll_timeout(hw, cq->sq.head,976 head, head == cq->sq.next_to_use,977 20, ICE_CTL_Q_SQ_CMD_TIMEOUT);978}979 980/**981 * ice_sq_send_cmd - send command to a control queue982 * @hw: pointer to the HW struct983 * @cq: pointer to the specific Control queue984 * @desc: prefilled descriptor describing the command985 * @buf: buffer to use for indirect commands (or NULL for direct commands)986 * @buf_size: size of buffer for indirect commands (or 0 for direct commands)987 * @cd: pointer to command details structure988 *989 * Main command for the transmit side of a control queue. It puts the command990 * on the queue, bumps the tail, waits for processing of the command, captures991 * command status and results, etc.992 */993int994ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq,995 struct ice_aq_desc *desc, void *buf, u16 buf_size,996 struct ice_sq_cd *cd)997{998 struct ice_dma_mem *dma_buf = NULL;999 struct ice_aq_desc *desc_on_ring;1000 bool cmd_completed = false;1001 int status = 0;1002 u16 retval = 0;1003 u32 val = 0;1004 1005 /* if reset is in progress return a soft error */1006 if (hw->reset_ongoing)1007 return -EBUSY;1008 mutex_lock(&cq->sq_lock);1009 1010 cq->sq_last_status = ICE_AQ_RC_OK;1011 1012 if (!cq->sq.count) {1013 ice_debug(hw, ICE_DBG_AQ_MSG, "Control Send queue not initialized.\n");1014 status = -EIO;1015 goto sq_send_command_error;1016 }1017 1018 if ((buf && !buf_size) || (!buf && buf_size)) {1019 status = -EINVAL;1020 goto sq_send_command_error;1021 }1022 1023 if (buf) {1024 if (buf_size > cq->sq_buf_size) {1025 ice_debug(hw, ICE_DBG_AQ_MSG, "Invalid buffer size for Control Send queue: %d.\n",1026 buf_size);1027 status = -EINVAL;1028 goto sq_send_command_error;1029 }1030 1031 desc->flags |= cpu_to_le16(ICE_AQ_FLAG_BUF);1032 if (buf_size > ICE_AQ_LG_BUF)1033 desc->flags |= cpu_to_le16(ICE_AQ_FLAG_LB);1034 }1035 1036 val = rd32(hw, cq->sq.head);1037 if (val >= cq->num_sq_entries) {1038 ice_debug(hw, ICE_DBG_AQ_MSG, "head overrun at %d in the Control Send Queue ring\n",1039 val);1040 status = -EIO;1041 goto sq_send_command_error;1042 }1043 1044 /* Call clean and check queue available function to reclaim the1045 * descriptors that were processed by FW/MBX; the function returns the1046 * number of desc available. The clean function called here could be1047 * called in a separate thread in case of asynchronous completions.1048 */1049 if (ice_clean_sq(hw, cq) == 0) {1050 ice_debug(hw, ICE_DBG_AQ_MSG, "Error: Control Send Queue is full.\n");1051 status = -ENOSPC;1052 goto sq_send_command_error;1053 }1054 1055 /* initialize the temp desc pointer with the right desc */1056 desc_on_ring = ICE_CTL_Q_DESC(cq->sq, cq->sq.next_to_use);1057 1058 /* if the desc is available copy the temp desc to the right place */1059 memcpy(desc_on_ring, desc, sizeof(*desc_on_ring));1060 1061 /* if buf is not NULL assume indirect command */1062 if (buf) {1063 dma_buf = &cq->sq.r.sq_bi[cq->sq.next_to_use];1064 /* copy the user buf into the respective DMA buf */1065 memcpy(dma_buf->va, buf, buf_size);1066 desc_on_ring->datalen = cpu_to_le16(buf_size);1067 1068 /* Update the address values in the desc with the pa value1069 * for respective buffer1070 */1071 desc_on_ring->params.generic.addr_high =1072 cpu_to_le32(upper_32_bits(dma_buf->pa));1073 desc_on_ring->params.generic.addr_low =1074 cpu_to_le32(lower_32_bits(dma_buf->pa));1075 }1076 1077 /* Debug desc and buffer */1078 ice_debug(hw, ICE_DBG_AQ_DESC, "ATQ: Control Send queue desc and buffer:\n");1079 1080 ice_debug_cq(hw, cq, (void *)desc_on_ring, buf, buf_size, false);1081 1082 (cq->sq.next_to_use)++;1083 if (cq->sq.next_to_use == cq->sq.count)1084 cq->sq.next_to_use = 0;1085 wr32(hw, cq->sq.tail, cq->sq.next_to_use);1086 ice_flush(hw);1087 1088 /* Wait for the command to complete. If it finishes within the1089 * timeout, copy the descriptor back to temp.1090 */1091 if (ice_sq_done(hw, cq)) {1092 memcpy(desc, desc_on_ring, sizeof(*desc));1093 if (buf) {1094 /* get returned length to copy */1095 u16 copy_size = le16_to_cpu(desc->datalen);1096 1097 if (copy_size > buf_size) {1098 ice_debug(hw, ICE_DBG_AQ_MSG, "Return len %d > than buf len %d\n",1099 copy_size, buf_size);1100 status = -EIO;1101 } else {1102 memcpy(buf, dma_buf->va, copy_size);1103 }1104 }1105 retval = le16_to_cpu(desc->retval);1106 if (retval) {1107 ice_debug(hw, ICE_DBG_AQ_MSG, "Control Send Queue command 0x%04X completed with error 0x%X\n",1108 le16_to_cpu(desc->opcode),1109 retval);1110 1111 /* strip off FW internal code */1112 retval &= 0xff;1113 }1114 cmd_completed = true;1115 if (!status && retval != ICE_AQ_RC_OK)1116 status = -EIO;1117 cq->sq_last_status = (enum ice_aq_err)retval;1118 }1119 1120 ice_debug(hw, ICE_DBG_AQ_MSG, "ATQ: desc and buffer writeback:\n");1121 1122 ice_debug_cq(hw, cq, (void *)desc, buf, buf_size, true);1123 1124 /* save writeback AQ if requested */1125 if (cd && cd->wb_desc)1126 memcpy(cd->wb_desc, desc_on_ring, sizeof(*cd->wb_desc));1127 1128 /* update the error if time out occurred */1129 if (!cmd_completed) {1130 if (rd32(hw, cq->rq.len) & cq->rq.len_crit_mask ||1131 rd32(hw, cq->sq.len) & cq->sq.len_crit_mask) {1132 ice_debug(hw, ICE_DBG_AQ_MSG, "Critical FW error.\n");1133 status = -EIO;1134 } else {1135 ice_debug(hw, ICE_DBG_AQ_MSG, "Control Send Queue Writeback timeout.\n");1136 status = -EIO;1137 }1138 }1139 1140sq_send_command_error:1141 mutex_unlock(&cq->sq_lock);1142 return status;1143}1144 1145/**1146 * ice_fill_dflt_direct_cmd_desc - AQ descriptor helper function1147 * @desc: pointer to the temp descriptor (non DMA mem)1148 * @opcode: the opcode can be used to decide which flags to turn off or on1149 *1150 * Fill the desc with default values1151 */1152void ice_fill_dflt_direct_cmd_desc(struct ice_aq_desc *desc, u16 opcode)1153{1154 /* zero out the desc */1155 memset(desc, 0, sizeof(*desc));1156 desc->opcode = cpu_to_le16(opcode);1157 desc->flags = cpu_to_le16(ICE_AQ_FLAG_SI);1158}1159 1160/**1161 * ice_clean_rq_elem1162 * @hw: pointer to the HW struct1163 * @cq: pointer to the specific Control queue1164 * @e: event info from the receive descriptor, includes any buffers1165 * @pending: number of events that could be left to process1166 *1167 * Clean one element from the receive side of a control queue. On return 'e'1168 * contains contents of the message, and 'pending' contains the number of1169 * events left to process.1170 */1171int1172ice_clean_rq_elem(struct ice_hw *hw, struct ice_ctl_q_info *cq,1173 struct ice_rq_event_info *e, u16 *pending)1174{1175 u16 ntc = cq->rq.next_to_clean;1176 enum ice_aq_err rq_last_status;1177 struct ice_aq_desc *desc;1178 struct ice_dma_mem *bi;1179 int ret_code = 0;1180 u16 desc_idx;1181 u16 datalen;1182 u16 flags;1183 u16 ntu;1184 1185 /* pre-clean the event info */1186 memset(&e->desc, 0, sizeof(e->desc));1187 1188 /* take the lock before we start messing with the ring */1189 mutex_lock(&cq->rq_lock);1190 1191 if (!cq->rq.count) {1192 ice_debug(hw, ICE_DBG_AQ_MSG, "Control Receive queue not initialized.\n");1193 ret_code = -EIO;1194 goto clean_rq_elem_err;1195 }1196 1197 /* set next_to_use to head */1198 ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);1199 1200 if (ntu == ntc) {1201 /* nothing to do - shouldn't need to update ring's values */1202 ret_code = -EALREADY;1203 goto clean_rq_elem_out;1204 }1205 1206 /* now clean the next descriptor */1207 desc = ICE_CTL_Q_DESC(cq->rq, ntc);1208 desc_idx = ntc;1209 1210 rq_last_status = (enum ice_aq_err)le16_to_cpu(desc->retval);1211 flags = le16_to_cpu(desc->flags);1212 if (flags & ICE_AQ_FLAG_ERR) {1213 ret_code = -EIO;1214 ice_debug(hw, ICE_DBG_AQ_MSG, "Control Receive Queue Event 0x%04X received with error 0x%X\n",1215 le16_to_cpu(desc->opcode), rq_last_status);1216 }1217 memcpy(&e->desc, desc, sizeof(e->desc));1218 datalen = le16_to_cpu(desc->datalen);1219 e->msg_len = min_t(u16, datalen, e->buf_len);1220 if (e->msg_buf && e->msg_len)1221 memcpy(e->msg_buf, cq->rq.r.rq_bi[desc_idx].va, e->msg_len);1222 1223 ice_debug(hw, ICE_DBG_AQ_DESC, "ARQ: desc and buffer:\n");1224 1225 ice_debug_cq(hw, cq, (void *)desc, e->msg_buf, cq->rq_buf_size, true);1226 1227 /* Restore the original datalen and buffer address in the desc,1228 * FW updates datalen to indicate the event message size1229 */1230 bi = &cq->rq.r.rq_bi[ntc];1231 memset(desc, 0, sizeof(*desc));1232 1233 desc->flags = cpu_to_le16(ICE_AQ_FLAG_BUF);1234 if (cq->rq_buf_size > ICE_AQ_LG_BUF)1235 desc->flags |= cpu_to_le16(ICE_AQ_FLAG_LB);1236 desc->datalen = cpu_to_le16(bi->size);1237 desc->params.generic.addr_high = cpu_to_le32(upper_32_bits(bi->pa));1238 desc->params.generic.addr_low = cpu_to_le32(lower_32_bits(bi->pa));1239 1240 /* set tail = the last cleaned desc index. */1241 wr32(hw, cq->rq.tail, ntc);1242 /* ntc is updated to tail + 1 */1243 ntc++;1244 if (ntc == cq->num_rq_entries)1245 ntc = 0;1246 cq->rq.next_to_clean = ntc;1247 cq->rq.next_to_use = ntu;1248 1249clean_rq_elem_out:1250 /* Set pending if needed, unlock and return */1251 if (pending) {1252 /* re-read HW head to calculate actual pending messages */1253 ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);1254 *pending = (u16)((ntc > ntu ? cq->rq.count : 0) + (ntu - ntc));1255 }1256clean_rq_elem_err:1257 mutex_unlock(&cq->rq_lock);1258 1259 return ret_code;1260}1261