619 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright (C) 2023 Intel Corporation */3 4#include "idpf_controlq.h"5 6/**7 * idpf_ctlq_setup_regs - initialize control queue registers8 * @cq: pointer to the specific control queue9 * @q_create_info: structs containing info for each queue to be initialized10 */11static void idpf_ctlq_setup_regs(struct idpf_ctlq_info *cq,12 struct idpf_ctlq_create_info *q_create_info)13{14 /* set control queue registers in our local struct */15 cq->reg.head = q_create_info->reg.head;16 cq->reg.tail = q_create_info->reg.tail;17 cq->reg.len = q_create_info->reg.len;18 cq->reg.bah = q_create_info->reg.bah;19 cq->reg.bal = q_create_info->reg.bal;20 cq->reg.len_mask = q_create_info->reg.len_mask;21 cq->reg.len_ena_mask = q_create_info->reg.len_ena_mask;22 cq->reg.head_mask = q_create_info->reg.head_mask;23}24 25/**26 * idpf_ctlq_init_regs - Initialize control queue registers27 * @hw: pointer to hw struct28 * @cq: pointer to the specific Control queue29 * @is_rxq: true if receive control queue, false otherwise30 *31 * Initialize registers. The caller is expected to have already initialized the32 * descriptor ring memory and buffer memory33 */34static void idpf_ctlq_init_regs(struct idpf_hw *hw, struct idpf_ctlq_info *cq,35 bool is_rxq)36{37 /* Update tail to post pre-allocated buffers for rx queues */38 if (is_rxq)39 wr32(hw, cq->reg.tail, (u32)(cq->ring_size - 1));40 41 /* For non-Mailbox control queues only TAIL need to be set */42 if (cq->q_id != -1)43 return;44 45 /* Clear Head for both send or receive */46 wr32(hw, cq->reg.head, 0);47 48 /* set starting point */49 wr32(hw, cq->reg.bal, lower_32_bits(cq->desc_ring.pa));50 wr32(hw, cq->reg.bah, upper_32_bits(cq->desc_ring.pa));51 wr32(hw, cq->reg.len, (cq->ring_size | cq->reg.len_ena_mask));52}53 54/**55 * idpf_ctlq_init_rxq_bufs - populate receive queue descriptors with buf56 * @cq: pointer to the specific Control queue57 *58 * Record the address of the receive queue DMA buffers in the descriptors.59 * The buffers must have been previously allocated.60 */61static void idpf_ctlq_init_rxq_bufs(struct idpf_ctlq_info *cq)62{63 int i;64 65 for (i = 0; i < cq->ring_size; i++) {66 struct idpf_ctlq_desc *desc = IDPF_CTLQ_DESC(cq, i);67 struct idpf_dma_mem *bi = cq->bi.rx_buff[i];68 69 /* No buffer to post to descriptor, continue */70 if (!bi)71 continue;72 73 desc->flags =74 cpu_to_le16(IDPF_CTLQ_FLAG_BUF | IDPF_CTLQ_FLAG_RD);75 desc->opcode = 0;76 desc->datalen = cpu_to_le16(bi->size);77 desc->ret_val = 0;78 desc->v_opcode_dtype = 0;79 desc->v_retval = 0;80 desc->params.indirect.addr_high =81 cpu_to_le32(upper_32_bits(bi->pa));82 desc->params.indirect.addr_low =83 cpu_to_le32(lower_32_bits(bi->pa));84 desc->params.indirect.param0 = 0;85 desc->params.indirect.sw_cookie = 0;86 desc->params.indirect.v_flags = 0;87 }88}89 90/**91 * idpf_ctlq_shutdown - shutdown the CQ92 * @hw: pointer to hw struct93 * @cq: pointer to the specific Control queue94 *95 * The main shutdown routine for any controq queue96 */97static void idpf_ctlq_shutdown(struct idpf_hw *hw, struct idpf_ctlq_info *cq)98{99 mutex_lock(&cq->cq_lock);100 101 /* free ring buffers and the ring itself */102 idpf_ctlq_dealloc_ring_res(hw, cq);103 104 /* Set ring_size to 0 to indicate uninitialized queue */105 cq->ring_size = 0;106 107 mutex_unlock(&cq->cq_lock);108 mutex_destroy(&cq->cq_lock);109}110 111/**112 * idpf_ctlq_add - add one control queue113 * @hw: pointer to hardware struct114 * @qinfo: info for queue to be created115 * @cq_out: (output) double pointer to control queue to be created116 *117 * Allocate and initialize a control queue and add it to the control queue list.118 * The cq parameter will be allocated/initialized and passed back to the caller119 * if no errors occur.120 *121 * Note: idpf_ctlq_init must be called prior to any calls to idpf_ctlq_add122 */123int idpf_ctlq_add(struct idpf_hw *hw,124 struct idpf_ctlq_create_info *qinfo,125 struct idpf_ctlq_info **cq_out)126{127 struct idpf_ctlq_info *cq;128 bool is_rxq = false;129 int err;130 131 cq = kzalloc(sizeof(*cq), GFP_KERNEL);132 if (!cq)133 return -ENOMEM;134 135 cq->cq_type = qinfo->type;136 cq->q_id = qinfo->id;137 cq->buf_size = qinfo->buf_size;138 cq->ring_size = qinfo->len;139 140 cq->next_to_use = 0;141 cq->next_to_clean = 0;142 cq->next_to_post = cq->ring_size - 1;143 144 switch (qinfo->type) {145 case IDPF_CTLQ_TYPE_MAILBOX_RX:146 is_rxq = true;147 fallthrough;148 case IDPF_CTLQ_TYPE_MAILBOX_TX:149 err = idpf_ctlq_alloc_ring_res(hw, cq);150 break;151 default:152 err = -EBADR;153 break;154 }155 156 if (err)157 goto init_free_q;158 159 if (is_rxq) {160 idpf_ctlq_init_rxq_bufs(cq);161 } else {162 /* Allocate the array of msg pointers for TX queues */163 cq->bi.tx_msg = kcalloc(qinfo->len,164 sizeof(struct idpf_ctlq_msg *),165 GFP_KERNEL);166 if (!cq->bi.tx_msg) {167 err = -ENOMEM;168 goto init_dealloc_q_mem;169 }170 }171 172 idpf_ctlq_setup_regs(cq, qinfo);173 174 idpf_ctlq_init_regs(hw, cq, is_rxq);175 176 mutex_init(&cq->cq_lock);177 178 list_add(&cq->cq_list, &hw->cq_list_head);179 180 *cq_out = cq;181 182 return 0;183 184init_dealloc_q_mem:185 /* free ring buffers and the ring itself */186 idpf_ctlq_dealloc_ring_res(hw, cq);187init_free_q:188 kfree(cq);189 190 return err;191}192 193/**194 * idpf_ctlq_remove - deallocate and remove specified control queue195 * @hw: pointer to hardware struct196 * @cq: pointer to control queue to be removed197 */198void idpf_ctlq_remove(struct idpf_hw *hw,199 struct idpf_ctlq_info *cq)200{201 list_del(&cq->cq_list);202 idpf_ctlq_shutdown(hw, cq);203 kfree(cq);204}205 206/**207 * idpf_ctlq_init - main initialization routine for all control queues208 * @hw: pointer to hardware struct209 * @num_q: number of queues to initialize210 * @q_info: array of structs containing info for each queue to be initialized211 *212 * This initializes any number and any type of control queues. This is an all213 * or nothing routine; if one fails, all previously allocated queues will be214 * destroyed. This must be called prior to using the individual add/remove215 * APIs.216 */217int idpf_ctlq_init(struct idpf_hw *hw, u8 num_q,218 struct idpf_ctlq_create_info *q_info)219{220 struct idpf_ctlq_info *cq, *tmp;221 int err;222 int i;223 224 INIT_LIST_HEAD(&hw->cq_list_head);225 226 for (i = 0; i < num_q; i++) {227 struct idpf_ctlq_create_info *qinfo = q_info + i;228 229 err = idpf_ctlq_add(hw, qinfo, &cq);230 if (err)231 goto init_destroy_qs;232 }233 234 return 0;235 236init_destroy_qs:237 list_for_each_entry_safe(cq, tmp, &hw->cq_list_head, cq_list)238 idpf_ctlq_remove(hw, cq);239 240 return err;241}242 243/**244 * idpf_ctlq_deinit - destroy all control queues245 * @hw: pointer to hw struct246 */247void idpf_ctlq_deinit(struct idpf_hw *hw)248{249 struct idpf_ctlq_info *cq, *tmp;250 251 list_for_each_entry_safe(cq, tmp, &hw->cq_list_head, cq_list)252 idpf_ctlq_remove(hw, cq);253}254 255/**256 * idpf_ctlq_send - send command to Control Queue (CTQ)257 * @hw: pointer to hw struct258 * @cq: handle to control queue struct to send on259 * @num_q_msg: number of messages to send on control queue260 * @q_msg: pointer to array of queue messages to be sent261 *262 * The caller is expected to allocate DMAable buffers and pass them to the263 * send routine via the q_msg struct / control queue specific data struct.264 * The control queue will hold a reference to each send message until265 * the completion for that message has been cleaned.266 */267int idpf_ctlq_send(struct idpf_hw *hw, struct idpf_ctlq_info *cq,268 u16 num_q_msg, struct idpf_ctlq_msg q_msg[])269{270 struct idpf_ctlq_desc *desc;271 int num_desc_avail;272 int err = 0;273 int i;274 275 mutex_lock(&cq->cq_lock);276 277 /* Ensure there are enough descriptors to send all messages */278 num_desc_avail = IDPF_CTLQ_DESC_UNUSED(cq);279 if (num_desc_avail == 0 || num_desc_avail < num_q_msg) {280 err = -ENOSPC;281 goto err_unlock;282 }283 284 for (i = 0; i < num_q_msg; i++) {285 struct idpf_ctlq_msg *msg = &q_msg[i];286 287 desc = IDPF_CTLQ_DESC(cq, cq->next_to_use);288 289 desc->opcode = cpu_to_le16(msg->opcode);290 desc->pfid_vfid = cpu_to_le16(msg->func_id);291 292 desc->v_opcode_dtype = cpu_to_le32(msg->cookie.mbx.chnl_opcode);293 desc->v_retval = cpu_to_le32(msg->cookie.mbx.chnl_retval);294 295 desc->flags = cpu_to_le16((msg->host_id & IDPF_HOST_ID_MASK) <<296 IDPF_CTLQ_FLAG_HOST_ID_S);297 if (msg->data_len) {298 struct idpf_dma_mem *buff = msg->ctx.indirect.payload;299 300 desc->datalen |= cpu_to_le16(msg->data_len);301 desc->flags |= cpu_to_le16(IDPF_CTLQ_FLAG_BUF);302 desc->flags |= cpu_to_le16(IDPF_CTLQ_FLAG_RD);303 304 /* Update the address values in the desc with the pa305 * value for respective buffer306 */307 desc->params.indirect.addr_high =308 cpu_to_le32(upper_32_bits(buff->pa));309 desc->params.indirect.addr_low =310 cpu_to_le32(lower_32_bits(buff->pa));311 312 memcpy(&desc->params, msg->ctx.indirect.context,313 IDPF_INDIRECT_CTX_SIZE);314 } else {315 memcpy(&desc->params, msg->ctx.direct,316 IDPF_DIRECT_CTX_SIZE);317 }318 319 /* Store buffer info */320 cq->bi.tx_msg[cq->next_to_use] = msg;321 322 (cq->next_to_use)++;323 if (cq->next_to_use == cq->ring_size)324 cq->next_to_use = 0;325 }326 327 /* Force memory write to complete before letting hardware328 * know that there are new descriptors to fetch.329 */330 dma_wmb();331 332 wr32(hw, cq->reg.tail, cq->next_to_use);333 334err_unlock:335 mutex_unlock(&cq->cq_lock);336 337 return err;338}339 340/**341 * idpf_ctlq_clean_sq - reclaim send descriptors on HW write back for the342 * requested queue343 * @cq: pointer to the specific Control queue344 * @clean_count: (input|output) number of descriptors to clean as input, and345 * number of descriptors actually cleaned as output346 * @msg_status: (output) pointer to msg pointer array to be populated; needs347 * to be allocated by caller348 *349 * Returns an array of message pointers associated with the cleaned350 * descriptors. The pointers are to the original ctlq_msgs sent on the cleaned351 * descriptors. The status will be returned for each; any messages that failed352 * to send will have a non-zero status. The caller is expected to free original353 * ctlq_msgs and free or reuse the DMA buffers.354 */355int idpf_ctlq_clean_sq(struct idpf_ctlq_info *cq, u16 *clean_count,356 struct idpf_ctlq_msg *msg_status[])357{358 struct idpf_ctlq_desc *desc;359 u16 i, num_to_clean;360 u16 ntc, desc_err;361 362 if (*clean_count == 0)363 return 0;364 if (*clean_count > cq->ring_size)365 return -EBADR;366 367 mutex_lock(&cq->cq_lock);368 369 ntc = cq->next_to_clean;370 371 num_to_clean = *clean_count;372 373 for (i = 0; i < num_to_clean; i++) {374 /* Fetch next descriptor and check if marked as done */375 desc = IDPF_CTLQ_DESC(cq, ntc);376 if (!(le16_to_cpu(desc->flags) & IDPF_CTLQ_FLAG_DD))377 break;378 379 /* strip off FW internal code */380 desc_err = le16_to_cpu(desc->ret_val) & 0xff;381 382 msg_status[i] = cq->bi.tx_msg[ntc];383 msg_status[i]->status = desc_err;384 385 cq->bi.tx_msg[ntc] = NULL;386 387 /* Zero out any stale data */388 memset(desc, 0, sizeof(*desc));389 390 ntc++;391 if (ntc == cq->ring_size)392 ntc = 0;393 }394 395 cq->next_to_clean = ntc;396 397 mutex_unlock(&cq->cq_lock);398 399 /* Return number of descriptors actually cleaned */400 *clean_count = i;401 402 return 0;403}404 405/**406 * idpf_ctlq_post_rx_buffs - post buffers to descriptor ring407 * @hw: pointer to hw struct408 * @cq: pointer to control queue handle409 * @buff_count: (input|output) input is number of buffers caller is trying to410 * return; output is number of buffers that were not posted411 * @buffs: array of pointers to dma mem structs to be given to hardware412 *413 * Caller uses this function to return DMA buffers to the descriptor ring after414 * consuming them; buff_count will be the number of buffers.415 *416 * Note: this function needs to be called after a receive call even417 * if there are no DMA buffers to be returned, i.e. buff_count = 0,418 * buffs = NULL to support direct commands419 */420int idpf_ctlq_post_rx_buffs(struct idpf_hw *hw, struct idpf_ctlq_info *cq,421 u16 *buff_count, struct idpf_dma_mem **buffs)422{423 struct idpf_ctlq_desc *desc;424 u16 ntp = cq->next_to_post;425 bool buffs_avail = false;426 u16 tbp = ntp + 1;427 int i = 0;428 429 if (*buff_count > cq->ring_size)430 return -EBADR;431 432 if (*buff_count > 0)433 buffs_avail = true;434 435 mutex_lock(&cq->cq_lock);436 437 if (tbp >= cq->ring_size)438 tbp = 0;439 440 if (tbp == cq->next_to_clean)441 /* Nothing to do */442 goto post_buffs_out;443 444 /* Post buffers for as many as provided or up until the last one used */445 while (ntp != cq->next_to_clean) {446 desc = IDPF_CTLQ_DESC(cq, ntp);447 448 if (cq->bi.rx_buff[ntp])449 goto fill_desc;450 if (!buffs_avail) {451 /* If the caller hasn't given us any buffers or452 * there are none left, search the ring itself453 * for an available buffer to move to this454 * entry starting at the next entry in the ring455 */456 tbp = ntp + 1;457 458 /* Wrap ring if necessary */459 if (tbp >= cq->ring_size)460 tbp = 0;461 462 while (tbp != cq->next_to_clean) {463 if (cq->bi.rx_buff[tbp]) {464 cq->bi.rx_buff[ntp] =465 cq->bi.rx_buff[tbp];466 cq->bi.rx_buff[tbp] = NULL;467 468 /* Found a buffer, no need to469 * search anymore470 */471 break;472 }473 474 /* Wrap ring if necessary */475 tbp++;476 if (tbp >= cq->ring_size)477 tbp = 0;478 }479 480 if (tbp == cq->next_to_clean)481 goto post_buffs_out;482 } else {483 /* Give back pointer to DMA buffer */484 cq->bi.rx_buff[ntp] = buffs[i];485 i++;486 487 if (i >= *buff_count)488 buffs_avail = false;489 }490 491fill_desc:492 desc->flags =493 cpu_to_le16(IDPF_CTLQ_FLAG_BUF | IDPF_CTLQ_FLAG_RD);494 495 /* Post buffers to descriptor */496 desc->datalen = cpu_to_le16(cq->bi.rx_buff[ntp]->size);497 desc->params.indirect.addr_high =498 cpu_to_le32(upper_32_bits(cq->bi.rx_buff[ntp]->pa));499 desc->params.indirect.addr_low =500 cpu_to_le32(lower_32_bits(cq->bi.rx_buff[ntp]->pa));501 502 ntp++;503 if (ntp == cq->ring_size)504 ntp = 0;505 }506 507post_buffs_out:508 /* Only update tail if buffers were actually posted */509 if (cq->next_to_post != ntp) {510 if (ntp)511 /* Update next_to_post to ntp - 1 since current ntp512 * will not have a buffer513 */514 cq->next_to_post = ntp - 1;515 else516 /* Wrap to end of end ring since current ntp is 0 */517 cq->next_to_post = cq->ring_size - 1;518 519 dma_wmb();520 521 wr32(hw, cq->reg.tail, cq->next_to_post);522 }523 524 mutex_unlock(&cq->cq_lock);525 526 /* return the number of buffers that were not posted */527 *buff_count = *buff_count - i;528 529 return 0;530}531 532/**533 * idpf_ctlq_recv - receive control queue message call back534 * @cq: pointer to control queue handle to receive on535 * @num_q_msg: (input|output) input number of messages that should be received;536 * output number of messages actually received537 * @q_msg: (output) array of received control queue messages on this q;538 * needs to be pre-allocated by caller for as many messages as requested539 *540 * Called by interrupt handler or polling mechanism. Caller is expected541 * to free buffers542 */543int idpf_ctlq_recv(struct idpf_ctlq_info *cq, u16 *num_q_msg,544 struct idpf_ctlq_msg *q_msg)545{546 u16 num_to_clean, ntc, flags;547 struct idpf_ctlq_desc *desc;548 int err = 0;549 u16 i;550 551 /* take the lock before we start messing with the ring */552 mutex_lock(&cq->cq_lock);553 554 ntc = cq->next_to_clean;555 556 num_to_clean = *num_q_msg;557 558 for (i = 0; i < num_to_clean; i++) {559 /* Fetch next descriptor and check if marked as done */560 desc = IDPF_CTLQ_DESC(cq, ntc);561 flags = le16_to_cpu(desc->flags);562 563 if (!(flags & IDPF_CTLQ_FLAG_DD))564 break;565 566 q_msg[i].vmvf_type = (flags &567 (IDPF_CTLQ_FLAG_FTYPE_VM |568 IDPF_CTLQ_FLAG_FTYPE_PF)) >>569 IDPF_CTLQ_FLAG_FTYPE_S;570 571 if (flags & IDPF_CTLQ_FLAG_ERR)572 err = -EBADMSG;573 574 q_msg[i].cookie.mbx.chnl_opcode =575 le32_to_cpu(desc->v_opcode_dtype);576 q_msg[i].cookie.mbx.chnl_retval =577 le32_to_cpu(desc->v_retval);578 579 q_msg[i].opcode = le16_to_cpu(desc->opcode);580 q_msg[i].data_len = le16_to_cpu(desc->datalen);581 q_msg[i].status = le16_to_cpu(desc->ret_val);582 583 if (desc->datalen) {584 memcpy(q_msg[i].ctx.indirect.context,585 &desc->params.indirect, IDPF_INDIRECT_CTX_SIZE);586 587 /* Assign pointer to dma buffer to ctlq_msg array588 * to be given to upper layer589 */590 q_msg[i].ctx.indirect.payload = cq->bi.rx_buff[ntc];591 592 /* Zero out pointer to DMA buffer info;593 * will be repopulated by post buffers API594 */595 cq->bi.rx_buff[ntc] = NULL;596 } else {597 memcpy(q_msg[i].ctx.direct, desc->params.raw,598 IDPF_DIRECT_CTX_SIZE);599 }600 601 /* Zero out stale data in descriptor */602 memset(desc, 0, sizeof(struct idpf_ctlq_desc));603 604 ntc++;605 if (ntc == cq->ring_size)606 ntc = 0;607 }608 609 cq->next_to_clean = ntc;610 611 mutex_unlock(&cq->cq_lock);612 613 *num_q_msg = i;614 if (*num_q_msg == 0)615 err = -ENOMSG;616 617 return err;618}619